Don't insist that the maintenance version of netlib match.

.coveragerc

@@ -2,5 +2,5 @@
 branch = True
 
 [report]
-omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
+omit = *contrib*, *tnetstring*, *platform*, *console*
 include = *libmproxy*

.gitattributes

@@ -1,2 +0,0 @@
-libmproxy/web/static/**/* -diff
-web/src/js/filt/filt.js -diff

.gitignore

@@ -1,4 +1,3 @@
-.DS_Store
 MANIFEST
 /build
 /dist
@@ -10,18 +9,9 @@ MANIFEST
 *.py[cdo]
 *.swp
 *.swo
-mitmproxy.egg-info/
 mitmproxyc
 mitmdumpc
 .coverage
 .idea
 netlib
-pathod
 libpathod
-
-# UI
-
-node_modules
-bower_components
-*.compiled.js
-*.map

.travis.yml

@@ -1,11 +1,12 @@
 language: python
-sudo: false
 python:
   - "2.7"
-  - pypy
 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
-install:
-  - "pip install --upgrade --src . -r requirements.txt"
+install: 
+  - "pip install --upgrade git+https://github.com/mitmproxy/netlib.git"
+  - "pip install --upgrade git+https://github.com/mitmproxy/pathod.git"
+  - "pip install -r requirements.txt --use-mirrors"
+  - "pip install -r test/requirements.txt --use-mirrors"
 # command to run tests, e.g. python setup.py test
 script: 
   - "nosetests --with-cov --cov-report term-missing"
@@ -17,7 +18,4 @@ notifications:
       - "irc.oftc.net#mitmproxy"
     on_success: change
     on_failure: always
-cache:
-  directories:
-  - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages
-  - /home/travis/virtualenv/pypy-2.4.0/site-packages
+

CHANGELOG

@@ -1,58 +1,6 @@
-29 Dec 2014: mitmproxy 0.11.2:
-
-    * Configuration files - mitmproxy.conf, mitmdump.conf, common.conf in the
-      .mitmproxy directory.
-    * Better handling of servers that reject connections that are not SNI.
-    * Many other small bugfixes and improvements.
-
-
-15 November 2014: mitmproxy 0.11.1:
-
-    * Bug fixes: connection leaks some crashes 
-
-
-7 November 2014: mitmproxy 0.11:
-    
-    * Performance improvements for mitmproxy console
-
-    * SOCKS5 proxy mode allows mitmproxy to act as a SOCKS5 proxy server
-
-    * Data streaming for response bodies exceeding a threshold
-      (bradpeabody@gmail.com)
-
-    * Ignore hosts or IP addresses, forwarding both HTTP and HTTPS traffic
-      untouched
-
-    * Finer-grained control of traffic replay, including options to ignore
-      contents or parameters when matching flows (marcelo.glezer@gmail.com)
-
-    * Pass arguments to inline scripts 
-
-    * Configurable size limit on HTTP request and response bodies
-
-    * Per-domain specification of interception certificates and keys (see
-      --cert option)
-
-    * Certificate forwarding, relaying upstream SSL certificates verbatim (see
-      --cert-forward)
-
-    * Search and highlighting for HTTP request and response bodies in
-      mitmproxy console (pedro@worcel.com)
-
-    * Transparent proxy support on Windows
-
-    * Improved error messages and logging
-
-    * Support for FreeBSD in transparent mode, using pf (zbrdge@gmail.com)
-
-    * Content view mode for WBXML (davidshaw835@air-watch.com)
-
-    * Better documentation, with a new section on proxy modes
-
-    * Generic TCP proxy mode
-
-    * Countless bugfixes and other small improvements
+26 Februrary 2014: mitmproxy 0.10.1:
 
+    * Fix compatibility with pyOpenSSL 0.14
 
 
 28 January 2014: mitmproxy 0.10:

CONTRIBUTORS

@@ -1,65 +1,51 @@
-   902	Aldo Cortesi
-   323	Maximilian Hils
+   854	Aldo Cortesi
+    64	Maximilian Hils
     18	Henrik Nordstrom
     13	Thomas Roth
-    12	Pedro Worcel
     11	Stephen Altamirano
     10	András Veres-Szentkirályi
-     8	Rouli
      8	Jason A. Novak
+     8	Rouli
      7	Alexis Hildebrandt
+     6	Pedro Worcel
      5	Tomaz Muraus
-     5	Brad Peabody
      5	Matthias Urlichs
      4	root
+     4	Bryan Bishop
      4	Marc Liyanage
      4	Valtteri Virtanen
-     4	Bryan Bishop
-     3	Chris Neasbitt
-     3	Zack B
-     3	Eli Shvartsman
      3	Kyle Manna
-     2	Michael Frister
-     2	Bennett Blodinger
-     2	Jim Lloyd
-     2	Rob Wills
-     2	israel
-     2	Jaime Soriano Pastor
-     2	Heikki Hannikainen
-     2	Mark E. Haase
+     3	Chris Neasbitt
      2	alts
-     1	davidpshaw
-     1	deployable
-     1	joebowbeer
-     1	meeee
-     1	phil plante
-     1	Michael Bisbjerg
-     1	Andy Smith
-     1	Dan Wilbraham
-     1	David Shaw
-     1	Eric Entzel
-     1	Felix Wolfsteller
-     1	Henrik Nordström
-     1	Ivaylo Popov
-     1	JC
-     1	Jakub Nawalaniec
-     1	James Billingham
-     1	Jean Regisser
-     1	Kit Randel
-     1	Marcelo Glezer
-     1	Mathieu Mitchell
-     1	Mikhail Korobov
-     1	Nicolas Esteves
-     1	Oleksandr Sheremet
+     2	Heikki Hannikainen
+     2	Jim Lloyd
+     2	Michael Frister
+     2	Rob Wills
+     2	Jaime Soriano Pastor
+     2	israel
+     2	Mark E. Haase
      1	Paul
      1	Rich Somerfield
      1	Rory McCann
+     1	Felix Wolfsteller
      1	Rune Halvorsen
      1	Sahn Lam
-     1	Seppo Yli-Olli
-     1	Sergey Chipiga
-     1	Steven Van Acker
+     1	Eric Entzel
+     1	Dan Wilbraham
      1	Ulrich Petri
-     1	Vyacheslav Bakhmutov
+     1	Andy Smith
      1	Yuangxuan Wang
      1	capt8bit
+     1	joebowbeer
+     1	meeee
+     1	James Billingham
+     1	Jakub Nawalaniec
+     1	JC
+     1	Kit Randel
+     1	phil plante
+     1	Mathieu Mitchell
+     1	Ivaylo Popov
+     1	Henrik Nordström
+     1	Michael Bisbjerg
+     1	Nicolas Esteves
+     1	Oleksandr Sheremet

MANIFEST.in

@@ -1,10 +1,11 @@
-include mitmproxy mitmdump
-include LICENSE CHANGELOG CONTRIBUTORS README.txt
+include LICENSE
+include CHANGELOG
+include CONTRIBUTORS
+include README.txt
+include setup.py
 exclude README.mkd
 recursive-include examples *
 recursive-include doc *
 recursive-include test *
-recursive-include libmproxy *
-recursive-exclude * *.pyc *.pyo *.swo *.swp
-recursive-exclude netlib *
-recursive-exclude libpathod *
+recursive-include libmproxy/resources *
+recursive-exclude test *.swo *.swp *.pyc

README.mkd

@@ -13,9 +13,6 @@ mitmproxy.org website:
 [mitmproxy.org](http://mitmproxy.org).
 
 
-You can find complete directions for installing mitmproxy [here](http://mitmproxy.org/doc/install.html). 
-
-
 Features
 --------
 
@@ -29,17 +26,9 @@ Features
 - SSL certificates for interception are generated on the fly.
 - And much, much more.
 
-__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows,
-only mitmdump is supported, which does not have a graphical user interface.
-
-
-
-Hacking
--------
-
-
-### Requirements
 
+Requirements
+------------
 
 * [Python](http://www.python.org) 2.7.x.
 * [netlib](http://pypi.python.org/pypi/netlib), version matching mitmproxy.
@@ -51,36 +40,22 @@ Optional packages for extended content decoding:
 * [protobuf](https://code.google.com/p/protobuf/) version 2.5.0 or newer.
 * [cssutils](http://cthedot.de/cssutils/) version 1.0 or newer.
 
-For convenience, all optional dependencies can be installed with 
-
-`pip install "mitmproxy[contentviews]"`
-
-### Setting up a dev environment
-
-The following procedure is recommended to set up your dev environment:
-
-```
-$ git clone https://github.com/mitmproxy/mitmproxy.git
-$ cd mitmproxy
-$ pip install --src . -r requirements.txt
-```
-
-This installs the latest GitHub versions of mitmproxy, netlib and pathod into `mitmproxy/`. All other development dependencies save countershape are installed into their usual locations. 
+__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. Windows is not
+officially supported at the moment.
 
 
-### Testing
+Hacking
+-------
 
-The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy. Install these with:
+The following components are needed if you plan to hack on mitmproxy:
 
-`pip install "mitmproxy[dev]"`
+* The test suite uses the [nose](http://readthedocs.org/docs/nose/en/latest/) unit testing
+  framework and requires [pathod](http://pathod.org) and [flask](http://flask.pocoo.org/).
+* Rendering the documentation requires [countershape](http://github.com/cortesi/countershape).
 
+For convenience, all dependencies save countershape can be installed from pypi
+to a virtualenv with 'pip install -r requirements.txt'.
 
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project maintains 100% test coverage.
 
-
-### Docs
-
-Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). After installation, you can render the documentation to the doc like this:
-
-`cshape doc-src doc`

doc-src/_nav.html

@@ -2,32 +2,24 @@
     $!nav(idxpath, this, state)!$
     $!nav("install.html", this, state)!$
     $!nav("howmitmproxy.html", this, state)!$
-    $!nav("modes.html", this, state)!$
 
     <li class="nav-header">Tools</li>
         $!nav("mitmproxy.html", this, state)!$
         $!nav("mitmdump.html", this, state)!$
-        $!nav("config.html", this, state)!$
 
     <li class="nav-header">Features</li>
         $!nav("anticache.html", this, state)!$
-
-        $!nav("filters.html", this, state)!$
-        $!nav("replacements.html", this, state)!$
         $!nav("clientreplay.html", this, state)!$
+        $!nav("filters.html", this, state)!$
+        $!nav("forwardproxy.html", this, state)!$
+        $!nav("proxyauth.html", this, state)!$
+        $!nav("replacements.html", this, state)!$
         $!nav("serverreplay.html", this, state)!$
         $!nav("setheaders.html", this, state)!$
-        $!nav("passthrough.html", this, state)!$
-        $!nav("proxyauth.html", this, state)!$
-        $!nav("reverseproxy.html", this, state)!$
-        $!nav("responsestreaming.html", this, state)!$
-        $!nav("socksproxy.html", this, state)!$
         $!nav("sticky.html", this, state)!$
-        $!nav("tcpproxy.html", this, state)!$
-        $!nav("upstreamproxy.html", this, state)!$
+        $!nav("reverseproxy.html", this, state)!$
         $!nav("upstreamcerts.html", this, state)!$
 
-
     <li class="nav-header">Installing Certificates</li>
         $!nav("ssl.html", this, state)!$
         $!nav("certinstall/webapp.html", this, state)!$
@@ -54,6 +46,5 @@
         $!nav("tutorials/transparent-dhcp.html", this, state)!$
 
     <li class="nav-header">Hacking</li>
-        $!nav("dev/architecture.html", this, state)!$
         $!nav("dev/testing.html", this, state)!$
 </ul>

doc-src/certinstall/firefox.html

@@ -7,7 +7,7 @@ app](@!urlTo("webapp.html")!@). If this fails, do the following:
 <ol class="tlist">
     <li> If needed, copy the ~/.mitmproxy/mitmproxy-ca-cert.pem file to the target. </li>
 
-    <li>Open preferences, click on "Advanced", then select"Certificates":
+    <li>Open preferences, click on "Advanced", then select"Encryption":
         <img src="@!urlTo('firefox3.jpg')!@"/>
     </li>
 
@@ -21,7 +21,7 @@ app](@!urlTo("webapp.html")!@). If this fails, do the following:
 ## Installing the certificate
 
 <ol class="tlist">
-    <li>Tick "Trust this CA to identify web sites", and click "Ok":
+    <li>Tick "Trust this CS to identify web sites", and click "Ok":
         <img src="@!urlTo('firefox3-trust.jpg')!@"/>
     </li>
 

doc-src/certinstall/index.py

@@ -9,5 +9,4 @@ pages = [
     Page("ios-simulator.html", "IOS Simulator"),
     Page("android.html", "Android"),
     Page("java.html", "Java"),
-    Page("mitm.it-error.html", "Error: No proxy configured"),
 ]

doc-src/certinstall/mitm.it-error.html

@@ -1,5 +0,0 @@
-**Looks like you wanted to install the mitmproxy CA using the web app?**
-
-Unfortunately, there's been no mitmproxy instance on the wire that could have intercepted your request.
-Please configure your client to use mitmproxy and try again.<br>
-The request to <a href="http://mitm.it/">http://mitm.it/</a> must go through your mitmproxy instance.

doc-src/certinstall/webapp.html

@@ -8,6 +8,3 @@ domain **mitm.it**. You should see something like this:
 
 Just click on the relevant icon, and then follow the setup instructions
 for the platform you're on.
-
-Make sure you aren't using a bandwith optimizer (like Google's Data Compression
-Proxy on Chrome for Android) or the page will not load.

doc-src/config.html

@@ -1,86 +0,0 @@
-
-
-Mitmproxy is configured through a set of files in the users ~/.mitmproxy
-directory. 
-
-<table class="table">
-	<tbody>
-		<tr>
-			<th>mitmproxy.conf</th> 
-			<td>Settings for the <b>mitmproxy</b>. This file can contain any options supported by mitmproxy.</td>
-		</tr>
-		<tr>
-			<th>mitmdump.conf</th> 
-			<td>Settings for the <b>mitmdump</b>. This file can contain any options supported by mitmdump.</td>
-		</tr>
-		<tr>
-			<th>common.conf</th>
-
-			<td>Settings shared between all command-line tools. Settings in
-			this file are over-ridden by those in the tool-specific
-			files. Only options shared by mitmproxy and mitmdump should be used in this file. </td>
-		</tr>
-	</tbody>
-</table>
-
-# Syntax
-
-## Comments
-
-<pre>
-# this is a comment
-; this is also a comment (.ini style)
---- and this is a comment too (yaml style)
-</pre>
-
-## Key/Value pairs
-
-- Keys and values are case-sensitive
-- Whitespace is ignored
-- Lists are comma-delimited, and enclosed in square brackets
-
-<pre>
-name = value   # (.ini style) 
-name: value    # (yaml style)
---name value   # (command-line option style)
-
-fruit = [apple, orange, lemon]
-indexes = [1, 12, 35 , 40]
-</pre>
-
-## Flags
-
-These are boolean options that take no value but true/false.
-
-<pre>
-name = true    # (.ini style)
-name
---name 	 	   # (command-line option style)
-</pre>
-
-# Options
-
-The options available in the config files are precisely those available as
-command-line flags, with the key being the option's long name. To get a
-complete list of these, use the __--help__ option on each of the tools. Be
-careful to only specify common options in the __common.conf__ file -
-unsupported options in this file will be detected as an error on startup.
-
-# Examples
-
-## common.conf
-
-Note that __port__ is an option supported by all tools. 
-
-<pre class="code">
-port = 8080
-</pre>
-
-
-## mitmproxy.conf
-
-<pre class="code">
-palette = light
-</pre>
-
-

doc-src/custom-routing.txt

@@ -1,24 +0,0 @@
-# Adapted from http://tldp.org/HOWTO/TransparentProxy-6.html (6.2 Second method)
-# Note that the choice of firewall mark (3) and routing table (2) was fairly arbitrary.
-# If you are already using policy routing or firewall marking for some other purpose,
-# make sure you choose unique numbers here. Otherwise, don't worry about it.
-
-
-
-# On the router, run
-
-PROXY_IP=192.168.1.100
-TARGET_IP=192.168.1.110
-
-iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s ! $TARGET_IP
-# Alternative to MITM the whole network:
-# iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s  $PROXY_IP
-iptables -t mangle -A PREROUTING -j MARK --set-mark 3 -p tcp -m multiport --dports 80,443
-ip rule add fwmark 3 table 2
-ip route add default via $PROXY_IP dev br0 table 2
-
-
-
-# On the proxy machine, run
-
-iptables -A PREROUTING -t nat -i eth0 -p tcp -m multiport --dports 80,443 -j REDIRECT --to-port 8080

doc-src/dev/architecture.html

@@ -1,8 +0,0 @@
-To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed
-in the following graphic:
-
-<img src="@!urlTo('schematics/architecture.png')!@">
-
-<a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
-<p>Please don't refrain from asking any further
-questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>

doc-src/dev/index.py

@@ -2,6 +2,5 @@ from countershape import Page
 
 pages = [
      Page("testing.html", "Testing"),
-     Page("architecture.html", "Architecture"),
 #    Page("addingviews.html", "Writing Content Views"),
 ]

doc-src/features/forwardproxy.html

@@ -0,0 +1,16 @@
+
+In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
+requests to a specified upstream server. This is in contrast to <a
+href="@!urlTo("reverseproxy.html")!@">reverse proxy mode</a>, in which
+mitmproxy forwards ordinary HTTP requests to an upstream server.
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>-F http[s]://hostname[:port]</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>F</b></td>
+        </tr>
+    </tbody>
+</table>

doc-src/features/index.py

@@ -4,16 +4,12 @@ pages = [
     Page("anticache.html", "Anticache"),
     Page("clientreplay.html", "Client-side replay"),
     Page("filters.html", "Filter expressions"),
-    Page("passthrough.html", "Ignore Domains"),
-    Page("proxyauth.html", "Proxy Authentication"),
-    Page("replacements.html", "Replacements"),
-    Page("responsestreaming.html", "Response Streaming"),
-    Page("reverseproxy.html", "Reverse proxy mode"),
-    Page("socksproxy.html", "SOCKS Mode"),
+    Page("forwardproxy.html", "Forward proxy mode"),
     Page("setheaders.html", "Set Headers"),
     Page("serverreplay.html", "Server-side replay"),
     Page("sticky.html", "Sticky cookies and auth"),
-    Page("tcpproxy.html", "TCP Proxy"),
+    Page("proxyauth.html", "Proxy Authentication"),
+    Page("replacements.html", "Replacements"),
+    Page("reverseproxy.html", "Reverse proxy mode"),
     Page("upstreamcerts.html", "Upstream Certs"),
-    Page("upstreamproxy.html", "Upstream proxy mode"),
-]
+]

doc-src/features/passthrough.html

@@ -1,81 +0,0 @@
-There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
-
-- **Certificate pinning:** Some traffic is is protected using
-  [certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's
-  interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active.
-- **Convenience:** You really don't care about some parts of the traffic and just want them to go away.
-
-If you want to peek into (SSL-protected) non-HTTP connections, check out the [tcp proxy](@!urlTo("tcpproxy.html")!@) feature.
-If you want to ignore traffic from mitmproxy's processing because of large response bodies, take a look at the
-[response streaming](@!urlTo("responsestreaming.html")!@) feature.
-
-## How it works
-
-
-<table class="table">
-    <tbody>
-        <tr>
-            <th width="20%">command-line</th> <td>--ignore regex</td>
-        </tr>
-        <tr>
-            <th>mitmproxy shortcut</th> <td><b>I</b></td>
-        </tr>
-    </tbody>
-</table>
-
-
-mitmproxy allows you to specify a regex which is matched against a <code>host:port</code> string (e.g. "example.com:443")
-to determine hosts that should be excluded. 
-
-There are two important quirks to consider:
-
-- **In transparent mode, the ignore pattern is matched against the IP.** While we usually infer the hostname from the 
-  Host header if the --host argument is passed to mitmproxy, we do not have access to this information before the SSL
-  handshake.
-- In regular mode, explicit HTTP requests are never ignored.[^explicithttp] The ignore pattern is applied on CONNECT
-  requests, which initiate HTTPS or clear-text WebSocket connections.
-
-
-### Tutorial
-
-If you just want to ignore one specific domain, there's usually a bulletproof method to do so:
-
-1. Run mitmproxy or mitmdump in verbose mode (-v) and observe the host:port information in the serverconnect 
-messages. mitmproxy will filter on these.
-2. Take the host:port string, surround it with ^ and $, escape all dots (. becomes \\.) 
-and use this as your ignore pattern:
-
-<pre class="terminal">
-$ mitmdump -v
-127.0.0.1:50588: clientconnect
-127.0.0.1:50588: request
-  -> CONNECT example.com:443 HTTP/1.1
-127.0.0.1:50588: Set new server address: example.com:443
-<span style="color: white">127.0.0.1:50588: serverconnect
-  -> example.com:443</span>
-^C
-$ <span style="color: white">mitmproxy --ignore ^example\.com:443$</span>    
-</pre>
-
-Here are some other examples for ignore patterns:
-<pre>
-# Exempt traffic from the iOS App Store (usually just works):
---ignore apple.com:443  
-# "Correct" version without false-positives:
---ignore ^(.+\.)?apple\.com:443$ 
-    
-# Ignore example.com on all ports, but no subdomains:
---ignore ^example.com:
-    
-# Transparent mode:
---ignore 17\.178\.96\.59:443
-# IP address range:
---ignore 17\.178\.\d+\.\d+:443
-</pre>
-
-### See Also
-
-- [TCP Proxy](@!urlTo("tcpproxy.html")!@)
-- [Response Streaming](@!urlTo("responsestreaming.html")!@)
-
-[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.

doc-src/features/responsestreaming.html

@@ -1,54 +0,0 @@
-By using mitmproxy's streaming feature, response contents can be passed to the client incrementally before they have been fully received by the proxy.
-This is especially useful for large binary files such as videos, where buffering the whole file slows down the client's browser.
-
-By default, mitmproxy will read the entire response, perform any indicated
-manipulations on it and then send the (possibly modified) response to
-the client. In some cases this is undesirable and you may wish to "stream"
-the reponse back to the client. When streaming is enabled, the response is
-not buffered on the proxy but directly sent back to the client instead.
-
-<h2>On the command-line</h2>
-
-Streaming can be enabled on the command line for all response bodies exceeding a certain size.  The SIZE argument understands
-k/m/g suffixes, e.g. 3m for 3 megabytes.
-
-<table class="table">
-    <tbody>
-    <tr>
-        <th width="20%">command-line</th>
-        <td>
-            --stream SIZE
-        </td>
-    </tr>
-    </tbody>
-</table>
-
-
-<h2>Caveats</h2>
-
-When response streaming is enabled, <strong>streamed response contents will not be
-    recorded or preserved in any way.</strong>
-
-When response streaming is enabled, the response body cannot be modified.
-
-<h2>Customizing Response Streaming</h2>
-
-You can also use an <a href="@!urlTo("scripting/inlinescripts.html")!@">inline script</a> to customize exactly
-which responses are streamed.
-
-Responses that should be tagged for streaming by setting their respective .stream attribute to True:
-
-$!example("examples/stream.py")!$
-
-
-<h2>Implementation Details</h2>
-
-When response streaming is enabled, portions of the code which would have otherwise performed changes
-on the response body will see an empty response body instead (<code>libmproxy.protocol.http.CONTENT_MISSING</code>). Any modifications will be ignored.
-
-Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding:
-    chunked</code> header, the response will be streamed one chunk at a time.
-
-### See Also
-
-- [Ignore Domains](@!urlTo("passthrough.html")!@)

doc-src/features/reverseproxy.html

@@ -1,52 +1,19 @@
 
 In reverse proxy mode, mitmproxy accepts standard HTTP requests and forwards
-them to the specified upstream server. This is in contrast to
-<a href="@!urlTo("upstreamproxy.html")!@">upstream proxy mode</a>, in which
-mitmproxy forwards HTTP proxy requests to an upstream proxy server.
+them to the specified upstream server. This is in contrast to <a
+href="@!urlTo("forwardproxy.html")!@">forward proxy mode</a>, in which
+mitmproxy forwards HTTP proxy requests to an upstream server.
+
+Note that the displayed URL for flows in this mode will use the value of the
+__Host__ header field from the request, not the reverse proxy server. 
 
 <table class="table">
     <tbody>
         <tr>
-            <th width="20%">command-line</th> <td>-R <i>schema</i>://hostname[:port]</td>
+            <th width="20%">command-line</th> <td>-P http[s]://hostname[:port]</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>P</b></td>
         </tr>
     </tbody>
 </table>
-
-Here, **schema** is one of http, https, http2https or https2http. The latter
-two extended schema specifications control the use of HTTP and HTTPS on
-mitmproxy and the upstream server. You can indicate that mitmproxy should use
-HTTP, and the upstream server uses HTTPS like this:
-
-	http2https://hostname:port
-
-And you can indicate that mitmproxy should use HTTPS while the upstream
-service uses HTTP like this:
-
-	https2http://hostname:port
-
-
-### Host Header
-
-In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this
-may lead to issues with public web servers. For example, consider the following scenario:
-
-    $ python mitmdump -d -R http://example.com/ &
-    $ curl http://localhost:8080/
-
-    >> GET https://example.com/
-        Host: localhost:8080
-        User-Agent: curl/7.35.0
-        [...]
-
-    << 404 Not Found 345B
-
-Since the Host header doesn't match <samp>example.com</samp>, an error is returned.<br>
-There are two ways to solve this:
-<ol>
-    <li>Modify the hosts file of your OS so that example.com resolves to 127.0.0.1.</li>
-    <li>
-        Instruct mitmproxy to rewrite the host header by passing <kbd>&#8209;&#8209;setheader&nbsp;:~q:Host:example.com</kbd>.
-        However, keep in mind that absolute URLs within the returned document or HTTP redirects will cause the client application
-        to bypass the proxy.
-    </li>
-</ol>

doc-src/features/socksproxy.html

@@ -1,10 +0,0 @@
-
-In this mode, mitmproxy acts as a SOCKS5 proxy server.
-
-<table class="table">
-    <tbody>
-        <tr>
-            <th width="20%">command-line</th> <td>--socks</td>
-        </tr>
-    </tbody>
-</table>

doc-src/features/tcpproxy.html

@@ -1,30 +0,0 @@
-WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt hostnames from
-processing, so that mitmproxy acts as a generic TCP forwarder. This feature is closely related to the
-[ignore domains](@!urlTo("passthrough.html")!@) functionality, but differs in two important aspects:
-
-- The raw TCP messages are printed to the event log.
-- SSL connections will be intercepted.
-
-Please note that message interception or modification are not possible yet.
-If you are not interested in the raw TCP messages, you should use the ignore domains feature.
-
-## How it works
-
-
-<table class="table">
-    <tbody>
-        <tr>
-            <th width="20%">command-line</th> <td>--tcp HOST</td>
-        </tr>
-        <tr>
-            <th>mitmproxy shortcut</th> <td><b>T</b></td>
-        </tr>
-    </tbody>
-</table>
-
-For a detailed description on the structure of the hostname pattern, please refer to the [Ignore Domains](@!urlTo("passthrough.html")!@) feature.
-
-### See Also
-
-- [Ignore Domains](@!urlTo("passthrough.html")!@)
-- [Response Streaming](@!urlTo("responsestreaming.html")!@)

doc-src/features/upstreamproxy.html

@@ -1,27 +0,0 @@
-
-In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
-requests to a specified upstream proxy server. This is in contrast to <a
-href="@!urlTo("reverseproxy.html")!@">reverse proxy mode</a>, in which
-mitmproxy forwards ordinary HTTP requests to an upstream server.
-
-<table class="table">
-    <tbody>
-        <tr>
-            <th width="20%">command-line</th> <td>-U http://hostname[:port]</td>
-        </tr>
-    </tbody>
-</table>
-
-Here, **schema** is one of http, https, http2https or https2http. The latter
-two extended schema specifications control the use of HTTP and HTTPS on
-mitmproxy and the upstream server. You can indicate that mitmproxy should use
-HTTP, and the upstream server uses HTTPS like this:
-
-	http2https://hostname:port
-
-And you can indicate that mitmproxy should use HTTPS while the upstream
-service uses HTTP like this:
-
-	https2http://hostname:port
-
-

doc-src/index.html

@@ -1,27 +1,4 @@
 
-__mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
-with a console interface.
-
-__mitmdump__ is the command-line version of mitmproxy. Think tcpdump for HTTP.
-
-__libmproxy__ is the library that mitmproxy and mitmdump are built on.
-
-Documentation, tutorials and distribution packages can be found on the
-mitmproxy.org website:
-
-[mitmproxy.org](http://mitmproxy.org).
+@!index_contents!@
 
 
-Features
---------
-
-- Intercept HTTP requests and responses and modify them on the fly.
-- Save complete HTTP conversations for later replay and analysis.
-- Replay the client-side of an HTTP conversations.
-- Replay HTTP responses of a previously recorded server.
-- Reverse proxy mode to forward traffic to a specified server.
-- Transparent proxy mode on OSX and Linux.
-- Make scripted changes to HTTP traffic using Python.
-- SSL certificates for interception are generated on the fly.
-- And much, much more.
-

doc-src/index.py

@@ -1,16 +1,11 @@
-import os
-import sys
-import datetime
+import os, sys, datetime
 import countershape
-from countershape import Page, Directory, markup, model
+from countershape import Page, Directory, PythonModule, markup, model
 import countershape.template
-
-MITMPROXY_SRC = os.path.abspath(
-    os.path.expanduser(os.environ.get("MITMPROXY_SRC", ".."))
-)
-sys.path.insert(0, MITMPROXY_SRC)
+sys.path.insert(0, "..")
 from libmproxy import filt, version
 
+MITMPROXY_SRC = os.environ.get("MITMPROXY_SRC", os.path.abspath(".."))
 ns.VERSION = version.VERSION
 
 if ns.options.website:
@@ -28,22 +23,44 @@ ns.docMaintainer = "Aldo Cortesi"
 ns.docMaintainerEmail = "aldo@corte.si"
 ns.copyright = u"\u00a9 mitmproxy project, %s" % datetime.date.today().year
 
-
 def mpath(p):
     p = os.path.join(MITMPROXY_SRC, p)
     return os.path.expanduser(p)
 
+with open(mpath("README.mkd")) as f:
+        readme = f.read()
+        ns.index_contents = readme.split("\n", 1)[1] #remove first line (contains build status)
 
 def example(s):
     d = file(mpath(s)).read().rstrip()
     extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
     return extemp%(countershape.template.Syntax("py")(d), s)
-
-
 ns.example = example
 
 
-ns.filt_help = filt.help
+filt_help = []
+for i in filt.filt_unary:
+    filt_help.append(
+        ("~%s"%i.code, i.help)
+    )
+for i in filt.filt_rex:
+    filt_help.append(
+        ("~%s regex"%i.code, i.help)
+    )
+for i in filt.filt_int:
+    filt_help.append(
+        ("~%s int"%i.code, i.help)
+    )
+filt_help.sort()
+filt_help.extend(
+    [
+        ("!", "unary not"),
+        ("&", "and"),
+        ("|", "or"),
+        ("(...)", "grouping"),
+    ]
+)
+ns.filt_help = filt_help
 
 
 def nav(page, current, state):
@@ -56,16 +73,12 @@ def nav(page, current, state):
 ns.nav = nav
 ns.navbar = countershape.template.File(None, "_nav.html")
 
-
 pages = [
     Page("index.html", "Introduction"),
     Page("install.html", "Installation"),
-    Page("howmitmproxy.html", "How mitmproxy works"),
-    Page("modes.html", "Modes of Operation"),
-
     Page("mitmproxy.html", "mitmproxy"),
     Page("mitmdump.html", "mitmdump"),
-    Page("config.html", "configuration"),
+    Page("howmitmproxy.html", "How mitmproxy works"),
 
     Page("ssl.html", "Overview"),
     Directory("certinstall"),

doc-src/install.html

@@ -1,33 +1,40 @@
 
-## Installing from source
-
 The preferred way to install mitmproxy - whether you're installing the latest
 release or from source - is to use [pip](http://www.pip-installer.org/). If you
 don't already have pip on your system, you can find installation instructions
 [here](http://www.pip-installer.org/en/latest/installing.html).
 
+
+## Installing the latest release
+
+A single command will download and install the latest release of mitmproxy,
+along with all its dependencies:
+
 <pre class="terminal">
 pip install mitmproxy
 </pre>
 
-If you also want to install the optional packages AMF, protobuf and CSS
-content views, do this:
 
+## Installing from source
+
+When installing from source, the easiest method is still to use pip. In this
+case run:
+    
 <pre class="terminal">
-pip install "mitmproxy[contentviews]"
+pip install /path/to/source
 </pre>
 
+Note that if you're installing current git master, you will also have to
+install the current git master of [netlib](http://github.com/mitmproxy/netlib) by
+hand.
 
 ## OSX
 
-The easiest way to get up and running on OSX is to download the pre-built
-binary packages from [mitmproxy.org](http://mitmproxy.org). If you still want
-to install using pip, there are a few things to keep in mind:
-
 - If you're running a Python interpreter installed with homebrew (or similar),
 you may have to install some dependencies by hand. 
 - Make sure that XCode is installed from the App Store, and that the
 command-line tools have been downloaded (XCode/Preferences/Downloads).
+- Now use __pip__ to do the installation, as above.
 
 There are a few bits of customization you might want to do to make mitmproxy
 comfortable to use on OSX. The default color scheme is optimized for a dark
@@ -43,17 +50,3 @@ image/*; /usr/bin/open -Wn %s
 video/*; /usr/bin/open -Wn %s
 </pre>
 
-
-## Ubuntu
-
-On Ubuntu, you will need the following native packages to install mitmproxy
-from source:
-
-- build-essential
-- python-dev
-- libffi-dev
-- libssl-dev
-- libxml2-dev
-- libxslt1-dev
-
-

doc-src/modes.html

@@ -1,222 +0,0 @@
-
-Mitmproxy has four modes of operation that allow you to use mitmproxy in a
-variety of scenarios:
-
-- **Regular** (the default)
-- **Transparent**
-- **Reverse Proxy**
-- **Upstream Proxy**
-
-Now, which one should you pick? Use this flow chart:
-
-<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
-
-<div class="page-header">
-    <h1>Regular Proxy</h1>
-</div>
-
-Mitmproxy's regular mode is the simplest and the easiest to set up.
-
-1. Start mitmproxy.
-2. Configure your client to use mitmproxy. For instance on IOS, the settings might look like <a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>.
-3. Quick Check: You should already be able to visit an unencrypted HTTP site
-through the proxy.
-4. Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.
-
-<div class="well">
-<strong>Heads Up:</strong> Unfortunately, some applications bypass the
-system HTTP proxy settings - Android applications are a common example. In
-these cases, you need to use mitmproxy's transparent mode.
-</div>
-
-If you are proxying an external device, your network will probably look like this:
-
-<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
-
-The square brackets signify the source and destination IP addresses. Your
-client explicitly connects to mitmproxy and mitmproxy explicitly connects
-to the target server.
-
-<div class="page-header">
-    <h1>Transparent Proxy</h1>
-</div>
-
-In transparent mode, traffic is directed into a proxy at the network layer,
-without any client configuration required. This makes transparent proxying
-ideal for situations where you can't change client behaviour. In the graphic
-below, a machine running mitmproxy has been inserted between the router and
-the internet:
-
-<a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
-</a>
-
-The square brackets signify the source and destination IP addresses. Round
-brackets mark the next hop on the *Ethernet/data link* layer. This distinction
-is important: when the packet arrives at the mitmproxy machine, it must still
-be addressed to the target server. This means that Network Address Translation
-should not be applied before the traffic reaches mitmproxy, since this would
-remove the target information, leaving mitmproxy unable to determine the real
-destination.
-
-<a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
-
-<h2>Common Configurations</h2>
-
-There are many ways to configure your network for transparent proxying. We'll
-look at three common scenarios:
-
-1. Configuring the client to use a custom gateway/router/"next hop"
-2. Implementing custom routing on the router
-
-In most cases, the first option is recommended due to its ease of use.
-
-<h3>(a) Custom Gateway</h3>
-
-One simple way to get traffic to the mitmproxy machine with the destination IP
-intact, is to simply configure the client with the mitmproxy box as the
-default gateway.
-
-<a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
-
-In this scenario, we would:
-
-- Configure the proxy machine for transparent mode. You can find instructions
-in the <em>Transparent Proxying</em> section of the mitmproxy docs.
-
-- Configure the client to use the proxy machine's IP as the default gateway.
-<a href="@!urlTo('screenshots/ios-gateway.png')!@">Here</a> is what this would
-look like on IOS.
-
-- Quick Check: At this point, you should already be able to visit an
-unencrypted HTTP site over the proxy.
-
-- Open the magic domain <strong>mitm.it</strong> and install the certificate
-for your device.
-
-Setting the custom gateway on clients can be automated by serving the settings
-out to clients over DHCP. This lets set up an interception network where all
-clients are proxied automatically, which can save time and effort. 
-
-
-<div class="well">
-    <strong style="text-align: center; display: block">Troubleshooting Transparent Mode</strong>
-
-    <p>Incorrect transparent mode configurations are a frequent source of
-    error. If it doesn't work for you, try the following things:</p>
-
-    <ul>
-        <li>
-            Open mitmproxy's event log (press `e`) - do you see clientconnect
-            messages? If not, the packets are not arriving at the proxy. One common
-            cause is the occurrence of ICMP redirects, which means that your
-            machine is telling the client that there's a faster way to the
-            internet by contacting your router directly (see the
-            <em>Transparent Proxying</em> section on how to disable them). If in
-            doubt, <a href="https://wireshark.org/">Wireshark</a> may help you
-            to see whether something arrives at your machine or not.
-        </li>
-        <li>
-            Make sure you have not explicitly configured an HTTP proxy on the
-            client. This is not needed in transparent mode.
-        </li>
-        <li>
-            Re-check the instructions in the <em>Transparent Proxying</em> section. Anything you missed?
-        </li>
-    </ul>
-
-    If you encounter any other pitfalls that should be listed here, please let us know!
-
-</div>
-
-<h3>(b) Custom Routing</h3>
-
-In some cases, you may need more fine-grained control of which traffic reaches
-the mitmproxy instance, and which doesn't. You may, for instance, choose only
-to divert traffic to some hosts into the transparent proxy. There are a huge
-number of ways to accomplish this, and much will depend on the router or
-packet filter you're using. In most cases, the configuration will look like
-this:
-
-<a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
-</a>
-
-
-<div class="page-header">
-    <h1>Reverse Proxy</h1>
-</div>
-
-Mitmproxy is usually used with a client that uses the proxy to access the
-Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
-HTTP server:
-
-<a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
-</a>
-
-There are various use-cases:
-
-- Say you have an internal API running at http://example.local/. You could now
-set up mitmproxy in reverse proxy mode at http://debug.example.local/ and
-dynamically point clients to this new API endpoint, which provides clients
-with the same data and you with debug information. Similarly, you could move
-your real server to a different IP/port and set up mitmproxy at the original
-place to debug all sessions.
-
-- Say you're a web developer working on example.com (with a development
-version running on localhost:8000). You can modify your hosts file so that
-example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy mode
-on port 80. You can test your app on the example.com domain and get all
-requests recorded in mitmproxy.
-
-- Say you have some toy project that should get SSL support. Simply set up
-mitmproxy with SSL termination and you're done (<code>mitmdump -p 443 -R
-https2http://localhost:80/</code>). There are better tools for this specific
-task, but mitmproxy is very quick and simple way to set up an SSL-speaking
-server.
-
-- Want to add a non-SSL-capable compression proxy in front of your server? You
-could even spawn a mitmproxy instance that terminates SSL (https2http://...),
-point it to the compression proxy and let the compression proxy point to a
-SSL-initiating mitmproxy (http2https://...), which then points to the real
-server. As you see, it's a fairly flexible thing.
-
-Note that mitmproxy supports either an HTTP or an HTTPS upstream server, not
-both at the same time. You can work around this by spawning a second mitmproxy
-instance.
-
-<div class="well">
-    <strong style="text-align: center; display: block">Caveat: Interactive Use</strong>
-
-
-One caveat is that reverse proxy mode is often not sufficient for interactive
-browsing. Consider trying to clone Google by using:
-
-<code>mitmproxy -R http://google.com/</code>
-
-This works for the initial request, but the HTML served to the client remains
-unchanged. As soon as the user clicks on an non-relative URL (or downloads a
-non-relative image resource), traffic no longer passes through mitmproxy, and
-the client connects to Google directly again.
-
-</div>
-
-
-
-<div class="page-header">
-    <h1>Upstream Proxy</h1>
-</div>
-
-If you want to chain proxies by adding mitmproxy in front of a different proxy
-appliance, you can use mitmproxy's upstream mode. In upstream mode, all
-requests are unconditionally transferred to an upstream proxy of your choice.
-
-<a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
-
-mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
-mode. You could in theory chain multiple mitmproxy instances in a row, but
-that doesn't make any sense in practice (i.e. outside of our tests). 

doc-src/scripting/inlinescripts.html

@@ -1,5 +1,5 @@
 __mitmproxy__ has a powerful scripting API that allows you to modify flows
-on-the-fly or rewrite previously saved flows locally.
+on-the-fly or rewrite previously saved flows locally. 
 
 The mitmproxy scripting API is event driven - a script is simply a Python
 module that exposes a set of event methods. Here's a complete mitmproxy script
@@ -21,11 +21,6 @@ We can now run this script using mitmdump or mitmproxy as follows:
 
 The new header will be added to all responses passing through the proxy.
 
-## Example Scripts
-
-mitmproxy comes with a variety of example inline scripts, which demonstrate
-many basic tasks. We encourage you to either browse them locally or in our
-[GitHub repo](https://github.com/mitmproxy/mitmproxy/tree/master/examples).
 
 
 ## Events
@@ -35,45 +30,38 @@ many basic tasks. We encourage you to either browse them locally or in our
 Called once on startup, before any other events.
 
 
-### clientconnect(ScriptContext, ConnectionHandler)
+### clientconnect(ScriptContext, ClientConnect)
 
 Called when a client initiates a connection to the proxy. Note that
 a connection can correspond to multiple HTTP requests.
 
-### serverconnect(ScriptContext, ConnectionHandler)
+
+### serverconnect(ScriptContext, ServerConnection)
 
 Called when the proxy initiates a connection to the target server. Note that
 a connection can correspond to multiple HTTP requests.
 
-### request(ScriptContext, HTTPFlow)
+### request(ScriptContext, Flow)
 
-Called when a client request has been received. The __HTTPFlow__ object is
+Called when a client request has been received. The __Flow__ object is
 guaranteed to have a non-None __request__ attribute.
 
-### responseheaders(ScriptContext, HTTPFlow)
 
-Called when the headers of a server response have been received.
-This will always be called before the response hook.
-The __HTTPFlow__ object is guaranteed to have non-None __request__ and
-__response__ attributes.  __response.content__ will be None,
-as the response body has not been read yet.
+### response(ScriptContext, Flow)
 
-### response(ScriptContext, HTTPFlow)
-
-Called when a server response has been received. The __HTTPFlow__ object is
+Called when a server response has been received. The __Flow__ object is
 guaranteed to have non-None __request__ and __response__ attributes.
-Note that if response streaming is enabled for this response,
-__response.content__ will not contain the response body.
 
-### error(ScriptContext, HTTPFlow)
+
+### error(ScriptContext, Flow)
 
 Called when a flow error has occurred, e.g. invalid server responses, or
 interrupted connections. This is distinct from a valid server HTTP error
-response, which is simply a response with an HTTP error code. The __HTTPFlow__
+response, which is simply a response with an HTTP error code. The __Flow__
 object is guaranteed to have non-None __request__ and __error__ attributes.
 
 
-### clientdisconnect(ScriptContext, ConnectionHandler)
+### clientdisconnect(ScriptContext, ClientDisconnect)
 
 Called when a client disconnects from the proxy.
 
@@ -88,46 +76,43 @@ The main classes you will deal with in writing mitmproxy scripts are:
 
 <table class="table">
     <tr>
-        <th>libmproxy.proxy.server.ConnectionHandler</th>
-        <td>Describes a proxy client connection session. Always has a client_conn attribute, might have a server_conn
-            attribute.
-        </td>
-    </tr>
-    <tr>
-        <th>libmproxy.proxy.connection.ClientConnection</th>
+        <th>libmproxy.flow.ClientConnection</th>
         <td>Describes a client connection.</td>
     </tr>
     <tr>
-        <th>libmproxy.proxy.connection.ServerConnection</th>
-        <td>Describes a server connection.</td>
+        <th>libmproxy.flow.ClientDisconnection</th>
+        <td>Describes a client disconnection.</td>
     </tr>
     <tr>
-        <th>libmproxy.protocol.http.HTTPFlow</th>
-        <td>A collection of objects representing a single HTTP transaction.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.protocol.http.HTTPResponse</th>
-        <td>An HTTP response.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.protocol.http.HTTPRequest</th>
-        <td>An HTTP request.</td>
-    </tr>
-    <tr>
-        <th>libmproxy.protocol.primitives.Error</th>
+        <th>libmproxy.flow.Error</th>
         <td>A communications error.</td>
     </tr>
     <tr>
-        <th>libmproxy.script.ScriptContext</th>
-        <td> A handle for interacting with mitmproxy's from within scripts.</td>
+        <th>libmproxy.flow.Flow</th>
+        <td>A collection of objects representing a single HTTP transaction.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.flow.Headers</th>
+        <td>HTTP headers for a request or response.</td>
     </tr>
     <tr>
         <th>libmproxy.flow.ODict</th>
 
         <td>A dictionary-like object for managing sets of key/value data. There
-            is also a variant called CaselessODict that ignores key case for some
-            calls (used mainly for headers).
-        </td>
+        is also a variant called CaselessODict that ignores key case for some
+        calls (used mainly for headers).</td>
+    </tr>
+    <tr>
+        <th>libmproxy.flow.Response</th>
+        <td>An HTTP response.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.flow.Request</th>
+        <td>An HTTP request.</td>
+    </tr>
+    <tr>
+        <th>libmproxy.flow.ScriptContext</th>
+        <td> A handle for interacting with mitmproxy's from within scripts.  </td>
     </tr>
     <tr>
         <th>libmproxy.certutils.SSLCert</th>
@@ -135,12 +120,11 @@ The main classes you will deal with in writing mitmproxy scripts are:
     </tr>
 </table>
 
-The canonical API documentation is the code, which you can browse locally or in our
-[GitHub repo](https://github.com/mitmproxy/mitmproxy).
-You can view the API documentation using pydoc (which is installed with Python by default), like this:
+The canonical API documentation is the code. You can view the API documentation
+using pydoc (which is installed with Python by default), like this:
 
 <pre class="terminal">
-> pydoc libmproxy.protocol.http.HTTPRequest
+> pydoc libmproxy.flow.Request
 </pre>
 
 
@@ -151,13 +135,6 @@ While that's a very desirable behaviour under some circumstances, scripts can be
 
 $!example("examples/nonblocking.py")!$
 
-## Make scripts configurable with arguments
-
-Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by surrounding the script call with quotes, e.g.
-<code>mitmdump -s "script.py --foo 42"</code>. The arguments are then exposed in the start event:
-
-$!example("examples/modify_response_body.py")!$
-
 ## Running scripts on saved flows
 
 Sometimes, we want to run a script on __Flow__ objects that are already
@@ -167,11 +144,6 @@ flows from a file (see the "scripted data transformation" example on the
 one-shot script on a single flow through the _|_ (pipe) shortcut in mitmproxy.
 
 In this case, there are no client connections, and the events are run in the
-following order: __start__, __request__, __responseheaders__, __response__, __error__, __done__.  If
+following order: __start__, __request__, __response__, __error__, __done__.  If
 the flow doesn't have a __response__ or __error__ associated with it, the
-matching events will be skipped.
-
-## Spaces in the script path
-By default, spaces are interpreted as separator between the inline script and its arguments (e.g. <code>-s "foo.py
-    42"</code>). Consequently, the script path needs to be wrapped in a separate pair of quotes if it contains spaces:
-<code>-s "'./foo bar/baz.py' 42"</code>.
+matching event will be skipped. 

doc-src/scripting/libmproxy.html

@@ -1,13 +1,3 @@
-<div class="well">
-    <strong>Heads up!</strong> We strongly encourage you to use <a href="@!urlTo("scripting/inlinescripts.html")!@">inline scripts</a> rather than libmproxy
-    directly.<br><br>
-    <ul>
-        <li>Inline Scripts are equally powerful and provide an easier syntax.</li>
-        <li>Most examples are written as inline scripts.</li>
-        <li>Multiple inline scripts can be combined and used together.</li>
-        <li>Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI.</li>
-    </ul>
-</div>
 
 All of mitmproxy's basic functionality is exposed through the __libmproxy__
 library. The example below shows a simple implementation of the "sticky cookie"

doc-src/ssl.html

@@ -41,26 +41,10 @@ The files created by mitmproxy in the .mitmproxy directory are as follows:
 Using a custom certificate
 --------------------------
 
-You can use your own certificate by passing the <kbd>--cert</kbd> option to mitmproxy. mitmproxy then uses the provided
-certificate for interception of the specified domains instead of generating a cert signed by its own CA.
+You can use your own certificate by passing the __--cert__ option to mitmproxy.
 
-The certificate file is expected to be in the PEM format.
-You can include intermediary certificates right below your leaf certificate, so that you PEM file roughly looks like
-this:
-
-<pre>
------BEGIN PRIVATE KEY-----
-&lt;private key&gt;
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-&lt;cert&gt;
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-&lt;intermediary cert (optional)&gt;
------END CERTIFICATE-----
-</pre>
-
-For example, you can generate a certificate in this format using these instructions:
+The certificate file is expected to be in the PEM format. You can generate
+a certificate in this format using these instructions:
 
 <pre class="terminal">
 > openssl genrsa -out cert.key 8192
@@ -71,15 +55,6 @@ For example, you can generate a certificate in this format using these instructi
 </pre>
 
 
-Using a custom certificate authority
-------------------------------------
-
-By default, mitmproxy will (generate and) use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as the default certificate
-authority to generate certificates for all domains for which no custom certificate is provided (see above).
-You can use your own certificate authority by passing the <kbd>--confdir</kbd> option to mitmproxy.
-mitmproxy will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If no such file exists,
-it will be generated automatically.
-
 Installing the mitmproxy CA
 ---------------------------
 

doc-src/transparent/linux.html

@@ -15,16 +15,6 @@ achieve transparent mode.
 
     </li>
 
-    <li> If your target machine is on the same physical network and you configured it to use a custom gateway,
-    disable ICMP redirects:
-
-    <pre class="terminal">echo 0 | sudo tee /proc/sys/net/ipv4/conf/*/send_redirects</pre>
-
-    You may also want to consider enabling this permanently in
-    <b>/etc/sysctl.conf</b> as demonstrated <a href="http://unix.stackexchange.com/a/58081">here</a>.
-
-    </li>
-
     <li> Create an iptables ruleset that redirects the desired traffic to the
     mitmproxy port. Details will differ according to your setup, but the
     ruleset should look something like this:

doc-src/tutorials/gamecenter.html

@@ -67,8 +67,8 @@ timestamp. Looks pretty simple to mess with.
 Lets edit the score submission. First, select it in mitmproxy, then press
 __enter__ to view it. Make sure you're viewing the request, not the response -
 you can use __tab__ to flick between the two. Now press __e__ for edit. You'll
-be prompted for the part of the request you want to change - press __r__ for
-raw body.  Your preferred editor (taken from the EDITOR environment variable) will
+be prompted for the part of the request you want to change - press __b__ for
+body.  Your preferred editor (taken from the EDITOR environment variable) will
 now fire up. Lets bump the score up to something a bit more ambitious:
 
 <!--(block|syntax("xml"))-->

examples/README

@@ -1,23 +1,10 @@
-# inline script examples
-add_header.py             Simple script that just adds a header to every request.
-change_upstream_proxy.py  Dynamically change the upstream proxy
-dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
-iframe_injector.py        Inject configurable iframe into pages.
-modify_form.py            Modify all form submissions to add a parameter.
-modify_querystring.py     Modify all query strings to add a parameters.
-modify_response_body.py   Replace arbitrary strings in all responses
-nonblocking.py            Demonstrate parallel processing with a blocking script.
-proxapp.py                How to embed a WSGI app in a mitmproxy server
-redirect_requests.py      Redirect requests or directly reply to them.
-stub.py                   Script stub with a method definition for every event.
-upsidedownternet.py       Rewrites traffic to turn images upside down.
-
-
-# libmproxy examples
-flowbasic                 Basic use of mitmproxy as a library.
-stickycookies             An example of writing a custom proxy with libmproxy.
-
-
-# misc
-read_dumpfile             Read a dumpfile generated by mitmproxy.
-mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X
+add_header.py           Simple script that just adds a header to every request.
+dup_and_replay.py       Duplicates each request, changes it, and then replays the modified request.
+flowbasic               Basic use of mitmproxy as a library.
+modify_form.py          Modify all form submissions to add a parameter.
+modify_querystring.py   Modify all query strings to add a parameters.
+proxapp                 How to embed a WSGI app in a mitmproxy server
+stub.py                 Script stub with a method definition for every event.
+stickycookies           An example of writing a custom proxy with libmproxy.
+upsidedownternet.py     Rewrites traffic to turn PNGs upside down.
+mitmproxywrapper.py     Bracket mitmproxy run with proxy enable/disable on OS X

examples/add_header.py

@@ -1,2 +1,2 @@
 def response(context, flow):
-    flow.response.headers["newheader"] = ["foo"]
+    flow.response.headers["newheader"] = ["foo"]

examples/change_upstream_proxy.py

@@ -1,21 +0,0 @@
-# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
-# in upstream proxy mode.
-#
-# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
-from libmproxy.protocol.http import send_connect_request
-
-alternative_upstream_proxy = ("localhost", 8082)
-def should_redirect(flow):
-    return flow.request.host == "example.com"
-
-
-def request(context, flow):
-    if flow.live and should_redirect(flow):
-
-        # If you want to change the target server, you should modify flow.request.host and flow.request.port
-        # flow.live.change_server should only be used by inline scripts to change the upstream proxy,
-        # unless you are sure that you know what you are doing.
-        server_changed = flow.live.change_server(alternative_upstream_proxy, persistent_change=True)
-        if flow.request.scheme == "https" and server_changed:
-            send_connect_request(flow.live.c.server_conn, flow.request.host, flow.request.port)
-            flow.live.c.establish_ssl(server=True)

examples/dup_and_replay.py

@@ -1,4 +1,4 @@
-def request(context, flow):
-    f = context.duplicate_flow(flow)
-    f.request.path = "/changed"
-    context.replay_request(f)
+def request(ctx, flow):
+   f = ctx.duplicate_flow(flow)
+   f.request.path = "/changed"
+   ctx.replay_request(f)

examples/flowbasic

@@ -3,15 +3,11 @@
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
 
-    Heads Up: In the majority of cases, you want to use inline scripts.
-
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
 import os
-from libmproxy import flow, proxy
-from libmproxy.proxy.server import ProxyServer
-
+from libmproxy import proxy, flow
 
 class MyMaster(flow.FlowMaster):
     def run(self):
@@ -20,25 +16,24 @@ class MyMaster(flow.FlowMaster):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, f):
-        f = flow.FlowMaster.handle_request(self, f)
+    def handle_request(self, r):
+        f = flow.FlowMaster.handle_request(self, r)
         if f:
-            f.reply()
+            r.reply()
         return f
 
-    def handle_response(self, f):
-        f = flow.FlowMaster.handle_response(self, f)
+    def handle_response(self, r):
+        f = flow.FlowMaster.handle_response(self, r)
         if f:
-            f.reply()
+            r.reply()
         print f
         return f
 
 
 config = proxy.ProxyConfig(
-    port=8080,
-    cadir="~/.mitmproxy/"  # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
+    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
 )
 state = flow.State()
-server = ProxyServer(config)
+server = proxy.ProxyServer(config, 8080)
 m = MyMaster(server, state)
 m.run()

examples/har_extractor.py

@@ -1,212 +0,0 @@
-"""
-    This inline script utilizes harparser.HAR from https://github.com/JustusW/harparser
-    to generate a HAR log object.
-"""
-try:
-    from harparser import HAR
-    from pytz import UTC
-except ImportError as e:
-    import sys
-    print >> sys.stderr, "\r\nMissing dependencies: please run `pip install mitmproxy[examples]`.\r\n"
-    raise
-
-from datetime import datetime, timedelta, tzinfo
-
-
-class _HARLog(HAR.log):
-    # The attributes need to be registered here for them to actually be available later via self. This is
-    # due to HAREncodable linking __getattr__ to __getitem__. Anything that is set only in __init__ will
-    # just be added as key/value pair to self.__classes__.
-    __page_list__ = []
-    __page_count__ = 0
-    __page_ref__ = {}
-
-    def __init__(self, page_list):
-        self.__page_list__ = page_list
-        self.__page_count__ = 0
-        self.__page_ref__ = {}
-
-        HAR.log.__init__(self, {"version": "1.2",
-                                "creator": {"name": "MITMPROXY HARExtractor",
-                                            "version": "0.1",
-                                            "comment": ""},
-                                "pages": [],
-                                "entries": []})
-
-    def reset(self):
-        self.__init__(self.__page_list__)
-
-    def add(self, obj):
-        if isinstance(obj, HAR.pages):
-            self['pages'].append(obj)
-        if isinstance(obj, HAR.entries):
-            self['entries'].append(obj)
-
-    def create_page_id(self):
-        self.__page_count__ += 1
-        return "autopage_%s" % str(self.__page_count__)
-
-    def set_page_ref(self, page, ref):
-        self.__page_ref__[page] = ref
-
-    def get_page_ref(self, page):
-        return self.__page_ref__.get(page, None)
-
-    def get_page_list(self):
-        return self.__page_list__
-
-
-def start(context, argv):
-    """
-        On start we create a HARLog instance. You will have to adapt this to suit your actual needs
-        of HAR generation. As it will probably be necessary to cluster logs by IPs or reset them
-        from time to time.
-    """
-    context.dump_file = None
-    if len(argv) > 1:
-        context.dump_file = argv[1]
-    else:
-        raise ValueError('Usage: -s "har_extractor.py filename" '
-                         '(- will output to stdout, filenames ending with .zhar will result in compressed har)')
-    context.HARLog = _HARLog(['https://github.com'])
-    context.seen_server = set()
-
-
-def response(context, flow):
-    """
-       Called when a server response has been received. At the time of this message both
-       a request and a response are present and completely done.
-    """
-    # Values are converted from float seconds to int milliseconds later.
-    ssl_time = -.001
-    connect_time = -.001
-    if flow.server_conn not in context.seen_server:
-        # Calculate the connect_time for this server_conn. Afterwards add it to seen list, in
-        # order to avoid the connect_time being present in entries that use an existing connection.
-        connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start
-        context.seen_server.add(flow.server_conn)
-
-        if flow.server_conn.timestamp_ssl_setup is not None:
-            # Get the ssl_time for this server_conn as the difference between the start of the successful
-            # tcp setup and the successful ssl setup. If  no ssl setup has been made it is left as -1 since
-            # it doesn't apply to this connection.
-            ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup
-
-    # Calculate the raw timings from the different timestamps present in the request and response object.
-    # For lack of a way to measure it dns timings can not be calculated. The same goes for HAR blocked:
-    # MITMProxy will open a server connection as soon as it receives the host and port from the client
-    # connection. So the time spent waiting is actually spent waiting between request.timestamp_end and
-    # response.timestamp_start thus it correlates to HAR wait instead.
-    timings_raw = {'send': flow.request.timestamp_end - flow.request.timestamp_start,
-                   'wait': flow.response.timestamp_start - flow.request.timestamp_end,
-                   'receive': flow.response.timestamp_end - flow.response.timestamp_start,
-                   'connect': connect_time,
-                   'ssl': ssl_time}
-
-    # HAR timings are integers in ms, so we have to re-encode the raw timings to that format.
-    timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()])
-
-    # The full_time is the sum of all timings. Timings set to -1 will be ignored as per spec.
-    full_time = 0
-    for item in timings.values():
-        if item > -1:
-            full_time += item
-
-    started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat()
-
-    request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()]
-    request_http_version = ".".join([str(v) for v in flow.request.httpversion])
-    # Cookies are shaped as tuples by MITMProxy.
-    request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()]
-    request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
-    request_headers_size = len(str(flow.request.headers))
-    request_body_size = len(flow.request.content)
-
-    response_http_version = ".".join([str(v) for v in flow.response.httpversion])
-    # Cookies are shaped as tuples by MITMProxy.
-    response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()]
-    response_headers = [{"name": k, "value": v} for k, v in flow.response.headers]
-    response_headers_size = len(str(flow.response.headers))
-    response_body_size = len(flow.response.content)
-    response_body_decoded_size = len(flow.response.get_decoded_content())
-    response_body_compression = response_body_decoded_size - response_body_size
-    response_mime_type = flow.response.headers.get_first('Content-Type', '')
-    response_redirect_url = flow.response.headers.get_first('Location', '')
-
-    entry = HAR.entries({"startedDateTime": started_date_time,
-                         "time": full_time,
-                         "request": {"method": flow.request.method,
-                                     "url": flow.request.url,
-                                     "httpVersion": request_http_version,
-                                     "cookies": request_cookies,
-                                     "headers": request_headers,
-                                     "queryString": request_query_string,
-                                     "headersSize": request_headers_size,
-                                     "bodySize": request_body_size, },
-                         "response": {"status": flow.response.code,
-                                      "statusText": flow.response.msg,
-                                      "httpVersion": response_http_version,
-                                      "cookies": response_cookies,
-                                      "headers": response_headers,
-                                      "content": {"size": response_body_size,
-                                                  "compression": response_body_compression,
-                                                  "mimeType": response_mime_type},
-                                      "redirectURL": response_redirect_url,
-                                      "headersSize": response_headers_size,
-                                      "bodySize": response_body_size, },
-                         "cache": {},
-                         "timings": timings, })
-
-    # If the current url is in the page list of context.HARLog or does not have a referrer we add it as a new
-    # pages object.
-    if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None:
-        page_id = context.HARLog.create_page_id()
-        context.HARLog.add(HAR.pages({"startedDateTime": entry['startedDateTime'],
-                                      "id": page_id,
-                                      "title": flow.request.url, }))
-        context.HARLog.set_page_ref(flow.request.url, page_id)
-        entry['pageref'] = page_id
-
-    # Lookup the referer in the page_ref of context.HARLog to point this entries pageref attribute to the right
-    # pages object, then set it as a new reference to build a reference tree.
-    elif context.HARLog.get_page_ref(flow.request.headers.get('Referer', (None, ))[0]) is not None:
-        entry['pageref'] = context.HARLog.get_page_ref(flow.request.headers['Referer'][0])
-        context.HARLog.set_page_ref(flow.request.headers['Referer'][0], entry['pageref'])
-
-    context.HARLog.add(entry)
-
-
-def done(context):
-    """
-        Called once on script shutdown, after any other events.
-    """
-    from pprint import pprint
-    import json
-
-    json_dump = context.HARLog.json()
-    compressed_json_dump = context.HARLog.compress()
-
-    print "=" * 100
-    if context.dump_file == '-':
-        pprint(json.loads(json_dump))
-    elif context.dump_file.endswith('.zhar'):
-        file(context.dump_file, "w").write(compressed_json_dump)
-    else:
-        file(context.dump_file, "w").write(json_dump)
-    print "=" * 100
-    print "HAR log finished with %s bytes (%s bytes compressed)" % (len(json_dump), len(compressed_json_dump))
-    print "Compression rate is %s%%" % str(100. * len(compressed_json_dump) / len(json_dump))
-    print "=" * 100
-
-
-def print_attributes(obj, filter_string=None, hide_privates=False):
-    """
-        Useful helper method to quickly get all attributes of an object and its values.
-    """
-    for attr in dir(obj):
-        if hide_privates and "__" in attr:
-            continue
-        if filter_string is not None and filter_string not in attr:
-            continue
-        value = getattr(obj, attr)
-        print "%s.%s" % ('obj', attr), value, type(value)

examples/iframe_injector

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+    Zap encoding in requests and inject iframe after body tag in html responses.
+    Usage:
+        iframe_injector http://someurl/somefile.html
+"""
+from libmproxy import controller, proxy
+import os
+import sys
+
+
+class InjectingMaster(controller.Master):
+    def __init__(self, server, iframe_url):
+        controller.Master.__init__(self, server)
+        self._iframe_url = iframe_url
+
+    def run(self):
+        try:
+            return controller.Master.run(self)
+        except KeyboardInterrupt:
+            self.shutdown()
+
+    def handle_request(self, msg):
+        if 'Accept-Encoding' in msg.headers:
+            msg.headers["Accept-Encoding"] = 'none'
+        msg.reply()
+
+    def handle_response(self, msg):
+        if msg.content:
+            c = msg.replace('<body>', '<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % self._iframe_url)
+            if c > 0:
+                print 'Iframe injected!'
+        msg.reply()
+
+
+def main(argv):
+    if len(argv) != 2:
+        print "Usage: %s IFRAME_URL" % argv[0]
+        sys.exit(1)
+    iframe_url = argv[1]
+    config = proxy.ProxyConfig(
+        cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
+    )
+    server = proxy.ProxyServer(config, 8080)
+    print 'Starting proxy...'
+    m = InjectingMaster(server, iframe_url)
+    m.run()
+
+if __name__ == '__main__':
+    main(sys.argv)

examples/iframe_injector.py

@@ -1,22 +0,0 @@
-# Usage: mitmdump -s "iframe_injector.py url"
-# (this script works best with --anticache)
-from bs4 import BeautifulSoup
-from libmproxy.protocol.http import decoded
-
-
-def start(context, argv):
-    if len(argv) != 2:
-        raise ValueError('Usage: -s "iframe_injector.py url"')
-    context.iframe_url = argv[1]
-
-
-def response(context, flow):
-    if flow.request.host in context.iframe_url:
-        return
-    with decoded(flow.response):  # Remove content encoding (gzip, ...)
-        html = BeautifulSoup(flow.response.content)
-        if html.body:
-            iframe = html.new_tag("iframe", src=context.iframe_url, frameborder=0, height=0, width=0)
-            html.body.insert(0, iframe)
-            flow.response.content = str(html)
-            context.log("Iframe inserted.")

examples/ignore_websocket.py

@@ -1,34 +0,0 @@
-# This script makes mitmproxy switch to passthrough mode for all HTTP
-# responses with "Connection: Upgrade" header. This is useful to make
-# WebSockets work in untrusted environments.
-#
-# Note: Chrome (and possibly other browsers), when explicitly configured
-# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request
-# to the proxy before they initiate the websocket connection.
-# To make WebSockets work in these cases, supply
-# `--ignore :80$` as an additional parameter.
-# (see http://mitmproxy.org/doc/features/passthrough.html)
-
-from libmproxy.protocol.http import HTTPRequest
-from libmproxy.protocol.tcp import TCPHandler
-from libmproxy.protocol import KILL
-from libmproxy.script import concurrent
-
-
-def start(context, argv):
-    HTTPRequest._headers_to_strip_off.remove("Connection")
-    HTTPRequest._headers_to_strip_off.remove("Upgrade")
-
-
-def done(context):
-    HTTPRequest._headers_to_strip_off.append("Connection")
-    HTTPRequest._headers_to_strip_off.append("Upgrade")
-
-@concurrent
-def response(context, flow):
-    if flow.response.headers.get_first("Connection", None) == "Upgrade":
-        # We need to send the response manually now...
-        flow.client_conn.send(flow.response.assemble())
-        # ...and then delegate to tcp passthrough.
-        TCPHandler(flow.live.c, log=False).handle_messages()
-        flow.reply(KILL)

examples/mitmproxywrapper.py

@@ -86,6 +86,7 @@ class Wrapper(object):
             cmd = ['mitmproxy', '-p', str(self.port)]
             if self.extra_arguments:
                 cmd.extend(self.extra_arguments)
+            cmd.extend(['--palette', 'light'])
             subprocess.check_call(cmd)
 
     def wrap_honeyproxy(self):

examples/modify_form.py

@@ -1,6 +1,8 @@
 
 def request(context, flow):
     if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
-        form = flow.request.get_form_urlencoded()
-        form["mitmproxy"] = ["rocks"]
-        flow.request.set_form_urlencoded(form)
+        frm = flow.request.get_form_urlencoded()
+        frm["mitmproxy"] = ["rocks"]
+        flow.request.set_form_urlencoded(frm)
+
+

examples/modify_querystring.py

@@ -3,4 +3,5 @@ def request(context, flow):
     q = flow.request.get_query()
     if q:
         q["mitmproxy"] = ["rocks"]
-        flow.request.set_query(q)
+        flow.request.set_query(q)
+

examples/modify_response_body.py

@@ -1,15 +0,0 @@
-# Usage: mitmdump -s "modify_response_body.py mitmproxy bananas"
-# (this script works best with --anticache)
-from libmproxy.protocol.http import decoded
-
-
-def start(context, argv):
-    if len(argv) != 3:
-        raise ValueError('Usage: -s "modify-response-body.py old new"')
-    # You may want to use Python's argparse for more sophisticated argument parsing.
-    context.old, context.new = argv[1], argv[2]
-
-
-def response(context, flow):
-    with decoded(flow.response):  # automatically decode gzipped responses.
-        flow.response.content = flow.response.content.replace(context.old, context.new)

examples/nonblocking.py

@@ -1,9 +1,8 @@
 import time
 from libmproxy.script import concurrent
 
-
-@concurrent  # Remove this and see what happens
+@concurrent
 def request(context, flow):
     print "handle request: %s%s" % (flow.request.host, flow.request.path)
     time.sleep(5)
-    print "start  request: %s%s" % (flow.request.host, flow.request.path)
+    print "start  request: %s%s" % (flow.request.host, flow.request.path)

examples/proxapp

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""
+This example shows how to graft a WSGI app onto mitmproxy. In this
+instance, we're using the Bottle framework (http://bottlepy.org/) to expose
+a single simplest-possible page.
+"""
+import bottle
+import os
+from libmproxy import proxy, flow
+
+@bottle.route('/')
+def index():
+    return 'Hi!'
+
+
+class MyMaster(flow.FlowMaster):
+    def run(self):
+        try:
+            flow.FlowMaster.run(self)
+        except KeyboardInterrupt:
+            self.shutdown()
+
+    def handle_request(self, r):
+        f = flow.FlowMaster.handle_request(self, r)
+        if f:
+            r.reply()
+        return f
+
+    def handle_response(self, r):
+        f = flow.FlowMaster.handle_response(self, r)
+        if f:
+            r.reply()
+        print f
+        return f
+
+
+config = proxy.ProxyConfig(
+    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
+)
+state = flow.State()
+server = proxy.ProxyServer(config, 8080)
+# Register the app using the magic domain "proxapp" on port 80. Requests to
+# this domain and port combination will now be routed to the WSGI app instance.
+server.apps.add(bottle.app(), "proxapp", 80)
+m = MyMaster(server, state)
+m.run()
+

examples/proxapp.py

@@ -1,24 +0,0 @@
-"""
-This example shows how to graft a WSGI app onto mitmproxy. In this
-instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
-a single simplest-possible page.
-"""
-from flask import Flask
-
-app = Flask("proxapp")
-
-
-@app.route('/')
-def hello_world():
-    return 'Hello World!'
-
-
-# Register the app using the magic domain "proxapp" on port 80. Requests to
-# this domain and port combination will now be routed to the WSGI app instance.
-def start(context, argv):
-    context.app_registry.add(app, "proxapp", 80)
-
-    # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
-    # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
-    # but won't send any data.
-    context.app_registry.add(app, "example.com", 443)

examples/read_dumpfile

@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-#
-# Simple script showing how to read a mitmproxy dump file
-#
-
-from libmproxy import flow
-import json, sys
-
-with open("logfile", "rb") as logfile:
-    freader = flow.FlowReader(logfile)
-    try:
-        for f in freader.stream():
-            print(f)
-            print(f.request.host)
-            json.dump(f.get_state(), sys.stdout, indent=4)
-            print ""
-    except flow.FlowReadError, v:
-        print "Flow file corrupted. Stopped loading."

examples/redirect_requests.py

@@ -1,24 +1,19 @@
-from libmproxy.protocol.http import HTTPResponse
+from libmproxy.flow import Response
 from netlib.odict import ODictCaseless
 
 """
 This example shows two ways to redirect flows to other destinations.
 """
 
-
 def request(context, flow):
-    # pretty_host(hostheader=True) takes the Host: header of the request into account,
-    # which is useful in transparent mode where we usually only have the IP otherwise.
-
-    # Method 1: Answer with a locally generated response
-    if flow.request.pretty_host(hostheader=True).endswith("example.com"):
-        resp = HTTPResponse(
-            [1, 1], 200, "OK",
-            ODictCaseless([["Content-Type", "text/html"]]),
-            "helloworld")
-        flow.reply(resp)
-
-    # Method 2: Redirect the request to a different server
-    if flow.request.pretty_host(hostheader=True).endswith("example.org"):
+    if flow.request.host.endswith("example.com"):
+        resp = Response(flow.request,
+                        [1,1],
+                        200, "OK",
+                        ODictCaseless([["Content-Type","text/html"]]),
+                        "helloworld",
+                        None)
+        flow.request.reply(resp)
+    if flow.request.host.endswith("example.org"):
         flow.request.host = "mitmproxy.org"
-        flow.request.update_host_header()
+        flow.request.headers["Host"] = ["mitmproxy.org"]

examples/stickycookies

@@ -1,14 +1,12 @@
 #!/usr/bin/env python
 """
 This example builds on mitmproxy's base proxying infrastructure to
-implement functionality similar to the "sticky cookies" option.
-
-Heads Up: In the majority of cases, you want to use inline scripts.
+implement functionality similar to the "sticky cookies" option. This is at
+a lower level than the Flow mechanism, so we're dealing directly with
+request and response objects.
 """
-import os
 from libmproxy import controller, proxy
-from libmproxy.proxy.server import ProxyServer
-
+import os
 
 class StickyMaster(controller.Master):
     def __init__(self, server):
@@ -21,22 +19,24 @@ class StickyMaster(controller.Master):
         except KeyboardInterrupt:
             self.shutdown()
 
-    def handle_request(self, flow):
-        hid = (flow.request.host, flow.request.port)
-        if flow.request.headers["cookie"]:
-            self.stickyhosts[hid] = flow.request.headers["cookie"]
+    def handle_request(self, msg):
+        hid = (msg.host, msg.port)
+        if msg.headers["cookie"]:
+            self.stickyhosts[hid] = msg.headers["cookie"]
         elif hid in self.stickyhosts:
-            flow.request.headers["cookie"] = self.stickyhosts[hid]
-        flow.reply()
+            msg.headers["cookie"] = self.stickyhosts[hid]
+        msg.reply()
 
-    def handle_response(self, flow):
-        hid = (flow.request.host, flow.request.port)
-        if flow.response.headers["set-cookie"]:
-            self.stickyhosts[hid] = flow.response.headers["set-cookie"]
-        flow.reply()
+    def handle_response(self, msg):
+        hid = (msg.request.host, msg.request.port)
+        if msg.headers["set-cookie"]:
+            self.stickyhosts[hid] = msg.headers["set-cookie"]
+        msg.reply()
 
 
-config = proxy.ProxyConfig(port=8080)
-server = ProxyServer(config)
+config = proxy.ProxyConfig(
+    cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
+)
+server = proxy.ProxyServer(config, 8080)
 m = StickyMaster(server)
 m.run()

examples/stream.py

@@ -1,5 +0,0 @@
-def responseheaders(context, flow):
-    """
-    Enables streaming for all responses.
-    """
-    flow.response.stream = True

examples/stub.py

@@ -1,63 +1,54 @@
 """
     This is a script stub, with definitions for all events.
 """
-def start(context, argv):
+def start(ctx, argv):
     """
         Called once on script startup, before any other events.
     """
-    context.log("start")
+    ctx.log("start")
 
-def clientconnect(context, conn_handler):
+def clientconnect(ctx, client_connect):
     """
         Called when a client initiates a connection to the proxy. Note that a
         connection can correspond to multiple HTTP requests
     """
-    context.log("clientconnect")
+    ctx.log("clientconnect")
 
-def serverconnect(context, conn_handler):
+def serverconnect(ctx, server_connection):
     """
         Called when the proxy initiates a connection to the target server. Note that a
         connection can correspond to multiple HTTP requests
     """
-    context.log("serverconnect")
+    ctx.log("serverconnect")
 
-def request(context, flow):
+def request(ctx, flow):
     """
         Called when a client request has been received.
     """
-    context.log("request")
+    ctx.log("request")
 
-
-def responseheaders(context, flow):
-    """
-        Called when the response headers for a server response have been received,
-        but the response body has not been processed yet. Can be used to tell mitmproxy
-        to stream the response.
-    """
-    context.log("responseheaders")
-
-def response(context, flow):
+def response(ctx, flow):
     """
        Called when a server response has been received.
     """
-    context.log("response")
+    ctx.log("response")
 
-def error(context, flow):
+def error(ctx, flow):
     """
         Called when a flow error has occured, e.g. invalid server responses, or
         interrupted connections. This is distinct from a valid server HTTP error
         response, which is simply a response with an HTTP error code.
     """
-    context.log("error")
+    ctx.log("error")
 
-def clientdisconnect(context, conn_handler):
+def clientdisconnect(ctx, client_disconnect):
     """
         Called when a client disconnects from the proxy.
     """
-    context.log("clientdisconnect")
+    ctx.log("clientdisconnect")
 
-def done(context):
+def done(ctx):
     """
         Called once on script shutdown, after any other events.
     """
-    context.log("done")
+    ctx.log("done")

examples/upsidedownternet.py

@@ -1,16 +1,10 @@
 import cStringIO
 from PIL import Image
-from libmproxy.protocol.http import decoded
 
 def response(context, flow):
-    if flow.response.headers.get_first("content-type", "").startswith("image"):
-        with decoded(flow.response):  # automatically decode gzipped responses.
-            try:
-                s = cStringIO.StringIO(flow.response.content)
-                img = Image.open(s).rotate(180)
-                s2 = cStringIO.StringIO()
-                img.save(s2, "png")
-                flow.response.content = s2.getvalue()
-                flow.response.headers["content-type"] = ["image/png"]
-            except:  # Unknown image types etc.
-                pass
+    if flow.response.headers["content-type"] == ["image/png"]:
+        s = cStringIO.StringIO(flow.response.content)
+        img = Image.open(s).rotate(180)
+        s2 = cStringIO.StringIO()
+        img.save(s2, "png")
+        flow.response.content = s2.getvalue()

libmproxy/app.py

@@ -0,0 +1,27 @@
+import flask
+import os.path
+
+mapp = flask.Flask(__name__)
+mapp.debug = True
+
+def master():
+    return flask.request.environ["mitmproxy.master"]
+
+@mapp.route("/")
+def index():
+    return flask.render_template("index.html", section="home")
+
+
+@mapp.route("/cert/pem")
+def certs_pem():
+    capath = master().server.config.cacert
+    p = os.path.splitext(capath)[0] + "-cert.pem"
+    return flask.Response(open(p).read(), mimetype='application/x-x509-ca-cert')
+
+
+@mapp.route("/cert/p12")
+def certs_p12():
+    capath = master().server.config.cacert
+    p = os.path.splitext(capath)[0] + "-cert.p12"
+    return flask.Response(open(p).read(), mimetype='application/x-pkcs12')
+

libmproxy/cmdline.py

@@ -1,18 +1,12 @@
-from __future__ import absolute_import
-import os
-import re
-import configargparse
-from netlib import http
-from . import filt, utils, version
-from .proxy import config
+import proxy
+import re, filt
+import argparse
 
 APP_HOST = "mitm.it"
 APP_PORT = 80
 
-
-class ParseException(Exception):
-    pass
-
+class ParseException(Exception): pass
+class OptionException(Exception): pass
 
 def _parse_hook(s):
     sep, rem = s[0], s[1:]
@@ -23,15 +17,13 @@ def _parse_hook(s):
     elif len(parts) == 3:
         patt, a, b = parts
     else:
-        raise ParseException(
-            "Malformed hook specifier - too few clauses: %s" % s
-        )
+        raise ParseException("Malformed hook specifier - too few clauses: %s"%s)
 
     if not a:
-        raise ParseException("Empty clause: %s" % str(patt))
+        raise ParseException("Empty clause: %s"%str(patt))
 
     if not filt.parse(patt):
-        raise ParseException("Malformed filter pattern: %s" % patt)
+        raise ParseException("Malformed filter pattern: %s"%patt)
 
     return patt, a, b
 
@@ -66,7 +58,7 @@ def parse_replace_hook(s):
     try:
         re.compile(regex)
     except re.error, e:
-        raise ParseException("Malformed replacement regex: %s" % str(e.message))
+        raise ParseException("Malformed replacement regex: %s"%str(e.message))
     return patt, regex, replacement
 
 
@@ -99,27 +91,6 @@ def parse_setheader(s):
     return _parse_hook(s)
 
 
-def parse_server_spec(url):
-    normalized_url = re.sub("^https?2", "", url)
-
-    p = http.parse_url(normalized_url)
-    if not p or not p[1]:
-        raise configargparse.ArgumentTypeError(
-            "Invalid server specification: %s" % url
-        )
-
-    if url.lower().startswith("https2http"):
-        ssl = [True, False]
-    elif url.lower().startswith("http2https"):
-        ssl = [False, True]
-    elif url.lower().startswith("https"):
-        ssl = [True, True]
-    else:
-        ssl = [False, False]
-
-    return ssl + list(p[1:3])
-
-
 def get_common_options(options):
     stickycookie, stickyauth = None, None
     if options.stickycookie_filt:
@@ -128,300 +99,206 @@ def get_common_options(options):
     if options.stickyauth_filt:
         stickyauth = options.stickyauth_filt
 
-    stream_large_bodies = utils.parse_size(options.stream_large_bodies)
-
     reps = []
     for i in options.replace:
         try:
             p = parse_replace_hook(i)
         except ParseException, e:
-            raise configargparse.ArgumentTypeError(e.message)
+            raise OptionException(e.message)
         reps.append(p)
     for i in options.replace_file:
         try:
             patt, rex, path = parse_replace_hook(i)
         except ParseException, e:
-            raise configargparse.ArgumentTypeError(e.message)
+            raise OptionException(e.message)
         try:
             v = open(path, "rb").read()
         except IOError, e:
-            raise configargparse.ArgumentTypeError(
-                "Could not read replace file: %s" % path
-            )
+            raise OptionException("Could not read replace file: %s"%path)
         reps.append((patt, rex, v))
 
+
     setheaders = []
     for i in options.setheader:
         try:
             p = parse_setheader(i)
         except ParseException, e:
-            raise configargparse.ArgumentTypeError(e.message)
+            raise OptionException(e.message)
         setheaders.append(p)
 
     return dict(
-        app=options.app,
-        app_host=options.app_host,
-        app_port=options.app_port,
+        app = options.app,
+        app_host = options.app_host,
+        app_port = options.app_port,
+        app_external = options.app_external,
 
-        anticache=options.anticache,
-        anticomp=options.anticomp,
-        client_replay=options.client_replay,
-        kill=options.kill,
-        no_server=options.no_server,
-        refresh_server_playback=not options.norefresh,
-        rheaders=options.rheaders,
-        rfile=options.rfile,
-        replacements=reps,
-        setheaders=setheaders,
-        server_replay=options.server_replay,
-        scripts=options.scripts,
-        stickycookie=stickycookie,
-        stickyauth=stickyauth,
-        stream_large_bodies=stream_large_bodies,
-        showhost=options.showhost,
-        outfile=options.outfile,
-        verbosity=options.verbose,
-        nopop=options.nopop,
-        replay_ignore_content = options.replay_ignore_content,
-        replay_ignore_params = options.replay_ignore_params,
-        replay_ignore_payload_params = options.replay_ignore_payload_params
+        anticache = options.anticache,
+        anticomp = options.anticomp,
+        client_replay = options.client_replay,
+        eventlog = options.eventlog,
+        kill = options.kill,
+        no_server = options.no_server,
+        refresh_server_playback = not options.norefresh,
+        rheaders = options.rheaders,
+        rfile = options.rfile,
+        replacements = reps,
+        setheaders = setheaders,
+        server_replay = options.server_replay,
+        scripts = options.scripts,
+        stickycookie = stickycookie,
+        stickyauth = stickyauth,
+        showhost = options.showhost,
+        wfile = options.wfile,
+        verbosity = options.verbose,
+        nopop = options.nopop,
     )
 
 
 def common_options(parser):
     parser.add_argument(
-        '--version',
-        action= 'version',
-        version= "%(prog)s" + " " + version.VERSION
-    )
-    parser.add_argument(
-        '--shortversion',
-        action= 'version',
-        help = "show program's short version number and exit",
-        version = version.VERSION
+        "-b",
+        action="store", type = str, dest="addr", default='',
+        help = "Address to bind proxy to (defaults to all interfaces)"
     )
     parser.add_argument(
         "--anticache",
         action="store_true", dest="anticache", default=False,
-
-        help="""
-            Strip out request headers that might cause the server to return
-            304-not-modified.
-        """
+        help="Strip out request headers that might cause the server to return 304-not-modified."
     )
     parser.add_argument(
-        "--cadir",
-        action="store", type=str, dest="cadir", default=config.CA_DIR,
-        help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
+        "--confdir",
+        action="store", type = str, dest="confdir", default='~/.mitmproxy',
+        help = "Configuration directory. (~/.mitmproxy)"
+    )
+    parser.add_argument(
+        "-e",
+        action="store_true", dest="eventlog",
+        help="Show event log."
+    )
+    parser.add_argument(
+        "-n",
+        action="store_true", dest="no_server",
+        help="Don't start a proxy server."
+    )
+    parser.add_argument(
+        "-p",
+        action="store", type = int, dest="port", default=8080,
+        help = "Proxy service port."
+    )
+    parser.add_argument(
+        "-P",
+        action="store", dest="reverse_proxy", default=None,
+        help="Reverse proxy to upstream server: http[s]://host[:port]"
+    )
+    parser.add_argument(
+        "-F",
+        action="store", dest="forward_proxy", default=None,
+        help="Proxy to unconditionally forward to: http[s]://host[:port]"
+    )
+    parser.add_argument(
+        "-q",
+        action="store_true", dest="quiet",
+        help="Quiet."
+    )
+    parser.add_argument(
+        "-r",
+        action="store", dest="rfile", default=None,
+        help="Read flows from file."
+    )
+    parser.add_argument(
+        "-s",
+        action="append", type=str, dest="scripts", default=[],
+        metavar='"script.py --bar"',
+        help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times."
+    )
+    parser.add_argument(
+        "-t",
+        action="store", dest="stickycookie_filt", default=None, metavar="FILTER",
+        help="Set sticky cookie filter. Matched against requests."
+    )
+    parser.add_argument(
+        "-T",
+        action="store_true", dest="transparent_proxy", default=False,
+        help="Set transparent proxy mode."
+    )
+    parser.add_argument(
+        "-u",
+        action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
+        help="Set sticky auth filter. Matched against requests."
+    )
+    parser.add_argument(
+        "-v",
+        action="count", dest="verbose", default=1,
+        help="Increase verbosity. Can be passed multiple times."
+    )
+    parser.add_argument(
+        "-w",
+        action="store", dest="wfile", default=None,
+        help="Write flows to file."
+    )
+    parser.add_argument(
+        "-z",
+        action="store_true", dest="anticomp", default=False,
+        help="Try to convince servers to send us un-compressed data."
+    )
+    parser.add_argument(
+        "-Z",
+        action="store", dest="body_size_limit", default=None,
+        metavar="SIZE",
+        help="Byte size limit of HTTP request and response bodies."\
+             " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
     parser.add_argument(
         "--host",
         action="store_true", dest="showhost", default=False,
         help="Use the Host header to construct URLs for display."
     )
+
     parser.add_argument(
-        "-q", "--quiet",
-        action="store_true", dest="quiet",
-        help="Quiet."
-    )
-    parser.add_argument(
-        "-r", "--read-flows",
-        action="store", dest="rfile", default=None,
-        help="Read flows from file."
-    )
-    parser.add_argument(
-        "-s", "--script",
-        action="append", type=str, dest="scripts", default=[],
-        metavar='"script.py --bar"',
-        help="""
-            Run a script. Surround with quotes to pass script arguments. Can be
-            passed multiple times.
-        """
-    )
-    parser.add_argument(
-        "-t", "--stickycookie",
-        action="store",
-        dest="stickycookie_filt",
-        default=None,
-        metavar="FILTER",
-        help="Set sticky cookie filter. Matched against requests."
-    )
-    parser.add_argument(
-        "-u", "--stickyauth",
-        action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
-        help="Set sticky auth filter. Matched against requests."
-    )
-    parser.add_argument(
-        "-v", "--verbose",
-        action="store_const", dest="verbose", default=1, const=2,
-        help="Increase event log verbosity."
-    )
-    outfile = parser.add_mutually_exclusive_group()
-    outfile.add_argument(
-        "-w", "--wfile",
-        action="store", dest="outfile", type=lambda f: (f, "wb"),
-        help="Write flows to file."
-    )
-    outfile.add_argument(
-        "-a", "--afile",
-        action="store", dest="outfile", type=lambda f: (f, "ab"),
-        help="Append flows to file."
-    )
-    parser.add_argument(
-        "-z", "--anticomp",
-        action="store_true", dest="anticomp", default=False,
-        help="Try to convince servers to send us un-compressed data."
-    )
-    parser.add_argument(
-        "-Z", "--body-size-limit",
-        action="store", dest="body_size_limit", default=None,
-        metavar="SIZE",
-        help="Byte size limit of HTTP request and response bodies."
-             " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
-    )
-    parser.add_argument(
-        "--stream",
-        action="store", dest="stream_large_bodies", default=None,
-        metavar="SIZE",
-        help="""
-            Stream data to the client if response body exceeds the given
-            threshold. If streamed, the body will not be stored in any way.
-            Understands k/m/g suffixes, i.e. 3m for 3 megabytes.
-         """
+        "--no-upstream-cert", default=False,
+        action="store_true", dest="no_upstream_cert",
+        help="Don't connect to upstream server to look up certificate details."
     )
 
-    group = parser.add_argument_group("Proxy Options")
-    # We could make a mutually exclusive group out of -R, -U, -T, but we don't
-    # do that because  - --upstream-server should be in that group as well, but
-    # it's already in a different group.  - our own error messages are more
-    # helpful
+    group = parser.add_argument_group("Web App")
     group.add_argument(
-        "-b", "--bind-address",
-        action="store", type=str, dest="addr", default='',
-        help="Address to bind proxy to (defaults to all interfaces)"
-    )
-    group.add_argument(
-        "-I", "--ignore",
-        action="append", type=str, dest="ignore_hosts", default=[],
-        metavar="HOST",
-        help="""
-            Ignore host and forward all traffic without processing it. In
-            transparent mode, it is recommended to use an IP address (range),
-            not the hostname. In regular mode, only SSL traffic is ignored and
-            the hostname should be used. The supplied value is interpreted as a
-            regular expression and matched on the ip or the hostname. Can be
-            passed multiple times.
-        """
-    )
-    group.add_argument(
-        "--tcp",
-        action="append", type=str, dest="tcp_hosts", default=[],
-        metavar="HOST",
-        help="""
-            Generic TCP SSL proxy mode for all hosts that match the pattern.
-            Similar to --ignore, but SSL connections are intercepted. The
-            communication contents are printed to the event log in verbose mode.
-        """
-    )
-    group.add_argument(
-        "-n", "--no-server",
-        action="store_true", dest="no_server",
-        help="Don't start a proxy server."
-    )
-    group.add_argument(
-        "-p", "--port",
-        action="store", type=int, dest="port", default=8080,
-        help="Proxy service port."
-    )
-    group.add_argument(
-        "-R", "--reverse",
-        action="store",
-        type=parse_server_spec,
-        dest="reverse_proxy",
-        default=None,
-        help="""
-            Forward all requests to upstream HTTP server:
-            http[s][2http[s]]://host[:port]
-        """
-    )
-    group.add_argument(
-        "--socks",
-        action="store_true", dest="socks_proxy", default=False,
-        help="Set SOCKS5 proxy mode."
-    )
-    group.add_argument(
-        "-T", "--transparent",
-        action="store_true", dest="transparent_proxy", default=False,
-        help="Set transparent proxy mode."
-    )
-    group.add_argument(
-        "-U", "--upstream",
-        action="store",
-        type=parse_server_spec,
-        dest="upstream_proxy",
-        default=None,
-        help="Forward all requests to upstream proxy server: http://host[:port]"
-    )
-
-    group = parser.add_argument_group(
-        "Advanced Proxy Options",
-        """
-            The following options allow a custom adjustment of the proxy
-            behavior. Normally, you don't want to use these options directly and
-            use the provided wrappers instead (-R, -U, -T).
-        """
-    )
-    group.add_argument(
-        "--http-form-in", dest="http_form_in", default=None,
-        action="store", choices=("relative", "absolute"),
-        help="Override the HTTP request form accepted by the proxy"
-    )
-    group.add_argument(
-        "--http-form-out", dest="http_form_out", default=None,
-        action="store", choices=("relative", "absolute"),
-        help="Override the HTTP request form sent upstream by the proxy"
-    )
-
-    group = parser.add_argument_group("Onboarding App")
-    group.add_argument(
-        "--noapp",
+        "-a",
         action="store_false", dest="app", default=True,
-        help="Disable the mitmproxy onboarding app."
+        help="Disable the mitmproxy web app."
     )
     group.add_argument(
         "--app-host",
         action="store", dest="app_host", default=APP_HOST, metavar="host",
-        help="""
-            Domain to serve the onboarding app from. For transparent mode, use
-            an IP when a DNS entry for the app domain is not present. Default:
-            %s
-        """ % APP_HOST
+        help="Domain to serve the app from. For transparent mode, use an IP when\
+                a DNS entry for the app domain is not present. Default: %s"%APP_HOST
+
     )
     group.add_argument(
         "--app-port",
-        action="store",
-        dest="app_port",
-        default=APP_PORT,
-        type=int,
-        metavar="80",
-        help="Port to serve the onboarding app from."
+        action="store", dest="app_port", default=APP_PORT, type=int, metavar="80",
+        help="Port to serve the app from."
+    )
+    group.add_argument(
+        "--app-external",
+        action="store_true", dest="app_external",
+        help="Serve the app outside of the proxy."
     )
 
     group = parser.add_argument_group("Client Replay")
     group.add_argument(
-        "-c", "--client-replay",
+        "-c",
         action="store", dest="client_replay", default=None, metavar="PATH",
         help="Replay client requests from a saved file."
     )
 
     group = parser.add_argument_group("Server Replay")
     group.add_argument(
-        "-S", "--server-replay",
+        "-S",
         action="store", dest="server_replay", default=None, metavar="PATH",
         help="Replay server responses from a saved file."
     )
     group.add_argument(
-        "-k", "--kill",
+        "-k",
         action="store_true", dest="kill", default=False,
         help="Kill extra requests during replay."
     )
@@ -429,47 +306,19 @@ def common_options(parser):
         "--rheader",
         action="append", dest="rheaders", type=str,
         help="Request headers to be considered during replay. "
-             "Can be passed multiple times."
+           "Can be passed multiple times."
     )
     group.add_argument(
         "--norefresh",
         action="store_true", dest="norefresh", default=False,
-        help="""
-            Disable response refresh, which updates times in cookies and headers
-            for replayed responses.
-        """
+        help= "Disable response refresh, "
+        "which updates times in cookies and headers for replayed responses."
     )
     group.add_argument(
         "--no-pop",
         action="store_true", dest="nopop", default=False,
         help="Disable response pop from response flow. "
-             "This makes it possible to replay same response multiple times."
-    )
-    payload = group.add_mutually_exclusive_group()
-    payload.add_argument(
-        "--replay-ignore-content",
-        action="store_true", dest="replay_ignore_content", default=False,
-        help="""
-            Ignore request's content while searching for a saved flow to replay
-        """
-    )
-    payload.add_argument(
-        "--replay-ignore-payload-param",
-        action="append", dest="replay_ignore_payload_params", type=str,
-        help="""
-            Request's payload parameters (application/x-www-form-urlencoded) to
-            be ignored while searching for a saved flow to replay.
-            Can be passed multiple times.
-        """
-    )
-
-    group.add_argument(
-        "--replay-ignore-param",
-        action="append", dest="replay_ignore_params", type=str,
-        help="""
-            Request's parameters to be ignored while searching for a saved flow
-            to replay. Can be passed multiple times.
-        """
+        "This makes it possible to replay same response multiple times."
     )
 
     group = parser.add_argument_group(
@@ -488,14 +337,12 @@ def common_options(parser):
     )
     group.add_argument(
         "--replace-from-file",
-        action = "append", type=str, dest="replace_file", default=[],
-        metavar = "PATH",
-        help = """
-            Replacement pattern, where the replacement clause is a path to a
-            file.
-        """
+        action="append", type=str, dest="replace_file", default=[],
+        metavar="PATH",
+        help="Replacement pattern, where the replacement clause is a path to a file."
     )
 
+
     group = parser.add_argument_group(
         "Set Headers",
         """
@@ -511,11 +358,13 @@ def common_options(parser):
         help="Header set pattern."
     )
 
+
     group = parser.add_argument_group(
         "Proxy Authentication",
         """
             Specify which users are allowed to access the proxy and the method
-            used for authenticating them.
+            used for authenticating them. These options are ignored if the
+            proxy is in transparent or reverse proxy mode.
         """
     )
     user_specification_group = group.add_mutually_exclusive_group()
@@ -529,129 +378,13 @@ def common_options(parser):
         "--singleuser",
         action="store", dest="auth_singleuser", type=str,
         metavar="USER",
-        help="""
-            Allows access to a a single user, specified in the form
-            username:password.
-        """
+        help="Allows access to a a single user, specified in the form username:password."
     )
     user_specification_group.add_argument(
         "--htpasswd",
-        action="store", dest="auth_htpasswd", type=str,
+        action="store", dest="auth_htpasswd", type=argparse.FileType('r'),
         metavar="PATH",
         help="Allow access to users specified in an Apache htpasswd file."
     )
 
-    config.ssl_option_group(parser)
-
-
-def mitmproxy():
-    # Don't import libmproxy.console for mitmdump, urwid is not available on all
-    # platforms.
-    from .console import palettes
-
-    parser = configargparse.ArgumentParser(
-        usage="%(prog)s [options]",
-        args_for_setting_config_path = ["--conf"],
-        default_config_files = [
-            os.path.join(config.CA_DIR, "common.conf"),
-            os.path.join(config.CA_DIR, "mitmproxy.conf")
-        ],
-        add_config_file_help = True,
-        add_env_var_help = True
-    )
-    common_options(parser)
-    parser.add_argument(
-        "--palette", type=str, default="dark",
-        action="store", dest="palette",
-        help="Select color palette: " + ", ".join(palettes.palettes.keys())
-    )
-    parser.add_argument(
-        "-e", "--eventlog",
-        action="store_true", dest="eventlog",
-        help="Show event log."
-    )
-    group = parser.add_argument_group(
-        "Filters",
-        "See help in mitmproxy for filter expression syntax."
-    )
-    group.add_argument(
-        "-i", "--intercept", action="store",
-        type=str, dest="intercept", default=None,
-        help="Intercept filter expression."
-    )
-    return parser
-
-
-def mitmdump():
-    parser = configargparse.ArgumentParser(
-        usage="%(prog)s [options] [filter]",
-        args_for_setting_config_path = ["--conf"],
-        default_config_files = [
-            os.path.join(config.CA_DIR, "common.conf"),
-            os.path.join(config.CA_DIR, "mitmdump.conf")
-        ],
-        add_config_file_help = True,
-        add_env_var_help = True
-    )
-
-    common_options(parser)
-    parser.add_argument(
-        "--keepserving",
-        action= "store_true", dest="keepserving", default=False,
-        help= """
-            Continue serving after client playback or file read. We exit by
-            default.
-        """
-    )
-    parser.add_argument(
-        "-d", "--detail",
-        action="count", dest="flow_detail", default=1,
-        help="Increase flow detail display level. Can be passed multiple times."
-    )
-    parser.add_argument('args', nargs="...")
-    return parser
-
-
-def mitmweb():
-    parser = configargparse.ArgumentParser(
-        usage="%(prog)s [options]",
-        args_for_setting_config_path = ["--conf"],
-        default_config_files = [
-            os.path.join(config.CA_DIR, "common.conf"),
-            os.path.join(config.CA_DIR, "mitmweb.conf")
-        ],
-        add_config_file_help = True,
-        add_env_var_help = True
-    )
-
-    group = parser.add_argument_group("Mitmweb")
-    group.add_argument(
-        "--wport",
-        action="store", type=int, dest="wport", default=8081,
-        metavar="PORT",
-        help="Mitmweb port."
-    )
-    group.add_argument(
-        "--wiface",
-        action="store", dest="wiface", default="127.0.0.1",
-        metavar="IFACE",
-        help="Mitmweb interface."
-    )
-    group.add_argument(
-        "--wdebug",
-        action="store_true", dest="wdebug",
-        help="Turn on mitmweb debugging"
-    )
-
-    common_options(parser)
-    group = parser.add_argument_group(
-        "Filters",
-        "See help in mitmproxy for filter expression syntax."
-    )
-    group.add_argument(
-        "-i", "--intercept", action="store",
-        type=str, dest="intercept", default=None,
-        help="Intercept filter expression."
-    )
-    return parser
-
+    proxy.certificate_option_group(parser)

libmproxy/console/__init__.py

@@ -1,9 +1,8 @@
-from __future__ import absolute_import
 import mailcap, mimetypes, tempfile, os, subprocess, glob, time, shlex, stat
-import os.path, sys, weakref, traceback
+import os.path, sys, weakref
 import urwid
-from .. import controller, utils, flow, script, proxy
-from . import flowlist, flowview, help, common, grideditor, palettes, contentview, flowdetailview
+from .. import controller, utils, flow, script
+import flowlist, flowview, help, common, grideditor, palettes, contentview, flowdetailview
 
 EVENTLOG_SIZE = 500
 
@@ -78,6 +77,7 @@ class PathEdit(urwid.Edit, _PathCompleter):
 class ActionBar(common.WWrap):
     def __init__(self):
         self.message("")
+        self.expire = None
 
     def selectable(self):
         return True
@@ -129,14 +129,6 @@ class StatusBar(common.WWrap):
                 r.append(":%s in file]"%self.master.server_playback.count())
             else:
                 r.append(":%s to go]"%self.master.server_playback.count())
-        if self.master.get_ignore_filter():
-            r.append("[")
-            r.append(("heading_key", "I"))
-            r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
-        if self.master.get_tcp_filter():
-            r.append("[")
-            r.append(("heading_key", "T"))
-            r.append("CP:%d]" % len(self.master.get_tcp_filter()))
         if self.master.state.intercept_txt:
             r.append("[")
             r.append(("heading_key", "i"))
@@ -153,6 +145,10 @@ class StatusBar(common.WWrap):
             r.append("[")
             r.append(("heading_key", "u"))
             r.append(":%s]"%self.master.stickyauth_txt)
+        if self.master.server.config.reverse_proxy:
+            r.append("[")
+            r.append(("heading_key", "P"))
+            r.append(":%s]"%utils.unparse_url(*self.master.server.config.reverse_proxy))
         if self.master.state.default_body_view.name != "Auto":
             r.append("[")
             r.append(("heading_key", "M"))
@@ -173,23 +169,14 @@ class StatusBar(common.WWrap):
             opts.append("no-upstream-cert")
         if self.master.state.follow_focus:
             opts.append("following")
-        if self.master.stream_large_bodies:
-            opts.append("stream:%s" % utils.pretty_size(self.master.stream_large_bodies.max_size))
 
         if opts:
             r.append("[%s]"%(":".join(opts)))
 
-        if self.master.server.config.mode in ["reverse", "upstream"]:
-            dst = self.master.server.config.mode.dst
-            scheme = "https" if dst[0] else "http"
-            if dst[1] != dst[0]:
-                scheme += "2https" if dst[1] else "http"
-            r.append("[dest:%s]"%utils.unparse_url(scheme, *dst[2:]))
         if self.master.scripts:
-            r.append("[")
-            r.append(("heading_key", "s"))
-            r.append("cripts:%s]"%len(self.master.scripts))
-        # r.append("[lt:%0.3f]"%self.master.looptime)
+            r.append("[scripts:%s]"%len(self.master.scripts))
+        if self.master.debug:
+            r.append("[lt:%0.3f]"%self.master.looptime)
 
         if self.master.stream:
             r.append("[W:%s]"%self.master.stream_path)
@@ -210,10 +197,7 @@ class StatusBar(common.WWrap):
         ]
 
         if self.master.server.bound:
-            host = self.master.server.address.host
-            if host == "0.0.0.0":
-                host = "*"
-            boundaddr = "[%s:%s]"%(host, self.master.server.address.port)
+            boundaddr = "[%s:%s]"%(self.master.server.address or "*", self.master.server.port)
         else:
             boundaddr = ""
         t.extend(self.get_status())
@@ -277,16 +261,16 @@ class ConsoleState(flow.State):
         d = self.flowsettings.get(flow, {})
         return d.get(key, default)
 
-    def add_flow(self, f):
-        super(ConsoleState, self).add_flow(f)
+    def add_request(self, req):
+        f = flow.State.add_request(self, req)
         if self.focus is None:
             self.set_focus(0)
         elif self.follow_focus:
             self.set_focus(len(self.view) - 1)
         return f
 
-    def update_flow(self, f):
-        super(ConsoleState, self).update_flow(f)
+    def add_response(self, resp):
+        f = flow.State.add_response(self, resp)
         if self.focus is None:
             self.set_focus(0)
         return f
@@ -332,10 +316,6 @@ class ConsoleState(flow.State):
         self.set_focus(self.focus)
         return ret
 
-    def clear(self):
-        self.focus = None
-        super(ConsoleState, self).clear()
-
 
 
 class Options(object):
@@ -346,6 +326,7 @@ class Options(object):
         "anticache",
         "anticomp",
         "client_replay",
+        "debug",
         "eventlog",
         "keepserving",
         "kill",
@@ -361,7 +342,6 @@ class Options(object):
         "server_replay",
         "stickycookie",
         "stickyauth",
-        "stream_large_bodies",
         "verbosity",
         "wfile",
         "nopop",
@@ -410,8 +390,6 @@ class ConsoleMaster(flow.FlowMaster):
             print >> sys.stderr, "Sticky auth error:", r
             sys.exit(1)
 
-        self.set_stream_large_bodies(options.stream_large_bodies)
-
         self.refresh_server_playback = options.refresh_server_playback
         self.anticache = options.anticache
         self.anticomp = options.anticomp
@@ -429,6 +407,8 @@ class ConsoleMaster(flow.FlowMaster):
         if options.server_replay:
             self.server_playback_path(options.server_replay)
 
+        self.debug = options.debug
+
         if options.scripts:
             for i in options.scripts:
                 err = self.load_script(i)
@@ -436,20 +416,20 @@ class ConsoleMaster(flow.FlowMaster):
                     print >> sys.stderr, "Script load error:", err
                     sys.exit(1)
 
-        if options.outfile:
-            err = self.start_stream_to_path(options.outfile[0], options.outfile[1])
+        if options.wfile:
+            err = self.start_stream(options.wfile)
             if err:
-                print >> sys.stderr, "Stream file error:", err
+                print >> sys.stderr, "Script load error:", err
                 sys.exit(1)
 
         if options.app:
-            self.start_app(self.options.app_host, self.options.app_port)
+            self.start_app(self.options.app_host, self.options.app_port, self.options.app_external)
 
-    def start_stream_to_path(self, path, mode="wb"):
+    def start_stream(self, path):
         path = os.path.expanduser(path)
         try:
-            f = file(path, mode)
-            self.start_stream(f, None)
+            f = file(path, "wb")
+            flow.FlowMaster.start_stream(self, f, None)
         except IOError, v:
             return str(v)
         self.stream_path = path
@@ -458,20 +438,20 @@ class ConsoleMaster(flow.FlowMaster):
         status, val = s.run(method, f)
         if val:
             if status:
-                self.add_event("Method %s return: %s"%(method, val), "debug")
+                self.add_event("Method %s return: %s"%(method, val))
             else:
-                self.add_event("Method %s error: %s"%(method, val[1]), "error")
+                self.add_event("Method %s error: %s"%(method, val[1]))
 
     def run_script_once(self, command, f):
         if not command:
             return
-        self.add_event("Running script on flow: %s"%command, "debug")
+        self.add_event("Running script on flow: %s"%command)
 
         try:
             s = script.Script(command, self)
         except script.ScriptError, v:
             self.statusbar.message("Error loading script.")
-            self.add_event("Error loading script:\n%s"%v.args[0], "error")
+            self.add_event("Error loading script:\n%s"%v.args[0])
             return
 
         if f.request:
@@ -520,8 +500,7 @@ class ConsoleMaster(flow.FlowMaster):
             self.start_server_playback(
                 ret,
                 self.killextra, self.rheaders,
-                False, self.nopop,
-                self.options.replay_ignore_params, self.options.replay_ignore_content
+                False, self.nopop
             )
 
     def spawn_editor(self, data):
@@ -589,35 +568,31 @@ class ConsoleMaster(flow.FlowMaster):
         self.ui.set_terminal_properties(256)
         self.ui.register_palette(self.palette)
         self.flow_list_walker = flowlist.FlowListWalker(self, self.state)
+
         self.view = None
         self.statusbar = None
         self.header = None
         self.body = None
         self.help_context = None
+
         self.prompting = False
         self.onekey = False
 
         self.view_flowlist()
 
-        self.server.start_slave(controller.Slave, controller.Channel(self.masterq, self.should_exit))
+        self.server.start_slave(controller.Slave, controller.Channel(self.masterq))
 
         if self.options.rfile:
             ret = self.load_flows(self.options.rfile)
             if ret and self.state.flow_count():
-                self.add_event("File truncated or corrupted. Loaded as many flows as possible.","error")
+                self.add_event("File truncated or corrupted. Loaded as many flows as possible.")
             elif not self.state.flow_count():
                 self.shutdown()
                 print >> sys.stderr, "Could not load file:", ret
                 sys.exit(1)
 
-        try:
-            self.ui.run_wrapper(self.loop)
-        except Exception:
-            self.ui.stop()
-            sys.stdout.flush()
-            print >> sys.stderr, traceback.format_exc()
-            print >> sys.stderr, "mitmproxy has crashed!"
-            print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy"
+        self.ui.run_wrapper(self.loop)
+        # If True, quit just pops out to flow list view.
         print >> sys.stderr, "Shutting down..."
         sys.stderr.flush()
         self.shutdown()
@@ -674,6 +649,7 @@ class ConsoleMaster(flow.FlowMaster):
         self.statusbar = StatusBar(self, flowview.footer)
         self.state.set_focus_flow(flow)
         self.state.view_mode = common.VIEW_FLOW
+
         self.make_view()
         self.help_context = flowview.help_context
 
@@ -771,7 +747,7 @@ class ConsoleMaster(flow.FlowMaster):
         self.prompt_done()
 
     def accept_all(self):
-        self.state.accept_all(self)
+        self.state.accept_all()
 
     def set_limit(self, txt):
         v = self.state.set_limit(txt)
@@ -786,6 +762,15 @@ class ConsoleMaster(flow.FlowMaster):
         self.state.default_body_view = v
         self.refresh_focus()
 
+    def set_reverse_proxy(self, txt):
+        if not txt:
+            self.server.config.reverse_proxy = None
+        else:
+            s = utils.parse_proxy_spec(txt)
+            if not s:
+                return "Invalid reverse proxy specification"
+            self.server.config.reverse_proxy = s
+
     def drawscreen(self):
         size = self.ui.get_cols_rows()
         canvas = self.view.render(size, focus=1)
@@ -798,33 +783,19 @@ class ConsoleMaster(flow.FlowMaster):
         else:
             self.view_flowlist()
 
-    def edit_scripts(self, scripts):
-        commands = [x[0] for x in scripts]  # remove outer array
-        if commands == [s.command for s in self.scripts]:
-            return
-
-        self.unload_scripts()
-        for command in commands:
-            self.load_script(command)
-
-    def edit_ignore_filter(self, ignore):
-        patterns = (x[0] for x in ignore)
-        self.set_ignore_filter(patterns)
-
-    def edit_tcp_filter(self, tcp):
-        patterns = (x[0] for x in tcp)
-        self.set_tcp_filter(patterns)
+    def edit_scripts(self, *args, **kwargs):
+        pass
 
     def loop(self):
         changed = True
         try:
-            while not self.should_exit.is_set():
+            while not controller.should_exit:
                 startloop = time.time()
                 if changed:
                     self.statusbar.redraw()
                     size = self.drawscreen()
-                changed = self.tick(self.masterq, 0.01)
-                self.ui.set_input_timeouts(max_wait=0.01)
+                changed = self.tick(self.masterq)
+                self.ui.set_input_timeouts(max_wait=0.1)
                 keys = self.ui.get_input()
                 if keys:
                     changed = True
@@ -871,22 +842,6 @@ class ConsoleMaster(flow.FlowMaster):
                                         self.setheaders.set
                                     )
                                 )
-                            elif k == "I":
-                                self.view_grideditor(
-                                    grideditor.HostPatternEditor(
-                                        self,
-                                        [[x] for x in self.get_ignore_filter()],
-                                        self.edit_ignore_filter
-                                    )
-                                )
-                            elif k == "T":
-                                self.view_grideditor(
-                                    grideditor.HostPatternEditor(
-                                        self,
-                                        [[x] for x in self.get_tcp_filter()],
-                                        self.edit_tcp_filter
-                                    )
-                                )
                             elif k == "i":
                                 self.prompt(
                                     "Intercept filter: ",
@@ -910,6 +865,16 @@ class ConsoleMaster(flow.FlowMaster):
                                     contentview.view_prompts,
                                     self.change_default_display_mode
                                 )
+                            elif k == "P":
+                                if self.server.config.reverse_proxy:
+                                    p = utils.unparse_url(*self.server.config.reverse_proxy)
+                                else:
+                                    p = ""
+                                self.prompt(
+                                    "Reverse proxy: ",
+                                    p,
+                                    self.set_reverse_proxy
+                                )
                             elif k == "R":
                                 self.view_grideditor(
                                     grideditor.ReplaceEditor(
@@ -922,7 +887,7 @@ class ConsoleMaster(flow.FlowMaster):
                                 self.view_grideditor(
                                     grideditor.ScriptEditor(
                                         self,
-                                        [[i.command] for i in self.scripts],
+                                        [[i.argv[0]] for i in self.scripts],
                                         self.edit_scripts
                                     )
                                 )
@@ -1042,11 +1007,11 @@ class ConsoleMaster(flow.FlowMaster):
         if hasattr(self.statusbar, "refresh_flow"):
             self.statusbar.refresh_flow(c)
 
-    def process_flow(self, f):
-        if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay:
-            f.intercept(self)
+    def process_flow(self, f, r):
+        if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay():
+            f.intercept()
         else:
-            f.reply()
+            r.reply()
         self.sync_list_view()
         self.refresh_flow(f)
 
@@ -1054,34 +1019,35 @@ class ConsoleMaster(flow.FlowMaster):
         self.eventlist[:] = []
 
     def add_event(self, e, level="info"):
-        needed = dict(error=0, info=1, debug=2).get(level, 1)
-        if self.options.verbosity < needed:
-            return
-
-        if level == "error":
-            e = urwid.Text(("error", str(e)))
-        else:
+        if level == "info":
             e = urwid.Text(str(e))
+        elif level == "error":
+            e = urwid.Text(("error", str(e)))
+
         self.eventlist.append(e)
         if len(self.eventlist) > EVENTLOG_SIZE:
             self.eventlist.pop(0)
         self.eventlist.set_focus(len(self.eventlist)-1)
 
     # Handlers
-    def handle_error(self, f):
-        f = flow.FlowMaster.handle_error(self, f)
+    def handle_log(self, l):
+        self.add_event(l.msg)
+        l.reply()
+
+    def handle_error(self, r):
+        f = flow.FlowMaster.handle_error(self, r)
         if f:
-            self.process_flow(f)
+            self.process_flow(f, r)
         return f
 
-    def handle_request(self, f):
-        f = flow.FlowMaster.handle_request(self, f)
+    def handle_request(self, r):
+        f = flow.FlowMaster.handle_request(self, r)
         if f:
-            self.process_flow(f)
+            self.process_flow(f, r)
         return f
 
-    def handle_response(self, f):
-        f = flow.FlowMaster.handle_response(self, f)
+    def handle_response(self, r):
+        f = flow.FlowMaster.handle_response(self, r)
         if f:
-            self.process_flow(f)
+            self.process_flow(f, r)
         return f

libmproxy/console/common.py

@@ -1,8 +1,6 @@
-from __future__ import absolute_import
 import urwid
 import urwid.util
-from .. import utils
-from ..protocol.http import CONTENT_MISSING
+from .. import utils, flow
 
 
 VIEW_LIST = 0
@@ -108,7 +106,7 @@ def raw_format_flow(f, focus, extended, padding):
 
     preamble = sum(i[1] for i in req) + len(req) -1
 
-    if f["intercepted"] and not f["acked"]:
+    if f["intercepting"] and not f["req_acked"]:
         uc = "intercept"
     elif f["resp_code"] or f["err_msg"]:
         uc = "text"
@@ -138,7 +136,7 @@ def raw_format_flow(f, focus, extended, padding):
         if f["resp_is_replay"]:
             resp.append(fcol(SYMBOL_REPLAY, "replay"))
         resp.append(fcol(f["resp_code"], ccol))
-        if f["intercepted"] and f["resp_code"] and not f["acked"]:
+        if f["intercepting"] and f["resp_code"] and not f["resp_acked"]:
             rc = "intercept"
         else:
             rc = "text"
@@ -171,13 +169,13 @@ flowcache = FlowCache()
 
 def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     d = dict(
-        intercepted = f.intercepted,
-        acked = f.reply.acked,
+        intercepting = f.intercepting,
 
         req_timestamp = f.request.timestamp_start,
-        req_is_replay = f.request.is_replay,
+        req_is_replay = f.request.is_replay(),
         req_method = f.request.method,
-        req_url = f.request.pretty_url(hostheader=hostheader),
+        req_acked = f.request.reply.acked,
+        req_url = f.request.get_url(hostheader=hostheader),
 
         err_msg = f.error.msg if f.error else None,
         resp_code = f.response.code if f.response else None,
@@ -185,21 +183,19 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     if f.response:
         if f.response.content:
             contentdesc = utils.pretty_size(len(f.response.content))
-        elif f.response.content == CONTENT_MISSING:
+        elif f.response.content == flow.CONTENT_MISSING:
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"
 
-        if f.response.timestamp_end:
-            delta = f.response.timestamp_end - f.response.timestamp_start
-        else:
-            delta = 0
-        size = f.response.size()
+        delta = f.response.timestamp_end - f.response.timestamp_start
+        size = len(f.response.content) + f.response.get_header_size()
         rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
 
         d.update(dict(
             resp_code = f.response.code,
-            resp_is_replay = f.response.is_replay,
+            resp_is_replay = f.response.is_replay(),
+            resp_acked = f.response.reply.acked,
             resp_clen = contentdesc,
             resp_rate = "{0}/s".format(rate),
         ))

libmproxy/console/contentview.py

@@ -1,14 +1,16 @@
-from __future__ import absolute_import
-import logging, subprocess, re, cStringIO, traceback, json, urwid
+import logging
+import re, cStringIO, traceback, json
+import urwid
+
 from PIL import Image
 from PIL.ExifTags import TAGS
 
 import lxml.html, lxml.etree
 import netlib.utils
-from . import common
+import common
 from .. import utils, encoding, flow
 from ..contrib import jsbeautifier, html2text
-from ..contrib.wbxml.ASCommandResponse import ASCommandResponse
+import subprocess
 try:
     import pyamf
     from pyamf import remoting, flex
@@ -427,31 +429,12 @@ class ViewProtobuf:
         txt = _view_text(decoded[:limit], len(decoded), limit)
         return "Protobuf", txt
 
-class ViewWBXML:
-    name = "WBXML"
-    prompt = ("wbxml", "w")
-    content_types = [
-        "application/vnd.wap.wbxml",
-        "application/vnd.ms-sync.wbxml"
-    ]
-
-    def __call__(self, hdrs, content, limit):
-        
-        try:
-            parser = ASCommandResponse(content)
-            parsedContent = parser.xmlString
-            txt = _view_text(parsedContent, len(parsedContent), limit)
-            return "WBXML", txt
-        except:
-        	return None
-
 views = [
     ViewAuto(),
     ViewRaw(),
     ViewHex(),
     ViewJSON(),
     ViewXML(),
-    ViewWBXML(),
     ViewHTML(),
     ViewHTMLOutline(),
     ViewJavaScript(),
@@ -510,7 +493,7 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc):
     except Exception:
         s = traceback.format_exc()
         s = "Content viewer failed: \n"  + s
-        logfunc(s, "error")
+        logfunc(s)
         ret = None
     if not ret:
         ret = get("Raw")(hdrs, content, limit)

libmproxy/console/flowdetailview.py

@@ -1,7 +1,5 @@
-from __future__ import absolute_import
 import urwid
-from . import common
-from .. import utils
+import common
 
 footer = [
     ('heading_key', "q"), ":back ",
@@ -35,17 +33,8 @@ class FlowDetailsView(urwid.ListBox):
         title = urwid.AttrWrap(title, "heading")
         text.append(title)
 
-        if self.flow.server_conn:
-            text.append(urwid.Text([("head", "Server Connection:")]))
-            sc = self.flow.server_conn
-            parts = [
-                ["Address", "%s:%s" % sc.address()],
-                ["Start time", utils.format_timestamp(sc.timestamp_start)],
-                ["End time", utils.format_timestamp(sc.timestamp_end) if sc.timestamp_end else "active"],
-            ]
-            text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
-
-            c = self.flow.server_conn.cert
+        if self.flow.response:
+            c = self.flow.response.cert
             if c:
                 text.append(urwid.Text([("head", "Server Certificate:")]))
                 parts = [
@@ -54,13 +43,19 @@ class FlowDetailsView(urwid.ListBox):
                     ["Valid to", str(c.notafter)],
                     ["Valid from", str(c.notbefore)],
                     ["Serial", str(c.serial)],
+                ]
+
+                parts.append(
                     [
                         "Subject",
                         urwid.BoxAdapter(
                             urwid.ListBox(common.format_keyvals(c.subject, key="highlight", val="text")),
                             len(c.subject)
                         )
-                    ],
+                    ]
+                )
+
+                parts.append(
                     [
                         "Issuer",
                         urwid.BoxAdapter(
@@ -68,7 +63,7 @@ class FlowDetailsView(urwid.ListBox):
                             len(c.issuer)
                         )
                     ]
-                ]
+                )
 
                 if c.altnames:
                     parts.append(
@@ -79,14 +74,13 @@ class FlowDetailsView(urwid.ListBox):
                     )
                 text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 
-        if self.flow.client_conn:
+        if self.flow.request.client_conn:
             text.append(urwid.Text([("head", "Client Connection:")]))
-            cc = self.flow.client_conn
+            cc = self.flow.request.client_conn
             parts = [
-                ["Address", "%s:%s" % cc.address()],
-                ["Start time", utils.format_timestamp(cc.timestamp_start)],
-                # ["Requests", "%s"%cc.requestcount],
-                ["End time", utils.format_timestamp(cc.timestamp_end) if cc.timestamp_end else "active"],
+                ["Address", "%s:%s"%tuple(cc.address)],
+                ["Requests", "%s"%cc.requestcount],
+                ["Closed", "%s"%cc.close],
             ]
             text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 

libmproxy/console/flowlist.py

@@ -1,6 +1,5 @@
-from __future__ import absolute_import
 import urwid
-from . import common
+import common
 
 def _mkhelp():
     text = []
@@ -69,7 +68,7 @@ class BodyPile(urwid.Pile):
             else:
                 self.widget_list[1].header = self.inactive_header
             key = None
-        elif key == "e":
+        elif key == "v":
             self.master.toggle_eventlog()
             key = None
 
@@ -120,15 +119,13 @@ class ConnectionItem(common.WWrap):
             self.master.start_server_playback(
                 [i.copy() for i in self.master.state.view],
                 self.master.killextra, self.master.rheaders,
-                False, self.master.nopop,
-                self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
+                False, self.master.nopop
             )
         elif k == "t":
             self.master.start_server_playback(
                 [self.flow.copy()],
                 self.master.killextra, self.master.rheaders,
-                False, self.master.nopop,
-                self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
+                False, self.master.nopop
             )
         else:
             self.master.path_prompt(
@@ -140,7 +137,7 @@ class ConnectionItem(common.WWrap):
     def keypress(self, (maxcol,), key):
         key = common.shortcuts(key)
         if key == "a":
-            self.flow.accept_intercept(self.master)
+            self.flow.accept_intercept()
             self.master.sync_list_view()
         elif key == "d":
             self.flow.kill(self.master)
@@ -150,6 +147,7 @@ class ConnectionItem(common.WWrap):
             f = self.master.duplicate_flow(self.flow)
             self.master.view_flow(f)
         elif key == "r":
+            self.flow.backup()
             r = self.master.replay_request(self.flow)
             if r:
                 self.master.statusbar.message(r)
@@ -263,7 +261,7 @@ class FlowListBox(urwid.ListBox):
                 self.master.path_prompt(
                     "Stream flows to: ",
                     self.master.state.last_saveload,
-                    self.master.start_stream_to_path
+                    self.master.start_stream
                 )
         else:
             return urwid.ListBox.keypress(self, size, key)

libmproxy/console/flowview.py

@@ -1,9 +1,7 @@
-from __future__ import absolute_import
 import os, sys, copy
 import urwid
-from . import common, grideditor, contentview
+import common, grideditor, contentview
 from .. import utils, flow, controller
-from ..protocol.http import HTTPResponse, CONTENT_MISSING, decoded
 
 
 class SearchError(Exception): pass
@@ -70,8 +68,7 @@ def _mkhelp():
         ("space", "next flow"),
         ("|", "run script on this flow"),
         ("/", "search in response body (case sensitive)"),
-        ("n", "repeat search forward"),
-        ("N", "repeat search backwards"),
+        ("n", "repeat previous search"),
     ]
     text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
     return text
@@ -152,8 +149,8 @@ class FlowView(common.WWrap):
         return (description, text_objects)
 
     def cont_view_handle_missing(self, conn, viewmode):
-            if conn.content == CONTENT_MISSING:
-                msg, body = "", [urwid.Text([("error", "[content missing]")])]
+            if conn.content == flow.CONTENT_MISSING:
+                msg, body = "", [urwid.Text([("error", "[content missing]")])], 0
             else:
                 msg, body = self.content_view(viewmode, conn)
 
@@ -176,9 +173,12 @@ class FlowView(common.WWrap):
                 key = "header",
                 val = "text"
             )
-        override = self.override_get()
-        viewmode = self.viewmode_get(override)
-        msg, body = self.cont_view_handle_missing(conn, viewmode)
+        if conn.content is not None:
+            override = self.override_get()
+            viewmode = self.viewmode_get(override)
+            msg, body = self.cont_view_handle_missing(conn, viewmode)
+        elif conn.content == flow.CONTENT_MISSING:
+            pass
         return headers, msg, body
 
     def conn_text_merge(self, headers, msg, body):
@@ -230,7 +230,7 @@ class FlowView(common.WWrap):
     def wrap_body(self, active, body):
         parts = []
 
-        if self.flow.intercepted and not self.flow.reply.acked and not self.flow.response:
+        if self.flow.intercepting and not self.flow.request.reply.acked:
             qt = "Request intercepted"
         else:
             qt = "Request"
@@ -239,7 +239,7 @@ class FlowView(common.WWrap):
         else:
             parts.append(self._tab(qt, "heading_inactive"))
 
-        if self.flow.intercepted and not self.flow.reply.acked and self.flow.response:
+        if self.flow.intercepting and self.flow.response and not self.flow.response.reply.acked:
             st = "Response intercepted"
         else:
             st = "Response"
@@ -255,7 +255,7 @@ class FlowView(common.WWrap):
                 )
         return f
 
-    def search_wrapped_around(self, last_find_line, last_search_index, backwards):
+    def search_wrapped_around(self, last_find_line, last_search_index):
         """
             returns true if search wrapped around the bottom.
         """
@@ -265,50 +265,24 @@ class FlowView(common.WWrap):
         current_search_index = self.state.get_flow_setting(self.flow,
                 "last_search_index")
 
-        if not backwards:
-            message = "search hit BOTTOM, continuing at TOP"
-            if current_find_line <= last_find_line:
-                return True, message
-            elif current_find_line == last_find_line:
-                if current_search_index <= last_search_index:
-                    return True, message
-        else:
-            message = "search hit TOP, continuing at BOTTOM"
-            if current_find_line >= last_find_line:
-                return True, message
-            elif current_find_line == last_find_line:
-                if current_search_index >= last_search_index:
-                    return True, message
+        if current_find_line <= last_find_line:
+            return True
+        elif current_find_line == last_find_line:
+            if current_search_index <= last_search_index:
+                return True
 
-        return False, ""
+        return False
 
-    def search_again(self, backwards=False):
-        """
-            runs the previous search again, forwards or backwards.
-        """
-        last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
-        if last_search_string:
-            message = self.search(last_search_string, backwards)
-            if message:
-                self.master.statusbar.message(message)
-        else:
-            message = "no previous searches have been made"
-            self.master.statusbar.message(message)
-
-        return message
-
-    def search(self, search_string, backwards=False):
+    def search(self, search_string):
         """
             similar to view_response or view_request, but instead of just
             displaying the conn, it highlights a word that the user is
             searching for and handles all the logic surrounding that.
         """
 
-        if not search_string:
+        if search_string == "":
             search_string = self.state.get_flow_setting(self.flow,
                     "last_search_string")
-            if not search_string:
-                return
 
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
             text = self.flow.request
@@ -327,7 +301,7 @@ class FlowView(common.WWrap):
         # generate the body, highlight the words and get focus
         headers, msg, body = self.conn_text_raw(text)
         try:
-            body, focus_position = self.search_highlight_text(body, search_string, backwards=backwards)
+            body, focus_position = self.search_highlight_text(body, search_string)
         except SearchError:
             return "Search not supported in this view."
 
@@ -344,10 +318,8 @@ class FlowView(common.WWrap):
 
         self.last_displayed_body = list_box
 
-        wrapped, wrapped_message = self.search_wrapped_around(last_find_line, last_search_index, backwards)
-
-        if wrapped:
-            return wrapped_message
+        if self.search_wrapped_around(last_find_line, last_search_index):
+            return "search hit BOTTOM, continuing at TOP"
 
     def search_get_start(self, search_string):
         start_line = 0
@@ -372,94 +344,57 @@ class FlowView(common.WWrap):
 
         return (start_line, start_index)
 
-    def search_get_range(self, len_text_objects, start_line, backwards):
-        if not backwards:
-            loop_range = xrange(start_line, len_text_objects)
-        else:
-            loop_range = xrange(start_line, -1, -1)
-
-        return loop_range
-
-    def search_find(self, text, search_string, start_index, backwards):
-            if backwards == False:
-                find_index = text.find(search_string, start_index)
-            else:
-                if start_index != 0:
-                    start_index -= len(search_string)
-                else:
-                    start_index = None
-
-                find_index = text.rfind(search_string, 0, start_index)
-
-            return find_index
-
-    def search_highlight_text(self, text_objects, search_string, looping = False, backwards = False):
+    def search_highlight_text(self, text_objects, search_string, looping = False):
         start_line, start_index = self.search_get_start(search_string)
         i = start_line
 
         found = False
         text_objects = copy.deepcopy(text_objects)
-        loop_range = self.search_get_range(len(text_objects), start_line, backwards)
-        for i in loop_range:
-            text_object = text_objects[i]
+        for text_object in text_objects[start_line:]:
+            if i != start_line:
+                start_index = 0
 
             try:
                 text, style = text_object.get_text()
             except AttributeError:
                 raise SearchError()
-
-            if i != start_line:
-                start_index = 0
-
-            find_index = self.search_find(text, search_string, start_index, backwards)
-
+            find_index = text.find(search_string, start_index)
             if find_index != -1:
-                new_text = self.search_highlight_object(text, find_index, search_string)
-                text_objects[i] = new_text
+                before = text[:find_index]
+                after = text[find_index+len(search_string):]
+                new_text = urwid.Text(
+                    [
+                        before,
+                        (self.highlight_color, search_string),
+                        after,
+                    ]
+                )
 
-                found = True
                 self.state.add_flow_setting(self.flow, "last_search_index",
                         find_index)
                 self.state.add_flow_setting(self.flow, "last_find_line", i)
 
+                text_objects[i] = new_text
+
+                found = True
+
                 break
 
-        # handle search WRAP
+            i += 1
+
         if found:
             focus_pos = i
         else :
-            if looping:
+            # loop from the beginning, but not forever.
+            if (start_line == 0 and start_index == 0) or looping:
                 focus_pos = None
             else:
-                if not backwards:
-                    self.state.add_flow_setting(self.flow, "last_search_index", 0)
-                    self.state.add_flow_setting(self.flow, "last_find_line", 0)
-                else:
-                    self.state.add_flow_setting(self.flow, "last_search_index", None)
-                    self.state.add_flow_setting(self.flow, "last_find_line", len(text_objects) - 1)
-
-                text_objects, focus_pos = self.search_highlight_text(text_objects,
-                        search_string, looping=True, backwards=backwards)
+                self.state.add_flow_setting(self.flow, "last_search_index", 0)
+                self.state.add_flow_setting(self.flow, "last_find_line", 0)
+                text_objects, focus_pos = self.search_highlight_text(text_objects, search_string, True)
 
         return text_objects, focus_pos
 
-    def search_highlight_object(self, text_object, find_index, search_string):
-        """
-            just a little abstraction
-        """
-        before = text_object[:find_index]
-        after = text_object[find_index+len(search_string):]
-
-        new_text = urwid.Text(
-            [
-                before,
-                (self.highlight_color, search_string),
-                after,
-            ]
-        )
-
-        return new_text
-
     def view_request(self):
         self.state.view_flow_mode = common.VIEW_FLOW_REQUEST
         body = self.conn_text(self.flow.request)
@@ -525,9 +460,7 @@ class FlowView(common.WWrap):
 
     def set_url(self, url):
         request = self.flow.request
-        try:
-            request.url = str(url)
-        except ValueError:
+        if not request.set_url(str(url)):
             return "Invalid URL."
         self.master.refresh_flow(self.flow)
 
@@ -570,27 +503,23 @@ class FlowView(common.WWrap):
 
     def edit(self, part):
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
-            message = self.flow.request
+            conn = self.flow.request
         else:
             if not self.flow.response:
-                self.flow.response = HTTPResponse(
+                self.flow.response = flow.Response(
+                    self.flow.request,
                     self.flow.request.httpversion,
-                    200, "OK", flow.ODictCaseless(), ""
+                    200, "OK", flow.ODictCaseless(), "", None
                 )
                 self.flow.response.reply = controller.DummyReply()
-            message = self.flow.response
+            conn = self.flow.response
 
         self.flow.backup()
         if part == "r":
-            with decoded(message):
-                # Fix an issue caused by some editors when editing a request/response body.
-                # Many editors make it hard to save a file without a terminating newline on the last
-                # line. When editing message bodies, this can cause problems. For now, I just
-                # strip the newlines off the end of the body when we return from an editor.
-                c = self.master.spawn_editor(message.content or "")
-                message.content = c.rstrip("\n")
+            c = self.master.spawn_editor(conn.content or "")
+            conn.content = c.rstrip("\n") # what?
         elif part == "f":
-            if not message.get_form_urlencoded() and message.content:
+            if not conn.get_form_urlencoded() and conn.content:
                 self.master.prompt_onekey(
                     "Existing body is not a URL-encoded form. Clear and edit?",
                     [
@@ -598,26 +527,26 @@ class FlowView(common.WWrap):
                         ("no", "n"),
                     ],
                     self.edit_form_confirm,
-                    message
+                    conn
                 )
             else:
-                self.edit_form(message)
+                self.edit_form(conn)
         elif part == "h":
-            self.master.view_grideditor(grideditor.HeaderEditor(self.master, message.headers.lst, self.set_headers, message))
+            self.master.view_grideditor(grideditor.HeaderEditor(self.master, conn.headers.lst, self.set_headers, conn))
         elif part == "p":
-            p = message.get_path_components()
+            p = conn.get_path_components()
             p = [[i] for i in p]
-            self.master.view_grideditor(grideditor.PathEditor(self.master, p, self.set_path_components, message))
+            self.master.view_grideditor(grideditor.PathEditor(self.master, p, self.set_path_components, conn))
         elif part == "q":
-            self.master.view_grideditor(grideditor.QueryEditor(self.master, message.get_query().lst, self.set_query, message))
+            self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.get_query().lst, self.set_query, conn))
         elif part == "u" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
-            self.master.prompt_edit("URL", message.url, self.set_url)
+            self.master.prompt_edit("URL", conn.get_url(), self.set_url)
         elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
             self.master.prompt_onekey("Method", self.method_options, self.edit_method)
         elif part == "c" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:
-            self.master.prompt_edit("Code", str(message.code), self.set_resp_code)
+            self.master.prompt_edit("Code", str(conn.code), self.set_resp_code)
         elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:
-            self.master.prompt_edit("Message", message.msg, self.set_resp_msg)
+            self.master.prompt_edit("Message", conn.msg, self.set_resp_msg)
         self.master.refresh_flow(self.flow)
 
     def _view_nextprev_flow(self, np, flow):
@@ -650,7 +579,7 @@ class FlowView(common.WWrap):
 
     def delete_body(self, t):
         if t == "m":
-            val = CONTENT_MISSING
+            val = flow.CONTENT_MISSING
         else:
             val = None
         if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
@@ -682,7 +611,7 @@ class FlowView(common.WWrap):
             # Why doesn't this just work??
             self.w.keypress(size, key)
         elif key == "a":
-            self.flow.accept_intercept(self.master)
+            self.flow.accept_intercept()
             self.master.view_flow(self.flow)
         elif key == "A":
             self.master.accept_all()
@@ -753,7 +682,7 @@ class FlowView(common.WWrap):
             self.master.statusbar.message("")
         elif key == "m":
             p = list(contentview.view_prompts)
-            p.insert(0, ("Clear", "C"))
+            p.insert(0, ("clear", "c"))
             self.master.prompt_onekey(
                 "Display mode",
                 p,
@@ -763,6 +692,7 @@ class FlowView(common.WWrap):
         elif key == "p":
             self.view_prev_flow(self.flow)
         elif key == "r":
+            self.flow.backup()
             r = self.master.replay_request(self.flow)
             if r:
                 self.master.statusbar.message(r)
@@ -831,9 +761,13 @@ class FlowView(common.WWrap):
                     None,
                     self.search)
         elif key == "n":
-            self.search_again(backwards=False)
-        elif key == "N":
-            self.search_again(backwards=True)
+            last_search_string = self.state.get_flow_setting(self.flow, "last_search_string")
+            if last_search_string:
+                message = self.search(last_search_string)
+                if message:
+                    self.master.statusbar.message(message)
+            else:
+                self.master.statusbar.message("no previous searches have been made")
         else:
             return key
 

libmproxy/console/grideditor.py

@@ -1,7 +1,6 @@
-from __future__ import absolute_import
 import copy, re, os
 import urwid
-from . import common
+import common
 from .. import utils, filt, script
 from netlib import http_uastrings
 
@@ -123,13 +122,12 @@ class GridWalker(urwid.ListWalker):
             except ValueError:
                 self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000)
                 return
+
         errors = self.lst[self.focus][1]
         emsg = self.editor.is_error(self.focus_col, val)
         if emsg:
             self.editor.master.statusbar.message(emsg, 1000)
             errors.add(self.focus_col)
-        else:
-            errors.discard(self.focus_col)
 
         row = list(self.lst[self.focus][0])
         row[self.focus_col] = val
@@ -321,11 +319,9 @@ class GridEditor(common.WWrap):
         elif key == "d":
             self.walker.delete_focus()
         elif key == "r":
-            if self.walker.get_current_value() is not None:
-                self.master.path_prompt("Read file: ", "", self.read_file)
+            self.master.path_prompt("Read file: ", "", self.read_file)
         elif key == "R":
-            if self.walker.get_current_value() is not None:
-                self.master.path_prompt("Read unescaped file: ", "", self.read_file, True)
+            self.master.path_prompt("Read unescaped file: ", "", self.read_file, True)
         elif key == "e":
             o = self.walker.get_current_value()
             if o is not None:
@@ -496,15 +492,3 @@ class ScriptEditor(GridEditor):
             script.Script.parse_command(val)
         except script.ScriptError, v:
             return str(v)
-
-
-class HostPatternEditor(GridEditor):
-    title = "Editing host patterns"
-    columns = 1
-    headings = ("Regex (matched on hostname:port / ip:port)",)
-
-    def is_error(self, col, val):
-        try:
-            re.compile(val, re.IGNORECASE)
-        except re.error as e:
-            return "Invalid regex: %s" % str(e)

libmproxy/console/help.py

@@ -1,6 +1,5 @@
-from __future__ import absolute_import
 import urwid
-from . import common
+import common
 from .. import filt, version
 
 footer = [
@@ -24,7 +23,7 @@ class HelpView(urwid.ListBox):
 
         text.append(urwid.Text([("head", "\n\nMovement:\n")]))
         keys = [
-            ("j, k", "down, up"),
+            ("j, k", "up, down"),
             ("h, l", "left, right (in some contexts)"),
             ("space", "page down"),
             ("pg up/down", "page up/down"),
@@ -36,7 +35,6 @@ class HelpView(urwid.ListBox):
         keys = [
             ("c", "client replay"),
             ("H", "edit global header set patterns"),
-            ("I", "set ignore pattern"),
             ("i", "set interception pattern"),
             ("M", "change global default display mode"),
                 (None,
@@ -79,10 +77,6 @@ class HelpView(urwid.ListBox):
                     common.highlight_key("xml", "x") +
                     [("text", ": XML")]
                 ),
-                (None,
-                    common.highlight_key("wbxml", "w") +
-                    [("text", ": WBXML")]
-                ),
                 (None,
                     common.highlight_key("amf", "f") +
                     [("text", ": AMF (requires PyAMF)")]
@@ -115,11 +109,11 @@ class HelpView(urwid.ListBox):
 
             ("q", "quit / return to flow list"),
             ("Q", "quit without confirm prompt"),
+            ("P", "set reverse proxy mode"),
             ("R", "edit replacement patterns"),
-            ("s", "add/remove scripts"),
+            ("s", "set/unset script"),
             ("S", "server replay"),
             ("t", "set sticky cookie expression"),
-            ("T", "set tcp proxying pattern"),
             ("u", "set sticky auth expression"),
         ]
         text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))

libmproxy/contrib/README

@@ -11,4 +11,5 @@ jsbeautifier, git checkout 25/03/12, MIT license
 
 html2text, git checkout 18/08/12, GPLv3
 
-WinDivert 1.1.4, LGPL license, http://reqrypt.org/windivert.html
+md5crypt, PSF license, http://code.activestate.com/recipes/325204/
+

libmproxy/contrib/jsbeautifier/unpackers/__init__.py

@@ -7,7 +7,7 @@
 
 import pkgutil
 import re
-from . import evalbased
+from jsbeautifier.unpackers import evalbased
 
 # NOTE: AT THE MOMENT, IT IS DEACTIVATED FOR YOUR SECURITY: it runs js!
 BLACKLIST = ['jsbeautifier.unpackers.evalbased']

libmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py

@@ -40,7 +40,7 @@ try:
 except ImportError:
     from urllib.parse import unquote
 
-from . import UnpackingError
+from jsbeautifier.unpackers import UnpackingError
 
 PRIORITY = 1
 

libmproxy/contrib/jsbeautifier/unpackers/packer.py

@@ -14,7 +14,7 @@
 
 import re
 import string
-from . import UnpackingError
+from jsbeautifier.unpackers import UnpackingError
 
 PRIORITY = 1
 

libmproxy/contrib/wbxml/ASCommandResponse.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-'''
-@author: David Shaw, david.shaw.aw@gmail.com
-
-Inspired by EAS Inspector for Fiddler
-https://easinspectorforfiddler.codeplex.com
-
------ The MIT License (MIT) ----- 
-Filename: ASCommandResponse.py
-Copyright (c) 2014, David P. Shaw
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-'''
-from ASWBXML import ASWBXML
-import logging
-
-class ASCommandResponse:
-
-	def __init__(self, response):
-		self.wbxmlBody = response
-		try:
-			if ( len(response) > 0):
-				self.xmlString = self.decodeWBXML(self.wbxmlBody)
-			else:
-				logging.error("Empty WBXML body passed")
-		except Exception as e:
-			logging.error("Error: {0}".format(e.message))
-			self.xmlString = None
-
-	def getWBXMLBytes(self):
-		return self.wbxmlBytes
-	
-	def getXMLString(self):
-		return self.xmlString
-	
-	def decodeWBXML(self, body):
-		self.instance = ASWBXML()
-		self.instance.loadBytes(body)
-		return self.instance.getXml()
-
-if __name__ == "__main__":
-	import os	
-	logging.basicConfig(level=logging.INFO)
-
-	projectDir = os.path.dirname(os.path.realpath("."))
-	samplesDir = os.path.join(projectDir, "Samples/")
-	listOfSamples = os.listdir(samplesDir)
-
-	for filename in listOfSamples:
-		byteWBXML = open(samplesDir + os.sep + filename, "rb").read()
-		
-		logging.info("-"*100)
-		logging.info(filename)
-		logging.info("-"*100)
-		instance = ASCommandResponse(byteWBXML)
-		logging.info(instance.xmlString)
-		

libmproxy/contrib/wbxml/ASWBXML.py

@@ -1,903 +0,0 @@
-#!/usr/bin/env python
-'''
-@author: David Shaw, david.shaw.aw@gmail.com
-
-Inspired by EAS Inspector for Fiddler
-https://easinspectorforfiddler.codeplex.com
-
------ The MIT License (MIT) ----- 
-Filename: ASWBXML.py
-Copyright (c) 2014, David P. Shaw
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-'''
-import xml.dom.minidom
-import logging
-
-from ASWBXMLCodePage import ASWBXMLCodePage
-from ASWBXMLByteQueue import ASWBXMLByteQueue
-from GlobalTokens import GlobalTokens
-from InvalidDataException import InvalidDataException
-
-class ASWBXML:
-	versionByte = 0x03
-	publicIdentifierByte = 0x01
-	characterSetByte = 0x6A
-	stringTableLengthByte = 0x00
-	
-	def __init__(self):
-		
-		# empty on init
-		self.xmlDoc = xml.dom.minidom.Document()
-		self.currentCodePage = 0
-		self.defaultCodePage = -1
-		
-		# Load up code pages
-		# Currently there are 25 code pages as per MS-ASWBXML
-		self.codePages = []
-
-		# region Code Page Initialization
-		# Code Page 0: AirSync
-		# region AirSync Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "AirSync:"
-		page.xmlns = "airsync"
-
-		page.addToken(0x05, "Sync")
-		page.addToken(0x06, "Responses")
-		page.addToken(0x07, "Add")
-		page.addToken(0x08, "Change")
-		page.addToken(0x09, "Delete")
-		page.addToken(0x0A, "Fetch")
-		page.addToken(0x0B, "SyncKey")
-		page.addToken(0x0C, "ClientId")
-		page.addToken(0x0D, "ServerId")
-		page.addToken(0x0E, "Status")
-		page.addToken(0x0F, "Collection")
-		page.addToken(0x10, "Class")
-		page.addToken(0x12, "CollectionId")
-		page.addToken(0x13, "GetChanges")
-		page.addToken(0x14, "MoreAvailable")
-		page.addToken(0x15, "WindowSize")
-		page.addToken(0x16, "Commands")
-		page.addToken(0x17, "Options")
-		page.addToken(0x18, "FilterType")
-		page.addToken(0x1B, "Conflict")
-		page.addToken(0x1C, "Collections")
-		page.addToken(0x1D, "ApplicationData")
-		page.addToken(0x1E, "DeletesAsMoves")
-		page.addToken(0x20, "Supported")
-		page.addToken(0x21, "SoftDelete")
-		page.addToken(0x22, "MIMESupport")
-		page.addToken(0x23, "MIMETruncation")
-		page.addToken(0x24, "Wait")
-		page.addToken(0x25, "Limit")
-		page.addToken(0x26, "Partial")
-		page.addToken(0x27, "ConversationMode")
-		page.addToken(0x28, "MaxItems")
-		page.addToken(0x29, "HeartbeatInterval")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 1: Contacts
-		# region Contacts Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Contacts:"
-		page.xmlns = "contacts"
-
-		page.addToken(0x05, "Anniversary")
-		page.addToken(0x06, "AssistantName")
-		page.addToken(0x07, "AssistantTelephoneNumber")
-		page.addToken(0x08, "Birthday")
-		page.addToken(0x0C, "Business2PhoneNumber")
-		page.addToken(0x0D, "BusinessCity")
-		page.addToken(0x0E, "BusinessCountry")
-		page.addToken(0x0F, "BusinessPostalCode")
-		page.addToken(0x10, "BusinessState")
-		page.addToken(0x11, "BusinessStreet")
-		page.addToken(0x12, "BusinessFaxNumber")
-		page.addToken(0x13, "BusinessPhoneNumber")
-		page.addToken(0x14, "CarPhoneNumber")
-		page.addToken(0x15, "Categories")
-		page.addToken(0x16, "Category")
-		page.addToken(0x17, "Children")
-		page.addToken(0x18, "Child")
-		page.addToken(0x19, "CompanyName")
-		page.addToken(0x1A, "Department")
-		page.addToken(0x1B, "Email1Address")
-		page.addToken(0x1C, "Email2Address")
-		page.addToken(0x1D, "Email3Address")
-		page.addToken(0x1E, "FileAs")
-		page.addToken(0x1F, "FirstName")
-		page.addToken(0x20, "Home2PhoneNumber")
-		page.addToken(0x21, "HomeCity")
-		page.addToken(0x22, "HomeCountry")
-		page.addToken(0x23, "HomePostalCode")
-		page.addToken(0x24, "HomeState")
-		page.addToken(0x25, "HomeStreet")
-		page.addToken(0x26, "HomeFaxNumber")
-		page.addToken(0x27, "HomePhoneNumber")
-		page.addToken(0x28, "JobTitle")
-		page.addToken(0x29, "LastName")
-		page.addToken(0x2A, "MiddleName")
-		page.addToken(0x2B, "MobilePhoneNumber")
-		page.addToken(0x2C, "OfficeLocation")
-		page.addToken(0x2D, "OtherCity")
-		page.addToken(0x2E, "OtherCountry")
-		page.addToken(0x2F, "OtherPostalCode")
-		page.addToken(0x30, "OtherState")
-		page.addToken(0x31, "OtherStreet")
-		page.addToken(0x32, "PagerNumber")
-		page.addToken(0x33, "RadioPhoneNumber")
-		page.addToken(0x34, "Spouse")
-		page.addToken(0x35, "Suffix")
-		page.addToken(0x36, "Title")
-		page.addToken(0x37, "Webpage")
-		page.addToken(0x38, "YomiCompanyName")
-		page.addToken(0x39, "YomiFirstName")
-		page.addToken(0x3A, "YomiLastName")
-		page.addToken(0x3C, "Picture")
-		page.addToken(0x3D, "Alias")
-		page.addToken(0x3E, "WeightedRank")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 2: Email
-		# region Email Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Email:"
-		page.xmlns = "email"
-
-		page.addToken(0x0F, "DateReceived")
-		page.addToken(0x11, "DisplayTo")
-		page.addToken(0x12, "Importance")
-		page.addToken(0x13, "MessageClass")
-		page.addToken(0x14, "Subject")
-		page.addToken(0x15, "Read")
-		page.addToken(0x16, "To")
-		page.addToken(0x17, "CC")
-		page.addToken(0x18, "From")
-		page.addToken(0x19, "ReplyTo")
-		page.addToken(0x1A, "AllDayEvent")
-		page.addToken(0x1B, "Categories")
-		page.addToken(0x1C, "Category")
-		page.addToken(0x1D, "DTStamp")
-		page.addToken(0x1E, "EndTime")
-		page.addToken(0x1F, "InstanceType")
-		page.addToken(0x20, "BusyStatus")
-		page.addToken(0x21, "Location")
-		page.addToken(0x22, "MeetingRequest")
-		page.addToken(0x23, "Organizer")
-		page.addToken(0x24, "RecurrenceId")
-		page.addToken(0x25, "Reminder")
-		page.addToken(0x26, "ResponseRequested")
-		page.addToken(0x27, "Recurrences")
-		page.addToken(0x28, "Recurrence")
-		page.addToken(0x29, "Recurrence_Type")
-		page.addToken(0x2A, "Recurrence_Until")
-		page.addToken(0x2B, "Recurrence_Occurrences")
-		page.addToken(0x2C, "Recurrence_Interval")
-		page.addToken(0x2D, "Recurrence_DayOfWeek")
-		page.addToken(0x2E, "Recurrence_DayOfMonth")
-		page.addToken(0x2F, "Recurrence_WeekOfMonth")
-		page.addToken(0x30, "Recurrence_MonthOfYear")
-		page.addToken(0x31, "StartTime")
-		page.addToken(0x32, "Sensitivity")
-		page.addToken(0x33, "TimeZone")
-		page.addToken(0x34, "GlobalObjId")
-		page.addToken(0x35, "ThreadTopic")
-		page.addToken(0x39, "InternetCPID")
-		page.addToken(0x3A, "Flag")
-		page.addToken(0x3B, "FlagStatus")
-		page.addToken(0x3C, "ContentClass")
-		page.addToken(0x3D, "FlagType")
-		page.addToken(0x3E, "CompleteTime")
-		page.addToken(0x3F, "DisallowNewTimeProposal")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 3: AirNotify - retired
-		# region AirNotify Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = ""
-		page.xmlns = ""
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 4: Calendar
-		# region Calendar Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Calendar:"
-		page.xmlns = "calendar"
-
-		page.addToken(0x05, "TimeZone")
-		page.addToken(0x06, "AllDayEvent")
-		page.addToken(0x07, "Attendees")
-		page.addToken(0x08, "Attendee")
-		page.addToken(0x09, "Attendee_Email")
-		page.addToken(0x0A, "Attendee_Name")
-		page.addToken(0x0D, "BusyStatus")
-		page.addToken(0x0E, "Categories")
-		page.addToken(0x0F, "Category")
-		page.addToken(0x11, "DTStamp")
-		page.addToken(0x12, "EndTime")
-		page.addToken(0x13, "Exception")
-		page.addToken(0x14, "Exceptions")
-		page.addToken(0x15, "Exception_Deleted")
-		page.addToken(0x16, "Exception_StartTime")
-		page.addToken(0x17, "Location")
-		page.addToken(0x18, "MeetingStatus")
-		page.addToken(0x19, "Organizer_Email")
-		page.addToken(0x1A, "Organizer_Name")
-		page.addToken(0x1B, "Recurrence")
-		page.addToken(0x1C, "Recurrence_Type")
-		page.addToken(0x1D, "Recurrence_Until")
-		page.addToken(0x1E, "Recurrence_Occurrences")
-		page.addToken(0x1F, "Recurrence_Interval")
-		page.addToken(0x20, "Recurrence_DayOfWeek")
-		page.addToken(0x21, "Recurrence_DayOfMonth")
-		page.addToken(0x22, "Recurrence_WeekOfMonth")
-		page.addToken(0x23, "Recurrence_MonthOfYear")
-		page.addToken(0x24, "Reminder")
-		page.addToken(0x25, "Sensitivity")
-		page.addToken(0x26, "Subject")
-		page.addToken(0x27, "StartTime")
-		page.addToken(0x28, "UID")
-		page.addToken(0x29, "Attendee_Status")
-		page.addToken(0x2A, "Attendee_Type")
-		page.addToken(0x33, "DisallowNewTimeProposal")
-		page.addToken(0x34, "ResponseRequested")
-		page.addToken(0x35, "AppointmentReplyTime")
-		page.addToken(0x36, "ResponseType")
-		page.addToken(0x37, "CalendarType")
-		page.addToken(0x38, "IsLeapMonth")
-		page.addToken(0x39, "FirstDayOfWeek")
-		page.addToken(0x3A, "OnlineMeetingConfLink")
-		page.addToken(0x3B, "OnlineMeetingExternalLink")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 5: Move
-		# region Move Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Move:"
-		page.xmlns = "move"
-
-		page.addToken(0x05, "MoveItems")
-		page.addToken(0x06, "Move")
-		page.addToken(0x07, "SrcMsgId")
-		page.addToken(0x08, "SrcFldId")
-		page.addToken(0x09, "DstFldId")
-		page.addToken(0x0A, "Response")
-		page.addToken(0x0B, "Status")
-		page.addToken(0x0C, "DstMsgId")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 6: ItemEstimate
-		# region ItemEstimate Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "GetItemEstimate:"
-		page.xmlns = "getitemestimate"
-
-		page.addToken(0x05, "GetItemEstimate")
-		page.addToken(0x06, "Version")
-		page.addToken(0x07, "Collections")
-		page.addToken(0x08, "Collection")
-		page.addToken(0x09, "Class")
-		page.addToken(0x0A, "CollectionId")
-		page.addToken(0x0B, "DateTime")
-		page.addToken(0x0C, "Estimate")
-		page.addToken(0x0D, "Response")
-		page.addToken(0x0E, "Status")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 7: FolderHierarchy
-		# region FolderHierarchy Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "FolderHierarchy:"
-		page.xmlns = "folderhierarchy"
-
-		page.addToken(0x07, "DisplayName")
-		page.addToken(0x08, "ServerId")
-		page.addToken(0x09, "ParentId")
-		page.addToken(0x0A, "Type")
-		page.addToken(0x0C, "Status")
-		page.addToken(0x0E, "Changes")
-		page.addToken(0x0F, "Add")
-		page.addToken(0x10, "Delete")
-		page.addToken(0x11, "Update")
-		page.addToken(0x12, "SyncKey")
-		page.addToken(0x13, "FolderCreate")
-		page.addToken(0x14, "FolderDelete")
-		page.addToken(0x15, "FolderUpdate")
-		page.addToken(0x16, "FolderSync")
-		page.addToken(0x17, "Count")
-
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 8: MeetingResponse
-		# region MeetingResponse Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "MeetingResponse:"
-		page.xmlns = "meetingresponse"
-
-		page.addToken(0x05, "CalendarId")
-		page.addToken(0x06, "CollectionId")
-		page.addToken(0x07, "MeetingResponse")
-		page.addToken(0x08, "RequestId")
-		page.addToken(0x09, "Request")
-		page.addToken(0x0A, "Result")
-		page.addToken(0x0B, "Status")
-		page.addToken(0x0C, "UserResponse")
-		page.addToken(0x0E, "InstanceId")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 9: Tasks
-		# region Tasks Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Tasks:"
-		page.xmlns = "tasks"
-
-		page.addToken(0x08, "Categories")
-		page.addToken(0x09, "Category")
-		page.addToken(0x0A, "Complete")
-		page.addToken(0x0B, "DateCompleted")
-		page.addToken(0x0C, "DueDate")
-		page.addToken(0x0D, "UTCDueDate")
-		page.addToken(0x0E, "Importance")
-		page.addToken(0x0F, "Recurrence")
-		page.addToken(0x10, "Recurrence_Type")
-		page.addToken(0x11, "Recurrence_Start")
-		page.addToken(0x12, "Recurrence_Until")
-		page.addToken(0x13, "Recurrence_Occurrences")
-		page.addToken(0x14, "Recurrence_Interval")
-		page.addToken(0x15, "Recurrence_DayOfMonth")
-		page.addToken(0x16, "Recurrence_DayOfWeek")
-		page.addToken(0x17, "Recurrence_WeekOfMonth")
-		page.addToken(0x18, "Recurrence_MonthOfYear")
-		page.addToken(0x19, "Recurrence_Regenerate")
-		page.addToken(0x1A, "Recurrence_DeadOccur")
-		page.addToken(0x1B, "ReminderSet")
-		page.addToken(0x1C, "ReminderTime")
-		page.addToken(0x1D, "Sensitivity")
-		page.addToken(0x1E, "StartDate")
-		page.addToken(0x1F, "UTCStartDate")
-		page.addToken(0x20, "Subject")
-		page.addToken(0x22, "OrdinalDate")
-		page.addToken(0x23, "SubOrdinalDate")
-		page.addToken(0x24, "CalendarType")
-		page.addToken(0x25, "IsLeapMonth")
-		page.addToken(0x26, "FirstDayOfWeek")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 10: ResolveRecipients
-		# region ResolveRecipients Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "ResolveRecipients:"
-		page.xmlns = "resolverecipients"
-
-		page.addToken(0x05, "ResolveRecipients")
-		page.addToken(0x06, "Response")
-		page.addToken(0x07, "Status")
-		page.addToken(0x08, "Type")
-		page.addToken(0x09, "Recipient")
-		page.addToken(0x0A, "DisplayName")
-		page.addToken(0x0B, "EmailAddress")
-		page.addToken(0x0C, "Certificates")
-		page.addToken(0x0D, "Certificate")
-		page.addToken(0x0E, "MiniCertificate")
-		page.addToken(0x0F, "Options")
-		page.addToken(0x10, "To")
-		page.addToken(0x11, "CertificateRetrieval")
-		page.addToken(0x12, "RecipientCount")
-		page.addToken(0x13, "MaxCertificates")
-		page.addToken(0x14, "MaxAmbiguousRecipients")
-		page.addToken(0x15, "CertificateCount")
-		page.addToken(0x16, "Availability")
-		page.addToken(0x17, "StartTime")
-		page.addToken(0x18, "EndTime")
-		page.addToken(0x19, "MergedFreeBusy")
-		page.addToken(0x1A, "Picture")
-		page.addToken(0x1B, "MaxSize")
-		page.addToken(0x1C, "Data")
-		page.addToken(0x1D, "MaxPictures")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 11: ValidateCert
-		# region ValidateCert Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "ValidateCert:"
-		page.xmlns = "validatecert"
-
-		page.addToken(0x05, "ValidateCert")
-		page.addToken(0x06, "Certificates")
-		page.addToken(0x07, "Certificate")
-		page.addToken(0x08, "CertificateChain")
-		page.addToken(0x09, "CheckCRL")
-		page.addToken(0x0A, "Status")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 12: Contacts2
-		# region Contacts2 Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Contacts2:"
-		page.xmlns = "contacts2"
-
-		page.addToken(0x05, "CustomerId")
-		page.addToken(0x06, "GovernmentId")
-		page.addToken(0x07, "IMAddress")
-		page.addToken(0x08, "IMAddress2")
-		page.addToken(0x09, "IMAddress3")
-		page.addToken(0x0A, "ManagerName")
-		page.addToken(0x0B, "CompanyMainPhone")
-		page.addToken(0x0C, "AccountName")
-		page.addToken(0x0D, "NickName")
-		page.addToken(0x0E, "MMS")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 13: Ping
-		# region Ping Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Ping:"
-		page.xmlns = "ping"
-
-		page.addToken(0x05, "Ping")
-		page.addToken(0x06, "AutdState")  # Per MS-ASWBXML, this tag is not used by protocol
-		page.addToken(0x07, "Status")
-		page.addToken(0x08, "HeartbeatInterval")
-		page.addToken(0x09, "Folders")
-		page.addToken(0x0A, "Folder")
-		page.addToken(0x0B, "Id")
-		page.addToken(0x0C, "Class")
-		page.addToken(0x0D, "MaxFolders")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 14: Provision
-		# region Provision Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Provision:"
-		page.xmlns = "provision"
-
-		page.addToken(0x05, "Provision")
-		page.addToken(0x06, "Policies")
-		page.addToken(0x07, "Policy")
-		page.addToken(0x08, "PolicyType")
-		page.addToken(0x09, "PolicyKey")
-		page.addToken(0x0A, "Data")
-		page.addToken(0x0B, "Status")
-		page.addToken(0x0C, "RemoteWipe")
-		page.addToken(0x0D, "EASProvisionDoc")
-		page.addToken(0x0E, "DevicePasswordEnabled")
-		page.addToken(0x0F, "AlphanumericDevicePasswordRequired")
-		page.addToken(0x10, "RequireStorageCardEncryption")
-		page.addToken(0x11, "PasswordRecoveryEnabled")
-		page.addToken(0x13, "AttachmentsEnabled")
-		page.addToken(0x14, "MinDevicePasswordLength")
-		page.addToken(0x15, "MaxInactivityTimeDeviceLock")
-		page.addToken(0x16, "MaxDevicePasswordFailedAttempts")
-		page.addToken(0x17, "MaxAttachmentSize")
-		page.addToken(0x18, "AllowSimpleDevicePassword")
-		page.addToken(0x19, "DevicePasswordExpiration")
-		page.addToken(0x1A, "DevicePasswordHistory")
-		page.addToken(0x1B, "AllowStorageCard")
-		page.addToken(0x1C, "AllowCamera")
-		page.addToken(0x1D, "RequireDeviceEncryption")
-		page.addToken(0x1E, "AllowUnsignedApplications")
-		page.addToken(0x1F, "AllowUnsignedInstallationPackages")
-		page.addToken(0x20, "MinDevicePasswordComplexCharacters")
-		page.addToken(0x21, "AllowWiFi")
-		page.addToken(0x22, "AllowTextMessaging")
-		page.addToken(0x23, "AllowPOPIMAPEmail")
-		page.addToken(0x24, "AllowBluetooth")
-		page.addToken(0x25, "AllowIrDA")
-		page.addToken(0x26, "RequireManualSyncWhenRoaming")
-		page.addToken(0x27, "AllowDesktopSync")
-		page.addToken(0x28, "MaxCalendarAgeFilter")
-		page.addToken(0x29, "AllowHTMLEmail")
-		page.addToken(0x2A, "MaxEmailAgeFilter")
-		page.addToken(0x2B, "MaxEmailBodyTruncationSize")
-		page.addToken(0x2C, "MaxEmailHTMLBodyTruncationSize")
-		page.addToken(0x2D, "RequireSignedSMIMEMessages")
-		page.addToken(0x2E, "RequireEncryptedSMIMEMessages")
-		page.addToken(0x2F, "RequireSignedSMIMEAlgorithm")
-		page.addToken(0x30, "RequireEncryptionSMIMEAlgorithm")
-		page.addToken(0x31, "AllowSMIMEEncryptionAlgorithmNegotiation")
-		page.addToken(0x32, "AllowSMIMESoftCerts")
-		page.addToken(0x33, "AllowBrowser")
-		page.addToken(0x34, "AllowConsumerEmail")
-		page.addToken(0x35, "AllowRemoteDesktop")
-		page.addToken(0x36, "AllowInternetSharing")
-		page.addToken(0x37, "UnapprovedInROMApplicationList")
-		page.addToken(0x38, "ApplicationName")
-		page.addToken(0x39, "ApprovedApplicationList")
-		page.addToken(0x3A, "Hash")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 15: Search
-		# region Search Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Search:"
-		page.xmlns = "search"
-
-		page.addToken(0x05, "Search")
-		page.addToken(0x07, "Store")
-		page.addToken(0x08, "Name")
-		page.addToken(0x09, "Query")
-		page.addToken(0x0A, "Options")
-		page.addToken(0x0B, "Range")
-		page.addToken(0x0C, "Status")
-		page.addToken(0x0D, "Response")
-		page.addToken(0x0E, "Result")
-		page.addToken(0x0F, "Properties")
-		page.addToken(0x10, "Total")
-		page.addToken(0x11, "EqualTo")
-		page.addToken(0x12, "Value")
-		page.addToken(0x13, "And")
-		page.addToken(0x14, "Or")
-		page.addToken(0x15, "FreeText")
-		page.addToken(0x17, "DeepTraversal")
-		page.addToken(0x18, "LongId")
-		page.addToken(0x19, "RebuildResults")
-		page.addToken(0x1A, "LessThan")
-		page.addToken(0x1B, "GreaterThan")
-		page.addToken(0x1E, "UserName")
-		page.addToken(0x1F, "Password")
-		page.addToken(0x20, "ConversationId")
-		page.addToken(0x21, "Picture")
-		page.addToken(0x22, "MaxSize")
-		page.addToken(0x23, "MaxPictures")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 16: GAL
-		# region GAL Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "GAL:"
-		page.xmlns = "gal"
-
-		page.addToken(0x05, "DisplayName")
-		page.addToken(0x06, "Phone")
-		page.addToken(0x07, "Office")
-		page.addToken(0x08, "Title")
-		page.addToken(0x09, "Company")
-		page.addToken(0x0A, "Alias")
-		page.addToken(0x0B, "FirstName")
-		page.addToken(0x0C, "LastName")
-		page.addToken(0x0D, "HomePhone")
-		page.addToken(0x0E, "MobilePhone")
-		page.addToken(0x0F, "EmailAddress")
-		page.addToken(0x10, "Picture")
-		page.addToken(0x11, "Status")
-		page.addToken(0x12, "Data")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 17: AirSyncBase
-		# region AirSyncBase Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "AirSyncBase:"
-		page.xmlns = "airsyncbase"
-
-		page.addToken(0x05, "BodyPreference")
-		page.addToken(0x06, "Type")
-		page.addToken(0x07, "TruncationSize")
-		page.addToken(0x08, "AllOrNone")
-		page.addToken(0x0A, "Body")
-		page.addToken(0x0B, "Data")
-		page.addToken(0x0C, "EstimatedDataSize")
-		page.addToken(0x0D, "Truncated")
-		page.addToken(0x0E, "Attachments")
-		page.addToken(0x0F, "Attachment")
-		page.addToken(0x10, "DisplayName")
-		page.addToken(0x11, "FileReference")
-		page.addToken(0x12, "Method")
-		page.addToken(0x13, "ContentId")
-		page.addToken(0x14, "ContentLocation")
-		page.addToken(0x15, "IsInline")
-		page.addToken(0x16, "NativeBodyType")
-		page.addToken(0x17, "ContentType")
-		page.addToken(0x18, "Preview")
-		page.addToken(0x19, "BodyPartPreference")
-		page.addToken(0x1A, "BodyPart")
-		page.addToken(0x1B, "Status")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 18: Settings
-		# region Settings Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Settings:"
-		page.xmlns = "settings"
-
-		page.addToken(0x05, "Settings")
-		page.addToken(0x06, "Status")
-		page.addToken(0x07, "Get")
-		page.addToken(0x08, "Set")
-		page.addToken(0x09, "Oof")
-		page.addToken(0x0A, "OofState")
-		page.addToken(0x0B, "StartTime")
-		page.addToken(0x0C, "EndTime")
-		page.addToken(0x0D, "OofMessage")
-		page.addToken(0x0E, "AppliesToInternal")
-		page.addToken(0x0F, "AppliesToExternalKnown")
-		page.addToken(0x10, "AppliesToExternalUnknown")
-		page.addToken(0x11, "Enabled")
-		page.addToken(0x12, "ReplyMessage")
-		page.addToken(0x13, "BodyType")
-		page.addToken(0x14, "DevicePassword")
-		page.addToken(0x15, "Password")
-		page.addToken(0x16, "DeviceInformation")
-		page.addToken(0x17, "Model")
-		page.addToken(0x18, "IMEI")
-		page.addToken(0x19, "FriendlyName")
-		page.addToken(0x1A, "OS")
-		page.addToken(0x1B, "OSLanguage")
-		page.addToken(0x1C, "PhoneNumber")
-		page.addToken(0x1D, "UserInformation")
-		page.addToken(0x1E, "EmailAddresses")
-		page.addToken(0x1F, "SmtpAddress")
-		page.addToken(0x20, "UserAgent")
-		page.addToken(0x21, "EnableOutboundSMS")
-		page.addToken(0x22, "MobileOperator")
-		page.addToken(0x23, "PrimarySmtpAddress")
-		page.addToken(0x24, "Accounts")
-		page.addToken(0x25, "Account")
-		page.addToken(0x26, "AccountId")
-		page.addToken(0x27, "AccountName")
-		page.addToken(0x28, "UserDisplayName")
-		page.addToken(0x29, "SendDisabled")
-		page.addToken(0x2B, "RightsManagementInformation")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 19: DocumentLibrary
-		# region DocumentLibrary Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "DocumentLibrary:"
-		page.xmlns = "documentlibrary"
-
-		page.addToken(0x05, "LinkId")
-		page.addToken(0x06, "DisplayName")
-		page.addToken(0x07, "IsFolder")
-		page.addToken(0x08, "CreationDate")
-		page.addToken(0x09, "LastModifiedDate")
-		page.addToken(0x0A, "IsHidden")
-		page.addToken(0x0B, "ContentLength")
-		page.addToken(0x0C, "ContentType")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 20: ItemOperations
-		# region ItemOperations Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "ItemOperations:"
-		page.xmlns = "itemoperations"
-
-		page.addToken(0x05, "ItemOperations")
-		page.addToken(0x06, "Fetch")
-		page.addToken(0x07, "Store")
-		page.addToken(0x08, "Options")
-		page.addToken(0x09, "Range")
-		page.addToken(0x0A, "Total")
-		page.addToken(0x0B, "Properties")
-		page.addToken(0x0C, "Data")
-		page.addToken(0x0D, "Status")
-		page.addToken(0x0E, "Response")
-		page.addToken(0x0F, "Version")
-		page.addToken(0x10, "Schema")
-		page.addToken(0x11, "Part")
-		page.addToken(0x12, "EmptyFolderContents")
-		page.addToken(0x13, "DeleteSubFolders")
-		page.addToken(0x14, "UserName")
-		page.addToken(0x15, "Password")
-		page.addToken(0x16, "Move")
-		page.addToken(0x17, "DstFldId")
-		page.addToken(0x18, "ConversationId")
-		page.addToken(0x19, "MoveAlways")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 21: ComposeMail
-		# region ComposeMail Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "ComposeMail:"
-		page.xmlns = "composemail"
-
-		page.addToken(0x05, "SendMail")
-		page.addToken(0x06, "SmartForward")
-		page.addToken(0x07, "SmartReply")
-		page.addToken(0x08, "SaveInSentItems")
-		page.addToken(0x09, "ReplaceMime")
-		page.addToken(0x0B, "Source")
-		page.addToken(0x0C, "FolderId")
-		page.addToken(0x0D, "ItemId")
-		page.addToken(0x0E, "LongId")
-		page.addToken(0x0F, "InstanceId")
-		page.addToken(0x10, "MIME")
-		page.addToken(0x11, "ClientId")
-		page.addToken(0x12, "Status")
-		page.addToken(0x13, "AccountId")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 22: Email2
-		# region Email2 Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Email2:"
-		page.xmlns = "email2"
-
-		page.addToken(0x05, "UmCallerID")
-		page.addToken(0x06, "UmUserNotes")
-		page.addToken(0x07, "UmAttDuration")
-		page.addToken(0x08, "UmAttOrder")
-		page.addToken(0x09, "ConversationId")
-		page.addToken(0x0A, "ConversationIndex")
-		page.addToken(0x0B, "LastVerbExecuted")
-		page.addToken(0x0C, "LastVerbExecutionTime")
-		page.addToken(0x0D, "ReceivedAsBcc")
-		page.addToken(0x0E, "Sender")
-		page.addToken(0x0F, "CalendarType")
-		page.addToken(0x10, "IsLeapMonth")
-		page.addToken(0x11, "AccountId")
-		page.addToken(0x12, "FirstDayOfWeek")
-		page.addToken(0x13, "MeetingMessageType")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 23: Notes
-		# region Notes Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "Notes:"
-		page.xmlns = "notes"
-
-		page.addToken(0x05, "Subject")
-		page.addToken(0x06, "MessageClass")
-		page.addToken(0x07, "LastModifiedDate")
-		page.addToken(0x08, "Categories")
-		page.addToken(0x09, "Category")
-		self.codePages.append(page)
-		# endregion
-
-		# Code Page 24: RightsManagement
-		# region RightsManagement Code Page
-		page = ASWBXMLCodePage()
-		page.namespace = "RightsManagement:"
-		page.xmlns = "rightsmanagement"
-
-		page.addToken(0x05, "RightsManagementSupport")
-		page.addToken(0x06, "RightsManagementTemplates")
-		page.addToken(0x07, "RightsManagementTemplate")
-		page.addToken(0x08, "RightsManagementLicense")
-		page.addToken(0x09, "EditAllowed")
-		page.addToken(0x0A, "ReplyAllowed")
-		page.addToken(0x0B, "ReplyAllAllowed")
-		page.addToken(0x0C, "ForwardAllowed")
-		page.addToken(0x0D, "ModifyRecipientsAllowed")
-		page.addToken(0x0E, "ExtractAllowed")
-		page.addToken(0x0F, "PrintAllowed")
-		page.addToken(0x10, "ExportAllowed")
-		page.addToken(0x11, "ProgrammaticAccessAllowed")
-		page.addToken(0x12, "RMOwner")
-		page.addToken(0x13, "ContentExpiryDate")
-		page.addToken(0x14, "TemplateID")
-		page.addToken(0x15, "TemplateName")
-		page.addToken(0x16, "TemplateDescription")
-		page.addToken(0x17, "ContentOwner")
-		page.addToken(0x18, "RemoveRightsManagementDistribution")
-		self.codePages.append(page)
-		# endregion
-		# endregion
-	
-	def loadXml(self, strXML):
-		# note xmlDoc has .childNodes and .parentNode
-		self.xmlDoc = xml.dom.minidom.parseString(strXML)
-
-	def getXml(self):
-		if (self.xmlDoc != None):
-			try:
-				return self.xmlDoc.toprettyxml(indent="    ", newl="\n")
-			except:
-				return self.xmlDoc.toxml()
-	
-	def loadBytes(self, byteWBXML):
-		
-		currentNode = self.xmlDoc
-		
-		wbXMLBytes = ASWBXMLByteQueue(byteWBXML)
-		# Version is ignored
-		version = wbXMLBytes.dequeueAndLog()
-
-		# Public Identifier is ignored
-		publicId = wbXMLBytes.dequeueMultibyteInt()
-		
-		logging.debug("Version: %d, Public Identifier: %d" % (version, publicId))
-		
-		# Character set
-		# Currently only UTF-8 is supported, throw if something else
-		charset = wbXMLBytes.dequeueMultibyteInt()
-		if (charset != 0x6A):
-			raise InvalidDataException("ASWBXML only supports UTF-8 encoded XML.")
-
-		# String table length
-		# This should be 0, MS-ASWBXML does not use string tables
-		stringTableLength = wbXMLBytes.dequeueMultibyteInt()
-		if (stringTableLength != 0):
-			raise InvalidDataException("WBXML data contains a string table.")
-
-		# Now we should be at the body of the data.
-		# Add the declaration
-		unusedArray = [GlobalTokens.ENTITY, GlobalTokens.EXT_0, GlobalTokens.EXT_1, GlobalTokens.EXT_2, GlobalTokens.EXT_I_0, GlobalTokens.EXT_I_1, GlobalTokens.EXT_I_2, GlobalTokens.EXT_T_0, GlobalTokens.EXT_T_1, GlobalTokens.EXT_T_2, GlobalTokens.LITERAL, GlobalTokens.LITERAL_A, GlobalTokens.LITERAL_AC, GlobalTokens.LITERAL_C, GlobalTokens.PI, GlobalTokens.STR_T]
-		
-		while ( wbXMLBytes.qsize() > 0):
-			currentByte = wbXMLBytes.dequeueAndLog()
-			if ( currentByte == GlobalTokens.SWITCH_PAGE ):
-				newCodePage = wbXMLBytes.dequeueAndLog()
-				if (newCodePage >= 0 and newCodePage < 25):
-					self.currentCodePage = newCodePage
-				else:
-					raise InvalidDataException("Unknown code page ID 0x{0:X} encountered in WBXML".format(currentByte))
-			elif  ( currentByte == GlobalTokens.END ):
-				if (currentNode != None and currentNode.parentNode != None):
-					currentNode = currentNode.parentNode
-				else:
-					raise InvalidDataException("END global token encountered out of sequence")
-					break
-			elif  ( currentByte == GlobalTokens.OPAQUE ):
-				CDATALength = wbXMLBytes.dequeueMultibyteInt()
-				newOpaqueNode = self.xmlDoc.createCDATASection(wbXMLBytes.dequeueString(CDATALength))
-				currentNode.appendChild(newOpaqueNode)
-
-			elif  ( currentByte == GlobalTokens.STR_I ):
-				newTextNode = self.xmlDoc.createTextNode(wbXMLBytes.dequeueString())
-				currentNode.appendChild(newTextNode)
-
-			elif ( currentByte in unusedArray):
-				raise InvalidDataException("Encountered unknown global token 0x{0:X}.".format(currentByte))
-			else:
-				hasAttributes = (currentByte & 0x80) > 0
-				hasContent = (currentByte & 0x40) > 0
-
-				token = currentByte & 0x3F
-				if (hasAttributes):
-					raise InvalidDataException("Token 0x{0:X} has attributes.".format(token))
-
-				strTag = self.codePages[self.currentCodePage].getTag(token)
-				if (strTag == None):
-					strTag = "UNKNOWN_TAG_{0,2:X}".format(token)
-
-				newNode = self.xmlDoc.createElement(strTag)
-				# not sure if this should be set on every node or not
-				#newNode.setAttribute("xmlns", self.codePages[self.currentCodePage].xmlns)
-				
-				currentNode.appendChild(newNode)
-
-				if (hasContent):
-					currentNode = newNode
-
-		logging.debug("Total bytes dequeued: %d" % wbXMLBytes.bytesDequeued)

libmproxy/contrib/wbxml/ASWBXMLByteQueue.py

@@ -1,103 +0,0 @@
-#!/usr/bin/env python
-'''
-@author: David Shaw, david.shaw.aw@gmail.com
-
-Inspired by EAS Inspector for Fiddler
-https://easinspectorforfiddler.codeplex.com
-
------ The MIT License (MIT) ----- 
-Filename: ASWBXMLByteQueue.py
-Copyright (c) 2014, David P. Shaw
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-'''
-from Queue import Queue
-import logging
-
-class ASWBXMLByteQueue(Queue):
-
-    def __init__(self, wbxmlBytes):
-        
-        self.bytesDequeued = 0
-        self.bytesEnqueued = 0
-        
-        Queue.__init__(self)
-
-        for byte in wbxmlBytes:
-            self.put(ord(byte))
-            self.bytesEnqueued += 1
-        
-        
-        logging.debug("Array byte count: %d, enqueued: %d" % (self.qsize(), self.bytesEnqueued))
-    
-    """
-    Created to debug the dequeueing of bytes
-    """
-    def dequeueAndLog(self):
-        singleByte = self.get()
-        self.bytesDequeued += 1
-        logging.debug("Dequeued byte 0x{0:X} ({1} total)".format(singleByte, self.bytesDequeued))
-        return singleByte
-    
-    """
-    Return true if the continuation bit is set in the byte
-    """
-    def checkContinuationBit(self, byteval):
-        continuationBitmask = 0x80
-        return (continuationBitmask & byteval) != 0
-    
-    def dequeueMultibyteInt(self):
-        iReturn = 0
-        singleByte = 0xFF
-         
-        while True:
-            iReturn <<= 7
-            if (self.qsize() == 0):
-                break
-            else:
-                singleByte = self.dequeueAndLog()
-            iReturn += int(singleByte & 0x7F)
-            if not self.checkContinuationBit(singleByte):
-                return iReturn
-
-    def dequeueString(self, length=None):
-        if ( length != None):
-            currentByte = 0x00
-            strReturn = ""
-            for i in range(0, length):
-                # TODO: Improve this handling. We are technically UTF-8, meaning
-                # that characters could be more than one byte long. This will fail if we have
-                # characters outside of the US-ASCII range
-                if ( self.qsize() == 0 ):
-                    break
-                currentByte = self.dequeueAndLog()
-                strReturn += chr(currentByte)
-
-        else:
-            currentByte = 0x00
-            strReturn = ""
-            while True:
-                currentByte = self.dequeueAndLog()
-                if (currentByte != 0x00):
-                    strReturn += chr(currentByte)
-                else:
-                    break
-
-        return strReturn
-

libmproxy/contrib/wbxml/ASWBXMLCodePage.py

@@ -1,52 +0,0 @@
-#!/usr/bin/env python
-'''
-@author: David Shaw, david.shaw.aw@gmail.com
-
-Inspired by EAS Inspector for Fiddler
-https://easinspectorforfiddler.codeplex.com
-
------ The MIT License (MIT) ----- 
-Filename: ASWBXMLCodePage.py
-Copyright (c) 2014, David P. Shaw
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-'''
-class ASWBXMLCodePage:
-	def __init__(self):
-		self.namespace = ""
-		self.xmlns = ""
-		self.tokenLookup = {}
-		self.tagLookup = {}
-	
-	def addToken(self, token, tag):
-		self.tokenLookup[token] = tag
-		self.tagLookup[tag] = token
-	
-	def getToken(self, tag):
-		if self.tagLookup.has_key(tag):
-			return self.tagLookup[tag]
-		return 0xFF
-	
-	def getTag(self, token):
-		if self.tokenLookup.has_key(token):
-			return self.tokenLookup[token]
-		return None
-	
-	def __repr__(self):
-		return str(self.tokenLookup)

libmproxy/contrib/wbxml/GlobalTokens.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-'''
-@author: David Shaw, david.shaw.aw@gmail.com
-
-Inspired by EAS Inspector for Fiddler
-https://easinspectorforfiddler.codeplex.com
-
------ The MIT License (MIT) ----- 
-Filename: GlobalTokens.py
-Copyright (c) 2014, David P. Shaw
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-'''
-class GlobalTokens:
-    SWITCH_PAGE = 0x00
-    END = 0x01
-    ENTITY = 0x02
-    STR_I = 0x03
-    LITERAL = 0x04
-    EXT_I_0 = 0x40
-    EXT_I_1 = 0x41
-    EXT_I_2 = 0x42
-    PI = 0x43
-    LITERAL_C = 0x44
-    EXT_T_0 = 0x80
-    EXT_T_1 = 0x81
-    EXT_T_2 = 0x82
-    STR_T = 0x83
-    LITERAL_A = 0x84
-    EXT_0 = 0xC0
-    EXT_1 = 0xC1
-    EXT_2 = 0xC2
-    OPAQUE = 0xC3
-    LITERAL_AC = 0xC4

libmproxy/contrib/wbxml/InvalidDataException.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-'''
-@author: David Shaw, david.shaw.aw@gmail.com
-
-Inspired by EAS Inspector for Fiddler
-https://easinspectorforfiddler.codeplex.com
-
------ The MIT License (MIT) ----- 
-Filename: InvalidDataException.py
-Copyright (c) 2014, David P. Shaw
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-'''
-class InvalidDataException(Exception):
-    pass

libmproxy/controller.py

@@ -1,6 +1,8 @@
-from __future__ import absolute_import
 import Queue, threading
 
+should_exit = False
+
+
 class DummyReply:
     """
         A reply object that does nothing. Useful when we need an object to seem
@@ -34,9 +36,8 @@ class Reply:
 
 
 class Channel:
-    def __init__(self, q, should_exit):
+    def __init__(self, q):
         self.q = q
-        self.should_exit = should_exit
 
     def ask(self, mtype, m):
         """
@@ -45,11 +46,11 @@ class Channel:
         """
         m.reply = Reply(m)
         self.q.put((mtype, m))
-        while not self.should_exit.is_set():
+        while not should_exit:
             try:
                 # The timeout is here so we can handle a should_exit event.
                 g = m.reply.q.get(timeout=0.5)
-            except Queue.Empty:  # pragma: nocover
+            except Queue.Empty: # pragma: nocover
                 continue
             return g
 
@@ -71,13 +72,12 @@ class Slave(threading.Thread):
         self.channel, self.server = channel, server
         self.server.set_channel(channel)
         threading.Thread.__init__(self)
-        self.name = "SlaveThread (%s:%s)" % (self.server.address.host, self.server.address.port)
 
     def run(self):
         self.server.serve_forever()
 
 
-class Master(object):
+class Master:
     """
         Masters get and respond to messages from slaves.
     """
@@ -87,9 +87,8 @@ class Master(object):
         """
         self.server = server
         self.masterq = Queue.Queue()
-        self.should_exit = threading.Event()
 
-    def tick(self, q, timeout):
+    def tick(self, q):
         changed = False
         try:
             # This endless loop runs until the 'Queue.Empty'
@@ -97,7 +96,8 @@ class Master(object):
             # the queue, this speeds up every request by 0.1 seconds,
             # because get_input(..) function is not blocking.
             while True:
-                msg = q.get(timeout=timeout)
+                # Small timeout to prevent pegging the CPU
+                msg = q.get(timeout=0.01)
                 self.handle(*msg)
                 changed = True
         except Queue.Empty:
@@ -105,10 +105,11 @@ class Master(object):
         return changed
 
     def run(self):
-        self.should_exit.clear()
-        self.server.start_slave(Slave, Channel(self.masterq, self.should_exit))
-        while not self.should_exit.is_set():
-            self.tick(self.masterq, 0.01)
+        global should_exit
+        should_exit = False
+        self.server.start_slave(Slave, Channel(self.masterq))
+        while not should_exit:
+            self.tick(self.masterq)
         self.shutdown()
 
     def handle(self, mtype, obj):
@@ -120,7 +121,8 @@ class Master(object):
             obj.reply()
 
     def shutdown(self):
-        if not self.should_exit.is_set():
-            self.should_exit.set()
+        global should_exit
+        if not should_exit:
+            should_exit = True
             if self.server:
                 self.server.shutdown()

libmproxy/dump.py

@@ -1,25 +1,20 @@
-from __future__ import absolute_import
-import sys
-import os
+import sys, os
 import netlib.utils
-from . import flow, filt, utils
-from .protocol import http
+import flow, filt, utils
 
-
-class DumpError(Exception):
-    pass
+class DumpError(Exception): pass
 
 
 class Options(object):
     attributes = [
         "app",
+        "app_external",
         "app_host",
         "app_port",
         "anticache",
         "anticomp",
         "client_replay",
-        "filtstr",
-        "flow_detail",
+        "eventlog",
         "keepserving",
         "kill",
         "no_server",
@@ -34,14 +29,9 @@ class Options(object):
         "showhost",
         "stickycookie",
         "stickyauth",
-        "stream_large_bodies",
         "verbosity",
-        "outfile",
-        "replay_ignore_content",
-        "replay_ignore_params",
-        "replay_ignore_payload_params",
+        "wfile",
     ]
-
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
             setattr(self, k, v)
@@ -52,39 +42,35 @@ class Options(object):
 
 def str_response(resp):
     r = "%s %s"%(resp.code, resp.msg)
-    if resp.is_replay:
+    if resp.is_replay():
         r = "[replay] " + r
     return r
 
 
-def str_request(f, showhost):
-    if f.client_conn:
-        c = f.client_conn.address.host
+def str_request(req, showhost):
+    if req.client_conn:
+        c = req.client_conn.address[0]
     else:
         c = "[replay]"
-    r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
-    if f.request.stickycookie:
+    r = "%s %s %s"%(c, req.method, req.get_url(showhost))
+    if req.stickycookie:
         r = "[stickycookie] " + r
     return r
 
 
 class DumpMaster(flow.FlowMaster):
-    def __init__(self, server, options, outfile=sys.stdout):
+    def __init__(self, server, options, filtstr, outfile=sys.stdout):
         flow.FlowMaster.__init__(self, server, flow.State())
         self.outfile = outfile
         self.o = options
         self.anticache = options.anticache
         self.anticomp = options.anticomp
+        self.eventlog = options.eventlog
         self.showhost = options.showhost
-        self.replay_ignore_params = options.replay_ignore_params
-        self.replay_ignore_content = options.replay_ignore_content
         self.refresh_server_playback = options.refresh_server_playback
-        self.replay_ignore_payload_params = options.replay_ignore_payload_params
 
-        self.set_stream_large_bodies(options.stream_large_bodies)
-
-        if options.filtstr:
-            self.filt = filt.parse(options.filtstr)
+        if filtstr:
+            self.filt = filt.parse(filtstr)
         else:
             self.filt = None
 
@@ -94,10 +80,10 @@ class DumpMaster(flow.FlowMaster):
         if options.stickyauth:
             self.set_stickyauth(options.stickyauth)
 
-        if options.outfile:
-            path = os.path.expanduser(options.outfile[0])
+        if options.wfile:
+            path = os.path.expanduser(options.wfile)
             try:
-                f = file(path, options.outfile[1])
+                f = file(path, "wb")
                 self.start_stream(f, self.filt)
             except IOError, v:
                 raise DumpError(v.strerror)
@@ -115,10 +101,7 @@ class DumpMaster(flow.FlowMaster):
                 self._readflow(options.server_replay),
                 options.kill, options.rheaders,
                 not options.keepserving,
-                options.nopop,
-                options.replay_ignore_params,
-                options.replay_ignore_content,
-                options.replay_ignore_payload_params,
+                options.nopop
             )
 
         if options.client_replay:
@@ -143,10 +126,10 @@ class DumpMaster(flow.FlowMaster):
             try:
                 self.load_flows(freader)
             except flow.FlowReadError, v:
-                self.add_event("Flow file corrupted. Stopped loading.", "error")
+                self.add_event("Flow file corrupted. Stopped loading.")
 
         if self.o.app:
-            self.start_app(self.o.app_host, self.o.app_port)
+            self.start_app(self.o.app_host, self.o.app_port, self.o.app_external)
 
     def _readflow(self, path):
         path = os.path.expanduser(path)
@@ -158,8 +141,7 @@ class DumpMaster(flow.FlowMaster):
         return flows
 
     def add_event(self, e, level="info"):
-        needed = dict(error=0, info=1, debug=2).get(level, 1)
-        if self.o.verbosity >= needed:
+        if self.eventlog:
             print >> self.outfile, e
             self.outfile.flush()
 
@@ -173,18 +155,13 @@ class DumpMaster(flow.FlowMaster):
             return
 
         if f.response:
-            if self.o.flow_detail > 0:
-                if f.response.content == http.CONTENT_MISSING:
-                    sz = "(content missing)"
-                else:
-                    sz = utils.pretty_size(len(f.response.content))
+            sz = utils.pretty_size(len(f.response.content))
+            if self.o.verbosity > 0:
                 result = " << %s %s"%(str_response(f.response), sz)
-            if self.o.flow_detail > 1:
+            if self.o.verbosity > 1:
                 result = result + "\n\n" + self.indent(4, f.response.headers)
-            if self.o.flow_detail > 2:
-                if f.response.content == http.CONTENT_MISSING:
-                    cont = self.indent(4, "(content missing)")
-                elif utils.isBin(f.response.content):
+            if self.o.verbosity > 2:
+                if utils.isBin(f.response.content):
                     d = netlib.utils.hexdump(f.response.content)
                     d = "\n".join("%s\t%s %s"%i for i in d)
                     cont = self.indent(4, d)
@@ -196,45 +173,47 @@ class DumpMaster(flow.FlowMaster):
         elif f.error:
             result = " << %s"%f.error.msg
 
-        if self.o.flow_detail == 1:
-            print >> self.outfile, str_request(f, self.showhost)
+        if self.o.verbosity == 1:
+            print >> self.outfile, str_request(f.request, self.showhost)
             print >> self.outfile, result
-        elif self.o.flow_detail == 2:
-            print >> self.outfile, str_request(f, self.showhost)
+        elif self.o.verbosity == 2:
+            print >> self.outfile, str_request(f.request, self.showhost)
             print >> self.outfile, self.indent(4, f.request.headers)
             print >> self.outfile
             print >> self.outfile, result
             print >> self.outfile, "\n"
-        elif self.o.flow_detail >= 3:
-            print >> self.outfile, str_request(f, self.showhost)
+        elif self.o.verbosity >= 3:
+            print >> self.outfile, str_request(f.request, self.showhost)
             print >> self.outfile, self.indent(4, f.request.headers)
-            if f.request.content != http.CONTENT_MISSING and utils.isBin(f.request.content):
-                d = netlib.utils.hexdump(f.request.content)
-                d = "\n".join("%s\t%s %s"%i for i in d)
-                print >> self.outfile, self.indent(4, d)
+            if utils.isBin(f.request.content):
+                print >> self.outfile, self.indent(4, netlib.utils.hexdump(f.request.content))
             elif f.request.content:
                 print >> self.outfile, self.indent(4, f.request.content)
             print >> self.outfile
             print >> self.outfile, result
             print >> self.outfile, "\n"
-        if self.o.flow_detail:
+        if self.o.verbosity:
             self.outfile.flush()
 
-    def handle_request(self, f):
-        flow.FlowMaster.handle_request(self, f)
+    def handle_log(self, l):
+        self.add_event(l.msg)
+        l.reply()
+
+    def handle_request(self, r):
+        f = flow.FlowMaster.handle_request(self, r)
         if f:
-            f.reply()
+            r.reply()
         return f
 
-    def handle_response(self, f):
-        flow.FlowMaster.handle_response(self, f)
+    def handle_response(self, msg):
+        f = flow.FlowMaster.handle_response(self, msg)
         if f:
-            f.reply()
+            msg.reply()
             self._process_flow(f)
         return f
 
-    def handle_error(self, f):
-        flow.FlowMaster.handle_error(self, f)
+    def handle_error(self, msg):
+        f = flow.FlowMaster.handle_error(self, msg)
         if f:
             self._process_flow(f)
         return f
@@ -244,7 +223,8 @@ class DumpMaster(flow.FlowMaster):
 
     def run(self):  # pragma: no cover
         if self.o.rfile and not self.o.keepserving:
-            self.shutdown()
+            for script in self.scripts:
+                self.unload_script(script)
             return
         try:
             return flow.FlowMaster.run(self)

libmproxy/encoding.py

@@ -1,7 +1,6 @@
 """
     Utility functions for decoding response bodies.
 """
-from __future__ import absolute_import
 import cStringIO
 import gzip, zlib
 

libmproxy/filt.py

@@ -31,10 +31,8 @@
         ~c CODE     Response code.
         rex         Equivalent to ~u rex
 """
-from __future__ import absolute_import
 import re, sys
-from .contrib import pyparsing as pp
-from .protocol.http import decoded
+import contrib.pyparsing as pp
 
 
 class _Token:
@@ -166,14 +164,10 @@ class FBod(_Rex):
     code = "b"
     help = "Body"
     def __call__(self, f):
-        if f.request and f.request.content:
-            with decoded(f.request):
-                if re.search(self.expr, f.request.content):
-                    return True
-        if f.response and f.response.content:
-            with decoded(f.response):
-                if re.search(self.expr, f.response.content):
-                    return True
+        if f.request.content and re.search(self.expr, f.request.content):
+            return True
+        elif f.response and f.response.content and re.search(self.expr, f.response.content):
+            return True
         return False
 
 
@@ -181,20 +175,16 @@ class FBodRequest(_Rex):
     code = "bq"
     help = "Request body"
     def __call__(self, f):
-        if f.request and f.request.content:
-            with decoded(f.request):
-                if re.search(self.expr, f.request.content):
-                    return True
+        if f.request.content and re.search(self.expr, f.request.content):
+            return True
 
 
 class FBodResponse(_Rex):
     code = "bs"
     help = "Response body"
     def __call__(self, f):
-        if f.response and f.response.content:
-            with decoded(f.response):
-                if re.search(self.expr, f.response.content):
-                    return True
+        if f.response and f.response.content and re.search(self.expr, f.response.content):
+            return True
 
 
 class FMethod(_Rex):
@@ -222,7 +212,7 @@ class FUrl(_Rex):
         return klass(*toks)
 
     def __call__(self, f):
-        return re.search(self.expr, f.request.url)
+        return re.search(self.expr, f.request.get_url())
 
 
 class _Int(_Action):
@@ -343,34 +333,9 @@ bnf = _make()
 
 def parse(s):
     try:
-        filt = bnf.parseString(s, parseAll=True)[0]
-        filt.pattern = s
-        return filt
+        return bnf.parseString(s, parseAll=True)[0]
     except pp.ParseException:
         return None
     except ValueError:
         return None
 
-
-help = []
-for i in filt_unary:
-    help.append(
-        ("~%s"%i.code, i.help)
-    )
-for i in filt_rex:
-    help.append(
-        ("~%s regex"%i.code, i.help)
-    )
-for i in filt_int:
-    help.append(
-        ("~%s int"%i.code, i.help)
-    )
-help.sort()
-help.extend(
-    [
-        ("!", "unary not"),
-        ("&", "and"),
-        ("|", "or"),
-        ("(...)", "grouping"),
-    ]
-)

libmproxy/main.py

@@ -1,161 +0,0 @@
-from __future__ import print_function, absolute_import
-import os
-import signal
-import sys
-import netlib.version
-from . import version, cmdline
-from .proxy import process_proxy_options, ProxyServerError
-from .proxy.server import DummyServer, ProxyServer
-
-
-# This file is not included in coverage analysis or tests - anything that can be
-# tested should live elsewhere.
-
-def check_versions():
-    """
-    Having installed a wrong version of pyOpenSSL or netlib is unfortunately a
-    very common source of error. Check before every start that both versions are
-    somewhat okay.
-    """
-    # We don't introduce backward-incompatible changes in patch versions. Only
-    # consider major and minor version.
-    if netlib.version.IVERSION[:2] != version.IVERSION[:2]:
-        print(
-            "Warning: You are using mitmdump %s with netlib %s. "
-            "Most likely, that won't work - please upgrade!" % (
-                version.VERSION, netlib.version.VERSION
-            ),
-            file=sys.stderr
-        )
-    import OpenSSL
-    import inspect
-    v = tuple([int(x) for x in OpenSSL.__version__.split(".")][:2])
-    if v < (0, 14):
-        print(
-            "You are using an outdated version of pyOpenSSL:"
-            " mitmproxy requires pyOpenSSL 0.14 or greater.",
-            file=sys.stderr
-        )
-        # Some users apparently have multiple versions of pyOpenSSL installed.
-        # Report which one we got.
-        pyopenssl_path = os.path.dirname(inspect.getfile(OpenSSL))
-        print(
-            "Your pyOpenSSL %s installation is located at %s" % (
-                OpenSSL.__version__, pyopenssl_path
-            ),
-            file=sys.stderr
-        )
-        sys.exit(1)
-
-
-def assert_utf8_env():
-    spec = ""
-    for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
-        spec += os.environ.get(i, "").lower()
-    if "utf" not in spec:
-        print(
-            "Error: mitmproxy requires a UTF console environment.",
-            file=sys.stderr
-        )
-        print(
-            "Set your LANG enviroment variable to something like en_US.UTF-8",
-            file=sys.stderr
-        )
-        sys.exit(1)
-
-
-def get_server(dummy_server, options):
-    if dummy_server:
-        return DummyServer(options)
-    else:
-        try:
-            return ProxyServer(options)
-        except ProxyServerError, v:
-            print(str(v), file=sys.stderr)
-            sys.exit(1)
-
-
-def mitmproxy():  # pragma: nocover
-    from . import console
-
-    check_versions()
-    assert_utf8_env()
-
-    parser = cmdline.mitmproxy()
-    options = parser.parse_args()
-    if options.quiet:
-        options.verbose = 0
-
-    proxy_config = process_proxy_options(parser, options)
-    console_options = console.Options(**cmdline.get_common_options(options))
-    console_options.palette = options.palette
-    console_options.eventlog = options.eventlog
-    console_options.intercept = options.intercept
-
-    server = get_server(console_options.no_server, proxy_config)
-
-    m = console.ConsoleMaster(server, console_options)
-    try:
-        m.run()
-    except KeyboardInterrupt:
-        pass
-
-
-def mitmdump():  # pragma: nocover
-    from . import dump
-
-    check_versions()
-
-    parser = cmdline.mitmdump()
-    options = parser.parse_args()
-    if options.quiet:
-        options.verbose = 0
-        options.flow_detail = 0
-
-    proxy_config = process_proxy_options(parser, options)
-    dump_options = dump.Options(**cmdline.get_common_options(options))
-    dump_options.flow_detail = options.flow_detail
-    dump_options.keepserving = options.keepserving
-    dump_options.filtstr = " ".join(options.args) if options.args else None
-
-    server = get_server(dump_options.no_server, proxy_config)
-
-    try:
-        master = dump.DumpMaster(server, dump_options)
-
-        def cleankill(*args, **kwargs):
-            master.shutdown()
-
-        signal.signal(signal.SIGTERM, cleankill)
-        master.run()
-    except dump.DumpError as e:
-        print("mitmdump: %s" % e, file=sys.stderr)
-        sys.exit(1)
-    except KeyboardInterrupt:
-        pass
-
-
-def mitmweb():  # pragma: nocover
-    from . import web
-
-    check_versions()
-    parser = cmdline.mitmweb()
-
-    options = parser.parse_args()
-    if options.quiet:
-        options.verbose = 0
-
-    proxy_config = process_proxy_options(parser, options)
-    web_options = web.Options(**cmdline.get_common_options(options))
-    web_options.intercept = options.intercept
-    web_options.wdebug = options.wdebug
-    web_options.wiface = options.wiface
-    web_options.wport = options.wport
-
-    server = get_server(web_options.no_server, proxy_config)
-
-    m = web.WebMaster(server, web_options)
-    try:
-        m.run()
-    except KeyboardInterrupt:
-        pass