Version bump.

Add travis CI

.gitignore

@@ -10,5 +10,6 @@ MANIFEST
 mitmproxyc
 mitmdumpc
 .coverage
+.idea
 netlib
 libpathod

.travis.yml

@@ -0,0 +1,10 @@
+language: python
+python:
+  - "2.7"
+# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
+install: 
+  - "pip install -r requirements.txt --use-mirrors"
+  - "pip install --upgrade git+https://github.com/mitmproxy/netlib.git"
+  - "pip install --upgrade git+https://github.com/mitmproxy/pathod.git"
+# command to run tests, e.g. python setup.py test
+script: nosetests

CHANGELOG

@@ -1,5 +1,40 @@
+25 August 2013: mitmproxy 0.9.2:
 
-x January 2013: mitmproxy 0.9:
+    * Improvements to the mitmproxywrapper.py helper script for OSX.
+    
+    * Don't take minor version into account when checking for serialized file
+      compatibility.
+
+    * Fix a bug causing resource exhaustion under some circumstances for SSL
+      connections.
+
+    * Revamp the way we store interception certificates. We used to store these
+      on disk, they're now in-memory. This fixes a race condition related to
+      cert handling, and improves compatibility with Windows, where the rules
+      governing permitted file names are weird, resulting in errors for some
+      valid IDNA-encoded names.
+
+    * Display transfer rates for responses in the flow list.
+    
+    * Many other small bugfixes and improvements.
+    
+
+
+
+16 June 2013: mitmproxy 0.9.1:
+
+    * Use "correct" case for Content-Type headers added by mitmproxy.
+    
+    * Make UTF environment detection more robust. 
+
+    * Improved MIME-type detection for viewers.
+    
+    * Always read files in binary mode (Windows compatibility fix).
+    
+    * Some developer documentation.
+    
+
+15 May 2013: mitmproxy 0.9:
 
     * Upstream certs mode is now the default.
 

CONTRIBUTORS

@@ -1,26 +1,28 @@
-   759	Aldo Cortesi
+   801	Aldo Cortesi
     18	Henrik Nordstrom
     13	Thomas Roth
+    13	Maximilian Hils
     11	Stephen Altamirano
     10	András Veres-Szentkirályi
+     8	Jason A. Novak
      8	Rouli
      7	Alexis Hildebrandt
-     5	Maximilian Hils
-     4	Bryan Bishop
+     4	Marc Liyanage
      4	Valtteri Virtanen
+     4	Bryan Bishop
      3	Chris Neasbitt
-     2	Michael Frister
-     2	Heikki Hannikainen
+     3	Kyle Manna
      2	Jim Lloyd
-     2	Mark E. Haase
-     2	Rob Wills
+     2	Matthias Urlichs
+     2	Michael Frister
      2	alts
+     2	Rob Wills
      2	israel
-     1	Jakub Nawalaniec
+     2	Mark E. Haase
+     2	Heikki Hannikainen
+     1	Oleksandr Sheremet
      1	Paul
-     1	phil plante
      1	Rory McCann
-     1	Henrik Nordström
      1	Rune Halvorsen
      1	Sahn Lam
      1	Felix Wolfsteller
@@ -28,8 +30,13 @@
      1	Ulrich Petri
      1	Andy Smith
      1	Yuangxuan Wang
-     1	meeee
      1	capt8bit
+     1	meeee
+     1	Jakub Nawalaniec
+     1	Kit Randel
+     1	phil plante
+     1	Ivaylo Popov
      1	Mathieu Mitchell
-     1	Jason A. Novak
+     1	Henrik Nordström
+     1	Michael Bisbjerg
      1	Nicolas Esteves

MANIFEST.in

@@ -2,6 +2,7 @@ include LICENSE
 include CHANGELOG
 include CONTRIBUTORS
 include README.txt
+include setup.py
 exclude README.mkd
 recursive-include examples *
 recursive-include doc *

README.mkd

@@ -55,6 +55,8 @@ The following components are needed if you plan to hack on mitmproxy:
   framework and requires [pathod](http://pathod.org) and [flask](http://flask.pocoo.org/).
 * Rendering the documentation requires [countershape](http://github.com/cortesi/countershape).
 
+For convenience, all dependencies save countershape, can be installed from pypi to a virtualenv with 'pip install -r requirements.txt'.
+
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project maintains 100% test coverage.
 

doc-src/_layout.html

@@ -58,6 +58,10 @@
             <li class="nav-header">Scripting mitmproxy</li>
                 $!nav("scripting/inlinescripts.html", this, state)!$
                 $!nav("scripting/libmproxy.html", this, state)!$
+
+            <li class="nav-header">Hacking</li>
+                $!nav("dev/testing.html", this, state)!$
+
         </ul>
       </div>
     </div>

doc-src/_websitelayout.html

@@ -64,6 +64,9 @@
             <li class="nav-header">Scripting mitmproxy</li>
                 $!nav("scripting/inlinescripts.html", this, state)!$
                 $!nav("scripting/libmproxy.html", this, state)!$
+
+            <li class="nav-header">Hacking</li>
+                $!nav("dev/testing.html", this, state)!$
         </ul>
       </div>
     </div>

doc-src/dev/addingviews.html

@@ -0,0 +1,52 @@
+As discussed in [the Flow View section of the mitmproxy
+overview](@!urlTo("mitmproxy.html")!@), mitmproxy allows you to inspect and
+manipulate flows.  When inspecting a single flow, mitmproxy uses a number of
+heuristics to show a friendly view of various content types; if mitmproxy
+cannot show a friendly view, mitmproxy defaults to a __raw__ view.
+
+Each content type invokes a different flow viewer to parse the data and display
+the friendly view. Users can add custom content viewers by adding a view class
+to contentview.py, discussed below.
+
+## Adding a new View class to contentview.py
+
+The content viewers used by mitmproxy to present a friendly view of various
+content types are stored in contentview.py.  Reviewing this file shows a number
+of classes named ViewSomeDataType, each with the properties: __name__,
+__prompt__, and __content\_types__ and a function named __\_\_call\_\___.
+
+Adding a new content viewer to parse a data type is as simple as writing a new
+View class.  Your new content viewer View class should have the same properties
+as the other View classes: __name__, __prompt__, and __content\_types__ and a
+__\_\_call\_\___ function to parse the content of the request/response. 
+
+* The __name__ property should be a string describing the contents and new content viewer; 
+* The __prompt__ property should be a two item tuple:
+
+  - __1__: A string that will be used to display the new content viewer's type; and
+  - __2__: A one character string that will be the hotkey used to select the new content viewer from the Flow View screen; 
+
+* The __content\_types__ property should be a list of strings of HTTP Content\-Types that the new content viewer can parse.  
+  * Note that mitmproxy will use the content\_types to try and heuristically show a friendly view of content and that you can override the built-in views by populating content\_types with values for content\_types that are already parsed -- e.g. "image/png".
+
+After defining the __name__, __prompt__, and __content\_types__ properties of
+the class, you should write the __\_\_call\_\___ function, which will parse the
+request/response data and provide a friendly view of the data.  The
+__\_\_call\_\___ function should take the following arguments: __self__,
+__hdrs__, __content__, __limit__; __hdrs__ is a ODictCaseless object containing
+the headers of the request/response; __content__ is the content of the
+request/response, and __limit__ is an integer representing the amount of data
+to display in the view window.
+
+The __\_\_call\_\___ function returns two values: (1) a string describing the
+parsed data; and (2) the parsed data for friendly display.  The parsed data to
+be displayed should be a list of strings formatted for display.  You can use
+the __\_view\_text__ function in contentview.py to format text for display.
+Alternatively, you can display content as a series of key-value pairs; to do
+so, prepare a list of lists, where each list item is a two item list -- a key
+that describes the data, and then the data itself; after preparing the list of
+lists, use the __common.format\_keyvals__ function on it to prepare it as text
+for display.  
+
+If the new content viewer fails or throws an exception, mitmproxy will default
+to a __raw__ view.

doc-src/dev/index.py

@@ -0,0 +1,6 @@
+from countershape import Page
+
+pages = [
+     Page("testing.html", "Testing"),
+#    Page("addingviews.html", "Writing Content Views"),
+]

doc-src/dev/testing.html

@@ -0,0 +1,43 @@
+
+All the mitmproxy projects strive to maintain 100% code coverage. In general,
+patches and pull requests will be declined unless they're accompanied by a
+suitable extension to the test suite. 
+
+Our tests are written for the [nose](https://nose.readthedocs.org/en/latest/).
+At the point where you send your pull request, a command like this:
+
+<pre class="terminal">
+> nosetests --with-cov --cov-report term-missing ./test
+</pre>
+
+Should give output something like this:
+
+<pre class="terminal">
+> ---------- coverage: platform darwin, python 2.7.2-final-0 --
+> Name                   Stmts   Miss  Cover   Missing
+> ----------------------------------------------------
+> libmproxy/__init__         0      0   100%
+> libmproxy/app              4      0   100%
+> libmproxy/cmdline        100      0   100%
+> libmproxy/controller      69      0   100%
+> libmproxy/dump           150      0   100%
+> libmproxy/encoding        39      0   100%
+> libmproxy/filt           201      0   100%
+> libmproxy/flow           891      0   100%
+> libmproxy/proxy          427      0   100%
+> libmproxy/script          27      0   100%
+> libmproxy/utils          133      0   100%
+> libmproxy/version          4      0   100%
+> ----------------------------------------------------
+> TOTAL                   2045      0   100%
+> ----------------------------------------------------
+> Ran 251 tests in 11.864s
+</pre>
+
+
+There are exceptions to the coverage requirement - for instance, much of the
+console interface code can't sensibly be unit tested. These portions are
+excluded from coverage analysis either in the **.coveragerc** file, or using
+**#pragma no-cover** directives. To keep our coverage analysis relevant, we use
+these measures as sparingly as possible.
+

doc-src/howmitmproxy.html

@@ -246,7 +246,7 @@ mechanism has a different way of exposing this data, so this introduces the
 second component required for working transparent proxying: a host module that
 knows how to retrieve the original destination address from the router. In
 mitmproxy, this takes the form of a built-in set of
-[modules](https://github.com/cortesi/mitmproxy/tree/master/libmproxy/platform)
+[modules](https://github.com/mitmproxy/mitmproxy/tree/master/libmproxy/platform)
 that know how to talk to each platform's redirection mechanism.  Once we have
 this information, the process is fairly straight-forward.
 

doc-src/index.py

@@ -5,7 +5,7 @@ import countershape.template
 sys.path.insert(0, "..")
 from libmproxy import filt
 
-MITMPROXY_SRC = "~/git/public/mitmproxy"
+MITMPROXY_SRC = "~/mitmproxy/mitmproxy"
 
 if ns.options.website:
     ns.idxpath = "doc/index.html"

doc-src/install.html

@@ -25,7 +25,7 @@ pip install /path/to/source
 </pre>
 
 Note that if you're installing current git master, you will also have to
-install the current git master of [netlib](http://github.com/cortesi/netlib) by
+install the current git master of [netlib](http://github.com/mitmproxy/netlib) by
 hand.
 
 ## OSX

doc-src/scripting/index.py

@@ -2,5 +2,5 @@ from countershape import Page
 
 pages = [
     Page("inlinescripts.html", "Inline Scripts"),
-    Page("libmproxy.html", "libmproxy")
+    Page("libmproxy.html", "libmproxy"),
 ]

examples/README

@@ -7,3 +7,4 @@ proxapp                 How to embed a WSGI app in a mitmproxy server
 stub.py                 Script stub with a method definition for every event.
 stickycookies           An example of writing a custom proxy with libmproxy.
 upsidedownternet.py     Rewrites traffic to turn PNGs upside down.
+mitmproxywrapper.py     Bracket mitmproxy run with proxy enable/disable on OS X

examples/mitmproxywrapper.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+#
+# Helper tool to enable/disable OS X proxy and wrap mitmproxy
+#
+# Get usage information with:
+#
+# mitmproxywrapper.py -h
+#
+
+import subprocess
+import re
+import argparse
+import contextlib
+import os
+import sys
+
+class Wrapper(object):
+    
+    def __init__(self, port, extra_arguments=None):
+        self.port = port
+        self.extra_arguments = extra_arguments
+
+    def run_networksetup_command(self, *arguments):
+        return subprocess.check_output(['sudo', 'networksetup'] + list(arguments))
+
+    def proxy_state_for_service(self, service):
+        state = self.run_networksetup_command('-getwebproxy', service).splitlines()
+        return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state])
+
+    def enable_proxy_for_service(self, service):
+        print 'Enabling proxy on {}...'.format(service)
+        for subcommand in ['-setwebproxy', '-setsecurewebproxy']:
+            self.run_networksetup_command(subcommand, service, '127.0.0.1', str(self.port))
+
+    def disable_proxy_for_service(self, service):
+        print 'Disabling proxy on {}...'.format(service)
+        for subcommand in ['-setwebproxystate', '-setsecurewebproxystate']:
+            self.run_networksetup_command(subcommand, service, 'Off')
+
+    def interface_name_to_service_name_map(self):
+        order = self.run_networksetup_command('-listnetworkserviceorder')
+        mapping = re.findall(r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$', order, re.MULTILINE)
+        return dict([(b, a) for (a, b) in mapping])
+
+    def run_command_with_input(self, command, input):
+        popen = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        (stdout, stderr) = popen.communicate(input)
+        return stdout
+    
+    def primary_interace_name(self):
+        scutil_script = 'get State:/Network/Global/IPv4\nd.show\n'
+        stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+        interface, = re.findall(r'PrimaryInterface\s*:\s*(.+)', stdout)
+        return interface
+
+    def primary_service_name(self):
+        return self.interface_name_to_service_name_map()[self.primary_interace_name()]
+
+    def proxy_enabled_for_service(self, service):
+        return self.proxy_state_for_service(service)['Enabled'] == 'Yes'
+
+    def toggle_proxy(self):
+        new_state = not self.proxy_enabled_for_service(self.primary_service_name())
+        for service_name in self.connected_service_names():
+            if self.proxy_enabled_for_service(service_name) and not new_state:
+                self.disable_proxy_for_service(service_name)
+            elif not self.proxy_enabled_for_service(service_name) and new_state:
+                self.enable_proxy_for_service(service_name)
+
+    def connected_service_names(self):
+        scutil_script = 'list\n'
+        stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+        service_ids = re.findall(r'State:/Network/Service/(.+)/IPv4', stdout)
+
+        service_names = []
+        for service_id in service_ids:
+            scutil_script = 'show Setup:/Network/Service/{}\n'.format(service_id)
+            stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+            service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout)
+            service_names.append(service_name)
+
+        return service_names
+
+    def wrap_mitmproxy(self):
+        with self.wrap_proxy():
+            cmd = ['mitmproxy', '-p', str(self.port)]
+            if self.extra_arguments:
+                cmd.extend(self.extra_arguments)
+            cmd.extend(['--palette', 'light'])
+            subprocess.check_call(cmd)
+
+    def wrap_honeyproxy(self):
+        with self.wrap_proxy():
+            popen = subprocess.Popen('honeyproxy.sh')
+            try:
+                popen.wait()
+            except KeyboardInterrupt:
+                popen.terminate()
+
+    @contextlib.contextmanager
+    def wrap_proxy(self):
+        connected_service_names = self.connected_service_names()
+        for service_name in connected_service_names:
+            if not self.proxy_enabled_for_service(service_name):
+                self.enable_proxy_for_service(service_name)
+        
+        yield
+
+        for service_name in connected_service_names:
+            if self.proxy_enabled_for_service(service_name):
+                self.disable_proxy_for_service(service_name)
+
+    @classmethod
+    def ensure_superuser(cls):
+        if os.getuid() != 0:
+            print 'Relaunching with sudo...'
+            os.execv('/usr/bin/sudo', ['/usr/bin/sudo'] + sys.argv)
+
+    @classmethod
+    def main(cls):
+        parser = argparse.ArgumentParser(
+            description='Helper tool for OS X proxy configuration and mitmproxy.',
+            epilog='Any additional arguments will be passed on unchanged to mitmproxy.'
+        )
+        parser.add_argument('-t', '--toggle', action='store_true', help='just toggle the proxy configuration')
+#         parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
+        parser.add_argument('-p', '--port', type=int, help='override the default port of 8080', default=8080)
+        args, extra_arguments = parser.parse_known_args()
+
+        wrapper = cls(port=args.port, extra_arguments=extra_arguments)
+        
+        if args.toggle:
+            wrapper.toggle_proxy()
+#         elif args.honeyproxy:
+#             wrapper.wrap_honeyproxy()
+        else:
+            wrapper.wrap_mitmproxy()
+
+
+if __name__ == '__main__':
+    Wrapper.ensure_superuser()
+    Wrapper.main()
+

libmproxy/app.py

@@ -1,7 +1,14 @@
 import flask
 
 mapp = flask.Flask(__name__)
+mapp.debug = True
+
 
 @mapp.route("/")
-def hello():
-    return "mitmproxy"
+def index():
+    return flask.render_template("index.html", section="home")
+
+
+@mapp.route("/certs")
+def certs():
+    return flask.render_template("certs.html", section="certs")

libmproxy/cmdline.py

@@ -1,22 +1,10 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import proxy
 import re, filt
 import argparse
 
+APP_DOMAIN = "mitm"
+APP_IP = "1.1.1.1"
+
 class ParseException(Exception): pass
 class OptionException(Exception): pass
 
@@ -124,7 +112,7 @@ def get_common_options(options):
         except ParseException, e:
             raise OptionException(e.message)
         try:
-            v = open(path, "r").read()
+            v = open(path, "rb").read()
         except IOError, e:
             raise OptionException("Could not read replace file: %s"%path)
         reps.append((patt, rex, v))
@@ -139,6 +127,10 @@ def get_common_options(options):
         setheaders.append(p)
 
     return dict(
+        app = options.app,
+        app_ip = options.app_ip,
+        app_domain = options.app_domain,
+
         anticache = options.anticache,
         anticomp = options.anticomp,
         client_replay = options.client_replay,
@@ -267,6 +259,17 @@ def common_options(parser):
         action="store_true", dest="app", default=False,
         help="Enable the mitmproxy web app."
     )
+    group.add_argument(
+        "--appdomain",
+        action="store", dest="app_domain", default=APP_DOMAIN, metavar="domain",
+        help="Domain to serve the app from."
+    )
+    group.add_argument(
+        "--appip",
+        action="store", dest="app_ip", default=APP_IP, metavar="ip",
+        help="""IP to serve the app from. Useful for transparent mode, when a DNS
+        entry for the app domain is not present."""
+    )
 
     group = parser.add_argument_group("Client Replay")
     group.add_argument(

libmproxy/console/__init__.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2010  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import mailcap, mimetypes, tempfile, os, subprocess, glob, time, shlex, stat
 import os.path, sys, weakref
 import urwid
@@ -204,15 +189,13 @@ class StatusBar(common.WWrap):
             self.message("")
 
         fc = self.master.state.flow_count()
-        if self.master.currentflow:
-            idx = self.master.state.view.index(self.master.currentflow) + 1
-            t = [
-                ('heading', ("[%s/%s]"%(idx, fc)).ljust(9))
-            ]
+        if self.master.state.focus is None:
+            offset = 0
         else:
-            t = [
-                ('heading', ("[%s]"%fc).ljust(9))
-            ]
+            offset = min(self.master.state.focus + 1, fc)
+        t = [
+            ('heading', ("[%s/%s]"%(offset, fc)).ljust(9))
+        ]
 
         if self.master.server.bound:
             boundaddr = "[%s:%s]"%(self.master.server.address or "*", self.master.server.port)
@@ -263,7 +246,10 @@ class ConsoleState(flow.State):
         self.focus = None
         self.follow_focus = None
         self.default_body_view = contentview.get("Auto")
+
+        self.view_mode = common.VIEW_LIST
         self.view_flow_mode = common.VIEW_FLOW_REQUEST
+
         self.last_script = ""
         self.last_saveload = ""
         self.flowsettings = weakref.WeakKeyDictionary()
@@ -308,6 +294,9 @@ class ConsoleState(flow.State):
                 idx = 0
             self.focus = idx
 
+    def set_focus_flow(self, f):
+        self.set_focus(self.view.index(f))
+
     def get_from_pos(self, pos):
         if len(self.view) <= pos or pos < 0:
             return None, None
@@ -320,6 +309,10 @@ class ConsoleState(flow.State):
         return self.get_from_pos(pos-1)
 
     def delete_flow(self, f):
+        if f in self.view and self.view.index(f) <= self.focus:
+            self.focus -= 1
+        if self.focus < 0:
+            self.focus = None
         ret = flow.State.delete_flow(self, f)
         self.set_focus(self.focus)
         return ret
@@ -328,6 +321,9 @@ class ConsoleState(flow.State):
 
 class Options(object):
     attributes = [
+        "app",
+        "app_domain",
+        "app_ip",
         "anticache",
         "anticomp",
         "client_replay",
@@ -426,6 +422,9 @@ class ConsoleMaster(flow.FlowMaster):
                 print >> sys.stderr, "Script load error:", err
                 sys.exit(1)
 
+        if options.app:
+            self.start_app(options.app_domain, options.app_ip)
+
     def start_stream(self, path):
         path = os.path.expanduser(path)
         try:
@@ -479,7 +478,7 @@ class ConsoleMaster(flow.FlowMaster):
     def _readflow(self, path):
         path = os.path.expanduser(path)
         try:
-            f = file(path, "r")
+            f = file(path, "rb")
             flows = list(flow.FlowReader(f).stream())
         except (IOError, flow.FlowReadError), v:
             return True, v.strerror
@@ -519,13 +518,14 @@ class ConsoleMaster(flow.FlowMaster):
         except:
             self.statusbar.message("Can't start editor: %s" % " ".join(c))
         else:
-            data = open(name).read()
+            data = open(name,"rb").read()
         self.ui.start()
         os.unlink(name)
         return data
 
     def spawn_external_viewer(self, data, contenttype):
         if contenttype:
+            contenttype = contenttype.split(";")[0]
             ext = mimetypes.guess_extension(contenttype) or ""
         else:
             ext = ""
@@ -563,8 +563,6 @@ class ConsoleMaster(flow.FlowMaster):
         self.palette = palettes.palettes[name]
 
     def run(self):
-        self.currentflow = None
-
         self.ui = urwid.raw_display.Screen()
         self.ui.set_terminal_properties(256)
         self.ui.register_palette(self.palette)
@@ -598,13 +596,6 @@ class ConsoleMaster(flow.FlowMaster):
         sys.stderr.flush()
         self.shutdown()
 
-    def focus_current(self):
-        if self.currentflow:
-            try:
-                self.flow_list_walker.set_focus(self.state.view.index(self.currentflow))
-            except (IndexError, ValueError):
-                pass
-
     def make_view(self):
         self.view = urwid.Frame(
                         self.body,
@@ -639,8 +630,6 @@ class ConsoleMaster(flow.FlowMaster):
             self.ui.clear()
         if self.state.follow_focus:
             self.state.set_focus(self.state.flow_count())
-        else:
-            self.focus_current()
 
         if self.eventlog:
             self.body = flowlist.BodyPile(self)
@@ -648,7 +637,7 @@ class ConsoleMaster(flow.FlowMaster):
             self.body = flowlist.FlowListBox(self)
         self.statusbar = StatusBar(self, flowlist.footer)
         self.header = None
-        self.currentflow = None
+        self.state.view_mode = common.VIEW_LIST
 
         self.make_view()
         self.help_context = flowlist.help_context
@@ -657,7 +646,8 @@ class ConsoleMaster(flow.FlowMaster):
         self.body = flowview.FlowView(self, self.state, flow)
         self.header = flowview.FlowViewHeader(self, flow)
         self.statusbar = StatusBar(self, flowview.footer)
-        self.currentflow = flow
+        self.state.set_focus_flow(flow)
+        self.state.view_mode = common.VIEW_FLOW
 
         self.make_view()
         self.help_context = flowview.help_context
@@ -704,7 +694,6 @@ class ConsoleMaster(flow.FlowMaster):
         f.close()
         if self.flow_list_walker:
             self.sync_list_view()
-            self.focus_current()
         return reterr
 
     def path_prompt(self, prompt, text, callback, *args):
@@ -770,8 +759,7 @@ class ConsoleMaster(flow.FlowMaster):
     def change_default_display_mode(self, t):
         v = contentview.get_by_shortcut(t)
         self.state.default_body_view = v
-        if self.currentflow:
-            self.refresh_flow(self.currentflow)
+        self.refresh_focus()
 
     def set_reverse_proxy(self, txt):
         if not txt:
@@ -789,8 +777,8 @@ class ConsoleMaster(flow.FlowMaster):
         return size
 
     def pop_view(self):
-        if self.currentflow:
-            self.view_flow(self.currentflow)
+        if self.state.view_mode == common.VIEW_FLOW:
+            self.view_flow(self.state.view[self.state.focus])
         else:
             self.view_flowlist()
 
@@ -965,7 +953,7 @@ class ConsoleMaster(flow.FlowMaster):
         if a == "h":
             self.showhost = not self.showhost
             self.sync_list_view()
-            self.refresh_flow(self.currentflow)
+            self.refresh_focus()
         elif a == "k":
             self.killextra = not self.killextra
         elif a == "n":
@@ -996,6 +984,10 @@ class ConsoleMaster(flow.FlowMaster):
         self.state.delete_flow(f)
         self.sync_list_view()
 
+    def refresh_focus(self):
+        if self.state.view:
+            self.refresh_flow(self.state.view[self.state.focus])
+
     def refresh_flow(self, c):
         if hasattr(self.header, "refresh_flow"):
             self.header.refresh_flow(c)

libmproxy/console/common.py

@@ -1,23 +1,11 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import urwid
 import urwid.util
 from .. import utils, flow
 
 
+VIEW_LIST = 0
+VIEW_FLOW = 1
+
 
 VIEW_FLOW_REQUEST = 0
 VIEW_FLOW_RESPONSE = 1
@@ -156,6 +144,8 @@ def raw_format_flow(f, focus, extended, padding):
         if f["resp_ctype"]:
             resp.append(fcol(f["resp_ctype"], rc))
         resp.append(fcol(f["resp_clen"], rc))
+        resp.append(fcol(f["resp_rate"], rc))
+
     elif f["err_msg"]:
         resp.append(fcol(SYMBOL_RETURN, "error"))
         resp.append(
@@ -197,11 +187,17 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"
+
+        delta = f.response.timestamp_end - f.response.timestamp_start
+        size = len(f.response.content) + f.response.get_header_size()
+        rate = utils.pretty_size(size / delta)
+
         d.update(dict(
             resp_code = f.response.code,
             resp_is_replay = f.response.is_replay(),
             resp_acked = f.response.reply.acked,
-            resp_clen = contentdesc
+            resp_clen = contentdesc,
+            resp_rate = "{0}/s".format(rate),
         ))
         t = f.response.headers["content-type"]
         if t:

libmproxy/console/flowdetailview.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import urwid
 import common
 

libmproxy/console/flowlist.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import urwid
 import common
 
@@ -160,8 +145,7 @@ class ConnectionItem(common.WWrap):
             self.master.sync_list_view()
         elif key == "D":
             f = self.master.duplicate_flow(self.flow)
-            self.master.currentflow = f
-            self.master.focus_current()
+            self.master.view_flow(f)
         elif key == "r":
             self.flow.backup()
             r = self.master.replay_request(self.flow)

libmproxy/console/flowview.py

@@ -1,22 +1,7 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import os, sys
 import urwid
 import common, grideditor, contentview
-from .. import utils, flow
+from .. import utils, flow, controller
 
 def _mkhelp():
     text = []
@@ -341,7 +326,12 @@ class FlowView(common.WWrap):
             conn = self.flow.request
         else:
             if not self.flow.response:
-                self.flow.response = flow.Response(self.flow.request, 200, "OK", flow.ODictCaseless(), "", None)
+                self.flow.response = flow.Response(
+                    self.flow.request,
+                    self.flow.request.httpversion,
+                    200, "OK", flow.ODictCaseless(), "", None
+                )
+                self.flow.response.reply = controller.DummyReply()
             conn = self.flow.response
 
         self.flow.backup()
@@ -388,7 +378,7 @@ class FlowView(common.WWrap):
             new_flow, new_idx = self.state.get_next(idx)
         else:
             new_flow, new_idx = self.state.get_prev(idx)
-        if new_idx is None:
+        if new_flow is None:
             self.master.statusbar.message("No more flows!")
             return
         self.master.view_flow(new_flow)
@@ -473,7 +463,6 @@ class FlowView(common.WWrap):
         elif key == "D":
             f = self.master.duplicate_flow(self.flow)
             self.master.view_flow(f)
-            self.master.currentflow = f
             self.master.statusbar.message("Duplicated.")
         elif key == "e":
             if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
@@ -572,7 +561,8 @@ class FlowView(common.WWrap):
                 self.flow.backup()
                 e = conn.headers.get_first("content-encoding", "identity")
                 if e != "identity":
-                    conn.decode()
+                    if not conn.decode():
+                        self.master.statusbar.message("Could not decode - invalid data?")
                 else:
                     self.master.prompt_onekey(
                         "Select encoding: ",

libmproxy/console/grideditor.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import copy, re, os
 import urwid
 import common
@@ -294,7 +279,7 @@ class GridEditor(common.WWrap):
         if p:
             try:
                 p = os.path.expanduser(p)
-                d = file(p, "r").read()
+                d = file(p, "rb").read()
                 self.walker.set_current_value(d, unescaped)
                 self.walker._modified()
             except IOError, v:

libmproxy/console/help.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import urwid
 import common
 from .. import filt, version

libmproxy/console/palettes.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 palettes = {
 
 # Default palette for dark background

libmproxy/controller.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2010  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import Queue, threading
 
 should_exit = False

libmproxy/dump.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import sys, os
 import netlib.utils
 import flow, filt, utils
@@ -22,6 +7,9 @@ class DumpError(Exception): pass
 
 class Options(object):
     attributes = [
+        "app",
+        "app_domain",
+        "app_ip",
         "anticache",
         "anticomp",
         "client_replay",
@@ -138,10 +126,13 @@ class DumpMaster(flow.FlowMaster):
             except flow.FlowReadError, v:
                 self.add_event("Flow file corrupted. Stopped loading.")
 
+        if self.o.app:
+            self.start_app(self.o.app_domain, self.o.app_ip)
+
     def _readflow(self, path):
         path = os.path.expanduser(path)
         try:
-            f = file(path, "r")
+            f = file(path, "rb")
             flows = list(flow.FlowReader(f).stream())
         except (IOError, flow.FlowReadError), v:
             raise DumpError(v.strerror)

libmproxy/encoding.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 """
     Utility functions for decoding response bodies.
 """
@@ -54,7 +39,7 @@ def decode_gzip(content):
     gfile = gzip.GzipFile(fileobj=cStringIO.StringIO(content))
     try:
         return gfile.read()
-    except IOError:
+    except (IOError, EOFError):
         return None
 
 def encode_gzip(content):

libmproxy/filt.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2010  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 """
     The following operators are understood:
 

libmproxy/flow.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 """
     This module provides more sophisticated flow tracking. These match requests
     with their responses, and provide filtering and interception facilities.
@@ -221,15 +206,21 @@ class HTTPMsg(StateObject):
             Decodes content based on the current Content-Encoding header, then
             removes the header. If there is no Content-Encoding header, no
             action is taken.
+
+            Returns True if decoding succeeded, False otherwise.
         """
         ce = self.headers.get_first("content-encoding")
         if not self.content or ce not in encoding.ENCODINGS:
-            return
-        self.content = encoding.decode(
+            return False
+        data = encoding.decode(
             ce,
             self.content
         )
+        if data is None:
+            return False
+        self.content = data
         del self.headers["content-encoding"]
+        return True
 
     def encode(self, e):
         """
@@ -542,7 +533,7 @@ class Request(HTTPMsg):
             headers["host"] = [utils.hostport(self.scheme, self.host, self.port)]
         content = self.content
         if content:
-            headers["content-length"] = [str(len(content))]
+            headers["Content-Length"] = [str(len(content))]
         else:
             content = ""
         if self.close:
@@ -627,7 +618,7 @@ class Response(HTTPMsg):
         self.headers, self.content = headers, content
         self.cert = cert
         self.timestamp_start = timestamp_start or utils.timestamp()
-        self.timestamp_end = max(timestamp_end or utils.timestamp(), timestamp_start)
+        self.timestamp_end = timestamp_end or utils.timestamp()
         self.replay = False
 
     def _refresh_cookie(self, c, delta):
@@ -737,7 +728,7 @@ class Response(HTTPMsg):
             ['proxy-connection', 'transfer-encoding']
         )
         if self.content:
-            headers["content-length"] = [str(len(self.content))]
+            headers["Content-Length"] = [str(len(self.content))]
         proto = "HTTP/%s.%s %s %s"%(self.httpversion[0], self.httpversion[1], self.code, str(self.msg))
         data = (proto, str(headers))
         return FMT%data
@@ -790,6 +781,7 @@ class Response(HTTPMsg):
             cookies.append((cookie_name, (cookie_value, cookie_parameters)))
         return dict(cookies)
 
+
 class ClientDisconnect:
     """
         A client disconnection event.
@@ -1375,6 +1367,18 @@ class FlowMaster(controller.Master):
         self.stream = None
         app.mapp.config["PMASTER"] = self
 
+    def start_app(self, domain, ip):
+        self.server.apps.add(
+            app.mapp,
+            domain,
+            80
+        )
+        self.server.apps.add(
+            app.mapp,
+            ip,
+            80
+        )
+
     def add_event(self, e, level="info"):
         """
             level: info, error
@@ -1548,7 +1552,7 @@ class FlowMaster(controller.Master):
         if f.request:
             f.request._set_replay()
             if f.request.content:
-                f.request.headers["content-length"] = [str(len(f.request.content))]
+                f.request.headers["Content-Length"] = [str(len(f.request.content))]
             f.response = None
             f.error = None
             self.process_new_request(f)
@@ -1654,7 +1658,7 @@ class FlowReader:
         try:
             while 1:
                 data = tnetstring.load(self.fo)
-                if tuple(data["version"]) != version.IVERSION:
+                if tuple(data["version"][:2]) != version.IVERSION[:2]:
                     v = ".".join(str(i) for i in data["version"])
                     raise FlowReadError("Incompatible serialized data version: %s"%v)
                 off = self.fo.tell()
@@ -1677,4 +1681,3 @@ class FilteredFlowWriter:
         d = f._get_state()
         tnetstring.dump(d, self.fo)
 
-

libmproxy/proxy.py

@@ -1,27 +1,11 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import sys, os, string, socket, time
 import shutil, tempfile, threading
 import SocketServer
 from OpenSSL import SSL
 from netlib import odict, tcp, http, wsgi, certutils, http_status, http_auth
-import utils, flow, version, platform, controller, app
+import utils, flow, version, platform, controller
 
 
-APP_DOMAIN = "mitm"
-APP_IP = "1.1.1.1"
 KILL = 0
 
 
@@ -39,8 +23,7 @@ class Log:
 
 
 class ProxyConfig:
-    def __init__(self, app=False, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, certdir = None, authenticator=None):
-        self.app = app
+    def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, authenticator=None):
         self.certfile = certfile
         self.cacert = cacert
         self.clientcerts = clientcerts
@@ -49,7 +32,7 @@ class ProxyConfig:
         self.reverse_proxy = reverse_proxy
         self.transparent_proxy = transparent_proxy
         self.authenticator = authenticator
-        self.certstore = certutils.CertStore(certdir)
+        self.certstore = certutils.CertStore()
 
 
 class ServerConnection(tcp.TCPClient):
@@ -61,7 +44,6 @@ class ServerConnection(tcp.TCPClient):
         self.tcp_setup_timestamp = None
         self.ssl_setup_timestamp = None
 
-
     def connect(self):
         tcp.TCPClient.connect(self)
         self.tcp_setup_timestamp = time.time()
@@ -86,11 +68,12 @@ class ServerConnection(tcp.TCPClient):
         self.wfile.flush()
 
     def terminate(self):
-        try:
-            self.wfile.flush()
+        if self.connection:
+            try:
+                self.wfile.flush()
+            except tcp.NetLibDisconnect: # pragma: no cover
+                pass
             self.connection.close()
-        except IOError:
-            pass
 
 
 
@@ -105,11 +88,13 @@ class RequestReplayThread(threading.Thread):
             server = ServerConnection(self.config, r.scheme, r.host, r.port, r.host)
             server.connect()
             server.send(r)
+            tsstart = utils.timestamp()
             httpversion, code, msg, headers, content = http.read_response(
                 server.rfile, r.method, self.config.body_size_limit
             )
             response = flow.Response(
-                self.flow.request, httpversion, code, msg, headers, content, server.cert
+                self.flow.request, httpversion, code, msg, headers, content, server.cert, 
+                server.rfile.first_byte_timestamp
             )
             self.channel.ask(response)
         except (ProxyError, http.HttpError, tcp.NetLibError), v:
@@ -129,7 +114,7 @@ class HandleSNI:
                 self.handler.get_server_connection(self.client_conn, "https", self.host, self.port, sn)
                 new_context = SSL.Context(SSL.TLSv1_METHOD)
                 new_context.use_privatekey_file(self.key)
-                new_context.use_certificate_file(self.cert)
+                new_context.use_certificate(self.cert.x509)
                 connection.set_context(new_context)
                 self.handler.sni = sn.decode("utf8").encode("idna")
         # An unhandled exception in this method will core dump PyOpenSSL, so
@@ -179,6 +164,8 @@ class ProxyHandler(tcp.BaseHandler):
         return self.server_conn
 
     def del_server_connection(self):
+        if self.server_conn:
+            self.server_conn.terminate()
         self.server_conn = None
 
     def handle(self):
@@ -188,6 +175,7 @@ class ProxyHandler(tcp.BaseHandler):
         while self.handle_request(cc) and not cc.close:
             pass
         cc.close = True
+        self.del_server_connection()
 
         cd = flow.ClientDisconnect(cc)
         self.log(
@@ -213,7 +201,7 @@ class ProxyHandler(tcp.BaseHandler):
                     return
             else:
                 request_reply = self.channel.ask(request)
-                if request_reply == KILL:
+                if request_reply is None or request_reply == KILL:
                     return
                 elif isinstance(request_reply, flow.Response):
                     request = False
@@ -238,6 +226,7 @@ class ProxyHandler(tcp.BaseHandler):
                             request.ssl_setup_timestamp = sc.ssl_setup_timestamp
                         sc.rfile.reset_timestamps()
                         try:
+                            tsstart = utils.timestamp()
                             httpversion, code, msg, headers, content = http.read_response(
                                 sc.rfile,
                                 request.method,
@@ -256,7 +245,7 @@ class ProxyHandler(tcp.BaseHandler):
 
                     response = flow.Response(
                         request, httpversion, code, msg, headers, content, sc.cert,
-                        sc.rfile.first_byte_timestamp, utils.timestamp()
+                        sc.rfile.first_byte_timestamp
                     )
                     response_reply = self.channel.ask(response)
                     # Not replying to the server invalidates the server
@@ -277,7 +266,7 @@ class ProxyHandler(tcp.BaseHandler):
                     # disconnect.
                     if http.response_connection_close(response.httpversion, response.headers):
                         return
-        except (IOError, ProxyError, http.HttpError, tcp.NetLibDisconnect), e:
+        except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
             if hasattr(e, "code"):
                 cc.error = "%s: %s"%(e.code, e.msg)
             else:
@@ -309,7 +298,7 @@ class ProxyHandler(tcp.BaseHandler):
 
     def find_cert(self, cc, host, port, sni):
         if self.config.certfile:
-            return self.config.certfile
+            return certutils.SSLCert.from_pem(file(self.config.certfile, "r").read())
         else:
             sans = []
             if not self.config.no_upstream_cert:
@@ -509,17 +498,6 @@ class ProxyServer(tcp.TCPServer):
             raise ProxyServerError('Error starting proxy server: ' + v.strerror)
         self.channel = None
         self.apps = AppRegistry()
-        if config.app:
-            self.apps.add(
-                app.mapp,
-                APP_DOMAIN,
-                80
-            )
-            self.apps.add(
-                app.mapp,
-                APP_IP,
-                80
-            )
 
     def start_slave(self, klass, channel):
         slave = klass(channel, self)
@@ -533,9 +511,6 @@ class ProxyServer(tcp.TCPServer):
         h.handle()
         h.finish()
 
-    def handle_shutdown(self):
-        self.config.certstore.cleanup()
-
 
 class AppRegistry:
     def __init__(self):
@@ -584,11 +559,6 @@ def certificate_option_group(parser):
         type = str, dest = "clientcerts", default=None,
         help = "Client certificate directory."
     )
-    group.add_argument(
-        "--dummy-certs", action="store",
-        type = str, dest = "certdir", default=None,
-        help = "Generated dummy certs directory."
-    )
 
 
 TRANSPARENT_SSL_PORTS = [443, 8443]
@@ -629,11 +599,6 @@ def process_proxy_options(parser, options):
         if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
             return parser.error("Client certificate directory does not exist or is not a directory: %s"%options.clientcerts)
 
-    if options.certdir:
-        options.certdir = os.path.expanduser(options.certdir)
-        if not os.path.exists(options.certdir) or not os.path.isdir(options.certdir):
-            return parser.error("Dummy cert directory does not exist or is not a directory: %s"%options.certdir)
-
     if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd):
         if options.auth_singleuser:
             if len(options.auth_singleuser.split(':')) != 2:
@@ -652,7 +617,6 @@ def process_proxy_options(parser, options):
         authenticator = http_auth.NullProxyAuth(None)
 
     return ProxyConfig(
-        app = options.app,
         certfile = options.cert,
         cacert = cacert,
         clientcerts = options.clientcerts,
@@ -660,6 +624,5 @@ def process_proxy_options(parser, options):
         no_upstream_cert = options.no_upstream_cert,
         reverse_proxy = rp,
         transparent_proxy = trans,
-        certdir = options.certdir,
         authenticator = authenticator
     )

libmproxy/script.py

@@ -1,18 +1,3 @@
-# Copyright (C) 2012  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import os, traceback
 
 class ScriptError(Exception):

libmproxy/static/jquery.localscroll-min.js

@@ -0,0 +1,9 @@
+/**
+ * jQuery.LocalScroll - Animated scrolling navigation, using anchors.
+ * Copyright (c) 2007-2009 Ariel Flesler - aflesler(at)gmail(dot)com | http://flesler.blogspot.com
+ * Dual licensed under MIT and GPL.
+ * Date: 3/11/2009
+ * @author Ariel Flesler
+ * @version 1.2.7
+ **/
+;(function($){var l=location.href.replace(/#.*/,'');var g=$.localScroll=function(a){$('body').localScroll(a)};g.defaults={duration:1e3,axis:'y',event:'click',stop:true,target:window,reset:true};g.hash=function(a){if(location.hash){a=$.extend({},g.defaults,a);a.hash=false;if(a.reset){var e=a.duration;delete a.duration;$(a.target).scrollTo(0,a);a.duration=e}i(0,location,a)}};$.fn.localScroll=function(b){b=$.extend({},g.defaults,b);return b.lazy?this.bind(b.event,function(a){var e=$([a.target,a.target.parentNode]).filter(d)[0];if(e)i(a,e,b)}):this.find('a,area').filter(d).bind(b.event,function(a){i(a,this,b)}).end().end();function d(){return!!this.href&&!!this.hash&&this.href.replace(this.hash,'')==l&&(!b.filter||$(this).is(b.filter))}};function i(a,e,b){var d=e.hash.slice(1),f=document.getElementById(d)||document.getElementsByName(d)[0];if(!f)return;if(a)a.preventDefault();var h=$(b.target);if(b.lock&&h.is(':animated')||b.onBefore&&b.onBefore.call(b,a,f,h)===false)return;if(b.stop)h.stop(true);if(b.hash){var j=f.id==d?'id':'name',k=$('<a> </a>').attr(j,d).css({position:'absolute',top:$(window).scrollTop(),left:$(window).scrollLeft()});f[j]='';$('body').prepend(k);location=e.hash;k.remove();f[j]=d}h.scrollTo(f,b).trigger('notify.serialScroll',[f])}})(jQuery);

libmproxy/static/jquery.scrollTo-min.js

@@ -0,0 +1,11 @@
+/**
+ * jQuery.ScrollTo - Easy element scrolling using jQuery.
+ * Copyright (c) 2007-2009 Ariel Flesler - aflesler(at)gmail(dot)com | http://flesler.blogspot.com
+ * Dual licensed under MIT and GPL.
+ * Date: 3/9/2009
+ * @author Ariel Flesler
+ * @version 1.4.1
+ *
+ * http://flesler.blogspot.com/2007/10/jqueryscrollto.html
+ */
+;(function($){var m=$.scrollTo=function(b,h,f){$(window).scrollTo(b,h,f)};m.defaults={axis:'xy',duration:parseFloat($.fn.jquery)>=1.3?0:1};m.window=function(b){return $(window).scrollable()};$.fn.scrollable=function(){return this.map(function(){var b=this,h=!b.nodeName||$.inArray(b.nodeName.toLowerCase(),['iframe','#document','html','body'])!=-1;if(!h)return b;var f=(b.contentWindow||b).document||b.ownerDocument||b;return $.browser.safari||f.compatMode=='BackCompat'?f.body:f.documentElement})};$.fn.scrollTo=function(l,j,a){if(typeof j=='object'){a=j;j=0}if(typeof a=='function')a={onAfter:a};if(l=='max')l=9e9;a=$.extend({},m.defaults,a);j=j||a.speed||a.duration;a.queue=a.queue&&a.axis.length>1;if(a.queue)j/=2;a.offset=n(a.offset);a.over=n(a.over);return this.scrollable().each(function(){var k=this,o=$(k),d=l,p,g={},q=o.is('html,body');switch(typeof d){case'number':case'string':if(/^([+-]=)?\d+(\.\d+)?(px)?$/.test(d)){d=n(d);break}d=$(d,this);case'object':if(d.is||d.style)p=(d=$(d)).offset()}$.each(a.axis.split(''),function(b,h){var f=h=='x'?'Left':'Top',i=f.toLowerCase(),c='scroll'+f,r=k[c],s=h=='x'?'Width':'Height';if(p){g[c]=p[i]+(q?0:r-o.offset()[i]);if(a.margin){g[c]-=parseInt(d.css('margin'+f))||0;g[c]-=parseInt(d.css('border'+f+'Width'))||0}g[c]+=a.offset[i]||0;if(a.over[i])g[c]+=d[s.toLowerCase()]()*a.over[i]}else g[c]=d[i];if(/^\d+$/.test(g[c]))g[c]=g[c]<=0?0:Math.min(g[c],u(s));if(!b&&a.queue){if(r!=g[c])t(a.onAfterFirst);delete g[c]}});t(a.onAfter);function t(b){o.animate(g,j,a.easing,b&&function(){b.call(this,l,a)})};function u(b){var h='scroll'+b;if(!q)return k[h];var f='client'+b,i=k.ownerDocument.documentElement,c=k.ownerDocument.body;return Math.max(i[h],c[h])-Math.min(i[f],c[f])}}).end()};function n(b){return typeof b=='object'?b:{top:b,left:b}}})(jQuery);

libmproxy/static/mitmproxy.css

@@ -0,0 +1,43 @@
+
+
+.fronttable {
+}
+
+.bigtitle {
+    font-weight: bold;
+    font-size: 50px;
+    line-height: 55px;
+    text-align: center;
+    display: table;
+    height: 300px;
+}
+
+.bigtitle>div {
+    display: table-cell;
+    vertical-align: middle;
+}
+
+section {
+    margin-top: 50px;
+}
+
+.example {
+    margin-top: 10px;
+    margin-bottom: 10px;
+}
+
+.innerlink {
+    text-decoration: none;
+    border-bottom:1px dotted;
+    margin-bottom: 15px;
+}
+
+.masthead {
+    padding: 50px 0 60px;
+    text-align: center;
+
+}
+
+.header {
+    font-size: 1.5em;
+}

libmproxy/templates/certs.html

@@ -0,0 +1,14 @@
+{% extends "frame.html" %}
+{% block body %}  
+
+
+<div class="masthead">
+    <div class="container">
+        <h1>mitmproxy: pathological HTTP</h1>
+
+        <p>Here are some certs.</p>
+    </div>
+</div>
+
+
+{% endblock %}  

libmproxy/templates/frame.html

@@ -0,0 +1,9 @@
+{% extends "layout.html" %}
+{% block content %}  
+<div class="row">
+    <div class="span12">
+        {% block body %}  
+        {% endblock %}  
+    </div>
+</div>
+{% endblock %}  

libmproxy/templates/index.html

@@ -0,0 +1,14 @@
+{% extends "frame.html" %}
+{% block body %}  
+
+
+<div class="masthead">
+    <div class="container">
+        <h1>mitmproxy: pathological HTTP</h1>
+
+        <p>This is an index page.</p>
+    </div>
+</div>
+
+
+{% endblock %}  

libmproxy/templates/layout.html

@@ -0,0 +1,68 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>mitmproxy</title>
+    <link href="/static/bootstrap.min.css" rel="stylesheet">
+    <link href="/static/mitmproxy.css" rel="stylesheet">
+    <link href="/static/syntax.css" rel="stylesheet">
+    <script src="/static/jquery-1.7.2.min.js"></script>
+    <script src="/static/jquery.scrollTo-min.js"></script>
+    <script src="/static/jquery.localscroll-min.js"></script>
+    <script src="/static/bootstrap.min.js"></script>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="">
+    <meta name="author" content="">
+    <style type="text/css">
+      body {
+        padding-top: 60px;
+        padding-bottom: 40px;
+      }
+    </style>
+    <!-- Le HTML5 shim, for IE6-8 support of HTML5 elements -->
+    <!--[if lt IE 9]>
+      <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+    <![endif]-->
+  </head>
+
+  <body>
+
+    <div class="navbar navbar-fixed-top">
+      <div class="navbar-inner">
+        <div class="container">
+          <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </a>
+          <a class="brand" href="/">mitmproxy</a>
+          <div class="nav-collapse">
+            <ul class="nav">
+              <li {% if section== "home" %} class="active" {% endif %}><a href="/">home</a></li>
+              <li {% if section== "certs" %} class="active" {% endif %}><a href="/certs">certs</a></li>
+            </ul>
+          </div>
+        </div>
+      </div>
+    </div>
+
+    <div class="container">
+        {% block content %}  
+        {% endblock %}  
+      <hr>
+      <footer>
+        <span><a href="http://mitmproxy.org">mitmproxy</a></span>
+      </footer>
+    </div>
+  </body>
+  <script>
+    $(function(){
+        $.localScroll(
+            {
+                duration: 300,
+                offset: {top: -45}
+            }
+        );
+    });
+  </script>
+</html>

libmproxy/utils.py

@@ -1,17 +1,3 @@
-# Copyright (C) 2010  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import os, datetime, urlparse, string, urllib, re
 import time, functools, cgi
 import json
@@ -88,7 +74,7 @@ def pretty_size(size):
     suffixes = [
         ("B",   2**10),
         ("kB",   2**20),
-        ("M",   2**30),
+        ("MB",   2**30),
     ]
     for suf, lim in suffixes:
         if size >= lim:

libmproxy/version.py

@@ -1,4 +1,4 @@
-IVERSION = (0, 9)
+IVERSION = (0, 9, 2)
 VERSION = ".".join(str(i) for i in IVERSION)
 NAME = "mitmproxy"
 NAMEVERSION = NAME + " " + VERSION

mitmdump

@@ -1,20 +1,4 @@
 #!/usr/bin/env python
-
-# Copyright (C) 2010  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import sys, signal
 # The unneccesary console import here is to work around a bug in pyinstaller
 from libmproxy import proxy, dump, cmdline, version, console
@@ -70,5 +54,3 @@ if __name__ == '__main__':
     except KeyboardInterrupt:
         pass
 
-
-

mitmproxy

@@ -1,20 +1,4 @@
 #!/usr/bin/env python
-
-# Copyright (C) 2010  Aldo Cortesi
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import sys, argparse, os
 from libmproxy import proxy, console, cmdline, version
 from libmproxy.console import palettes
@@ -62,12 +46,14 @@ if __name__ == '__main__':
     opts.debug = options.debug
     opts.palette = options.palette
 
-    if "utf" not in os.environ.get("LANG", "").lower():
+    spec = ""
+    for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
+        spec += os.environ.get(i, "").lower()
+    if "utf" not in spec:
         print >> sys.stderr, "Error: mitmproxy requires a UTF console environment."
         print >> sys.stderr, "Set your LANG enviroment variable to something like en_US.UTF-8"
         sys.exit(1)
 
-
     m = console.ConsoleMaster(server, opts)
     try:
         m.run()

release/osx-binaries

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# Quick and dangerous script for building OSX binaries.
+
+# A few quirks to note, which should be re-checked every release:
+# - We require the latest development version of PyInstaller.
+ 
+# - PyInstaller has trouble detecting the zope.interfaces package. This is
+# required by Twisted, which for mysterious reasons is required by Urwid. The
+# answer is to touch the __init__.py file in the zope directory. On my system:
+#   touch /Library/Python/2.7/site-packages/zope/__init__.py
+
+# To run, change into the pyinstaller directory, and then run this script.
+
+DST=/tmp/osx-mitmproxy
+MITMPROXY=~/mitmproxy/mitmproxy
+PYINST_CMD="./pyinstaller.py -F --clean"
+
+rm -rf $DST
+mkdir -p $DST
+rm -rf mitmproxy
+rm -rf mitmdump
+
+$PYINST_CMD $MITMPROXY/mitmproxy
+cp mitmproxy/dist/mitmproxy $DST
+
+$PYINST_CMD $MITMPROXY/mitmdump
+cp mitmdump/dist/mitmdump $DST
+
+cshape $MITMPROXY/doc-src $DST/doc

release/pyinstaller-mitmdump.spec

@@ -1,20 +0,0 @@
-# -*- mode: python -*-
-
-# Copy into the pyinstaller directory
-# ./pyinstaller.py --clean -F ./pyinstaller-mitmdump.spec 
-
-a = Analysis(['/Users/aldo/git/public/mitmproxy/mitmdump'],
-             hiddenimports=["pyamf"],
-             hookspath=None,
-             runtime_hooks=None)
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='mitmdump',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )

release/pyinstaller-mitmproxy.spec

@@ -1,16 +0,0 @@
-# -*- mode: python -*-
-a = Analysis(['/Users/aldo/git/public/mitmproxy/mitmproxy'],
-             hiddenimports=["pyamf"],
-             hookspath=None,
-             runtime_hooks=None)
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='mitmproxy',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )

setup.py

@@ -79,7 +79,7 @@ setup(
         package_data = package_data,
         scripts = ["mitmproxy", "mitmdump"],
         classifiers = [
-            "License :: OSI Approved :: GNU General Public License (GPL)",
+            "License :: OSI Approved :: MIT License",
             "Development Status :: 5 - Production/Stable",
             "Environment :: Console",
             "Environment :: Console :: Curses",
@@ -96,7 +96,7 @@ setup(
             "netlib>=%s"%version.VERSION,
             "urwid>=1.1",
             "pyasn1>0.1.2",
-            "pyopenssl>=0.12",
+            "pyopenssl>=0.13",
             "PIL",
             "lxml",
             "flask"

test/test_console_contentview.py

@@ -114,16 +114,16 @@ class TestContentView:
     def test_view_image(self):
         v = cv.ViewImage()
         p = tutils.test_data.path("data/image.png")
-        assert v([], file(p).read(), sys.maxint)
+        assert v([], file(p,"rb").read(), sys.maxint)
 
         p = tutils.test_data.path("data/image.gif")
-        assert v([], file(p).read(), sys.maxint)
+        assert v([], file(p,"rb").read(), sys.maxint)
 
         p = tutils.test_data.path("data/image-err1.jpg")
-        assert v([], file(p).read(), sys.maxint)
+        assert v([], file(p,"rb").read(), sys.maxint)
 
         p = tutils.test_data.path("data/image.ico")
-        assert v([], file(p).read(), sys.maxint)
+        assert v([], file(p,"rb").read(), sys.maxint)
 
         assert not v([], "flibble", sys.maxint)
 
@@ -224,22 +224,22 @@ if pyamf:
         v = cv.ViewAMF()
 
         p = tutils.test_data.path("data/amf01")
-        assert v([], file(p).read(), sys.maxint)
+        assert v([], file(p,"rb").read(), sys.maxint)
 
         p = tutils.test_data.path("data/amf02")
-        assert v([], file(p).read(), sys.maxint)
+        assert v([], file(p,"rb").read(), sys.maxint)
 
     def test_view_amf_response():
         v = cv.ViewAMF()
         p = tutils.test_data.path("data/amf03")
-        assert v([], file(p).read(), sys.maxint)
+        assert v([], file(p,"rb").read(), sys.maxint)
 
 if cv.ViewProtobuf.is_available():
     def test_view_protobuf_request():
         v = cv.ViewProtobuf()
 
         p = tutils.test_data.path("data/protobuf01")
-        content_type, output = v([], file(p).read(), sys.maxint)
+        content_type, output = v([], file(p,"rb").read(), sys.maxint)
         assert content_type == "Protobuf"
         assert output[0].text == '1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"'
 

test/test_dump.py

@@ -46,7 +46,7 @@ class TestDumpMaster:
         return cs.getvalue()
 
     def _flowfile(self, path):
-        f = open(path, "w")
+        f = open(path, "wb")
         fw = flow.FlowWriter(f)
         t = tutils.tflow_full()
         t.response = tutils.tresp(t.request)
@@ -106,6 +106,12 @@ class TestDumpMaster:
     def test_filter(self):
         assert not "GET" in self._dummy_cycle(1, "~u foo", "", verbosity=1)
 
+    def test_app(self):
+        o = dump.Options(app=True)
+        s = mock.MagicMock()
+        m = dump.DumpMaster(s, o, None)
+        assert s.apps.add.call_count == 2
+
     def test_replacements(self):
         o = dump.Options(replacements=[(".*", "content", "foo")])
         m = dump.DumpMaster(None, o, None)
@@ -128,7 +134,7 @@ class TestDumpMaster:
         with tutils.tmpdir() as d:
             p = os.path.join(d, "a")
             self._dummy_cycle(1, None, "", wfile=p, verbosity=0)
-            assert len(list(flow.FlowReader(open(p)).stream())) == 1
+            assert len(list(flow.FlowReader(open(p,"rb")).stream())) == 1
 
     def test_write_err(self):
         tutils.raises(

test/test_flow.py

@@ -591,7 +591,7 @@ class TestFlowMaster:
         s = flow.State()
         fm = flow.FlowMaster(None, s)
         f = tutils.tflow_full()
-        fm.load_flow(f)
+        f = fm.load_flow(f)
         assert s.flow_count() == 1
         f2 = fm.duplicate_flow(f)
         assert f2.response
@@ -733,7 +733,7 @@ class TestFlowMaster:
         with tutils.tmpdir() as tdir:
             p = os.path.join(tdir, "foo")
             def r():
-                r = flow.FlowReader(open(p))
+                r = flow.FlowReader(open(p,"rb"))
                 return list(r.stream())
 
             s = flow.State()

test/test_platform_pf.py

@@ -5,7 +5,7 @@ from libmproxy.platform import pf
 class TestLookup:
     def test_simple(self):
         p = tutils.test_data.path("data/pf01")
-        d = open(p).read()
+        d = open(p,"rb").read()
         assert pf.lookup("192.168.1.111", 40000, d) == ("5.5.5.5", 80)
         assert not pf.lookup("192.168.1.112", 40000, d)
         assert not pf.lookup("192.168.1.111", 40001, d)

test/test_proxy.py

@@ -56,7 +56,7 @@ class TestServerConnection:
         sc = proxy.ServerConnection(proxy.ProxyConfig(), "http", self.d.IFACE, self.d.port, "host.com")
         sc.connect()
         sc.connection = mock.Mock()
-        sc.connection.close = mock.Mock(side_effect=IOError)
+        sc.connection.flush = mock.Mock(side_effect=tcp.NetLibDisconnect)
         sc.terminate()
 
 
@@ -116,9 +116,6 @@ class TestProcessProxyOptions:
             self.assert_noerr("--client-certs", confdir)
             self.assert_err("directory does not exist", "--client-certs", "nonexistent")
 
-            self.assert_noerr("--dummy-certs", confdir)
-            self.assert_err("directory does not exist", "--dummy-certs", "nonexistent")
-
     def test_auth(self):
         p = self.assert_noerr("--nonanonymous")
         assert p.authenticator

test/test_utils.py

@@ -38,7 +38,7 @@ def test_pretty_size():
     assert utils.pretty_size(100) == "100B"
     assert utils.pretty_size(1024) == "1kB"
     assert utils.pretty_size(1024 + (1024/2.0)) == "1.5kB"
-    assert utils.pretty_size(1024*1024) == "1M"
+    assert utils.pretty_size(1024*1024) == "1MB"
 
 
 def test_pkg_data():

test/tservers.py

@@ -4,6 +4,9 @@ import libpathod.test, libpathod.pathoc
 from libmproxy import proxy, flow, controller
 import tutils
 
+APP_DOMAIN = "mitm"
+APP_IP = "1.1.1.1"
+
 testapp = flask.Flask(__name__)
 
 @testapp.route("/")
@@ -28,6 +31,7 @@ class TestMaster(flow.FlowMaster):
         flow.FlowMaster.__init__(self, s, state)
         self.testq = testq
         self.clear_log()
+        self.start_app(APP_DOMAIN, APP_IP)
 
     def handle_request(self, m):
         flow.FlowMaster.handle_request(self, m)
@@ -85,7 +89,6 @@ class ProxTestBase:
             no_upstream_cert = cls.no_upstream_cert,
             cacert = tutils.test_data.path("data/serverkey.pem"),
             authenticator = cls.authenticator,
-            app = True,
             **pconf
         )
         tmaster = cls.masterclass(cls.tqueue, config)
@@ -162,12 +165,12 @@ class HTTPProxTest(ProxTestBase):
         if self.ssl:
             p = libpathod.pathoc.Pathoc("127.0.0.1", self.proxy.port, True)
             print "PRE"
-            p.connect((proxy.APP_IP, 80))
+            p.connect((APP_IP, 80))
             print "POST"
             return p.request("get:'/%s'"%page)
         else:
             p = self.pathoc()
-            return p.request("get:'http://%s/%s'"%(proxy.APP_DOMAIN, page))
+            return p.request("get:'http://%s/%s'"%(APP_DOMAIN, page))
 
 
 class TResolver:

test/tutils.py

@@ -20,7 +20,7 @@ def tresp(req=None):
         req = treq()
     headers = flow.ODictCaseless()
     headers["header_response"] = ["svalue"]
-    cert = certutils.SSLCert.from_der(file(test_data.path("data/dercert")).read())
+    cert = certutils.SSLCert.from_der(file(test_data.path("data/dercert"),"rb").read())
     resp = flow.Response(req, (1, 1), 200, "message", headers, "content_response", cert)
     resp.reply = controller.DummyReply()
     return resp