Prep for 0.5 release

- Update CHANGELOG and CONTRIBUTORS
- Bump version
- Include Apple Gamecenter highscore setting tutorial in docs

CHANGELOG

@@ -1,3 +1,43 @@
+27 June 2011: mitmproxy 0.5:
+
+    * An -n option to start the tools without binding to a proxy port.
+
+    * Allow scripts, hooks, sticky cookies etc. to run on flows loaded from
+      save files.
+
+    * Regularize command-line options for mitmproxy and mitmdump.
+
+    * Add an "SSL exception" to mitmproxy's license to remove possible
+      distribution issues.
+
+    * Add a --cert-wait-time option to make mitmproxy pause after a new SSL
+      certificate is generated. This can pave over small discrepancies in
+      system time between the client and server.
+
+    * Handle viewing big request and response bodies more elegantly. Only
+      render the first 100k of large documents, and try to avoid running the
+      XML indenter on non-XML data.
+
+    * BUGFIX: Make the "revert" keyboard shortcut in mitmproxy work after a
+      flow has been replayed.
+
+    * BUGFIX: Repair a problem that sometimes caused SSL connections to consume
+      100% of CPU.
+
+
+30 March 2011: mitmproxy 0.4 
+
+    * Full serialization of HTTP conversations
+    
+    * Client and server replay
+    
+    * On-the-fly generation of dummy SSL certificates
+   
+    * mitmdump has "grown up" into a powerful tcpdump-like tool for HTTP/S
+
+    * Dozens of improvements to the mitmproxy console interface
+
+    * Python scripting hooks for programmatic modification of traffic
 
 
 1 March 2010: mitmproxy 0.2 
@@ -5,7 +45,7 @@
     * Big speed and responsiveness improvements, thanks to Thomas Roth
     
     * Support urwid 0.9.9
-    *
+    
     * Terminal beeping based on filter expressions
     
     * Filter expressions for terminal beeps, limits, interceptions and sticky

CONTRIBUTORS

@@ -1,4 +1,5 @@
-    179 Aldo Cortesi
+    203 Aldo Cortesi
      18 Henrik Nordstrom
      13 Thomas Roth
+      1 Yuangxuan Wang
       1 Henrik Nordström

LICENSE

@@ -1,3 +1,19 @@
+mitmproxy is licensed under the terms of the GNU General Public
+License version 3, with the following addition:
+
+  In addition, as a special exception, the copyright holders give
+permission to link the code of this program or portions of this
+program with the OpenSSL project's "OpenSSL" library (or with modified
+versions of it that use the same license as the "OpenSSL" library),
+and distribute linked combinations including the two. 
+
+  You must obey the GNU General Public License in all respects for all
+of the code used other than "OpenSSL".  If you modify file(s) provided
+under this license, you may extend this exception to your version of
+the file, but you are not obligated to do so. If you do not wish to do
+so, delete this exception statement from your version.
+
+
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 

doc-src/02-docstyle.css

@@ -67,6 +67,13 @@ pre {
     background-color: #dddddd;
 }
 
+.terminal {
+    color: #ffffff;
+    background: #000000;
+
+
+}
+
 .docindex, .docindex ul {
     margin-top: 0.1em;
     margin-bottom: 0;

doc-src/index.html

@@ -18,10 +18,8 @@
      <li><a href="@!urlTo("library.html")!@">libmproxy</a></li>
      <li>Tutorials</li>
         <ul>
-            <li> Client replay: a 30 second example [coming soon] </li>
-            <li> Scripting: On-the-fly modifications to HTTP conversations [coming soon] </li>
-            <li> Sticky cookies  [coming soon] </li>
-            <li> Breaking iPhone apps for fun and profit [coming soon] </li>
+            <li> <a href="@!urlTo("tutorials/30second.html")!@">Client replay: a 30 second example</a> </li>
+            <li> <a href="@!urlTo("tutorials/gamecenter.html")!@">Setting highscores on Apple's GameCenter</a> </li>
         </ul>
      <li><a href="@!urlTo("faq.html")!@">FAQ</a></li>
      <li><a href="@!urlTo("admin.html")!@">Administrivia</a></li>

doc-src/index.py

@@ -1,7 +1,7 @@
 import os, sys
 import countershape
 from countershape import Page, Directory, PythonModule, markup
-import countershape.grok, countershape.template    
+import countershape.template
 sys.path.insert(0, "..")
 from libmproxy import filt
 
@@ -33,7 +33,7 @@ ns.index_contents = file(mpath("README.mkd")).read()
 top = os.path.abspath(os.getcwd())
 def example(s):
     d = file(mpath(s)).read()
-    return countershape.template.pySyntax(d)
+    return countershape.template.Syntax("py")(d)
 
 
 ns.example = example
@@ -74,9 +74,10 @@ pages = [
     Page("anticache.html", "Anticache"),
     Page("filters.html", "Filter expressions"),
     Page("scripts.html", "External scripts"),
-    Page("library.html", "libmproxy: mitmproxy as a library"),
     Page("ssl.html", "SSL"),
     Directory("certinstall"),
+    Page("library.html", "libmproxy: mitmproxy as a library"),
+    Directory("tutorials"),
     Page("faq.html", "FAQ"),
     Page("admin.html", "Administrivia")
 ]

doc-src/tutorials/30second.html

@@ -0,0 +1,61 @@
+
+My local cafe is serviced by a rickety and unreliable wireless network,
+generously sponsored with ratepayers' money by our city council. After
+connecting, you  are redirected to an SSL-protected page that prompts you for a
+username and password. Once you've entered your details, you are free to enjoy
+the intermittent dropouts, treacle-like speeds and incorrectly configured
+transparent proxy. 
+
+I tend to automate this kind of thing at the first opportunity, on the theory
+that time spent now will be more than made up in the long run. In this case, I
+might use [Firebug](http://getfirebug.com/) to ferret out the form post
+parameters and target URL, then fire up an editor to write a little script
+using Python's [urllib](http://docs.python.org/library/urllib.html) to simulate
+a submission. That's a lot of futzing about. With mitmproxy we can do the job
+in literally 30 seconds, without having to worry about any of the details.
+Here's how.
+
+## 1. Run mitmdump to record our HTTP conversation to a file.
+
+<pre class="terminal">
+> mitmdump -w wireless-login
+</pre>
+
+## 2. Point your browser at the mitmdump instance. 
+
+I use a tiny Firefox addon called [Toggle
+Proxy](https://addons.mozilla.org/en-us/firefox/addon/toggle-proxy-51740/) to
+switch quickly to and from mitmproxy. I'm assuming you've already [configured
+your browser with mitmproxy's SSL certificate
+authority](http://mitmproxy.org/doc/ssl.html).
+
+## 3. Log in as usual. 
+
+
+And that's it! You now have a serialized version of the login process in the
+file wireless-login, and you can replay it at any time like this:
+
+<pre class="terminal">
+> mitmdump -c wireless-login
+</pre>
+
+## Embellishments
+
+We're really done at this point, but there are a couple of embellishments we
+could make if we wanted. I use [wicd](http://wicd.sourceforge.net/) to
+automatically join wireless networks I frequent, and it lets me specify a
+command to run after connecting. I used the client replay command above and
+voila! - totally hands-free wireless network startup.
+
+We might also want to prune requests that download CSS, JS, images and so
+forth. These add only a few moments to the time it takes to replay, but they're
+not really needed and I somehow feel compelled trim them anyway. So, we fire up
+the mitmproxy console tool on our serialized conversation, like so:
+
+<pre class="terminal">
+> mitmproxy wireless-login
+</pre>
+
+We can now go through and manually delete (using the __d__ keyboard shortcut)
+everything we want to trim. When we're done, we use __S__ to save the
+conversation back to the file.

doc-src/tutorials/gamecenter.html

@@ -0,0 +1,105 @@
+
+## The setup
+
+In this tutorial, I'm going to show you how simple it is to creatively
+interfere with Apple Game Center traffic using mitmproxy. To set things up, I
+registered my mitmproxy CA certificate with my iPhone - there's a [step by step
+set of instructions](@!urlTo("certinstall/ios.html")!@) elsewhere in this manual. I then
+started mitmproxy on my desktop, and configured the iPhone to use it as a
+proxy. 
+
+
+## Taking a look at the Game Center traffic
+
+Lets take a first look at the Game Center traffic. The game I'll use in this
+tutorial is [Super Mega
+Worm](http://itunes.apple.com/us/app/super-mega-worm/id388541990?mt=8) - a
+great little retro-apocalyptic sidescroller for the iPhone: 
+
+<center>
+    <img src="@!urlTo("tutorials/supermega.png")!@"/>
+</center>
+
+After finishing a game (take your time), watch the traffic flowing through
+mitmproxy:
+
+<center>
+    <img src="@!urlTo("tutorials/one.png")!@"/>
+</center>
+
+We see a bunch of things we might expect - initialisation, the retrieval of
+leaderboards and so forth. Then, right at the end, there's a POST to this
+tantalising URL:
+
+<pre>
+https://service.gc.apple.com/WebObjects/GKGameStatsService.woa/wa/submitScore
+</pre>
+
+The contents of the submission are particularly interesting:
+
+<!--(block|syntax("xml"))-->
+<plist version="1.0">
+<dict>
+    <key>category</key>
+    <string>SMW_Adv_USA1</string>
+    <key>score-value</key>
+    <integer>55</integer>
+    <key>timestamp</key>
+    <integer>1301553284461</integer>
+</dict>
+</plist>
+<!--(end)-->
+
+This is a [property list](http://en.wikipedia.org/wiki/Property_list),
+containing an identifier for the game, a score (55, in this case), and a
+timestamp. Looks pretty simple to mess with.
+
+
+## Modifying and replaying the score submission
+
+Lets edit the score submission. First, select it in mitmproxy, then press
+__enter__ to view it. Make sure you're viewing the request, not the response -
+you can use __tab__ to flick between the two. Now press __e__ for edit. You'll
+be prompted for the part of the request you want to change - press __b__ for
+body.  Your preferred editor (taken from the EDITOR environment variable) will
+now fire up. Lets bump the score up to something a bit more ambitious:
+
+<!--(block|syntax("xml"))-->
+<plist version="1.0">
+<dict>
+    <key>category</key>
+    <string>SMW_Adv_USA1</string>
+    <key>score-value</key>
+    <integer>2200272667</integer>
+    <key>timestamp</key>
+    <integer>1301553284461</integer>
+</dict>
+</plist>
+<!--(end)-->
+
+Save the file and exit your editor. 
+
+The final step is to replay this modified request. Simply press __r__ for
+replay.
+
+## The glorious result and some intrigue
+
+<center>
+    <img src="@!urlTo("tutorials/leaderboard.png")!@"/>
+</center>
+
+And that's it - according to the records, I am the greatest Super Mega Worm
+player of all time. 
+
+Curiously, the top competitors' scores are all the same: 2,147,483,647. If you
+think that number seems familiar, you're right: it's 2^31-1, the maximum value
+you can fit into a signed 32-bit int. Now let me tell you another peculiar
+thing about Super Mega Worm - at the end of every game, it submits your highest
+previous score to the Game Center, not your current score.  This means that it
+stores your highscore somewhere, and I'm guessing that it reads that stored
+score back into a signed integer. So, if you _were_ to cheat by the relatively
+pedestrian means of modifying the saved score on your jailbroken phone, then
+2^31-1 might well be the maximum score you could get. Then again, if the game
+itself stores its score in a signed 32-bit int, you could get the same score
+through perfect play, effectively beating the game. So, which is it in this
+case? I'll leave that for you to decide.

doc-src/tutorials/index.py

@@ -0,0 +1,6 @@
+from countershape import Page
+
+pages = [
+    Page("30second.html", "Client playback: a 30 second example"),
+    Page("gamecenter.html", "Setting highscores on Apple's GameCenter"),
+]

libmproxy/cmdline.py

@@ -15,18 +15,20 @@ def get_common_options(options):
         stickyauth = options.stickyauth_filt
 
     return dict(
-        verbosity = options.verbose,
-        wfile = options.wfile,
+        anticache = options.anticache,
+        client_replay = options.client_replay,
+        kill = options.kill,
+        no_server = options.no_server,
+        refresh_server_playback = not options.norefresh,
+        rheaders = options.rheaders,
+        rfile = options.rfile,
         request_script = options.request_script,
         response_script = options.response_script,
         server_replay = options.server_replay,
-        kill = options.kill,
-        rheaders = options.rheaders,
-        client_replay = options.client_replay,
         stickycookie = stickycookie,
         stickyauth = stickyauth,
-        anticache = options.anticache,
-        refresh_server_playback = not options.norefresh,
+        wfile = options.wfile,
+        verbosity = options.verbose,
     )
 
 
@@ -41,6 +43,11 @@ def common_options(parser):
         action="store", type = "str", dest="confdir", default='~/.mitmproxy',
         help = "Configuration directory. (~/.mitmproxy)"
     )
+    parser.add_option(
+        "-n",
+        action="store_true", dest="no_server",
+        help="Don't start a proxy server."
+    )
     parser.add_option(
         "-p",
         action="store", type = "int", dest="port", default=8080,
@@ -51,6 +58,11 @@ def common_options(parser):
         action="store_true", dest="quiet",
         help="Quiet."
     )
+    parser.add_option(
+        "-r",
+        action="store", dest="rfile", default=None,
+        help="Read flows from file."
+    )
     parser.add_option(
         "--anticache",
         action="store_true", dest="anticache", default=False,
@@ -104,6 +116,12 @@ def common_options(parser):
     )
     parser.add_option_group(group)
 
+    parser.add_option(
+        "--cert-wait-time",
+        action="store", dest="cert_wait_time", default=0,
+        help="Wait for specified number of seconds after a new cert is generated. This can smooth over small discrepancies between the client and server times."
+    )
+
     group = optparse.OptionGroup(parser, "Server Replay")
     group.add_option(
         "-s",

libmproxy/console.py

@@ -13,13 +13,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import Queue, mailcap, mimetypes, tempfile, os, subprocess, glob, time
+import mailcap, mimetypes, tempfile, os, subprocess, glob, time
 import os.path, sys
 import cStringIO
 import urwid.raw_display
 import urwid
 import controller, utils, filt, proxy, flow
 
+VIEW_CUTOFF = 1024*100
+
 
 class Stop(Exception): pass
 
@@ -114,7 +116,7 @@ def format_flow(f, focus, extended=False, padding=2):
 
 def int_version(v):
     SIG = 3
-    v = urwid.__version__.split(".")
+    v = urwid.__version__.split("-")[0].split(".")
     x = 0
     for i in range(min(SIG, len(v))):
         x += int(v[i]) * 10**(SIG-i)
@@ -664,16 +666,19 @@ class StatusBar(WWrap):
                 ('statusbar_text', ("[%s]"%len(self.master.state.flow_list)).ljust(7)),
             ]
         t.extend(self.get_status())
+            
+        if self.master.server:
+            boundaddr = "[%s:%s]"%(self.master.server.address or "*", self.master.server.port)
+        else:
+            boundaddr = "[no proxy]"
+
         status = urwid.AttrWrap(urwid.Columns([
             urwid.Text(t),
             urwid.Text(
                 [
                     self.helptext,
                     " ",
-                    (
-                        'statusbar_text',
-                        "[%s:%s]"%(self.master.server.address or "*", self.master.server.port)
-                    ),
+                    ('statusbar_text', boundaddr),
                 ],
                 align="right"
             ),
@@ -775,9 +780,11 @@ class Options(object):
         "kill",
         "intercept",
         "limit",
+        "no_server",
         "refresh_server_playback",
         "request_script",
         "response_script",
+        "rfile",
         "rheaders",
         "server_replay",
         "stickycookie",
@@ -814,6 +821,7 @@ class ConsoleMaster(flow.FlowMaster):
     def __init__(self, server, options):
         flow.FlowMaster.__init__(self, server, ConsoleState())
         self.looptime = 0
+        self.options = options
 
         self.conn_list_view = None
         self.set_palette()
@@ -873,14 +881,27 @@ class ConsoleMaster(flow.FlowMaster):
             self.spawn_external_viewer(serr, None)
         self.refresh_connection(f)
 
+    def _trailer(self, content, txt):
+        rem = len(content) - VIEW_CUTOFF
+        if rem > 0:
+            txt.append(urwid.Text(""))
+            txt.append(
+                urwid.Text(
+                    [
+                        ("highlight", "... %s of data not shown"%utils.pretty_size(rem))
+                    ]
+                )
+            )
+
     def _view_conn_normal(self, content, txt):
-        for i in content.splitlines():
+        for i in content[:VIEW_CUTOFF].splitlines():
             txt.append(
                 urwid.Text(("text", i))
             )
+        self._trailer(content, txt)
 
     def _view_conn_binary(self, content, txt):
-        for offset, hex, s in utils.hexdump(content):
+        for offset, hex, s in utils.hexdump(content[:VIEW_CUTOFF]):
             txt.append(urwid.Text([
                 ("offset", offset),
                 " ",
@@ -888,12 +909,14 @@ class ConsoleMaster(flow.FlowMaster):
                 "   ",
                 ("text", s),
             ]))
+        self._trailer(content, txt)
 
     def _view_conn_pretty(self, content, txt):
-        for i in utils.pretty_xmlish(content):
+        for i in utils.pretty_xmlish(content[:VIEW_CUTOFF]):
             txt.append(
                 urwid.Text(("text", i)),
             )
+        self._trailer(content, txt)
 
     @utils.LRUCache(20)
     def _cached_conn_text(self, content, hdrItems, viewmode):
@@ -912,7 +935,13 @@ class ConsoleMaster(flow.FlowMaster):
             if viewmode == VIEW_BODY_BINARY:
                 self._view_conn_binary(content, txt)
             elif viewmode == VIEW_BODY_INDENT:
-                self._view_conn_pretty(content, txt)
+                if utils.isXML(content):
+                    self._view_conn_pretty(content, txt)
+                else:
+                    if utils.isBin(content):
+                        self._view_conn_binary(content, txt)
+                    else:
+                        self._view_conn_normal(content, txt)
             else:
                 if utils.isBin(content):
                     self._view_conn_binary(content, txt)
@@ -1023,9 +1052,12 @@ class ConsoleMaster(flow.FlowMaster):
         self.onekey = False
         self.view_connlist()
 
-        self.masterq = Queue.Queue()
-        slave = controller.Slave(self.masterq, self.server)
-        slave.start()
+        if self.server:
+            slave = controller.Slave(self.masterq, self.server)
+            slave.start()
+
+        if self.options.rfile:
+            self.load_flows(self.options.rfile)
 
         self.ui.run_wrapper(self.loop)
         # If True, quit just pops out to connection list view.
@@ -1123,11 +1155,10 @@ class ConsoleMaster(flow.FlowMaster):
         try:
             f = file(path, "r")
             fr = flow.FlowReader(f)
-            data = list(fr.stream())
-            f.close()
         except IOError, v:
             return v.strerror
-        self.state.load_flows(data)
+        flow.FlowMaster.load_flows(self, fr)
+        f.close()
         if self.conn_list_view:
             self.sync_list_view()
             self.focus_current()
@@ -1298,14 +1329,18 @@ class ConsoleMaster(flow.FlowMaster):
         return size
 
     def loop(self):
+        changed = True
         try:
             while not controller.exit:
                 startloop = time.time()
-                self.statusbar.redraw()
-                size = self.drawscreen()
-                self.tick(self.masterq)
+                if changed:
+                    self.statusbar.redraw()
+                    size = self.drawscreen()
+                changed = self.tick(self.masterq)
                 self.ui.set_input_timeouts(max_wait=0.1)
                 keys = self.ui.get_input()
+                if keys:
+                    changed = True
                 for k in keys:
                     if self.prompting:
                         if k == "esc":

libmproxy/controller.py

@@ -58,10 +58,14 @@ class Slave(threading.Thread):
 
 class Master:
     def __init__(self, server):
+        """
+            server may be None if no server is needed.
+        """
         self.server = server
         self.masterq = Queue.Queue()
 
     def tick(self, q):
+        changed = False
         try:
             # This endless loop runs until the 'Queue.Empty'
             # exception is thrown. If more than one request is in
@@ -71,12 +75,15 @@ class Master:
                 # Small timeout to prevent pegging the CPU
                 msg = q.get(timeout=0.01)
                 self.handle(msg)
+                changed = True
         except Queue.Empty:
             pass
+        return changed
 
     def run(self):
-        slave = Slave(self.masterq, self.server)
-        slave.start()
+        if self.server:
+            slave = Slave(self.masterq, self.server)
+            slave.start()
         while not exit:
             self.tick(self.masterq)
         self.shutdown()

libmproxy/dump.py

@@ -10,9 +10,11 @@ class Options(object):
         "client_replay",
         "keepserving",
         "kill",
+        "no_server",
         "refresh_server_playback",
         "request_script",
         "response_script",
+        "rfile",
         "rheaders",
         "server_replay",
         "stickycookie",
@@ -51,6 +53,8 @@ class DumpMaster(flow.FlowMaster):
         flow.FlowMaster.__init__(self, server, flow.State())
         self.outfile = outfile
         self.o = options
+        self.anticache = options.anticache
+        self.refresh_server_playback = options.refresh_server_playback
 
         if filtstr:
             self.filt = filt.parse(filtstr)
@@ -83,14 +87,21 @@ class DumpMaster(flow.FlowMaster):
                 not options.keepserving
             )
 
+        if options.rfile:
+            path = os.path.expanduser(options.rfile)
+            try:
+                f = file(path, "r")
+                freader = flow.FlowReader(f)
+            except IOError, v:
+                raise DumpError(v.strerror)
+            self.load_flows(freader)
+
         if options.client_replay:
             self.start_client_playback(
                 self._readflow(options.client_replay),
                 not options.keepserving
             )
 
-        self.anticache = options.anticache
-        self.refresh_server_playback = options.refresh_server_playback
 
     def _readflow(self, path):
         path = os.path.expanduser(path)
@@ -188,6 +199,8 @@ class DumpMaster(flow.FlowMaster):
 
 # begin nocover
     def run(self):
+        if self.o.rfile and not self.o.keepserving:
+            return
         try:
             return flow.FlowMaster.run(self)
         except BaseException, v:

libmproxy/flow.py

@@ -540,7 +540,19 @@ class FlowMaster(controller.Master):
             if self.server_playback.exit and self.server_playback.count() == 0:
                 self.shutdown()
 
-        controller.Master.tick(self, q)
+        return controller.Master.tick(self, q)
+
+    def load_flows(self, fr):
+        """
+            Load flows from a FlowReader object.
+        """
+        for i in fr.stream():
+            if i.request:
+                self.handle_request(i.request)
+            if i.response:
+                self.handle_response(i.response)
+            if i.error:
+                self.handle_error(i.error)
 
     def process_new_request(self, f):
         if self.stickycookie_state:
@@ -560,6 +572,12 @@ class FlowMaster(controller.Master):
                 else:
                     f.request.ack()
 
+    def process_new_response(self, f):
+        if self.stickycookie_state:
+            self.stickycookie_state.handle_response(f)
+        if "response" in self.scripts:
+            self._runscript(f, self.scripts["response"])
+
     def replay_request(self, f):
         """
             Returns None if successful, or error message if not.
@@ -568,7 +586,6 @@ class FlowMaster(controller.Master):
         if f.intercepting:
             return "Can't replay while intercepting..."
         if f.request:
-            f.backup()
             f.request.set_replay()
             if f.request.content:
                 f.request.headers["content-length"] = [str(len(f.request.content))]
@@ -605,10 +622,7 @@ class FlowMaster(controller.Master):
             self.client_playback.clear(f)
         if not f:
             r.ack()
-        if self.stickycookie_state:
-            self.stickycookie_state.handle_response(f)
-        if "response" in self.scripts:
-            self._runscript(f, self.scripts["response"])
+        self.process_new_response(f)
         return f
 
 

libmproxy/proxy.py

@@ -23,11 +23,12 @@ class ProxyError(Exception):
 
 
 class SSLConfig:
-    def __init__(self, certfile = None, ciphers = None, cacert = None):
+    def __init__(self, certfile = None, ciphers = None, cacert = None, cert_wait_time=None):
         self.certfile = certfile
         self.ciphers = ciphers
         self.cacert = cacert
         self.certdir = None
+        self.cert_wait_time = cert_wait_time
 
 
 def read_chunked(fp):
@@ -472,7 +473,10 @@ class FileLike:
     def read(self, length):
         result = ''
         while len(result) < length:
-            data = self.o.read(length)
+            try:
+                data = self.o.read(length)
+            except AttributeError:
+                break
             if not data:
                 break
             result += data
@@ -596,7 +600,7 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
                 return
             self.send_response(response)
         except IOError:
-            pass
+            cc.close = True
         except ProxyError, e:
             err = Error(request, e.msg)
             err.send(self.mqueue)
@@ -610,6 +614,7 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
             return self.config.certfile
         else:
             ret = utils.dummy_cert(self.config.certdir, self.config.cacert, host)
+            time.sleep(self.config.cert_wait_time)
             if not ret:
                 raise ProxyError(400, "mitmproxy: Unable to generate dummy cert.")
             return ret
@@ -714,7 +719,7 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
             self.wfile.write('<html><head>\n<title>%d %s</title>\n</head>\n'
                     '<body>\n%s\n</body>\n</html>' % (code, response, body))
             self.wfile.flush()
-        except IOError:
+        except:
             pass
 
 
@@ -781,5 +786,6 @@ def process_certificate_option_group(parser, options):
     return SSLConfig(
         certfile = options.cert,
         cacert = cacert,
-        ciphers = options.ciphers
+        ciphers = options.ciphers,
+        cert_wait_time = options.cert_wait_time
     )

libmproxy/utils.py

@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import re, os, subprocess, datetime, textwrap, errno, sys, time, functools
 
+CERT_SLEEP_TIME = 1
 
 def timestamp():
     """
@@ -43,6 +44,16 @@ def isBin(s):
     return False
 
 
+def isXML(s):
+    for i in s:
+        if i in "\n \t":
+            continue
+        elif i == "<":
+            return True
+        else:
+            return False
+
+
 def cleanBin(s):
     parts = []
     for i in s:
@@ -50,7 +61,8 @@ def cleanBin(s):
         if o > 31 and o < 127:
             parts.append(i)
         else:
-            parts.append(".")
+            if i not in "\n\r\t":
+                parts.append(".")
     return "".join(parts)
     
 

libmproxy/version.py

@@ -1,2 +1,2 @@
-IVERSION = (0, 4)
+IVERSION = (0, 5)
 VERSION = ".".join([str(i) for i in IVERSION])

mitmdump

@@ -30,7 +30,7 @@ if __name__ == '__main__':
     parser.add_option(
         "--keepserving",
         action="store_true", dest="keepserving", default=False,
-        help="Continue serving after playback. We exit by default."
+        help="Continue serving after client playback or file read. We exit by default."
     )
 
     options, args = parser.parse_args()
@@ -39,11 +39,15 @@ if __name__ == '__main__':
         options.verbose = 0
 
     config = proxy.process_certificate_option_group(parser, options)
-    try:
-        server = proxy.ProxyServer(config, options.port, options.addr)
-    except proxy.ProxyServerError, v:
-        print >> sys.stderr, "mitmdump:", v.args[0]
-        sys.exit(1)
+    if options.no_server:
+        server = None
+    else:
+        try:
+            server = proxy.ProxyServer(config, options.port, options.addr)
+        except proxy.ProxyServerError, v:
+            print >> sys.stderr, "mitmdump:", v.args[0]
+            sys.exit(1)
+
 
     dumpopts = dump.Options(**cmdline.get_common_options(options))
     dumpopts.keepserving = options.keepserving

mitmproxy

@@ -23,7 +23,7 @@ from optparse import OptionParser, OptionGroup
 
 if __name__ == '__main__':
     parser = OptionParser(
-                usage = "%prog [options] [flowdump path]",
+                usage = "%prog [options]",
                 version="%%prog %s"%VERSION,
             )
     cmdline.common_options(parser)
@@ -49,11 +49,15 @@ if __name__ == '__main__':
     options, args = parser.parse_args()
 
     config = proxy.process_certificate_option_group(parser, options)
-    try:
-        server = proxy.ProxyServer(config, options.port, options.addr)
-    except proxy.ProxyServerError, v:
-        print >> sys.stderr, "mitmproxy:", v.args[0]
-        sys.exit(1)
+
+    if options.no_server:
+        server = None
+    else:
+        try:
+            server = proxy.ProxyServer(config, options.port, options.addr)
+        except proxy.ProxyServerError, v:
+            print >> sys.stderr, "mitmproxy:", v.args[0]
+            sys.exit(1)
 
     opts = console.Options(**cmdline.get_common_options(options))
     opts.intercept = options.intercept
@@ -61,13 +65,6 @@ if __name__ == '__main__':
     opts.debug = options.debug
     m = console.ConsoleMaster(server, opts)
 
-    for i in args:
-        try:
-            m.load_flows(i)
-        except flow.FlowReadError, v:
-            print >> sys.stderr, "mitmproxy:", v.args[0]
-            sys.exit(1)
-
     m.run()
 
 

test/test_flow.py

@@ -369,6 +369,19 @@ class uState(libpry.AutoTree):
 
 
 class uSerialize(libpry.AutoTree):
+    def _treader(self):
+        sio = StringIO()
+        w = flow.FlowWriter(sio)
+        for i in range(3):
+            f = tutils.tflow_full()
+            w.add(f)
+        for i in range(3):
+            f = tutils.tflow_err()
+            w.add(f)
+
+        sio.seek(0)
+        return flow.FlowReader(sio)
+
     def test_roundtrip(self):
         sio = StringIO()
         f = tutils.tflow()
@@ -381,6 +394,14 @@ class uSerialize(libpry.AutoTree):
         assert len(l) == 1
         assert l[0] == f
 
+    def test_load_flows(self):
+        r = self._treader()
+        s = flow.State()
+        fm = flow.FlowMaster(None, s)
+        fm.load_flows(r)
+        assert len(s.flow_list) == 6
+
+
     def test_error(self):
         sio = StringIO()
         sio.write("bogus")

test/test_utils.py

@@ -2,6 +2,8 @@ import textwrap, cStringIO, os, time, re
 import libpry
 from libmproxy import utils
 
+utils.CERT_SLEEP_TIME = 0
+
 
 class uformat_timestamp(libpry.AutoTree):
     def test_simple(self):
@@ -16,6 +18,13 @@ class uisBin(libpry.AutoTree):
         assert utils.isBin("testing\x7f")
 
 
+class uisXML(libpry.AutoTree):
+    def test_simple(self):
+        assert not utils.isXML("foo")
+        assert utils.isXML("<foo")
+        assert utils.isXML("  \n<foo")
+
+
 class uhexdump(libpry.AutoTree):
     def test_simple(self):
         assert utils.hexdump("one\0"*10)
@@ -350,6 +359,7 @@ class uLRUCache(libpry.AutoTree):
 tests = [
     uformat_timestamp(),
     uisBin(),
+    uisXML(),
     uhexdump(),
     upretty_size(),
     uisStringLike(),

test/tutils.py

@@ -32,6 +32,13 @@ def tflow_full():
     return f
 
 
+def tflow_err():
+    r = treq()
+    f = flow.Flow(r)
+    f.error = proxy.Error(r, "error")
+    return f
+
+
 # Yes, the random ports are horrible. During development, sockets are often not
 # properly closed during error conditions, which means you have to wait until
 # you can re-bind to the same port. This is a pain in the ass, so we just pick