0.18.3: warn about Python 2 usage

Exempt localhost from upstream binding introduced in #879

CONTRIBUTORS

@@ -1,88 +1,98 @@
-  2118	Aldo Cortesi
-  1666	Maximilian Hils
-   450	Thomas Kriechbaumer
-   210	Shadab Zafar
-    94	Jason
+  2184	Aldo Cortesi
+  1745	Maximilian Hils
+   507	Thomas Kriechbaumer
+   258	Shadab Zafar
+    97	Jason
     83	Marcelo Glezer
-    36	Clemens
+    68	Clemens
     28	Jim Shaver
     18	Henrik Nordstrom
     16	Matthew Shao
-    14	David Weinstein
     14	Pedro Worcel
+    14	David Weinstein
     13	Thomas Roth
-    11	Jake Drahos
-    11	Justus Wingert
     11	Stephen Altamirano
-    10	András Veres-Szentkirályi
-    10	Chris Czub
+    11	Jake Drahos
+    11	arjun23496
+    11	Justus Wingert
     10	Sandor Nemes
     10	Zohar Lorberbaum
+    10	András Veres-Szentkirályi
+    10	Chris Czub
+    10	smill
+     9	ikoz
      9	Kyle Morton
      9	Legend Tang
      9	Rouli
-     9	ikoz
-     8	Chandler Abraham
      8	Jason A. Novak
+     8	Chandler Abraham
      7	Alexis Hildebrandt
-     7	Brad Peabody
      7	Matthias Urlichs
+     7	Brad Peabody
      7	dufferzafar
      6	Felix Yan
-     5	Choongwoo Han
-     5	Sam Cleveland
      5	Tomaz Muraus
-     5	Will Coster
      5	elitest
      5	iroiro123
-     4	Bryan Bishop
+     5	Sam Cleveland
+     5	Choongwoo Han
+     5	Will Coster
+     4	root
      4	Clemens Brunner
-     4	Marc Liyanage
-     4	Michael J. Bazzinotti
+     4	Schamper
      4	Valtteri Virtanen
      4	Wade 524
      4	Youhei Sakurai
-     4	root
+     4	Bryan Bishop
+     4	Marc Liyanage
+     4	Michael J. Bazzinotti
      4	yonder
-     3	Benjamin Lee
-     3	Chris Neasbitt
      3	Eli Shvartsman
+     3	Chris Neasbitt
      3	Guillem Anguera
-     3	Kyle Manna
      3	MatthewShao
      3	Ryan Welton
+     3	smill@cuckoo.sh
+     3	Manish Kumar
+     3	Benjamin Lee
+     3	Ryan Laughlin
      3	Zack B
+     3	Kyle Manna
      3	redfast00
      3	requires.io
-     2	Anant
-     2	Bennett Blodinger
-     2	Colin Bendell
-     2	Cory Benfield
-     2	Heikki Hannikainen
-     2	Israel Nir
-     2	Jaime Soriano Pastor
-     2	Jim Lloyd
-     2	Krzysztof Bielicki
-     2	Mark E. Haase
-     2	Michael Frister
-     2	Nick Badger
-     2	Niko Kommenda
-     2	Paul
-     2	Rob Wills
-     2	Sean Coates
-     2	Steven Van Acker
-     2	Terry Long
-     2	Wade Catron
-     2	alts
      2	isra17
      2	israel
+     2	Colin Bendell
      2	jpkrause
+     2	Paul
      2	lilydjwg
-     2	strohu
+     2	Michael Frister
      2	依云
-     1	Aditya
+     2	Jaime Soriano Pastor
+     2	Nick Badger
+     2	Rob Wills
+     2	Heikki Hannikainen
+     2	Vincent Haupert
+     2	strohu
+     2	Wade Catron
+     2	Krzysztof Bielicki
+     2	Sachin Kelkar
+     2	Israel Nir
+     2	Anant
+     2	alts
+     2	Doug Freed
+     2	Niko Kommenda
+     2	Terry Long
+     2	Mark E. Haase
+     2	Steven Van Acker
+     2	Jim Lloyd
+     2	Bennett Blodinger
+     2	Sean Coates
+     2	Cory Benfield
+     1	Sergey Chipiga
      1	Andrey Plotnikov
      1	Andy Smith
+     1	Angelo Agatino Nicolosi
      1	Anthony Zhang
      1	BSalita
      1	Ben Lerner
@@ -93,7 +103,6 @@
      1	Dan Wilbraham
      1	David Dworken
      1	David Shaw
-     1	Doug Freed
      1	Doug Lethin
      1	Drake Caraker
      1	Eric Entzel
@@ -131,10 +140,10 @@
      1	Rory McCann
      1	Rune Halvorsen
      1	Ryo Onodera
-     1	Sachin Kelkar
      1	Sahn Lam
+     1	Sanchit Sokhey
      1	Seppo Yli-Olli
-     1	Sergey Chipiga
+     1	Aditya
      1	Stefan Wärting
      1	Steve Phillips
      1	Steven Noble
@@ -145,12 +154,14 @@
      1	Thiago Arrais
      1	Tim Becker
      1	Timothy Elliott
+     1	Tyler St. Onge
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
      1	Wes Turner
      1	Yoginski
      1	Yuangxuan Wang
      1	capt8bit
+     1	chhsiao90
      1	cle1000
      1	davidpshaw
      1	deployable
@@ -161,8 +172,10 @@
      1	meeee
      1	michaeljau
      1	peralta
+     1	phackt
      1	phil plante
      1	sentient07
      1	sethp-jive
      1	starenka
+     1	vulnminer
      1	vzvu3k6k

mitmproxy/builtins/clientplayback.py

@@ -19,7 +19,7 @@ class ClientPlayback:
     def configure(self, options, updated):
         if "client_replay" in updated:
             if options.client_replay:
-                ctx.log.info(options.client_replay)
+                ctx.log.info("Client Replay: {}".format(options.client_replay))
                 try:
                     flows = flow.read_flows_from_paths(options.client_replay)
                 except exceptions.FlowReadException as e:

mitmproxy/cmdline.py

@@ -208,6 +208,11 @@ def get_common_options(args):
     if args.quiet:
         args.verbose = 0
 
+    if args.addr in ("localhost", "127.0.0.1", "::1"):
+        upstream_bind_address = ""
+    else:
+        upstream_bind_address = args.addr
+
     return dict(
         app=args.app,
         app_host=args.app_host,
@@ -251,6 +256,7 @@ def get_common_options(args):
         ignore_hosts = args.ignore_hosts,
         listen_host = args.addr,
         listen_port = args.port,
+        upstream_bind_address = upstream_bind_address,
         mode = mode,
         no_upstream_cert = args.no_upstream_cert,
         spoof_source_address = args.spoof_source_address,

mitmproxy/console/master.py

@@ -285,9 +285,6 @@ class ConsoleMaster(flow.FlowMaster):
             self.logbuffer.pop(0)
         self.logbuffer.set_focus(len(self.logbuffer) - 1)
 
-    def add_log(self, e, level):
-        signals.add_log(e, level)
-
     def sig_call_in(self, sender, seconds, callback, args=()):
         def cb(*_):
             return callback(*args)
@@ -668,11 +665,10 @@ class ConsoleMaster(flow.FlowMaster):
             )
 
     def process_flow(self, f):
-        should_intercept = any(
-            [
-                self.state.intercept and flowfilter.match(self.state.intercept, f) and not f.request.is_replay,
-                f.intercepted,
-            ]
+        should_intercept = (
+            self.state.intercept and flowfilter.match(self.state.intercept, f)
+            and not f.request.is_replay
+            and f.reply.state == "handled"
         )
         if should_intercept:
             f.intercept(self)
@@ -703,9 +699,13 @@ class ConsoleMaster(flow.FlowMaster):
         super(ConsoleMaster, self).tcp_message(f)
         message = f.messages[-1]
         direction = "->" if message.from_client else "<-"
-        self.add_log("{client} {direction} tcp {direction} {server}".format(
+        signals.add_log("{client} {direction} tcp {direction} {server}".format(
             client=repr(f.client_conn.address),
             server=repr(f.server_conn.address),
             direction=direction,
         ), "info")
-        self.add_log(strutils.bytes_to_escaped_str(message.content), "debug")
+        signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug")
+
+    @controller.handler
+    def log(self, evt):
+        signals.add_log(evt.msg, evt.level)

mitmproxy/main.py

@@ -14,6 +14,31 @@ from netlib import version_check
 from netlib import debug
 
 
+def py3():
+    print("""
+Error: mitmproxy requires Python 3.5 or above
+
+Starting with version 1.0 released in 12/2016, mitmproxy no longer supports Python 2.
+
+To install the latest version of mitmproxy using pip on Python 3, run:
+
+    pip uninstall mitmproxy
+    pip3 install mitmproxy
+
+You can also use our standalone binaries that come with their own Python interpreter:
+
+    http://docs.mitmproxy.org/en/stable/install.html
+
+To get rid of this message and use the last version of mitmproxy that supports Python 2, run:
+
+    pip install "mitmproxy==0.18.2"
+
+
+Apologies for the inconvenience!
+""")
+    sys.exit(1)
+
+
 def assert_utf8_env():
     spec = ""
     for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
@@ -47,6 +72,7 @@ def process_options(parser, options, args):
 
 
 def mitmproxy(args=None):  # pragma: no cover
+    py3()
     if os.name == "nt":
         print("Error: mitmproxy's console interface is not supported on Windows. "
               "You can run mitmdump or mitmweb instead.", file=sys.stderr)
@@ -83,6 +109,7 @@ def mitmproxy(args=None):  # pragma: no cover
 
 
 def mitmdump(args=None):  # pragma: no cover
+    py3()
     from . import dump
 
     version_check.check_pyopenssl_version()
@@ -117,6 +144,7 @@ def mitmdump(args=None):  # pragma: no cover
 
 
 def mitmweb(args=None):  # pragma: no cover
+    py3()
     from . import web
 
     version_check.check_pyopenssl_version()

mitmproxy/options.py

@@ -67,6 +67,7 @@ class Options(optmanager.OptManager):
             ignore_hosts = (),  # type: Sequence[str]
             listen_host = "",  # type: str
             listen_port = LISTEN_PORT,  # type: int
+            upstream_bind_address = "",  # type: str
             mode = "regular",  # type: str
             no_upstream_cert = False,  # type: bool
             rawtcp = False,  # type: bool
@@ -127,6 +128,7 @@ class Options(optmanager.OptManager):
         self.ignore_hosts = ignore_hosts
         self.listen_host = listen_host
         self.listen_port = listen_port
+        self.upstream_bind_address = upstream_bind_address
         self.mode = mode
         self.no_upstream_cert = no_upstream_cert
         self.rawtcp = rawtcp

mitmproxy/protocol/base.py

@@ -121,7 +121,7 @@ class ServerConnectionMixin(object):
                 server_address, (self.ctx.client_conn.address.host, 0), True)
         else:
             self.server_conn = models.ServerConnection(
-                server_address, (self.config.options.listen_host, 0))
+                server_address, (self.config.options.upstream_bind_address, 0))
 
         self.__check_self_connect()
 

mitmproxy/web/master.py

@@ -12,6 +12,7 @@ from mitmproxy import builtins
 from mitmproxy import controller
 from mitmproxy import exceptions
 from mitmproxy import flow
+from mitmproxy import flowfilter
 from mitmproxy import options
 from mitmproxy.web import app
 from netlib.http import authentication
@@ -180,8 +181,12 @@ class WebMaster(flow.FlowMaster):
             self.shutdown()
 
     def _process_flow(self, f):
-        if self.state.intercept and self.state.intercept(
-                f) and not f.request.is_replay:
+        should_intercept = (
+            self.state.intercept and flowfilter.match(self.state.intercept, f)
+            and not f.request.is_replay
+            and f.reply.state == "handled"
+        )
+        if should_intercept:
             f.intercept(self)
         return f
 

netlib/version.py

@@ -1,6 +1,6 @@
 from __future__ import (absolute_import, print_function, division)
 
-IVERSION = (0, 18)
+IVERSION = (0, 18, 3)
 VERSION = ".".join(str(i) for i in IVERSION)
 PATHOD = "pathod " + VERSION
 MITMPROXY = "mitmproxy " + VERSION

release/README.mkd

@@ -10,63 +10,11 @@
         "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)
 
 
-
 # Release Checklist
 
-## Check out release versions
-
-  - Check out the versions of pathod, netlib and mitmproxy due to be released
-
-  - Verify that repositories are in a clean state:
-
-        `./build git status`
-
-  - Ensure that the website style assets have been compiled for production, and
-    synced to the docs.
-
-  - Render the docs, update CONTRIBUTORS file:
-
-        ./build docs contributors
-
-
-## Test
-
-  - Test the source distributions:
-
-        ./build test
-
-    This does the following:
-     - creates a venv in release/venv
-     - creates source distributions in release/release
-     - installs the source distributions in the venv
-     - and runs all installed tools
-
-
-## Release
-
-  - Make a release commit for all projects, tag and push it:
-
-        ./build git commit -am "Release v0.13"
-        ./build git tag v0.13
-        ./build git push --tags
-
-  - Build the OSX binaries
-    - Follow instructions in osx-binaries
-    - Move to download dir:
-
-          mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
-
-  - Move all source distributions from `./dist` to the server:
-
-        mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download
-
-  - Upload distributions in `./dist` to PyPI:
-
-        ./build upload
-
-    You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`.
-    ([more info](https://tom-christie.github.io/articles/pypi/))
-
-  - Now bump the version number to be ready for the next cycle
-
-    `./build set-version 0.13`
+- Verify that all CI tests pass for current master
+- Tag the release, and push to Github
+- Wait for tag CI to complete
+- Download assets from snapshots.mitmproxy.org
+- Create release notice on Github
+- Upload wheel to pypi

setup.cfg

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 140
 max-complexity = 25
-ignore = E251,C901
+ignore = E251,C901,W503
 exclude = mitmproxy/contrib/*,test/mitmproxy/data/*
 builtins = file,open,basestring,xrange,unicode,long,cmp
 

setup.py

@@ -69,7 +69,7 @@ setup(
         "cryptography>=1.3, <1.6",
         "cssutils>=1.0.1, <1.1",
         "Flask>=0.10.1, <0.12",
-        "h2>=2.4.1, <3",
+        "h2>=2.4.1, <2.5",
         "html2text>=2016.1.8, <=2016.9.19",
         "hyperframe>=4.0.1, <5",
         "jsbeautifier>=1.6.3, <1.7",

test/netlib/test_tcp.py

@@ -379,14 +379,14 @@ class TestSNI(tservers.ServerTestBase):
 class TestServerCipherList(tservers.ServerTestBase):
     handler = ClientCipherListHandler
     ssl = dict(
-        cipher_list='RC4-SHA'
+        cipher_list='AES128-SHA'
     )
 
     def test_echo(self):
         c = tcp.TCPClient(("127.0.0.1", self.port))
         with c.connect():
             c.convert_to_ssl(sni="foo.com")
-            assert c.rfile.readline() == b"['RC4-SHA']"
+            assert c.rfile.readline() == b"['AES128-SHA']"
 
 
 class TestServerCurrentCipher(tservers.ServerTestBase):
@@ -399,14 +399,14 @@ class TestServerCurrentCipher(tservers.ServerTestBase):
             self.wfile.flush()
 
     ssl = dict(
-        cipher_list='RC4-SHA'
+        cipher_list='AES128-SHA'
     )
 
     def test_echo(self):
         c = tcp.TCPClient(("127.0.0.1", self.port))
         with c.connect():
             c.convert_to_ssl(sni="foo.com")
-            assert b"RC4-SHA" in c.rfile.readline()
+            assert b"AES128-SHA" in c.rfile.readline()
 
 
 class TestServerCipherListError(tservers.ServerTestBase):
@@ -424,7 +424,7 @@ class TestServerCipherListError(tservers.ServerTestBase):
 class TestClientCipherListError(tservers.ServerTestBase):
     handler = ClientCipherListHandler
     ssl = dict(
-        cipher_list='RC4-SHA'
+        cipher_list='AES128-SHA'
     )
 
     def test_echo(self):