bump version to 0.18.2

console: correct handling of logs

.appveyor.yml

@@ -7,9 +7,8 @@ environment:
   matrix:
     - PYTHON: "C:\\Python35"
       TOXENV: "py35"
-    # TODO: ENABLE WHEN AVAILABLE
-    # - PYTHON: "C:\\Python36"
-    #   TOXENV: "py36"
+    - PYTHON: "C:\\Python27"
+      TOXENV: "py27"
 
   SNAPSHOT_HOST:
     secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
@@ -19,8 +18,6 @@ environment:
     secure: 6yBwmO5gv4vAwoFYII8qjQ==
   SNAPSHOT_PASS:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
-  RTOOL_KEY:
-    secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -28,48 +25,21 @@ install:
   - "pip install -U tox"
 
 test_script:
-  - ps: "tox -- --cov mitmproxy --cov pathod -v"
-  - ps: |
-      $Env:VERSION = $(python mitmproxy/version.py)
-      $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
-      tox -e wheel
-      tox -e rtool -- bdist
-
-  - ps: |
-      if(
-        ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
-        (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-        tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
-        if (!(Test-Path "C:\projects\mitmproxy\release\installbuilder-installer.exe")) {
-          "Download InstallBuilder..."
-          (New-Object System.Net.WebClient).DownloadFile(
-            "https://installbuilder.bitrock.com/installbuilder-enterprise-16.11.1-windows-installer.exe",
-            "C:\projects\mitmproxy\release\installbuilder-installer.exe"
-          )
-        }
-        Start-Process "C:\projects\mitmproxy\release\installbuilder-installer.exe" "--mode unattended --unattendedmodeui none" -Wait
-        & 'C:\Program Files (x86)\BitRock InstallBuilder Enterprise 16.11.1\bin\builder-cli.exe' `
-          build `
-          .\release\installbuilder\mitmproxy.xml `
-          windows `
-          --license .\release\installbuilder\license.xml `
-          --setvars project.version=$Env:VERSION `
-          --verbose
-      }
+  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v"
 
 deploy_script:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
   ps: |
     if(
       ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-    ) {
-      tox -e rtool -- upload-snapshot --bdist --wheel --installer
+      (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+      ) {
+      pip install -U virtualenv
+      .\dev.ps1
+      cmd /c "python -u .\release\rtool.py bdist 2>&1"
+      python -u .\release\rtool.py upload-snapshot --bdist --wheel
     }
 
 cache:
-  - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml
   - C:\Users\appveyor\AppData\Local\pip\cache
 
 notifications:

.gitattributes

@@ -1,2 +1,2 @@
-mitmproxy/tools/web/static/**/* -diff
+mitmproxy/web/static/**/* -diff
 web/src/js/filt/filt.js -diff

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 MANIFEST
-**/tmp
+*/tmp
 /venv*
 *.py[cdo]
 *.swp

.travis.yml

@@ -1,6 +1,15 @@
 sudo: false
 language: python
 
+addons:
+  apt:
+    sources:
+      # Debian sid currently holds OpenSSL 1.0.2
+      # change this with future releases!
+      - debian-sid
+    packages:
+      - libssl-dev
+
 env:
   global:
     - CI_DEPS=codecov>=2.0.5
@@ -16,35 +25,15 @@ matrix:
       language: generic
       env: TOXENV=py35 BDIST=1
     - python: 3.5
-      env: TOXENV=py35 OPENSSL_OLD
-      addons:
-        apt:
-          packages:
-            - libssl-dev
+      env: TOXENV=py35 BDIST=1
     - python: 3.5
-      env: TOXENV=py35 BDIST=1 OPENSSL_ALPN
-      addons:
-        apt:
-          sources:
-            # Debian sid currently holds OpenSSL 1.1.0
-            # change this with future releases!
-            - debian-sid
-          packages:
-            - libssl-dev
-    - python: 3.6
-      env: TOXENV=py36 OPENSSL_ALPN
-      addons:
-        apt:
-          sources:
-            # Debian sid currently holds OpenSSL 1.1.0
-            # change this with future releases!
-            - debian-sid
-          packages:
-            - libssl-dev
+      env: TOXENV=py35 NO_ALPN=1
+    - python: 2.7
+      env: TOXENV=py27
+    - python: 2.7
+      env: TOXENV=py27 NO_ALPN=1
     - python: 3.5
       env: TOXENV=docs
-      git:
-        depth: 10000
   allow_failures:
     - python: pypy
 
@@ -52,8 +41,10 @@ install:
   - |
     if [[ $TRAVIS_OS_NAME == "osx" ]]
     then
-      brew update || brew update
-      brew outdated pyenv || brew upgrade pyenv
+      brew update || brew update # try again if it fails
+      brew upgrade
+      brew reinstall openssl
+      brew reinstall pyenv
       eval "$(pyenv init -)"
       env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.2
       pyenv global 3.5.2
@@ -62,21 +53,18 @@ install:
     fi
   - pip install tox
 
-script:
-  - tox -- --cov mitmproxy --cov pathod -v
-  - |
-    if [[ $BDIST == "1" ]]
-    then
-        git fetch --unshallow --tags
-        tox -e rtool -- bdist
-    fi
+script: tox -- --cov netlib --cov mitmproxy --cov pathod -v
 
 after_success:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
-        tox -e rtool -- upload-snapshot --bdist
+        git fetch --unshallow
+        ./dev.sh 3.5
+        source venv3.5/bin/activate
+        pip install -e ./release
+        python -u ./release/rtool.py bdist
+        python -u ./release/rtool.py upload-snapshot --bdist
     fi
 
 notifications:
@@ -91,4 +79,3 @@ cache:
   directories:
     - $HOME/.pyenv
     - $HOME/.cache/pip
-    # - $HOME/build/mitmproxy/mitmproxy/.tox

CHANGELOG

@@ -1,33 +1,3 @@
-26 December 2016: mitmproxy 1.0
-
-    * All mitmproxy tools are now Python 3 only! We plan to support Python 3.5 and higher.
-
-    * Web-Based User Interface: Mitmproxy now offically has a web-based user interface
-      called mitmweb. We consider it stable for all features currently exposed 
-      in the UI, but it still misses a lot of mitmproxy’s options.
-      
-    * Windows Compatibility: With mitmweb, mitmproxy is now useable on Windows. 
-      We are also introducing an installer (kindly sponsored by BitRock) that 
-      simplifies setup.
-      
-    * Configuration: The config file format is now a single YAML file. In most cases,
-      converting to the new format should be trivial - please see the docs for
-      more information.
-
-    * Console: Significant UI improvements - including sorting of flows by
-      size, type and url, status bar improvements, much faster indentation for
-      HTTP views, and more.
-
-    * HTTP/2: Significant improvements, but is temporarily disabled by default
-      due to wide-spread protocol implementation errors on some large website
-
-    * WebSocket: The protocol implementation is now mature, and is enabled by
-      default. Complete UI support is coming in the next release. Hooks for 
-      message interception and manipulation are available.
-
-    * A myriad of other small improvements throughout the project.
-
-
 16 October 2016: mitmproxy 0.18
 
     * Python 3 Compatibility for mitmproxy and pathod (Shadab Zafar, GSoC 2016)

CONTRIBUTORS

@@ -1,6 +1,6 @@
-  2407	Aldo Cortesi
-  1873	Maximilian Hils
-   556	Thomas Kriechbaumer
+  2184	Aldo Cortesi
+  1745	Maximilian Hils
+   507	Thomas Kriechbaumer
    258	Shadab Zafar
     97	Jason
     83	Marcelo Glezer
@@ -11,91 +11,85 @@
     14	Pedro Worcel
     14	David Weinstein
     13	Thomas Roth
-    11	Jake Drahos
     11	Stephen Altamirano
+    11	Jake Drahos
     11	arjun23496
     11	Justus Wingert
-    10	András Veres-Szentkirályi
-    10	Zohar Lorberbaum
-    10	smill
-    10	Chris Czub
     10	Sandor Nemes
-    10	Doug Freed
+    10	Zohar Lorberbaum
+    10	András Veres-Szentkirályi
+    10	Chris Czub
+    10	smill
      9	ikoz
+     9	Kyle Morton
      9	Legend Tang
      9	Rouli
-     9	Kyle Morton
      8	Jason A. Novak
      8	Chandler Abraham
+     7	Alexis Hildebrandt
      7	Matthias Urlichs
      7	Brad Peabody
      7	dufferzafar
-     7	Alexis Hildebrandt
      6	Felix Yan
-     5	Will Coster
-     5	Sam Cleveland
-     5	iroiro123
-     5	elitest
      5	Tomaz Muraus
+     5	elitest
+     5	iroiro123
+     5	Sam Cleveland
      5	Choongwoo Han
+     5	Will Coster
+     4	root
+     4	Clemens Brunner
      4	Schamper
+     4	Valtteri Virtanen
+     4	Wade 524
      4	Youhei Sakurai
      4	Bryan Bishop
-     4	root
-     4	Valtteri Virtanen
-     4	Clemens Brunner
      4	Marc Liyanage
-     4	Wade 524
-     4	chhsiao90
-     4	yonder
      4	Michael J. Bazzinotti
-     3	Ryan Welton
-     3	Ryan Laughlin
-     3	Kyle Manna
+     4	yonder
      3	Eli Shvartsman
-     3	Vincent Haupert
-     3	Manish Kumar
-     3	Zack B
+     3	Chris Neasbitt
+     3	Guillem Anguera
      3	MatthewShao
+     3	Ryan Welton
+     3	smill@cuckoo.sh
+     3	Manish Kumar
+     3	Benjamin Lee
+     3	Ryan Laughlin
+     3	Zack B
+     3	Kyle Manna
      3	redfast00
      3	requires.io
-     3	Guillem Anguera
-     3	smill@cuckoo.sh
-     3	Chris Neasbitt
-     3	Benjamin Lee
-     2	Steven Van Acker
-     2	Slobodan Mišković
-     2	Jim Lloyd
      2	isra17
      2	israel
-     2	Sean Coates
-     2	Sachin Kelkar
+     2	Colin Bendell
      2	jpkrause
-     2	Bennett Blodinger
+     2	Paul
      2	lilydjwg
      2	Michael Frister
-     2	Israel Nir
-     2	Cory Benfield
-     2	phackt
-     2	Anant
-     2	Jaime Soriano Pastor
-     2	Paul
-     2	Colin Bendell
      2	依云
-     2	Heikki Hannikainen
-     2	Rob Wills
-     2	Niko Kommenda
-     2	Naveen Pai
-     2	strohu
-     2	alts
-     2	Yoginski
-     2	Mark E. Haase
-     2	Wade Catron
-     2	Terry Long
-     2	Krzysztof Bielicki
+     2	Jaime Soriano Pastor
      2	Nick Badger
-     1	Nicolas Esteves
-     1	Andrew Orr
+     2	Rob Wills
+     2	Heikki Hannikainen
+     2	Vincent Haupert
+     2	strohu
+     2	Wade Catron
+     2	Krzysztof Bielicki
+     2	Sachin Kelkar
+     2	Israel Nir
+     2	Anant
+     2	alts
+     2	Doug Freed
+     2	Niko Kommenda
+     2	Terry Long
+     2	Mark E. Haase
+     2	Steven Van Acker
+     2	Jim Lloyd
+     2	Bennett Blodinger
+     2	Sean Coates
+     2	Cory Benfield
+     1	Sergey Chipiga
      1	Andrey Plotnikov
      1	Andy Smith
      1	Angelo Agatino Nicolosi
@@ -103,7 +97,6 @@
      1	BSalita
      1	Ben Lerner
      1	Bradley Baetz
-     1	Brady Law
      1	Brett Randall
      1	Chris Hamant
      1	Christian Frichot
@@ -112,7 +105,6 @@
      1	David Shaw
      1	Doug Lethin
      1	Drake Caraker
-     1	Edgar Boda-Majer
      1	Eric Entzel
      1	Felix Wolfsteller
      1	FreeArtMan
@@ -136,25 +128,22 @@
      1	Mathieu Mitchell
      1	Michael Bisbjerg
      1	Mike C
-     1	Mike Fotinakis
      1	Mikhail Korobov
      1	Morton Fox
      1	Nick HS
      1	Nick Raptis
-     1	Aditya
+     1	Nicolas Esteves
      1	Oleksandr Sheremet
      1	Parth Ganatra
      1	Pritam Baral
-     1	Quentin Pradet
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
      1	Ryo Onodera
-     1	Sahil Chelaramani
      1	Sahn Lam
      1	Sanchit Sokhey
      1	Seppo Yli-Olli
-     1	Sergey Chipiga
+     1	Aditya
      1	Stefan Wärting
      1	Steve Phillips
      1	Steven Noble
@@ -169,8 +158,10 @@
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
      1	Wes Turner
+     1	Yoginski
      1	Yuangxuan Wang
      1	capt8bit
+     1	chhsiao90
      1	cle1000
      1	davidpshaw
      1	deployable
@@ -181,6 +172,7 @@
      1	meeee
      1	michaeljau
      1	peralta
+     1	phackt
      1	phil plante
      1	sentient07
      1	sethp-jive

MANIFEST.in

@@ -1,3 +1,4 @@
 graft mitmproxy
 graft pathod
-recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
+graft netlib
+recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

README.rst

@@ -3,7 +3,8 @@ mitmproxy
 
 |travis| |appveyor| |coverage| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects.
+This repository contains the **mitmproxy** and **pathod** projects, as well as
+their shared networking library, **netlib**.
 
 ``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
 interface.
@@ -22,7 +23,8 @@ Documentation & Help
 General information, tutorials, and precompiled binaries can be found on the mitmproxy
 and pathod websites.
 
-|mitmproxy_site|
+|mitmproxy_site| |pathod_site|
+
 
 The latest documentation for mitmproxy is also available on ReadTheDocs.
 
@@ -43,7 +45,7 @@ Join our developer chat on Slack if you would like to hack on mitmproxy itself.
 Installation
 ------------
 
-The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`__.
+The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`_.
 If you want to contribute changes, keep on reading.
 
 
@@ -52,7 +54,7 @@ Hacking
 
 To get started hacking on mitmproxy, make sure you have Python_ 3.5.x or above with
 virtualenv_ installed (you can find installation instructions for virtualenv
-`here <http://virtualenv.readthedocs.org/en/latest/>`__). Then do the following:
+`here <http://virtualenv.readthedocs.org/en/latest/>`_). Then do the following:
 
 .. code-block:: text
 
@@ -63,7 +65,7 @@ virtualenv_ installed (you can find installation instructions for virtualenv
 
 The *dev* script will create a virtualenv environment in a directory called
 "venv", and install all mandatory and optional dependencies into it. The
-primary mitmproxy components - mitmproxy and pathod - are installed as
+primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
 "editable", so any changes to the source in the repository will be reflected
 live in the virtualenv.
 
@@ -101,17 +103,21 @@ requirements installed, and you can simply run the test suite:
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project tries to maintain 100% test coverage.
 
-You can also use `tox` to run the full suite of tests, including a quick test
-to check documentation and code linting.
+You can also use `tox` to run a full suite of tests in Python 2.7 and 3.5,
+including a quick test to check documentation and code linting.
 
 The following tox environments are relevant for local testing:
 
 .. code-block:: text
 
+    tox -e py27  # runs all tests with Python 2.7
     tox -e py35  # runs all tests with Python 3.5
     tox -e docs  # runs a does-it-compile check on the documentation
     tox -e lint  # runs the linter for coding style checks
 
+We support Python 2.7 and 3.5, so please make sure all tests pass in both
+environments. Running `tox` ensures all necessary tests are executed.
+
 
 Documentation
 -------------
@@ -139,18 +145,21 @@ contribute and collaborate. Please stick to the guidelines in
 good reason not to.
 
 This is automatically enforced on every PR. If we detect a linting error, the
-PR checks will fail and block merging. You can run our lint checks yourself
-with the following command:
+PR checks will fail and block merging. We are using this command to check for style compliance:
 
 .. code-block:: text
 
-    tox -e lint
+    flake8 --jobs 8 --count mitmproxy netlib pathod examples test
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg
     :target: https://mitmproxy.org/
     :alt: mitmproxy.org
 
+.. |pathod_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-pathod.net-blue.svg
+    :target: https://pathod.net/
+    :alt: pathod.net
+
 .. |mitmproxy_docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
     :target: http://docs.mitmproxy.org/en/latest/
     :alt: mitmproxy documentation
@@ -189,5 +198,6 @@ with the following command:
 .. _.env: https://github.com/mitmproxy/mitmproxy/blob/master/.env
 .. _Sphinx: http://sphinx-doc.org/
 .. _sphinx-autobuild: https://pypi.python.org/pypi/sphinx-autobuild
+.. _issue_tracker: https://github.com/mitmproxy/mitmproxy/issues
 .. _PEP8: https://www.python.org/dev/peps/pep-0008
 .. _Google Style Guide: https://google.github.io/styleguide/pyguide.html

dev.ps1

@@ -1,12 +1,7 @@
 $ErrorActionPreference = "Stop"
 $VENV = ".\venv"
 
-$pyver = python --version
-if($pyver -notmatch "3\.[5-9]") {
-    Write-Warning "Unexpected Python version, expected Python 3.5 or above: $pyver"
-}
-
-python -m venv $VENV --copies
+virtualenv $VENV --always-copy
 & $VENV\Scripts\activate.ps1
 
 python -m pip install --disable-pip-version-check -U pip

dev.sh

@@ -2,12 +2,12 @@
 set -e
 set -x
 
-PYVERSION=${1:-3.5}
-VENV="venv$PYVERSION"
+PYVERSION=$1
+VENV="venv$1"
 
 echo "Creating dev environment in $VENV using Python $PYVERSION"
 
-python$PYVERSION -m venv "$VENV"
+python$PYVERSION -m virtualenv "$VENV" --always-copy
 . "$VENV/bin/activate"
 pip$PYVERSION install -U pip setuptools
 pip$PYVERSION install -r requirements.txt

docs/conf.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 
 sys.path.insert(0, os.path.abspath('..'))
-from mitmproxy import version as mversion
+import netlib.version
 
 
 extensions = [
@@ -47,9 +47,9 @@ author = u'The mitmproxy project'
 # built documents.
 #
 # The short X.Y version.
-version = mversion.VERSION
+version = netlib.version.VERSION
 # The full version, including alpha/beta/rc tags.
-release = mversion.VERSION
+release = netlib.version.VERSION
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -231,7 +231,10 @@ def linkcode_resolve(domain, info):
         _, line = inspect.getsourcelines(obj)
     except (TypeError, IOError):
         return None
-    if spath.rfind("mitmproxy") > -1:
+    if spath.rfind("netlib") > -1:
+        off = spath.rfind("netlib")
+        mpath = spath[off:]
+    elif spath.rfind("mitmproxy") > -1:
         off = spath.rfind("mitmproxy")
         mpath = spath[off:]
     else:

docs/config.rst

@@ -3,11 +3,84 @@
 Configuration
 =============
 
-Mitmproxy is configured with a YAML_ file, located at
-``~/.mitmproxy/config.yaml``. We'll have complete documentation for all
-supported options in the next release in the meantime, please consult the
-source_ for a complete list of options and types.
+Mitmproxy is configured through a set of files in the users ~/.mitmproxy
+directory.
 
+mitmproxy.conf
+    Settings for the :program:`mitmproxy`. This file can contain any options supported by
+    mitmproxy.
 
-.. _YAML: http://www.yaml.org/start.html
-.. _source: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/options.py
+mitmdump.conf
+    Settings for the :program:`mitmdump`. This file can contain any options supported by mitmdump.
+
+common.conf
+    Settings shared between all command-line tools. Settings in this file are over-ridden by those
+    in the tool-specific files. Only options shared by mitmproxy and mitmdump should be used in
+    this file.
+
+Syntax
+------
+
+Comments
+^^^^^^^^
+
+.. code-block:: none
+
+    # this is a comment
+    ; this is also a comment (.ini style)
+    --- and this is a comment too (yaml style)
+
+Key/Value pairs
+^^^^^^^^^^^^^^^
+
+- Keys and values are case-sensitive
+- Whitespace is ignored
+- Lists are comma-delimited, and enclosed in square brackets
+
+.. code-block:: none
+
+    name = value   # (.ini style)
+    name: value    # (yaml style)
+    --name value   # (command-line option style)
+
+    fruit = [apple, orange, lemon]
+    indexes = [1, 12, 35 , 40]
+
+Flags
+^^^^^
+
+These are boolean options that take no value but true/false.
+
+.. code-block:: none
+
+    name = true    # (.ini style)
+    name
+    --name 	 	   # (command-line option style)
+
+Options
+-------
+
+The options available in the config files are precisely those available as
+command-line flags, with the key being the option's long name. To get a
+complete list of these, use the ``--help`` option on each of the tools. Be
+careful to only specify common options in the **common.conf** file -
+unsupported options in this file will be detected as an error on startup.
+
+Examples
+--------
+
+common.conf
+^^^^^^^^^^^
+
+Note that ``--port`` is an option supported by all tools.
+
+.. code-block:: none
+
+    port = 8080
+
+mitmproxy.conf
+^^^^^^^^^^^^^^
+
+.. code-block:: none
+
+    palette = light

docs/dev/testing.rst

@@ -10,7 +10,7 @@ suitable extension to the test suite.
 Our tests are written for the `py.test`_ or nose_ test frameworks.
 At the point where you send your pull request, a command like this:
 
->>> py.test --cov mitmproxy
+>>> py.test --cov mitmproxy --cov netlib
 
 Should give output something like this:
 

docs/features/responsestreaming.rst

@@ -40,8 +40,8 @@ You can also use a script to customize exactly which responses are streamed.
 Responses that should be tagged for streaming by setting their ``.stream``
 attribute to ``True``:
 
-.. literalinclude:: ../../examples/complex/stream.py
-   :caption: examples/complex/stream.py
+.. literalinclude:: ../../examples/stream.py
+   :caption: examples/stream.py
    :language: python
 
 Implementation Details
@@ -59,8 +59,8 @@ Modifying streamed data
 If the ``.stream`` attribute is callable, ``.stream`` will wrap the generator that yields all
 chunks.
 
-.. literalinclude:: ../../examples/complex/stream_modify.py
-   :caption: examples/complex/stream_modify.py
+.. literalinclude:: ../../examples/stream_modify.py
+   :caption: examples/stream_modify.py
    :language: python
 
 .. seealso::

docs/features/tcpproxy.rst

@@ -3,7 +3,7 @@
 TCP Proxy
 =========
 
-In case mitmproxy does not handle a specific protocol, you can exempt
+WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt
 hostnames from processing, so that mitmproxy acts as a generic TCP forwarder.
 This feature is closely related to the :ref:`passthrough` functionality,
 but differs in two important aspects:

docs/index.rst

@@ -17,7 +17,6 @@
 
     mitmproxy
     mitmdump
-    mitmweb
     config
 
 .. toctree::

docs/install.rst

@@ -3,149 +3,130 @@
 Installation
 ============
 
-Please follow the steps for your operating system.
+.. _install-ubuntu:
 
-Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or
-:ref:`mitmweb` from a terminal.
+Installation On Ubuntu
+----------------------
 
-
-.. _install-macos:
-
-Installation on macOS
----------------------
-
-You can use Homebrew to install everything:
+Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
+This was tested on a fully patched installation of Ubuntu 14.04.
 
 .. code:: bash
 
-    brew install mitmproxy
+   sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev g++
+   sudo pip install mitmproxy  # or pip install --user mitmproxy
 
-Or you can download the pre-built binary packages from `mitmproxy.org`_.
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
+On **Ubuntu 12.04** (and other systems with an outdated version of pip),
+you may need to update pip using ``pip install -U pip`` before installing mitmproxy.
 
-.. _install-windows:
+Installation From Source (Ubuntu)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Installation on Windows
------------------------
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, install the dependencies as you would for a regular
+mitmproxy installation (see :ref:`install-ubuntu`).
+Then see the Hacking_ section of the README on GitHub.
 
-The recommended way to install mitmproxy on Windows is to use the installer
-provided at `mitmproxy.org`_. After installation, you'll find shortcuts for
-:ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu.
-Both executables are  added to your PATH and can be invoked from the command
-line.
+.. _install-fedora:
 
-.. note::
-    mitmproxy's console interface is not supported on Windows, but you can use
-    mitmweb (the web-based interface) and mitmdump.
+Installation On Fedora
+----------------------
 
-.. _install-linux:
+Fedora comes with Python but we need to install pip, python-dev and several libraries.
+This was tested on a fully patched installation of Fedora 23.
 
-Installation on Linux
----------------------
+.. code:: bash
 
-The recommended way to run mitmproxy on Linux is to use the pre-built binaries
-provided at `mitmproxy.org`_.
+   sudo dnf install -y python-pip python-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libpng-devel libjpeg-devel
+   sudo pip install mitmproxy  # or pip install --user mitmproxy
 
-Our pre-built binaries provide you with the latest version of mitmproxy, a
-self-contained Python 3.5 environment and a recent version of OpenSSL that
-supports HTTP/2. Of course, you can also install mitmproxy from source if you
-prefer that (see :ref:`install-advanced`).
-
-.. _install-advanced:
-
-Advanced Installation
----------------------
-
-.. _install-docker:
-
-Docker Images
-^^^^^^^^^^^^^
-
-You can also use the official mitmproxy images from `DockerHub`_. That being
-said, our portable binaries are just as easy to install and even easier to use. 😊
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
 
 .. _install-arch:
 
-Installation on Arch Linux
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+Installation On Arch Linux
+--------------------------
 
 mitmproxy has been added into the [community] repository. Use pacman to install it:
 
 >>> sudo pacman -S mitmproxy
 
 
-.. _install-source-ubuntu:
 
-Installation from Source on Ubuntu
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Installation On Mac OS X
+------------------------
 
-Ubuntu comes with Python but we need to install pip3, python3-dev and several
-libraries. This was tested on a fully patched installation of Ubuntu 16.04.
+The easiest way to get up and running on OSX is to download the pre-built binary packages from
+`mitmproxy.org`_.
 
-.. code:: bash
+There are a few bits of customization you might want to do to make mitmproxy comfortable to use on
+OSX. The default color scheme is optimized for a dark background terminal, but you can select a
+palette for a light terminal background with the ``--palette`` option.
+You can use the OSX **open** program to create a simple and effective ``~/.mailcap`` file to view
+request and response bodies:
 
-   sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev
-   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
+.. code-block:: none
 
-On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install
-a newer version of Python. mitmproxy requires Python 3.5 or higher. Please take
-a look at pyenv_. Make sure to have an up-to-date version of pip by running
-``pip3 install -U pip``.
+    application/*; /usr/bin/open -Wn %s
+    audio/*; /usr/bin/open -Wn %s
+    image/*; /usr/bin/open -Wn %s
+    video/*; /usr/bin/open -Wn %s
+
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
 
-.. _install-source-fedora:
+Installation From Source (Mac OS X)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Installation from Source on Fedora
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, there are a few OS X specific things to keep in mind.
 
-Fedora comes with Python but we need to install pip3, python3-dev and several
-libraries. This was tested on a fully patched installation of Fedora 24.
+- Make sure that XCode is installed from the App Store, and that the command-line tools have been
+  downloaded (XCode/Preferences/Downloads).
+- If you're running a Python interpreter installed with homebrew (or similar), you may have to
+  install some dependencies by hand.
 
-.. code:: bash
+Then see the Hacking_ section of the README on GitHub.
 
-   sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel
-   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
-
-Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
-
-
-
-.. _install-source-windows:
-
-🐱💻 Installation from Source on Windows
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Installation On Windows
+-----------------------
 
 .. note::
-    mitmproxy's console interface is not supported on Windows, but you can use
-    mitmweb (the web-based interface) and mitmdump.
+    Please note that mitmdump is the only component of mitmproxy that is supported on Windows at
+    the moment.
 
-First, install the latest version of Python 3.5 or later from the `Python
-website`_. During installation, make sure to select `Add Python to PATH`.
+    **There is no interactive user interface on Windows.**
+
+
+First, install the latest version of Python 2.7 from the `Python website`_.
+If you already have an older version of Python 2.7 installed, make sure to install pip_
+(pip is included in Python 2.7.9+ by default). If pip aborts with an error, make sure you are using the current version of pip.
+
+>>> python -m pip install --upgrade pip
+
+Next, add Python and the Python Scripts directory to your **PATH** variable.
+You can do this easily by running the following in powershell:
+
+>>> [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User")
 
 Now, you can install mitmproxy by running
 
-.. code:: powershell
+>>> pip install mitmproxy
 
-    pip3 install mitmproxy
+Once the installation is complete, you can run :ref:`mitmdump` from a command prompt.
 
+Installation From Source (Windows)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-
-.. _install-dev-version:
-
-Latest Development Version
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-If you would like to install mitmproxy directly from the master branch on GitHub
-or would like to get set up to contribute to the project, install the
-dependencies as you would for a regular installation from source. Then see the
-Hacking_ section of the README on GitHub. You can check your system information
-by running: ``mitmproxy --sysinfo``
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, install Python as outlined above, then see the
+Hacking_ section of the README on GitHub.
 
 
 .. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst#hacking
 .. _mitmproxy.org: https://mitmproxy.org/
 .. _`Python website`: https://www.python.org/downloads/windows/
 .. _pip: https://pip.pypa.io/en/latest/installing.html
-.. _pyenv: https://github.com/yyuu/pyenv
-.. _DockerHub: https://hub.docker.com/r/mitmproxy/mitmproxy/

docs/introduction.rst

@@ -6,8 +6,6 @@ with a console interface.
 
 **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
-**mitmweb** is a web-based interface for mitmproxy.
-
 Documentation, tutorials and distribution packages can be found on the
 mitmproxy website: `mitmproxy.org <https://mitmproxy.org/>`_
 

docs/mitmweb.rst

@@ -1,18 +0,0 @@
-.. _mitmweb:
-.. program:: mitmweb
-
-mitmweb
-=======
-
-**mitmweb** is mitmproxy's web-based user interface that allows interactive
-examination and modification of HTTP traffic. Like mitmproxy, it differs from
-mitmdump in that all flows are kept in memory, which means that it's intended
-for taking and manipulating small-ish samples.
-
-.. warning::
-
-  Mitmweb is currently in beta. We consider it stable for all features currently
-  exposed in the UI, but it still misses a lot of mitmproxy's features.
-
-
-.. image:: screenshots/mitmweb.png

docs/modd.conf

@@ -1,6 +1,6 @@
 @build = ./_build
 
-** !_build/** ../mitmproxy/**/*.py {
+** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py {
     prep: sphinx-build -W -d @build/doctrees -b html . @build/html
     daemon: devd -m @build/html
 }

docs/pathod/test.rst

@@ -14,7 +14,7 @@ The canonical docs can be accessed using pydoc:
 >>> pydoc pathod.test
 
 The remainder of this page demonstrates some common interaction patterns using
-`Nose`_. These examples are
+<a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are
 also applicable with only minor modification to most commonly used Python testing
 engines.
 
@@ -33,6 +33,3 @@ One instance per test
 .. literalinclude:: ../../examples/pathod/test_setup.py
    :caption: examples/pathod/test_setup.py
    :language: python
-
-
-.. _Nose: https://nose.readthedocs.org/en/latest/

docs/scripting/api.rst

@@ -5,38 +5,36 @@ API
 ===
 
 - Errors
-    - `mitmproxy.flow.Error <#mitmproxy.flow.Error>`_
+    - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_
 - HTTP
-    - `mitmproxy.http.HTTPRequest <#mitmproxy.http.HTTPRequest>`_
-    - `mitmproxy.http.HTTPResponse <#mitmproxy.http.HTTPResponse>`_
-    - `mitmproxy.http.HTTPFlow <#mitmproxy.http.HTTPFlow>`_
+    - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_
+    - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_
+    - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_
 - Logging
-    - `mitmproxy.log.Log <#mitmproxy.controller.Log>`_
-    - `mitmproxy.log.LogEntry <#mitmproxy.controller.LogEntry>`_
+    - `mitmproxy.controller.Log <#mitmproxy.controller.Log>`_
+    - `mitmproxy.controller.LogEntry <#mitmproxy.controller.LogEntry>`_
 
 
 Errors
 ------
 
-.. autoclass:: mitmproxy.flow.Error
+.. autoclass:: mitmproxy.models.flow.Error
     :inherited-members:
 
 HTTP
 ----
 
-.. autoclass:: mitmproxy.http.HTTPRequest
+.. autoclass:: mitmproxy.models.http.HTTPRequest
     :inherited-members:
 
-.. autoclass:: mitmproxy.http.HTTPResponse
+.. autoclass:: mitmproxy.models.http.HTTPResponse
     :inherited-members:
 
-.. autoclass:: mitmproxy.http.HTTPFlow
+.. autoclass:: mitmproxy.models.http.HTTPFlow
     :inherited-members:
 
 Logging
 --------
 
-.. autoclass:: mitmproxy.log.Log
-    :inherited-members:
-.. autoclass:: mitmproxy.log.LogEntry
+.. autoclass:: mitmproxy.controller.Log
     :inherited-members:

docs/scripting/events.rst

@@ -56,7 +56,7 @@ Connection
           connection can correspond to multiple HTTP requests.
 
           *root_layer*
-            The root layer (see `mitmproxy.proxy.protocol` for an explanation what
+            The root layer (see `mitmproxy.protocol` for an explanation what
             the root layer is), provides transparent access to all attributes
             of the :py:class:`~mitmproxy.proxy.RootContext`. For example,
             ``root_layer.client_conn.address`` gives the remote address of the
@@ -98,18 +98,6 @@ HTTP Events
     :widths: 40 60
     :header-rows: 0
 
-    *   - .. py:function:: http_connect(flow)
-        - Called when we receive an HTTP CONNECT request. Setting a non 2xx
-          response on the flow will return the response to the client abort the
-          connection. CONNECT requests and responses do not generate the usual
-          HTTP handler events. CONNECT requests are only valid in regular and
-          upstream proxy modes.
-
-          *flow*
-            A ``models.HTTPFlow`` object. The flow is guaranteed to have
-            non-None ``request``  and ``requestheaders`` attributes.
-
-
     *   - .. py:function:: request(flow)
         - Called when a client request has been received.
 
@@ -158,54 +146,21 @@ HTTP Events
 WebSocket Events
 -----------------
 
-These events are called only after a connection made an HTTP upgrade with
-"101 Switching Protocols". No further HTTP-related events after the handshake
-are issued, only new WebSocket messages are called.
-
 .. list-table::
     :widths: 40 60
     :header-rows: 0
 
-    *   - .. py:function:: websocket_handshake(flow)
-        - Called when a client wants to establish a WebSocket connection. The
-          WebSocket-specific headers can be manipulated to alter the
+    *   - .. py:function:: websockets_handshake(flow)
+
+        - Called when a client wants to establish a WebSockets connection. The
+          WebSockets-specific headers can be manipulated to manipulate the
           handshake. The ``flow`` object is guaranteed to have a non-None
           ``request`` attribute.
 
           *flow*
-            The flow containing the HTTP WebSocket handshake request. The
+            The flow containing the HTTP websocket handshake request. The
             object is guaranteed to have a non-None ``request`` attribute.
 
-    *   - .. py:function:: websocket_start(flow)
-        - Called when WebSocket connection is established after a successful
-          handshake.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_message(flow)
-
-        - Called when a WebSocket message is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable. Currently there are
-          two types of messages, corresponding to the BINARY and TEXT frame types.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_end(flow)
-        - Called when WebSocket connection ends.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_error(flow)
-        - Called when a WebSocket error occurs - e.g. the connection closing
-          unexpectedly.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
 
 TCP Events
 ----------
@@ -218,22 +173,6 @@ connections.
     :widths: 40 60
     :header-rows: 0
 
-
-    *   - .. py:function:: tcp_start(flow)
-        - Called when TCP streaming starts.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_message(flow)
-
-        - Called when a TCP payload is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
     *   - .. py:function:: tcp_end(flow)
         - Called when TCP streaming ends.
 
@@ -246,3 +185,18 @@ connections.
 
           *flow*
             A ``models.TCPFlow`` object.
+
+    *   - .. py:function:: tcp_message(flow)
+
+        - Called a TCP payload is received from the client or server. The
+          sender and receiver are identifiable. The most recent message will be
+          ``flow.messages[-1]``. The message is user-modifiable.
+
+          *flow*
+            A ``models.TCPFlow`` object.
+
+    *   - .. py:function:: tcp_start(flow)
+        - Called when TCP streaming starts.
+
+          *flow*
+            A ``models.TCPFlow`` object.

docs/scripting/overview.rst

@@ -6,7 +6,7 @@ Overview
 Mitmproxy has a powerful scripting API that allows you to control almost any
 aspect of traffic being proxied. In fact, much of mitmproxy's own core
 functionality is implemented using the exact same API exposed to scripters (see
-:src:`mitmproxy/addons`).
+:src:`mitmproxy/builtins`).
 
 
 A simple example
@@ -17,8 +17,8 @@ appropriate points of mitmproxy's operation. Here's a complete mitmproxy script
 that adds a new header to every HTTP response before it is returned to the
 client:
 
-.. literalinclude:: ../../examples/simple/add_header.py
-   :caption: :src:`examples/simple/add_header.py`
+.. literalinclude:: ../../examples/add_header.py
+   :caption: :src:`examples/add_header.py`
    :language: python
 
 All events that deal with an HTTP request get an instance of `HTTPFlow
@@ -42,8 +42,8 @@ called before anything else happens. You can replace the current script object
 by returning it from this handler. Here's how this looks when applied to the
 example above:
 
-.. literalinclude:: ../../examples/simple/add_header_class.py
-   :caption: :src:`examples/simple/add_header_class.py`
+.. literalinclude:: ../../examples/classes.py
+   :caption: :src:`examples/classes.py`
    :language: python
 
 So here, we're using a module-level script to "boot up" into a class instance.
@@ -62,13 +62,13 @@ sophisticated - replace one value with another in all responses. Mitmproxy's
 <api.html#mitmproxy.models.http.HTTPResponse.replace>`_ method that takes care
 of all the details for us.
 
-.. literalinclude:: ../../examples/simple/script_arguments.py
-   :caption: :src:`examples/simple/script_arguments.py`
+.. literalinclude:: ../../examples/arguments.py
+   :caption: :src:`examples/arguments.py`
    :language: python
 
 We can now call this script on the command-line like this:
 
->>> mitmdump -dd -s "./script_arguments.py html faketml"
+>>> mitmdump -dd -s "./arguments.py html faketml"
 
 Whenever a handler is called, mitpmroxy rewrites the script environment so that
 it sees its own arguments as if it was invoked from the command-line.
@@ -78,15 +78,15 @@ Logging and the context
 -----------------------
 
 Scripts should not output straight to stderr or stdout. Instead, the `log
-<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` context module
+<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` contexzt module
 should be used, so that the mitmproxy host program can handle output
-appropriately. So, mitmdump can print colorised script output to the terminal,
+appropriately. So, mitmdump can print colorised sript output to the terminal,
 and mitmproxy console can place script output in the event buffer.
 
 Here's how this looks:
 
-.. literalinclude:: ../../examples/simple/log_events.py
-   :caption: :src:`examples/simple/log_events.py`
+.. literalinclude:: ../../examples/logging.py
+   :caption: :src:`examples/logging.py`
    :language: python
 
 The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
@@ -126,32 +126,15 @@ It's possible to implement a concurrent mechanism on top of the blocking
 framework, and mitmproxy includes a handy example of this that is fit for most
 purposes. You can use it as follows:
 
-.. literalinclude:: ../../examples/complex/nonblocking.py
-   :caption: :src:`examples/complex/nonblocking.py`
-   :language: python
-
-
-Testing
--------
-
-Mitmproxy includes a number of helpers for testing addons. The
-``mitmproxy.test.taddons`` module contains a context helper that takes care of
-setting up and tearing down the addon event context. The
-``mitmproxy.test.tflow`` module contains helpers for quickly creating test
-flows. Pydoc is the canonical reference for these modules, and mitmproxy's own
-test suite is an excellent source of examples of usage. Here, for instance, is
-the mitmproxy unit tests for the `anticache` option, demonstrating a good
-cross-section of the test helpers:
-
-.. literalinclude:: ../../test/mitmproxy/addons/test_anticache.py
-   :caption: :src:`test/mitmproxy/addons/test_anticache.py`
+.. literalinclude:: ../../examples/nonblocking.py
+   :caption: :src:`examples/nonblocking.py`
    :language: python
 
 
 Developing scripts
 ------------------
 
-Mitmproxy monitors scripts for modifications, and reloads them on change. When
+Mitmprxoy monitors scripts for modifications, and reloads them on change. When
 this happens, the script is shut down (the `done <events.html#done>`_  event is
 called), and the new instance is started up as if the script had just been
 loaded (the `start <events.html#start>`_ and `configure

docs/tutorials/transparent-dhcp.rst

@@ -38,14 +38,8 @@ DHCP and TFTP) services to a small-scale network.
     **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default
     `[1] <https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/>`_. For our use case, this needs
     to be disabled by changing ``dns=dnsmasq`` to ``#dns=dnsmasq`` in
-    **/etc/NetworkManager/NetworkManager.conf** and
-    
-    if on Ubuntu 16.04 or newer running:
-    
-    >>> sudo systemctl restart NetworkManager
-    
-    if on Ubuntu 12.04 or 14.04 running:
-    
+    **/etc/NetworkManager/NetworkManager.conf** and running
+
     >>> sudo restart network-manager
 
     afterwards.
@@ -67,12 +61,6 @@ DHCP and TFTP) services to a small-scale network.
 
     Apply changes:
 
-    if on Ubuntu 16.04 or newer:
-    
-    >>> sudo systemctl restart dnsmasq
-    
-    if on Ubuntu 12.04 or 14.04:
-    
     >>> sudo service dnsmasq restart
 
     Your **proxied machine** in the internal virtual network should now receive an IP address via DHCP:
@@ -86,8 +74,8 @@ To redirect traffic to mitmproxy, we need to add two iptables rules:
 
 .. code-block:: none
 
-    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
-    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
 
 4. Run mitmproxy
 ----------------

examples/README

@@ -0,0 +1,31 @@
+Some inline scripts may require additional dependencies, which can be installed using
+`pip install mitmproxy[examples]`.
+
+
+# inline script examples
+add_header.py             Simple script that just adds a header to every request.
+change_upstream_proxy.py  Dynamically change the upstream proxy
+dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
+dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+fail_with_500.py          Turn every response into an Internal Server Error.
+filt.py                   Use mitmproxy's filter expressions in your script.
+flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
+iframe_injector.py        Inject configurable iframe into pages.
+modify_form.py            Modify all form submissions to add a parameter.
+modify_querystring.py     Modify all query strings to add a parameters.
+modify_response_body.py   Replace arbitrary strings in all responses
+nonblocking.py            Demonstrate parallel processing with a blocking script.
+proxapp.py                How to embed a WSGI app in a mitmproxy server
+redirect_requests.py      Redirect requests or directly reply to them.
+stub.py                   Script stub with a method definition for every event.
+upsidedownternet.py       Rewrites traffic to turn images upside down.
+
+
+# mitmproxy examples
+flowbasic                 Basic use of mitmproxy as a library.
+stickycookies             An example of writing a custom proxy with mitmproxy.
+
+
+# misc
+read_dumpfile             Read a dumpfile generated by mitmproxy.
+mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/README.md

@@ -1,15 +0,0 @@
-# Mitmproxy Scripting API
-
-Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being 
-proxied. In fact, much of mitmproxy’s own core functionality is implemented using the exact same API 
-exposed to scripters (see [mitmproxy/addons](../mitmproxy/addons)).
-
-This directory contains some examples of the scripting API. We recommend to start with the
-ones in [simple/](./simple).
-
-|  :warning: | If you are browsing this on GitHub, make sure to select the git tag matching your mitmproxy version. |
-|------------|------------------------------------------------------------------------------------------------------|
-
-
-Some inline scripts may require additional dependencies, which can be installed using
-`pip install mitmproxy[examples]`.

examples/complex/README.md

@@ -1,18 +0,0 @@
-## Complex Examples
-
-| Filename                 | Description                                                                                   |
-|:-------------------------|:----------------------------------------------------------------------------------------------|
-| change_upstream_proxy.py | Dynamically change the upstream proxy.                                                        |
-| dns_spoofing.py          | Use mitmproxy in a DNS spoofing scenario.                                                     |
-| dup_and_replay.py        | Duplicates each request, changes it, and then replays the modified request.                   |
-| flowbasic.py             | Basic use of mitmproxy's FlowMaster directly.                                                 |
-| full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. |
-| har_dump.py              | Dump flows as HAR files.                                                                      |
-| mitmproxywrapper.py      | Bracket mitmproxy run with proxy enable/disable on OS X                                       |
-| nonblocking.py           | Demonstrate parallel processing with a blocking script                                        |
-| remote_debug.py          | This script enables remote debugging of the mitmproxy _UI_ with PyCharm.                      |
-| sslstrip.py              | sslstrip-like funtionality implemented with mitmproxy                                         |
-| stream                   | Enable streaming for all responses.                                                           |
-| stream_modify.py         | Modify a streamed response body.                                                              |
-| tcp_message.py           | Modify a raw TCP connection                                                                   |
-| tls_passthrough.py       | Use conditional TLS interception based on a user-defined strategy.                            |

examples/complex/nonblocking.py

@@ -1,11 +0,0 @@
-import time
-
-from mitmproxy.script import concurrent
-
-
-@concurrent  # Remove this and see what happens
-def request(flow):
-    # You don't want to use mitmproxy.ctx from a different thread
-    print("handle request: %s%s" % (flow.request.host, flow.request.path))
-    time.sleep(5)
-    print("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/custom_contentviews.py

@@ -0,0 +1,70 @@
+import string
+import lxml.html
+import lxml.etree
+from mitmproxy import contentviews
+from netlib import strutils
+
+
+class ViewPigLatin(contentviews.View):
+    name = "pig_latin_HTML"
+    prompt = ("pig latin HTML", "l")
+    content_types = ["text/html"]
+
+    def __call__(self, data, **metadata):
+        if strutils.is_xml(data):
+            parser = lxml.etree.HTMLParser(
+                strip_cdata=True,
+                remove_blank_text=True
+            )
+            d = lxml.html.fromstring(data, parser=parser)
+            docinfo = d.getroottree().docinfo
+
+            def piglify(src):
+                words = src.split()
+                ret = ''
+                for word in words:
+                    idx = -1
+                    while word[idx] in string.punctuation and (idx * -1) != len(word):
+                        idx -= 1
+                    if word[0].lower() in 'aeiou':
+                        if idx == -1:
+                            ret += word[0:] + "hay"
+                        else:
+                            ret += word[0:len(word) + idx + 1] + "hay" + word[idx + 1:]
+                    else:
+                        if idx == -1:
+                            ret += word[1:] + word[0] + "ay"
+                        else:
+                            ret += word[1:len(word) + idx + 1] + word[0] + "ay" + word[idx + 1:]
+                    ret += ' '
+                return ret.strip()
+
+            def recurse(root):
+                if hasattr(root, 'text') and root.text:
+                    root.text = piglify(root.text)
+                if hasattr(root, 'tail') and root.tail:
+                    root.tail = piglify(root.tail)
+
+                if len(root):
+                    for child in root:
+                        recurse(child)
+
+            recurse(d)
+
+            s = lxml.etree.tostring(
+                d,
+                pretty_print=True,
+                doctype=docinfo.doctype
+            )
+            return "HTML", contentviews.format_text(s)
+
+
+pig_view = ViewPigLatin()
+
+
+def start():
+    contentviews.add(pig_view)
+
+
+def done():
+    contentviews.remove(pig_view)

examples/dup_and_replay.py

@@ -0,0 +1,7 @@
+from mitmproxy import master
+
+
+def request(flow):
+    f = master.duplicate_flow(flow)
+    f.request.path = "/changed"
+    master.replay_request(f, block=True, run_scripthooks=False)

examples/fail_with_500.py

@@ -0,0 +1,3 @@
+def response(flow):
+    flow.response.status_code = 500
+    flow.response.content = b""

examples/flowbasic

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 """
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
@@ -8,14 +8,14 @@
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
-from mitmproxy import controller, options, master
+from mitmproxy import flow, controller, options
 from mitmproxy.proxy import ProxyServer, ProxyConfig
 
 
-class MyMaster(master.Master):
+class MyMaster(flow.FlowMaster):
     def run(self):
         try:
-            master.Master.run(self)
+            flow.FlowMaster.run(self)
         except KeyboardInterrupt:
             self.shutdown()
 
@@ -35,9 +35,9 @@ class MyMaster(master.Master):
     def log(self, l):
         print("log", l.msg)
 
-
 opts = options.Options(cadir="~/.mitmproxy/")
 config = ProxyConfig(opts)
+state = flow.State()
 server = ProxyServer(config)
-m = MyMaster(opts, server)
+m = MyMaster(opts, server, state)
 m.run()

examples/flowfilter.py

@@ -1,8 +1,6 @@
-"""
-This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
-Usage:
-    mitmdump -s "flowfilter.py FILTER"
-"""
+# This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
+# Usage: mitmdump -s "flowfilter.py FILTER"
+
 import sys
 from mitmproxy import flowfilter
 
@@ -12,7 +10,7 @@ class Filter:
         self.filter = flowfilter.parse(spec)
 
     def response(self, flow):
-        if flowfilter.match(self.filter, flow):
+        if flowfilter.match(flow, self.filter):
             print("Flow matches filter:")
             print(flow)
 

examples/flowwriter.py

@@ -1,13 +1,7 @@
-"""
-This script how to generate a mitmproxy dump file,
-as it would also be generated by passing `-w` to mitmproxy.
-In contrast to `-w`, this gives you full control over which
-flows should be saved and also allows you to rotate files or log
-to multiple files in parallel.
-"""
 import random
 import sys
-from mitmproxy import io
+
+from mitmproxy.flow import FlowWriter
 
 
 class Writer:
@@ -16,7 +10,7 @@ class Writer:
             f = sys.stdout
         else:
             f = open(path, "wb")
-        self.w = io.FlowWriter(f)
+        self.w = FlowWriter(f)
 
     def response(self, flow):
         if random.choice([True, False]):

examples/har_dump.py

@@ -3,6 +3,7 @@ This inline script can be used to dump flows as HAR files.
 """
 
 
+import pprint
 import json
 import sys
 import base64
@@ -13,9 +14,9 @@ import pytz
 
 import mitmproxy
 
-from mitmproxy import version
-from mitmproxy.utils import strutils
-from mitmproxy.net.http import cookies
+from netlib import version
+from netlib import strutils
+from netlib.http import cookies
 
 HAR = {}
 
@@ -127,25 +128,22 @@ def response(flow):
         "timings": timings,
     }
 
-    # Store binary data as base64
+    # Store binay data as base64
     if strutils.is_mostly_bin(flow.response.content):
-        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
+        b64 = base64.b64encode(flow.response.content)
+        entry["response"]["content"]["text"] = b64.decode('ascii')
         entry["response"]["content"]["encoding"] = "base64"
     else:
-        entry["response"]["content"]["text"] = flow.response.get_text(strict=False)
+        entry["response"]["content"]["text"] = flow.response.text
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
-        params = [
-            {"name": a, "value": b}
-            for a, b in flow.request.urlencoded_form.items(multi=True)
-        ]
         entry["request"]["postData"] = {
-            "mimeType": flow.request.headers.get("Content-Type", ""),
-            "text": flow.request.get_text(strict=False),
-            "params": params
+            "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
+            "text": flow.request.content,
+            "params": name_value(flow.request.urlencoded_form)
         }
 
-    if flow.server_conn.connected():
+    if flow.server_conn:
         entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
 
     HAR["log"]["entries"].append(entry)
@@ -157,17 +155,16 @@ def done():
     """
     dump_file = sys.argv[1]
 
-    json_dump = json.dumps(HAR, indent=2)  # type: str
-
     if dump_file == '-':
-        mitmproxy.ctx.log(json_dump)
+        mitmproxy.ctx.log(pprint.pformat(HAR))
     else:
-        raw = json_dump.encode()  # type: bytes
-        if dump_file.endswith('.zhar'):
-            raw = zlib.compress(raw, 9)
+        json_dump = json.dumps(HAR, indent=2)
 
-        with open(dump_file, "wb") as f:
-            f.write(raw)
+        if dump_file.endswith('.zhar'):
+            json_dump = zlib.compress(json_dump, 9)
+
+        with open(dump_file, "w") as f:
+            f.write(json_dump)
 
         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
 

examples/iframe_injector.py

@@ -11,7 +11,7 @@ class Injector:
     def response(self, flow):
         if flow.request.host in self.iframe_url:
             return
-        html = BeautifulSoup(flow.response.content, "html.parser")
+        html = BeautifulSoup(flow.response.content, "lxml")
         if html.body:
             iframe = html.new_tag(
                 "iframe",

examples/logging.py

@@ -0,0 +1,6 @@
+from mitmproxy import ctx
+
+
+def start():
+    ctx.log.info("This is some informative text.")
+    ctx.log.error("This is an error.")

examples/mitmproxywrapper.py

@@ -15,7 +15,7 @@ import os
 import sys
 
 
-class Wrapper:
+class Wrapper(object):
     def __init__(self, port, extra_arguments=None):
         self.port = port
         self.extra_arguments = extra_arguments

examples/modify_form.py

@@ -1,9 +1,7 @@
 def request(flow):
     if flow.request.urlencoded_form:
-        # If there's already a form, one can just add items to the dict:
         flow.request.urlencoded_form["mitmproxy"] = "rocks"
     else:
-        # One can also just pass new form data.
         # This sets the proper content type and overrides the body.
         flow.request.urlencoded_form = [
             ("foo", "bar")

examples/nonblocking.py

@@ -0,0 +1,10 @@
+import time
+import mitmproxy
+from mitmproxy.script import concurrent
+
+
+@concurrent  # Remove this and see what happens
+def request(flow):
+    mitmproxy.ctx.log("handle request: %s%s" % (flow.request.host, flow.request.path))
+    time.sleep(5)
+    mitmproxy.ctx.log("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/proxapp.py

@@ -4,7 +4,7 @@ instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
 a single simplest-possible page.
 """
 from flask import Flask
-from mitmproxy.addons import wsgiapp
+import mitmproxy
 
 app = Flask("proxapp")
 
@@ -14,12 +14,12 @@ def hello_world():
     return 'Hello World!'
 
 
+# Register the app using the magic domain "proxapp" on port 80. Requests to
+# this domain and port combination will now be routed to the WSGI app instance.
 def start():
-    # Host app at the magic domain "proxapp" on port 80. Requests to this
-    # domain and port combination will now be routed to the WSGI app instance.
-    return wsgiapp.WSGIApp(app, "proxapp", 80)
+    mitmproxy.ctx.master.apps.add(app, "proxapp", 80)
 
     # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
     # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
     # but won't send any data.
-    # mitmproxy.ctx.master.apps.add(app, "example.com", 443)
+    mitmproxy.ctx.master.apps.add(app, "example.com", 443)

examples/read_dumpfile

@@ -3,13 +3,13 @@
 # Simple script showing how to read a mitmproxy dump file
 #
 
-from mitmproxy import io
+from mitmproxy import flow
 from mitmproxy.exceptions import FlowReadException
 import pprint
 import sys
 
 with open(sys.argv[1], "rb") as logfile:
-    freader = io.FlowReader(logfile)
+    freader = flow.FlowReader(logfile)
     pp = pprint.PrettyPrinter(indent=4)
     try:
         for f in freader.stream():

examples/redirect_requests.py

@@ -0,0 +1,19 @@
+"""
+This example shows two ways to redirect flows to other destinations.
+"""
+from mitmproxy.models import HTTPResponse
+
+
+def request(flow):
+    # pretty_host takes the "Host" header of the request into account,
+    # which is useful in transparent mode where we usually only have the IP
+    # otherwise.
+
+    # Method 1: Answer with a locally generated response
+    if flow.request.pretty_host.endswith("example.com"):
+        resp = HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"})
+        flow.reply.send(resp)
+
+    # Method 2: Redirect the request to a different server
+    if flow.request.pretty_host.endswith("example.org"):
+        flow.request.host = "mitmproxy.org"

examples/simple/README.md

@@ -1,18 +0,0 @@
-## Simple Examples
-
-| Filename                     | Description                                                                |
-|:-----------------------------|:---------------------------------------------------------------------------|
-| add_header.py                | Simple script that just adds a header to every request.                    |
-| custom_contentview.py        | Add a custom content view to the mitmproxy UI.                             |
-| filter_flows.py              | This script demonstrates how to use mitmproxy's filter pattern in scripts. |
-| io_read_dumpfile.py          | Read a dumpfile generated by mitmproxy.                                    |
-| io_write_dumpfile.py         | Only write selected flows into a mitmproxy dumpfile.                       |
-| logging.py                   | Use mitmproxy's logging API.                                               |
-| modify_body_inject_iframe.py | Inject configurable iframe into pages.                                     |
-| modify_form.py               | Modify HTTP form submissions.                                              |
-| modify_querystring.py        | Modify HTTP query strings.                                                 |
-| redirect_requests.py         | Redirect a request to a different server.                                  |
-| script_arguments.py          | Add arguments to a script.                                                 |
-| send_reply_from_proxy.py     | Send a HTTP response directly from the proxy.                              |
-| upsidedownternet.py          | Turn all images upside down.                                               |
-| wsgi_flask_app.py            | Embed a WSGI app into mitmproxy.                                           |

examples/simple/custom_contentview.py

@@ -1,28 +0,0 @@
-"""
-This example shows how one can add a custom contentview to mitmproxy.
-The content view API is explained in the mitmproxy.contentviews module.
-"""
-from mitmproxy import contentviews
-
-
-class ViewSwapCase(contentviews.View):
-    name = "swapcase"
-
-    # We don't have a good solution for the keyboard shortcut yet -
-    # you manually need to find a free letter. Contributions welcome :)
-    prompt = ("swap case text", "p")
-    content_types = ["text/plain"]
-
-    def __call__(self, data: bytes, **metadata):
-        return "case-swapped text", contentviews.format_text(data.swapcase())
-
-
-view = ViewSwapCase()
-
-
-def start():
-    contentviews.add(view)
-
-
-def done():
-    contentviews.remove(view)

examples/simple/log_events.py

@@ -1,12 +0,0 @@
-"""
-It is recommended to use `ctx.log` for logging within a script.
-This goes to the event log in mitmproxy and to stdout in mitmdump.
-
-If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 :-)
-"""
-from mitmproxy import ctx
-
-
-def start():
-    ctx.log.info("This is some informative text.")
-    ctx.log.error("This is an error.")

examples/simple/redirect_requests.py

@@ -1,11 +0,0 @@
-"""
-This example shows two ways to redirect flows to another server.
-"""
-
-
-def request(flow):
-    # pretty_host takes the "Host" header of the request into account,
-    # which is useful in transparent mode where we usually only have the IP
-    # otherwise.
-    if flow.request.pretty_host == "example.org":
-        flow.request.host = "mitmproxy.org"

examples/simple/send_reply_from_proxy.py

@@ -1,17 +0,0 @@
-"""
-This example shows how to send a reply from the proxy immediately
-without sending any data to the remote server.
-"""
-from mitmproxy import http
-
-
-def request(flow):
-    # pretty_url takes the "Host" header of the request into account, which
-    # is useful in transparent mode where we usually only have the IP otherwise.
-
-    if flow.request.pretty_url == "http://example.com/path":
-        flow.response = http.HTTPResponse.make(
-            200,  # (optional) status code
-            b"Hello World",  # (optional) content
-            {"Content-Type": "text/html"}  # (optional) headers
-        )

examples/simple/upsidedownternet.py

@@ -1,16 +0,0 @@
-"""
-This script rotates all images passing through the proxy by 180 degrees.
-"""
-import io
-
-from PIL import Image
-
-
-def response(flow):
-    if flow.response.headers.get("content-type", "").startswith("image"):
-        s = io.BytesIO(flow.response.content)
-        img = Image.open(s).rotate(180)
-        s2 = io.BytesIO()
-        img.save(s2, "png")
-        flow.response.content = s2.getvalue()
-        flow.response.headers["content-type"] = "image/png"

examples/sslstrip.py

@@ -1,9 +1,5 @@
-"""
-This script implements an sslstrip-like attack based on mitmproxy.
-https://moxie.org/software/sslstrip/
-"""
 import re
-import urllib
+from six.moves import urllib
 
 # set of SSL/TLS capable hosts
 secure_hosts = set()
@@ -21,18 +17,13 @@ def request(flow):
         flow.request.scheme = 'https'
         flow.request.port = 443
 
-        # We need to update the request destination to whatever is specified in the host header:
-        # Having no TLS Server Name Indication from the client and just an IP address as request.host
-        # in transparent mode, TLS server name certificate validation would fail.
-        flow.request.host = flow.request.pretty_host
-
 
 def response(flow):
     flow.response.headers.pop('Strict-Transport-Security', None)
     flow.response.headers.pop('Public-Key-Pins', None)
 
     # strip links in response body
-    flow.response.content = flow.response.content.replace(b'https://', b'http://')
+    flow.response.content = flow.response.content.replace('https://', 'http://')
 
     # strip meta tag upgrade-insecure-requests in response body
     csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>'

examples/stickycookies

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+"""
+This example builds on mitmproxy's base proxying infrastructure to
+implement functionality similar to the "sticky cookies" option.
+
+Heads Up: In the majority of cases, you want to use inline scripts.
+"""
+import os
+from mitmproxy import controller, proxy
+from mitmproxy.proxy.server import ProxyServer
+
+
+class StickyMaster(controller.Master):
+    def __init__(self, server):
+        controller.Master.__init__(self, server)
+        self.stickyhosts = {}
+
+    def run(self):
+        try:
+            return controller.Master.run(self)
+        except KeyboardInterrupt:
+            self.shutdown()
+
+    @controller.handler
+    def request(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if "cookie" in flow.request.headers:
+            self.stickyhosts[hid] = flow.request.headers.get_all("cookie")
+        elif hid in self.stickyhosts:
+            flow.request.headers.set_all("cookie", self.stickyhosts[hid])
+
+    @controller.handler
+    def response(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if "set-cookie" in flow.response.headers:
+            self.stickyhosts[hid] = flow.response.headers.get_all("set-cookie")
+
+
+config = proxy.ProxyConfig(port=8080)
+server = ProxyServer(config)
+m = StickyMaster(server)
+m.run()

examples/stream.py

@@ -1,6 +1,5 @@
 def responseheaders(flow):
     """
     Enables streaming for all responses.
-    This is equivalent to passing `--stream 0` to mitmproxy.
     """
     flow.response.stream = True

examples/stub.py

@@ -0,0 +1,87 @@
+import mitmproxy
+"""
+    This is a script stub, with definitions for all events.
+"""
+
+
+def start():
+    """
+        Called once on script startup before any other events
+    """
+    mitmproxy.ctx.log("start")
+
+
+def configure(options, updated):
+    """
+        Called once on script startup before any other events, and whenever options changes.
+    """
+    mitmproxy.ctx.log("configure")
+
+
+def clientconnect(root_layer):
+    """
+        Called when a client initiates a connection to the proxy. Note that a
+        connection can correspond to multiple HTTP requests
+    """
+    mitmproxy.ctx.log("clientconnect")
+
+
+def request(flow):
+    """
+        Called when a client request has been received.
+    """
+    mitmproxy.ctx.log("request")
+
+
+def serverconnect(server_conn):
+    """
+        Called when the proxy initiates a connection to the target server. Note that a
+        connection can correspond to multiple HTTP requests
+    """
+    mitmproxy.ctx.log("serverconnect")
+
+
+def responseheaders(flow):
+    """
+        Called when the response headers for a server response have been received,
+        but the response body has not been processed yet. Can be used to tell mitmproxy
+        to stream the response.
+    """
+    mitmproxy.ctx.log("responseheaders")
+
+
+def response(flow):
+    """
+       Called when a server response has been received.
+    """
+    mitmproxy.ctx.log("response")
+
+
+def error(flow):
+    """
+        Called when a flow error has occured, e.g. invalid server responses, or
+        interrupted connections. This is distinct from a valid server HTTP error
+        response, which is simply a response with an HTTP error code.
+    """
+    mitmproxy.ctx.log("error")
+
+
+def serverdisconnect(server_conn):
+    """
+        Called when the proxy closes the connection to the target server.
+    """
+    mitmproxy.ctx.log("serverdisconnect")
+
+
+def clientdisconnect(root_layer):
+    """
+        Called when a client disconnects from the proxy.
+    """
+    mitmproxy.ctx.log("clientdisconnect")
+
+
+def done():
+    """
+        Called once on script shutdown, after any other events.
+    """
+    mitmproxy.ctx.log("done")

examples/tcp_message.py

@@ -8,7 +8,7 @@ tcp_message Inline Script Hook API Demonstration
 example cmdline invocation:
 mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py
 """
-from mitmproxy.utils import strutils
+from netlib import strutils
 
 
 def tcp_message(tcp_msg):

examples/tls_passthrough.py

@@ -20,6 +20,7 @@ Example:
 
 Authors: Maximilian Hils, Matthew Tuusberg
 """
+from __future__ import absolute_import, print_function, division
 import collections
 import random
 
@@ -28,7 +29,7 @@ from enum import Enum
 
 import mitmproxy
 from mitmproxy.exceptions import TlsProtocolException
-from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
+from mitmproxy.protocol import TlsLayer, RawTCPLayer
 
 
 class InterceptionResult(Enum):
@@ -37,7 +38,7 @@ class InterceptionResult(Enum):
     skipped = None
 
 
-class _TlsStrategy:
+class _TlsStrategy(object):
     """
     Abstract base class for interception strategies.
     """

examples/upsidedownternet.py

@@ -0,0 +1,15 @@
+from six.moves import cStringIO as StringIO
+from PIL import Image
+
+
+def response(flow):
+    if flow.response.headers.get("content-type", "").startswith("image"):
+        try:
+            s = StringIO(flow.response.content)
+            img = Image.open(s).rotate(180)
+            s2 = StringIO()
+            img.save(s2, "png")
+            flow.response.content = s2.getvalue()
+            flow.response.headers["content-type"] = "image/png"
+        except:  # Unknown image types etc.
+            pass

issue_template.md

@@ -1,20 +1,22 @@
 ##### Steps to reproduce the problem:
 
-1.
-2.
-3.
+1. 
+2. 
+3. 
+
+##### What is the expected behavior?
+
+
+##### What went wrong?
 
 
 ##### Any other comments? What have you tried so far?
 
 
+---
 
-##### System information
+Mitmproxy Version:
+Operating System:
 
 
-<!--
-    Cut and paste the output of "mitmdump --sysinfo".
-
-    If you're using an older version if mitmproxy, please specify the version
-    and OS.
--->
+<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) -->

mitmproxy/__init__.py

@@ -1,3 +0,0 @@
-# https://github.com/mitmproxy/mitmproxy/issues/1809
-# import script here so that pyinstaller registers it.
-from . import script  # noqa

mitmproxy/addons.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import, print_function, division
 from mitmproxy import exceptions
 import pprint
 
@@ -6,19 +7,12 @@ def _get_name(itm):
     return getattr(itm, "name", itm.__class__.__name__.lower())
 
 
-class AddonManager:
+class Addons(object):
     def __init__(self, master):
         self.chain = []
         self.master = master
         master.options.changed.connect(self._options_update)
 
-    def clear(self):
-        """
-            Remove all addons.
-        """
-        self.done()
-        self.chain = []
-
     def get(self, name):
         """
             Retrieve an addon by name. Addon names are equal to the .name
@@ -32,7 +26,7 @@ class AddonManager:
     def _options_update(self, options, updated):
         for i in self.chain:
             with self.master.handlecontext():
-                self.invoke_with_context(i, "configure", options, updated)
+                i.configure(options, updated)
 
     def startup(self, s):
         """
@@ -50,6 +44,8 @@ class AddonManager:
         """
             Add addons to the end of the chain, and run their startup events.
         """
+        if not addons:
+            raise ValueError("No addons specified.")
         self.chain.extend(addons)
         for i in addons:
             self.startup(i)
@@ -86,7 +82,4 @@ class AddonManager:
 
     def __call__(self, name, *args, **kwargs):
         for i in self.chain:
-            try:
-                self.invoke(i, name, *args, **kwargs)
-            except exceptions.AddonHalt:
-                return
+            self.invoke(i, name, *args, **kwargs)

mitmproxy/addons/__init__.py

@@ -1,35 +0,0 @@
-from mitmproxy.addons import anticache
-from mitmproxy.addons import anticomp
-from mitmproxy.addons import clientplayback
-from mitmproxy.addons import streamfile
-from mitmproxy.addons import onboarding
-from mitmproxy.addons import proxyauth
-from mitmproxy.addons import replace
-from mitmproxy.addons import script
-from mitmproxy.addons import setheaders
-from mitmproxy.addons import serverplayback
-from mitmproxy.addons import stickyauth
-from mitmproxy.addons import stickycookie
-from mitmproxy.addons import streambodies
-from mitmproxy.addons import upstream_auth
-from mitmproxy.addons import disable_h2c_upgrade
-
-
-def default_addons():
-    return [
-        onboarding.Onboarding(),
-        proxyauth.ProxyAuth(),
-        anticache.AntiCache(),
-        anticomp.AntiComp(),
-        stickyauth.StickyAuth(),
-        stickycookie.StickyCookie(),
-        script.ScriptLoader(),
-        streamfile.StreamFile(),
-        streambodies.StreamBodies(),
-        replace.Replace(),
-        setheaders.SetHeaders(),
-        serverplayback.ServerPlayback(),
-        clientplayback.ClientPlayback(),
-        upstream_auth.UpstreamAuth(),
-        disable_h2c_upgrade.DisableH2CleartextUpgrade(),
-    ]

mitmproxy/addons/disable_h2c_upgrade.py

@@ -1,21 +0,0 @@
-class DisableH2CleartextUpgrade:
-
-    """
-    We currently only support HTTP/2 over a TLS connection. Some clients try
-    to upgrade a connection from HTTP/1.1 to h2c, so we need to remove those
-    headers to avoid protocol errors if one endpoints suddenly starts sending
-    HTTP/2 frames.
-    """
-
-    def process_flow(self, f):
-        if f.request.headers.get('upgrade', '') == 'h2c':
-            del f.request.headers['upgrade']
-            if 'connection' in f.request.headers:
-                del f.request.headers['connection']
-            if 'http2-settings' in f.request.headers:
-                del f.request.headers['http2-settings']
-
-    # Handlers
-
-    def request(self, f):
-        self.process_flow(f)

mitmproxy/addons/eventstore.py

@@ -1,19 +0,0 @@
-from typing import List  # noqa
-
-import blinker
-from mitmproxy.log import LogEntry
-
-
-class EventStore:
-    def __init__(self):
-        self.data = []  # type: List[LogEntry]
-        self.sig_add = blinker.Signal()
-        self.sig_refresh = blinker.Signal()
-
-    def log(self, entry: LogEntry):
-        self.data.append(entry)
-        self.sig_add.send(self, entry=entry)
-
-    def clear(self):
-        self.data.clear()
-        self.sig_refresh.send(self)

mitmproxy/addons/intercept.py

@@ -1,35 +0,0 @@
-from mitmproxy import flowfilter
-from mitmproxy import exceptions
-
-
-class Intercept:
-    def __init__(self):
-        self.filt = None
-
-    def configure(self, opts, updated):
-        if "intercept" in updated:
-            if not opts.intercept:
-                self.filt = None
-                return
-            self.filt = flowfilter.parse(opts.intercept)
-            if not self.filt:
-                raise exceptions.OptionsError(
-                    "Invalid interception filter: %s" % opts.intercept
-                )
-
-    def process_flow(self, f):
-        if self.filt:
-            should_intercept = all([
-                self.filt(f),
-                not f.request.is_replay,
-            ])
-            if should_intercept:
-                f.intercept()
-
-    # Handlers
-
-    def request(self, f):
-        self.process_flow(f)
-
-    def response(self, f):
-        self.process_flow(f)

mitmproxy/addons/onboarding.py

@@ -1,17 +0,0 @@
-from mitmproxy.addons import wsgiapp
-from mitmproxy.addons.onboardingapp import app
-
-
-class Onboarding(wsgiapp.WSGIApp):
-    def __init__(self):
-        super().__init__(app.Adapter(app.application), None, None)
-        self.enabled = False
-
-    def configure(self, options, updated):
-        self.host = options.app_host
-        self.port = options.app_port
-        self.enabled = options.app
-
-    def request(self, f):
-        if self.enabled:
-            super().request(f)

mitmproxy/addons/proxyauth.py

@@ -1,148 +0,0 @@
-import binascii
-
-import passlib.apache
-
-from mitmproxy import exceptions
-from mitmproxy import http
-import mitmproxy.net.http
-
-
-REALM = "mitmproxy"
-
-
-def mkauth(username, password, scheme="basic"):
-    v = binascii.b2a_base64(
-        (username + ":" + password).encode("utf8")
-    ).decode("ascii")
-    return scheme + " " + v
-
-
-def parse_http_basic_auth(s):
-    words = s.split()
-    if len(words) != 2:
-        return None
-    scheme = words[0]
-    try:
-        user = binascii.a2b_base64(words[1]).decode("utf8", "replace")
-    except binascii.Error:
-        return None
-    parts = user.split(':')
-    if len(parts) != 2:
-        return None
-    return scheme, parts[0], parts[1]
-
-
-class ProxyAuth:
-    def __init__(self):
-        self.nonanonymous = False
-        self.htpasswd = None
-        self.singleuser = None
-
-    def enabled(self):
-        return any([self.nonanonymous, self.htpasswd, self.singleuser])
-
-    def which_auth_header(self, f):
-        if f.mode == "regular":
-            return 'Proxy-Authorization'
-        else:
-            return 'Authorization'
-
-    def auth_required_response(self, f):
-        if f.mode == "regular":
-            hdrname = 'Proxy-Authenticate'
-        else:
-            hdrname = 'WWW-Authenticate'
-
-        headers = mitmproxy.net.http.Headers()
-        headers[hdrname] = 'Basic realm="%s"' % REALM
-
-        if f.mode == "transparent":
-            return http.make_error_response(
-                401,
-                "Authentication Required",
-                headers
-            )
-        else:
-            return http.make_error_response(
-                407,
-                "Proxy Authentication Required",
-                headers,
-            )
-
-    def check(self, f):
-        auth_value = f.request.headers.get(self.which_auth_header(f), None)
-        if not auth_value:
-            return False
-        parts = parse_http_basic_auth(auth_value)
-        if not parts:
-            return False
-        scheme, username, password = parts
-        if scheme.lower() != 'basic':
-            return False
-
-        if self.nonanonymous:
-            pass
-        elif self.singleuser:
-            if [username, password] != self.singleuser:
-                return False
-        elif self.htpasswd:
-            if not self.htpasswd.check_password(username, password):
-                return False
-        else:
-            raise NotImplementedError("Should never happen.")
-
-        return True
-
-    def authenticate(self, f):
-        if self.check(f):
-            del f.request.headers[self.which_auth_header(f)]
-        else:
-            f.response = self.auth_required_response(f)
-
-    # Handlers
-    def configure(self, options, updated):
-        if "auth_nonanonymous" in updated:
-            self.nonanonymous = options.auth_nonanonymous
-        if "auth_singleuser" in updated:
-            if options.auth_singleuser:
-                parts = options.auth_singleuser.split(':')
-                if len(parts) != 2:
-                    raise exceptions.OptionsError(
-                        "Invalid single-user auth specification."
-                    )
-                self.singleuser = parts
-            else:
-                self.singleuser = None
-        if "auth_htpasswd" in updated:
-            if options.auth_htpasswd:
-                try:
-                    self.htpasswd = passlib.apache.HtpasswdFile(
-                        options.auth_htpasswd
-                    )
-                except (ValueError, OSError) as v:
-                    raise exceptions.OptionsError(
-                        "Could not open htpasswd file: %s" % v
-                    )
-            else:
-                self.htpasswd = None
-        if self.enabled():
-            if options.mode == "transparent":
-                raise exceptions.OptionsError(
-                    "Proxy Authentication not supported in transparent mode."
-                )
-            elif options.mode == "socks5":
-                raise exceptions.OptionsError(
-                    "Proxy Authentication not supported in SOCKS mode. "
-                    "https://github.com/mitmproxy/mitmproxy/issues/738"
-                )
-            # TODO: check for multiple auth options
-
-    def http_connect(self, f):
-        if self.enabled() and f.mode == "regular":
-            self.authenticate(f)
-
-    def requestheaders(self, f):
-        if self.enabled():
-            # Are we already authenticated in CONNECT?
-            if not (f.mode == "regular" and f.server_conn.via):
-                self.authenticate(f)

mitmproxy/addons/stickyauth.py

@@ -1,29 +0,0 @@
-from mitmproxy import exceptions
-from mitmproxy import flowfilter
-
-
-class StickyAuth:
-    def __init__(self):
-        self.flt = None
-        self.hosts = {}
-
-    def configure(self, options, updated):
-        if "stickyauth" in updated:
-            if options.stickyauth:
-                flt = flowfilter.parse(options.stickyauth)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "stickyauth: invalid filter expression: %s" % options.stickyauth
-                    )
-                self.flt = flt
-            else:
-                self.flt = None
-
-    def request(self, flow):
-        if self.flt:
-            host = flow.request.host
-            if "authorization" in flow.request.headers:
-                self.hosts[host] = flow.request.headers["authorization"]
-            elif flowfilter.match(self.flt, flow):
-                if host in self.hosts:
-                    flow.request.headers["authorization"] = self.hosts[host]

mitmproxy/addons/streambodies.py

@@ -1,34 +0,0 @@
-from mitmproxy.net.http import http1
-from mitmproxy import exceptions
-from mitmproxy import ctx
-
-
-class StreamBodies:
-    def __init__(self):
-        self.max_size = None
-
-    def configure(self, options, updated):
-        self.max_size = options.stream_large_bodies
-
-    def run(self, f, is_request):
-        if self.max_size:
-            r = f.request if is_request else f.response
-            try:
-                expected_size = http1.expected_http_body_size(
-                    f.request, f.response if not is_request else None
-                )
-            except exceptions.HttpException:
-                f.reply.kill()
-                return
-            if expected_size and not r.raw_content and not (0 <= expected_size <= self.max_size):
-                # r.stream may already be a callable, which we want to preserve.
-                r.stream = r.stream or True
-                # FIXME: make message generic when we add rquest streaming
-                ctx.log.info("Streaming response from %s" % f.request.host)
-
-    # FIXME! Request streaming doesn't work at the moment.
-    def requestheaders(self, f):
-        self.run(f, True)
-
-    def responseheaders(self, f):
-        self.run(f, False)

mitmproxy/addons/upstream_auth.py

@@ -1,53 +0,0 @@
-import re
-import base64
-
-from mitmproxy import exceptions
-from mitmproxy.utils import strutils
-
-
-def parse_upstream_auth(auth):
-    pattern = re.compile(".+:")
-    if pattern.search(auth) is None:
-        raise exceptions.OptionsError(
-            "Invalid upstream auth specification: %s" % auth
-        )
-    return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
-
-
-class UpstreamAuth():
-    """
-        This addon handles authentication to systems upstream from us for the
-        upstream proxy and reverse proxy mode. There are 3 cases:
-
-        - Upstream proxy CONNECT requests should have authentication added, and
-          subsequent already connected requests should not.
-        - Upstream proxy regular requests
-        - Reverse proxy regular requests (CONNECT is invalid in this mode)
-    """
-    def __init__(self):
-        self.auth = None
-        self.root_mode = None
-
-    def configure(self, options, updated):
-        # FIXME: We're doing this because our proxy core is terminally confused
-        # at the moment. Ideally, we should be able to check if we're in
-        # reverse proxy mode at the HTTP layer, so that scripts can put the
-        # proxy in reverse proxy mode for specific reuests.
-        if "mode" in updated:
-            self.root_mode = options.mode
-        if "upstream_auth" in updated:
-            if options.upstream_auth is None:
-                self.auth = None
-            else:
-                self.auth = parse_upstream_auth(options.upstream_auth)
-
-    def http_connect(self, f):
-        if self.auth and f.mode == "upstream":
-            f.request.headers["Proxy-Authorization"] = self.auth
-
-    def requestheaders(self, f):
-        if self.auth:
-            if f.mode == "upstream" and not f.server_conn.via:
-                f.request.headers["Proxy-Authorization"] = self.auth
-            elif self.root_mode == "reverse":
-                f.request.headers["Proxy-Authorization"] = self.auth

mitmproxy/addons/view.py

@@ -1,418 +0,0 @@
-"""
-The View:
-
-- Keeps track of a store of flows
-- Maintains a filtered, ordered view onto that list of flows
-- Exposes a number of signals so the view can be monitored
-- Tracks focus within the view
-- Exposes a settings store for flows that automatically expires if the flow is
-  removed from the store.
-"""
-import collections
-import typing
-import datetime
-
-import blinker
-import sortedcontainers
-
-import mitmproxy.flow
-from mitmproxy import flowfilter
-from mitmproxy import exceptions
-
-# The underlying sorted list implementation expects the sort key to be stable
-# for the lifetime of the object. However, if we sort by size, for instance,
-# the sort order changes as the flow progresses through its lifecycle. We
-# address this through two means:
-#
-# - Let order keys cache the sort value by flow ID.
-#
-# - Add a facility to refresh items in the list by removing and re-adding them
-# when they are updated.
-
-
-class _OrderKey:
-    def __init__(self, view):
-        self.view = view
-
-    def generate(self, f: mitmproxy.flow.Flow) -> typing.Any:  # pragma: no cover
-        pass
-
-    def refresh(self, f):
-        k = self._key()
-        old = self.view.settings[f][k]
-        new = self.generate(f)
-        if old != new:
-            self.view._view.remove(f)
-            self.view.settings[f][k] = new
-            self.view._view.add(f)
-            self.view.sig_view_refresh.send(self.view)
-
-    def _key(self):
-        return "_order_%s" % id(self)
-
-    def __call__(self, f):
-        if f.id in self.view._store:
-            k = self._key()
-            s = self.view.settings[f]
-            if k in s:
-                return s[k]
-            val = self.generate(f)
-            s[k] = val
-            return val
-        else:
-            return self.generate(f)
-
-
-class OrderRequestStart(_OrderKey):
-    def generate(self, f: mitmproxy.flow.Flow) -> datetime.datetime:
-        return f.request.timestamp_start or 0
-
-
-class OrderRequestMethod(_OrderKey):
-    def generate(self, f: mitmproxy.flow.Flow) -> str:
-        return f.request.method
-
-
-class OrderRequestURL(_OrderKey):
-    def generate(self, f: mitmproxy.flow.Flow) -> str:
-        return f.request.url
-
-
-class OrderKeySize(_OrderKey):
-    def generate(self, f: mitmproxy.flow.Flow) -> int:
-        s = 0
-        if f.request.raw_content:
-            s += len(f.request.raw_content)
-        if f.response and f.response.raw_content:
-            s += len(f.response.raw_content)
-        return s
-
-
-matchall = flowfilter.parse(".")
-
-
-orders = [
-    ("t", "time"),
-    ("m", "method"),
-    ("u", "url"),
-    ("z", "size"),
-]
-
-
-class View(collections.Sequence):
-    def __init__(self):
-        super().__init__()
-        self._store = {}
-        self.filter = matchall
-        # Should we show only marked flows?
-        self.show_marked = False
-
-        self.default_order = OrderRequestStart(self)
-        self.orders = dict(
-            time = self.default_order,
-            method = OrderRequestMethod(self),
-            url = OrderRequestURL(self),
-            size = OrderKeySize(self),
-        )
-        self.order_key = self.default_order
-        self.order_reversed = False
-        self.focus_follow = False
-
-        self._view = sortedcontainers.SortedListWithKey(key = self.order_key)
-
-        # The sig_view* signals broadcast events that affect the view. That is,
-        # an update to a flow in the store but not in the view does not trigger
-        # a signal. All signals are called after the view has been updated.
-        self.sig_view_update = blinker.Signal()
-        self.sig_view_add = blinker.Signal()
-        self.sig_view_remove = blinker.Signal()
-        # Signals that the view should be refreshed completely
-        self.sig_view_refresh = blinker.Signal()
-
-        # The sig_store* signals broadcast events that affect the underlying
-        # store. If a flow is removed from just the view, sig_view_remove is
-        # triggered. If it is removed from the store while it is also in the
-        # view, both sig_store_remove and sig_view_remove are triggered.
-        self.sig_store_remove = blinker.Signal()
-        # Signals that the store should be refreshed completely
-        self.sig_store_refresh = blinker.Signal()
-
-        self.focus = Focus(self)
-        self.settings = Settings(self)
-
-    def store_count(self):
-        return len(self._store)
-
-    def inbounds(self, index: int) -> bool:
-        """
-            Is this 0 <= index < len(self)
-        """
-        return 0 <= index < len(self)
-
-    def _rev(self, idx: int) -> int:
-        """
-            Reverses an index, if needed
-        """
-        if self.order_reversed:
-            if idx < 0:
-                idx = -idx - 1
-            else:
-                idx = len(self._view) - idx - 1
-                if idx < 0:
-                    raise IndexError
-        return idx
-
-    def __len__(self):
-        return len(self._view)
-
-    def __getitem__(self, offset) -> mitmproxy.flow.Flow:
-        return self._view[self._rev(offset)]
-
-    # Reflect some methods to the efficient underlying implementation
-
-    def _bisect(self, f: mitmproxy.flow.Flow) -> int:
-        v = self._view.bisect_right(f)
-        return self._rev(v - 1) + 1
-
-    def index(self, f: mitmproxy.flow.Flow, start: int = 0, stop: typing.Optional[int] = None) -> int:
-        return self._rev(self._view.index(f, start, stop))
-
-    def __contains__(self, f: mitmproxy.flow.Flow) -> bool:
-        return self._view.__contains__(f)
-
-    def _order_key_name(self):
-        return "_order_%s" % id(self.order_key)
-
-    def _base_add(self, f):
-        self.settings[f][self._order_key_name()] = self.order_key(f)
-        self._view.add(f)
-
-    def _refilter(self):
-        self._view.clear()
-        for i in self._store.values():
-            if self.show_marked and not i.marked:
-                continue
-            if self.filter(i):
-                self._base_add(i)
-        self.sig_view_refresh.send(self)
-
-    # API
-    def toggle_marked(self):
-        self.show_marked = not self.show_marked
-        self._refilter()
-
-    def set_reversed(self, value: bool):
-        self.order_reversed = value
-        self.sig_view_refresh.send(self)
-
-    def set_order(self, order_key: typing.Callable):
-        """
-            Sets the current view order.
-        """
-        self.order_key = order_key
-        newview = sortedcontainers.SortedListWithKey(key=order_key)
-        newview.update(self._view)
-        self._view = newview
-
-    def set_filter(self, flt: typing.Optional[flowfilter.TFilter]):
-        """
-            Sets the current view filter.
-        """
-        self.filter = flt or matchall
-        self._refilter()
-
-    def clear(self):
-        """
-            Clears both the store and view.
-        """
-        self._store.clear()
-        self._view.clear()
-        self.sig_view_refresh.send(self)
-        self.sig_store_refresh.send(self)
-
-    def add(self, f: mitmproxy.flow.Flow) -> bool:
-        """
-            Adds a flow to the state. If the flow already exists, it is
-            ignored.
-        """
-        if f.id not in self._store:
-            self._store[f.id] = f
-            if self.filter(f):
-                self._base_add(f)
-                if self.focus_follow:
-                    self.focus.flow = f
-                self.sig_view_add.send(self, flow=f)
-
-    def remove(self, f: mitmproxy.flow.Flow):
-        """
-            Removes the flow from the underlying store and the view.
-        """
-        if f.id in self._store:
-            if f in self._view:
-                self._view.remove(f)
-                self.sig_view_remove.send(self, flow=f)
-            del self._store[f.id]
-            self.sig_store_remove.send(self, flow=f)
-
-    def update(self, f: mitmproxy.flow.Flow):
-        """
-            Updates a flow. If the flow is not in the state, it's ignored.
-        """
-        if f.id in self._store:
-            if self.filter(f):
-                if f not in self._view:
-                    self._base_add(f)
-                    if self.focus_follow:
-                        self.focus.flow = f
-                    self.sig_view_add.send(self, flow=f)
-                else:
-                    # This is a tad complicated. The sortedcontainers
-                    # implementation assumes that the order key is stable. If
-                    # it changes mid-way Very Bad Things happen. We detect when
-                    # this happens, and re-fresh the item.
-                    self.order_key.refresh(f)
-                    self.sig_view_update.send(self, flow=f)
-            else:
-                try:
-                    self._view.remove(f)
-                    self.sig_view_remove.send(self, flow=f)
-                except ValueError:
-                    # The value was not in the view
-                    pass
-
-    def get_by_id(self, flow_id: str) -> typing.Optional[mitmproxy.flow.Flow]:
-        """
-        Get flow with the given id from the store.
-        Returns None if the flow is not found.
-        """
-        return self._store.get(flow_id)
-
-    # Event handlers
-    def configure(self, opts, updated):
-        if "filter" in updated:
-            filt = None
-            if opts.filter:
-                filt = flowfilter.parse(opts.filter)
-                if not filt:
-                    raise exceptions.OptionsError(
-                        "Invalid interception filter: %s" % opts.filter
-                    )
-            self.set_filter(filt)
-        if "order" in updated:
-            if opts.order is None:
-                self.set_order(self.default_order)
-            else:
-                if opts.order not in self.orders:
-                    raise exceptions.OptionsError(
-                        "Unknown flow order: %s" % opts.order
-                    )
-                self.set_order(self.orders[opts.order])
-        if "order_reversed" in updated:
-            self.set_reversed(opts.order_reversed)
-        if "focus_follow" in updated:
-            self.focus_follow = opts.focus_follow
-
-    def request(self, f):
-        self.add(f)
-
-    def error(self, f):
-        self.update(f)
-
-    def response(self, f):
-        self.update(f)
-
-    def intercept(self, f):
-        self.update(f)
-
-    def resume(self, f):
-        self.update(f)
-
-    def kill(self, f):
-        self.update(f)
-
-
-class Focus:
-    """
-        Tracks a focus element within a View.
-    """
-    def __init__(self, v: View) -> None:
-        self.view = v
-        self._flow = None  # type: mitmproxy.flow.Flow
-        self.sig_change = blinker.Signal()
-        if len(self.view):
-            self.flow = self.view[0]
-        v.sig_view_add.connect(self._sig_view_add)
-        v.sig_view_remove.connect(self._sig_view_remove)
-        v.sig_view_refresh.connect(self._sig_view_refresh)
-
-    @property
-    def flow(self) -> typing.Optional[mitmproxy.flow.Flow]:
-        return self._flow
-
-    @flow.setter
-    def flow(self, f: typing.Optional[mitmproxy.flow.Flow]):
-        if f is not None and f not in self.view:
-            raise ValueError("Attempt to set focus to flow not in view")
-        self._flow = f
-        self.sig_change.send(self)
-
-    @property
-    def index(self) -> typing.Optional[int]:
-        if self.flow:
-            return self.view.index(self.flow)
-
-    @index.setter
-    def index(self, idx):
-        if idx < 0 or idx > len(self.view) - 1:
-            raise ValueError("Index out of view bounds")
-        self.flow = self.view[idx]
-
-    def _nearest(self, f, v):
-        return min(v._bisect(f), len(v) - 1)
-
-    def _sig_view_remove(self, view, flow):
-        if len(view) == 0:
-            self.flow = None
-        elif flow is self.flow:
-            self.flow = view[self._nearest(self.flow, view)]
-
-    def _sig_view_refresh(self, view):
-        if len(view) == 0:
-            self.flow = None
-        elif self.flow is None:
-            self.flow = view[0]
-        elif self.flow not in view:
-            self.flow = view[self._nearest(self.flow, view)]
-
-    def _sig_view_add(self, view, flow):
-        # We only have to act if we don't have a focus element
-        if not self.flow:
-            self.flow = flow
-
-
-class Settings(collections.Mapping):
-    def __init__(self, view: View) -> None:
-        self.view = view
-        self._values = {}  # type: typing.MutableMapping[str, mitmproxy.flow.Flow]
-        view.sig_store_remove.connect(self._sig_store_remove)
-        view.sig_store_refresh.connect(self._sig_store_refresh)
-
-    def __iter__(self) -> typing.Iterator:
-        return iter(self._values)
-
-    def __len__(self) -> int:
-        return len(self._values)
-
-    def __getitem__(self, f: mitmproxy.flow.Flow) -> dict:
-        if f.id not in self.view._store:
-            raise KeyError
-        return self._values.setdefault(f.id, {})
-
-    def _sig_store_remove(self, view, flow):
-        if flow.id in self._values:
-            del self._values[flow.id]
-
-    def _sig_store_refresh(self, view):
-        for fid in list(self._values.keys()):
-            if fid not in view._store:
-                del self._values[fid]

mitmproxy/addons/wsgiapp.py

@@ -1,38 +0,0 @@
-from mitmproxy import ctx
-from mitmproxy import exceptions
-
-from mitmproxy.net import wsgi
-from mitmproxy import version
-
-
-class WSGIApp:
-    """
-        An addon that hosts a WSGI app withing mitproxy, at a specified
-        hostname and port.
-    """
-    def __init__(self, app, host, port):
-        self.app, self.host, self.port = app, host, port
-
-    def serve(self, app, flow):
-        """
-            Serves app on flow, and prevents further handling of the flow.
-        """
-        app = wsgi.WSGIAdaptor(
-            app,
-            flow.request.pretty_host,
-            flow.request.port,
-            version.MITMPROXY
-        )
-        err = app.serve(
-            flow,
-            flow.client_conn.wfile,
-            **{"mitmproxy.master": ctx.master}
-        )
-        if err:
-            ctx.log.error("Error in wsgi app. %s" % err)
-            raise exceptions.AddonHalt()
-        flow.reply.kill()
-
-    def request(self, f):
-        if (f.request.pretty_host, f.request.port) == (self.host, self.port):
-            self.serve(self.app, f)

mitmproxy/builtins/__init__.py

@@ -0,0 +1,27 @@
+from __future__ import absolute_import, print_function, division
+
+from mitmproxy.builtins import anticache
+from mitmproxy.builtins import anticomp
+from mitmproxy.builtins import filestreamer
+from mitmproxy.builtins import stickyauth
+from mitmproxy.builtins import stickycookie
+from mitmproxy.builtins import script
+from mitmproxy.builtins import replace
+from mitmproxy.builtins import setheaders
+from mitmproxy.builtins import serverplayback
+from mitmproxy.builtins import clientplayback
+
+
+def default_addons():
+    return [
+        anticache.AntiCache(),
+        anticomp.AntiComp(),
+        stickyauth.StickyAuth(),
+        stickycookie.StickyCookie(),
+        script.ScriptLoader(),
+        filestreamer.FileStreamer(),
+        replace.Replace(),
+        setheaders.SetHeaders(),
+        serverplayback.ServerPlayback(),
+        clientplayback.ClientPlayback(),
+    ]

mitmproxy/builtins/anticache.py

@@ -1,3 +1,6 @@
+from __future__ import absolute_import, print_function, division
+
+
 class AntiCache:
     def __init__(self):
         self.enabled = False

mitmproxy/builtins/anticomp.py

@@ -1,3 +1,6 @@
+from __future__ import absolute_import, print_function, division
+
+
 class AntiComp:
     def __init__(self):
         self.enabled = False

mitmproxy/builtins/clientplayback.py

@@ -1,24 +1,19 @@
-from mitmproxy import exceptions
-from mitmproxy import ctx
-from mitmproxy import io
-from mitmproxy import flow
-
-import typing
+from mitmproxy import exceptions, flow, ctx
 
 
 class ClientPlayback:
     def __init__(self):
         self.flows = None
-        self.current_thread = None
-        self.keepserving = False
+        self.current = None
+        self.keepserving = None
         self.has_replayed = False
 
-    def count(self) -> int:
+    def count(self):
         if self.flows:
             return len(self.flows)
         return 0
 
-    def load(self, flows: typing.Sequence[flow.Flow]):
+    def load(self, flows):
         self.flows = flows
 
     def configure(self, options, updated):
@@ -26,7 +21,7 @@ class ClientPlayback:
             if options.client_replay:
                 ctx.log.info("Client Replay: {}".format(options.client_replay))
                 try:
-                    flows = io.read_flows_from_paths(options.client_replay)
+                    flows = flow.read_flows_from_paths(options.client_replay)
                 except exceptions.FlowReadException as e:
                     raise exceptions.OptionsError(str(e))
                 self.load(flows)
@@ -35,11 +30,11 @@ class ClientPlayback:
         self.keepserving = options.keepserving
 
     def tick(self):
-        if self.current_thread and not self.current_thread.is_alive():
-            self.current_thread = None
-        if self.flows and not self.current_thread:
-            self.current_thread = ctx.master.replay_request(self.flows.pop(0))
+        if self.current and not self.current.is_alive():
+            self.current = None
+        if self.flows and not self.current:
+            self.current = ctx.master.replay_request(self.flows.pop(0))
             self.has_replayed = True
         if self.has_replayed:
-            if not self.flows and not self.current_thread and not self.keepserving:
+            if not self.flows and not self.current and not self.keepserving:
                 ctx.master.shutdown()

mitmproxy/builtins/dumper.py

@@ -1,5 +1,6 @@
+from __future__ import absolute_import, print_function, division
+
 import itertools
-import sys
 
 import click
 
@@ -9,43 +10,35 @@ from mitmproxy import contentviews
 from mitmproxy import ctx
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
-from mitmproxy.utils import human
-from mitmproxy.utils import strutils
+from netlib import human
+from netlib import strutils
 
 
-def indent(n: int, text: str) -> str:
+def indent(n, text):
     l = str(text).strip().splitlines()
     pad = " " * n
     return "\n".join(pad + i for i in l)
 
 
-def colorful(line, styles):
-    yield u"    "  # we can already indent here
-    for (style, text) in line:
-        yield click.style(text, **styles.get(style, {}))
-
-
-class Dumper:
-    def __init__(self, outfile=sys.stdout):
+class Dumper(object):
+    def __init__(self):
         self.filter = None  # type: flowfilter.TFilter
         self.flow_detail = None  # type: int
-        self.outfp = outfile  # type: typing.io.TextIO
+        self.outfp = None  # type: typing.io.TextIO
         self.showhost = None  # type: bool
-        self.default_contentview = "auto"  # type: str
 
     def configure(self, options, updated):
-        if "filtstr" in updated:
-            if options.filtstr:
-                self.filter = flowfilter.parse(options.filtstr)
-                if not self.filter:
-                    raise exceptions.OptionsError(
-                        "Invalid filter expression: %s" % options.filtstr
-                    )
-            else:
-                self.filter = None
+        if options.filtstr:
+            self.filter = flowfilter.parse(options.filtstr)
+            if not self.filter:
+                raise exceptions.OptionsError(
+                    "Invalid filter expression: %s" % options.filtstr
+                )
+        else:
+            self.filter = None
         self.flow_detail = options.flow_detail
+        self.outfp = options.tfile
         self.showhost = options.showhost
-        self.default_contentview = options.default_contentview
 
     def echo(self, text, ident=None, **style):
         if ident:
@@ -54,50 +47,66 @@ class Dumper:
         if self.outfp:
             self.outfp.flush()
 
-    def _echo_headers(self, headers):
-        for k, v in headers.fields:
-            k = strutils.bytes_to_escaped_str(k)
-            v = strutils.bytes_to_escaped_str(v)
-            out = "{}: {}".format(
-                click.style(k, fg="blue"),
-                click.style(v)
-            )
-            self.echo(out, ident=4)
-
     def _echo_message(self, message):
-        _, lines, error = contentviews.get_message_content_view(
-            self.default_contentview,
-            message
-        )
-        if error:
-            ctx.log.debug(error)
+        if self.flow_detail >= 2 and hasattr(message, "headers"):
+            headers = "\r\n".join(
+                "{}: {}".format(
+                    click.style(
+                        strutils.bytes_to_escaped_str(k), fg="blue", bold=True
+                    ),
+                    click.style(
+                        strutils.bytes_to_escaped_str(v), fg="blue"
+                    )
+                )
+                for k, v in message.headers.fields
+            )
+            self.echo(headers, ident=4)
+        if self.flow_detail >= 3:
+                _, lines, error = contentviews.get_message_content_view(
+                    contentviews.get("Auto"),
+                    message
+                )
+                if error:
+                    ctx.log.debug(error)
 
-        if self.flow_detail == 3:
-            lines_to_echo = itertools.islice(lines, 70)
-        else:
-            lines_to_echo = lines
+                styles = dict(
+                    highlight=dict(bold=True),
+                    offset=dict(fg="blue"),
+                    header=dict(fg="green", bold=True),
+                    text=dict(fg="green")
+                )
 
-        styles = dict(
-            highlight=dict(bold=True),
-            offset=dict(fg="blue"),
-            header=dict(fg="green", bold=True),
-            text=dict(fg="green")
-        )
+                def colorful(line):
+                    yield u"    "  # we can already indent here
+                    for (style, text) in line:
+                        yield click.style(text, **styles.get(style, {}))
 
-        content = u"\r\n".join(
-            u"".join(colorful(line, styles)) for line in lines_to_echo
-        )
-        if content:
-            self.echo("")
-            self.echo(content)
+                if self.flow_detail == 3:
+                    lines_to_echo = itertools.islice(lines, 70)
+                else:
+                    lines_to_echo = lines
 
-        if next(lines, None):
-            self.echo("(cut off)", ident=4, dim=True)
+                content = u"\r\n".join(
+                    u"".join(colorful(line)) for line in lines_to_echo
+                )
+                if content:
+                    self.echo("")
+                    self.echo(content)
+
+                if next(lines, None):
+                    self.echo("(cut off)", ident=4, dim=True)
 
         if self.flow_detail >= 2:
             self.echo("")
 
     def _echo_request_line(self, flow):
+        if flow.request.stickycookie:
+            stickycookie = click.style(
+                "[stickycookie] ", fg="yellow", bold=True
+            )
+        else:
+            stickycookie = ""
+
         if flow.client_conn:
             client = click.style(
                 strutils.escape_control_characters(
@@ -109,8 +118,7 @@ class Dumper:
         else:
             client = ""
 
-        pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
-        method = flow.request.method + pushed
+        method = flow.request.method
         method_color = dict(
             GET="green",
             DELETE="red"
@@ -131,8 +139,15 @@ class Dumper:
             # We hide "normal" HTTP 1.
             http_version = " " + flow.request.http_version
 
-        line = "{client}: {method} {url}{http_version}".format(
+        if self.flow_detail >= 2:
+            linebreak = "\n    "
+        else:
+            linebreak = ""
+
+        line = "{client}: {linebreak}{stickycookie}{method} {url}{http_version}".format(
             client=client,
+            stickycookie=stickycookie,
+            linebreak=linebreak,
             method=method,
             url=url,
             http_version=http_version
@@ -190,17 +205,11 @@ class Dumper:
     def echo_flow(self, f):
         if f.request:
             self._echo_request_line(f)
-            if self.flow_detail >= 2:
-                self._echo_headers(f.request.headers)
-            if self.flow_detail >= 3:
-                self._echo_message(f.request)
+            self._echo_message(f.request)
 
         if f.response:
             self._echo_response_line(f)
-            if self.flow_detail >= 2:
-                self._echo_headers(f.response.headers)
-            if self.flow_detail >= 3:
-                self._echo_message(f.response)
+            self._echo_message(f.response)
 
         if f.error:
             msg = strutils.escape_control_characters(f.error.msg)
@@ -223,29 +232,6 @@ class Dumper:
         if self.match(f):
             self.echo_flow(f)
 
-    def websocket_error(self, f):
-        self.echo(
-            "Error in WebSocket connection to {}: {}".format(
-                repr(f.server_conn.address), f.error
-            ),
-            fg="red"
-        )
-
-    def websocket_message(self, f):
-        if self.match(f):
-            message = f.messages[-1]
-            self.echo(message.info)
-            if self.flow_detail >= 3:
-                self._echo_message(message)
-
-    def websocket_end(self, f):
-        if self.match(f):
-            self.echo("WebSocket connection closed by {}: {} {}, {}".format(
-                f.close_sender,
-                f.close_code,
-                f.close_message,
-                f.close_reason))
-
     def tcp_error(self, f):
         self.echo(
             "Error in TCP connection to {}: {}".format(
@@ -255,13 +241,13 @@ class Dumper:
         )
 
     def tcp_message(self, f):
-        if self.match(f):
-            message = f.messages[-1]
-            direction = "->" if message.from_client else "<-"
-            self.echo("{client} {direction} tcp {direction} {server}".format(
-                client=repr(f.client_conn.address),
-                server=repr(f.server_conn.address),
-                direction=direction,
-            ))
-            if self.flow_detail >= 3:
-                self._echo_message(message)
+        if not self.match(f):
+            return
+        message = f.messages[-1]
+        direction = "->" if message.from_client else "<-"
+        self.echo("{client} {direction} tcp {direction} {server}".format(
+            client=repr(f.client_conn.address),
+            server=repr(f.server_conn.address),
+            direction=direction,
+        ))
+        self._echo_message(message)

mitmproxy/builtins/filestreamer.py

@@ -1,45 +1,44 @@
+from __future__ import absolute_import, print_function, division
 import os.path
 
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
-from mitmproxy import io
+from mitmproxy.flow import io
 
 
-class StreamFile:
+class FileStreamer:
     def __init__(self):
         self.stream = None
-        self.filt = None
-        self.active_flows = set()  # type: Set[flow.Flow]
+        self.active_flows = set()  # type: Set[models.Flow]
 
     def start_stream_to_path(self, path, mode, flt):
         path = os.path.expanduser(path)
         try:
             f = open(path, mode)
         except IOError as v:
-            raise exceptions.OptionsError(str(v))
+            return str(v)
         self.stream = io.FilteredFlowWriter(f, flt)
         self.active_flows = set()
 
     def configure(self, options, updated):
         # We're already streaming - stop the previous stream and restart
-        if "filtstr" in updated:
-            if options.filtstr:
-                self.filt = flowfilter.parse(options.filtstr)
-                if not self.filt:
+        if self.stream:
+            self.done()
+
+        if options.outfile:
+            flt = None
+            if options.get("filtstr"):
+                flt = flowfilter.parse(options.filtstr)
+                if not flt:
                     raise exceptions.OptionsError(
                         "Invalid filter specification: %s" % options.filtstr
                     )
-            else:
-                self.filt = None
-        if "streamfile" in updated:
-            if self.stream:
-                self.done()
-            if options.streamfile:
-                if options.streamfile_append:
-                    mode = "ab"
-                else:
-                    mode = "wb"
-                self.start_stream_to_path(options.streamfile, mode, self.filt)
+            path, mode = options.outfile
+            if mode not in ("wb", "ab"):
+                raise exceptions.OptionsError("Invalid mode.")
+            err = self.start_stream_to_path(path, mode, flt)
+            if err:
+                raise exceptions.OptionsError(err)
 
     def tcp_start(self, flow):
         if self.stream:

mitmproxy/builtins/replace.py

@@ -16,22 +16,21 @@ class Replace:
             rex: a regular expression, as bytes.
             s: the replacement string, as bytes
         """
-        if "replacements" in updated:
-            lst = []
-            for fpatt, rex, s in options.replacements:
-                flt = flowfilter.parse(fpatt)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "Invalid filter pattern: %s" % fpatt
-                    )
-                try:
-                    re.compile(rex)
-                except re.error as e:
-                    raise exceptions.OptionsError(
-                        "Invalid regular expression: %s - %s" % (rex, str(e))
-                    )
-                lst.append((rex, s, flt))
-            self.lst = lst
+        lst = []
+        for fpatt, rex, s in options.replacements:
+            flt = flowfilter.parse(fpatt)
+            if not flt:
+                raise exceptions.OptionsError(
+                    "Invalid filter pattern: %s" % fpatt
+                )
+            try:
+                re.compile(rex)
+            except re.error as e:
+                raise exceptions.OptionsError(
+                    "Invalid regular expression: %s - %s" % (rex, str(e))
+                )
+            lst.append((rex, s, flt))
+        self.lst = lst
 
     def execute(self, f):
         for rex, s, flt in self.lst:

mitmproxy/builtins/script.py

@@ -1,26 +1,41 @@
+from __future__ import absolute_import, print_function, division
+
 import contextlib
 import os
 import shlex
 import sys
 import threading
 import traceback
-import types
 
 from mitmproxy import exceptions
+from mitmproxy import controller
 from mitmproxy import ctx
-from mitmproxy import events
+from mitmproxy.flow import master as flowmaster
 
 
 import watchdog.events
 from watchdog.observers import polling
 
 
+class NS:
+    def __init__(self, ns):
+        self.__dict__["ns"] = ns
+
+    def __getattr__(self, key):
+        if key not in self.ns:
+            raise AttributeError("No such element: %s", key)
+        return self.ns[key]
+
+    def __setattr__(self, key, value):
+        self.__dict__["ns"][key] = value
+
+
 def parse_command(command):
     """
         Returns a (path, args) tuple.
     """
     if not command or not command.strip():
-        raise exceptions.OptionsError("Empty script command.")
+        raise exceptions.AddonError("Empty script command.")
     # Windows: escape all backslashes in the path.
     if os.name == "nt":  # pragma: no cover
         backslashes = shlex.split(command, posix=False)[0].count("\\")
@@ -28,13 +43,13 @@ def parse_command(command):
     args = shlex.split(command)  # pragma: no cover
     args[0] = os.path.expanduser(args[0])
     if not os.path.exists(args[0]):
-        raise exceptions.OptionsError(
+        raise exceptions.AddonError(
             ("Script file not found: %s.\r\n"
              "If your script path contains spaces, "
              "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") %
             args[0])
     elif os.path.isdir(args[0]):
-        raise exceptions.OptionsError("Not a file: %s" % args[0])
+        raise exceptions.AddonError("Not a file: %s" % args[0])
     return args[0], args[1:]
 
 
@@ -101,8 +116,8 @@ def load_script(path, args):
             return
     ns = {'__file__': os.path.abspath(path)}
     with scriptenv(path, args):
-        exec(code, ns)
-    return types.SimpleNamespace(**ns)
+        exec(code, ns, ns)
+    return NS(ns)
 
 
 class ReloadHandler(watchdog.events.FileSystemEventHandler):
@@ -141,7 +156,7 @@ class Script:
         self.last_options = None
         self.should_reload = threading.Event()
 
-        for i in events.Events:
+        for i in controller.Events:
             if not hasattr(self, i):
                 def mkprox():
                     evt = i
@@ -200,7 +215,7 @@ class Script:
         self.dead = True
 
 
-class ScriptLoader:
+class ScriptLoader():
     """
         An addon that manages loading scripts from options.
     """
@@ -208,7 +223,7 @@ class ScriptLoader:
         sc = Script(command)
         sc.load_script()
         for f in flows:
-            for evt, o in events.event_sequence(f):
+            for evt, o in flowmaster.event_sequence(f):
                 sc.run(evt, o)
         sc.done()
         return sc
@@ -221,8 +236,8 @@ class ScriptLoader:
 
             for a in ctx.master.addons.chain[:]:
                 if isinstance(a, Script) and a.name not in options.scripts:
-                    ctx.log.info("Un-loading script: %s" % a.name)
-                    ctx.master.addons.remove(a)
+                        ctx.log.info("Un-loading script: %s" % a.name)
+                        ctx.master.addons.remove(a)
 
             # The machinations below are to ensure that:
             #   - Scripts remain in the same order

mitmproxy/builtins/serverplayback.py

@@ -1,14 +1,12 @@
+from __future__ import absolute_import, print_function, division
+from six.moves import urllib
 import hashlib
-import urllib
-from typing import Any  # noqa
-from typing import List  # noqa
 
-from mitmproxy import ctx
-from mitmproxy import exceptions
-from mitmproxy import io
+from netlib import strutils
+from mitmproxy import exceptions, flow, ctx
 
 
-class ServerPlayback:
+class ServerPlayback(object):
     def __init__(self):
         self.options = None
 
@@ -37,20 +35,17 @@ class ServerPlayback:
         _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
         queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
 
-        key = [str(r.port), str(r.scheme), str(r.method), str(path)]  # type: List[Any]
+        key = [str(r.port), str(r.scheme), str(r.method), str(path)]
         if not self.options.server_replay_ignore_content:
-            if self.options.server_replay_ignore_payload_params and r.multipart_form:
-                key.extend(
-                    (k, v)
-                    for k, v in r.multipart_form.items(multi=True)
-                    if k.decode(errors="replace") not in self.options.server_replay_ignore_payload_params
-                )
-            elif self.options.server_replay_ignore_payload_params and r.urlencoded_form:
-                key.extend(
-                    (k, v)
-                    for k, v in r.urlencoded_form.items(multi=True)
-                    if k not in self.options.server_replay_ignore_payload_params
-                )
+            form_contents = r.urlencoded_form or r.multipart_form
+            if self.options.server_replay_ignore_payload_params and form_contents:
+                params = [
+                    strutils.always_bytes(i)
+                    for i in self.options.server_replay_ignore_payload_params
+                ]
+                for p in form_contents.items(multi=True):
+                    if p[0] not in params:
+                        key.append(p)
             else:
                 key.append(str(r.raw_content))
 
@@ -97,7 +92,7 @@ class ServerPlayback:
             self.clear()
             if options.server_replay:
                 try:
-                    flows = io.read_flows_from_paths(options.server_replay)
+                    flows = flow.read_flows_from_paths(options.server_replay)
                 except exceptions.FlowReadException as e:
                     raise exceptions.OptionsError(str(e))
                 self.load(flows)

mitmproxy/builtins/setheaders.py

@@ -14,15 +14,13 @@ class SetHeaders:
             header: Header name.
             value: Header value string
         """
-        if "setheaders" in updated:
-            self.lst = []
-            for fpatt, header, value in options.setheaders:
-                flt = flowfilter.parse(fpatt)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "Invalid setheader filter pattern %s" % fpatt
-                    )
-                self.lst.append((fpatt, header, value, flt))
+        for fpatt, header, value in options.setheaders:
+            flt = flowfilter.parse(fpatt)
+            if not flt:
+                raise exceptions.OptionsError(
+                    "Invalid setheader filter pattern %s" % fpatt
+                )
+            self.lst.append((fpatt, header, value, flt))
 
     def run(self, f, hdrs):
         for _, header, value, flt in self.lst:

mitmproxy/builtins/stickyauth.py

@@ -0,0 +1,27 @@
+from __future__ import absolute_import, print_function, division
+
+from mitmproxy import exceptions
+from mitmproxy import flowfilter
+
+
+class StickyAuth:
+    def __init__(self):
+        self.flt = None
+        self.hosts = {}
+
+    def configure(self, options, updated):
+        if options.stickyauth:
+            flt = flowfilter.parse(options.stickyauth)
+            if not flt:
+                raise exceptions.OptionsError(
+                    "stickyauth: invalid filter expression: %s" % options.stickyauth
+                )
+            self.flt = flt
+
+    def request(self, flow):
+        host = flow.request.host
+        if "authorization" in flow.request.headers:
+            self.hosts[host] = flow.request.headers["authorization"]
+        elif flowfilter.match(self.flt, flow):
+            if host in self.hosts:
+                flow.request.headers["authorization"] = self.hosts[host]

mitmproxy/builtins/stickycookie.py

@@ -1,7 +1,6 @@
 import collections
-from http import cookiejar
-
-from mitmproxy.net.http import cookies
+from six.moves import http_cookiejar
+from netlib.http import cookies
 
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
@@ -21,9 +20,9 @@ def ckey(attrs, f):
 
 
 def domain_match(a, b):
-    if cookiejar.domain_match(a, b):
+    if http_cookiejar.domain_match(a, b):
         return True
-    elif cookiejar.domain_match(a, b.strip(".")):
+    elif http_cookiejar.domain_match(a, b.strip(".")):
         return True
     return False
 
@@ -34,16 +33,13 @@ class StickyCookie:
         self.flt = None
 
     def configure(self, options, updated):
-        if "stickycookie" in updated:
-            if options.stickycookie:
-                flt = flowfilter.parse(options.stickycookie)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "stickycookie: invalid filter expression: %s" % options.stickycookie
-                    )
-                self.flt = flt
-            else:
-                self.flt = None
+        if options.stickycookie:
+            flt = flowfilter.parse(options.stickycookie)
+            if not flt:
+                raise exceptions.OptionsError(
+                    "stickycookie: invalid filter expression: %s" % options.stickycookie
+                )
+            self.flt = flt
 
     def response(self, flow):
         if self.flt:
@@ -62,8 +58,7 @@ class StickyCookie:
                         if not self.jar[dom_port_path]:
                             self.jar.pop(dom_port_path, None)
                     else:
-                        b = attrs.copy()
-                        b.insert(0, name, value)
+                        b = attrs.with_insert(0, name, value)
                         self.jar[dom_port_path][name] = b
 
     def request(self, flow):

mitmproxy/builtins/termlog.py

@@ -1,22 +1,22 @@
-import sys
+from __future__ import absolute_import, print_function, division
+
 import click
 
-from mitmproxy import log
+from mitmproxy import utils
 
 
 class TermLog:
-    def __init__(self, outfile=sys.stdout):
+    def __init__(self):
         self.options = None
-        self.outfile = outfile
 
     def configure(self, options, updated):
         self.options = options
 
     def log(self, e):
-        if self.options.verbosity >= log.log_tier(e.level):
+        if self.options.verbosity >= utils.log_tier(e.level):
             click.secho(
                 e.msg,
-                file=self.outfile,
+                file=self.options.tfile,
                 fg=dict(error="red", warn="yellow").get(e.level),
                 dim=(e.level == "debug"),
                 err=(e.level == "error")

mitmproxy/cmdline.py

@@ -1,18 +1,15 @@
-import argparse
-import re
-import os
+from __future__ import absolute_import, print_function, division
 
+import configargparse
+import os
+import re
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
 from mitmproxy import options
 from mitmproxy import platform
-from mitmproxy.utils import human
-from mitmproxy.net import tcp
-from mitmproxy import version
-from mitmproxy.addons import view
-
-
-CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml")
+from netlib import human
+from netlib import tcp
+from netlib import version
 
 
 class ParseException(Exception):
@@ -117,13 +114,13 @@ def get_common_options(args):
         stream_large_bodies = human.parse_size(stream_large_bodies)
 
     reps = []
-    for i in args.replace or []:
+    for i in args.replace:
         try:
             p = parse_replace_hook(i)
         except ParseException as e:
             raise exceptions.OptionsError(e)
         reps.append(p)
-    for i in args.replace_file or []:
+    for i in args.replace_file:
         try:
             patt, rex, path = parse_replace_hook(i)
         except ParseException as e:
@@ -137,15 +134,15 @@ def get_common_options(args):
         reps.append((patt, rex, v))
 
     setheaders = []
-    for i in args.setheader or []:
+    for i in args.setheader:
         try:
             p = parse_setheader(i)
         except ParseException as e:
             raise exceptions.OptionsError(e)
         setheaders.append(p)
 
-    if args.streamfile and args.streamfile[0] == args.rfile:
-        if args.streamfile[1] == "wb":
+    if args.outfile and args.outfile[0] == args.rfile:
+        if args.outfile[1] == "wb":
             raise exceptions.OptionsError(
                 "Cannot use '{}' for both reading and writing flows. "
                 "Are you looking for --afile?".format(args.rfile)
@@ -158,7 +155,7 @@ def get_common_options(args):
 
     # Proxy config
     certs = []
-    for i in args.certs or []:
+    for i in args.certs:
         parts = i.split("=", 1)
         if len(parts) == 1:
             parts = ["*", parts[0]]
@@ -178,7 +175,7 @@ def get_common_options(args):
     mode, upstream_server = "regular", None
     if args.transparent_proxy:
         c += 1
-        if not platform.original_addr:
+        if not platform.resolver:
             raise exceptions.OptionsError(
                 "Transparent mode not supported on this platform."
             )
@@ -232,8 +229,7 @@ def get_common_options(args):
         stickyauth=stickyauth,
         stream_large_bodies=stream_large_bodies,
         showhost=args.showhost,
-        streamfile=args.streamfile[0] if args.streamfile else None,
-        streamfile_append=True if args.streamfile and args.streamfile[1] == "a" else False,
+        outfile=args.outfile,
         verbosity=args.verbose,
         server_replay_nopop=args.server_replay_nopop,
         server_replay_ignore_content=args.server_replay_ignore_content,
@@ -255,12 +251,11 @@ def get_common_options(args):
         ignore_hosts = args.ignore_hosts,
         listen_host = args.addr,
         listen_port = args.port,
-        upstream_bind_address = args.upstream_bind_address,
         mode = mode,
         no_upstream_cert = args.no_upstream_cert,
         spoof_source_address = args.spoof_source_address,
         rawtcp = args.rawtcp,
-        websocket = args.websocket,
+        websockets = args.websockets,
         upstream_server = upstream_server,
         upstream_auth = args.upstream_auth,
         ssl_version_client = args.ssl_version_client,
@@ -291,7 +286,8 @@ def basic_options(parser):
     )
     parser.add_argument(
         "--anticache",
-        action="store_true", dest="anticache",
+        action="store_true", dest="anticache", default=False,
+
         help="""
             Strip out request headers that might cause the server to return
             304-not-modified.
@@ -299,12 +295,12 @@ def basic_options(parser):
     )
     parser.add_argument(
         "--cadir",
-        action="store", type=str, dest="cadir",
+        action="store", type=str, dest="cadir", default=options.CA_DIR,
         help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR
     )
     parser.add_argument(
         "--host",
-        action="store_true", dest="showhost",
+        action="store_true", dest="showhost", default=False,
         help="Use the Host header to construct URLs for display."
     )
     parser.add_argument(
@@ -314,12 +310,12 @@ def basic_options(parser):
     )
     parser.add_argument(
         "-r", "--read-flows",
-        action="store", dest="rfile",
+        action="store", dest="rfile", default=None,
         help="Read flows from file."
     )
     parser.add_argument(
         "-s", "--script",
-        action="append", type=str, dest="scripts",
+        action="append", type=str, dest="scripts", default=[],
         metavar='"script.py --bar"',
         help="""
             Run a script. Surround with quotes to pass script arguments. Can be
@@ -330,45 +326,46 @@ def basic_options(parser):
         "-t", "--stickycookie",
         action="store",
         dest="stickycookie_filt",
+        default=None,
         metavar="FILTER",
         help="Set sticky cookie filter. Matched against requests."
     )
     parser.add_argument(
         "-u", "--stickyauth",
-        action="store", dest="stickyauth_filt", metavar="FILTER",
+        action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
         help="Set sticky auth filter. Matched against requests."
     )
     parser.add_argument(
         "-v", "--verbose",
-        action="store_const", dest="verbose", const=3,
+        action="store_const", dest="verbose", default=2, const=3,
         help="Increase log verbosity."
     )
-    streamfile = parser.add_mutually_exclusive_group()
-    streamfile.add_argument(
+    outfile = parser.add_mutually_exclusive_group()
+    outfile.add_argument(
         "-w", "--wfile",
-        action="store", dest="streamfile", type=lambda f: (f, "w"),
+        action="store", dest="outfile", type=lambda f: (f, "wb"),
         help="Write flows to file."
     )
-    streamfile.add_argument(
+    outfile.add_argument(
         "-a", "--afile",
-        action="store", dest="streamfile", type=lambda f: (f, "a"),
+        action="store", dest="outfile", type=lambda f: (f, "ab"),
         help="Append flows to file."
     )
     parser.add_argument(
         "-z", "--anticomp",
-        action="store_true", dest="anticomp",
+        action="store_true", dest="anticomp", default=False,
         help="Try to convince servers to send us un-compressed data."
     )
     parser.add_argument(
         "-Z", "--body-size-limit",
-        action="store", dest="body_size_limit",
+        action="store", dest="body_size_limit", default=None,
         metavar="SIZE",
         help="Byte size limit of HTTP request and response bodies."
              " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
     parser.add_argument(
         "--stream",
-        action="store", dest="stream_large_bodies",
+        action="store", dest="stream_large_bodies", default=None,
         metavar="SIZE",
         help="""
             Stream data to the client if response body exceeds the given
@@ -385,6 +382,7 @@ def proxy_modes(parser):
         action="store",
         type=str,
         dest="reverse_proxy",
+        default=None,
         help="""
             Forward all requests to upstream HTTP server:
             http[s]://host[:port]. Clients can always connect both
@@ -394,12 +392,12 @@ def proxy_modes(parser):
     )
     group.add_argument(
         "--socks",
-        action="store_true", dest="socks_proxy",
+        action="store_true", dest="socks_proxy", default=False,
         help="Set SOCKS5 proxy mode."
     )
     group.add_argument(
         "-T", "--transparent",
-        action="store_true", dest="transparent_proxy",
+        action="store_true", dest="transparent_proxy", default=False,
         help="Set transparent proxy mode."
     )
     group.add_argument(
@@ -407,6 +405,7 @@ def proxy_modes(parser):
         action="store",
         type=str,
         dest="upstream_proxy",
+        default=None,
         help="Forward all requests to upstream proxy server: http://host[:port]"
     )
 
@@ -415,12 +414,12 @@ def proxy_options(parser):
     group = parser.add_argument_group("Proxy Options")
     group.add_argument(
         "-b", "--bind-address",
-        action="store", type=str, dest="addr",
+        action="store", type=str, dest="addr", default='',
         help="Address to bind proxy to (defaults to all interfaces)"
     )
     group.add_argument(
         "-I", "--ignore",
-        action="append", type=str, dest="ignore_hosts",
+        action="append", type=str, dest="ignore_hosts", default=[],
         metavar="HOST",
         help="""
             Ignore host and forward all traffic without processing it. In
@@ -433,7 +432,7 @@ def proxy_options(parser):
     )
     group.add_argument(
         "--tcp",
-        action="append", type=str, dest="tcp_hosts",
+        action="append", type=str, dest="tcp_hosts", default=[],
         metavar="HOST",
         help="""
             Generic TCP SSL proxy mode for all hosts that match the pattern.
@@ -448,35 +447,26 @@ def proxy_options(parser):
     )
     group.add_argument(
         "-p", "--port",
-        action="store", type=int, dest="port",
+        action="store", type=int, dest="port", default=options.LISTEN_PORT,
         help="Proxy service port."
     )
-
-    http2 = group.add_mutually_exclusive_group()
-    http2.add_argument("--http2", action="store_true", dest="http2")
-    http2.add_argument("--no-http2", action="store_false", dest="http2",
-                       help="Explicitly enable/disable HTTP/2 support. "
-                            "Disabled by default until major websites implement the spec correctly. "
-                            "Default value will change in a future version."
-                       )
-
-    websocket = group.add_mutually_exclusive_group()
-    websocket.add_argument("--no-websocket", action="store_false", dest="websocket",
-                           help="Explicitly enable/disable WebSocket support. "
-                                "Enabled by default."
-                           )
-    websocket.add_argument("--websocket", action="store_true", dest="websocket")
-
-    parser.add_argument(
-        "--upstream-auth",
-        action="store", dest="upstream_auth",
-        type=str,
+    group.add_argument(
+        "--no-http2",
+        action="store_false", dest="http2",
         help="""
-            Add HTTP Basic authentcation to upstream proxy and reverse proxy
-            requests. Format: username:password
+            Explicitly disable HTTP/2 support.
+            If your OpenSSL version supports ALPN, HTTP/2 is enabled by default.
+        """
+    )
+    parser.add_argument(
+        "--upstream-auth",
+        action="store", dest="upstream_auth", default=None,
+        type=str,
+        help="""
+            Proxy Authentication:
+            username:password
         """
     )
-
     rawtcp = group.add_mutually_exclusive_group()
     rawtcp.add_argument("--raw-tcp", action="store_true", dest="rawtcp")
     rawtcp.add_argument("--no-raw-tcp", action="store_false", dest="rawtcp",
@@ -484,17 +474,17 @@ def proxy_options(parser):
                         "Disabled by default. "
                         "Default value will change in a future version."
                         )
-
+    websockets = group.add_mutually_exclusive_group()
+    websockets.add_argument("--websockets", action="store_true", dest="websockets")
+    websockets.add_argument("--no-websockets", action="store_false", dest="websockets",
+                            help="Explicitly enable/disable experimental WebSocket support. "
+                                 "Disabled by default as messages are only printed to the event log and not retained. "
+                                 "Default value will change in a future version."
+                            )
     group.add_argument(
         "--spoof-source-address",
         action="store_true", dest="spoof_source_address",
-        help="Use the client's IP for server-side connections. "
-             "Combine with --upstream-bind-address to spoof a fixed source address."
-    )
-    group.add_argument(
-        "--upstream-bind-address",
-        action="store", type=str, dest="upstream_bind_address",
-        help="Address to bind upstream requests to (defaults to none)"
+        help="Use the client's IP for server-side connections"
     )
 
 
@@ -504,6 +494,7 @@ def proxy_ssl_options(parser):
     group.add_argument(
         "--cert",
         dest='certs',
+        default=[],
         type=str,
         metavar="SPEC",
         action="append",
@@ -515,56 +506,56 @@ def proxy_ssl_options(parser):
              'as the first entry. Can be passed multiple times.')
     group.add_argument(
         "--ciphers-client", action="store",
-        type=str, dest="ciphers_client",
+        type=str, dest="ciphers_client", default=options.DEFAULT_CLIENT_CIPHERS,
         help="Set supported ciphers for client connections. (OpenSSL Syntax)"
     )
     group.add_argument(
         "--ciphers-server", action="store",
-        type=str, dest="ciphers_server",
+        type=str, dest="ciphers_server", default=None,
         help="Set supported ciphers for server connections. (OpenSSL Syntax)"
     )
     group.add_argument(
         "--client-certs", action="store",
-        type=str, dest="clientcerts",
+        type=str, dest="clientcerts", default=None,
         help="Client certificate file or directory."
     )
     group.add_argument(
-        "--no-upstream-cert",
+        "--no-upstream-cert", default=False,
         action="store_true", dest="no_upstream_cert",
         help="Don't connect to upstream server to look up certificate details."
     )
     group.add_argument(
-        "--add-upstream-certs-to-client-chain",
+        "--add-upstream-certs-to-client-chain", default=False,
         action="store_true", dest="add_upstream_certs_to_client_chain",
         help="Add all certificates of the upstream server to the certificate chain "
              "that will be served to the proxy client, as extras."
     )
     group.add_argument(
-        "--insecure",
+        "--insecure", default=False,
         action="store_true", dest="ssl_insecure",
         help="Do not verify upstream server SSL/TLS certificates."
     )
     group.add_argument(
-        "--upstream-trusted-cadir", action="store",
+        "--upstream-trusted-cadir", default=None, action="store",
         dest="ssl_verify_upstream_trusted_cadir",
         help="Path to a directory of trusted CA certificates for upstream "
              "server verification prepared using the c_rehash tool."
     )
     group.add_argument(
-        "--upstream-trusted-ca", action="store",
+        "--upstream-trusted-ca", default=None, action="store",
         dest="ssl_verify_upstream_trusted_ca",
         help="Path to a PEM formatted trusted CA certificate."
     )
     group.add_argument(
         "--ssl-version-client", dest="ssl_version_client",
-        action="store",
+        default="secure", action="store",
         choices=tcp.sslversion_choices.keys(),
         help="Set supported SSL/TLS versions for client connections. "
              "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
     )
     group.add_argument(
         "--ssl-version-server", dest="ssl_version_server",
-        action="store",
+        default="secure", action="store",
         choices=tcp.sslversion_choices.keys(),
         help="Set supported SSL/TLS versions for server connections. "
              "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
@@ -575,12 +566,12 @@ def onboarding_app(parser):
     group = parser.add_argument_group("Onboarding App")
     group.add_argument(
         "--noapp",
-        action="store_false", dest="app",
+        action="store_false", dest="app", default=True,
         help="Disable the mitmproxy onboarding app."
     )
     group.add_argument(
         "--app-host",
-        action="store", dest="app_host",
+        action="store", dest="app_host", default=options.APP_HOST, metavar="host",
         help="""
             Domain to serve the onboarding app from. For transparent mode, use
             an IP when a DNS entry for the app domain is not present. Default:
@@ -591,6 +582,7 @@ def onboarding_app(parser):
         "--app-port",
         action="store",
         dest="app_port",
+        default=options.APP_PORT,
         type=int,
         metavar="80",
         help="Port to serve the onboarding app from."
@@ -601,7 +593,7 @@ def client_replay(parser):
     group = parser.add_argument_group("Client Replay")
     group.add_argument(
         "-c", "--client-replay",
-        action="append", dest="client_replay", metavar="PATH",
+        action="append", dest="client_replay", default=None, metavar="PATH",
         help="Replay client requests from a saved file."
     )
 
@@ -610,12 +602,12 @@ def server_replay(parser):
     group = parser.add_argument_group("Server Replay")
     group.add_argument(
         "-S", "--server-replay",
-        action="append", dest="server_replay", metavar="PATH",
+        action="append", dest="server_replay", default=None, metavar="PATH",
         help="Replay server responses from a saved file."
     )
     group.add_argument(
         "-k", "--replay-kill-extra",
-        action="store_true", dest="replay_kill_extra",
+        action="store_true", dest="replay_kill_extra", default=False,
         help="Kill extra requests during replay."
     )
     group.add_argument(
@@ -626,7 +618,7 @@ def server_replay(parser):
     )
     group.add_argument(
         "--norefresh",
-        action="store_true", dest="norefresh",
+        action="store_true", dest="norefresh", default=False,
         help="""
             Disable response refresh, which updates times in cookies and headers
             for replayed responses.
@@ -634,14 +626,14 @@ def server_replay(parser):
     )
     group.add_argument(
         "--no-pop",
-        action="store_true", dest="server_replay_nopop",
+        action="store_true", dest="server_replay_nopop", default=False,
         help="Disable response pop from response flow. "
              "This makes it possible to replay same response multiple times."
     )
     payload = group.add_mutually_exclusive_group()
     payload.add_argument(
         "--replay-ignore-content",
-        action="store_true", dest="server_replay_ignore_content",
+        action="store_true", dest="server_replay_ignore_content", default=False,
         help="""
             Ignore request's content while searching for a saved flow to replay
         """
@@ -668,6 +660,7 @@ def server_replay(parser):
         "--replay-ignore-host",
         action="store_true",
         dest="server_replay_ignore_host",
+        default=False,
         help="Ignore request's destination host while searching for a saved flow to replay")
 
 
@@ -682,13 +675,13 @@ def replacements(parser):
     )
     group.add_argument(
         "--replace",
-        action="append", type=str, dest="replace",
+        action="append", type=str, dest="replace", default=[],
         metavar="PATTERN",
         help="Replacement pattern."
     )
     group.add_argument(
         "--replace-from-file",
-        action="append", type=str, dest="replace_file",
+        action="append", type=str, dest="replace_file", default=[],
         metavar="PATH",
         help="""
             Replacement pattern, where the replacement clause is a path to a
@@ -708,7 +701,7 @@ def set_headers(parser):
     )
     group.add_argument(
         "--setheader",
-        action="append", type=str, dest="setheader",
+        action="append", type=str, dest="setheader", default=[],
         metavar="PATTERN",
         help="Header set pattern."
     )
@@ -746,15 +739,6 @@ def proxy_authentication(parser):
 
 
 def common_options(parser):
-    parser.add_argument(
-        "--conf",
-        type=str, dest="conf", default=CONFIG_PATH,
-        metavar="PATH",
-        help="""
-            Configuration file
-        """
-    )
-
     basic_options(parser)
     proxy_modes(parser)
     proxy_options(parser)
@@ -768,21 +752,30 @@ def common_options(parser):
 
 
 def mitmproxy():
-    # Don't import mitmproxy.tools.console for mitmdump, urwid is not available on all
+    # Don't import mitmproxy.console for mitmdump, urwid is not available on all
     # platforms.
     from .console import palettes
 
-    parser = argparse.ArgumentParser(usage="%(prog)s [options]")
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options]",
+        args_for_setting_config_path=["--conf"],
+        default_config_files=[
+            os.path.join(options.CA_DIR, "common.conf"),
+            os.path.join(options.CA_DIR, "mitmproxy.conf")
+        ],
+        add_config_file_help=True,
+        add_env_var_help=True
+    )
     common_options(parser)
     parser.add_argument(
-        "--palette", type=str,
+        "--palette", type=str, default=palettes.DEFAULT,
         action="store", dest="palette",
         choices=sorted(palettes.palettes.keys()),
         help="Select color palette: " + ", ".join(palettes.palettes.keys())
     )
     parser.add_argument(
         "--palette-transparent",
-        action="store_true", dest="palette_transparent",
+        action="store_true", dest="palette_transparent", default=False,
         help="Set transparent background for palette."
     )
     parser.add_argument(
@@ -792,14 +785,8 @@ def mitmproxy():
     )
     parser.add_argument(
         "--follow",
-        action="store_true", dest="focus_follow",
-        help="Focus follows new flows."
-    )
-    parser.add_argument(
-        "--order",
-        type=str, dest="order",
-        choices=[o[1] for o in view.orders],
-        help="Flow sort order."
+        action="store_true", dest="follow",
+        help="Follow flow list."
     )
     parser.add_argument(
         "--no-mouse",
@@ -812,24 +799,33 @@ def mitmproxy():
     )
     group.add_argument(
         "-i", "--intercept", action="store",
-        type=str, dest="intercept",
+        type=str, dest="intercept", default=None,
         help="Intercept filter expression."
     )
     group.add_argument(
         "-f", "--filter", action="store",
-        type=str, dest="filter",
+        type=str, dest="filter", default=None,
         help="Filter view expression."
     )
     return parser
 
 
 def mitmdump():
-    parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]")
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options] [filter]",
+        args_for_setting_config_path=["--conf"],
+        default_config_files=[
+            os.path.join(options.CA_DIR, "common.conf"),
+            os.path.join(options.CA_DIR, "mitmdump.conf")
+        ],
+        add_config_file_help=True,
+        add_env_var_help=True
+    )
 
     common_options(parser)
     parser.add_argument(
         "--keepserving",
-        action="store_true", dest="keepserving",
+        action="store_true", dest="keepserving", default=False,
         help="""
             Continue serving after client playback or file read. We exit by
             default.
@@ -837,38 +833,35 @@ def mitmdump():
     )
     parser.add_argument(
         "-d", "--detail",
-        action="count", dest="flow_detail",
+        action="count", dest="flow_detail", default=1,
         help="Increase flow detail display level. Can be passed multiple times."
     )
-    parser.add_argument(
-        'filter',
-        nargs="...",
-        help="""
-            Filter view expression, used to only show flows that match a certain filter.
-            See help in mitmproxy for filter expression syntax.
-        """
-    )
+    parser.add_argument('args', nargs="...")
     return parser
 
 
 def mitmweb():
-    parser = argparse.ArgumentParser(usage="%(prog)s [options]")
+    parser = configargparse.ArgumentParser(
+        usage="%(prog)s [options]",
+        args_for_setting_config_path=["--conf"],
+        default_config_files=[
+            os.path.join(options.CA_DIR, "common.conf"),
+            os.path.join(options.CA_DIR, "mitmweb.conf")
+        ],
+        add_config_file_help=True,
+        add_env_var_help=True
+    )
 
     group = parser.add_argument_group("Mitmweb")
-    group.add_argument(
-        "--no-browser",
-        action="store_false", dest="open_browser",
-        help="Don't start a browser"
-    )
     group.add_argument(
         "--wport",
-        action="store", type=int, dest="wport",
+        action="store", type=int, dest="wport", default=8081,
         metavar="PORT",
         help="Mitmweb port."
     )
     group.add_argument(
         "--wiface",
-        action="store", dest="wiface",
+        action="store", dest="wiface", default="127.0.0.1",
         metavar="IFACE",
         help="Mitmweb interface."
     )
@@ -877,6 +870,21 @@ def mitmweb():
         action="store_true", dest="wdebug",
         help="Turn on mitmweb debugging"
     )
+    group.add_argument(
+        "--wsingleuser",
+        action="store", dest="wsingleuser", type=str,
+        metavar="USER",
+        help="""
+            Allows access to a a single user, specified in the form
+            username:password.
+        """
+    )
+    group.add_argument(
+        "--whtpasswd",
+        action="store", dest="whtpasswd", type=str,
+        metavar="PATH",
+        help="Allow access to users specified in an Apache htpasswd file."
+    )
 
     common_options(parser)
     group = parser.add_argument_group(
@@ -885,7 +893,7 @@ def mitmweb():
     )
     group.add_argument(
         "-i", "--intercept", action="store",
-        type=str, dest="intercept",
+        type=str, dest="intercept", default=None,
         help="Intercept filter expression."
     )
     return parser

mitmproxy/console/__init__.py

@@ -0,0 +1,4 @@
+from mitmproxy.console import master
+
+
+__all__ = ["master"]

mitmproxy/console/common.py

@@ -1,16 +1,18 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import, print_function, division
 
 import os
 
 import urwid
 import urwid.util
+import six
 
-import mitmproxy.net
-from functools import lru_cache
-from mitmproxy.tools.console import signals
-from mitmproxy import export
-from mitmproxy.utils import human
+import netlib
+from mitmproxy import utils
+from mitmproxy.console import signals
+from mitmproxy.flow import export
+from netlib import human
 
 try:
     import pyperclip
@@ -37,7 +39,7 @@ def is_keypress(k):
     """
         Is this input event a keypress?
     """
-    if isinstance(k, str):
+    if isinstance(k, six.string_types):
         return True
 
 
@@ -108,7 +110,7 @@ def shortcuts(k):
 
 
 def fcol(s, attr):
-    s = str(s)
+    s = six.text_type(s)
     return (
         "fixed",
         len(s),
@@ -119,19 +121,14 @@ def fcol(s, attr):
         )
     )
 
-
 if urwid.util.detected_encoding:
     SYMBOL_REPLAY = u"\u21ba"
     SYMBOL_RETURN = u"\u2190"
     SYMBOL_MARK = u"\u25cf"
-    SYMBOL_UP = u"\u21E7"
-    SYMBOL_DOWN = u"\u21E9"
 else:
     SYMBOL_REPLAY = u"[r]"
     SYMBOL_RETURN = u"<-"
     SYMBOL_MARK = "[m]"
-    SYMBOL_UP = "^"
-    SYMBOL_DOWN = " "
 
 
 # Save file to disk
@@ -150,6 +147,9 @@ def save_data(path, data):
 
 
 def ask_save_overwrite(path, data):
+    if not path:
+        return
+    path = os.path.expanduser(path)
     if os.path.exists(path):
         def save_overwrite(k):
             if k == "y":
@@ -228,7 +228,7 @@ def format_flow_data(key, scope, flow):
         if request.content is None:
             return None, "Request content is missing"
         if key == "h":
-            data += mitmproxy.net.http.http1.assemble_request(request)
+            data += netlib.http.http1.assemble_request(request)
         elif key == "c":
             data += request.get_content(strict=False)
         else:
@@ -242,7 +242,7 @@ def format_flow_data(key, scope, flow):
         if response.content is None:
             return None, "Response content is missing"
         if key == "h":
-            data += mitmproxy.net.http.http1.assemble_response(response)
+            data += netlib.http.http1.assemble_response(response)
         elif key == "c":
             data += response.get_content(strict=False)
         else:
@@ -327,9 +327,10 @@ def export_to_clip_or_file(key, scope, flow, writer):
             else:  # other keys
                 writer(exporter(flow))
 
+flowcache = utils.LRUCache(800)
 
-@lru_cache(maxsize=800)
-def raw_format_flow(f, flow):
+
+def raw_format_flow(f):
     f = dict(f)
     pile = []
     req = []
@@ -348,9 +349,7 @@ def raw_format_flow(f, flow):
 
     if f["req_is_replay"]:
         req.append(fcol(SYMBOL_REPLAY, "replay"))
-
-    pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
-    req.append(fcol(f["req_method"] + pushed, "method"))
+    req.append(fcol(f["req_method"], "method"))
 
     preamble = sum(i[1] for i in req) + len(req) - 1
 
@@ -455,11 +454,10 @@ def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False):
             resp_clen = contentdesc,
             roundtrip = roundtrip,
         ))
-
         t = f.response.headers.get("content-type")
         if t:
             d["resp_ctype"] = t.split(";")[0]
         else:
             d["resp_ctype"] = ""
 
-    return raw_format_flow(tuple(sorted(d.items())), f)
+    return flowcache.get(raw_format_flow, tuple(sorted(d.items())))

mitmproxy/console/flowdetailview.py

@@ -1,8 +1,9 @@
+from __future__ import absolute_import, print_function, division
+
 import urwid
 
-from mitmproxy.tools.console import common, searchable
-from mitmproxy.utils import human
-from mitmproxy.utils import strutils
+from mitmproxy.console import common, searchable
+from netlib import human
 
 
 def maybe_timestamp(base, attr):
@@ -15,16 +16,10 @@ def maybe_timestamp(base, attr):
 def flowdetails(state, flow):
     text = []
 
-    sc = flow.server_conn
     cc = flow.client_conn
+    sc = flow.server_conn
     req = flow.request
     resp = flow.response
-    metadata = flow.metadata
-
-    if metadata is not None and len(metadata.items()) > 0:
-        parts = [[str(k), repr(v)] for k, v in metadata.items()]
-        text.append(urwid.Text([("head", "Metadata:")]))
-        text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
 
     if sc is not None:
         text.append(urwid.Text([("head", "Server Connection:")]))
@@ -32,8 +27,6 @@ def flowdetails(state, flow):
             ["Address", repr(sc.address)],
             ["Resolved Address", repr(sc.ip_address)],
         ]
-        if sc.alpn_proto_negotiated:
-            parts.append(["ALPN", sc.alpn_proto_negotiated])
 
         text.extend(
             common.format_keyvals(parts, key="key", val="text", indent=4)
@@ -78,7 +71,7 @@ def flowdetails(state, flow):
                 parts.append(
                     [
                         "Alt names",
-                        ", ".join(strutils.bytes_to_escaped_str(x) for x in c.altnames)
+                        ", ".join(str(x) for x in c.altnames)
                     ]
                 )
             text.extend(
@@ -91,14 +84,6 @@ def flowdetails(state, flow):
         parts = [
             ["Address", repr(cc.address)],
         ]
-        if cc.tls_version:
-            parts.append(["TLS Version", cc.tls_version])
-        if cc.sni:
-            parts.append(["Server Name Indication", cc.sni])
-        if cc.cipher_name:
-            parts.append(["Cipher Name", cc.cipher_name])
-        if cc.alpn_proto_negotiated:
-            parts.append(["ALPN", cc.alpn_proto_negotiated])
 
         text.extend(
             common.format_keyvals(parts, key="key", val="text", indent=4)
@@ -120,7 +105,6 @@ def flowdetails(state, flow):
                     maybe_timestamp(cc, "timestamp_ssl_setup")
                 ]
             )
-
     if sc is not None and sc.timestamp_start:
         parts.append(
             [
@@ -141,7 +125,6 @@ def flowdetails(state, flow):
                     maybe_timestamp(sc, "timestamp_ssl_setup")
                 ]
             )
-
     if req is not None and req.timestamp_start:
         parts.append(
             [
@@ -155,7 +138,6 @@ def flowdetails(state, flow):
                 maybe_timestamp(req, "timestamp_end")
             ]
         )
-
     if resp is not None and resp.timestamp_start:
         parts.append(
             [
@@ -176,5 +158,4 @@ def flowdetails(state, flow):
 
         text.append(urwid.Text([("head", "Timing:")]))
         text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
-
     return searchable.Searchable(state, text)

mitmproxy/console/flowlist.py

@@ -1,11 +1,12 @@
+from __future__ import absolute_import, print_function, division
+
 import urwid
 
-import mitmproxy.net.http.url
+import netlib.http.url
 from mitmproxy import exceptions
-from mitmproxy.tools.console import common
-from mitmproxy.tools.console import signals
-from mitmproxy.addons import view
-from mitmproxy import export
+from mitmproxy.console import common
+from mitmproxy.console import signals
+from mitmproxy.flow import export
 
 
 def _mkhelp():
@@ -25,11 +26,9 @@ def _mkhelp():
         ("m", "toggle flow mark"),
         ("M", "toggle marked flow view"),
         ("n", "create a new request"),
-        ("o", "set flow order"),
         ("r", "replay request"),
         ("S", "server replay request/s"),
         ("U", "unmark all marked flows"),
-        ("v", "reverse flow order"),
         ("V", "revert changes to request"),
         ("w", "save flows "),
         ("W", "stream flows to file"),
@@ -41,8 +40,6 @@ def _mkhelp():
     ]
     text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
     return text
-
-
 help_context = _mkhelp()
 
 footer = [
@@ -65,9 +62,6 @@ class LogBufferBox(urwid.ListBox):
             self.set_focus(len(self.master.logbuffer) - 1)
         elif key == "g":
             self.set_focus(0)
-        elif key == "F":
-            o = self.master.options
-            o.focus_follow = not o.focus_follow
         return urwid.ListBox.keypress(self, size, key)
 
 
@@ -115,10 +109,11 @@ class BodyPile(urwid.Pile):
         return self.focus_item.keypress(tsize, key)
 
 
-class FlowItem(urwid.WidgetWrap):
+class ConnectionItem(urwid.WidgetWrap):
 
-    def __init__(self, master, flow):
-        self.master, self.flow = master, flow
+    def __init__(self, master, state, flow, focus):
+        self.master, self.state, self.flow = master, state, flow
+        self.f = focus
         w = self.get_text()
         urwid.WidgetWrap.__init__(self, w)
 
@@ -126,7 +121,7 @@ class FlowItem(urwid.WidgetWrap):
         cols, _ = self.master.ui.get_cols_rows()
         return common.format_flow(
             self.flow,
-            self.flow is self.master.view.focus.flow,
+            self.f,
             hostheader=self.master.options.showhost,
             max_url_len=cols,
         )
@@ -150,7 +145,7 @@ class FlowItem(urwid.WidgetWrap):
     def server_replay_prompt(self, k):
         a = self.master.addons.get("serverplayback")
         if k == "a":
-            a.load([i.copy() for i in self.master.view])
+            a.load([i.copy() for i in self.master.state.view])
         elif k == "t":
             a.load([self.flow.copy()])
         signals.update_settings.send(self)
@@ -165,19 +160,26 @@ class FlowItem(urwid.WidgetWrap):
         (maxcol,) = xxx_todo_changeme
         key = common.shortcuts(key)
         if key == "a":
-            self.flow.resume()
-            self.master.view.update(self.flow)
+            self.flow.accept_intercept(self.master)
+            signals.flowlist_change.send(self)
         elif key == "d":
             if self.flow.killable:
-                self.flow.kill()
-            self.master.view.remove(self.flow)
+                self.flow.kill(self.master)
+            self.state.delete_flow(self.flow)
+            signals.flowlist_change.send(self)
         elif key == "D":
-            cp = self.flow.copy()
-            self.master.view.add(cp)
-            self.master.view.focus.flow = cp
+            f = self.master.duplicate_flow(self.flow)
+            self.master.state.set_focus_flow(f)
+            signals.flowlist_change.send(self)
         elif key == "m":
             self.flow.marked = not self.flow.marked
             signals.flowlist_change.send(self)
+        elif key == "M":
+            if self.state.mark_filter:
+                self.state.disable_marked_filter()
+            else:
+                self.state.enable_marked_filter()
+            signals.flowlist_change.send(self)
         elif key == "r":
             try:
                 self.master.replay_request(self.flow)
@@ -208,14 +210,14 @@ class FlowItem(urwid.WidgetWrap):
                     callback = self.server_replay_prompt,
                 )
         elif key == "U":
-            for f in self.master.view:
+            for f in self.state.flows:
                 f.marked = False
             signals.flowlist_change.send(self)
         elif key == "V":
             if not self.flow.modified():
                 signals.status_message.send(message="Flow not modified.")
                 return
-            self.flow.revert()
+            self.state.revert(self.flow)
             signals.flowlist_change.send(self)
             signals.status_message.send(message="Reverted.")
         elif key == "w":
@@ -230,8 +232,7 @@ class FlowItem(urwid.WidgetWrap):
             )
         elif key == "X":
             if self.flow.killable:
-                self.flow.kill()
-                self.master.view.update(self.flow)
+                self.flow.kill(self.master)
         elif key == "enter":
             if self.flow.request:
                 self.master.view_flow(self.flow)
@@ -265,49 +266,39 @@ class FlowItem(urwid.WidgetWrap):
 
 class FlowListWalker(urwid.ListWalker):
 
-    def __init__(self, master):
-        self.master = master
-        self.master.view.sig_view_refresh.connect(self.sig_mod)
-        self.master.view.sig_view_add.connect(self.sig_mod)
-        self.master.view.sig_view_remove.connect(self.sig_mod)
-        self.master.view.sig_view_update.connect(self.sig_mod)
-        self.master.view.focus.sig_change.connect(self.sig_mod)
-        signals.flowlist_change.connect(self.sig_mod)
+    def __init__(self, master, state):
+        self.master, self.state = master, state
+        signals.flowlist_change.connect(self.sig_flowlist_change)
 
-    def sig_mod(self, *args, **kwargs):
+    def sig_flowlist_change(self, sender):
         self._modified()
 
     def get_focus(self):
-        if not self.master.view.focus.flow:
-            return None, 0
-        f = FlowItem(self.master, self.master.view.focus.flow)
-        return f, self.master.view.focus.index
+        f, i = self.state.get_focus()
+        f = ConnectionItem(self.master, self.state, f, True) if f else None
+        return f, i
 
-    def set_focus(self, index):
-        if self.master.view.inbounds(index):
-            self.master.view.focus.index = index
-            signals.flowlist_change.send(self)
+    def set_focus(self, focus):
+        ret = self.state.set_focus(focus)
+        return ret
 
     def get_next(self, pos):
-        pos = pos + 1
-        if not self.master.view.inbounds(pos):
-            return None, None
-        f = FlowItem(self.master, self.master.view[pos])
-        return f, pos
+        f, i = self.state.get_next(pos)
+        f = ConnectionItem(self.master, self.state, f, False) if f else None
+        return f, i
 
     def get_prev(self, pos):
-        pos = pos - 1
-        if not self.master.view.inbounds(pos):
-            return None, None
-        f = FlowItem(self.master, self.master.view[pos])
-        return f, pos
+        f, i = self.state.get_prev(pos)
+        f = ConnectionItem(self.master, self.state, f, False) if f else None
+        return f, i
 
 
 class FlowListBox(urwid.ListBox):
 
-    def __init__(self, master: "mitmproxy.tools.console.master.ConsoleMaster"):
-        self.master = master  # type: "mitmproxy.tools.console.master.ConsoleMaster"
-        super().__init__(FlowListWalker(master))
+    def __init__(self, master):
+        # type: (mitmproxy.console.master.ConsoleMaster) -> None
+        self.master = master
+        super(FlowListBox, self).__init__(FlowListWalker(master, master.state))
 
     def get_method_raw(self, k):
         if k:
@@ -337,36 +328,35 @@ class FlowListBox(urwid.ListBox):
         )
 
     def new_request(self, url, method):
-        parts = mitmproxy.net.http.url.parse(str(url))
+        parts = netlib.http.url.parse(str(url))
         if not parts:
             signals.status_message.send(message="Invalid Url")
             return
         scheme, host, port, path = parts
         f = self.master.create_request(method, scheme, host, port, path)
-        self.master.view.focus.flow = f
+        self.master.state.set_focus_flow(f)
+        signals.flowlist_change.send(self)
 
     def keypress(self, size, key):
         key = common.shortcuts(key)
         if key == "A":
-            for f in self.master.view:
-                if f.intercepted:
-                    f.resume()
-                    self.master.view.update(f)
+            self.master.accept_all()
+            signals.flowlist_change.send(self)
         elif key == "z":
-            self.master.view.clear()
+            self.master.clear_flows()
         elif key == "e":
             self.master.toggle_eventlog()
         elif key == "g":
-            if len(self.master.view):
-                self.master.view.focus.index = 0
+            self.master.state.set_focus(0)
+            signals.flowlist_change.send(self)
         elif key == "G":
-            if len(self.master.view):
-                self.master.view.focus.index = len(self.master.view) - 1
+            self.master.state.set_focus(self.master.state.flow_count())
+            signals.flowlist_change.send(self)
         elif key == "f":
             signals.status_prompt.send(
                 prompt = "Filter View",
-                text = self.master.options.filter,
-                callback = self.master.options.setter("filter")
+                text = self.master.state.filter_txt,
+                callback = self.master.set_view_filter
             )
         elif key == "L":
             signals.status_prompt_path.send(
@@ -374,40 +364,22 @@ class FlowListBox(urwid.ListBox):
                 prompt = "Load flows",
                 callback = self.master.load_flows_callback
             )
-        elif key == "M":
-            self.master.view.toggle_marked()
         elif key == "n":
             signals.status_prompt_onekey.send(
                 prompt = "Method",
                 keys = common.METHOD_OPTIONS,
                 callback = self.get_method
             )
-        elif key == "o":
-            orders = [(i[1], i[0]) for i in view.orders]
-            lookup = dict([(i[0], i[1]) for i in view.orders])
-
-            def change_order(k):
-                self.master.options.order = lookup[k]
-
-            signals.status_prompt_onekey.send(
-                prompt = "Order",
-                keys = orders,
-                callback = change_order
-            )
         elif key == "F":
-            o = self.master.options
-            o.focus_follow = not o.focus_follow
-        elif key == "v":
-            val = not self.master.options.order_reversed
-            self.master.options.order_reversed = val
+            self.master.toggle_follow_flows()
         elif key == "W":
-            if self.master.options.streamfile:
-                self.master.options.streamfile = None
+            if self.master.options.outfile:
+                self.master.options.outfile = None
             else:
                 signals.status_prompt_path.send(
                     self,
                     prompt="Stream flows to",
-                    callback= lambda path: self.master.options.update(streamfile=path)
+                    callback= lambda path: self.master.options.update(outfile=(path, "ab"))
                 )
         else:
             return urwid.ListBox.keypress(self, size, key)

mitmproxy/console/flowview.py

@@ -1,23 +1,25 @@
+from __future__ import absolute_import, print_function, division
+
 import math
 import os
 import sys
-from functools import lru_cache
-from typing import Optional, Union  # noqa
 
 import urwid
+from mitmproxy import exceptions
+from typing import Optional, Union  # noqa
 
 from mitmproxy import contentviews
-from mitmproxy import exceptions
-from mitmproxy import export
-from mitmproxy import http
-from mitmproxy.net.http import Headers
-from mitmproxy.net.http import status_codes
-from mitmproxy.tools.console import common
-from mitmproxy.tools.console import flowdetailview
-from mitmproxy.tools.console import grideditor
-from mitmproxy.tools.console import searchable
-from mitmproxy.tools.console import signals
-from mitmproxy.tools.console import tabs
+from mitmproxy import models
+from mitmproxy import utils
+from mitmproxy.console import common
+from mitmproxy.console import flowdetailview
+from mitmproxy.console import grideditor
+from mitmproxy.console import searchable
+from mitmproxy.console import signals
+from mitmproxy.console import tabs
+from mitmproxy.flow import export
+from netlib.http import Headers
+from netlib.http import status_codes
 
 
 class SearchError(Exception):
@@ -90,8 +92,6 @@ def _mkhelp():
     ]
     text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
     return text
-
-
 help_context = _mkhelp()
 
 footer = [
@@ -102,9 +102,9 @@ footer = [
 
 class FlowViewHeader(urwid.WidgetWrap):
 
-    def __init__(self, master: "mitmproxy.console.master.ConsoleMaster", f: http.HTTPFlow):
-        self.master = master
-        self.flow = f
+    def __init__(self, master, f):
+        self.master = master  # type: "mitmproxy.console.master.ConsoleMaster"
+        self.flow = f  # type: models.HTTPFlow
         self._w = common.format_flow(
             f,
             False,
@@ -123,6 +123,8 @@ class FlowViewHeader(urwid.WidgetWrap):
             )
 
 
+cache = utils.LRUCache(200)
+
 TAB_REQ = 0
 TAB_RESP = 1
 
@@ -130,9 +132,9 @@ TAB_RESP = 1
 class FlowView(tabs.Tabs):
     highlight_color = "focusfield"
 
-    def __init__(self, master, view, flow, tab_offset):
-        self.master, self.view, self.flow = master, view, flow
-        super().__init__(
+    def __init__(self, master, state, flow, tab_offset):
+        self.master, self.state, self.flow = master, state, flow
+        super(FlowView, self).__init__(
             [
                 (self.tab_request, self.view_request),
                 (self.tab_response, self.view_response),
@@ -167,7 +169,7 @@ class FlowView(tabs.Tabs):
         return self.conn_text(self.flow.response)
 
     def view_details(self):
-        return flowdetailview.flowdetails(self.view, self.flow)
+        return flowdetailview.flowdetails(self.state, self.flow)
 
     def sig_flow_change(self, sender, flow):
         if flow == self.flow:
@@ -178,8 +180,11 @@ class FlowView(tabs.Tabs):
             msg, body = "", [urwid.Text([("error", "[content missing]")])]
             return msg, body
         else:
-            s = self.view.settings[self.flow]
-            full = s.get((self.tab_offset, "fullcontents"), False)
+            full = self.state.get_flow_setting(
+                self.flow,
+                (self.tab_offset, "fullcontents"),
+                False
+            )
             if full:
                 limit = sys.maxsize
             else:
@@ -190,21 +195,22 @@ class FlowView(tabs.Tabs):
                 message.headers.fields,
                 getattr(message, "path", None),
             ))
-            # we need to pass the message off-band because it's not hashable
-            self._get_content_view_message = message
-            return self._get_content_view(viewmode, limit, flow_modify_cache_invalidation)
+            return cache.get(
+                # We move message into this partial function as it is not hashable.
+                lambda *args: self._get_content_view(message, *args),
+                viewmode,
+                limit,
+                flow_modify_cache_invalidation
+            )
 
-    @lru_cache(maxsize=200)
-    def _get_content_view(self, viewmode, max_lines, _):
-        message = self._get_content_view_message
-        self._get_content_view_message = None
+    def _get_content_view(self, message, viewmode, max_lines, _):
         description, lines, error = contentviews.get_message_content_view(
             viewmode, message
         )
         if error:
             signals.add_log(error, "error")
         # Give hint that you have to tab for the response.
-        if description == "No content" and isinstance(message, http.HTTPRequest):
+        if description == "No content" and isinstance(message, models.HTTPRequest):
             description = "No request content (press tab to view response)"
 
         # If the users has a wide terminal, he gets fewer lines; this should not be an issue.
@@ -237,11 +243,11 @@ class FlowView(tabs.Tabs):
         return description, text_objects
 
     def viewmode_get(self):
-        override = self.view.settings[self.flow].get(
-            (self.tab_offset, "prettyview"),
-            None
+        override = self.state.get_flow_setting(
+            self.flow,
+            (self.tab_offset, "prettyview")
         )
-        return self.master.options.default_contentview if override is None else override
+        return self.state.default_body_view if override is None else override
 
     def conn_text(self, conn):
         if conn:
@@ -264,7 +270,7 @@ class FlowView(tabs.Tabs):
                         " ",
                         ('heading', "["),
                         ('heading_key', "m"),
-                        ('heading', (":%s]" % viewmode)),
+                        ('heading', (":%s]" % viewmode.name)),
                     ],
                     align="right"
                 )
@@ -284,7 +290,7 @@ class FlowView(tabs.Tabs):
                     ]
                 )
             ]
-        return searchable.Searchable(self.view, txt)
+        return searchable.Searchable(self.state, txt)
 
     def set_method_raw(self, m):
         if m:
@@ -369,7 +375,7 @@ class FlowView(tabs.Tabs):
             message = self.flow.request
         else:
             if not self.flow.response:
-                self.flow.response = http.HTTPResponse.make(200, b"")
+                self.flow.response = models.HTTPResponse.make(200, b"")
             message = self.flow.response
 
         self.flow.backup()
@@ -466,38 +472,43 @@ class FlowView(tabs.Tabs):
             )
         signals.flow_change.send(self, flow = self.flow)
 
-    def view_flow(self, flow):
-        signals.pop_view_state.send(self)
-        self.master.view_flow(flow, self.tab_offset)
-
-    def _view_nextprev_flow(self, idx, flow):
-        if not self.view.inbounds(idx):
-            signals.status_message.send(message="No more flows")
+    def _view_nextprev_flow(self, np, flow):
+        try:
+            idx = self.state.view.index(flow)
+        except IndexError:
             return
-        self.view_flow(self.view[idx])
+        if np == "next":
+            new_flow, new_idx = self.state.get_next(idx)
+        else:
+            new_flow, new_idx = self.state.get_prev(idx)
+        if new_flow is None:
+            signals.status_message.send(message="No more flows!")
+        else:
+            signals.pop_view_state.send(self)
+            self.master.view_flow(new_flow, self.tab_offset)
 
     def view_next_flow(self, flow):
-        return self._view_nextprev_flow(self.view.index(flow) + 1, flow)
+        return self._view_nextprev_flow("next", flow)
 
     def view_prev_flow(self, flow):
-        return self._view_nextprev_flow(self.view.index(flow) - 1, flow)
+        return self._view_nextprev_flow("prev", flow)
 
     def change_this_display_mode(self, t):
-        view = contentviews.get_by_shortcut(t)
-        if view:
-            self.view.settings[self.flow][(self.tab_offset, "prettyview")] = view.name
-        else:
-            self.view.settings[self.flow][(self.tab_offset, "prettyview")] = None
-        signals.flow_change.send(self, flow=self.flow)
+        self.state.add_flow_setting(
+            self.flow,
+            (self.tab_offset, "prettyview"),
+            contentviews.get_by_shortcut(t)
+        )
+        signals.flow_change.send(self, flow = self.flow)
 
     def keypress(self, size, key):
-        conn = None  # type: Optional[Union[http.HTTPRequest, http.HTTPResponse]]
+        conn = None  # type: Optional[Union[models.HTTPRequest, models.HTTPResponse]]
         if self.tab_offset == TAB_REQ:
             conn = self.flow.request
         elif self.tab_offset == TAB_RESP:
             conn = self.flow.response
 
-        key = super().keypress(size, key)
+        key = super(self.__class__, self).keypress(size, key)
 
         # Special case: Space moves over to the next flow.
         # We need to catch that before applying common.shortcuts()
@@ -510,26 +521,26 @@ class FlowView(tabs.Tabs):
             # Pass scroll events to the wrapped widget
             self._w.keypress(size, key)
         elif key == "a":
-            self.flow.resume()
-            self.master.view.update(self.flow)
+            self.flow.accept_intercept(self.master)
+            signals.flow_change.send(self, flow = self.flow)
         elif key == "A":
-            for f in self.view:
-                if f.intercepted:
-                    f.resume()
-                    self.master.view.update(self.flow)
+            self.master.accept_all()
+            signals.flow_change.send(self, flow = self.flow)
         elif key == "d":
-            if self.flow.killable:
-                self.flow.kill()
-            self.view.remove(self.flow)
-            if not self.view.focus.flow:
+            if self.state.flow_count() == 1:
                 self.master.view_flowlist()
+            elif self.state.view.index(self.flow) == len(self.state.view) - 1:
+                self.view_prev_flow(self.flow)
             else:
-                self.view_flow(self.view.focus.flow)
+                self.view_next_flow(self.flow)
+            f = self.flow
+            if f.killable:
+                f.kill(self.master)
+            self.state.delete_flow(f)
         elif key == "D":
-            cp = self.flow.copy()
-            self.master.view.add(cp)
-            self.master.view.focus.flow = cp
-            self.view_flow(cp)
+            f = self.master.duplicate_flow(self.flow)
+            signals.pop_view_state.send(self)
+            self.master.view_flow(f)
             signals.status_message.send(message="Duplicated.")
         elif key == "p":
             self.view_prev_flow(self.flow)
@@ -541,7 +552,7 @@ class FlowView(tabs.Tabs):
             signals.flow_change.send(self, flow = self.flow)
         elif key == "V":
             if self.flow.modified():
-                self.flow.revert()
+                self.state.revert(self.flow)
                 signals.flow_change.send(self, flow = self.flow)
                 signals.status_message.send(message="Reverted.")
             else:
@@ -603,9 +614,14 @@ class FlowView(tabs.Tabs):
             else:
                 common.ask_save_body("s", self.flow)
         elif key == "f":
-            self.view.settings[self.flow][(self.tab_offset, "fullcontents")] = True
-            signals.flow_change.send(self, flow = self.flow)
             signals.status_message.send(message="Loading all body data...")
+            self.state.add_flow_setting(
+                self.flow,
+                (self.tab_offset, "fullcontents"),
+                True
+            )
+            signals.flow_change.send(self, flow = self.flow)
+            signals.status_message.send(message="")
         elif key == "m":
             p = list(contentviews.view_prompts)
             p.insert(0, ("Clear", "C"))