bump version

Fix #1928, @concurrent decorator in class

.appveyor.yml

@@ -7,8 +7,9 @@ environment:
   matrix:
     - PYTHON: "C:\\Python35"
       TOXENV: "py35"
-    - PYTHON: "C:\\Python27"
-      TOXENV: "py27"
+    # TODO: ENABLE WHEN AVAILABLE
+    # - PYTHON: "C:\\Python36"
+    #   TOXENV: "py36"
 
   SNAPSHOT_HOST:
     secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
@@ -18,6 +19,8 @@ environment:
     secure: 6yBwmO5gv4vAwoFYII8qjQ==
   SNAPSHOT_PASS:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
+  RTOOL_KEY:
+    secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -25,21 +28,48 @@ install:
   - "pip install -U tox"
 
 test_script:
-  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v"
+  - ps: "tox -- --verbose --cov-report=term"
+  - ps: |
+      $Env:VERSION = $(python mitmproxy/version.py)
+      $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
+      tox -e wheel
+      tox -e rtool -- bdist
+
+  - ps: |
+      if(
+        ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
+        (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+      ) {
+        tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
+        if (!(Test-Path "C:\projects\mitmproxy\release\installbuilder-installer.exe")) {
+          "Download InstallBuilder..."
+          (New-Object System.Net.WebClient).DownloadFile(
+            "https://installbuilder.bitrock.com/installbuilder-enterprise-17.1.0-windows-installer.exe",
+            "C:\projects\mitmproxy\release\installbuilder-installer.exe"
+          )
+        }
+        Start-Process "C:\projects\mitmproxy\release\installbuilder-installer.exe" "--mode unattended --unattendedmodeui none" -Wait
+        & 'C:\Program Files (x86)\BitRock InstallBuilder Enterprise 17.1.0\bin\builder-cli.exe' `
+          build `
+          .\release\installbuilder\mitmproxy.xml `
+          windows `
+          --license .\release\installbuilder\license.xml `
+          --setvars project.version=$Env:VERSION `
+          --verbose
+      }
 
 deploy_script:
+  # we build binaries on every run, but we only upload them for master snapshots or tags.
   ps: |
     if(
       ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-      pip install -U virtualenv
-      .\dev.ps1
-      cmd /c "python -u .\release\rtool.py bdist 2>&1"
-      python -u .\release\rtool.py upload-snapshot --bdist --wheel
+      (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+    ) {
+      tox -e rtool -- upload-snapshot --bdist --wheel --installer
     }
 
 cache:
+  - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml
   - C:\Users\appveyor\AppData\Local\pip\cache
 
 notifications:

.env

@@ -1,6 +0,0 @@
-DIR="$( dirname "${BASH_SOURCE[0]}" )"
-ACTIVATE_DIR="$(if [ -f "$DIR/venv/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)"
-if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/venv/$ACTIVATE_DIR/activate" ]; then
-    echo "Activating mitmproxy virtualenv..."
-    source "$DIR/venv/$ACTIVATE_DIR/activate"
-fi

.gitattributes

@@ -1,2 +1,2 @@
-mitmproxy/web/static/**/* -diff
+mitmproxy/tools/web/static/**/* -diff linguist-vendored
 web/src/js/filt/filt.js -diff

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 MANIFEST
-*/tmp
+**/tmp
 /venv*
 *.py[cdo]
 *.swp
@@ -19,3 +19,5 @@ bower_components
 *.map
 sslkeylogfile.log
 .tox/
+.python-version
+coverage.xml

.travis.yml

@@ -1,19 +1,12 @@
 sudo: false
 language: python
 
-addons:
-  apt:
-    sources:
-      # Debian sid currently holds OpenSSL 1.0.2
-      # change this with future releases!
-      - debian-sid
-    packages:
-      - libssl-dev
-
 env:
   global:
     - CI_DEPS=codecov>=2.0.5
     - CI_COMMANDS=codecov
+git:
+  depth: 10000
 
 matrix:
   fast_finish: true
@@ -25,26 +18,42 @@ matrix:
       language: generic
       env: TOXENV=py35 BDIST=1
     - python: 3.5
-      env: TOXENV=py35 BDIST=1
+      env: TOXENV=py35 OPENSSL_OLD
+      addons:
+        apt:
+          packages:
+            - libssl-dev
     - python: 3.5
-      env: TOXENV=py35 NO_ALPN=1
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 2.7
-      env: TOXENV=py27 NO_ALPN=1
+      env: TOXENV=py35 BDIST=1 OPENSSL_ALPN
+      addons:
+        apt:
+          sources:
+            # Debian sid currently holds OpenSSL 1.1.0
+            # change this with future releases!
+            - debian-sid
+          packages:
+            - libssl-dev
+    - python: 3.6
+      env: TOXENV=py36 OPENSSL_ALPN
+      addons:
+        apt:
+          sources:
+            # Debian sid currently holds OpenSSL 1.1.0
+            # change this with future releases!
+            - debian-sid
+          packages:
+            - libssl-dev
+    - python: 3.5
+      env: TOXENV=individual_coverage
     - python: 3.5
       env: TOXENV=docs
-  allow_failures:
-    - python: pypy
 
 install:
   - |
     if [[ $TRAVIS_OS_NAME == "osx" ]]
     then
-      brew update || brew update # try again if it fails
-      brew upgrade
-      brew reinstall openssl
-      brew reinstall pyenv
+      brew update || brew update
+      brew outdated pyenv || brew upgrade pyenv
       eval "$(pyenv init -)"
       env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.2
       pyenv global 3.5.2
@@ -53,18 +62,21 @@ install:
     fi
   - pip install tox
 
-script: tox -- --cov netlib --cov mitmproxy --cov pathod -v
+script:
+  - tox -- --verbose --cov-report=term
+  - |
+    if [[ $BDIST == "1" ]]
+    then
+        git fetch --unshallow --tags
+        tox -e rtool -- bdist
+    fi
 
 after_success:
+  # we build binaries on every run, but we only upload them for master snapshots or tags.
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
-        git fetch --unshallow
-        ./dev.sh 3.5
-        source venv3.5/bin/activate
-        pip install -e ./release
-        python -u ./release/rtool.py bdist
-        python -u ./release/rtool.py upload-snapshot --bdist
+        tox -e rtool -- upload-snapshot --bdist
     fi
 
 notifications:
@@ -79,3 +91,4 @@ cache:
   directories:
     - $HOME/.pyenv
     - $HOME/.cache/pip
+    # - $HOME/build/mitmproxy/mitmproxy/.tox

CHANGELOG

@@ -1,3 +1,72 @@
+28 April 2017: mitmproxy 2.0.2
+
+    * Fix mitmweb's Content-Security-Policy to work with Chrome 58+
+
+    * HTTP/2: actually use header normalization from hyper-h2
+
+
+15 March 2017: mitmproxy 2.0.1
+
+    * bump cryptography dependency
+
+    * bump pyparsing dependency
+
+    * HTTP/2: use header normalization from hyper-h2
+
+
+21 February 2017: mitmproxy 2.0
+
+    * HTTP/2 is now enabled by default.
+
+    * Image ContentView: Parse images with Kaitai Struct (kaitai.io) instead of Pillow. 
+      This simplifies installation, reduces binary size, and allows parsing in pure Python.
+
+    * Web: Add missing flow filters.
+
+    * Add transparent proxy support for OpenBSD.
+
+    * Check the mitmproxy CA for expiration and warn the user to regenerate it if necessary.
+
+    * Testing: Tremendous improvements, enforced 100% coverage for large parts of the 
+      codebase, increased overall coverage.
+
+    * Enforce individual coverage: one source file -> one test file with 100% coverage.
+
+    * A myriad of other small improvements throughout the project.
+
+    * Numerous bugfixes.
+
+
+26 December 2016: mitmproxy 1.0
+
+    * All mitmproxy tools are now Python 3 only! We plan to support Python 3.5 and higher.
+
+    * Web-Based User Interface: Mitmproxy now offically has a web-based user interface
+      called mitmweb. We consider it stable for all features currently exposed 
+      in the UI, but it still misses a lot of mitmproxy’s options.
+      
+    * Windows Compatibility: With mitmweb, mitmproxy is now useable on Windows. 
+      We are also introducing an installer (kindly sponsored by BitRock) that 
+      simplifies setup.
+      
+    * Configuration: The config file format is now a single YAML file. In most cases,
+      converting to the new format should be trivial - please see the docs for
+      more information.
+
+    * Console: Significant UI improvements - including sorting of flows by
+      size, type and url, status bar improvements, much faster indentation for
+      HTTP views, and more.
+
+    * HTTP/2: Significant improvements, but is temporarily disabled by default
+      due to wide-spread protocol implementation errors on some large website
+
+    * WebSocket: The protocol implementation is now mature, and is enabled by
+      default. Complete UI support is coming in the next release. Hooks for 
+      message interception and manipulation are available.
+
+    * A myriad of other small improvements throughout the project.
+
+
 16 October 2016: mitmproxy 0.18
 
     * Python 3 Compatibility for mitmproxy and pathod (Shadab Zafar, GSoC 2016)

CONTRIBUTORS

@@ -1,6 +1,6 @@
-  2184	Aldo Cortesi
-  1745	Maximilian Hils
-   507	Thomas Kriechbaumer
+  2407	Aldo Cortesi
+  1873	Maximilian Hils
+   556	Thomas Kriechbaumer
    258	Shadab Zafar
     97	Jason
     83	Marcelo Glezer
@@ -11,85 +11,91 @@
     14	Pedro Worcel
     14	David Weinstein
     13	Thomas Roth
-    11	Stephen Altamirano
     11	Jake Drahos
+    11	Stephen Altamirano
     11	arjun23496
     11	Justus Wingert
-    10	Sandor Nemes
-    10	Zohar Lorberbaum
     10	András Veres-Szentkirályi
-    10	Chris Czub
+    10	Zohar Lorberbaum
     10	smill
+    10	Chris Czub
+    10	Sandor Nemes
+    10	Doug Freed
      9	ikoz
-     9	Kyle Morton
      9	Legend Tang
      9	Rouli
+     9	Kyle Morton
      8	Jason A. Novak
      8	Chandler Abraham
-     7	Alexis Hildebrandt
      7	Matthias Urlichs
      7	Brad Peabody
      7	dufferzafar
+     7	Alexis Hildebrandt
      6	Felix Yan
-     5	Tomaz Muraus
-     5	elitest
-     5	iroiro123
-     5	Sam Cleveland
-     5	Choongwoo Han
      5	Will Coster
-     4	root
-     4	Clemens Brunner
+     5	Sam Cleveland
+     5	iroiro123
+     5	elitest
+     5	Tomaz Muraus
+     5	Choongwoo Han
      4	Schamper
-     4	Valtteri Virtanen
-     4	Wade 524
      4	Youhei Sakurai
      4	Bryan Bishop
+     4	root
+     4	Valtteri Virtanen
+     4	Clemens Brunner
      4	Marc Liyanage
-     4	Michael J. Bazzinotti
+     4	Wade 524
+     4	chhsiao90
      4	yonder
-     3	Eli Shvartsman
-     3	Chris Neasbitt
-     3	Guillem Anguera
-     3	MatthewShao
+     4	Michael J. Bazzinotti
      3	Ryan Welton
-     3	smill@cuckoo.sh
-     3	Manish Kumar
-     3	Benjamin Lee
      3	Ryan Laughlin
-     3	Zack B
      3	Kyle Manna
+     3	Eli Shvartsman
+     3	Vincent Haupert
+     3	Manish Kumar
+     3	Zack B
+     3	MatthewShao
      3	redfast00
      3	requires.io
+     3	Guillem Anguera
+     3	smill@cuckoo.sh
+     3	Chris Neasbitt
+     3	Benjamin Lee
+     2	Steven Van Acker
+     2	Slobodan Mišković
+     2	Jim Lloyd
      2	isra17
      2	israel
-     2	Colin Bendell
+     2	Sean Coates
+     2	Sachin Kelkar
      2	jpkrause
-     2	Paul
+     2	Bennett Blodinger
      2	lilydjwg
      2	Michael Frister
-     2	依云
-     2	Jaime Soriano Pastor
-     2	Nick Badger
-     2	Rob Wills
-     2	Heikki Hannikainen
-     2	Vincent Haupert
-     2	strohu
-     2	Wade Catron
-     2	Krzysztof Bielicki
-     2	Sachin Kelkar
      2	Israel Nir
-     2	Anant
-     2	alts
-     2	Doug Freed
-     2	Niko Kommenda
-     2	Terry Long
-     2	Mark E. Haase
-     2	Steven Van Acker
-     2	Jim Lloyd
-     2	Bennett Blodinger
-     2	Sean Coates
      2	Cory Benfield
-     1	Sergey Chipiga
+     2	phackt
+     2	Anant
+     2	Jaime Soriano Pastor
+     2	Paul
+     2	Colin Bendell
+     2	依云
+     2	Heikki Hannikainen
+     2	Rob Wills
+     2	Niko Kommenda
+     2	Naveen Pai
+     2	strohu
+     2	alts
+     2	Yoginski
+     2	Mark E. Haase
+     2	Wade Catron
+     2	Terry Long
+     2	Krzysztof Bielicki
+     2	Nick Badger
+     1	Nicolas Esteves
+     1	Andrew Orr
      1	Andrey Plotnikov
      1	Andy Smith
      1	Angelo Agatino Nicolosi
@@ -97,6 +103,7 @@
      1	BSalita
      1	Ben Lerner
      1	Bradley Baetz
+     1	Brady Law
      1	Brett Randall
      1	Chris Hamant
      1	Christian Frichot
@@ -105,6 +112,7 @@
      1	David Shaw
      1	Doug Lethin
      1	Drake Caraker
+     1	Edgar Boda-Majer
      1	Eric Entzel
      1	Felix Wolfsteller
      1	FreeArtMan
@@ -128,22 +136,25 @@
      1	Mathieu Mitchell
      1	Michael Bisbjerg
      1	Mike C
+     1	Mike Fotinakis
      1	Mikhail Korobov
      1	Morton Fox
      1	Nick HS
      1	Nick Raptis
-     1	Nicolas Esteves
+     1	Aditya
      1	Oleksandr Sheremet
      1	Parth Ganatra
      1	Pritam Baral
+     1	Quentin Pradet
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
      1	Ryo Onodera
+     1	Sahil Chelaramani
      1	Sahn Lam
      1	Sanchit Sokhey
      1	Seppo Yli-Olli
-     1	Aditya
+     1	Sergey Chipiga
      1	Stefan Wärting
      1	Steve Phillips
      1	Steven Noble
@@ -158,10 +169,8 @@
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
      1	Wes Turner
-     1	Yoginski
      1	Yuangxuan Wang
      1	capt8bit
-     1	chhsiao90
      1	cle1000
      1	davidpshaw
      1	deployable
@@ -172,7 +181,6 @@
      1	meeee
      1	michaeljau
      1	peralta
-     1	phackt
      1	phil plante
      1	sentient07
      1	sethp-jive

MANIFEST.in

@@ -1,4 +1,3 @@
 graft mitmproxy
 graft pathod
-graft netlib
-recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
+recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

README.rst

@@ -3,14 +3,15 @@ mitmproxy
 
 |travis| |appveyor| |coverage| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects, as well as
-their shared networking library, **netlib**.
+This repository contains the **mitmproxy** and **pathod** projects.
 
 ``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
 interface.
 
 ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
+``mitmweb`` is a web-based interface for mitmproxy.
+
 ``pathoc`` and ``pathod`` are perverse HTTP client and server applications
 designed to let you craft almost any conceivable HTTP request, including ones
 that creatively violate the standards.
@@ -23,8 +24,7 @@ Documentation & Help
 General information, tutorials, and precompiled binaries can be found on the mitmproxy
 and pathod websites.
 
-|mitmproxy_site| |pathod_site|
-
+|mitmproxy_site|
 
 The latest documentation for mitmproxy is also available on ReadTheDocs.
 
@@ -37,7 +37,7 @@ each other solve problems, and come up with new ideas for the project.
 |mitmproxy_discourse|
 
 
-Join our developer chat on Slack if you would like to hack on mitmproxy itself.
+Join our developer chat on Slack if you would like to contribute to mitmproxy itself.
 
 |slack|
 
@@ -45,79 +45,67 @@ Join our developer chat on Slack if you would like to hack on mitmproxy itself.
 Installation
 ------------
 
-The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`_.
+The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`__.
 If you want to contribute changes, keep on reading.
 
+Contributing
+------------
 
-Hacking
--------
+As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project forward,
+please consider contributing in the following areas:
 
-To get started hacking on mitmproxy, make sure you have Python_ 3.5.x or above with
-virtualenv_ installed (you can find installation instructions for virtualenv
-`here <http://virtualenv.readthedocs.org/en/latest/>`_). Then do the following:
+- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users in our forums_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported.
+- **Code Contributions:** We actively mark issues that we consider are `good first contributions`_. If you intend to work on a larger contribution to the project, please come talk to us first.
+
+Development Setup
+-----------------
+
+To get started hacking on mitmproxy, please follow the `advanced installation`_ steps to install mitmproxy from source, but stop right before running ``pip3 install mitmproxy``. Instead, do the following:
 
 .. code-block:: text
 
     git clone https://github.com/mitmproxy/mitmproxy.git
     cd mitmproxy
-    ./dev.sh  # powershell .\dev.ps1 on Windows
+    ./dev.sh  # "powershell .\dev.ps1" on Windows
 
 
-The *dev* script will create a virtualenv environment in a directory called
-"venv", and install all mandatory and optional dependencies into it. The
-primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
+The *dev* script will create a `virtualenv`_ environment in a directory called "venv"
+and install all mandatory and optional dependencies into it. The primary
+mitmproxy components - mitmproxy and pathod - are installed as
 "editable", so any changes to the source in the repository will be reflected
 live in the virtualenv.
 
-To confirm that you're up and running, activate the virtualenv, and run the
-mitmproxy test suite:
-
-.. code-block:: text
-
-    . venv/bin/activate  # venv\Scripts\activate on Windows
-    py.test
-
-Note that the main executables for the project - ``mitmdump``, ``mitmproxy``,
+The main executables for the project - ``mitmdump``, ``mitmproxy``,
 ``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the
 virtualenv. After activating the virtualenv, they will be on your $PATH, and
 you can run them like any other command:
 
 .. code-block:: text
 
+    . venv/bin/activate  # "venv\Scripts\activate" on Windows
     mitmdump --version
 
-For convenience, the project includes an autoenv_ file (`.env`_) that
-auto-activates the virtualenv when you cd into the mitmproxy directory.
-
-
 Testing
 -------
 
 If you've followed the procedure above, you already have all the development
-requirements installed, and you can simply run the test suite:
+requirements installed, and you can run the full test suite (including tests for code style and documentation) with tox_:
 
 .. code-block:: text
 
-    py.test
+    tox
+
+For speedier testing, we recommend you run `pytest`_ directly on individual test files or folders:
+
+.. code-block:: text
+
+    cd test/mitmproxy/addons
+    pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py
+
+As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes.
 
 Please ensure that all patches are accompanied by matching changes in the test
-suite. The project tries to maintain 100% test coverage.
-
-You can also use `tox` to run a full suite of tests in Python 2.7 and 3.5,
-including a quick test to check documentation and code linting.
-
-The following tox environments are relevant for local testing:
-
-.. code-block:: text
-
-    tox -e py27  # runs all tests with Python 2.7
-    tox -e py35  # runs all tests with Python 3.5
-    tox -e docs  # runs a does-it-compile check on the documentation
-    tox -e lint  # runs the linter for coding style checks
-
-We support Python 2.7 and 3.5, so please make sure all tests pass in both
-environments. Running `tox` ensures all necessary tests are executed.
-
+suite. The project tries to maintain 100% test coverage and enforces this strictly for some parts of the codebase.
 
 Documentation
 -------------
@@ -136,8 +124,8 @@ installation, you can render the documentation like this:
 The last command invokes `sphinx-autobuild`_, which watches the Sphinx directory and rebuilds
 the documentation when a change is detected.
 
-Style
------
+Code Style
+----------
 
 Keeping to a consistent code style throughout the project makes it easier to
 contribute and collaborate. Please stick to the guidelines in
@@ -145,22 +133,19 @@ contribute and collaborate. Please stick to the guidelines in
 good reason not to.
 
 This is automatically enforced on every PR. If we detect a linting error, the
-PR checks will fail and block merging. We are using this command to check for style compliance:
+PR checks will fail and block merging. You can run our lint checks yourself
+with the following command:
 
 .. code-block:: text
 
-    flake8 --jobs 8 --count mitmproxy netlib pathod examples test
+    tox -e lint
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg
     :target: https://mitmproxy.org/
     :alt: mitmproxy.org
 
-.. |pathod_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-pathod.net-blue.svg
-    :target: https://pathod.net/
-    :alt: pathod.net
-
-.. |mitmproxy_docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
+.. |mitmproxy_docs| image:: https://shields.mitmproxy.org/api/docs-latest-brightgreen.svg
     :target: http://docs.mitmproxy.org/en/latest/
     :alt: mitmproxy documentation
 
@@ -172,15 +157,15 @@ PR checks will fail and block merging. We are using this command to check for st
     :target: http://slack.mitmproxy.org/
     :alt: Slack Developer Chat
 
-.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=Travis%20build
+.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=travis%20ci
     :target: https://travis-ci.org/mitmproxy/mitmproxy
     :alt: Travis Build Status
 
-.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mhils/mitmproxy/master.svg?label=Appveyor%20build
+.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mhils/mitmproxy/master.svg?label=appveyor%20ci
     :target: https://ci.appveyor.com/project/mhils/mitmproxy
     :alt: Appveyor Build Status
 
-.. |coverage| image:: https://codecov.io/gh/mitmproxy/mitmproxy/branch/master/graph/badge.svg
+.. |coverage| image:: https://shields.mitmproxy.org/codecov/c/github/mitmproxy/mitmproxy/master.svg?label=codecov
     :target: https://codecov.io/gh/mitmproxy/mitmproxy
     :alt: Coverage Status
 
@@ -192,12 +177,13 @@ PR checks will fail and block merging. We are using this command to check for st
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Supported Python versions
 
-.. _Python: https://www.python.org/
-.. _virtualenv: http://virtualenv.readthedocs.org/en/latest/
-.. _autoenv: https://github.com/kennethreitz/autoenv
-.. _.env: https://github.com/mitmproxy/mitmproxy/blob/master/.env
+.. _`advanced installation`: http://docs.mitmproxy.org/en/latest/install.html#advanced-installation
+.. _virtualenv: https://virtualenv.pypa.io/
+.. _`pytest`: http://pytest.org/
+.. _tox: https://tox.readthedocs.io/
 .. _Sphinx: http://sphinx-doc.org/
 .. _sphinx-autobuild: https://pypi.python.org/pypi/sphinx-autobuild
-.. _issue_tracker: https://github.com/mitmproxy/mitmproxy/issues
 .. _PEP8: https://www.python.org/dev/peps/pep-0008
-.. _Google Style Guide: https://google.github.io/styleguide/pyguide.html
+.. _`Google Style Guide`: https://google.github.io/styleguide/pyguide.html
+.. _forums: https://discourse.mitmproxy.org/
+.. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3Agood-first-contribution

dev.ps1

@@ -1,15 +1,19 @@
 $ErrorActionPreference = "Stop"
-$VENV = ".\venv"
 
-virtualenv $VENV --always-copy
-& $VENV\Scripts\activate.ps1
+$pyver = python --version
+if($pyver -notmatch "3\.[5-9]") {
+    Write-Warning "Unexpected Python version, expected Python 3.5 or above: $pyver"
+}
+
+python -m venv .\venv --copies
+& .\venv\Scripts\activate.ps1
 
 python -m pip install --disable-pip-version-check -U pip
 cmd /c "pip install -r requirements.txt 2>&1"
 
 echo @"
 
-  * Created virtualenv environment in $VENV.
+  * Created virtualenv environment in .\venv.
   * Installed all dependencies into the virtualenv.
   * Activated virtualenv environment.
 

dev.sh

@@ -2,16 +2,14 @@
 set -e
 set -x
 
-PYVERSION=$1
-VENV="venv$1"
+echo "Creating dev environment in ./venv..."
 
-echo "Creating dev environment in $VENV using Python $PYVERSION"
-
-python$PYVERSION -m virtualenv "$VENV" --always-copy
-. "$VENV/bin/activate"
-pip$PYVERSION install -U pip setuptools
-pip$PYVERSION install -r requirements.txt
+python3 -m venv venv
+. venv/bin/activate
+pip3 install -U pip setuptools
+pip3 install -r requirements.txt
 
 echo ""
-echo "* Virtualenv created in $VENV and all dependencies installed."
-echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV/bin/activate\`"
+echo "  * Created virtualenv environment in ./venv."
+echo "  * Installed all dependencies into the virtualenv."
+echo "  * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`"

docs/certinstall.rst

@@ -40,7 +40,9 @@ start of mitmproxy.
 iOS
 ^^^
 
-http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
+See http://jasdev.me/intercepting-ios-traffic
+
+and http://web.archive.org/web/20150920082614/http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
 
 iOS Simulator
 ^^^^^^^^^^^^^

docs/conf.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 
 sys.path.insert(0, os.path.abspath('..'))
-import netlib.version
+from mitmproxy import version as mversion
 
 
 extensions = [
@@ -47,9 +47,9 @@ author = u'The mitmproxy project'
 # built documents.
 #
 # The short X.Y version.
-version = netlib.version.VERSION
+version = mversion.VERSION
 # The full version, including alpha/beta/rc tags.
-release = netlib.version.VERSION
+release = mversion.VERSION
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -231,10 +231,7 @@ def linkcode_resolve(domain, info):
         _, line = inspect.getsourcelines(obj)
     except (TypeError, IOError):
         return None
-    if spath.rfind("netlib") > -1:
-        off = spath.rfind("netlib")
-        mpath = spath[off:]
-    elif spath.rfind("mitmproxy") > -1:
+    if spath.rfind("mitmproxy") > -1:
         off = spath.rfind("mitmproxy")
         mpath = spath[off:]
     else:

docs/config.rst

@@ -3,84 +3,11 @@
 Configuration
 =============
 
-Mitmproxy is configured through a set of files in the users ~/.mitmproxy
-directory.
+Mitmproxy is configured with a YAML_ file, located at
+``~/.mitmproxy/config.yaml``. We'll have complete documentation for all
+supported options in the next release in the meantime, please consult the
+source_ for a complete list of options and types.
 
-mitmproxy.conf
-    Settings for the :program:`mitmproxy`. This file can contain any options supported by
-    mitmproxy.
 
-mitmdump.conf
-    Settings for the :program:`mitmdump`. This file can contain any options supported by mitmdump.
-
-common.conf
-    Settings shared between all command-line tools. Settings in this file are over-ridden by those
-    in the tool-specific files. Only options shared by mitmproxy and mitmdump should be used in
-    this file.
-
-Syntax
-------
-
-Comments
-^^^^^^^^
-
-.. code-block:: none
-
-    # this is a comment
-    ; this is also a comment (.ini style)
-    --- and this is a comment too (yaml style)
-
-Key/Value pairs
-^^^^^^^^^^^^^^^
-
-- Keys and values are case-sensitive
-- Whitespace is ignored
-- Lists are comma-delimited, and enclosed in square brackets
-
-.. code-block:: none
-
-    name = value   # (.ini style)
-    name: value    # (yaml style)
-    --name value   # (command-line option style)
-
-    fruit = [apple, orange, lemon]
-    indexes = [1, 12, 35 , 40]
-
-Flags
-^^^^^
-
-These are boolean options that take no value but true/false.
-
-.. code-block:: none
-
-    name = true    # (.ini style)
-    name
-    --name 	 	   # (command-line option style)
-
-Options
--------
-
-The options available in the config files are precisely those available as
-command-line flags, with the key being the option's long name. To get a
-complete list of these, use the ``--help`` option on each of the tools. Be
-careful to only specify common options in the **common.conf** file -
-unsupported options in this file will be detected as an error on startup.
-
-Examples
---------
-
-common.conf
-^^^^^^^^^^^
-
-Note that ``--port`` is an option supported by all tools.
-
-.. code-block:: none
-
-    port = 8080
-
-mitmproxy.conf
-^^^^^^^^^^^^^^
-
-.. code-block:: none
-
-    palette = light
+.. _YAML: http://www.yaml.org/start.html
+.. _source: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/options.py

docs/dev/architecture.rst

@@ -1,14 +0,0 @@
-.. _architecture:
-
-Architecture
-============
-
-To give you a better understanding of how mitmproxy works, mitmproxy's
-high-level architecture is detailed in the following graphic:
-
-.. image:: ../schematics/architecture.png
-
-:download:`architecture.pdf <../schematics/architecture.pdf>`
-
-Please don't refrain from asking any further
-questions on the mailing list, the Slack channel or the GitHub issue tracker.

docs/dev/contributing.rst

@@ -0,0 +1,11 @@
+.. _contributing:
+
+Contributing
+============
+
+As an open source project, **mitmproxy** welcomes contributions of all forms. 
+
+Please head over to the README_ to get started! 😃
+
+
+.. _README: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst

docs/dev/testing.rst

@@ -1,47 +0,0 @@
-.. _testing:
-
-Testing
-=======
-
-All the mitmproxy projects strive to maintain 100% code coverage. In general,
-patches and pull requests will be declined unless they're accompanied by a
-suitable extension to the test suite.
-
-Our tests are written for the `py.test`_ or nose_ test frameworks.
-At the point where you send your pull request, a command like this:
-
->>> py.test --cov mitmproxy --cov netlib
-
-Should give output something like this:
-
-.. code-block:: none
-
-    > ---------- coverage: platform darwin, python 2.7.2-final-0 --
-    > Name                   Stmts   Miss  Cover   Missing
-    > ----------------------------------------------------
-    > mitmproxy/__init__         0      0   100%
-    > mitmproxy/app              4      0   100%
-    > mitmproxy/cmdline        100      0   100%
-    > mitmproxy/controller      69      0   100%
-    > mitmproxy/dump           150      0   100%
-    > mitmproxy/encoding        39      0   100%
-    > mitmproxy/flowfilter     201      0   100%
-    > mitmproxy/flow           891      0   100%
-    > mitmproxy/proxy          427      0   100%
-    > mitmproxy/script          27      0   100%
-    > mitmproxy/utils          133      0   100%
-    > mitmproxy/version          4      0   100%
-    > ----------------------------------------------------
-    > TOTAL                   2045      0   100%
-    > ----------------------------------------------------
-    > Ran 251 tests in 11.864s
-
-
-There are exceptions to the coverage requirement - for instance, much of the
-console interface code can't sensibly be unit tested. These portions are
-excluded from coverage analysis either in the **.coveragerc** file, or using
-**#pragma no-cover** directives. To keep our coverage analysis relevant, we use
-these measures as sparingly as possible.
-
-.. _nose: https://nose.readthedocs.org/en/latest/
-.. _py.test: https://pytest.org/

docs/features/responsestreaming.rst

@@ -40,8 +40,8 @@ You can also use a script to customize exactly which responses are streamed.
 Responses that should be tagged for streaming by setting their ``.stream``
 attribute to ``True``:
 
-.. literalinclude:: ../../examples/stream.py
-   :caption: examples/stream.py
+.. literalinclude:: ../../examples/complex/stream.py
+   :caption: examples/complex/stream.py
    :language: python
 
 Implementation Details
@@ -59,8 +59,8 @@ Modifying streamed data
 If the ``.stream`` attribute is callable, ``.stream`` will wrap the generator that yields all
 chunks.
 
-.. literalinclude:: ../../examples/stream_modify.py
-   :caption: examples/stream_modify.py
+.. literalinclude:: ../../examples/complex/stream_modify.py
+   :caption: examples/complex/stream_modify.py
    :language: python
 
 .. seealso::

docs/features/serverreplay.rst

@@ -33,6 +33,19 @@ updated in a similar way.
 You can turn off response refreshing using the ``--norefresh`` argument, or using
 the :kbd:`o` options shortcut within :program:`mitmproxy`.
 
+
+Replaying a session recorded in Reverse-proxy Mode
+--------------------------------------------------
+
+If you have captured the session in reverse proxy mode, in order to replay it you 
+still have to specify the server URL, otherwise you may get the error: 
+'HTTP protocol error in client request: Invalid HTTP request form (expected authority or absolute...)'.
+
+During replay, when the client's requests match previously recorded requests, then the
+respective recorded responses are simply replayed  by mitmproxy. 
+Otherwise, the unmatched requests is forwarded to the upstream server.
+If forwarding is not desired, you can use the --kill (-k) switch to prevent that.
+
 ================== ===========
 command-line       ``-S path``
 mitmproxy shortcut :kbd:`R` then :kbd:`s`

docs/features/tcpproxy.rst

@@ -3,7 +3,7 @@
 TCP Proxy
 =========
 
-WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt
+In case mitmproxy does not handle a specific protocol, you can exempt
 hostnames from processing, so that mitmproxy acts as a generic TCP forwarder.
 This feature is closely related to the :ref:`passthrough` functionality,
 but differs in two important aspects:

docs/index.rst

@@ -17,6 +17,7 @@
 
     mitmproxy
     mitmdump
+    mitmweb
     config
 
 .. toctree::
@@ -46,6 +47,7 @@
     transparent
     transparent/linux
     transparent/osx
+    transparent/openbsd
 
 .. toctree::
     :hidden:
@@ -77,10 +79,9 @@
 
 .. toctree::
     :hidden:
-    :caption: Hacking
+    :caption: Development
 
-    dev/architecture
-    dev/testing
+    dev/contributing
     dev/sslkeylogfile
 
 ..  Indices and tables

docs/install.rst

@@ -3,130 +3,150 @@
 Installation
 ============
 
-.. _install-ubuntu:
+Please follow the steps for your operating system.
 
-Installation On Ubuntu
-----------------------
+Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or
+:ref:`mitmweb` from a terminal.
 
-Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Ubuntu 14.04.
+
+.. _install-macos:
+
+Installation on macOS
+---------------------
+
+You can use Homebrew to install everything:
 
 .. code:: bash
 
-   sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev g++
-   sudo pip install mitmproxy  # or pip install --user mitmproxy
+    brew install mitmproxy
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+Or you can download the pre-built binary packages from our `releases`_.
 
-On **Ubuntu 12.04** (and other systems with an outdated version of pip),
-you may need to update pip using ``pip install -U pip`` before installing mitmproxy.
 
-Installation From Source (Ubuntu)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. _install-windows:
 
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, install the dependencies as you would for a regular
-mitmproxy installation (see :ref:`install-ubuntu`).
-Then see the Hacking_ section of the README on GitHub.
+Installation on Windows
+-----------------------
 
-.. _install-fedora:
+The recommended way to install mitmproxy on Windows is to use the installer
+provided at `mitmproxy.org`_. After installation, you'll find shortcuts for
+:ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu.
+Both executables are  added to your PATH and can be invoked from the command
+line.
 
-Installation On Fedora
-----------------------
+.. note::
+    Mitmproxy's console interface is not supported on Windows, but you can use
+    mitmweb (the web-based interface) and mitmdump.
 
-Fedora comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Fedora 23.
+.. _install-linux:
 
-.. code:: bash
+Installation on Linux
+---------------------
 
-   sudo dnf install -y python-pip python-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libpng-devel libjpeg-devel
-   sudo pip install mitmproxy  # or pip install --user mitmproxy
+The recommended way to run mitmproxy on Linux is to use the pre-built binaries
+provided at `releases`_.
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+Our pre-built binaries provide you with the latest version of mitmproxy, a
+self-contained Python 3.5 environment and a recent version of OpenSSL that
+supports HTTP/2. Of course, you can also install mitmproxy from source if you
+prefer that (see :ref:`install-advanced`).
+
+.. _install-advanced:
+
+Advanced Installation
+---------------------
+
+.. _install-docker:
+
+Docker Images
+^^^^^^^^^^^^^
+
+You can also use the official mitmproxy images from `DockerHub`_. That being
+said, our portable binaries are just as easy to install and even easier to use. 😊
 
 
 .. _install-arch:
 
-Installation On Arch Linux
---------------------------
+Installation on Arch Linux
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 mitmproxy has been added into the [community] repository. Use pacman to install it:
 
 >>> sudo pacman -S mitmproxy
 
 
+.. _install-source-ubuntu:
 
-Installation On Mac OS X
-------------------------
-
-The easiest way to get up and running on OSX is to download the pre-built binary packages from
-`mitmproxy.org`_.
-
-There are a few bits of customization you might want to do to make mitmproxy comfortable to use on
-OSX. The default color scheme is optimized for a dark background terminal, but you can select a
-palette for a light terminal background with the ``--palette`` option.
-You can use the OSX **open** program to create a simple and effective ``~/.mailcap`` file to view
-request and response bodies:
-
-.. code-block:: none
-
-    application/*; /usr/bin/open -Wn %s
-    audio/*; /usr/bin/open -Wn %s
-    image/*; /usr/bin/open -Wn %s
-    video/*; /usr/bin/open -Wn %s
-
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
-
-
-Installation From Source (Mac OS X)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, there are a few OS X specific things to keep in mind.
-
-- Make sure that XCode is installed from the App Store, and that the command-line tools have been
-  downloaded (XCode/Preferences/Downloads).
-- If you're running a Python interpreter installed with homebrew (or similar), you may have to
-  install some dependencies by hand.
-
-Then see the Hacking_ section of the README on GitHub.
-
-Installation On Windows
------------------------
-
-.. note::
-    Please note that mitmdump is the only component of mitmproxy that is supported on Windows at
-    the moment.
-
-    **There is no interactive user interface on Windows.**
-
-
-First, install the latest version of Python 2.7 from the `Python website`_.
-If you already have an older version of Python 2.7 installed, make sure to install pip_
-(pip is included in Python 2.7.9+ by default). If pip aborts with an error, make sure you are using the current version of pip.
-
->>> python -m pip install --upgrade pip
-
-Next, add Python and the Python Scripts directory to your **PATH** variable.
-You can do this easily by running the following in powershell:
-
->>> [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User")
-
-Now, you can install mitmproxy by running
-
->>> pip install mitmproxy
-
-Once the installation is complete, you can run :ref:`mitmdump` from a command prompt.
-
-Installation From Source (Windows)
+Installation from Source on Ubuntu
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, install Python as outlined above, then see the
-Hacking_ section of the README on GitHub.
+Ubuntu comes with Python but we need to install pip3, python3-dev and several
+libraries. This was tested on a fully patched installation of Ubuntu 16.04.
+
+.. code:: bash
+
+   sudo apt-get install python3-dev python3-pip libffi-dev libssl-dev
+   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
+
+On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install
+a newer version of Python. mitmproxy requires Python 3.5 or higher. Please take
+a look at pyenv_. Make sure to have an up-to-date version of pip by running
+``pip3 install -U pip``.
 
 
-.. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst#hacking
+.. _install-source-fedora:
+
+Installation from Source on Fedora
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Fedora comes with Python but we need to install pip3, python3-dev and several
+libraries. This was tested on a fully patched installation of Fedora 24.
+
+.. code:: bash
+
+   sudo dnf install make gcc redhat-rpm-config python3-devel python3-pip libffi-devel openssl-devel
+   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
+
+Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
+
+
+
+.. _install-source-windows:
+
+🐱💻 Installation from Source on Windows
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. note::
+    Mitmproxy's console interface is not supported on Windows, but you can use
+    mitmweb (the web-based interface) and mitmdump.
+
+First, install the latest version of Python 3.5 or later from the `Python
+website`_. During installation, make sure to select `Add Python to PATH`.
+
+Mitmproxy has no other dependencies on Windows. You can now install mitmproxy by running
+
+.. code:: powershell
+
+    pip3 install mitmproxy
+
+
+
+.. _install-dev-version:
+
+Latest Development Version
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you would like to install mitmproxy directly from the master branch on GitHub
+or would like to get set up to contribute to the project, install the
+dependencies as you would for a regular installation from source. Then see the
+project's README_ on GitHub. You can check your system information
+by running: ``mitmproxy --version``
+
+
+.. _README: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst
+.. _releases: https://github.com/mitmproxy/mitmproxy/releases
 .. _mitmproxy.org: https://mitmproxy.org/
 .. _`Python website`: https://www.python.org/downloads/windows/
 .. _pip: https://pip.pypa.io/en/latest/installing.html
+.. _pyenv: https://github.com/yyuu/pyenv
+.. _DockerHub: https://hub.docker.com/r/mitmproxy/mitmproxy/

docs/introduction.rst

@@ -6,6 +6,8 @@ with a console interface.
 
 **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
+**mitmweb** is a web-based interface for mitmproxy.
+
 Documentation, tutorials and distribution packages can be found on the
 mitmproxy website: `mitmproxy.org <https://mitmproxy.org/>`_
 

docs/mitmweb.rst

@@ -0,0 +1,18 @@
+.. _mitmweb:
+.. program:: mitmweb
+
+mitmweb
+=======
+
+**mitmweb** is mitmproxy's web-based user interface that allows interactive
+examination and modification of HTTP traffic. Like mitmproxy, it differs from
+mitmdump in that all flows are kept in memory, which means that it's intended
+for taking and manipulating small-ish samples.
+
+.. warning::
+
+  Mitmweb is currently in beta. We consider it stable for all features currently
+  exposed in the UI, but it still misses a lot of mitmproxy's features.
+
+
+.. image:: screenshots/mitmweb.png

docs/modd.conf

@@ -1,6 +1,6 @@
 @build = ./_build
 
-** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py {
+** !_build/** ../mitmproxy/**/*.py {
     prep: sphinx-build -W -d @build/doctrees -b html . @build/html
     daemon: devd -m @build/html
 }

docs/pathod/test.rst

@@ -14,7 +14,7 @@ The canonical docs can be accessed using pydoc:
 >>> pydoc pathod.test
 
 The remainder of this page demonstrates some common interaction patterns using
-<a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are
+`Nose`_. These examples are
 also applicable with only minor modification to most commonly used Python testing
 engines.
 
@@ -33,3 +33,6 @@ One instance per test
 .. literalinclude:: ../../examples/pathod/test_setup.py
    :caption: examples/pathod/test_setup.py
    :language: python
+
+
+.. _Nose: https://nose.readthedocs.org/en/latest/

docs/scripting/api.rst

@@ -5,36 +5,38 @@ API
 ===
 
 - Errors
-    - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_
+    - `mitmproxy.flow.Error <#mitmproxy.flow.Error>`_
 - HTTP
-    - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_
-    - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_
-    - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_
+    - `mitmproxy.http.HTTPRequest <#mitmproxy.http.HTTPRequest>`_
+    - `mitmproxy.http.HTTPResponse <#mitmproxy.http.HTTPResponse>`_
+    - `mitmproxy.http.HTTPFlow <#mitmproxy.http.HTTPFlow>`_
 - Logging
-    - `mitmproxy.controller.Log <#mitmproxy.controller.Log>`_
-    - `mitmproxy.controller.LogEntry <#mitmproxy.controller.LogEntry>`_
+    - `mitmproxy.log.Log <#mitmproxy.controller.Log>`_
+    - `mitmproxy.log.LogEntry <#mitmproxy.controller.LogEntry>`_
 
 
 Errors
 ------
 
-.. autoclass:: mitmproxy.models.flow.Error
+.. autoclass:: mitmproxy.flow.Error
     :inherited-members:
 
 HTTP
 ----
 
-.. autoclass:: mitmproxy.models.http.HTTPRequest
+.. autoclass:: mitmproxy.http.HTTPRequest
     :inherited-members:
 
-.. autoclass:: mitmproxy.models.http.HTTPResponse
+.. autoclass:: mitmproxy.http.HTTPResponse
     :inherited-members:
 
-.. autoclass:: mitmproxy.models.http.HTTPFlow
+.. autoclass:: mitmproxy.http.HTTPFlow
     :inherited-members:
 
 Logging
 --------
 
-.. autoclass:: mitmproxy.controller.Log
+.. autoclass:: mitmproxy.log.Log
+    :inherited-members:
+.. autoclass:: mitmproxy.log.LogEntry
     :inherited-members:

docs/scripting/events.rst

@@ -56,7 +56,7 @@ Connection
           connection can correspond to multiple HTTP requests.
 
           *root_layer*
-            The root layer (see `mitmproxy.protocol` for an explanation what
+            The root layer (see `mitmproxy.proxy.protocol` for an explanation what
             the root layer is), provides transparent access to all attributes
             of the :py:class:`~mitmproxy.proxy.RootContext`. For example,
             ``root_layer.client_conn.address`` gives the remote address of the
@@ -98,6 +98,18 @@ HTTP Events
     :widths: 40 60
     :header-rows: 0
 
+    *   - .. py:function:: http_connect(flow)
+        - Called when we receive an HTTP CONNECT request. Setting a non 2xx
+          response on the flow will return the response to the client abort the
+          connection. CONNECT requests and responses do not generate the usual
+          HTTP handler events. CONNECT requests are only valid in regular and
+          upstream proxy modes.
+
+          *flow*
+            A ``models.HTTPFlow`` object. The flow is guaranteed to have
+            non-None ``request``  and ``requestheaders`` attributes.
+
+
     *   - .. py:function:: request(flow)
         - Called when a client request has been received.
 
@@ -146,21 +158,54 @@ HTTP Events
 WebSocket Events
 -----------------
 
+These events are called only after a connection made an HTTP upgrade with
+"101 Switching Protocols". No further HTTP-related events after the handshake
+are issued, only new WebSocket messages are called.
+
 .. list-table::
     :widths: 40 60
     :header-rows: 0
 
-    *   - .. py:function:: websockets_handshake(flow)
-
-        - Called when a client wants to establish a WebSockets connection. The
-          WebSockets-specific headers can be manipulated to manipulate the
+    *   - .. py:function:: websocket_handshake(flow)
+        - Called when a client wants to establish a WebSocket connection. The
+          WebSocket-specific headers can be manipulated to alter the
           handshake. The ``flow`` object is guaranteed to have a non-None
           ``request`` attribute.
 
           *flow*
-            The flow containing the HTTP websocket handshake request. The
+            The flow containing the HTTP WebSocket handshake request. The
             object is guaranteed to have a non-None ``request`` attribute.
 
+    *   - .. py:function:: websocket_start(flow)
+        - Called when WebSocket connection is established after a successful
+          handshake.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_message(flow)
+
+        - Called when a WebSocket message is received from the client or server. The
+          sender and receiver are identifiable. The most recent message will be
+          ``flow.messages[-1]``. The message is user-modifiable. Currently there are
+          two types of messages, corresponding to the BINARY and TEXT frame types.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_end(flow)
+        - Called when WebSocket connection ends.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_error(flow)
+        - Called when a WebSocket error occurs - e.g. the connection closing
+          unexpectedly.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
 
 TCP Events
 ----------
@@ -173,6 +218,22 @@ connections.
     :widths: 40 60
     :header-rows: 0
 
+
+    *   - .. py:function:: tcp_start(flow)
+        - Called when TCP streaming starts.
+
+          *flow*
+            A ``models.TCPFlow`` object.
+
+    *   - .. py:function:: tcp_message(flow)
+
+        - Called when a TCP payload is received from the client or server. The
+          sender and receiver are identifiable. The most recent message will be
+          ``flow.messages[-1]``. The message is user-modifiable.
+
+          *flow*
+            A ``models.TCPFlow`` object.
+
     *   - .. py:function:: tcp_end(flow)
         - Called when TCP streaming ends.
 
@@ -185,18 +246,3 @@ connections.
 
           *flow*
             A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_message(flow)
-
-        - Called a TCP payload is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_start(flow)
-        - Called when TCP streaming starts.
-
-          *flow*
-            A ``models.TCPFlow`` object.

docs/scripting/overview.rst

@@ -6,7 +6,7 @@ Overview
 Mitmproxy has a powerful scripting API that allows you to control almost any
 aspect of traffic being proxied. In fact, much of mitmproxy's own core
 functionality is implemented using the exact same API exposed to scripters (see
-:src:`mitmproxy/builtins`).
+:src:`mitmproxy/addons`).
 
 
 A simple example
@@ -17,8 +17,8 @@ appropriate points of mitmproxy's operation. Here's a complete mitmproxy script
 that adds a new header to every HTTP response before it is returned to the
 client:
 
-.. literalinclude:: ../../examples/add_header.py
-   :caption: :src:`examples/add_header.py`
+.. literalinclude:: ../../examples/simple/add_header.py
+   :caption: :src:`examples/simple/add_header.py`
    :language: python
 
 All events that deal with an HTTP request get an instance of `HTTPFlow
@@ -42,8 +42,8 @@ called before anything else happens. You can replace the current script object
 by returning it from this handler. Here's how this looks when applied to the
 example above:
 
-.. literalinclude:: ../../examples/classes.py
-   :caption: :src:`examples/classes.py`
+.. literalinclude:: ../../examples/simple/add_header_class.py
+   :caption: :src:`examples/simple/add_header_class.py`
    :language: python
 
 So here, we're using a module-level script to "boot up" into a class instance.
@@ -62,13 +62,13 @@ sophisticated - replace one value with another in all responses. Mitmproxy's
 <api.html#mitmproxy.models.http.HTTPResponse.replace>`_ method that takes care
 of all the details for us.
 
-.. literalinclude:: ../../examples/arguments.py
-   :caption: :src:`examples/arguments.py`
+.. literalinclude:: ../../examples/simple/script_arguments.py
+   :caption: :src:`examples/simple/script_arguments.py`
    :language: python
 
 We can now call this script on the command-line like this:
 
->>> mitmdump -dd -s "./arguments.py html faketml"
+>>> mitmdump -dd -s "./script_arguments.py html faketml"
 
 Whenever a handler is called, mitpmroxy rewrites the script environment so that
 it sees its own arguments as if it was invoked from the command-line.
@@ -78,15 +78,15 @@ Logging and the context
 -----------------------
 
 Scripts should not output straight to stderr or stdout. Instead, the `log
-<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` contexzt module
+<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` context module
 should be used, so that the mitmproxy host program can handle output
-appropriately. So, mitmdump can print colorised sript output to the terminal,
+appropriately. So, mitmdump can print colorised script output to the terminal,
 and mitmproxy console can place script output in the event buffer.
 
 Here's how this looks:
 
-.. literalinclude:: ../../examples/logging.py
-   :caption: :src:`examples/logging.py`
+.. literalinclude:: ../../examples/simple/log_events.py
+   :caption: :src:`examples/simple/log_events.py`
    :language: python
 
 The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
@@ -126,15 +126,32 @@ It's possible to implement a concurrent mechanism on top of the blocking
 framework, and mitmproxy includes a handy example of this that is fit for most
 purposes. You can use it as follows:
 
-.. literalinclude:: ../../examples/nonblocking.py
-   :caption: :src:`examples/nonblocking.py`
+.. literalinclude:: ../../examples/complex/nonblocking.py
+   :caption: :src:`examples/complex/nonblocking.py`
+   :language: python
+
+
+Testing
+-------
+
+Mitmproxy includes a number of helpers for testing addons. The
+``mitmproxy.test.taddons`` module contains a context helper that takes care of
+setting up and tearing down the addon event context. The
+``mitmproxy.test.tflow`` module contains helpers for quickly creating test
+flows. Pydoc is the canonical reference for these modules, and mitmproxy's own
+test suite is an excellent source of examples of usage. Here, for instance, is
+the mitmproxy unit tests for the `anticache` option, demonstrating a good
+cross-section of the test helpers:
+
+.. literalinclude:: ../../test/mitmproxy/addons/test_anticache.py
+   :caption: :src:`test/mitmproxy/addons/test_anticache.py`
    :language: python
 
 
 Developing scripts
 ------------------
 
-Mitmprxoy monitors scripts for modifications, and reloads them on change. When
+Mitmproxy monitors scripts for modifications, and reloads them on change. When
 this happens, the script is shut down (the `done <events.html#done>`_  event is
 called), and the new instance is started up as if the script had just been
 loaded (the `start <events.html#start>`_ and `configure

docs/transparent.rst

@@ -27,7 +27,7 @@ Fully transparent mode
 By default mitmproxy will use its own local ip address for its server-side connections.
 In case this isn't desired, the --spoof-source-address argument can be used to
 use the client's ip address for server-side connections. The following config is
-required for this mode to work:
+required for this mode to work::
 
     CLIENT_NET=192.168.1.0/24
     TABLE_ID=100
@@ -42,9 +42,9 @@ required for this mode to work:
 
 This mode does require root privileges though. There's a wrapper in the examples directory
 called 'mitmproxy_shim.c', which will enable you to use this mode with dropped priviliges.
-It can be used as follows:
+It can be used as follows::
 
-    gcc examples/mitmproxy_shim.c -o mitmproxy_shim -lcap
+    gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap
     sudo chown root:root mitmproxy_shim
     sudo chmod u+s mitmproxy_shim
     ./mitmproxy_shim $(which mitmproxy) -T --spoof-source-address

docs/transparent/openbsd.rst

@@ -0,0 +1,53 @@
+.. _openbsd:
+
+OpenBSD
+=======
+
+ 1. :ref:`Install the mitmproxy certificate on the test device <certinstall>`
+
+ 2. Enable IP forwarding:
+
+    >>> sudo sysctl -w net.inet.ip.forwarding=1
+
+ 3. Place the following two lines in **/etc/pf.conf**:
+
+    .. code-block:: none
+
+        mitm_if = "re2"
+        pass in quick proto tcp from $mitm_if to port { 80, 443 } divert-to 127.0.0.1 port 8080
+
+    These rules tell pf to divert all traffic from ``$mitm_if`` destined for
+    port 80 or 443 to the local mitmproxy instance running on port 8080. You
+    should replace ``$mitm_if`` value with the interface on which your test
+    device will appear.
+
+ 4. Configure pf with the rules:
+
+    >>> doas pfctl -f /etc/pf.conf
+
+ 5. And now enable it:
+
+    >>> doas pfctl -e
+
+ 6. Fire up mitmproxy. You probably want a command like this:
+
+    >>> mitmproxy -T --host
+
+    The ``-T`` flag turns on transparent mode, and the ``--host``
+    argument tells mitmproxy to use the value of the Host header for URL display.
+
+ 7. Finally, configure your test device to use the host on which mitmproxy is
+    running as the default gateway.
+
+.. note::
+
+    Note that the **divert-to** rules in the pf.conf given above only apply to
+    inbound traffic. **This means that they will NOT redirect traffic coming
+    from the box running pf itself.** We can't distinguish between an outbound
+    connection from a non-mitmproxy app, and an outbound connection from
+    mitmproxy itself - if you want to intercept your traffic, you should use an
+    external host to run mitmproxy. Nonetheless, pf is flexible to cater for a
+    range of creative possibilities, like intercepting traffic emanating from
+    VMs.  See the **pf.conf** man page for more.
+
+.. _pf: http://man.openbsd.org/OpenBSD-current/man5/pf.conf.5

docs/transparent/osx.rst

@@ -63,7 +63,7 @@ Note that this means we don't support transparent mode for earlier versions of O
     running pf itself.** We can't distinguish between an outbound connection from a
     non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
     want to intercept your OSX traffic, you should use an external host to run
-    mitmproxy. None the less, pf is flexible to cater for a range of creative
+    mitmproxy. Nonetheless, pf is flexible to cater for a range of creative
     possibilities, like intercepting traffic emanating from VMs.  See the
     **pf.conf** man page for more.
 

docs/tutorials/transparent-dhcp.rst

@@ -38,8 +38,14 @@ DHCP and TFTP) services to a small-scale network.
     **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default
     `[1] <https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/>`_. For our use case, this needs
     to be disabled by changing ``dns=dnsmasq`` to ``#dns=dnsmasq`` in
-    **/etc/NetworkManager/NetworkManager.conf** and running
-
+    **/etc/NetworkManager/NetworkManager.conf** and
+    
+    if on Ubuntu 16.04 or newer running:
+    
+    >>> sudo systemctl restart NetworkManager
+    
+    if on Ubuntu 12.04 or 14.04 running:
+    
     >>> sudo restart network-manager
 
     afterwards.
@@ -61,6 +67,12 @@ DHCP and TFTP) services to a small-scale network.
 
     Apply changes:
 
+    if on Ubuntu 16.04 or newer:
+    
+    >>> sudo systemctl restart dnsmasq
+    
+    if on Ubuntu 12.04 or 14.04:
+    
     >>> sudo service dnsmasq restart
 
     Your **proxied machine** in the internal virtual network should now receive an IP address via DHCP:
@@ -74,8 +86,8 @@ To redirect traffic to mitmproxy, we need to add two iptables rules:
 
 .. code-block:: none
 
-    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
-    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
+    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
+    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
 
 4. Run mitmproxy
 ----------------

examples/README

@@ -1,31 +0,0 @@
-Some inline scripts may require additional dependencies, which can be installed using
-`pip install mitmproxy[examples]`.
-
-
-# inline script examples
-add_header.py             Simple script that just adds a header to every request.
-change_upstream_proxy.py  Dynamically change the upstream proxy
-dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
-dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
-fail_with_500.py          Turn every response into an Internal Server Error.
-filt.py                   Use mitmproxy's filter expressions in your script.
-flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
-iframe_injector.py        Inject configurable iframe into pages.
-modify_form.py            Modify all form submissions to add a parameter.
-modify_querystring.py     Modify all query strings to add a parameters.
-modify_response_body.py   Replace arbitrary strings in all responses
-nonblocking.py            Demonstrate parallel processing with a blocking script.
-proxapp.py                How to embed a WSGI app in a mitmproxy server
-redirect_requests.py      Redirect requests or directly reply to them.
-stub.py                   Script stub with a method definition for every event.
-upsidedownternet.py       Rewrites traffic to turn images upside down.
-
-
-# mitmproxy examples
-flowbasic                 Basic use of mitmproxy as a library.
-stickycookies             An example of writing a custom proxy with mitmproxy.
-
-
-# misc
-read_dumpfile             Read a dumpfile generated by mitmproxy.
-mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/README.md

@@ -0,0 +1,15 @@
+# Mitmproxy Scripting API
+
+Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being 
+proxied. In fact, much of mitmproxy’s own core functionality is implemented using the exact same API 
+exposed to scripters (see [mitmproxy/addons](../mitmproxy/addons)).
+
+This directory contains some examples of the scripting API. We recommend to start with the
+ones in [simple/](./simple).
+
+|  :warning: | If you are browsing this on GitHub, make sure to select the git tag matching your mitmproxy version. |
+|------------|------------------------------------------------------------------------------------------------------|
+
+
+Some inline scripts may require additional dependencies, which can be installed using
+`pip install mitmproxy[examples]`.

examples/complex/README.md

@@ -0,0 +1,18 @@
+## Complex Examples
+
+| Filename                 | Description                                                                                   |
+|:-------------------------|:----------------------------------------------------------------------------------------------|
+| change_upstream_proxy.py | Dynamically change the upstream proxy.                                                        |
+| dns_spoofing.py          | Use mitmproxy in a DNS spoofing scenario.                                                     |
+| dup_and_replay.py        | Duplicates each request, changes it, and then replays the modified request.                   |
+| flowbasic.py             | Basic use of mitmproxy's FlowMaster directly.                                                 |
+| full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. |
+| har_dump.py              | Dump flows as HAR files.                                                                      |
+| mitmproxywrapper.py      | Bracket mitmproxy run with proxy enable/disable on OS X                                       |
+| nonblocking.py           | Demonstrate parallel processing with a blocking script                                        |
+| remote_debug.py          | This script enables remote debugging of the mitmproxy _UI_ with PyCharm.                      |
+| sslstrip.py              | sslstrip-like funtionality implemented with mitmproxy                                         |
+| stream.py                | Enable streaming for all responses.                                                           |
+| stream_modify.py         | Modify a streamed response body.                                                              |
+| tcp_message.py           | Modify a raw TCP connection                                                                   |
+| tls_passthrough.py       | Use conditional TLS interception based on a user-defined strategy.                            |

examples/complex/dns_spoofing.py

@@ -0,0 +1,62 @@
+"""
+This script makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
+connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
+Host header of the HTTP request.
+Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
+know the actual target and cannot construct a certificate that looks valid.
+Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
+Using transparent mode is the better option most of the time.
+
+Usage:
+    mitmproxy
+        -p 443
+        -s dns_spoofing.py
+        # Used as the target location if neither SNI nor host header are present.
+        -R http://example.com/
+    mitmdump
+        -p 80
+        -R http://localhost:443/
+
+    (Setting up a single proxy instance and using iptables to redirect to it
+    works as well)
+"""
+import re
+
+# This regex extracts splits the host header into host and port.
+# Handles the edge case of IPv6 addresses containing colons.
+# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
+parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
+
+
+class Rerouter:
+    def requestheaders(self, flow):
+        """
+        The original host header is retrieved early
+        before flow.request is replaced by mitmproxy new outgoing request
+        """
+        flow.metadata["original_host"] = flow.request.host_header
+
+    def request(self, flow):
+        if flow.client_conn.ssl_established:
+            flow.request.scheme = "https"
+            sni = flow.client_conn.connection.get_servername()
+            port = 443
+        else:
+            flow.request.scheme = "http"
+            sni = None
+            port = 80
+
+        host_header = flow.metadata["original_host"]
+        m = parse_host_header.match(host_header)
+        if m:
+            host_header = m.group("host").strip("[]")
+            if m.group("port"):
+                port = int(m.group("port"))
+
+        flow.request.host_header = host_header
+        flow.request.host = sni or host_header
+        flow.request.port = port
+
+
+def start():
+    return Rerouter()

examples/complex/dup_and_replay.py

@@ -2,5 +2,7 @@ from mitmproxy import ctx
 
 
 def request(flow):
-    f = ctx.master.duplicate_flow(flow)
+    f = flow.copy()
+    ctx.master.view.add(f)
+    f.request.path = "/changed"
     ctx.master.replay_request(f, block=True)

examples/complex/flowbasic.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
@@ -8,14 +8,14 @@
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
-from mitmproxy import flow, controller, options
+from mitmproxy import controller, options, master
 from mitmproxy.proxy import ProxyServer, ProxyConfig
 
 
-class MyMaster(flow.FlowMaster):
+class MyMaster(master.Master):
     def run(self):
         try:
-            flow.FlowMaster.run(self)
+            master.Master.run(self)
         except KeyboardInterrupt:
             self.shutdown()
 
@@ -35,9 +35,9 @@ class MyMaster(flow.FlowMaster):
     def log(self, l):
         print("log", l.msg)
 
+
 opts = options.Options(cadir="~/.mitmproxy/")
 config = ProxyConfig(opts)
-state = flow.State()
 server = ProxyServer(config)
-m = MyMaster(opts, server, state)
+m = MyMaster(opts, server)
 m.run()

examples/complex/har_dump.py

@@ -3,20 +3,20 @@ This inline script can be used to dump flows as HAR files.
 """
 
 
-import pprint
 import json
 import sys
 import base64
 import zlib
+import os
 
 from datetime import datetime
 import pytz
 
 import mitmproxy
 
-from netlib import version
-from netlib import strutils
-from netlib.http import cookies
+from mitmproxy import version
+from mitmproxy.utils import strutils
+from mitmproxy.net.http import cookies
 
 HAR = {}
 
@@ -128,22 +128,25 @@ def response(flow):
         "timings": timings,
     }
 
-    # Store binay data as base64
+    # Store binary data as base64
     if strutils.is_mostly_bin(flow.response.content):
-        b64 = base64.b64encode(flow.response.content)
-        entry["response"]["content"]["text"] = b64.decode('ascii')
+        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
         entry["response"]["content"]["encoding"] = "base64"
     else:
-        entry["response"]["content"]["text"] = flow.response.text
+        entry["response"]["content"]["text"] = flow.response.get_text(strict=False)
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
+        params = [
+            {"name": a, "value": b}
+            for a, b in flow.request.urlencoded_form.items(multi=True)
+        ]
         entry["request"]["postData"] = {
-            "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
-            "text": flow.request.content,
-            "params": name_value(flow.request.urlencoded_form)
+            "mimeType": flow.request.headers.get("Content-Type", ""),
+            "text": flow.request.get_text(strict=False),
+            "params": params
         }
 
-    if flow.server_conn:
+    if flow.server_conn.connected():
         entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
 
     HAR["log"]["entries"].append(entry)
@@ -155,16 +158,17 @@ def done():
     """
     dump_file = sys.argv[1]
 
+    json_dump = json.dumps(HAR, indent=2)  # type: str
+
     if dump_file == '-':
-        mitmproxy.ctx.log(pprint.pformat(HAR))
+        mitmproxy.ctx.log(json_dump)
     else:
-        json_dump = json.dumps(HAR, indent=2)
-
+        raw = json_dump.encode()  # type: bytes
         if dump_file.endswith('.zhar'):
-            json_dump = zlib.compress(json_dump, 9)
+            raw = zlib.compress(raw, 9)
 
-        with open(dump_file, "w") as f:
-            f.write(json_dump)
+        with open(os.path.expanduser(dump_file), "wb") as f:
+            f.write(raw)
 
         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
 

examples/complex/mitmproxywrapper.py

@@ -15,7 +15,7 @@ import os
 import sys
 
 
-class Wrapper(object):
+class Wrapper:
     def __init__(self, port, extra_arguments=None):
         self.port = port
         self.extra_arguments = extra_arguments

examples/complex/nonblocking.py

@@ -0,0 +1,11 @@
+import time
+
+from mitmproxy.script import concurrent
+
+
+@concurrent  # Remove this and see what happens
+def request(flow):
+    # You don't want to use mitmproxy.ctx from a different thread
+    print("handle request: %s%s" % (flow.request.host, flow.request.path))
+    time.sleep(5)
+    print("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/complex/sslstrip.py

@@ -1,5 +1,9 @@
+"""
+This script implements an sslstrip-like attack based on mitmproxy.
+https://moxie.org/software/sslstrip/
+"""
 import re
-from six.moves import urllib
+import urllib
 
 # set of SSL/TLS capable hosts
 secure_hosts = set()
@@ -17,13 +21,18 @@ def request(flow):
         flow.request.scheme = 'https'
         flow.request.port = 443
 
+        # We need to update the request destination to whatever is specified in the host header:
+        # Having no TLS Server Name Indication from the client and just an IP address as request.host
+        # in transparent mode, TLS server name certificate validation would fail.
+        flow.request.host = flow.request.pretty_host
+
 
 def response(flow):
     flow.response.headers.pop('Strict-Transport-Security', None)
     flow.response.headers.pop('Public-Key-Pins', None)
 
     # strip links in response body
-    flow.response.content = flow.response.content.replace('https://', 'http://')
+    flow.response.content = flow.response.content.replace(b'https://', b'http://')
 
     # strip meta tag upgrade-insecure-requests in response body
     csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>'

examples/complex/stream.py

@@ -1,5 +1,6 @@
 def responseheaders(flow):
     """
     Enables streaming for all responses.
+    This is equivalent to passing `--stream 0` to mitmproxy.
     """
     flow.response.stream = True

examples/complex/tcp_message.py

@@ -8,7 +8,7 @@ tcp_message Inline Script Hook API Demonstration
 example cmdline invocation:
 mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py
 """
-from netlib import strutils
+from mitmproxy.utils import strutils
 
 
 def tcp_message(tcp_msg):

examples/complex/tls_passthrough.py

@@ -20,7 +20,6 @@ Example:
 
 Authors: Maximilian Hils, Matthew Tuusberg
 """
-from __future__ import absolute_import, print_function, division
 import collections
 import random
 
@@ -29,7 +28,7 @@ from enum import Enum
 
 import mitmproxy
 from mitmproxy.exceptions import TlsProtocolException
-from mitmproxy.protocol import TlsLayer, RawTCPLayer
+from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
 
 
 class InterceptionResult(Enum):
@@ -38,7 +37,7 @@ class InterceptionResult(Enum):
     skipped = None
 
 
-class _TlsStrategy(object):
+class _TlsStrategy:
     """
     Abstract base class for interception strategies.
     """

examples/custom_contentviews.py

@@ -1,70 +0,0 @@
-import string
-import lxml.html
-import lxml.etree
-from mitmproxy import contentviews
-from netlib import strutils
-
-
-class ViewPigLatin(contentviews.View):
-    name = "pig_latin_HTML"
-    prompt = ("pig latin HTML", "l")
-    content_types = ["text/html"]
-
-    def __call__(self, data, **metadata):
-        if strutils.is_xml(data):
-            parser = lxml.etree.HTMLParser(
-                strip_cdata=True,
-                remove_blank_text=True
-            )
-            d = lxml.html.fromstring(data, parser=parser)
-            docinfo = d.getroottree().docinfo
-
-            def piglify(src):
-                words = src.split()
-                ret = ''
-                for word in words:
-                    idx = -1
-                    while word[idx] in string.punctuation and (idx * -1) != len(word):
-                        idx -= 1
-                    if word[0].lower() in 'aeiou':
-                        if idx == -1:
-                            ret += word[0:] + "hay"
-                        else:
-                            ret += word[0:len(word) + idx + 1] + "hay" + word[idx + 1:]
-                    else:
-                        if idx == -1:
-                            ret += word[1:] + word[0] + "ay"
-                        else:
-                            ret += word[1:len(word) + idx + 1] + word[0] + "ay" + word[idx + 1:]
-                    ret += ' '
-                return ret.strip()
-
-            def recurse(root):
-                if hasattr(root, 'text') and root.text:
-                    root.text = piglify(root.text)
-                if hasattr(root, 'tail') and root.tail:
-                    root.tail = piglify(root.tail)
-
-                if len(root):
-                    for child in root:
-                        recurse(child)
-
-            recurse(d)
-
-            s = lxml.etree.tostring(
-                d,
-                pretty_print=True,
-                doctype=docinfo.doctype
-            )
-            return "HTML", contentviews.format_text(s)
-
-
-pig_view = ViewPigLatin()
-
-
-def start():
-    contentviews.add(pig_view)
-
-
-def done():
-    contentviews.remove(pig_view)

examples/dns_spoofing.py

@@ -1,49 +0,0 @@
-"""
-This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
-connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
-Host header of the HTTP request.
-Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
-know the actual target and cannot construct a certificate that looks valid.
-Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
-Using transparent mode is the better option most of the time.
-
-Usage:
-    mitmproxy
-        -p 443
-        -s dns_spoofing.py
-        # Used as the target location if neither SNI nor host header are present.
-        -R http://example.com/
-    mitmdump
-        -p 80
-        -R http://localhost:443/
-
-    (Setting up a single proxy instance and using iptables to redirect to it
-    works as well)
-"""
-import re
-
-# This regex extracts splits the host header into host and port.
-# Handles the edge case of IPv6 addresses containing colons.
-# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
-parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
-
-
-def request(flow):
-    if flow.client_conn.ssl_established:
-        flow.request.scheme = "https"
-        sni = flow.client_conn.connection.get_servername()
-        port = 443
-    else:
-        flow.request.scheme = "http"
-        sni = None
-        port = 80
-
-    host_header = flow.request.pretty_host
-    m = parse_host_header.match(host_header)
-    if m:
-        host_header = m.group("host").strip("[]")
-        if m.group("port"):
-            port = int(m.group("port"))
-
-    flow.request.host = sni or host_header
-    flow.request.port = port

examples/dup_and_replay.py

@@ -1,7 +0,0 @@
-from mitmproxy import master
-
-
-def request(flow):
-    f = master.duplicate_flow(flow)
-    f.request.path = "/changed"
-    master.replay_request(f, block=True, run_scripthooks=False)

examples/fail_with_500.py

@@ -1,3 +0,0 @@
-def response(flow):
-    flow.response.status_code = 500
-    flow.response.content = b""

examples/logging.py

@@ -1,6 +0,0 @@
-from mitmproxy import ctx
-
-
-def start():
-    ctx.log.info("This is some informative text.")
-    ctx.log.error("This is an error.")

examples/nonblocking.py

@@ -1,10 +0,0 @@
-import time
-import mitmproxy
-from mitmproxy.script import concurrent
-
-
-@concurrent  # Remove this and see what happens
-def request(flow):
-    mitmproxy.ctx.log("handle request: %s%s" % (flow.request.host, flow.request.path))
-    time.sleep(5)
-    mitmproxy.ctx.log("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/redirect_requests.py

@@ -1,19 +0,0 @@
-"""
-This example shows two ways to redirect flows to other destinations.
-"""
-from mitmproxy.models import HTTPResponse
-
-
-def request(flow):
-    # pretty_host takes the "Host" header of the request into account,
-    # which is useful in transparent mode where we usually only have the IP
-    # otherwise.
-
-    # Method 1: Answer with a locally generated response
-    if flow.request.pretty_host.endswith("example.com"):
-        resp = HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"})
-        flow.reply.send(resp)
-
-    # Method 2: Redirect the request to a different server
-    if flow.request.pretty_host.endswith("example.org"):
-        flow.request.host = "mitmproxy.org"

examples/simple/README.md

@@ -0,0 +1,18 @@
+## Simple Examples
+
+| Filename                     | Description                                                                |
+|:-----------------------------|:---------------------------------------------------------------------------|
+| add_header.py                | Simple script that just adds a header to every request.                    |
+| custom_contentview.py        | Add a custom content view to the mitmproxy UI.                             |
+| filter_flows.py              | This script demonstrates how to use mitmproxy's filter pattern in scripts. |
+| io_read_dumpfile.py          | Read a dumpfile generated by mitmproxy.                                    |
+| io_write_dumpfile.py         | Only write selected flows into a mitmproxy dumpfile.                       |
+| log_events.py                   | Use mitmproxy's logging API.                                            |
+| modify_body_inject_iframe.py | Inject configurable iframe into pages.                                     |
+| modify_form.py               | Modify HTTP form submissions.                                              |
+| modify_querystring.py        | Modify HTTP query strings.                                                 |
+| redirect_requests.py         | Redirect a request to a different server.                                  |
+| script_arguments.py          | Add arguments to a script.                                                 |
+| send_reply_from_proxy.py     | Send a HTTP response directly from the proxy.                              |
+| upsidedownternet.py          | Turn all images upside down.                                               |
+| wsgi_flask_app.py            | Embed a WSGI app into mitmproxy.                                           |

examples/simple/custom_contentview.py

@@ -0,0 +1,28 @@
+"""
+This example shows how one can add a custom contentview to mitmproxy.
+The content view API is explained in the mitmproxy.contentviews module.
+"""
+from mitmproxy import contentviews
+
+
+class ViewSwapCase(contentviews.View):
+    name = "swapcase"
+
+    # We don't have a good solution for the keyboard shortcut yet -
+    # you manually need to find a free letter. Contributions welcome :)
+    prompt = ("swap case text", "z")
+    content_types = ["text/plain"]
+
+    def __call__(self, data: bytes, **metadata):
+        return "case-swapped text", contentviews.format_text(data.swapcase())
+
+
+view = ViewSwapCase()
+
+
+def start():
+    contentviews.add(view)
+
+
+def done():
+    contentviews.remove(view)

examples/simple/filter_flows.py

@@ -1,6 +1,8 @@
-# This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
-# Usage: mitmdump -s "flowfilter.py FILTER"
-
+"""
+This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
+Usage:
+    mitmdump -s "flowfilter.py FILTER"
+"""
 import sys
 from mitmproxy import flowfilter
 
@@ -10,7 +12,7 @@ class Filter:
         self.filter = flowfilter.parse(spec)
 
     def response(self, flow):
-        if flowfilter.match(flow, self.filter):
+        if flowfilter.match(self.filter, flow):
             print("Flow matches filter:")
             print(flow)
 

examples/simple/io_read_dumpfile.py

@@ -3,13 +3,13 @@
 # Simple script showing how to read a mitmproxy dump file
 #
 
-from mitmproxy import flow
+from mitmproxy import io
 from mitmproxy.exceptions import FlowReadException
 import pprint
 import sys
 
 with open(sys.argv[1], "rb") as logfile:
-    freader = flow.FlowReader(logfile)
+    freader = io.FlowReader(logfile)
     pp = pprint.PrettyPrinter(indent=4)
     try:
         for f in freader.stream():

examples/simple/io_write_dumpfile.py

@@ -1,7 +1,13 @@
+"""
+This script how to generate a mitmproxy dump file,
+as it would also be generated by passing `-w` to mitmproxy.
+In contrast to `-w`, this gives you full control over which
+flows should be saved and also allows you to rotate files or log
+to multiple files in parallel.
+"""
 import random
 import sys
-
-from mitmproxy.flow import FlowWriter
+from mitmproxy import io
 
 
 class Writer:
@@ -10,7 +16,7 @@ class Writer:
             f = sys.stdout
         else:
             f = open(path, "wb")
-        self.w = FlowWriter(f)
+        self.w = io.FlowWriter(f)
 
     def response(self, flow):
         if random.choice([True, False]):

examples/simple/log_events.py

@@ -0,0 +1,12 @@
+"""
+It is recommended to use `ctx.log` for logging within a script.
+This goes to the event log in mitmproxy and to stdout in mitmdump.
+
+If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 :-)
+"""
+from mitmproxy import ctx
+
+
+def start():
+    ctx.log.info("This is some informative text.")
+    ctx.log.error("This is an error.")

examples/simple/modify_body_inject_iframe.py

@@ -11,7 +11,7 @@ class Injector:
     def response(self, flow):
         if flow.request.host in self.iframe_url:
             return
-        html = BeautifulSoup(flow.response.content, "lxml")
+        html = BeautifulSoup(flow.response.content, "html.parser")
         if html.body:
             iframe = html.new_tag(
                 "iframe",

examples/simple/modify_form.py

@@ -1,7 +1,9 @@
 def request(flow):
     if flow.request.urlencoded_form:
+        # If there's already a form, one can just add items to the dict:
         flow.request.urlencoded_form["mitmproxy"] = "rocks"
     else:
+        # One can also just pass new form data.
         # This sets the proper content type and overrides the body.
         flow.request.urlencoded_form = [
             ("foo", "bar")

examples/simple/redirect_requests.py

@@ -0,0 +1,11 @@
+"""
+This example shows two ways to redirect flows to another server.
+"""
+
+
+def request(flow):
+    # pretty_host takes the "Host" header of the request into account,
+    # which is useful in transparent mode where we usually only have the IP
+    # otherwise.
+    if flow.request.pretty_host == "example.org":
+        flow.request.host = "mitmproxy.org"

examples/simple/send_reply_from_proxy.py

@@ -0,0 +1,17 @@
+"""
+This example shows how to send a reply from the proxy immediately
+without sending any data to the remote server.
+"""
+from mitmproxy import http
+
+
+def request(flow):
+    # pretty_url takes the "Host" header of the request into account, which
+    # is useful in transparent mode where we usually only have the IP otherwise.
+
+    if flow.request.pretty_url == "http://example.com/path":
+        flow.response = http.HTTPResponse.make(
+            200,  # (optional) status code
+            b"Hello World",  # (optional) content
+            {"Content-Type": "text/html"}  # (optional) headers
+        )

examples/simple/upsidedownternet.py

@@ -0,0 +1,16 @@
+"""
+This script rotates all images passing through the proxy by 180 degrees.
+"""
+import io
+
+from PIL import Image
+
+
+def response(flow):
+    if flow.response.headers.get("content-type", "").startswith("image"):
+        s = io.BytesIO(flow.response.content)
+        img = Image.open(s).rotate(180)
+        s2 = io.BytesIO()
+        img.save(s2, "png")
+        flow.response.content = s2.getvalue()
+        flow.response.headers["content-type"] = "image/png"

examples/simple/wsgi_flask_app.py

@@ -4,7 +4,7 @@ instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
 a single simplest-possible page.
 """
 from flask import Flask
-import mitmproxy
+from mitmproxy.addons import wsgiapp
 
 app = Flask("proxapp")
 
@@ -14,12 +14,12 @@ def hello_world():
     return 'Hello World!'
 
 
-# Register the app using the magic domain "proxapp" on port 80. Requests to
-# this domain and port combination will now be routed to the WSGI app instance.
 def start():
-    mitmproxy.ctx.master.apps.add(app, "proxapp", 80)
+    # Host app at the magic domain "proxapp" on port 80. Requests to this
+    # domain and port combination will now be routed to the WSGI app instance.
+    return wsgiapp.WSGIApp(app, "proxapp", 80)
 
     # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
     # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
     # but won't send any data.
-    mitmproxy.ctx.master.apps.add(app, "example.com", 443)
+    # mitmproxy.ctx.master.apps.add(app, "example.com", 443)

examples/stickycookies

@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-"""
-This example builds on mitmproxy's base proxying infrastructure to
-implement functionality similar to the "sticky cookies" option.
-
-Heads Up: In the majority of cases, you want to use inline scripts.
-"""
-import os
-from mitmproxy import controller, proxy
-from mitmproxy.proxy.server import ProxyServer
-
-
-class StickyMaster(controller.Master):
-    def __init__(self, server):
-        controller.Master.__init__(self, server)
-        self.stickyhosts = {}
-
-    def run(self):
-        try:
-            return controller.Master.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    @controller.handler
-    def request(self, flow):
-        hid = (flow.request.host, flow.request.port)
-        if "cookie" in flow.request.headers:
-            self.stickyhosts[hid] = flow.request.headers.get_all("cookie")
-        elif hid in self.stickyhosts:
-            flow.request.headers.set_all("cookie", self.stickyhosts[hid])
-
-    @controller.handler
-    def response(self, flow):
-        hid = (flow.request.host, flow.request.port)
-        if "set-cookie" in flow.response.headers:
-            self.stickyhosts[hid] = flow.response.headers.get_all("set-cookie")
-
-
-config = proxy.ProxyConfig(port=8080)
-server = ProxyServer(config)
-m = StickyMaster(server)
-m.run()

examples/stub.py

@@ -1,87 +0,0 @@
-import mitmproxy
-"""
-    This is a script stub, with definitions for all events.
-"""
-
-
-def start():
-    """
-        Called once on script startup before any other events
-    """
-    mitmproxy.ctx.log("start")
-
-
-def configure(options, updated):
-    """
-        Called once on script startup before any other events, and whenever options changes.
-    """
-    mitmproxy.ctx.log("configure")
-
-
-def clientconnect(root_layer):
-    """
-        Called when a client initiates a connection to the proxy. Note that a
-        connection can correspond to multiple HTTP requests
-    """
-    mitmproxy.ctx.log("clientconnect")
-
-
-def request(flow):
-    """
-        Called when a client request has been received.
-    """
-    mitmproxy.ctx.log("request")
-
-
-def serverconnect(server_conn):
-    """
-        Called when the proxy initiates a connection to the target server. Note that a
-        connection can correspond to multiple HTTP requests
-    """
-    mitmproxy.ctx.log("serverconnect")
-
-
-def responseheaders(flow):
-    """
-        Called when the response headers for a server response have been received,
-        but the response body has not been processed yet. Can be used to tell mitmproxy
-        to stream the response.
-    """
-    mitmproxy.ctx.log("responseheaders")
-
-
-def response(flow):
-    """
-       Called when a server response has been received.
-    """
-    mitmproxy.ctx.log("response")
-
-
-def error(flow):
-    """
-        Called when a flow error has occured, e.g. invalid server responses, or
-        interrupted connections. This is distinct from a valid server HTTP error
-        response, which is simply a response with an HTTP error code.
-    """
-    mitmproxy.ctx.log("error")
-
-
-def serverdisconnect(server_conn):
-    """
-        Called when the proxy closes the connection to the target server.
-    """
-    mitmproxy.ctx.log("serverdisconnect")
-
-
-def clientdisconnect(root_layer):
-    """
-        Called when a client disconnects from the proxy.
-    """
-    mitmproxy.ctx.log("clientdisconnect")
-
-
-def done():
-    """
-        Called once on script shutdown, after any other events.
-    """
-    mitmproxy.ctx.log("done")

examples/upsidedownternet.py

@@ -1,15 +0,0 @@
-from six.moves import cStringIO as StringIO
-from PIL import Image
-
-
-def response(flow):
-    if flow.response.headers.get("content-type", "").startswith("image"):
-        try:
-            s = StringIO(flow.response.content)
-            img = Image.open(s).rotate(180)
-            s2 = StringIO()
-            img.save(s2, "png")
-            flow.response.content = s2.getvalue()
-            flow.response.headers["content-type"] = "image/png"
-        except:  # Unknown image types etc.
-            pass

issue_template.md

@@ -1,22 +1,20 @@
 ##### Steps to reproduce the problem:
 
-1. 
-2. 
-3. 
-
-##### What is the expected behavior?
-
-
-##### What went wrong?
+1.
+2.
+3.
 
 
 ##### Any other comments? What have you tried so far?
 
 
----
 
-Mitmproxy Version:
-Operating System:
+##### System information
 
 
-<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) -->
+<!--
+    Cut and paste the output of "mitmproxy --version".
+
+    If you're using an older version if mitmproxy, please specify the version
+    and OS.
+-->

mitmproxy/__init__.py

@@ -0,0 +1,3 @@
+# https://github.com/mitmproxy/mitmproxy/issues/1809
+# import script here so that pyinstaller registers it.
+from . import script  # noqa

mitmproxy/addonmanager.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import, print_function, division
 from mitmproxy import exceptions
 import pprint
 
@@ -7,12 +6,19 @@ def _get_name(itm):
     return getattr(itm, "name", itm.__class__.__name__.lower())
 
 
-class Addons(object):
+class AddonManager:
     def __init__(self, master):
         self.chain = []
         self.master = master
         master.options.changed.connect(self._options_update)
 
+    def clear(self):
+        """
+            Remove all addons.
+        """
+        self.done()
+        self.chain = []
+
     def get(self, name):
         """
             Retrieve an addon by name. Addon names are equal to the .name
@@ -26,7 +32,7 @@ class Addons(object):
     def _options_update(self, options, updated):
         for i in self.chain:
             with self.master.handlecontext():
-                i.configure(options, updated)
+                self.invoke_with_context(i, "configure", options, updated)
 
     def startup(self, s):
         """
@@ -44,8 +50,6 @@ class Addons(object):
         """
             Add addons to the end of the chain, and run their startup events.
         """
-        if not addons:
-            raise ValueError("No addons specified.")
         self.chain.extend(addons)
         for i in addons:
             self.startup(i)
@@ -82,4 +86,7 @@ class Addons(object):
 
     def __call__(self, name, *args, **kwargs):
         for i in self.chain:
-            self.invoke(i, name, *args, **kwargs)
+            try:
+                self.invoke(i, name, *args, **kwargs)
+            except exceptions.AddonHalt:
+                return

mitmproxy/addons/__init__.py

@@ -0,0 +1,40 @@
+from mitmproxy.addons import anticache
+from mitmproxy.addons import anticomp
+from mitmproxy.addons import check_alpn
+from mitmproxy.addons import check_ca
+from mitmproxy.addons import clientplayback
+from mitmproxy.addons import disable_h2c_upgrade
+from mitmproxy.addons import onboarding
+from mitmproxy.addons import proxyauth
+from mitmproxy.addons import replace
+from mitmproxy.addons import script
+from mitmproxy.addons import serverplayback
+from mitmproxy.addons import setheaders
+from mitmproxy.addons import stickyauth
+from mitmproxy.addons import stickycookie
+from mitmproxy.addons import streambodies
+from mitmproxy.addons import streamfile
+from mitmproxy.addons import upstream_auth
+
+
+def default_addons():
+    return [
+        anticache.AntiCache(),
+        anticomp.AntiComp(),
+        check_alpn.CheckALPN(),
+        check_ca.CheckCA(),
+        clientplayback.ClientPlayback(),
+        disable_h2c_upgrade.DisableH2CleartextUpgrade(),
+        onboarding.Onboarding(),
+        proxyauth.ProxyAuth(),
+        replace.Replace(),
+        replace.ReplaceFile(),
+        script.ScriptLoader(),
+        serverplayback.ServerPlayback(),
+        setheaders.SetHeaders(),
+        stickyauth.StickyAuth(),
+        stickycookie.StickyCookie(),
+        streambodies.StreamBodies(),
+        streamfile.StreamFile(),
+        upstream_auth.UpstreamAuth(),
+    ]

mitmproxy/addons/anticache.py

@@ -1,6 +1,3 @@
-from __future__ import absolute_import, print_function, division
-
-
 class AntiCache:
     def __init__(self):
         self.enabled = False

mitmproxy/addons/anticomp.py

@@ -1,6 +1,3 @@
-from __future__ import absolute_import, print_function, division
-
-
 class AntiComp:
     def __init__(self):
         self.enabled = False

mitmproxy/addons/check_alpn.py

@@ -0,0 +1,17 @@
+import mitmproxy
+from mitmproxy.net import tcp
+
+
+class CheckALPN:
+    def __init__(self):
+        self.failed = False
+
+    def configure(self, options, updated):
+        self.failed = mitmproxy.ctx.master.options.http2 and not tcp.HAS_ALPN
+        if self.failed:
+            mitmproxy.ctx.master.add_log(
+                "HTTP/2 is disabled because ALPN support missing!\n"
+                "OpenSSL 1.0.2+ required to support HTTP/2 connections.\n"
+                "Use --no-http2 to silence this warning.",
+                "warn",
+            )

mitmproxy/addons/check_ca.py

@@ -0,0 +1,24 @@
+import mitmproxy
+
+
+class CheckCA:
+    def __init__(self):
+        self.failed = False
+
+    def configure(self, options, updated):
+        has_ca = (
+            mitmproxy.ctx.master.server and
+            mitmproxy.ctx.master.server.config and
+            mitmproxy.ctx.master.server.config.certstore and
+            mitmproxy.ctx.master.server.config.certstore.default_ca
+        )
+        if has_ca:
+            self.failed = mitmproxy.ctx.master.server.config.certstore.default_ca.has_expired()
+            if self.failed:
+                mitmproxy.ctx.master.add_log(
+                    "The mitmproxy certificate authority has expired!\n"
+                    "Please delete all CA-related files in your ~/.mitmproxy folder.\n"
+                    "The CA will be regenerated automatically after restarting mitmproxy.\n"
+                    "Then make sure all your clients have the new CA installed.",
+                    "warn",
+                )

mitmproxy/addons/clientplayback.py

@@ -1,19 +1,24 @@
-from mitmproxy import exceptions, flow, ctx
+from mitmproxy import exceptions
+from mitmproxy import ctx
+from mitmproxy import io
+from mitmproxy import flow
+
+import typing
 
 
 class ClientPlayback:
     def __init__(self):
         self.flows = None
-        self.current = None
-        self.keepserving = None
+        self.current_thread = None
+        self.keepserving = False
         self.has_replayed = False
 
-    def count(self):
+    def count(self) -> int:
         if self.flows:
             return len(self.flows)
         return 0
 
-    def load(self, flows):
+    def load(self, flows: typing.Sequence[flow.Flow]):
         self.flows = flows
 
     def configure(self, options, updated):
@@ -21,7 +26,7 @@ class ClientPlayback:
             if options.client_replay:
                 ctx.log.info("Client Replay: {}".format(options.client_replay))
                 try:
-                    flows = flow.read_flows_from_paths(options.client_replay)
+                    flows = io.read_flows_from_paths(options.client_replay)
                 except exceptions.FlowReadException as e:
                     raise exceptions.OptionsError(str(e))
                 self.load(flows)
@@ -30,11 +35,11 @@ class ClientPlayback:
         self.keepserving = options.keepserving
 
     def tick(self):
-        if self.current and not self.current.is_alive():
-            self.current = None
-        if self.flows and not self.current:
-            self.current = ctx.master.replay_request(self.flows.pop(0))
+        if self.current_thread and not self.current_thread.is_alive():
+            self.current_thread = None
+        if self.flows and not self.current_thread:
+            self.current_thread = ctx.master.replay_request(self.flows.pop(0))
             self.has_replayed = True
         if self.has_replayed:
-            if not self.flows and not self.current and not self.keepserving:
+            if not self.flows and not self.current_thread and not self.keepserving:
                 ctx.master.shutdown()

mitmproxy/addons/disable_h2c_upgrade.py

@@ -0,0 +1,21 @@
+class DisableH2CleartextUpgrade:
+
+    """
+    We currently only support HTTP/2 over a TLS connection. Some clients try
+    to upgrade a connection from HTTP/1.1 to h2c, so we need to remove those
+    headers to avoid protocol errors if one endpoints suddenly starts sending
+    HTTP/2 frames.
+    """
+
+    def process_flow(self, f):
+        if f.request.headers.get('upgrade', '') == 'h2c':
+            del f.request.headers['upgrade']
+            if 'connection' in f.request.headers:
+                del f.request.headers['connection']
+            if 'http2-settings' in f.request.headers:
+                del f.request.headers['http2-settings']
+
+    # Handlers
+
+    def request(self, f):
+        self.process_flow(f)

mitmproxy/addons/dumper.py

@@ -1,8 +1,8 @@
-from __future__ import absolute_import, print_function, division
-
 import itertools
+import sys
 
 import click
+import shutil
 
 import typing  # noqa
 
@@ -10,35 +10,43 @@ from mitmproxy import contentviews
 from mitmproxy import ctx
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
-from netlib import human
-from netlib import strutils
+from mitmproxy.utils import human
+from mitmproxy.utils import strutils
 
 
-def indent(n, text):
+def indent(n: int, text: str) -> str:
     l = str(text).strip().splitlines()
     pad = " " * n
     return "\n".join(pad + i for i in l)
 
 
-class Dumper(object):
-    def __init__(self):
+def colorful(line, styles):
+    yield u"    "  # we can already indent here
+    for (style, text) in line:
+        yield click.style(text, **styles.get(style, {}))
+
+
+class Dumper:
+    def __init__(self, outfile=sys.stdout):
         self.filter = None  # type: flowfilter.TFilter
         self.flow_detail = None  # type: int
-        self.outfp = None  # type: typing.io.TextIO
+        self.outfp = outfile  # type: typing.io.TextIO
         self.showhost = None  # type: bool
+        self.default_contentview = "auto"  # type: str
 
     def configure(self, options, updated):
-        if options.filtstr:
-            self.filter = flowfilter.parse(options.filtstr)
-            if not self.filter:
-                raise exceptions.OptionsError(
-                    "Invalid filter expression: %s" % options.filtstr
-                )
-        else:
-            self.filter = None
+        if "filtstr" in updated:
+            if options.filtstr:
+                self.filter = flowfilter.parse(options.filtstr)
+                if not self.filter:
+                    raise exceptions.OptionsError(
+                        "Invalid filter expression: %s" % options.filtstr
+                    )
+            else:
+                self.filter = None
         self.flow_detail = options.flow_detail
-        self.outfp = options.tfile
         self.showhost = options.showhost
+        self.default_contentview = options.default_contentview
 
     def echo(self, text, ident=None, **style):
         if ident:
@@ -47,66 +55,50 @@ class Dumper(object):
         if self.outfp:
             self.outfp.flush()
 
-    def _echo_message(self, message):
-        if self.flow_detail >= 2 and hasattr(message, "headers"):
-            headers = "\r\n".join(
-                "{}: {}".format(
-                    click.style(
-                        strutils.bytes_to_escaped_str(k), fg="blue", bold=True
-                    ),
-                    click.style(
-                        strutils.bytes_to_escaped_str(v), fg="blue"
-                    )
-                )
-                for k, v in message.headers.fields
+    def _echo_headers(self, headers):
+        for k, v in headers.fields:
+            k = strutils.bytes_to_escaped_str(k)
+            v = strutils.bytes_to_escaped_str(v)
+            out = "{}: {}".format(
+                click.style(k, fg="blue"),
+                click.style(v)
             )
-            self.echo(headers, ident=4)
-        if self.flow_detail >= 3:
-                _, lines, error = contentviews.get_message_content_view(
-                    contentviews.get("Auto"),
-                    message
-                )
-                if error:
-                    ctx.log.debug(error)
+            self.echo(out, ident=4)
 
-                styles = dict(
-                    highlight=dict(bold=True),
-                    offset=dict(fg="blue"),
-                    header=dict(fg="green", bold=True),
-                    text=dict(fg="green")
-                )
+    def _echo_message(self, message):
+        _, lines, error = contentviews.get_message_content_view(
+            self.default_contentview,
+            message
+        )
+        if error:
+            ctx.log.debug(error)
 
-                def colorful(line):
-                    yield u"    "  # we can already indent here
-                    for (style, text) in line:
-                        yield click.style(text, **styles.get(style, {}))
+        if self.flow_detail == 3:
+            lines_to_echo = itertools.islice(lines, 70)
+        else:
+            lines_to_echo = lines
 
-                if self.flow_detail == 3:
-                    lines_to_echo = itertools.islice(lines, 70)
-                else:
-                    lines_to_echo = lines
+        styles = dict(
+            highlight=dict(bold=True),
+            offset=dict(fg="blue"),
+            header=dict(fg="green", bold=True),
+            text=dict(fg="green")
+        )
 
-                content = u"\r\n".join(
-                    u"".join(colorful(line)) for line in lines_to_echo
-                )
-                if content:
-                    self.echo("")
-                    self.echo(content)
+        content = u"\r\n".join(
+            u"".join(colorful(line, styles)) for line in lines_to_echo
+        )
+        if content:
+            self.echo("")
+            self.echo(content)
 
-                if next(lines, None):
-                    self.echo("(cut off)", ident=4, dim=True)
+        if next(lines, None):
+            self.echo("(cut off)", ident=4, dim=True)
 
         if self.flow_detail >= 2:
             self.echo("")
 
     def _echo_request_line(self, flow):
-        if flow.request.stickycookie:
-            stickycookie = click.style(
-                "[stickycookie] ", fg="yellow", bold=True
-            )
-        else:
-            stickycookie = ""
-
         if flow.client_conn:
             client = click.style(
                 strutils.escape_control_characters(
@@ -118,7 +110,8 @@ class Dumper(object):
         else:
             client = ""
 
-        method = flow.request.method
+        pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
+        method = flow.request.method + pushed
         method_color = dict(
             GET="green",
             DELETE="red"
@@ -132,6 +125,9 @@ class Dumper(object):
             url = flow.request.pretty_url
         else:
             url = flow.request.url
+        terminalWidthLimit = max(shutil.get_terminal_size()[0] - 25, 50)
+        if self.flow_detail < 1 and len(url) > terminalWidthLimit:
+            url = url[:terminalWidthLimit] + "…"
         url = click.style(strutils.escape_control_characters(url), bold=True)
 
         http_version = ""
@@ -139,15 +135,8 @@ class Dumper(object):
             # We hide "normal" HTTP 1.
             http_version = " " + flow.request.http_version
 
-        if self.flow_detail >= 2:
-            linebreak = "\n    "
-        else:
-            linebreak = ""
-
-        line = "{client}: {linebreak}{stickycookie}{method} {url}{http_version}".format(
+        line = "{client}: {method} {url}{http_version}".format(
             client=client,
-            stickycookie=stickycookie,
-            linebreak=linebreak,
             method=method,
             url=url,
             http_version=http_version
@@ -205,11 +194,17 @@ class Dumper(object):
     def echo_flow(self, f):
         if f.request:
             self._echo_request_line(f)
-            self._echo_message(f.request)
+            if self.flow_detail >= 2:
+                self._echo_headers(f.request.headers)
+            if self.flow_detail >= 3:
+                self._echo_message(f.request)
 
         if f.response:
             self._echo_response_line(f)
-            self._echo_message(f.response)
+            if self.flow_detail >= 2:
+                self._echo_headers(f.response.headers)
+            if self.flow_detail >= 3:
+                self._echo_message(f.response)
 
         if f.error:
             msg = strutils.escape_control_characters(f.error.msg)
@@ -232,6 +227,29 @@ class Dumper(object):
         if self.match(f):
             self.echo_flow(f)
 
+    def websocket_error(self, f):
+        self.echo(
+            "Error in WebSocket connection to {}: {}".format(
+                repr(f.server_conn.address), f.error
+            ),
+            fg="red"
+        )
+
+    def websocket_message(self, f):
+        if self.match(f):
+            message = f.messages[-1]
+            self.echo(f.message_info(message))
+            if self.flow_detail >= 3:
+                self._echo_message(message)
+
+    def websocket_end(self, f):
+        if self.match(f):
+            self.echo("WebSocket connection closed by {}: {} {}, {}".format(
+                f.close_sender,
+                f.close_code,
+                f.close_message,
+                f.close_reason))
+
     def tcp_error(self, f):
         self.echo(
             "Error in TCP connection to {}: {}".format(
@@ -241,13 +259,13 @@ class Dumper(object):
         )
 
     def tcp_message(self, f):
-        if not self.match(f):
-            return
-        message = f.messages[-1]
-        direction = "->" if message.from_client else "<-"
-        self.echo("{client} {direction} tcp {direction} {server}".format(
-            client=repr(f.client_conn.address),
-            server=repr(f.server_conn.address),
-            direction=direction,
-        ))
-        self._echo_message(message)
+        if self.match(f):
+            message = f.messages[-1]
+            direction = "->" if message.from_client else "<-"
+            self.echo("{client} {direction} tcp {direction} {server}".format(
+                client=repr(f.client_conn.address),
+                server=repr(f.server_conn.address),
+                direction=direction,
+            ))
+            if self.flow_detail >= 3:
+                self._echo_message(message)

mitmproxy/addons/eventstore.py

@@ -0,0 +1,19 @@
+from typing import List  # noqa
+
+import blinker
+from mitmproxy.log import LogEntry
+
+
+class EventStore:
+    def __init__(self):
+        self.data = []  # type: List[LogEntry]
+        self.sig_add = blinker.Signal()
+        self.sig_refresh = blinker.Signal()
+
+    def log(self, entry: LogEntry):
+        self.data.append(entry)
+        self.sig_add.send(self, entry=entry)
+
+    def clear(self):
+        self.data.clear()
+        self.sig_refresh.send(self)

mitmproxy/addons/intercept.py

@@ -0,0 +1,35 @@
+from mitmproxy import flowfilter
+from mitmproxy import exceptions
+
+
+class Intercept:
+    def __init__(self):
+        self.filt = None
+
+    def configure(self, opts, updated):
+        if "intercept" in updated:
+            if not opts.intercept:
+                self.filt = None
+                return
+            self.filt = flowfilter.parse(opts.intercept)
+            if not self.filt:
+                raise exceptions.OptionsError(
+                    "Invalid interception filter: %s" % opts.intercept
+                )
+
+    def process_flow(self, f):
+        if self.filt:
+            should_intercept = all([
+                self.filt(f),
+                not f.request.is_replay,
+            ])
+            if should_intercept:
+                f.intercept()
+
+    # Handlers
+
+    def request(self, f):
+        self.process_flow(f)
+
+    def response(self, f):
+        self.process_flow(f)

mitmproxy/addons/onboarding.py

@@ -0,0 +1,17 @@
+from mitmproxy.addons import wsgiapp
+from mitmproxy.addons.onboardingapp import app
+
+
+class Onboarding(wsgiapp.WSGIApp):
+    def __init__(self):
+        super().__init__(app.Adapter(app.application), None, None)
+        self.enabled = False
+
+    def configure(self, options, updated):
+        self.host = options.onboarding_host
+        self.port = options.onboarding_port
+        self.enabled = options.onboarding
+
+    def request(self, f):
+        if self.enabled:
+            super().request(f)

mitmproxy/addons/onboardingapp/app.py

@@ -1,15 +1,13 @@
-from __future__ import absolute_import, print_function, division
-
 import os
 
 import tornado.template
 import tornado.web
 import tornado.wsgi
 
-from mitmproxy import utils
+from mitmproxy.utils import data
 from mitmproxy.proxy import config
 
-loader = tornado.template.Loader(utils.pkg_data.path("onboarding/templates"))
+loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates"))
 
 
 class Adapter(tornado.wsgi.WSGIAdapter):
@@ -87,10 +85,9 @@ application = tornado.web.Application(
             r"/static/(.*)",
             tornado.web.StaticFileHandler,
             {
-                "path": utils.pkg_data.path("onboarding/static")
+                "path": data.pkg_data.path("addons/onboardingapp/static")
             }
         ),
     ],
     # debug=True
 )
-mapp = Adapter(application)