bump version to 1.0.2

This is a simpler version of @Kriechi's patch.

.appveyor.yml

@@ -7,8 +7,9 @@ environment:
   matrix:
     - PYTHON: "C:\\Python35"
       TOXENV: "py35"
-    - PYTHON: "C:\\Python27"
-      TOXENV: "py27"
+    # TODO: ENABLE WHEN AVAILABLE
+    # - PYTHON: "C:\\Python36"
+    #   TOXENV: "py36"
 
   SNAPSHOT_HOST:
     secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
@@ -18,6 +19,8 @@ environment:
     secure: 6yBwmO5gv4vAwoFYII8qjQ==
   SNAPSHOT_PASS:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
+  RTOOL_KEY:
+    secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -25,21 +28,48 @@ install:
   - "pip install -U tox"
 
 test_script:
-  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v"
+  - ps: "tox -- --cov mitmproxy --cov pathod -v"
+  - ps: |
+      $Env:VERSION = $(python mitmproxy/version.py)
+      $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
+      tox -e wheel
+      tox -e rtool -- bdist
+
+  - ps: |
+      if(
+        ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
+        (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+      ) {
+        tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
+        if (!(Test-Path "C:\projects\mitmproxy\release\installbuilder-installer.exe")) {
+          "Download InstallBuilder..."
+          (New-Object System.Net.WebClient).DownloadFile(
+            "https://installbuilder.bitrock.com/installbuilder-enterprise-16.11.1-windows-installer.exe",
+            "C:\projects\mitmproxy\release\installbuilder-installer.exe"
+          )
+        }
+        Start-Process "C:\projects\mitmproxy\release\installbuilder-installer.exe" "--mode unattended --unattendedmodeui none" -Wait
+        & 'C:\Program Files (x86)\BitRock InstallBuilder Enterprise 16.11.1\bin\builder-cli.exe' `
+          build `
+          .\release\installbuilder\mitmproxy.xml `
+          windows `
+          --license .\release\installbuilder\license.xml `
+          --setvars project.version=$Env:VERSION `
+          --verbose
+      }
 
 deploy_script:
+  # we build binaries on every run, but we only upload them for master snapshots or tags.
   ps: |
     if(
       ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-      pip install -U virtualenv
-      .\dev.ps1
-      cmd /c "python -u .\release\rtool.py bdist 2>&1"
-      python -u .\release\rtool.py upload-snapshot --bdist --wheel
+      (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+    ) {
+      tox -e rtool -- upload-snapshot --bdist --wheel --installer
     }
 
 cache:
+  - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml
   - C:\Users\appveyor\AppData\Local\pip\cache
 
 notifications:

.gitattributes

@@ -1,2 +1,2 @@
-mitmproxy/web/static/**/* -diff
+mitmproxy/tools/web/static/**/* -diff
 web/src/js/filt/filt.js -diff

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 MANIFEST
-*/tmp
+**/tmp
 /venv*
 *.py[cdo]
 *.swp

.travis.yml

@@ -1,15 +1,6 @@
 sudo: false
 language: python
 
-addons:
-  apt:
-    sources:
-      # Debian sid currently holds OpenSSL 1.0.2
-      # change this with future releases!
-      - debian-sid
-    packages:
-      - libssl-dev
-
 env:
   global:
     - CI_DEPS=codecov>=2.0.5
@@ -25,15 +16,35 @@ matrix:
       language: generic
       env: TOXENV=py35 BDIST=1
     - python: 3.5
-      env: TOXENV=py35 BDIST=1
+      env: TOXENV=py35 OPENSSL_OLD
+      addons:
+        apt:
+          packages:
+            - libssl-dev
     - python: 3.5
-      env: TOXENV=py35 NO_ALPN=1
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 2.7
-      env: TOXENV=py27 NO_ALPN=1
+      env: TOXENV=py35 BDIST=1 OPENSSL_ALPN
+      addons:
+        apt:
+          sources:
+            # Debian sid currently holds OpenSSL 1.1.0
+            # change this with future releases!
+            - debian-sid
+          packages:
+            - libssl-dev
+    - python: 3.6
+      env: TOXENV=py36 OPENSSL_ALPN
+      addons:
+        apt:
+          sources:
+            # Debian sid currently holds OpenSSL 1.1.0
+            # change this with future releases!
+            - debian-sid
+          packages:
+            - libssl-dev
     - python: 3.5
       env: TOXENV=docs
+      git:
+        depth: 10000
   allow_failures:
     - python: pypy
 
@@ -41,10 +52,8 @@ install:
   - |
     if [[ $TRAVIS_OS_NAME == "osx" ]]
     then
-      brew update || brew update # try again if it fails
-      brew upgrade
-      brew reinstall openssl
-      brew reinstall pyenv
+      brew update || brew update
+      brew outdated pyenv || brew upgrade pyenv
       eval "$(pyenv init -)"
       env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.2
       pyenv global 3.5.2
@@ -53,18 +62,21 @@ install:
     fi
   - pip install tox
 
-script: tox -- --cov netlib --cov mitmproxy --cov pathod -v
+script:
+  - tox -- --cov mitmproxy --cov pathod -v
+  - |
+    if [[ $BDIST == "1" ]]
+    then
+        git fetch --unshallow --tags
+        tox -e rtool -- bdist
+    fi
 
 after_success:
+  # we build binaries on every run, but we only upload them for master snapshots or tags.
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
-        git fetch --unshallow
-        ./dev.sh 3.5
-        source venv3.5/bin/activate
-        pip install -e ./release
-        python -u ./release/rtool.py bdist
-        python -u ./release/rtool.py upload-snapshot --bdist
+        tox -e rtool -- upload-snapshot --bdist
     fi
 
 notifications:
@@ -79,3 +91,4 @@ cache:
   directories:
     - $HOME/.pyenv
     - $HOME/.cache/pip
+    # - $HOME/build/mitmproxy/mitmproxy/.tox

CHANGELOG

@@ -1,3 +1,33 @@
+26 December 2016: mitmproxy 1.0
+
+    * All mitmproxy tools are now Python 3 only! We plan to support Python 3.5 and higher.
+
+    * Web-Based User Interface: Mitmproxy now offically has a web-based user interface
+      called mitmweb. We consider it stable for all features currently exposed 
+      in the UI, but it still misses a lot of mitmproxy’s options.
+      
+    * Windows Compatibility: With mitmweb, mitmproxy is now useable on Windows. 
+      We are also introducing an installer (kindly sponsored by BitRock) that 
+      simplifies setup.
+      
+    * Configuration: The config file format is now a single YAML file. In most cases,
+      converting to the new format should be trivial - please see the docs for
+      more information.
+
+    * Console: Significant UI improvements - including sorting of flows by
+      size, type and url, status bar improvements, much faster indentation for
+      HTTP views, and more.
+
+    * HTTP/2: Significant improvements, but is temporarily disabled by default
+      due to wide-spread protocol implementation errors on some large website
+
+    * WebSocket: The protocol implementation is now mature, and is enabled by
+      default. Complete UI support is coming in the next release. Hooks for 
+      message interception and manipulation are available.
+
+    * A myriad of other small improvements throughout the project.
+
+
 16 October 2016: mitmproxy 0.18
 
     * Python 3 Compatibility for mitmproxy and pathod (Shadab Zafar, GSoC 2016)

CONTRIBUTORS

@@ -1,6 +1,6 @@
-  2184	Aldo Cortesi
-  1745	Maximilian Hils
-   507	Thomas Kriechbaumer
+  2407	Aldo Cortesi
+  1873	Maximilian Hils
+   556	Thomas Kriechbaumer
    258	Shadab Zafar
     97	Jason
     83	Marcelo Glezer
@@ -11,85 +11,91 @@
     14	Pedro Worcel
     14	David Weinstein
     13	Thomas Roth
-    11	Stephen Altamirano
     11	Jake Drahos
+    11	Stephen Altamirano
     11	arjun23496
     11	Justus Wingert
-    10	Sandor Nemes
-    10	Zohar Lorberbaum
     10	András Veres-Szentkirályi
-    10	Chris Czub
+    10	Zohar Lorberbaum
     10	smill
+    10	Chris Czub
+    10	Sandor Nemes
+    10	Doug Freed
      9	ikoz
-     9	Kyle Morton
      9	Legend Tang
      9	Rouli
+     9	Kyle Morton
      8	Jason A. Novak
      8	Chandler Abraham
-     7	Alexis Hildebrandt
      7	Matthias Urlichs
      7	Brad Peabody
      7	dufferzafar
+     7	Alexis Hildebrandt
      6	Felix Yan
-     5	Tomaz Muraus
-     5	elitest
-     5	iroiro123
-     5	Sam Cleveland
-     5	Choongwoo Han
      5	Will Coster
-     4	root
-     4	Clemens Brunner
+     5	Sam Cleveland
+     5	iroiro123
+     5	elitest
+     5	Tomaz Muraus
+     5	Choongwoo Han
      4	Schamper
-     4	Valtteri Virtanen
-     4	Wade 524
      4	Youhei Sakurai
      4	Bryan Bishop
+     4	root
+     4	Valtteri Virtanen
+     4	Clemens Brunner
      4	Marc Liyanage
-     4	Michael J. Bazzinotti
+     4	Wade 524
+     4	chhsiao90
      4	yonder
-     3	Eli Shvartsman
-     3	Chris Neasbitt
-     3	Guillem Anguera
-     3	MatthewShao
+     4	Michael J. Bazzinotti
      3	Ryan Welton
-     3	smill@cuckoo.sh
-     3	Manish Kumar
-     3	Benjamin Lee
      3	Ryan Laughlin
-     3	Zack B
      3	Kyle Manna
+     3	Eli Shvartsman
+     3	Vincent Haupert
+     3	Manish Kumar
+     3	Zack B
+     3	MatthewShao
      3	redfast00
      3	requires.io
+     3	Guillem Anguera
+     3	smill@cuckoo.sh
+     3	Chris Neasbitt
+     3	Benjamin Lee
+     2	Steven Van Acker
+     2	Slobodan Mišković
+     2	Jim Lloyd
      2	isra17
      2	israel
-     2	Colin Bendell
+     2	Sean Coates
+     2	Sachin Kelkar
      2	jpkrause
-     2	Paul
+     2	Bennett Blodinger
      2	lilydjwg
      2	Michael Frister
-     2	依云
-     2	Jaime Soriano Pastor
-     2	Nick Badger
-     2	Rob Wills
-     2	Heikki Hannikainen
-     2	Vincent Haupert
-     2	strohu
-     2	Wade Catron
-     2	Krzysztof Bielicki
-     2	Sachin Kelkar
      2	Israel Nir
-     2	Anant
-     2	alts
-     2	Doug Freed
-     2	Niko Kommenda
-     2	Terry Long
-     2	Mark E. Haase
-     2	Steven Van Acker
-     2	Jim Lloyd
-     2	Bennett Blodinger
-     2	Sean Coates
      2	Cory Benfield
-     1	Sergey Chipiga
+     2	phackt
+     2	Anant
+     2	Jaime Soriano Pastor
+     2	Paul
+     2	Colin Bendell
+     2	依云
+     2	Heikki Hannikainen
+     2	Rob Wills
+     2	Niko Kommenda
+     2	Naveen Pai
+     2	strohu
+     2	alts
+     2	Yoginski
+     2	Mark E. Haase
+     2	Wade Catron
+     2	Terry Long
+     2	Krzysztof Bielicki
+     2	Nick Badger
+     1	Nicolas Esteves
+     1	Andrew Orr
      1	Andrey Plotnikov
      1	Andy Smith
      1	Angelo Agatino Nicolosi
@@ -97,6 +103,7 @@
      1	BSalita
      1	Ben Lerner
      1	Bradley Baetz
+     1	Brady Law
      1	Brett Randall
      1	Chris Hamant
      1	Christian Frichot
@@ -105,6 +112,7 @@
      1	David Shaw
      1	Doug Lethin
      1	Drake Caraker
+     1	Edgar Boda-Majer
      1	Eric Entzel
      1	Felix Wolfsteller
      1	FreeArtMan
@@ -128,22 +136,25 @@
      1	Mathieu Mitchell
      1	Michael Bisbjerg
      1	Mike C
+     1	Mike Fotinakis
      1	Mikhail Korobov
      1	Morton Fox
      1	Nick HS
      1	Nick Raptis
-     1	Nicolas Esteves
+     1	Aditya
      1	Oleksandr Sheremet
      1	Parth Ganatra
      1	Pritam Baral
+     1	Quentin Pradet
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
      1	Ryo Onodera
+     1	Sahil Chelaramani
      1	Sahn Lam
      1	Sanchit Sokhey
      1	Seppo Yli-Olli
-     1	Aditya
+     1	Sergey Chipiga
      1	Stefan Wärting
      1	Steve Phillips
      1	Steven Noble
@@ -158,10 +169,8 @@
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
      1	Wes Turner
-     1	Yoginski
      1	Yuangxuan Wang
      1	capt8bit
-     1	chhsiao90
      1	cle1000
      1	davidpshaw
      1	deployable
@@ -172,7 +181,6 @@
      1	meeee
      1	michaeljau
      1	peralta
-     1	phackt
      1	phil plante
      1	sentient07
      1	sethp-jive

MANIFEST.in

@@ -1,4 +1,3 @@
 graft mitmproxy
 graft pathod
-graft netlib
-recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
+recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

README.rst

@@ -3,8 +3,7 @@ mitmproxy
 
 |travis| |appveyor| |coverage| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects, as well as
-their shared networking library, **netlib**.
+This repository contains the **mitmproxy** and **pathod** projects.
 
 ``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
 interface.
@@ -23,8 +22,7 @@ Documentation & Help
 General information, tutorials, and precompiled binaries can be found on the mitmproxy
 and pathod websites.
 
-|mitmproxy_site| |pathod_site|
-
+|mitmproxy_site|
 
 The latest documentation for mitmproxy is also available on ReadTheDocs.
 
@@ -45,7 +43,7 @@ Join our developer chat on Slack if you would like to hack on mitmproxy itself.
 Installation
 ------------
 
-The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`_.
+The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`__.
 If you want to contribute changes, keep on reading.
 
 
@@ -54,7 +52,7 @@ Hacking
 
 To get started hacking on mitmproxy, make sure you have Python_ 3.5.x or above with
 virtualenv_ installed (you can find installation instructions for virtualenv
-`here <http://virtualenv.readthedocs.org/en/latest/>`_). Then do the following:
+`here <http://virtualenv.readthedocs.org/en/latest/>`__). Then do the following:
 
 .. code-block:: text
 
@@ -65,7 +63,7 @@ virtualenv_ installed (you can find installation instructions for virtualenv
 
 The *dev* script will create a virtualenv environment in a directory called
 "venv", and install all mandatory and optional dependencies into it. The
-primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
+primary mitmproxy components - mitmproxy and pathod - are installed as
 "editable", so any changes to the source in the repository will be reflected
 live in the virtualenv.
 
@@ -103,21 +101,17 @@ requirements installed, and you can simply run the test suite:
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project tries to maintain 100% test coverage.
 
-You can also use `tox` to run a full suite of tests in Python 2.7 and 3.5,
-including a quick test to check documentation and code linting.
+You can also use `tox` to run the full suite of tests, including a quick test
+to check documentation and code linting.
 
 The following tox environments are relevant for local testing:
 
 .. code-block:: text
 
-    tox -e py27  # runs all tests with Python 2.7
     tox -e py35  # runs all tests with Python 3.5
     tox -e docs  # runs a does-it-compile check on the documentation
     tox -e lint  # runs the linter for coding style checks
 
-We support Python 2.7 and 3.5, so please make sure all tests pass in both
-environments. Running `tox` ensures all necessary tests are executed.
-
 
 Documentation
 -------------
@@ -145,21 +139,18 @@ contribute and collaborate. Please stick to the guidelines in
 good reason not to.
 
 This is automatically enforced on every PR. If we detect a linting error, the
-PR checks will fail and block merging. We are using this command to check for style compliance:
+PR checks will fail and block merging. You can run our lint checks yourself
+with the following command:
 
 .. code-block:: text
 
-    flake8 --jobs 8 --count mitmproxy netlib pathod examples test
+    tox -e lint
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg
     :target: https://mitmproxy.org/
     :alt: mitmproxy.org
 
-.. |pathod_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-pathod.net-blue.svg
-    :target: https://pathod.net/
-    :alt: pathod.net
-
 .. |mitmproxy_docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
     :target: http://docs.mitmproxy.org/en/latest/
     :alt: mitmproxy documentation
@@ -198,6 +189,5 @@ PR checks will fail and block merging. We are using this command to check for st
 .. _.env: https://github.com/mitmproxy/mitmproxy/blob/master/.env
 .. _Sphinx: http://sphinx-doc.org/
 .. _sphinx-autobuild: https://pypi.python.org/pypi/sphinx-autobuild
-.. _issue_tracker: https://github.com/mitmproxy/mitmproxy/issues
 .. _PEP8: https://www.python.org/dev/peps/pep-0008
 .. _Google Style Guide: https://google.github.io/styleguide/pyguide.html

dev.ps1

@@ -1,7 +1,12 @@
 $ErrorActionPreference = "Stop"
 $VENV = ".\venv"
 
-virtualenv $VENV --always-copy
+$pyver = python --version
+if($pyver -notmatch "3\.[5-9]") {
+    Write-Warning "Unexpected Python version, expected Python 3.5 or above: $pyver"
+}
+
+python -m venv $VENV --copies
 & $VENV\Scripts\activate.ps1
 
 python -m pip install --disable-pip-version-check -U pip

dev.sh

@@ -2,12 +2,12 @@
 set -e
 set -x
 
-PYVERSION=$1
-VENV="venv$1"
+PYVERSION=${1:-3.5}
+VENV="venv$PYVERSION"
 
 echo "Creating dev environment in $VENV using Python $PYVERSION"
 
-python$PYVERSION -m virtualenv "$VENV" --always-copy
+python$PYVERSION -m venv "$VENV"
 . "$VENV/bin/activate"
 pip$PYVERSION install -U pip setuptools
 pip$PYVERSION install -r requirements.txt

docs/conf.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 
 sys.path.insert(0, os.path.abspath('..'))
-import netlib.version
+from mitmproxy import version as mversion
 
 
 extensions = [
@@ -47,9 +47,9 @@ author = u'The mitmproxy project'
 # built documents.
 #
 # The short X.Y version.
-version = netlib.version.VERSION
+version = mversion.VERSION
 # The full version, including alpha/beta/rc tags.
-release = netlib.version.VERSION
+release = mversion.VERSION
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -231,10 +231,7 @@ def linkcode_resolve(domain, info):
         _, line = inspect.getsourcelines(obj)
     except (TypeError, IOError):
         return None
-    if spath.rfind("netlib") > -1:
-        off = spath.rfind("netlib")
-        mpath = spath[off:]
-    elif spath.rfind("mitmproxy") > -1:
+    if spath.rfind("mitmproxy") > -1:
         off = spath.rfind("mitmproxy")
         mpath = spath[off:]
     else:

docs/config.rst

@@ -3,84 +3,11 @@
 Configuration
 =============
 
-Mitmproxy is configured through a set of files in the users ~/.mitmproxy
-directory.
+Mitmproxy is configured with a YAML_ file, located at
+``~/.mitmproxy/config.yaml``. We'll have complete documentation for all
+supported options in the next release in the meantime, please consult the
+source_ for a complete list of options and types.
 
-mitmproxy.conf
-    Settings for the :program:`mitmproxy`. This file can contain any options supported by
-    mitmproxy.
 
-mitmdump.conf
-    Settings for the :program:`mitmdump`. This file can contain any options supported by mitmdump.
-
-common.conf
-    Settings shared between all command-line tools. Settings in this file are over-ridden by those
-    in the tool-specific files. Only options shared by mitmproxy and mitmdump should be used in
-    this file.
-
-Syntax
-------
-
-Comments
-^^^^^^^^
-
-.. code-block:: none
-
-    # this is a comment
-    ; this is also a comment (.ini style)
-    --- and this is a comment too (yaml style)
-
-Key/Value pairs
-^^^^^^^^^^^^^^^
-
-- Keys and values are case-sensitive
-- Whitespace is ignored
-- Lists are comma-delimited, and enclosed in square brackets
-
-.. code-block:: none
-
-    name = value   # (.ini style)
-    name: value    # (yaml style)
-    --name value   # (command-line option style)
-
-    fruit = [apple, orange, lemon]
-    indexes = [1, 12, 35 , 40]
-
-Flags
-^^^^^
-
-These are boolean options that take no value but true/false.
-
-.. code-block:: none
-
-    name = true    # (.ini style)
-    name
-    --name 	 	   # (command-line option style)
-
-Options
--------
-
-The options available in the config files are precisely those available as
-command-line flags, with the key being the option's long name. To get a
-complete list of these, use the ``--help`` option on each of the tools. Be
-careful to only specify common options in the **common.conf** file -
-unsupported options in this file will be detected as an error on startup.
-
-Examples
---------
-
-common.conf
-^^^^^^^^^^^
-
-Note that ``--port`` is an option supported by all tools.
-
-.. code-block:: none
-
-    port = 8080
-
-mitmproxy.conf
-^^^^^^^^^^^^^^
-
-.. code-block:: none
-
-    palette = light
+.. _YAML: http://www.yaml.org/start.html
+.. _source: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/options.py

docs/dev/testing.rst

@@ -10,7 +10,7 @@ suitable extension to the test suite.
 Our tests are written for the `py.test`_ or nose_ test frameworks.
 At the point where you send your pull request, a command like this:
 
->>> py.test --cov mitmproxy --cov netlib
+>>> py.test --cov mitmproxy
 
 Should give output something like this:
 

docs/features/responsestreaming.rst

@@ -40,8 +40,8 @@ You can also use a script to customize exactly which responses are streamed.
 Responses that should be tagged for streaming by setting their ``.stream``
 attribute to ``True``:
 
-.. literalinclude:: ../../examples/stream.py
-   :caption: examples/stream.py
+.. literalinclude:: ../../examples/complex/stream.py
+   :caption: examples/complex/stream.py
    :language: python
 
 Implementation Details
@@ -59,8 +59,8 @@ Modifying streamed data
 If the ``.stream`` attribute is callable, ``.stream`` will wrap the generator that yields all
 chunks.
 
-.. literalinclude:: ../../examples/stream_modify.py
-   :caption: examples/stream_modify.py
+.. literalinclude:: ../../examples/complex/stream_modify.py
+   :caption: examples/complex/stream_modify.py
    :language: python
 
 .. seealso::

docs/features/tcpproxy.rst

@@ -3,7 +3,7 @@
 TCP Proxy
 =========
 
-WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt
+In case mitmproxy does not handle a specific protocol, you can exempt
 hostnames from processing, so that mitmproxy acts as a generic TCP forwarder.
 This feature is closely related to the :ref:`passthrough` functionality,
 but differs in two important aspects:

docs/index.rst

@@ -17,6 +17,7 @@
 
     mitmproxy
     mitmdump
+    mitmweb
     config
 
 .. toctree::

docs/install.rst

@@ -3,130 +3,149 @@
 Installation
 ============
 
-.. _install-ubuntu:
+Please follow the steps for your operating system.
 
-Installation On Ubuntu
-----------------------
+Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or
+:ref:`mitmweb` from a terminal.
 
-Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Ubuntu 14.04.
+
+.. _install-macos:
+
+Installation on macOS
+---------------------
+
+You can use Homebrew to install everything:
 
 .. code:: bash
 
-   sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev g++
-   sudo pip install mitmproxy  # or pip install --user mitmproxy
+    brew install mitmproxy
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+Or you can download the pre-built binary packages from `mitmproxy.org`_.
 
-On **Ubuntu 12.04** (and other systems with an outdated version of pip),
-you may need to update pip using ``pip install -U pip`` before installing mitmproxy.
 
-Installation From Source (Ubuntu)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. _install-windows:
 
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, install the dependencies as you would for a regular
-mitmproxy installation (see :ref:`install-ubuntu`).
-Then see the Hacking_ section of the README on GitHub.
+Installation on Windows
+-----------------------
 
-.. _install-fedora:
+The recommended way to install mitmproxy on Windows is to use the installer
+provided at `mitmproxy.org`_. After installation, you'll find shortcuts for
+:ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu.
+Both executables are  added to your PATH and can be invoked from the command
+line.
 
-Installation On Fedora
-----------------------
+.. note::
+    mitmproxy's console interface is not supported on Windows, but you can use
+    mitmweb (the web-based interface) and mitmdump.
 
-Fedora comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Fedora 23.
+.. _install-linux:
 
-.. code:: bash
+Installation on Linux
+---------------------
 
-   sudo dnf install -y python-pip python-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libpng-devel libjpeg-devel
-   sudo pip install mitmproxy  # or pip install --user mitmproxy
+The recommended way to run mitmproxy on Linux is to use the pre-built binaries
+provided at `mitmproxy.org`_.
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+Our pre-built binaries provide you with the latest version of mitmproxy, a
+self-contained Python 3.5 environment and a recent version of OpenSSL that
+supports HTTP/2. Of course, you can also install mitmproxy from source if you
+prefer that (see :ref:`install-advanced`).
+
+.. _install-advanced:
+
+Advanced Installation
+---------------------
+
+.. _install-docker:
+
+Docker Images
+^^^^^^^^^^^^^
+
+You can also use the official mitmproxy images from `DockerHub`_. That being
+said, our portable binaries are just as easy to install and even easier to use. 😊
 
 
 .. _install-arch:
 
-Installation On Arch Linux
---------------------------
+Installation on Arch Linux
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 mitmproxy has been added into the [community] repository. Use pacman to install it:
 
 >>> sudo pacman -S mitmproxy
 
 
+.. _install-source-ubuntu:
 
-Installation On Mac OS X
-------------------------
+Installation from Source on Ubuntu
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The easiest way to get up and running on OSX is to download the pre-built binary packages from
-`mitmproxy.org`_.
+Ubuntu comes with Python but we need to install pip3, python3-dev and several
+libraries. This was tested on a fully patched installation of Ubuntu 16.04.
 
-There are a few bits of customization you might want to do to make mitmproxy comfortable to use on
-OSX. The default color scheme is optimized for a dark background terminal, but you can select a
-palette for a light terminal background with the ``--palette`` option.
-You can use the OSX **open** program to create a simple and effective ``~/.mailcap`` file to view
-request and response bodies:
+.. code:: bash
 
-.. code-block:: none
+   sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev
+   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
 
-    application/*; /usr/bin/open -Wn %s
-    audio/*; /usr/bin/open -Wn %s
-    image/*; /usr/bin/open -Wn %s
-    video/*; /usr/bin/open -Wn %s
-
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install
+a newer version of Python. mitmproxy requires Python 3.5 or higher. Please take
+a look at pyenv_. Make sure to have an up-to-date version of pip by running
+``pip3 install -U pip``.
 
 
-Installation From Source (Mac OS X)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. _install-source-fedora:
 
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, there are a few OS X specific things to keep in mind.
+Installation from Source on Fedora
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Make sure that XCode is installed from the App Store, and that the command-line tools have been
-  downloaded (XCode/Preferences/Downloads).
-- If you're running a Python interpreter installed with homebrew (or similar), you may have to
-  install some dependencies by hand.
+Fedora comes with Python but we need to install pip3, python3-dev and several
+libraries. This was tested on a fully patched installation of Fedora 24.
 
-Then see the Hacking_ section of the README on GitHub.
+.. code:: bash
 
-Installation On Windows
------------------------
+   sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel
+   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
+
+Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
+
+
+
+.. _install-source-windows:
+
+🐱💻 Installation from Source on Windows
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. note::
-    Please note that mitmdump is the only component of mitmproxy that is supported on Windows at
-    the moment.
+    mitmproxy's console interface is not supported on Windows, but you can use
+    mitmweb (the web-based interface) and mitmdump.
 
-    **There is no interactive user interface on Windows.**
-
-
-First, install the latest version of Python 2.7 from the `Python website`_.
-If you already have an older version of Python 2.7 installed, make sure to install pip_
-(pip is included in Python 2.7.9+ by default). If pip aborts with an error, make sure you are using the current version of pip.
-
->>> python -m pip install --upgrade pip
-
-Next, add Python and the Python Scripts directory to your **PATH** variable.
-You can do this easily by running the following in powershell:
-
->>> [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User")
+First, install the latest version of Python 3.5 or later from the `Python
+website`_. During installation, make sure to select `Add Python to PATH`.
 
 Now, you can install mitmproxy by running
 
->>> pip install mitmproxy
+.. code:: powershell
 
-Once the installation is complete, you can run :ref:`mitmdump` from a command prompt.
+    pip3 install mitmproxy
 
-Installation From Source (Windows)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, install Python as outlined above, then see the
-Hacking_ section of the README on GitHub.
+
+.. _install-dev-version:
+
+Latest Development Version
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you would like to install mitmproxy directly from the master branch on GitHub
+or would like to get set up to contribute to the project, install the
+dependencies as you would for a regular installation from source. Then see the
+Hacking_ section of the README on GitHub. You can check your system information
+by running: ``mitmproxy --sysinfo``
 
 
 .. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst#hacking
 .. _mitmproxy.org: https://mitmproxy.org/
 .. _`Python website`: https://www.python.org/downloads/windows/
 .. _pip: https://pip.pypa.io/en/latest/installing.html
+.. _pyenv: https://github.com/yyuu/pyenv
+.. _DockerHub: https://hub.docker.com/r/mitmproxy/mitmproxy/

docs/introduction.rst

@@ -6,6 +6,8 @@ with a console interface.
 
 **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
+**mitmweb** is a web-based interface for mitmproxy.
+
 Documentation, tutorials and distribution packages can be found on the
 mitmproxy website: `mitmproxy.org <https://mitmproxy.org/>`_
 

docs/mitmweb.rst

@@ -0,0 +1,18 @@
+.. _mitmweb:
+.. program:: mitmweb
+
+mitmweb
+=======
+
+**mitmweb** is mitmproxy's web-based user interface that allows interactive
+examination and modification of HTTP traffic. Like mitmproxy, it differs from
+mitmdump in that all flows are kept in memory, which means that it's intended
+for taking and manipulating small-ish samples.
+
+.. warning::
+
+  Mitmweb is currently in beta. We consider it stable for all features currently
+  exposed in the UI, but it still misses a lot of mitmproxy's features.
+
+
+.. image:: screenshots/mitmweb.png

docs/modd.conf

@@ -1,6 +1,6 @@
 @build = ./_build
 
-** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py {
+** !_build/** ../mitmproxy/**/*.py {
     prep: sphinx-build -W -d @build/doctrees -b html . @build/html
     daemon: devd -m @build/html
 }

docs/pathod/test.rst

@@ -14,7 +14,7 @@ The canonical docs can be accessed using pydoc:
 >>> pydoc pathod.test
 
 The remainder of this page demonstrates some common interaction patterns using
-<a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are
+`Nose`_. These examples are
 also applicable with only minor modification to most commonly used Python testing
 engines.
 
@@ -33,3 +33,6 @@ One instance per test
 .. literalinclude:: ../../examples/pathod/test_setup.py
    :caption: examples/pathod/test_setup.py
    :language: python
+
+
+.. _Nose: https://nose.readthedocs.org/en/latest/

docs/scripting/api.rst

@@ -5,36 +5,38 @@ API
 ===
 
 - Errors
-    - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_
+    - `mitmproxy.flow.Error <#mitmproxy.flow.Error>`_
 - HTTP
-    - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_
-    - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_
-    - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_
+    - `mitmproxy.http.HTTPRequest <#mitmproxy.http.HTTPRequest>`_
+    - `mitmproxy.http.HTTPResponse <#mitmproxy.http.HTTPResponse>`_
+    - `mitmproxy.http.HTTPFlow <#mitmproxy.http.HTTPFlow>`_
 - Logging
-    - `mitmproxy.controller.Log <#mitmproxy.controller.Log>`_
-    - `mitmproxy.controller.LogEntry <#mitmproxy.controller.LogEntry>`_
+    - `mitmproxy.log.Log <#mitmproxy.controller.Log>`_
+    - `mitmproxy.log.LogEntry <#mitmproxy.controller.LogEntry>`_
 
 
 Errors
 ------
 
-.. autoclass:: mitmproxy.models.flow.Error
+.. autoclass:: mitmproxy.flow.Error
     :inherited-members:
 
 HTTP
 ----
 
-.. autoclass:: mitmproxy.models.http.HTTPRequest
+.. autoclass:: mitmproxy.http.HTTPRequest
     :inherited-members:
 
-.. autoclass:: mitmproxy.models.http.HTTPResponse
+.. autoclass:: mitmproxy.http.HTTPResponse
     :inherited-members:
 
-.. autoclass:: mitmproxy.models.http.HTTPFlow
+.. autoclass:: mitmproxy.http.HTTPFlow
     :inherited-members:
 
 Logging
 --------
 
-.. autoclass:: mitmproxy.controller.Log
+.. autoclass:: mitmproxy.log.Log
+    :inherited-members:
+.. autoclass:: mitmproxy.log.LogEntry
     :inherited-members:

docs/scripting/events.rst

@@ -56,7 +56,7 @@ Connection
           connection can correspond to multiple HTTP requests.
 
           *root_layer*
-            The root layer (see `mitmproxy.protocol` for an explanation what
+            The root layer (see `mitmproxy.proxy.protocol` for an explanation what
             the root layer is), provides transparent access to all attributes
             of the :py:class:`~mitmproxy.proxy.RootContext`. For example,
             ``root_layer.client_conn.address`` gives the remote address of the
@@ -98,6 +98,18 @@ HTTP Events
     :widths: 40 60
     :header-rows: 0
 
+    *   - .. py:function:: http_connect(flow)
+        - Called when we receive an HTTP CONNECT request. Setting a non 2xx
+          response on the flow will return the response to the client abort the
+          connection. CONNECT requests and responses do not generate the usual
+          HTTP handler events. CONNECT requests are only valid in regular and
+          upstream proxy modes.
+
+          *flow*
+            A ``models.HTTPFlow`` object. The flow is guaranteed to have
+            non-None ``request``  and ``requestheaders`` attributes.
+
+
     *   - .. py:function:: request(flow)
         - Called when a client request has been received.
 
@@ -146,21 +158,54 @@ HTTP Events
 WebSocket Events
 -----------------
 
+These events are called only after a connection made an HTTP upgrade with
+"101 Switching Protocols". No further HTTP-related events after the handshake
+are issued, only new WebSocket messages are called.
+
 .. list-table::
     :widths: 40 60
     :header-rows: 0
 
-    *   - .. py:function:: websockets_handshake(flow)
-
-        - Called when a client wants to establish a WebSockets connection. The
-          WebSockets-specific headers can be manipulated to manipulate the
+    *   - .. py:function:: websocket_handshake(flow)
+        - Called when a client wants to establish a WebSocket connection. The
+          WebSocket-specific headers can be manipulated to alter the
           handshake. The ``flow`` object is guaranteed to have a non-None
           ``request`` attribute.
 
           *flow*
-            The flow containing the HTTP websocket handshake request. The
+            The flow containing the HTTP WebSocket handshake request. The
             object is guaranteed to have a non-None ``request`` attribute.
 
+    *   - .. py:function:: websocket_start(flow)
+        - Called when WebSocket connection is established after a successful
+          handshake.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_message(flow)
+
+        - Called when a WebSocket message is received from the client or server. The
+          sender and receiver are identifiable. The most recent message will be
+          ``flow.messages[-1]``. The message is user-modifiable. Currently there are
+          two types of messages, corresponding to the BINARY and TEXT frame types.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_end(flow)
+        - Called when WebSocket connection ends.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_error(flow)
+        - Called when a WebSocket error occurs - e.g. the connection closing
+          unexpectedly.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
 
 TCP Events
 ----------
@@ -173,6 +218,22 @@ connections.
     :widths: 40 60
     :header-rows: 0
 
+
+    *   - .. py:function:: tcp_start(flow)
+        - Called when TCP streaming starts.
+
+          *flow*
+            A ``models.TCPFlow`` object.
+
+    *   - .. py:function:: tcp_message(flow)
+
+        - Called when a TCP payload is received from the client or server. The
+          sender and receiver are identifiable. The most recent message will be
+          ``flow.messages[-1]``. The message is user-modifiable.
+
+          *flow*
+            A ``models.TCPFlow`` object.
+
     *   - .. py:function:: tcp_end(flow)
         - Called when TCP streaming ends.
 
@@ -185,18 +246,3 @@ connections.
 
           *flow*
             A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_message(flow)
-
-        - Called a TCP payload is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_start(flow)
-        - Called when TCP streaming starts.
-
-          *flow*
-            A ``models.TCPFlow`` object.

docs/scripting/overview.rst

@@ -6,7 +6,7 @@ Overview
 Mitmproxy has a powerful scripting API that allows you to control almost any
 aspect of traffic being proxied. In fact, much of mitmproxy's own core
 functionality is implemented using the exact same API exposed to scripters (see
-:src:`mitmproxy/builtins`).
+:src:`mitmproxy/addons`).
 
 
 A simple example
@@ -17,8 +17,8 @@ appropriate points of mitmproxy's operation. Here's a complete mitmproxy script
 that adds a new header to every HTTP response before it is returned to the
 client:
 
-.. literalinclude:: ../../examples/add_header.py
-   :caption: :src:`examples/add_header.py`
+.. literalinclude:: ../../examples/simple/add_header.py
+   :caption: :src:`examples/simple/add_header.py`
    :language: python
 
 All events that deal with an HTTP request get an instance of `HTTPFlow
@@ -42,8 +42,8 @@ called before anything else happens. You can replace the current script object
 by returning it from this handler. Here's how this looks when applied to the
 example above:
 
-.. literalinclude:: ../../examples/classes.py
-   :caption: :src:`examples/classes.py`
+.. literalinclude:: ../../examples/simple/add_header_class.py
+   :caption: :src:`examples/simple/add_header_class.py`
    :language: python
 
 So here, we're using a module-level script to "boot up" into a class instance.
@@ -62,13 +62,13 @@ sophisticated - replace one value with another in all responses. Mitmproxy's
 <api.html#mitmproxy.models.http.HTTPResponse.replace>`_ method that takes care
 of all the details for us.
 
-.. literalinclude:: ../../examples/arguments.py
-   :caption: :src:`examples/arguments.py`
+.. literalinclude:: ../../examples/simple/script_arguments.py
+   :caption: :src:`examples/simple/script_arguments.py`
    :language: python
 
 We can now call this script on the command-line like this:
 
->>> mitmdump -dd -s "./arguments.py html faketml"
+>>> mitmdump -dd -s "./script_arguments.py html faketml"
 
 Whenever a handler is called, mitpmroxy rewrites the script environment so that
 it sees its own arguments as if it was invoked from the command-line.
@@ -78,15 +78,15 @@ Logging and the context
 -----------------------
 
 Scripts should not output straight to stderr or stdout. Instead, the `log
-<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` contexzt module
+<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` context module
 should be used, so that the mitmproxy host program can handle output
-appropriately. So, mitmdump can print colorised sript output to the terminal,
+appropriately. So, mitmdump can print colorised script output to the terminal,
 and mitmproxy console can place script output in the event buffer.
 
 Here's how this looks:
 
-.. literalinclude:: ../../examples/logging.py
-   :caption: :src:`examples/logging.py`
+.. literalinclude:: ../../examples/simple/log_events.py
+   :caption: :src:`examples/simple/log_events.py`
    :language: python
 
 The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
@@ -126,15 +126,32 @@ It's possible to implement a concurrent mechanism on top of the blocking
 framework, and mitmproxy includes a handy example of this that is fit for most
 purposes. You can use it as follows:
 
-.. literalinclude:: ../../examples/nonblocking.py
-   :caption: :src:`examples/nonblocking.py`
+.. literalinclude:: ../../examples/complex/nonblocking.py
+   :caption: :src:`examples/complex/nonblocking.py`
+   :language: python
+
+
+Testing
+-------
+
+Mitmproxy includes a number of helpers for testing addons. The
+``mitmproxy.test.taddons`` module contains a context helper that takes care of
+setting up and tearing down the addon event context. The
+``mitmproxy.test.tflow`` module contains helpers for quickly creating test
+flows. Pydoc is the canonical reference for these modules, and mitmproxy's own
+test suite is an excellent source of examples of usage. Here, for instance, is
+the mitmproxy unit tests for the `anticache` option, demonstrating a good
+cross-section of the test helpers:
+
+.. literalinclude:: ../../test/mitmproxy/addons/test_anticache.py
+   :caption: :src:`test/mitmproxy/addons/test_anticache.py`
    :language: python
 
 
 Developing scripts
 ------------------
 
-Mitmprxoy monitors scripts for modifications, and reloads them on change. When
+Mitmproxy monitors scripts for modifications, and reloads them on change. When
 this happens, the script is shut down (the `done <events.html#done>`_  event is
 called), and the new instance is started up as if the script had just been
 loaded (the `start <events.html#start>`_ and `configure

docs/tutorials/transparent-dhcp.rst

@@ -38,8 +38,14 @@ DHCP and TFTP) services to a small-scale network.
     **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default
     `[1] <https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/>`_. For our use case, this needs
     to be disabled by changing ``dns=dnsmasq`` to ``#dns=dnsmasq`` in
-    **/etc/NetworkManager/NetworkManager.conf** and running
-
+    **/etc/NetworkManager/NetworkManager.conf** and
+    
+    if on Ubuntu 16.04 or newer running:
+    
+    >>> sudo systemctl restart NetworkManager
+    
+    if on Ubuntu 12.04 or 14.04 running:
+    
     >>> sudo restart network-manager
 
     afterwards.
@@ -61,6 +67,12 @@ DHCP and TFTP) services to a small-scale network.
 
     Apply changes:
 
+    if on Ubuntu 16.04 or newer:
+    
+    >>> sudo systemctl restart dnsmasq
+    
+    if on Ubuntu 12.04 or 14.04:
+    
     >>> sudo service dnsmasq restart
 
     Your **proxied machine** in the internal virtual network should now receive an IP address via DHCP:
@@ -74,8 +86,8 @@ To redirect traffic to mitmproxy, we need to add two iptables rules:
 
 .. code-block:: none
 
-    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
-    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
+    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
+    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
 
 4. Run mitmproxy
 ----------------

examples/README

@@ -1,31 +0,0 @@
-Some inline scripts may require additional dependencies, which can be installed using
-`pip install mitmproxy[examples]`.
-
-
-# inline script examples
-add_header.py             Simple script that just adds a header to every request.
-change_upstream_proxy.py  Dynamically change the upstream proxy
-dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
-dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
-fail_with_500.py          Turn every response into an Internal Server Error.
-filt.py                   Use mitmproxy's filter expressions in your script.
-flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
-iframe_injector.py        Inject configurable iframe into pages.
-modify_form.py            Modify all form submissions to add a parameter.
-modify_querystring.py     Modify all query strings to add a parameters.
-modify_response_body.py   Replace arbitrary strings in all responses
-nonblocking.py            Demonstrate parallel processing with a blocking script.
-proxapp.py                How to embed a WSGI app in a mitmproxy server
-redirect_requests.py      Redirect requests or directly reply to them.
-stub.py                   Script stub with a method definition for every event.
-upsidedownternet.py       Rewrites traffic to turn images upside down.
-
-
-# mitmproxy examples
-flowbasic                 Basic use of mitmproxy as a library.
-stickycookies             An example of writing a custom proxy with mitmproxy.
-
-
-# misc
-read_dumpfile             Read a dumpfile generated by mitmproxy.
-mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/README.md

@@ -0,0 +1,15 @@
+# Mitmproxy Scripting API
+
+Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being 
+proxied. In fact, much of mitmproxy’s own core functionality is implemented using the exact same API 
+exposed to scripters (see [mitmproxy/addons](../mitmproxy/addons)).
+
+This directory contains some examples of the scripting API. We recommend to start with the
+ones in [simple/](./simple).
+
+|  :warning: | If you are browsing this on GitHub, make sure to select the git tag matching your mitmproxy version. |
+|------------|------------------------------------------------------------------------------------------------------|
+
+
+Some inline scripts may require additional dependencies, which can be installed using
+`pip install mitmproxy[examples]`.

examples/complex/README.md

@@ -0,0 +1,18 @@
+## Complex Examples
+
+| Filename                 | Description                                                                                   |
+|:-------------------------|:----------------------------------------------------------------------------------------------|
+| change_upstream_proxy.py | Dynamically change the upstream proxy.                                                        |
+| dns_spoofing.py          | Use mitmproxy in a DNS spoofing scenario.                                                     |
+| dup_and_replay.py        | Duplicates each request, changes it, and then replays the modified request.                   |
+| flowbasic.py             | Basic use of mitmproxy's FlowMaster directly.                                                 |
+| full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. |
+| har_dump.py              | Dump flows as HAR files.                                                                      |
+| mitmproxywrapper.py      | Bracket mitmproxy run with proxy enable/disable on OS X                                       |
+| nonblocking.py           | Demonstrate parallel processing with a blocking script                                        |
+| remote_debug.py          | This script enables remote debugging of the mitmproxy _UI_ with PyCharm.                      |
+| sslstrip.py              | sslstrip-like funtionality implemented with mitmproxy                                         |
+| stream                   | Enable streaming for all responses.                                                           |
+| stream_modify.py         | Modify a streamed response body.                                                              |
+| tcp_message.py           | Modify a raw TCP connection                                                                   |
+| tls_passthrough.py       | Use conditional TLS interception based on a user-defined strategy.                            |

examples/complex/dup_and_replay.py

@@ -2,5 +2,7 @@ from mitmproxy import ctx
 
 
 def request(flow):
-    f = ctx.master.duplicate_flow(flow)
+    f = flow.copy()
+    ctx.master.view.add(f)
+    f.request.path = "/changed"
     ctx.master.replay_request(f, block=True)

examples/complex/flowbasic.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
@@ -8,14 +8,14 @@
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
-from mitmproxy import flow, controller, options
+from mitmproxy import controller, options, master
 from mitmproxy.proxy import ProxyServer, ProxyConfig
 
 
-class MyMaster(flow.FlowMaster):
+class MyMaster(master.Master):
     def run(self):
         try:
-            flow.FlowMaster.run(self)
+            master.Master.run(self)
         except KeyboardInterrupt:
             self.shutdown()
 
@@ -35,9 +35,9 @@ class MyMaster(flow.FlowMaster):
     def log(self, l):
         print("log", l.msg)
 
+
 opts = options.Options(cadir="~/.mitmproxy/")
 config = ProxyConfig(opts)
-state = flow.State()
 server = ProxyServer(config)
-m = MyMaster(opts, server, state)
+m = MyMaster(opts, server)
 m.run()

examples/complex/har_dump.py

@@ -3,7 +3,6 @@ This inline script can be used to dump flows as HAR files.
 """
 
 
-import pprint
 import json
 import sys
 import base64
@@ -14,9 +13,9 @@ import pytz
 
 import mitmproxy
 
-from netlib import version
-from netlib import strutils
-from netlib.http import cookies
+from mitmproxy import version
+from mitmproxy.utils import strutils
+from mitmproxy.net.http import cookies
 
 HAR = {}
 
@@ -128,22 +127,25 @@ def response(flow):
         "timings": timings,
     }
 
-    # Store binay data as base64
+    # Store binary data as base64
     if strutils.is_mostly_bin(flow.response.content):
-        b64 = base64.b64encode(flow.response.content)
-        entry["response"]["content"]["text"] = b64.decode('ascii')
+        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
         entry["response"]["content"]["encoding"] = "base64"
     else:
-        entry["response"]["content"]["text"] = flow.response.text
+        entry["response"]["content"]["text"] = flow.response.get_text(strict=False)
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
+        params = [
+            {"name": a, "value": b}
+            for a, b in flow.request.urlencoded_form.items(multi=True)
+        ]
         entry["request"]["postData"] = {
-            "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
-            "text": flow.request.content,
-            "params": name_value(flow.request.urlencoded_form)
+            "mimeType": flow.request.headers.get("Content-Type", ""),
+            "text": flow.request.get_text(strict=False),
+            "params": params
         }
 
-    if flow.server_conn:
+    if flow.server_conn.connected():
         entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
 
     HAR["log"]["entries"].append(entry)
@@ -155,16 +157,17 @@ def done():
     """
     dump_file = sys.argv[1]
 
+    json_dump = json.dumps(HAR, indent=2)  # type: str
+
     if dump_file == '-':
-        mitmproxy.ctx.log(pprint.pformat(HAR))
+        mitmproxy.ctx.log(json_dump)
     else:
-        json_dump = json.dumps(HAR, indent=2)
-
+        raw = json_dump.encode()  # type: bytes
         if dump_file.endswith('.zhar'):
-            json_dump = zlib.compress(json_dump, 9)
+            raw = zlib.compress(raw, 9)
 
-        with open(dump_file, "w") as f:
-            f.write(json_dump)
+        with open(dump_file, "wb") as f:
+            f.write(raw)
 
         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
 

examples/complex/mitmproxywrapper.py

@@ -15,7 +15,7 @@ import os
 import sys
 
 
-class Wrapper(object):
+class Wrapper:
     def __init__(self, port, extra_arguments=None):
         self.port = port
         self.extra_arguments = extra_arguments

examples/complex/nonblocking.py

@@ -0,0 +1,11 @@
+import time
+
+from mitmproxy.script import concurrent
+
+
+@concurrent  # Remove this and see what happens
+def request(flow):
+    # You don't want to use mitmproxy.ctx from a different thread
+    print("handle request: %s%s" % (flow.request.host, flow.request.path))
+    time.sleep(5)
+    print("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/complex/sslstrip.py

@@ -1,5 +1,9 @@
+"""
+This script implements an sslstrip-like attack based on mitmproxy.
+https://moxie.org/software/sslstrip/
+"""
 import re
-from six.moves import urllib
+import urllib
 
 # set of SSL/TLS capable hosts
 secure_hosts = set()
@@ -17,13 +21,18 @@ def request(flow):
         flow.request.scheme = 'https'
         flow.request.port = 443
 
+        # We need to update the request destination to whatever is specified in the host header:
+        # Having no TLS Server Name Indication from the client and just an IP address as request.host
+        # in transparent mode, TLS server name certificate validation would fail.
+        flow.request.host = flow.request.pretty_host
+
 
 def response(flow):
     flow.response.headers.pop('Strict-Transport-Security', None)
     flow.response.headers.pop('Public-Key-Pins', None)
 
     # strip links in response body
-    flow.response.content = flow.response.content.replace('https://', 'http://')
+    flow.response.content = flow.response.content.replace(b'https://', b'http://')
 
     # strip meta tag upgrade-insecure-requests in response body
     csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>'

examples/complex/stream.py

@@ -1,5 +1,6 @@
 def responseheaders(flow):
     """
     Enables streaming for all responses.
+    This is equivalent to passing `--stream 0` to mitmproxy.
     """
     flow.response.stream = True

examples/complex/tcp_message.py

@@ -8,7 +8,7 @@ tcp_message Inline Script Hook API Demonstration
 example cmdline invocation:
 mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py
 """
-from netlib import strutils
+from mitmproxy.utils import strutils
 
 
 def tcp_message(tcp_msg):

examples/complex/tls_passthrough.py

@@ -20,7 +20,6 @@ Example:
 
 Authors: Maximilian Hils, Matthew Tuusberg
 """
-from __future__ import absolute_import, print_function, division
 import collections
 import random
 
@@ -29,7 +28,7 @@ from enum import Enum
 
 import mitmproxy
 from mitmproxy.exceptions import TlsProtocolException
-from mitmproxy.protocol import TlsLayer, RawTCPLayer
+from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
 
 
 class InterceptionResult(Enum):
@@ -38,7 +37,7 @@ class InterceptionResult(Enum):
     skipped = None
 
 
-class _TlsStrategy(object):
+class _TlsStrategy:
     """
     Abstract base class for interception strategies.
     """

examples/custom_contentviews.py

@@ -1,70 +0,0 @@
-import string
-import lxml.html
-import lxml.etree
-from mitmproxy import contentviews
-from netlib import strutils
-
-
-class ViewPigLatin(contentviews.View):
-    name = "pig_latin_HTML"
-    prompt = ("pig latin HTML", "l")
-    content_types = ["text/html"]
-
-    def __call__(self, data, **metadata):
-        if strutils.is_xml(data):
-            parser = lxml.etree.HTMLParser(
-                strip_cdata=True,
-                remove_blank_text=True
-            )
-            d = lxml.html.fromstring(data, parser=parser)
-            docinfo = d.getroottree().docinfo
-
-            def piglify(src):
-                words = src.split()
-                ret = ''
-                for word in words:
-                    idx = -1
-                    while word[idx] in string.punctuation and (idx * -1) != len(word):
-                        idx -= 1
-                    if word[0].lower() in 'aeiou':
-                        if idx == -1:
-                            ret += word[0:] + "hay"
-                        else:
-                            ret += word[0:len(word) + idx + 1] + "hay" + word[idx + 1:]
-                    else:
-                        if idx == -1:
-                            ret += word[1:] + word[0] + "ay"
-                        else:
-                            ret += word[1:len(word) + idx + 1] + word[0] + "ay" + word[idx + 1:]
-                    ret += ' '
-                return ret.strip()
-
-            def recurse(root):
-                if hasattr(root, 'text') and root.text:
-                    root.text = piglify(root.text)
-                if hasattr(root, 'tail') and root.tail:
-                    root.tail = piglify(root.tail)
-
-                if len(root):
-                    for child in root:
-                        recurse(child)
-
-            recurse(d)
-
-            s = lxml.etree.tostring(
-                d,
-                pretty_print=True,
-                doctype=docinfo.doctype
-            )
-            return "HTML", contentviews.format_text(s)
-
-
-pig_view = ViewPigLatin()
-
-
-def start():
-    contentviews.add(pig_view)
-
-
-def done():
-    contentviews.remove(pig_view)

examples/dup_and_replay.py

@@ -1,7 +0,0 @@
-from mitmproxy import master
-
-
-def request(flow):
-    f = master.duplicate_flow(flow)
-    f.request.path = "/changed"
-    master.replay_request(f, block=True, run_scripthooks=False)

examples/fail_with_500.py

@@ -1,3 +0,0 @@
-def response(flow):
-    flow.response.status_code = 500
-    flow.response.content = b""

examples/logging.py

@@ -1,6 +0,0 @@
-from mitmproxy import ctx
-
-
-def start():
-    ctx.log.info("This is some informative text.")
-    ctx.log.error("This is an error.")

examples/nonblocking.py

@@ -1,10 +0,0 @@
-import time
-import mitmproxy
-from mitmproxy.script import concurrent
-
-
-@concurrent  # Remove this and see what happens
-def request(flow):
-    mitmproxy.ctx.log("handle request: %s%s" % (flow.request.host, flow.request.path))
-    time.sleep(5)
-    mitmproxy.ctx.log("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/redirect_requests.py

@@ -1,19 +0,0 @@
-"""
-This example shows two ways to redirect flows to other destinations.
-"""
-from mitmproxy.models import HTTPResponse
-
-
-def request(flow):
-    # pretty_host takes the "Host" header of the request into account,
-    # which is useful in transparent mode where we usually only have the IP
-    # otherwise.
-
-    # Method 1: Answer with a locally generated response
-    if flow.request.pretty_host.endswith("example.com"):
-        resp = HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"})
-        flow.reply.send(resp)
-
-    # Method 2: Redirect the request to a different server
-    if flow.request.pretty_host.endswith("example.org"):
-        flow.request.host = "mitmproxy.org"

examples/simple/README.md

@@ -0,0 +1,18 @@
+## Simple Examples
+
+| Filename                     | Description                                                                |
+|:-----------------------------|:---------------------------------------------------------------------------|
+| add_header.py                | Simple script that just adds a header to every request.                    |
+| custom_contentview.py        | Add a custom content view to the mitmproxy UI.                             |
+| filter_flows.py              | This script demonstrates how to use mitmproxy's filter pattern in scripts. |
+| io_read_dumpfile.py          | Read a dumpfile generated by mitmproxy.                                    |
+| io_write_dumpfile.py         | Only write selected flows into a mitmproxy dumpfile.                       |
+| logging.py                   | Use mitmproxy's logging API.                                               |
+| modify_body_inject_iframe.py | Inject configurable iframe into pages.                                     |
+| modify_form.py               | Modify HTTP form submissions.                                              |
+| modify_querystring.py        | Modify HTTP query strings.                                                 |
+| redirect_requests.py         | Redirect a request to a different server.                                  |
+| script_arguments.py          | Add arguments to a script.                                                 |
+| send_reply_from_proxy.py     | Send a HTTP response directly from the proxy.                              |
+| upsidedownternet.py          | Turn all images upside down.                                               |
+| wsgi_flask_app.py            | Embed a WSGI app into mitmproxy.                                           |

examples/simple/custom_contentview.py

@@ -0,0 +1,28 @@
+"""
+This example shows how one can add a custom contentview to mitmproxy.
+The content view API is explained in the mitmproxy.contentviews module.
+"""
+from mitmproxy import contentviews
+
+
+class ViewSwapCase(contentviews.View):
+    name = "swapcase"
+
+    # We don't have a good solution for the keyboard shortcut yet -
+    # you manually need to find a free letter. Contributions welcome :)
+    prompt = ("swap case text", "p")
+    content_types = ["text/plain"]
+
+    def __call__(self, data: bytes, **metadata):
+        return "case-swapped text", contentviews.format_text(data.swapcase())
+
+
+view = ViewSwapCase()
+
+
+def start():
+    contentviews.add(view)
+
+
+def done():
+    contentviews.remove(view)

examples/simple/filter_flows.py

@@ -1,6 +1,8 @@
-# This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
-# Usage: mitmdump -s "flowfilter.py FILTER"
-
+"""
+This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
+Usage:
+    mitmdump -s "flowfilter.py FILTER"
+"""
 import sys
 from mitmproxy import flowfilter
 
@@ -10,7 +12,7 @@ class Filter:
         self.filter = flowfilter.parse(spec)
 
     def response(self, flow):
-        if flowfilter.match(flow, self.filter):
+        if flowfilter.match(self.filter, flow):
             print("Flow matches filter:")
             print(flow)
 

examples/simple/io_read_dumpfile.py

@@ -3,13 +3,13 @@
 # Simple script showing how to read a mitmproxy dump file
 #
 
-from mitmproxy import flow
+from mitmproxy import io
 from mitmproxy.exceptions import FlowReadException
 import pprint
 import sys
 
 with open(sys.argv[1], "rb") as logfile:
-    freader = flow.FlowReader(logfile)
+    freader = io.FlowReader(logfile)
     pp = pprint.PrettyPrinter(indent=4)
     try:
         for f in freader.stream():

examples/simple/io_write_dumpfile.py

@@ -1,7 +1,13 @@
+"""
+This script how to generate a mitmproxy dump file,
+as it would also be generated by passing `-w` to mitmproxy.
+In contrast to `-w`, this gives you full control over which
+flows should be saved and also allows you to rotate files or log
+to multiple files in parallel.
+"""
 import random
 import sys
-
-from mitmproxy.flow import FlowWriter
+from mitmproxy import io
 
 
 class Writer:
@@ -10,7 +16,7 @@ class Writer:
             f = sys.stdout
         else:
             f = open(path, "wb")
-        self.w = FlowWriter(f)
+        self.w = io.FlowWriter(f)
 
     def response(self, flow):
         if random.choice([True, False]):

examples/simple/log_events.py

@@ -0,0 +1,12 @@
+"""
+It is recommended to use `ctx.log` for logging within a script.
+This goes to the event log in mitmproxy and to stdout in mitmdump.
+
+If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 :-)
+"""
+from mitmproxy import ctx
+
+
+def start():
+    ctx.log.info("This is some informative text.")
+    ctx.log.error("This is an error.")

examples/simple/modify_body_inject_iframe.py

@@ -11,7 +11,7 @@ class Injector:
     def response(self, flow):
         if flow.request.host in self.iframe_url:
             return
-        html = BeautifulSoup(flow.response.content, "lxml")
+        html = BeautifulSoup(flow.response.content, "html.parser")
         if html.body:
             iframe = html.new_tag(
                 "iframe",

examples/simple/modify_form.py

@@ -1,7 +1,9 @@
 def request(flow):
     if flow.request.urlencoded_form:
+        # If there's already a form, one can just add items to the dict:
         flow.request.urlencoded_form["mitmproxy"] = "rocks"
     else:
+        # One can also just pass new form data.
         # This sets the proper content type and overrides the body.
         flow.request.urlencoded_form = [
             ("foo", "bar")

examples/simple/redirect_requests.py

@@ -0,0 +1,11 @@
+"""
+This example shows two ways to redirect flows to another server.
+"""
+
+
+def request(flow):
+    # pretty_host takes the "Host" header of the request into account,
+    # which is useful in transparent mode where we usually only have the IP
+    # otherwise.
+    if flow.request.pretty_host == "example.org":
+        flow.request.host = "mitmproxy.org"

examples/simple/send_reply_from_proxy.py

@@ -0,0 +1,17 @@
+"""
+This example shows how to send a reply from the proxy immediately
+without sending any data to the remote server.
+"""
+from mitmproxy import http
+
+
+def request(flow):
+    # pretty_url takes the "Host" header of the request into account, which
+    # is useful in transparent mode where we usually only have the IP otherwise.
+
+    if flow.request.pretty_url == "http://example.com/path":
+        flow.response = http.HTTPResponse.make(
+            200,  # (optional) status code
+            b"Hello World",  # (optional) content
+            {"Content-Type": "text/html"}  # (optional) headers
+        )

examples/simple/upsidedownternet.py

@@ -0,0 +1,16 @@
+"""
+This script rotates all images passing through the proxy by 180 degrees.
+"""
+import io
+
+from PIL import Image
+
+
+def response(flow):
+    if flow.response.headers.get("content-type", "").startswith("image"):
+        s = io.BytesIO(flow.response.content)
+        img = Image.open(s).rotate(180)
+        s2 = io.BytesIO()
+        img.save(s2, "png")
+        flow.response.content = s2.getvalue()
+        flow.response.headers["content-type"] = "image/png"

examples/simple/wsgi_flask_app.py

@@ -4,7 +4,7 @@ instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
 a single simplest-possible page.
 """
 from flask import Flask
-import mitmproxy
+from mitmproxy.addons import wsgiapp
 
 app = Flask("proxapp")
 
@@ -14,12 +14,12 @@ def hello_world():
     return 'Hello World!'
 
 
-# Register the app using the magic domain "proxapp" on port 80. Requests to
-# this domain and port combination will now be routed to the WSGI app instance.
 def start():
-    mitmproxy.ctx.master.apps.add(app, "proxapp", 80)
+    # Host app at the magic domain "proxapp" on port 80. Requests to this
+    # domain and port combination will now be routed to the WSGI app instance.
+    return wsgiapp.WSGIApp(app, "proxapp", 80)
 
     # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
     # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
     # but won't send any data.
-    mitmproxy.ctx.master.apps.add(app, "example.com", 443)
+    # mitmproxy.ctx.master.apps.add(app, "example.com", 443)

examples/stickycookies

@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-"""
-This example builds on mitmproxy's base proxying infrastructure to
-implement functionality similar to the "sticky cookies" option.
-
-Heads Up: In the majority of cases, you want to use inline scripts.
-"""
-import os
-from mitmproxy import controller, proxy
-from mitmproxy.proxy.server import ProxyServer
-
-
-class StickyMaster(controller.Master):
-    def __init__(self, server):
-        controller.Master.__init__(self, server)
-        self.stickyhosts = {}
-
-    def run(self):
-        try:
-            return controller.Master.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    @controller.handler
-    def request(self, flow):
-        hid = (flow.request.host, flow.request.port)
-        if "cookie" in flow.request.headers:
-            self.stickyhosts[hid] = flow.request.headers.get_all("cookie")
-        elif hid in self.stickyhosts:
-            flow.request.headers.set_all("cookie", self.stickyhosts[hid])
-
-    @controller.handler
-    def response(self, flow):
-        hid = (flow.request.host, flow.request.port)
-        if "set-cookie" in flow.response.headers:
-            self.stickyhosts[hid] = flow.response.headers.get_all("set-cookie")
-
-
-config = proxy.ProxyConfig(port=8080)
-server = ProxyServer(config)
-m = StickyMaster(server)
-m.run()

examples/stub.py

@@ -1,87 +0,0 @@
-import mitmproxy
-"""
-    This is a script stub, with definitions for all events.
-"""
-
-
-def start():
-    """
-        Called once on script startup before any other events
-    """
-    mitmproxy.ctx.log("start")
-
-
-def configure(options, updated):
-    """
-        Called once on script startup before any other events, and whenever options changes.
-    """
-    mitmproxy.ctx.log("configure")
-
-
-def clientconnect(root_layer):
-    """
-        Called when a client initiates a connection to the proxy. Note that a
-        connection can correspond to multiple HTTP requests
-    """
-    mitmproxy.ctx.log("clientconnect")
-
-
-def request(flow):
-    """
-        Called when a client request has been received.
-    """
-    mitmproxy.ctx.log("request")
-
-
-def serverconnect(server_conn):
-    """
-        Called when the proxy initiates a connection to the target server. Note that a
-        connection can correspond to multiple HTTP requests
-    """
-    mitmproxy.ctx.log("serverconnect")
-
-
-def responseheaders(flow):
-    """
-        Called when the response headers for a server response have been received,
-        but the response body has not been processed yet. Can be used to tell mitmproxy
-        to stream the response.
-    """
-    mitmproxy.ctx.log("responseheaders")
-
-
-def response(flow):
-    """
-       Called when a server response has been received.
-    """
-    mitmproxy.ctx.log("response")
-
-
-def error(flow):
-    """
-        Called when a flow error has occured, e.g. invalid server responses, or
-        interrupted connections. This is distinct from a valid server HTTP error
-        response, which is simply a response with an HTTP error code.
-    """
-    mitmproxy.ctx.log("error")
-
-
-def serverdisconnect(server_conn):
-    """
-        Called when the proxy closes the connection to the target server.
-    """
-    mitmproxy.ctx.log("serverdisconnect")
-
-
-def clientdisconnect(root_layer):
-    """
-        Called when a client disconnects from the proxy.
-    """
-    mitmproxy.ctx.log("clientdisconnect")
-
-
-def done():
-    """
-        Called once on script shutdown, after any other events.
-    """
-    mitmproxy.ctx.log("done")

examples/upsidedownternet.py

@@ -1,15 +0,0 @@
-from six.moves import cStringIO as StringIO
-from PIL import Image
-
-
-def response(flow):
-    if flow.response.headers.get("content-type", "").startswith("image"):
-        try:
-            s = StringIO(flow.response.content)
-            img = Image.open(s).rotate(180)
-            s2 = StringIO()
-            img.save(s2, "png")
-            flow.response.content = s2.getvalue()
-            flow.response.headers["content-type"] = "image/png"
-        except:  # Unknown image types etc.
-            pass

issue_template.md

@@ -1,22 +1,20 @@
 ##### Steps to reproduce the problem:
 
-1. 
-2. 
-3. 
-
-##### What is the expected behavior?
-
-
-##### What went wrong?
+1.
+2.
+3.
 
 
 ##### Any other comments? What have you tried so far?
 
 
----
 
-Mitmproxy Version:
-Operating System:
+##### System information
 
 
-<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) -->
+<!--
+    Cut and paste the output of "mitmdump --sysinfo".
+
+    If you're using an older version if mitmproxy, please specify the version
+    and OS.
+-->

mitmproxy/__init__.py

@@ -0,0 +1,3 @@
+# https://github.com/mitmproxy/mitmproxy/issues/1809
+# import script here so that pyinstaller registers it.
+from . import script  # noqa

mitmproxy/addonmanager.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import, print_function, division
 from mitmproxy import exceptions
 import pprint
 
@@ -7,12 +6,19 @@ def _get_name(itm):
     return getattr(itm, "name", itm.__class__.__name__.lower())
 
 
-class Addons(object):
+class AddonManager:
     def __init__(self, master):
         self.chain = []
         self.master = master
         master.options.changed.connect(self._options_update)
 
+    def clear(self):
+        """
+            Remove all addons.
+        """
+        self.done()
+        self.chain = []
+
     def get(self, name):
         """
             Retrieve an addon by name. Addon names are equal to the .name
@@ -26,7 +32,7 @@ class Addons(object):
     def _options_update(self, options, updated):
         for i in self.chain:
             with self.master.handlecontext():
-                i.configure(options, updated)
+                self.invoke_with_context(i, "configure", options, updated)
 
     def startup(self, s):
         """
@@ -44,8 +50,6 @@ class Addons(object):
         """
             Add addons to the end of the chain, and run their startup events.
         """
-        if not addons:
-            raise ValueError("No addons specified.")
         self.chain.extend(addons)
         for i in addons:
             self.startup(i)
@@ -82,4 +86,7 @@ class Addons(object):
 
     def __call__(self, name, *args, **kwargs):
         for i in self.chain:
-            self.invoke(i, name, *args, **kwargs)
+            try:
+                self.invoke(i, name, *args, **kwargs)
+            except exceptions.AddonHalt:
+                return

mitmproxy/addons/__init__.py

@@ -0,0 +1,35 @@
+from mitmproxy.addons import anticache
+from mitmproxy.addons import anticomp
+from mitmproxy.addons import clientplayback
+from mitmproxy.addons import streamfile
+from mitmproxy.addons import onboarding
+from mitmproxy.addons import proxyauth
+from mitmproxy.addons import replace
+from mitmproxy.addons import script
+from mitmproxy.addons import setheaders
+from mitmproxy.addons import serverplayback
+from mitmproxy.addons import stickyauth
+from mitmproxy.addons import stickycookie
+from mitmproxy.addons import streambodies
+from mitmproxy.addons import upstream_auth
+from mitmproxy.addons import disable_h2c_upgrade
+
+
+def default_addons():
+    return [
+        onboarding.Onboarding(),
+        proxyauth.ProxyAuth(),
+        anticache.AntiCache(),
+        anticomp.AntiComp(),
+        stickyauth.StickyAuth(),
+        stickycookie.StickyCookie(),
+        script.ScriptLoader(),
+        streamfile.StreamFile(),
+        streambodies.StreamBodies(),
+        replace.Replace(),
+        setheaders.SetHeaders(),
+        serverplayback.ServerPlayback(),
+        clientplayback.ClientPlayback(),
+        upstream_auth.UpstreamAuth(),
+        disable_h2c_upgrade.DisableH2CleartextUpgrade(),
+    ]

mitmproxy/addons/anticache.py

@@ -1,6 +1,3 @@
-from __future__ import absolute_import, print_function, division
-
-
 class AntiCache:
     def __init__(self):
         self.enabled = False

mitmproxy/addons/anticomp.py

@@ -1,6 +1,3 @@
-from __future__ import absolute_import, print_function, division
-
-
 class AntiComp:
     def __init__(self):
         self.enabled = False

mitmproxy/addons/clientplayback.py

@@ -1,19 +1,24 @@
-from mitmproxy import exceptions, flow, ctx
+from mitmproxy import exceptions
+from mitmproxy import ctx
+from mitmproxy import io
+from mitmproxy import flow
+
+import typing
 
 
 class ClientPlayback:
     def __init__(self):
         self.flows = None
-        self.current = None
-        self.keepserving = None
+        self.current_thread = None
+        self.keepserving = False
         self.has_replayed = False
 
-    def count(self):
+    def count(self) -> int:
         if self.flows:
             return len(self.flows)
         return 0
 
-    def load(self, flows):
+    def load(self, flows: typing.Sequence[flow.Flow]):
         self.flows = flows
 
     def configure(self, options, updated):
@@ -21,7 +26,7 @@ class ClientPlayback:
             if options.client_replay:
                 ctx.log.info("Client Replay: {}".format(options.client_replay))
                 try:
-                    flows = flow.read_flows_from_paths(options.client_replay)
+                    flows = io.read_flows_from_paths(options.client_replay)
                 except exceptions.FlowReadException as e:
                     raise exceptions.OptionsError(str(e))
                 self.load(flows)
@@ -30,11 +35,11 @@ class ClientPlayback:
         self.keepserving = options.keepserving
 
     def tick(self):
-        if self.current and not self.current.is_alive():
-            self.current = None
-        if self.flows and not self.current:
-            self.current = ctx.master.replay_request(self.flows.pop(0))
+        if self.current_thread and not self.current_thread.is_alive():
+            self.current_thread = None
+        if self.flows and not self.current_thread:
+            self.current_thread = ctx.master.replay_request(self.flows.pop(0))
             self.has_replayed = True
         if self.has_replayed:
-            if not self.flows and not self.current and not self.keepserving:
+            if not self.flows and not self.current_thread and not self.keepserving:
                 ctx.master.shutdown()

mitmproxy/addons/disable_h2c_upgrade.py

@@ -0,0 +1,21 @@
+class DisableH2CleartextUpgrade:
+
+    """
+    We currently only support HTTP/2 over a TLS connection. Some clients try
+    to upgrade a connection from HTTP/1.1 to h2c, so we need to remove those
+    headers to avoid protocol errors if one endpoints suddenly starts sending
+    HTTP/2 frames.
+    """
+
+    def process_flow(self, f):
+        if f.request.headers.get('upgrade', '') == 'h2c':
+            del f.request.headers['upgrade']
+            if 'connection' in f.request.headers:
+                del f.request.headers['connection']
+            if 'http2-settings' in f.request.headers:
+                del f.request.headers['http2-settings']
+
+    # Handlers
+
+    def request(self, f):
+        self.process_flow(f)

mitmproxy/addons/dumper.py

@@ -1,6 +1,5 @@
-from __future__ import absolute_import, print_function, division
-
 import itertools
+import sys
 
 import click
 
@@ -10,35 +9,43 @@ from mitmproxy import contentviews
 from mitmproxy import ctx
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
-from netlib import human
-from netlib import strutils
+from mitmproxy.utils import human
+from mitmproxy.utils import strutils
 
 
-def indent(n, text):
+def indent(n: int, text: str) -> str:
     l = str(text).strip().splitlines()
     pad = " " * n
     return "\n".join(pad + i for i in l)
 
 
-class Dumper(object):
-    def __init__(self):
+def colorful(line, styles):
+    yield u"    "  # we can already indent here
+    for (style, text) in line:
+        yield click.style(text, **styles.get(style, {}))
+
+
+class Dumper:
+    def __init__(self, outfile=sys.stdout):
         self.filter = None  # type: flowfilter.TFilter
         self.flow_detail = None  # type: int
-        self.outfp = None  # type: typing.io.TextIO
+        self.outfp = outfile  # type: typing.io.TextIO
         self.showhost = None  # type: bool
+        self.default_contentview = "auto"  # type: str
 
     def configure(self, options, updated):
-        if options.filtstr:
-            self.filter = flowfilter.parse(options.filtstr)
-            if not self.filter:
-                raise exceptions.OptionsError(
-                    "Invalid filter expression: %s" % options.filtstr
-                )
-        else:
-            self.filter = None
+        if "filtstr" in updated:
+            if options.filtstr:
+                self.filter = flowfilter.parse(options.filtstr)
+                if not self.filter:
+                    raise exceptions.OptionsError(
+                        "Invalid filter expression: %s" % options.filtstr
+                    )
+            else:
+                self.filter = None
         self.flow_detail = options.flow_detail
-        self.outfp = options.tfile
         self.showhost = options.showhost
+        self.default_contentview = options.default_contentview
 
     def echo(self, text, ident=None, **style):
         if ident:
@@ -47,66 +54,50 @@ class Dumper(object):
         if self.outfp:
             self.outfp.flush()
 
-    def _echo_message(self, message):
-        if self.flow_detail >= 2 and hasattr(message, "headers"):
-            headers = "\r\n".join(
-                "{}: {}".format(
-                    click.style(
-                        strutils.bytes_to_escaped_str(k), fg="blue", bold=True
-                    ),
-                    click.style(
-                        strutils.bytes_to_escaped_str(v), fg="blue"
-                    )
-                )
-                for k, v in message.headers.fields
+    def _echo_headers(self, headers):
+        for k, v in headers.fields:
+            k = strutils.bytes_to_escaped_str(k)
+            v = strutils.bytes_to_escaped_str(v)
+            out = "{}: {}".format(
+                click.style(k, fg="blue"),
+                click.style(v)
             )
-            self.echo(headers, ident=4)
-        if self.flow_detail >= 3:
-                _, lines, error = contentviews.get_message_content_view(
-                    contentviews.get("Auto"),
-                    message
-                )
-                if error:
-                    ctx.log.debug(error)
+            self.echo(out, ident=4)
 
-                styles = dict(
-                    highlight=dict(bold=True),
-                    offset=dict(fg="blue"),
-                    header=dict(fg="green", bold=True),
-                    text=dict(fg="green")
-                )
+    def _echo_message(self, message):
+        _, lines, error = contentviews.get_message_content_view(
+            self.default_contentview,
+            message
+        )
+        if error:
+            ctx.log.debug(error)
 
-                def colorful(line):
-                    yield u"    "  # we can already indent here
-                    for (style, text) in line:
-                        yield click.style(text, **styles.get(style, {}))
+        if self.flow_detail == 3:
+            lines_to_echo = itertools.islice(lines, 70)
+        else:
+            lines_to_echo = lines
 
-                if self.flow_detail == 3:
-                    lines_to_echo = itertools.islice(lines, 70)
-                else:
-                    lines_to_echo = lines
+        styles = dict(
+            highlight=dict(bold=True),
+            offset=dict(fg="blue"),
+            header=dict(fg="green", bold=True),
+            text=dict(fg="green")
+        )
 
-                content = u"\r\n".join(
-                    u"".join(colorful(line)) for line in lines_to_echo
-                )
-                if content:
-                    self.echo("")
-                    self.echo(content)
+        content = u"\r\n".join(
+            u"".join(colorful(line, styles)) for line in lines_to_echo
+        )
+        if content:
+            self.echo("")
+            self.echo(content)
 
-                if next(lines, None):
-                    self.echo("(cut off)", ident=4, dim=True)
+        if next(lines, None):
+            self.echo("(cut off)", ident=4, dim=True)
 
         if self.flow_detail >= 2:
             self.echo("")
 
     def _echo_request_line(self, flow):
-        if flow.request.stickycookie:
-            stickycookie = click.style(
-                "[stickycookie] ", fg="yellow", bold=True
-            )
-        else:
-            stickycookie = ""
-
         if flow.client_conn:
             client = click.style(
                 strutils.escape_control_characters(
@@ -118,7 +109,8 @@ class Dumper(object):
         else:
             client = ""
 
-        method = flow.request.method
+        pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
+        method = flow.request.method + pushed
         method_color = dict(
             GET="green",
             DELETE="red"
@@ -139,15 +131,8 @@ class Dumper(object):
             # We hide "normal" HTTP 1.
             http_version = " " + flow.request.http_version
 
-        if self.flow_detail >= 2:
-            linebreak = "\n    "
-        else:
-            linebreak = ""
-
-        line = "{client}: {linebreak}{stickycookie}{method} {url}{http_version}".format(
+        line = "{client}: {method} {url}{http_version}".format(
             client=client,
-            stickycookie=stickycookie,
-            linebreak=linebreak,
             method=method,
             url=url,
             http_version=http_version
@@ -205,11 +190,17 @@ class Dumper(object):
     def echo_flow(self, f):
         if f.request:
             self._echo_request_line(f)
-            self._echo_message(f.request)
+            if self.flow_detail >= 2:
+                self._echo_headers(f.request.headers)
+            if self.flow_detail >= 3:
+                self._echo_message(f.request)
 
         if f.response:
             self._echo_response_line(f)
-            self._echo_message(f.response)
+            if self.flow_detail >= 2:
+                self._echo_headers(f.response.headers)
+            if self.flow_detail >= 3:
+                self._echo_message(f.response)
 
         if f.error:
             msg = strutils.escape_control_characters(f.error.msg)
@@ -232,6 +223,29 @@ class Dumper(object):
         if self.match(f):
             self.echo_flow(f)
 
+    def websocket_error(self, f):
+        self.echo(
+            "Error in WebSocket connection to {}: {}".format(
+                repr(f.server_conn.address), f.error
+            ),
+            fg="red"
+        )
+
+    def websocket_message(self, f):
+        if self.match(f):
+            message = f.messages[-1]
+            self.echo(message.info)
+            if self.flow_detail >= 3:
+                self._echo_message(message)
+
+    def websocket_end(self, f):
+        if self.match(f):
+            self.echo("WebSocket connection closed by {}: {} {}, {}".format(
+                f.close_sender,
+                f.close_code,
+                f.close_message,
+                f.close_reason))
+
     def tcp_error(self, f):
         self.echo(
             "Error in TCP connection to {}: {}".format(
@@ -241,13 +255,13 @@ class Dumper(object):
         )
 
     def tcp_message(self, f):
-        if not self.match(f):
-            return
-        message = f.messages[-1]
-        direction = "->" if message.from_client else "<-"
-        self.echo("{client} {direction} tcp {direction} {server}".format(
-            client=repr(f.client_conn.address),
-            server=repr(f.server_conn.address),
-            direction=direction,
-        ))
-        self._echo_message(message)
+        if self.match(f):
+            message = f.messages[-1]
+            direction = "->" if message.from_client else "<-"
+            self.echo("{client} {direction} tcp {direction} {server}".format(
+                client=repr(f.client_conn.address),
+                server=repr(f.server_conn.address),
+                direction=direction,
+            ))
+            if self.flow_detail >= 3:
+                self._echo_message(message)

mitmproxy/addons/eventstore.py

@@ -0,0 +1,19 @@
+from typing import List  # noqa
+
+import blinker
+from mitmproxy.log import LogEntry
+
+
+class EventStore:
+    def __init__(self):
+        self.data = []  # type: List[LogEntry]
+        self.sig_add = blinker.Signal()
+        self.sig_refresh = blinker.Signal()
+
+    def log(self, entry: LogEntry):
+        self.data.append(entry)
+        self.sig_add.send(self, entry=entry)
+
+    def clear(self):
+        self.data.clear()
+        self.sig_refresh.send(self)

mitmproxy/addons/intercept.py

@@ -0,0 +1,35 @@
+from mitmproxy import flowfilter
+from mitmproxy import exceptions
+
+
+class Intercept:
+    def __init__(self):
+        self.filt = None
+
+    def configure(self, opts, updated):
+        if "intercept" in updated:
+            if not opts.intercept:
+                self.filt = None
+                return
+            self.filt = flowfilter.parse(opts.intercept)
+            if not self.filt:
+                raise exceptions.OptionsError(
+                    "Invalid interception filter: %s" % opts.intercept
+                )
+
+    def process_flow(self, f):
+        if self.filt:
+            should_intercept = all([
+                self.filt(f),
+                not f.request.is_replay,
+            ])
+            if should_intercept:
+                f.intercept()
+
+    # Handlers
+
+    def request(self, f):
+        self.process_flow(f)
+
+    def response(self, f):
+        self.process_flow(f)

mitmproxy/addons/onboarding.py

@@ -0,0 +1,17 @@
+from mitmproxy.addons import wsgiapp
+from mitmproxy.addons.onboardingapp import app
+
+
+class Onboarding(wsgiapp.WSGIApp):
+    def __init__(self):
+        super().__init__(app.Adapter(app.application), None, None)
+        self.enabled = False
+
+    def configure(self, options, updated):
+        self.host = options.app_host
+        self.port = options.app_port
+        self.enabled = options.app
+
+    def request(self, f):
+        if self.enabled:
+            super().request(f)

mitmproxy/addons/onboardingapp/app.py

@@ -1,15 +1,13 @@
-from __future__ import absolute_import, print_function, division
-
 import os
 
 import tornado.template
 import tornado.web
 import tornado.wsgi
 
-from mitmproxy import utils
+from mitmproxy.utils import data
 from mitmproxy.proxy import config
 
-loader = tornado.template.Loader(utils.pkg_data.path("onboarding/templates"))
+loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates"))
 
 
 class Adapter(tornado.wsgi.WSGIAdapter):
@@ -87,10 +85,9 @@ application = tornado.web.Application(
             r"/static/(.*)",
             tornado.web.StaticFileHandler,
             {
-                "path": utils.pkg_data.path("onboarding/static")
+                "path": data.pkg_data.path("addons/onboardingapp/static")
             }
         ),
     ],
     # debug=True
 )
-mapp = Adapter(application)

mitmproxy/addons/onboardingapp/templates/index.html

@@ -24,6 +24,26 @@
 </div>
 
 <hr/>
+
+<div class="text-left">
+  <h3>Apple: How to install on macOS / OSX</h3>
+  <ul>
+    <li>Download PEM file (from above link)</li>
+    <li>Double-click the PEM file</li>
+    <li>The "Keychain Access" applications opens</li>
+    <li>Find the new certificate "mitmproxy" in the list</li>
+    <li>Double-click the "mitmproxy" entry</li>
+    <li>A dialog window openes up</li>
+    <li>Change "Secure Socket Layer (SSL)" to "Always Trust"</li>
+    <li>Close the dialog window (and enter your password if prompted)</li>
+    <li>Done!</li>
+  </ul>
+</div>
+
+
+
+<hr/>
+
 <div class="text-center">
     Other mitmproxy users cannot intercept your connection.
 </div>
@@ -32,4 +52,4 @@
     between mitmproxy installations.
 </div>
 
-{% end %}  
+{% end %}

mitmproxy/addons/proxyauth.py

@@ -0,0 +1,148 @@
+import binascii
+
+import passlib.apache
+
+from mitmproxy import exceptions
+from mitmproxy import http
+import mitmproxy.net.http
+
+
+REALM = "mitmproxy"
+
+
+def mkauth(username, password, scheme="basic"):
+    v = binascii.b2a_base64(
+        (username + ":" + password).encode("utf8")
+    ).decode("ascii")
+    return scheme + " " + v
+
+
+def parse_http_basic_auth(s):
+    words = s.split()
+    if len(words) != 2:
+        return None
+    scheme = words[0]
+    try:
+        user = binascii.a2b_base64(words[1]).decode("utf8", "replace")
+    except binascii.Error:
+        return None
+    parts = user.split(':')
+    if len(parts) != 2:
+        return None
+    return scheme, parts[0], parts[1]
+
+
+class ProxyAuth:
+    def __init__(self):
+        self.nonanonymous = False
+        self.htpasswd = None
+        self.singleuser = None
+
+    def enabled(self):
+        return any([self.nonanonymous, self.htpasswd, self.singleuser])
+
+    def which_auth_header(self, f):
+        if f.mode == "regular":
+            return 'Proxy-Authorization'
+        else:
+            return 'Authorization'
+
+    def auth_required_response(self, f):
+        if f.mode == "regular":
+            hdrname = 'Proxy-Authenticate'
+        else:
+            hdrname = 'WWW-Authenticate'
+
+        headers = mitmproxy.net.http.Headers()
+        headers[hdrname] = 'Basic realm="%s"' % REALM
+
+        if f.mode == "transparent":
+            return http.make_error_response(
+                401,
+                "Authentication Required",
+                headers
+            )
+        else:
+            return http.make_error_response(
+                407,
+                "Proxy Authentication Required",
+                headers,
+            )
+
+    def check(self, f):
+        auth_value = f.request.headers.get(self.which_auth_header(f), None)
+        if not auth_value:
+            return False
+        parts = parse_http_basic_auth(auth_value)
+        if not parts:
+            return False
+        scheme, username, password = parts
+        if scheme.lower() != 'basic':
+            return False
+
+        if self.nonanonymous:
+            pass
+        elif self.singleuser:
+            if [username, password] != self.singleuser:
+                return False
+        elif self.htpasswd:
+            if not self.htpasswd.check_password(username, password):
+                return False
+        else:
+            raise NotImplementedError("Should never happen.")
+
+        return True
+
+    def authenticate(self, f):
+        if self.check(f):
+            del f.request.headers[self.which_auth_header(f)]
+        else:
+            f.response = self.auth_required_response(f)
+
+    # Handlers
+    def configure(self, options, updated):
+        if "auth_nonanonymous" in updated:
+            self.nonanonymous = options.auth_nonanonymous
+        if "auth_singleuser" in updated:
+            if options.auth_singleuser:
+                parts = options.auth_singleuser.split(':')
+                if len(parts) != 2:
+                    raise exceptions.OptionsError(
+                        "Invalid single-user auth specification."
+                    )
+                self.singleuser = parts
+            else:
+                self.singleuser = None
+        if "auth_htpasswd" in updated:
+            if options.auth_htpasswd:
+                try:
+                    self.htpasswd = passlib.apache.HtpasswdFile(
+                        options.auth_htpasswd
+                    )
+                except (ValueError, OSError) as v:
+                    raise exceptions.OptionsError(
+                        "Could not open htpasswd file: %s" % v
+                    )
+            else:
+                self.htpasswd = None
+        if self.enabled():
+            if options.mode == "transparent":
+                raise exceptions.OptionsError(
+                    "Proxy Authentication not supported in transparent mode."
+                )
+            elif options.mode == "socks5":
+                raise exceptions.OptionsError(
+                    "Proxy Authentication not supported in SOCKS mode. "
+                    "https://github.com/mitmproxy/mitmproxy/issues/738"
+                )
+            # TODO: check for multiple auth options
+
+    def http_connect(self, f):
+        if self.enabled() and f.mode == "regular":
+            self.authenticate(f)
+
+    def requestheaders(self, f):
+        if self.enabled():
+            # Are we already authenticated in CONNECT?
+            if not (f.mode == "regular" and f.server_conn.via):
+                self.authenticate(f)

mitmproxy/addons/replace.py

@@ -16,21 +16,22 @@ class Replace:
             rex: a regular expression, as bytes.
             s: the replacement string, as bytes
         """
-        lst = []
-        for fpatt, rex, s in options.replacements:
-            flt = flowfilter.parse(fpatt)
-            if not flt:
-                raise exceptions.OptionsError(
-                    "Invalid filter pattern: %s" % fpatt
-                )
-            try:
-                re.compile(rex)
-            except re.error as e:
-                raise exceptions.OptionsError(
-                    "Invalid regular expression: %s - %s" % (rex, str(e))
-                )
-            lst.append((rex, s, flt))
-        self.lst = lst
+        if "replacements" in updated:
+            lst = []
+            for fpatt, rex, s in options.replacements:
+                flt = flowfilter.parse(fpatt)
+                if not flt:
+                    raise exceptions.OptionsError(
+                        "Invalid filter pattern: %s" % fpatt
+                    )
+                try:
+                    re.compile(rex)
+                except re.error as e:
+                    raise exceptions.OptionsError(
+                        "Invalid regular expression: %s - %s" % (rex, str(e))
+                    )
+                lst.append((rex, s, flt))
+            self.lst = lst
 
     def execute(self, f):
         for rex, s, flt in self.lst:

mitmproxy/addons/script.py

@@ -1,41 +1,26 @@
-from __future__ import absolute_import, print_function, division
-
 import contextlib
 import os
 import shlex
 import sys
 import threading
 import traceback
+import types
 
 from mitmproxy import exceptions
-from mitmproxy import controller
 from mitmproxy import ctx
-from mitmproxy.flow import master as flowmaster
+from mitmproxy import events
 
 
 import watchdog.events
 from watchdog.observers import polling
 
 
-class NS:
-    def __init__(self, ns):
-        self.__dict__["ns"] = ns
-
-    def __getattr__(self, key):
-        if key not in self.ns:
-            raise AttributeError("No such element: %s", key)
-        return self.ns[key]
-
-    def __setattr__(self, key, value):
-        self.__dict__["ns"][key] = value
-
-
 def parse_command(command):
     """
         Returns a (path, args) tuple.
     """
     if not command or not command.strip():
-        raise exceptions.AddonError("Empty script command.")
+        raise exceptions.OptionsError("Empty script command.")
     # Windows: escape all backslashes in the path.
     if os.name == "nt":  # pragma: no cover
         backslashes = shlex.split(command, posix=False)[0].count("\\")
@@ -43,13 +28,13 @@ def parse_command(command):
     args = shlex.split(command)  # pragma: no cover
     args[0] = os.path.expanduser(args[0])
     if not os.path.exists(args[0]):
-        raise exceptions.AddonError(
+        raise exceptions.OptionsError(
             ("Script file not found: %s.\r\n"
              "If your script path contains spaces, "
              "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") %
             args[0])
     elif os.path.isdir(args[0]):
-        raise exceptions.AddonError("Not a file: %s" % args[0])
+        raise exceptions.OptionsError("Not a file: %s" % args[0])
     return args[0], args[1:]
 
 
@@ -116,8 +101,8 @@ def load_script(path, args):
             return
     ns = {'__file__': os.path.abspath(path)}
     with scriptenv(path, args):
-        exec(code, ns, ns)
-    return NS(ns)
+        exec(code, ns)
+    return types.SimpleNamespace(**ns)
 
 
 class ReloadHandler(watchdog.events.FileSystemEventHandler):
@@ -156,7 +141,7 @@ class Script:
         self.last_options = None
         self.should_reload = threading.Event()
 
-        for i in controller.Events:
+        for i in events.Events:
             if not hasattr(self, i):
                 def mkprox():
                     evt = i
@@ -215,7 +200,7 @@ class Script:
         self.dead = True
 
 
-class ScriptLoader():
+class ScriptLoader:
     """
         An addon that manages loading scripts from options.
     """
@@ -223,7 +208,7 @@ class ScriptLoader():
         sc = Script(command)
         sc.load_script()
         for f in flows:
-            for evt, o in flowmaster.event_sequence(f):
+            for evt, o in events.event_sequence(f):
                 sc.run(evt, o)
         sc.done()
         return sc
@@ -236,8 +221,8 @@ class ScriptLoader():
 
             for a in ctx.master.addons.chain[:]:
                 if isinstance(a, Script) and a.name not in options.scripts:
-                        ctx.log.info("Un-loading script: %s" % a.name)
-                        ctx.master.addons.remove(a)
+                    ctx.log.info("Un-loading script: %s" % a.name)
+                    ctx.master.addons.remove(a)
 
             # The machinations below are to ensure that:
             #   - Scripts remain in the same order

mitmproxy/addons/serverplayback.py

@@ -1,12 +1,14 @@
-from __future__ import absolute_import, print_function, division
-from six.moves import urllib
 import hashlib
+import urllib
+from typing import Any  # noqa
+from typing import List  # noqa
 
-from netlib import strutils
-from mitmproxy import exceptions, flow, ctx
+from mitmproxy import ctx
+from mitmproxy import exceptions
+from mitmproxy import io
 
 
-class ServerPlayback(object):
+class ServerPlayback:
     def __init__(self):
         self.options = None
 
@@ -35,17 +37,20 @@ class ServerPlayback(object):
         _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
         queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
 
-        key = [str(r.port), str(r.scheme), str(r.method), str(path)]
+        key = [str(r.port), str(r.scheme), str(r.method), str(path)]  # type: List[Any]
         if not self.options.server_replay_ignore_content:
-            form_contents = r.urlencoded_form or r.multipart_form
-            if self.options.server_replay_ignore_payload_params and form_contents:
-                params = [
-                    strutils.always_bytes(i)
-                    for i in self.options.server_replay_ignore_payload_params
-                ]
-                for p in form_contents.items(multi=True):
-                    if p[0] not in params:
-                        key.append(p)
+            if self.options.server_replay_ignore_payload_params and r.multipart_form:
+                key.extend(
+                    (k, v)
+                    for k, v in r.multipart_form.items(multi=True)
+                    if k.decode(errors="replace") not in self.options.server_replay_ignore_payload_params
+                )
+            elif self.options.server_replay_ignore_payload_params and r.urlencoded_form:
+                key.extend(
+                    (k, v)
+                    for k, v in r.urlencoded_form.items(multi=True)
+                    if k not in self.options.server_replay_ignore_payload_params
+                )
             else:
                 key.append(str(r.raw_content))
 
@@ -92,7 +97,7 @@ class ServerPlayback(object):
             self.clear()
             if options.server_replay:
                 try:
-                    flows = flow.read_flows_from_paths(options.server_replay)
+                    flows = io.read_flows_from_paths(options.server_replay)
                 except exceptions.FlowReadException as e:
                     raise exceptions.OptionsError(str(e))
                 self.load(flows)

mitmproxy/addons/setheaders.py

@@ -14,13 +14,15 @@ class SetHeaders:
             header: Header name.
             value: Header value string
         """
-        for fpatt, header, value in options.setheaders:
-            flt = flowfilter.parse(fpatt)
-            if not flt:
-                raise exceptions.OptionsError(
-                    "Invalid setheader filter pattern %s" % fpatt
-                )
-            self.lst.append((fpatt, header, value, flt))
+        if "setheaders" in updated:
+            self.lst = []
+            for fpatt, header, value in options.setheaders:
+                flt = flowfilter.parse(fpatt)
+                if not flt:
+                    raise exceptions.OptionsError(
+                        "Invalid setheader filter pattern %s" % fpatt
+                    )
+                self.lst.append((fpatt, header, value, flt))
 
     def run(self, f, hdrs):
         for _, header, value, flt in self.lst:

mitmproxy/addons/stickyauth.py

@@ -0,0 +1,29 @@
+from mitmproxy import exceptions
+from mitmproxy import flowfilter
+
+
+class StickyAuth:
+    def __init__(self):
+        self.flt = None
+        self.hosts = {}
+
+    def configure(self, options, updated):
+        if "stickyauth" in updated:
+            if options.stickyauth:
+                flt = flowfilter.parse(options.stickyauth)
+                if not flt:
+                    raise exceptions.OptionsError(
+                        "stickyauth: invalid filter expression: %s" % options.stickyauth
+                    )
+                self.flt = flt
+            else:
+                self.flt = None
+
+    def request(self, flow):
+        if self.flt:
+            host = flow.request.host
+            if "authorization" in flow.request.headers:
+                self.hosts[host] = flow.request.headers["authorization"]
+            elif flowfilter.match(self.flt, flow):
+                if host in self.hosts:
+                    flow.request.headers["authorization"] = self.hosts[host]