0.18.3: warn about Python 2 usage

Exempt localhost from upstream binding introduced in #879

.appveyor.yml

@@ -7,9 +7,8 @@ environment:
   matrix:
     - PYTHON: "C:\\Python35"
       TOXENV: "py35"
-    # TODO: ENABLE WHEN AVAILABLE
-    # - PYTHON: "C:\\Python36"
-    #   TOXENV: "py36"
+    - PYTHON: "C:\\Python27"
+      TOXENV: "py27"
 
   SNAPSHOT_HOST:
     secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
@@ -19,8 +18,6 @@ environment:
     secure: 6yBwmO5gv4vAwoFYII8qjQ==
   SNAPSHOT_PASS:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
-  RTOOL_KEY:
-    secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -28,48 +25,21 @@ install:
   - "pip install -U tox"
 
 test_script:
-  - ps: "tox -- --verbose --cov-report=term"
-  - ps: |
-      $Env:VERSION = $(python mitmproxy/version.py)
-      $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
-      tox -e wheel
-      tox -e rtool -- bdist
-
-  - ps: |
-      if(
-        ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
-        (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-        tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
-        if (!(Test-Path "C:\projects\mitmproxy\release\installbuilder-installer.exe")) {
-          "Download InstallBuilder..."
-          (New-Object System.Net.WebClient).DownloadFile(
-            "https://installbuilder.bitrock.com/installbuilder-enterprise-17.1.0-windows-installer.exe",
-            "C:\projects\mitmproxy\release\installbuilder-installer.exe"
-          )
-        }
-        Start-Process "C:\projects\mitmproxy\release\installbuilder-installer.exe" "--mode unattended --unattendedmodeui none" -Wait
-        & 'C:\Program Files (x86)\BitRock InstallBuilder Enterprise 17.1.0\bin\builder-cli.exe' `
-          build `
-          .\release\installbuilder\mitmproxy.xml `
-          windows `
-          --license .\release\installbuilder\license.xml `
-          --setvars project.version=$Env:VERSION `
-          --verbose
-      }
+  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v"
 
 deploy_script:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
   ps: |
     if(
       ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-    ) {
-      tox -e rtool -- upload-snapshot --bdist --wheel --installer
+      (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+      ) {
+      pip install -U virtualenv
+      .\dev.ps1
+      cmd /c "python -u .\release\rtool.py bdist 2>&1"
+      python -u .\release\rtool.py upload-snapshot --bdist --wheel
     }
 
 cache:
-  - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml
   - C:\Users\appveyor\AppData\Local\pip\cache
 
 notifications:

.env

@@ -0,0 +1,6 @@
+DIR="$( dirname "${BASH_SOURCE[0]}" )"
+ACTIVATE_DIR="$(if [ -f "$DIR/venv/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)"
+if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/venv/$ACTIVATE_DIR/activate" ]; then
+    echo "Activating mitmproxy virtualenv..."
+    source "$DIR/venv/$ACTIVATE_DIR/activate"
+fi

.gitattributes

@@ -1,2 +1,2 @@
-mitmproxy/tools/web/static/**/* -diff linguist-vendored
+mitmproxy/web/static/**/* -diff
 web/src/js/filt/filt.js -diff

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 MANIFEST
-**/tmp
+*/tmp
 /venv*
 *.py[cdo]
 *.swp
@@ -19,5 +19,3 @@ bower_components
 *.map
 sslkeylogfile.log
 .tox/
-.python-version
-coverage.xml

.travis.yml

@@ -1,12 +1,19 @@
 sudo: false
 language: python
 
+addons:
+  apt:
+    sources:
+      # Debian sid currently holds OpenSSL 1.0.2
+      # change this with future releases!
+      - debian-sid
+    packages:
+      - libssl-dev
+
 env:
   global:
     - CI_DEPS=codecov>=2.0.5
     - CI_COMMANDS=codecov
-git:
-  depth: 10000
 
 matrix:
   fast_finish: true
@@ -18,42 +25,26 @@ matrix:
       language: generic
       env: TOXENV=py35 BDIST=1
     - python: 3.5
-      env: TOXENV=py35 OPENSSL_OLD
-      addons:
-        apt:
-          packages:
-            - libssl-dev
+      env: TOXENV=py35 BDIST=1
     - python: 3.5
-      env: TOXENV=py35 BDIST=1 OPENSSL_ALPN
-      addons:
-        apt:
-          sources:
-            # Debian sid currently holds OpenSSL 1.1.0
-            # change this with future releases!
-            - debian-sid
-          packages:
-            - libssl-dev
-    - python: 3.6
-      env: TOXENV=py36 OPENSSL_ALPN
-      addons:
-        apt:
-          sources:
-            # Debian sid currently holds OpenSSL 1.1.0
-            # change this with future releases!
-            - debian-sid
-          packages:
-            - libssl-dev
-    - python: 3.5
-      env: TOXENV=individual_coverage
+      env: TOXENV=py35 NO_ALPN=1
+    - python: 2.7
+      env: TOXENV=py27
+    - python: 2.7
+      env: TOXENV=py27 NO_ALPN=1
     - python: 3.5
       env: TOXENV=docs
+  allow_failures:
+    - python: pypy
 
 install:
   - |
     if [[ $TRAVIS_OS_NAME == "osx" ]]
     then
-      brew update || brew update
-      brew outdated pyenv || brew upgrade pyenv
+      brew update || brew update # try again if it fails
+      brew upgrade
+      brew reinstall openssl
+      brew reinstall pyenv
       eval "$(pyenv init -)"
       env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.2
       pyenv global 3.5.2
@@ -62,21 +53,18 @@ install:
     fi
   - pip install tox
 
-script:
-  - tox -- --verbose --cov-report=term
-  - |
-    if [[ $BDIST == "1" ]]
-    then
-        git fetch --unshallow --tags
-        tox -e rtool -- bdist
-    fi
+script: tox -- --cov netlib --cov mitmproxy --cov pathod -v
 
 after_success:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
-        tox -e rtool -- upload-snapshot --bdist
+        git fetch --unshallow
+        ./dev.sh 3.5
+        source venv3.5/bin/activate
+        pip install -e ./release
+        python -u ./release/rtool.py bdist
+        python -u ./release/rtool.py upload-snapshot --bdist
     fi
 
 notifications:
@@ -91,4 +79,3 @@ cache:
   directories:
     - $HOME/.pyenv
     - $HOME/.cache/pip
-    # - $HOME/build/mitmproxy/mitmproxy/.tox

CHANGELOG

@@ -1,56 +1,3 @@
-21 February 2017: mitmproxy 2.0
-
-    * HTTP/2 is now enabled by default.
-
-    * Image ContentView: Parse images with Kaitai Struct (kaitai.io) instead of Pillow. 
-      This simplifies installation, reduces binary size, and allows parsing in pure Python.
-
-    * Web: Add missing flow filters.
-
-    * Add transparent proxy support for OpenBSD.
-
-    * Check the mitmproxy CA for expiration and warn the user to regenerate it if necessary.
-
-    * Testing: Tremendous improvements, enforced 100% coverage for large parts of the 
-      codebase, increased overall coverage.
-
-    * Enforce individual coverage: one source file -> one test file with 100% coverage.
-
-    * A myriad of other small improvements throughout the project.
-
-    * Numerous bugfixes.
-
-
-26 December 2016: mitmproxy 1.0
-
-    * All mitmproxy tools are now Python 3 only! We plan to support Python 3.5 and higher.
-
-    * Web-Based User Interface: Mitmproxy now offically has a web-based user interface
-      called mitmweb. We consider it stable for all features currently exposed 
-      in the UI, but it still misses a lot of mitmproxy’s options.
-      
-    * Windows Compatibility: With mitmweb, mitmproxy is now useable on Windows. 
-      We are also introducing an installer (kindly sponsored by BitRock) that 
-      simplifies setup.
-      
-    * Configuration: The config file format is now a single YAML file. In most cases,
-      converting to the new format should be trivial - please see the docs for
-      more information.
-
-    * Console: Significant UI improvements - including sorting of flows by
-      size, type and url, status bar improvements, much faster indentation for
-      HTTP views, and more.
-
-    * HTTP/2: Significant improvements, but is temporarily disabled by default
-      due to wide-spread protocol implementation errors on some large website
-
-    * WebSocket: The protocol implementation is now mature, and is enabled by
-      default. Complete UI support is coming in the next release. Hooks for 
-      message interception and manipulation are available.
-
-    * A myriad of other small improvements throughout the project.
-
-
 16 October 2016: mitmproxy 0.18
 
     * Python 3 Compatibility for mitmproxy and pathod (Shadab Zafar, GSoC 2016)

CONTRIBUTORS

@@ -1,6 +1,6 @@
-  2407	Aldo Cortesi
-  1873	Maximilian Hils
-   556	Thomas Kriechbaumer
+  2184	Aldo Cortesi
+  1745	Maximilian Hils
+   507	Thomas Kriechbaumer
    258	Shadab Zafar
     97	Jason
     83	Marcelo Glezer
@@ -11,91 +11,85 @@
     14	Pedro Worcel
     14	David Weinstein
     13	Thomas Roth
-    11	Jake Drahos
     11	Stephen Altamirano
+    11	Jake Drahos
     11	arjun23496
     11	Justus Wingert
-    10	András Veres-Szentkirályi
-    10	Zohar Lorberbaum
-    10	smill
-    10	Chris Czub
     10	Sandor Nemes
-    10	Doug Freed
+    10	Zohar Lorberbaum
+    10	András Veres-Szentkirályi
+    10	Chris Czub
+    10	smill
      9	ikoz
+     9	Kyle Morton
      9	Legend Tang
      9	Rouli
-     9	Kyle Morton
      8	Jason A. Novak
      8	Chandler Abraham
+     7	Alexis Hildebrandt
      7	Matthias Urlichs
      7	Brad Peabody
      7	dufferzafar
-     7	Alexis Hildebrandt
      6	Felix Yan
-     5	Will Coster
-     5	Sam Cleveland
-     5	iroiro123
-     5	elitest
      5	Tomaz Muraus
+     5	elitest
+     5	iroiro123
+     5	Sam Cleveland
      5	Choongwoo Han
+     5	Will Coster
+     4	root
+     4	Clemens Brunner
      4	Schamper
+     4	Valtteri Virtanen
+     4	Wade 524
      4	Youhei Sakurai
      4	Bryan Bishop
-     4	root
-     4	Valtteri Virtanen
-     4	Clemens Brunner
      4	Marc Liyanage
-     4	Wade 524
-     4	chhsiao90
-     4	yonder
      4	Michael J. Bazzinotti
-     3	Ryan Welton
-     3	Ryan Laughlin
-     3	Kyle Manna
+     4	yonder
      3	Eli Shvartsman
-     3	Vincent Haupert
-     3	Manish Kumar
-     3	Zack B
+     3	Chris Neasbitt
+     3	Guillem Anguera
      3	MatthewShao
+     3	Ryan Welton
+     3	smill@cuckoo.sh
+     3	Manish Kumar
+     3	Benjamin Lee
+     3	Ryan Laughlin
+     3	Zack B
+     3	Kyle Manna
      3	redfast00
      3	requires.io
-     3	Guillem Anguera
-     3	smill@cuckoo.sh
-     3	Chris Neasbitt
-     3	Benjamin Lee
-     2	Steven Van Acker
-     2	Slobodan Mišković
-     2	Jim Lloyd
      2	isra17
      2	israel
-     2	Sean Coates
-     2	Sachin Kelkar
+     2	Colin Bendell
      2	jpkrause
-     2	Bennett Blodinger
+     2	Paul
      2	lilydjwg
      2	Michael Frister
-     2	Israel Nir
-     2	Cory Benfield
-     2	phackt
-     2	Anant
-     2	Jaime Soriano Pastor
-     2	Paul
-     2	Colin Bendell
      2	依云
-     2	Heikki Hannikainen
-     2	Rob Wills
-     2	Niko Kommenda
-     2	Naveen Pai
-     2	strohu
-     2	alts
-     2	Yoginski
-     2	Mark E. Haase
-     2	Wade Catron
-     2	Terry Long
-     2	Krzysztof Bielicki
+     2	Jaime Soriano Pastor
      2	Nick Badger
-     1	Nicolas Esteves
-     1	Andrew Orr
+     2	Rob Wills
+     2	Heikki Hannikainen
+     2	Vincent Haupert
+     2	strohu
+     2	Wade Catron
+     2	Krzysztof Bielicki
+     2	Sachin Kelkar
+     2	Israel Nir
+     2	Anant
+     2	alts
+     2	Doug Freed
+     2	Niko Kommenda
+     2	Terry Long
+     2	Mark E. Haase
+     2	Steven Van Acker
+     2	Jim Lloyd
+     2	Bennett Blodinger
+     2	Sean Coates
+     2	Cory Benfield
+     1	Sergey Chipiga
      1	Andrey Plotnikov
      1	Andy Smith
      1	Angelo Agatino Nicolosi
@@ -103,7 +97,6 @@
      1	BSalita
      1	Ben Lerner
      1	Bradley Baetz
-     1	Brady Law
      1	Brett Randall
      1	Chris Hamant
      1	Christian Frichot
@@ -112,7 +105,6 @@
      1	David Shaw
      1	Doug Lethin
      1	Drake Caraker
-     1	Edgar Boda-Majer
      1	Eric Entzel
      1	Felix Wolfsteller
      1	FreeArtMan
@@ -136,25 +128,22 @@
      1	Mathieu Mitchell
      1	Michael Bisbjerg
      1	Mike C
-     1	Mike Fotinakis
      1	Mikhail Korobov
      1	Morton Fox
      1	Nick HS
      1	Nick Raptis
-     1	Aditya
+     1	Nicolas Esteves
      1	Oleksandr Sheremet
      1	Parth Ganatra
      1	Pritam Baral
-     1	Quentin Pradet
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
      1	Ryo Onodera
-     1	Sahil Chelaramani
      1	Sahn Lam
      1	Sanchit Sokhey
      1	Seppo Yli-Olli
-     1	Sergey Chipiga
+     1	Aditya
      1	Stefan Wärting
      1	Steve Phillips
      1	Steven Noble
@@ -169,8 +158,10 @@
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
      1	Wes Turner
+     1	Yoginski
      1	Yuangxuan Wang
      1	capt8bit
+     1	chhsiao90
      1	cle1000
      1	davidpshaw
      1	deployable
@@ -181,6 +172,7 @@
      1	meeee
      1	michaeljau
      1	peralta
+     1	phackt
      1	phil plante
      1	sentient07
      1	sethp-jive

MANIFEST.in

@@ -1,3 +1,4 @@
 graft mitmproxy
 graft pathod
-recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
+graft netlib
+recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

README.rst

@@ -3,15 +3,14 @@ mitmproxy
 
 |travis| |appveyor| |coverage| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects.
+This repository contains the **mitmproxy** and **pathod** projects, as well as
+their shared networking library, **netlib**.
 
 ``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
 interface.
 
 ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
-``mitmweb`` is a web-based interface for mitmproxy.
-
 ``pathoc`` and ``pathod`` are perverse HTTP client and server applications
 designed to let you craft almost any conceivable HTTP request, including ones
 that creatively violate the standards.
@@ -24,7 +23,8 @@ Documentation & Help
 General information, tutorials, and precompiled binaries can be found on the mitmproxy
 and pathod websites.
 
-|mitmproxy_site|
+|mitmproxy_site| |pathod_site|
+
 
 The latest documentation for mitmproxy is also available on ReadTheDocs.
 
@@ -37,7 +37,7 @@ each other solve problems, and come up with new ideas for the project.
 |mitmproxy_discourse|
 
 
-Join our developer chat on Slack if you would like to contribute to mitmproxy itself.
+Join our developer chat on Slack if you would like to hack on mitmproxy itself.
 
 |slack|
 
@@ -45,67 +45,79 @@ Join our developer chat on Slack if you would like to contribute to mitmproxy it
 Installation
 ------------
 
-The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`__.
+The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`_.
 If you want to contribute changes, keep on reading.
 
-Contributing
-------------
 
-As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project forward,
-please consider contributing in the following areas:
+Hacking
+-------
 
-- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users in our forums_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported.
-- **Code Contributions:** We actively mark issues that we consider are `good first contributions`_. If you intend to work on a larger contribution to the project, please come talk to us first.
-
-Development Setup
------------------
-
-To get started hacking on mitmproxy, please follow the `advanced installation`_ steps to install mitmproxy from source, but stop right before running ``pip3 install mitmproxy``. Instead, do the following:
+To get started hacking on mitmproxy, make sure you have Python_ 3.5.x or above with
+virtualenv_ installed (you can find installation instructions for virtualenv
+`here <http://virtualenv.readthedocs.org/en/latest/>`_). Then do the following:
 
 .. code-block:: text
 
     git clone https://github.com/mitmproxy/mitmproxy.git
     cd mitmproxy
-    ./dev.sh  # "powershell .\dev.ps1" on Windows
+    ./dev.sh  # powershell .\dev.ps1 on Windows
 
 
-The *dev* script will create a `virtualenv`_ environment in a directory called "venv"
-and install all mandatory and optional dependencies into it. The primary
-mitmproxy components - mitmproxy and pathod - are installed as
+The *dev* script will create a virtualenv environment in a directory called
+"venv", and install all mandatory and optional dependencies into it. The
+primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
 "editable", so any changes to the source in the repository will be reflected
 live in the virtualenv.
 
-The main executables for the project - ``mitmdump``, ``mitmproxy``,
+To confirm that you're up and running, activate the virtualenv, and run the
+mitmproxy test suite:
+
+.. code-block:: text
+
+    . venv/bin/activate  # venv\Scripts\activate on Windows
+    py.test
+
+Note that the main executables for the project - ``mitmdump``, ``mitmproxy``,
 ``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the
 virtualenv. After activating the virtualenv, they will be on your $PATH, and
 you can run them like any other command:
 
 .. code-block:: text
 
-    . venv/bin/activate  # "venv\Scripts\activate" on Windows
     mitmdump --version
 
+For convenience, the project includes an autoenv_ file (`.env`_) that
+auto-activates the virtualenv when you cd into the mitmproxy directory.
+
+
 Testing
 -------
 
 If you've followed the procedure above, you already have all the development
-requirements installed, and you can run the full test suite (including tests for code style and documentation) with tox_:
+requirements installed, and you can simply run the test suite:
 
 .. code-block:: text
 
-    tox
-
-For speedier testing, we recommend you run `pytest`_ directly on individual test files or folders:
-
-.. code-block:: text
-
-    cd test/mitmproxy/addons
-    pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py
-
-As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes.
+    py.test
 
 Please ensure that all patches are accompanied by matching changes in the test
-suite. The project tries to maintain 100% test coverage and enforces this strictly for some parts of the codebase.
+suite. The project tries to maintain 100% test coverage.
+
+You can also use `tox` to run a full suite of tests in Python 2.7 and 3.5,
+including a quick test to check documentation and code linting.
+
+The following tox environments are relevant for local testing:
+
+.. code-block:: text
+
+    tox -e py27  # runs all tests with Python 2.7
+    tox -e py35  # runs all tests with Python 3.5
+    tox -e docs  # runs a does-it-compile check on the documentation
+    tox -e lint  # runs the linter for coding style checks
+
+We support Python 2.7 and 3.5, so please make sure all tests pass in both
+environments. Running `tox` ensures all necessary tests are executed.
+
 
 Documentation
 -------------
@@ -124,8 +136,8 @@ installation, you can render the documentation like this:
 The last command invokes `sphinx-autobuild`_, which watches the Sphinx directory and rebuilds
 the documentation when a change is detected.
 
-Code Style
-----------
+Style
+-----
 
 Keeping to a consistent code style throughout the project makes it easier to
 contribute and collaborate. Please stick to the guidelines in
@@ -133,19 +145,22 @@ contribute and collaborate. Please stick to the guidelines in
 good reason not to.
 
 This is automatically enforced on every PR. If we detect a linting error, the
-PR checks will fail and block merging. You can run our lint checks yourself
-with the following command:
+PR checks will fail and block merging. We are using this command to check for style compliance:
 
 .. code-block:: text
 
-    tox -e lint
+    flake8 --jobs 8 --count mitmproxy netlib pathod examples test
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg
     :target: https://mitmproxy.org/
     :alt: mitmproxy.org
 
-.. |mitmproxy_docs| image:: https://shields.mitmproxy.org/api/docs-latest-brightgreen.svg
+.. |pathod_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-pathod.net-blue.svg
+    :target: https://pathod.net/
+    :alt: pathod.net
+
+.. |mitmproxy_docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
     :target: http://docs.mitmproxy.org/en/latest/
     :alt: mitmproxy documentation
 
@@ -157,15 +172,15 @@ with the following command:
     :target: http://slack.mitmproxy.org/
     :alt: Slack Developer Chat
 
-.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=travis%20ci
+.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=Travis%20build
     :target: https://travis-ci.org/mitmproxy/mitmproxy
     :alt: Travis Build Status
 
-.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mhils/mitmproxy/master.svg?label=appveyor%20ci
+.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mhils/mitmproxy/master.svg?label=Appveyor%20build
     :target: https://ci.appveyor.com/project/mhils/mitmproxy
     :alt: Appveyor Build Status
 
-.. |coverage| image:: https://shields.mitmproxy.org/codecov/c/github/mitmproxy/mitmproxy/master.svg?label=codecov
+.. |coverage| image:: https://codecov.io/gh/mitmproxy/mitmproxy/branch/master/graph/badge.svg
     :target: https://codecov.io/gh/mitmproxy/mitmproxy
     :alt: Coverage Status
 
@@ -177,13 +192,12 @@ with the following command:
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Supported Python versions
 
-.. _`advanced installation`: http://docs.mitmproxy.org/en/latest/install.html#advanced-installation
-.. _virtualenv: https://virtualenv.pypa.io/
-.. _`pytest`: http://pytest.org/
-.. _tox: https://tox.readthedocs.io/
+.. _Python: https://www.python.org/
+.. _virtualenv: http://virtualenv.readthedocs.org/en/latest/
+.. _autoenv: https://github.com/kennethreitz/autoenv
+.. _.env: https://github.com/mitmproxy/mitmproxy/blob/master/.env
 .. _Sphinx: http://sphinx-doc.org/
 .. _sphinx-autobuild: https://pypi.python.org/pypi/sphinx-autobuild
+.. _issue_tracker: https://github.com/mitmproxy/mitmproxy/issues
 .. _PEP8: https://www.python.org/dev/peps/pep-0008
-.. _`Google Style Guide`: https://google.github.io/styleguide/pyguide.html
-.. _forums: https://discourse.mitmproxy.org/
-.. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3Agood-first-contribution
+.. _Google Style Guide: https://google.github.io/styleguide/pyguide.html

dev.ps1

@@ -1,19 +1,15 @@
 $ErrorActionPreference = "Stop"
+$VENV = ".\venv"
 
-$pyver = python --version
-if($pyver -notmatch "3\.[5-9]") {
-    Write-Warning "Unexpected Python version, expected Python 3.5 or above: $pyver"
-}
-
-python -m venv .\venv --copies
-& .\venv\Scripts\activate.ps1
+virtualenv $VENV --always-copy
+& $VENV\Scripts\activate.ps1
 
 python -m pip install --disable-pip-version-check -U pip
 cmd /c "pip install -r requirements.txt 2>&1"
 
 echo @"
 
-  * Created virtualenv environment in .\venv.
+  * Created virtualenv environment in $VENV.
   * Installed all dependencies into the virtualenv.
   * Activated virtualenv environment.
 

dev.sh

@@ -2,14 +2,16 @@
 set -e
 set -x
 
-echo "Creating dev environment in ./venv..."
+PYVERSION=$1
+VENV="venv$1"
 
-python3 -m venv venv
-. venv/bin/activate
-pip3 install -U pip setuptools
-pip3 install -r requirements.txt
+echo "Creating dev environment in $VENV using Python $PYVERSION"
+
+python$PYVERSION -m virtualenv "$VENV" --always-copy
+. "$VENV/bin/activate"
+pip$PYVERSION install -U pip setuptools
+pip$PYVERSION install -r requirements.txt
 
 echo ""
-echo "  * Created virtualenv environment in ./venv."
-echo "  * Installed all dependencies into the virtualenv."
-echo "  * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`"
+echo "* Virtualenv created in $VENV and all dependencies installed."
+echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV/bin/activate\`"

docs/certinstall.rst

@@ -40,9 +40,7 @@ start of mitmproxy.
 iOS
 ^^^
 
-See http://jasdev.me/intercepting-ios-traffic
-
-and http://web.archive.org/web/20150920082614/http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
+http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
 
 iOS Simulator
 ^^^^^^^^^^^^^

docs/conf.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 
 sys.path.insert(0, os.path.abspath('..'))
-from mitmproxy import version as mversion
+import netlib.version
 
 
 extensions = [
@@ -47,9 +47,9 @@ author = u'The mitmproxy project'
 # built documents.
 #
 # The short X.Y version.
-version = mversion.VERSION
+version = netlib.version.VERSION
 # The full version, including alpha/beta/rc tags.
-release = mversion.VERSION
+release = netlib.version.VERSION
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -231,7 +231,10 @@ def linkcode_resolve(domain, info):
         _, line = inspect.getsourcelines(obj)
     except (TypeError, IOError):
         return None
-    if spath.rfind("mitmproxy") > -1:
+    if spath.rfind("netlib") > -1:
+        off = spath.rfind("netlib")
+        mpath = spath[off:]
+    elif spath.rfind("mitmproxy") > -1:
         off = spath.rfind("mitmproxy")
         mpath = spath[off:]
     else:

docs/config.rst

@@ -3,11 +3,84 @@
 Configuration
 =============
 
-Mitmproxy is configured with a YAML_ file, located at
-``~/.mitmproxy/config.yaml``. We'll have complete documentation for all
-supported options in the next release in the meantime, please consult the
-source_ for a complete list of options and types.
+Mitmproxy is configured through a set of files in the users ~/.mitmproxy
+directory.
 
+mitmproxy.conf
+    Settings for the :program:`mitmproxy`. This file can contain any options supported by
+    mitmproxy.
 
-.. _YAML: http://www.yaml.org/start.html
-.. _source: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/options.py
+mitmdump.conf
+    Settings for the :program:`mitmdump`. This file can contain any options supported by mitmdump.
+
+common.conf
+    Settings shared between all command-line tools. Settings in this file are over-ridden by those
+    in the tool-specific files. Only options shared by mitmproxy and mitmdump should be used in
+    this file.
+
+Syntax
+------
+
+Comments
+^^^^^^^^
+
+.. code-block:: none
+
+    # this is a comment
+    ; this is also a comment (.ini style)
+    --- and this is a comment too (yaml style)
+
+Key/Value pairs
+^^^^^^^^^^^^^^^
+
+- Keys and values are case-sensitive
+- Whitespace is ignored
+- Lists are comma-delimited, and enclosed in square brackets
+
+.. code-block:: none
+
+    name = value   # (.ini style)
+    name: value    # (yaml style)
+    --name value   # (command-line option style)
+
+    fruit = [apple, orange, lemon]
+    indexes = [1, 12, 35 , 40]
+
+Flags
+^^^^^
+
+These are boolean options that take no value but true/false.
+
+.. code-block:: none
+
+    name = true    # (.ini style)
+    name
+    --name 	 	   # (command-line option style)
+
+Options
+-------
+
+The options available in the config files are precisely those available as
+command-line flags, with the key being the option's long name. To get a
+complete list of these, use the ``--help`` option on each of the tools. Be
+careful to only specify common options in the **common.conf** file -
+unsupported options in this file will be detected as an error on startup.
+
+Examples
+--------
+
+common.conf
+^^^^^^^^^^^
+
+Note that ``--port`` is an option supported by all tools.
+
+.. code-block:: none
+
+    port = 8080
+
+mitmproxy.conf
+^^^^^^^^^^^^^^
+
+.. code-block:: none
+
+    palette = light

docs/dev/architecture.rst

@@ -0,0 +1,14 @@
+.. _architecture:
+
+Architecture
+============
+
+To give you a better understanding of how mitmproxy works, mitmproxy's
+high-level architecture is detailed in the following graphic:
+
+.. image:: ../schematics/architecture.png
+
+:download:`architecture.pdf <../schematics/architecture.pdf>`
+
+Please don't refrain from asking any further
+questions on the mailing list, the Slack channel or the GitHub issue tracker.

docs/dev/contributing.rst

@@ -1,11 +0,0 @@
-.. _contributing:
-
-Contributing
-============
-
-As an open source project, **mitmproxy** welcomes contributions of all forms. 
-
-Please head over to the README_ to get started! 😃
-
-
-.. _README: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst

docs/dev/testing.rst

@@ -0,0 +1,47 @@
+.. _testing:
+
+Testing
+=======
+
+All the mitmproxy projects strive to maintain 100% code coverage. In general,
+patches and pull requests will be declined unless they're accompanied by a
+suitable extension to the test suite.
+
+Our tests are written for the `py.test`_ or nose_ test frameworks.
+At the point where you send your pull request, a command like this:
+
+>>> py.test --cov mitmproxy --cov netlib
+
+Should give output something like this:
+
+.. code-block:: none
+
+    > ---------- coverage: platform darwin, python 2.7.2-final-0 --
+    > Name                   Stmts   Miss  Cover   Missing
+    > ----------------------------------------------------
+    > mitmproxy/__init__         0      0   100%
+    > mitmproxy/app              4      0   100%
+    > mitmproxy/cmdline        100      0   100%
+    > mitmproxy/controller      69      0   100%
+    > mitmproxy/dump           150      0   100%
+    > mitmproxy/encoding        39      0   100%
+    > mitmproxy/flowfilter     201      0   100%
+    > mitmproxy/flow           891      0   100%
+    > mitmproxy/proxy          427      0   100%
+    > mitmproxy/script          27      0   100%
+    > mitmproxy/utils          133      0   100%
+    > mitmproxy/version          4      0   100%
+    > ----------------------------------------------------
+    > TOTAL                   2045      0   100%
+    > ----------------------------------------------------
+    > Ran 251 tests in 11.864s
+
+
+There are exceptions to the coverage requirement - for instance, much of the
+console interface code can't sensibly be unit tested. These portions are
+excluded from coverage analysis either in the **.coveragerc** file, or using
+**#pragma no-cover** directives. To keep our coverage analysis relevant, we use
+these measures as sparingly as possible.
+
+.. _nose: https://nose.readthedocs.org/en/latest/
+.. _py.test: https://pytest.org/

docs/features/responsestreaming.rst

@@ -40,8 +40,8 @@ You can also use a script to customize exactly which responses are streamed.
 Responses that should be tagged for streaming by setting their ``.stream``
 attribute to ``True``:
 
-.. literalinclude:: ../../examples/complex/stream.py
-   :caption: examples/complex/stream.py
+.. literalinclude:: ../../examples/stream.py
+   :caption: examples/stream.py
    :language: python
 
 Implementation Details
@@ -59,8 +59,8 @@ Modifying streamed data
 If the ``.stream`` attribute is callable, ``.stream`` will wrap the generator that yields all
 chunks.
 
-.. literalinclude:: ../../examples/complex/stream_modify.py
-   :caption: examples/complex/stream_modify.py
+.. literalinclude:: ../../examples/stream_modify.py
+   :caption: examples/stream_modify.py
    :language: python
 
 .. seealso::

docs/features/serverreplay.rst

@@ -33,19 +33,6 @@ updated in a similar way.
 You can turn off response refreshing using the ``--norefresh`` argument, or using
 the :kbd:`o` options shortcut within :program:`mitmproxy`.
 
-
-Replaying a session recorded in Reverse-proxy Mode
---------------------------------------------------
-
-If you have captured the session in reverse proxy mode, in order to replay it you 
-still have to specify the server URL, otherwise you may get the error: 
-'HTTP protocol error in client request: Invalid HTTP request form (expected authority or absolute...)'.
-
-During replay, when the client's requests match previously recorded requests, then the
-respective recorded responses are simply replayed  by mitmproxy. 
-Otherwise, the unmatched requests is forwarded to the upstream server.
-If forwarding is not desired, you can use the --kill (-k) switch to prevent that.
-
 ================== ===========
 command-line       ``-S path``
 mitmproxy shortcut :kbd:`R` then :kbd:`s`

docs/features/tcpproxy.rst

@@ -3,7 +3,7 @@
 TCP Proxy
 =========
 
-In case mitmproxy does not handle a specific protocol, you can exempt
+WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt
 hostnames from processing, so that mitmproxy acts as a generic TCP forwarder.
 This feature is closely related to the :ref:`passthrough` functionality,
 but differs in two important aspects:

docs/index.rst

@@ -17,7 +17,6 @@
 
     mitmproxy
     mitmdump
-    mitmweb
     config
 
 .. toctree::
@@ -47,7 +46,6 @@
     transparent
     transparent/linux
     transparent/osx
-    transparent/openbsd
 
 .. toctree::
     :hidden:
@@ -79,9 +77,10 @@
 
 .. toctree::
     :hidden:
-    :caption: Development
+    :caption: Hacking
 
-    dev/contributing
+    dev/architecture
+    dev/testing
     dev/sslkeylogfile
 
 ..  Indices and tables

docs/install.rst

@@ -3,150 +3,130 @@
 Installation
 ============
 
-Please follow the steps for your operating system.
+.. _install-ubuntu:
 
-Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or
-:ref:`mitmweb` from a terminal.
+Installation On Ubuntu
+----------------------
 
-
-.. _install-macos:
-
-Installation on macOS
----------------------
-
-You can use Homebrew to install everything:
+Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
+This was tested on a fully patched installation of Ubuntu 14.04.
 
 .. code:: bash
 
-    brew install mitmproxy
+   sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev g++
+   sudo pip install mitmproxy  # or pip install --user mitmproxy
 
-Or you can download the pre-built binary packages from our `releases`_.
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
+On **Ubuntu 12.04** (and other systems with an outdated version of pip),
+you may need to update pip using ``pip install -U pip`` before installing mitmproxy.
 
-.. _install-windows:
+Installation From Source (Ubuntu)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Installation on Windows
------------------------
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, install the dependencies as you would for a regular
+mitmproxy installation (see :ref:`install-ubuntu`).
+Then see the Hacking_ section of the README on GitHub.
 
-The recommended way to install mitmproxy on Windows is to use the installer
-provided at `mitmproxy.org`_. After installation, you'll find shortcuts for
-:ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu.
-Both executables are  added to your PATH and can be invoked from the command
-line.
+.. _install-fedora:
 
-.. note::
-    Mitmproxy's console interface is not supported on Windows, but you can use
-    mitmweb (the web-based interface) and mitmdump.
+Installation On Fedora
+----------------------
 
-.. _install-linux:
+Fedora comes with Python but we need to install pip, python-dev and several libraries.
+This was tested on a fully patched installation of Fedora 23.
 
-Installation on Linux
----------------------
+.. code:: bash
 
-The recommended way to run mitmproxy on Linux is to use the pre-built binaries
-provided at `releases`_.
+   sudo dnf install -y python-pip python-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libpng-devel libjpeg-devel
+   sudo pip install mitmproxy  # or pip install --user mitmproxy
 
-Our pre-built binaries provide you with the latest version of mitmproxy, a
-self-contained Python 3.5 environment and a recent version of OpenSSL that
-supports HTTP/2. Of course, you can also install mitmproxy from source if you
-prefer that (see :ref:`install-advanced`).
-
-.. _install-advanced:
-
-Advanced Installation
----------------------
-
-.. _install-docker:
-
-Docker Images
-^^^^^^^^^^^^^
-
-You can also use the official mitmproxy images from `DockerHub`_. That being
-said, our portable binaries are just as easy to install and even easier to use. 😊
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
 
 .. _install-arch:
 
-Installation on Arch Linux
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+Installation On Arch Linux
+--------------------------
 
 mitmproxy has been added into the [community] repository. Use pacman to install it:
 
 >>> sudo pacman -S mitmproxy
 
 
-.. _install-source-ubuntu:
 
-Installation from Source on Ubuntu
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Installation On Mac OS X
+------------------------
 
-Ubuntu comes with Python but we need to install pip3, python3-dev and several
-libraries. This was tested on a fully patched installation of Ubuntu 16.04.
+The easiest way to get up and running on OSX is to download the pre-built binary packages from
+`mitmproxy.org`_.
 
-.. code:: bash
+There are a few bits of customization you might want to do to make mitmproxy comfortable to use on
+OSX. The default color scheme is optimized for a dark background terminal, but you can select a
+palette for a light terminal background with the ``--palette`` option.
+You can use the OSX **open** program to create a simple and effective ``~/.mailcap`` file to view
+request and response bodies:
 
-   sudo apt-get install python3-dev python3-pip libffi-dev libssl-dev
-   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
+.. code-block:: none
 
-On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install
-a newer version of Python. mitmproxy requires Python 3.5 or higher. Please take
-a look at pyenv_. Make sure to have an up-to-date version of pip by running
-``pip3 install -U pip``.
+    application/*; /usr/bin/open -Wn %s
+    audio/*; /usr/bin/open -Wn %s
+    image/*; /usr/bin/open -Wn %s
+    video/*; /usr/bin/open -Wn %s
+
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
 
-.. _install-source-fedora:
+Installation From Source (Mac OS X)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Installation from Source on Fedora
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, there are a few OS X specific things to keep in mind.
 
-Fedora comes with Python but we need to install pip3, python3-dev and several
-libraries. This was tested on a fully patched installation of Fedora 24.
+- Make sure that XCode is installed from the App Store, and that the command-line tools have been
+  downloaded (XCode/Preferences/Downloads).
+- If you're running a Python interpreter installed with homebrew (or similar), you may have to
+  install some dependencies by hand.
 
-.. code:: bash
+Then see the Hacking_ section of the README on GitHub.
 
-   sudo dnf install make gcc redhat-rpm-config python3-devel python3-pip libffi-devel openssl-devel
-   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
-
-Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
-
-
-
-.. _install-source-windows:
-
-🐱💻 Installation from Source on Windows
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Installation On Windows
+-----------------------
 
 .. note::
-    Mitmproxy's console interface is not supported on Windows, but you can use
-    mitmweb (the web-based interface) and mitmdump.
+    Please note that mitmdump is the only component of mitmproxy that is supported on Windows at
+    the moment.
 
-First, install the latest version of Python 3.5 or later from the `Python
-website`_. During installation, make sure to select `Add Python to PATH`.
-
-Mitmproxy has no other dependencies on Windows. You can now install mitmproxy by running
-
-.. code:: powershell
-
-    pip3 install mitmproxy
+    **There is no interactive user interface on Windows.**
 
 
+First, install the latest version of Python 2.7 from the `Python website`_.
+If you already have an older version of Python 2.7 installed, make sure to install pip_
+(pip is included in Python 2.7.9+ by default). If pip aborts with an error, make sure you are using the current version of pip.
 
-.. _install-dev-version:
+>>> python -m pip install --upgrade pip
 
-Latest Development Version
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+Next, add Python and the Python Scripts directory to your **PATH** variable.
+You can do this easily by running the following in powershell:
 
-If you would like to install mitmproxy directly from the master branch on GitHub
-or would like to get set up to contribute to the project, install the
-dependencies as you would for a regular installation from source. Then see the
-project's README_ on GitHub. You can check your system information
-by running: ``mitmproxy --version``
+>>> [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User")
+
+Now, you can install mitmproxy by running
+
+>>> pip install mitmproxy
+
+Once the installation is complete, you can run :ref:`mitmdump` from a command prompt.
+
+Installation From Source (Windows)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, install Python as outlined above, then see the
+Hacking_ section of the README on GitHub.
 
 
-.. _README: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst
-.. _releases: https://github.com/mitmproxy/mitmproxy/releases
+.. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst#hacking
 .. _mitmproxy.org: https://mitmproxy.org/
 .. _`Python website`: https://www.python.org/downloads/windows/
 .. _pip: https://pip.pypa.io/en/latest/installing.html
-.. _pyenv: https://github.com/yyuu/pyenv
-.. _DockerHub: https://hub.docker.com/r/mitmproxy/mitmproxy/

docs/introduction.rst

@@ -6,8 +6,6 @@ with a console interface.
 
 **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
-**mitmweb** is a web-based interface for mitmproxy.
-
 Documentation, tutorials and distribution packages can be found on the
 mitmproxy website: `mitmproxy.org <https://mitmproxy.org/>`_
 

docs/mitmweb.rst

@@ -1,18 +0,0 @@
-.. _mitmweb:
-.. program:: mitmweb
-
-mitmweb
-=======
-
-**mitmweb** is mitmproxy's web-based user interface that allows interactive
-examination and modification of HTTP traffic. Like mitmproxy, it differs from
-mitmdump in that all flows are kept in memory, which means that it's intended
-for taking and manipulating small-ish samples.
-
-.. warning::
-
-  Mitmweb is currently in beta. We consider it stable for all features currently
-  exposed in the UI, but it still misses a lot of mitmproxy's features.
-
-
-.. image:: screenshots/mitmweb.png

docs/modd.conf

@@ -1,6 +1,6 @@
 @build = ./_build
 
-** !_build/** ../mitmproxy/**/*.py {
+** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py {
     prep: sphinx-build -W -d @build/doctrees -b html . @build/html
     daemon: devd -m @build/html
 }

docs/pathod/test.rst

@@ -14,7 +14,7 @@ The canonical docs can be accessed using pydoc:
 >>> pydoc pathod.test
 
 The remainder of this page demonstrates some common interaction patterns using
-`Nose`_. These examples are
+<a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are
 also applicable with only minor modification to most commonly used Python testing
 engines.
 
@@ -33,6 +33,3 @@ One instance per test
 .. literalinclude:: ../../examples/pathod/test_setup.py
    :caption: examples/pathod/test_setup.py
    :language: python
-
-
-.. _Nose: https://nose.readthedocs.org/en/latest/

docs/scripting/api.rst

@@ -5,38 +5,36 @@ API
 ===
 
 - Errors
-    - `mitmproxy.flow.Error <#mitmproxy.flow.Error>`_
+    - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_
 - HTTP
-    - `mitmproxy.http.HTTPRequest <#mitmproxy.http.HTTPRequest>`_
-    - `mitmproxy.http.HTTPResponse <#mitmproxy.http.HTTPResponse>`_
-    - `mitmproxy.http.HTTPFlow <#mitmproxy.http.HTTPFlow>`_
+    - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_
+    - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_
+    - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_
 - Logging
-    - `mitmproxy.log.Log <#mitmproxy.controller.Log>`_
-    - `mitmproxy.log.LogEntry <#mitmproxy.controller.LogEntry>`_
+    - `mitmproxy.controller.Log <#mitmproxy.controller.Log>`_
+    - `mitmproxy.controller.LogEntry <#mitmproxy.controller.LogEntry>`_
 
 
 Errors
 ------
 
-.. autoclass:: mitmproxy.flow.Error
+.. autoclass:: mitmproxy.models.flow.Error
     :inherited-members:
 
 HTTP
 ----
 
-.. autoclass:: mitmproxy.http.HTTPRequest
+.. autoclass:: mitmproxy.models.http.HTTPRequest
     :inherited-members:
 
-.. autoclass:: mitmproxy.http.HTTPResponse
+.. autoclass:: mitmproxy.models.http.HTTPResponse
     :inherited-members:
 
-.. autoclass:: mitmproxy.http.HTTPFlow
+.. autoclass:: mitmproxy.models.http.HTTPFlow
     :inherited-members:
 
 Logging
 --------
 
-.. autoclass:: mitmproxy.log.Log
-    :inherited-members:
-.. autoclass:: mitmproxy.log.LogEntry
+.. autoclass:: mitmproxy.controller.Log
     :inherited-members:

docs/scripting/events.rst

@@ -56,7 +56,7 @@ Connection
           connection can correspond to multiple HTTP requests.
 
           *root_layer*
-            The root layer (see `mitmproxy.proxy.protocol` for an explanation what
+            The root layer (see `mitmproxy.protocol` for an explanation what
             the root layer is), provides transparent access to all attributes
             of the :py:class:`~mitmproxy.proxy.RootContext`. For example,
             ``root_layer.client_conn.address`` gives the remote address of the
@@ -98,18 +98,6 @@ HTTP Events
     :widths: 40 60
     :header-rows: 0
 
-    *   - .. py:function:: http_connect(flow)
-        - Called when we receive an HTTP CONNECT request. Setting a non 2xx
-          response on the flow will return the response to the client abort the
-          connection. CONNECT requests and responses do not generate the usual
-          HTTP handler events. CONNECT requests are only valid in regular and
-          upstream proxy modes.
-
-          *flow*
-            A ``models.HTTPFlow`` object. The flow is guaranteed to have
-            non-None ``request``  and ``requestheaders`` attributes.
-
-
     *   - .. py:function:: request(flow)
         - Called when a client request has been received.
 
@@ -158,54 +146,21 @@ HTTP Events
 WebSocket Events
 -----------------
 
-These events are called only after a connection made an HTTP upgrade with
-"101 Switching Protocols". No further HTTP-related events after the handshake
-are issued, only new WebSocket messages are called.
-
 .. list-table::
     :widths: 40 60
     :header-rows: 0
 
-    *   - .. py:function:: websocket_handshake(flow)
-        - Called when a client wants to establish a WebSocket connection. The
-          WebSocket-specific headers can be manipulated to alter the
+    *   - .. py:function:: websockets_handshake(flow)
+
+        - Called when a client wants to establish a WebSockets connection. The
+          WebSockets-specific headers can be manipulated to manipulate the
           handshake. The ``flow`` object is guaranteed to have a non-None
           ``request`` attribute.
 
           *flow*
-            The flow containing the HTTP WebSocket handshake request. The
+            The flow containing the HTTP websocket handshake request. The
             object is guaranteed to have a non-None ``request`` attribute.
 
-    *   - .. py:function:: websocket_start(flow)
-        - Called when WebSocket connection is established after a successful
-          handshake.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_message(flow)
-
-        - Called when a WebSocket message is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable. Currently there are
-          two types of messages, corresponding to the BINARY and TEXT frame types.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_end(flow)
-        - Called when WebSocket connection ends.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_error(flow)
-        - Called when a WebSocket error occurs - e.g. the connection closing
-          unexpectedly.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
 
 TCP Events
 ----------
@@ -218,22 +173,6 @@ connections.
     :widths: 40 60
     :header-rows: 0
 
-
-    *   - .. py:function:: tcp_start(flow)
-        - Called when TCP streaming starts.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_message(flow)
-
-        - Called when a TCP payload is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
     *   - .. py:function:: tcp_end(flow)
         - Called when TCP streaming ends.
 
@@ -246,3 +185,18 @@ connections.
 
           *flow*
             A ``models.TCPFlow`` object.
+
+    *   - .. py:function:: tcp_message(flow)
+
+        - Called a TCP payload is received from the client or server. The
+          sender and receiver are identifiable. The most recent message will be
+          ``flow.messages[-1]``. The message is user-modifiable.
+
+          *flow*
+            A ``models.TCPFlow`` object.
+
+    *   - .. py:function:: tcp_start(flow)
+        - Called when TCP streaming starts.
+
+          *flow*
+            A ``models.TCPFlow`` object.

docs/scripting/overview.rst

@@ -6,7 +6,7 @@ Overview
 Mitmproxy has a powerful scripting API that allows you to control almost any
 aspect of traffic being proxied. In fact, much of mitmproxy's own core
 functionality is implemented using the exact same API exposed to scripters (see
-:src:`mitmproxy/addons`).
+:src:`mitmproxy/builtins`).
 
 
 A simple example
@@ -17,8 +17,8 @@ appropriate points of mitmproxy's operation. Here's a complete mitmproxy script
 that adds a new header to every HTTP response before it is returned to the
 client:
 
-.. literalinclude:: ../../examples/simple/add_header.py
-   :caption: :src:`examples/simple/add_header.py`
+.. literalinclude:: ../../examples/add_header.py
+   :caption: :src:`examples/add_header.py`
    :language: python
 
 All events that deal with an HTTP request get an instance of `HTTPFlow
@@ -42,8 +42,8 @@ called before anything else happens. You can replace the current script object
 by returning it from this handler. Here's how this looks when applied to the
 example above:
 
-.. literalinclude:: ../../examples/simple/add_header_class.py
-   :caption: :src:`examples/simple/add_header_class.py`
+.. literalinclude:: ../../examples/classes.py
+   :caption: :src:`examples/classes.py`
    :language: python
 
 So here, we're using a module-level script to "boot up" into a class instance.
@@ -62,13 +62,13 @@ sophisticated - replace one value with another in all responses. Mitmproxy's
 <api.html#mitmproxy.models.http.HTTPResponse.replace>`_ method that takes care
 of all the details for us.
 
-.. literalinclude:: ../../examples/simple/script_arguments.py
-   :caption: :src:`examples/simple/script_arguments.py`
+.. literalinclude:: ../../examples/arguments.py
+   :caption: :src:`examples/arguments.py`
    :language: python
 
 We can now call this script on the command-line like this:
 
->>> mitmdump -dd -s "./script_arguments.py html faketml"
+>>> mitmdump -dd -s "./arguments.py html faketml"
 
 Whenever a handler is called, mitpmroxy rewrites the script environment so that
 it sees its own arguments as if it was invoked from the command-line.
@@ -78,15 +78,15 @@ Logging and the context
 -----------------------
 
 Scripts should not output straight to stderr or stdout. Instead, the `log
-<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` context module
+<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` contexzt module
 should be used, so that the mitmproxy host program can handle output
-appropriately. So, mitmdump can print colorised script output to the terminal,
+appropriately. So, mitmdump can print colorised sript output to the terminal,
 and mitmproxy console can place script output in the event buffer.
 
 Here's how this looks:
 
-.. literalinclude:: ../../examples/simple/log_events.py
-   :caption: :src:`examples/simple/log_events.py`
+.. literalinclude:: ../../examples/logging.py
+   :caption: :src:`examples/logging.py`
    :language: python
 
 The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
@@ -126,32 +126,15 @@ It's possible to implement a concurrent mechanism on top of the blocking
 framework, and mitmproxy includes a handy example of this that is fit for most
 purposes. You can use it as follows:
 
-.. literalinclude:: ../../examples/complex/nonblocking.py
-   :caption: :src:`examples/complex/nonblocking.py`
-   :language: python
-
-
-Testing
--------
-
-Mitmproxy includes a number of helpers for testing addons. The
-``mitmproxy.test.taddons`` module contains a context helper that takes care of
-setting up and tearing down the addon event context. The
-``mitmproxy.test.tflow`` module contains helpers for quickly creating test
-flows. Pydoc is the canonical reference for these modules, and mitmproxy's own
-test suite is an excellent source of examples of usage. Here, for instance, is
-the mitmproxy unit tests for the `anticache` option, demonstrating a good
-cross-section of the test helpers:
-
-.. literalinclude:: ../../test/mitmproxy/addons/test_anticache.py
-   :caption: :src:`test/mitmproxy/addons/test_anticache.py`
+.. literalinclude:: ../../examples/nonblocking.py
+   :caption: :src:`examples/nonblocking.py`
    :language: python
 
 
 Developing scripts
 ------------------
 
-Mitmproxy monitors scripts for modifications, and reloads them on change. When
+Mitmprxoy monitors scripts for modifications, and reloads them on change. When
 this happens, the script is shut down (the `done <events.html#done>`_  event is
 called), and the new instance is started up as if the script had just been
 loaded (the `start <events.html#start>`_ and `configure

docs/transparent.rst

@@ -27,7 +27,7 @@ Fully transparent mode
 By default mitmproxy will use its own local ip address for its server-side connections.
 In case this isn't desired, the --spoof-source-address argument can be used to
 use the client's ip address for server-side connections. The following config is
-required for this mode to work::
+required for this mode to work:
 
     CLIENT_NET=192.168.1.0/24
     TABLE_ID=100
@@ -42,9 +42,9 @@ required for this mode to work::
 
 This mode does require root privileges though. There's a wrapper in the examples directory
 called 'mitmproxy_shim.c', which will enable you to use this mode with dropped priviliges.
-It can be used as follows::
+It can be used as follows:
 
-    gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap
+    gcc examples/mitmproxy_shim.c -o mitmproxy_shim -lcap
     sudo chown root:root mitmproxy_shim
     sudo chmod u+s mitmproxy_shim
     ./mitmproxy_shim $(which mitmproxy) -T --spoof-source-address

docs/transparent/openbsd.rst

@@ -1,53 +0,0 @@
-.. _openbsd:
-
-OpenBSD
-=======
-
- 1. :ref:`Install the mitmproxy certificate on the test device <certinstall>`
-
- 2. Enable IP forwarding:
-
-    >>> sudo sysctl -w net.inet.ip.forwarding=1
-
- 3. Place the following two lines in **/etc/pf.conf**:
-
-    .. code-block:: none
-
-        mitm_if = "re2"
-        pass in quick proto tcp from $mitm_if to port { 80, 443 } divert-to 127.0.0.1 port 8080
-
-    These rules tell pf to divert all traffic from ``$mitm_if`` destined for
-    port 80 or 443 to the local mitmproxy instance running on port 8080. You
-    should replace ``$mitm_if`` value with the interface on which your test
-    device will appear.
-
- 4. Configure pf with the rules:
-
-    >>> doas pfctl -f /etc/pf.conf
-
- 5. And now enable it:
-
-    >>> doas pfctl -e
-
- 6. Fire up mitmproxy. You probably want a command like this:
-
-    >>> mitmproxy -T --host
-
-    The ``-T`` flag turns on transparent mode, and the ``--host``
-    argument tells mitmproxy to use the value of the Host header for URL display.
-
- 7. Finally, configure your test device to use the host on which mitmproxy is
-    running as the default gateway.
-
-.. note::
-
-    Note that the **divert-to** rules in the pf.conf given above only apply to
-    inbound traffic. **This means that they will NOT redirect traffic coming
-    from the box running pf itself.** We can't distinguish between an outbound
-    connection from a non-mitmproxy app, and an outbound connection from
-    mitmproxy itself - if you want to intercept your traffic, you should use an
-    external host to run mitmproxy. Nonetheless, pf is flexible to cater for a
-    range of creative possibilities, like intercepting traffic emanating from
-    VMs.  See the **pf.conf** man page for more.
-
-.. _pf: http://man.openbsd.org/OpenBSD-current/man5/pf.conf.5

docs/transparent/osx.rst

@@ -63,7 +63,7 @@ Note that this means we don't support transparent mode for earlier versions of O
     running pf itself.** We can't distinguish between an outbound connection from a
     non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
     want to intercept your OSX traffic, you should use an external host to run
-    mitmproxy. Nonetheless, pf is flexible to cater for a range of creative
+    mitmproxy. None the less, pf is flexible to cater for a range of creative
     possibilities, like intercepting traffic emanating from VMs.  See the
     **pf.conf** man page for more.
 

docs/tutorials/transparent-dhcp.rst

@@ -38,14 +38,8 @@ DHCP and TFTP) services to a small-scale network.
     **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default
     `[1] <https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/>`_. For our use case, this needs
     to be disabled by changing ``dns=dnsmasq`` to ``#dns=dnsmasq`` in
-    **/etc/NetworkManager/NetworkManager.conf** and
-    
-    if on Ubuntu 16.04 or newer running:
-    
-    >>> sudo systemctl restart NetworkManager
-    
-    if on Ubuntu 12.04 or 14.04 running:
-    
+    **/etc/NetworkManager/NetworkManager.conf** and running
+
     >>> sudo restart network-manager
 
     afterwards.
@@ -67,12 +61,6 @@ DHCP and TFTP) services to a small-scale network.
 
     Apply changes:
 
-    if on Ubuntu 16.04 or newer:
-    
-    >>> sudo systemctl restart dnsmasq
-    
-    if on Ubuntu 12.04 or 14.04:
-    
     >>> sudo service dnsmasq restart
 
     Your **proxied machine** in the internal virtual network should now receive an IP address via DHCP:
@@ -86,8 +74,8 @@ To redirect traffic to mitmproxy, we need to add two iptables rules:
 
 .. code-block:: none
 
-    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
-    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
 
 4. Run mitmproxy
 ----------------

examples/README

@@ -0,0 +1,31 @@
+Some inline scripts may require additional dependencies, which can be installed using
+`pip install mitmproxy[examples]`.
+
+
+# inline script examples
+add_header.py             Simple script that just adds a header to every request.
+change_upstream_proxy.py  Dynamically change the upstream proxy
+dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
+dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+fail_with_500.py          Turn every response into an Internal Server Error.
+filt.py                   Use mitmproxy's filter expressions in your script.
+flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
+iframe_injector.py        Inject configurable iframe into pages.
+modify_form.py            Modify all form submissions to add a parameter.
+modify_querystring.py     Modify all query strings to add a parameters.
+modify_response_body.py   Replace arbitrary strings in all responses
+nonblocking.py            Demonstrate parallel processing with a blocking script.
+proxapp.py                How to embed a WSGI app in a mitmproxy server
+redirect_requests.py      Redirect requests or directly reply to them.
+stub.py                   Script stub with a method definition for every event.
+upsidedownternet.py       Rewrites traffic to turn images upside down.
+
+
+# mitmproxy examples
+flowbasic                 Basic use of mitmproxy as a library.
+stickycookies             An example of writing a custom proxy with mitmproxy.
+
+
+# misc
+read_dumpfile             Read a dumpfile generated by mitmproxy.
+mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/README.md

@@ -1,15 +0,0 @@
-# Mitmproxy Scripting API
-
-Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being 
-proxied. In fact, much of mitmproxy’s own core functionality is implemented using the exact same API 
-exposed to scripters (see [mitmproxy/addons](../mitmproxy/addons)).
-
-This directory contains some examples of the scripting API. We recommend to start with the
-ones in [simple/](./simple).
-
-|  :warning: | If you are browsing this on GitHub, make sure to select the git tag matching your mitmproxy version. |
-|------------|------------------------------------------------------------------------------------------------------|
-
-
-Some inline scripts may require additional dependencies, which can be installed using
-`pip install mitmproxy[examples]`.

examples/complex/README.md

@@ -1,18 +0,0 @@
-## Complex Examples
-
-| Filename                 | Description                                                                                   |
-|:-------------------------|:----------------------------------------------------------------------------------------------|
-| change_upstream_proxy.py | Dynamically change the upstream proxy.                                                        |
-| dns_spoofing.py          | Use mitmproxy in a DNS spoofing scenario.                                                     |
-| dup_and_replay.py        | Duplicates each request, changes it, and then replays the modified request.                   |
-| flowbasic.py             | Basic use of mitmproxy's FlowMaster directly.                                                 |
-| full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. |
-| har_dump.py              | Dump flows as HAR files.                                                                      |
-| mitmproxywrapper.py      | Bracket mitmproxy run with proxy enable/disable on OS X                                       |
-| nonblocking.py           | Demonstrate parallel processing with a blocking script                                        |
-| remote_debug.py          | This script enables remote debugging of the mitmproxy _UI_ with PyCharm.                      |
-| sslstrip.py              | sslstrip-like funtionality implemented with mitmproxy                                         |
-| stream.py                | Enable streaming for all responses.                                                           |
-| stream_modify.py         | Modify a streamed response body.                                                              |
-| tcp_message.py           | Modify a raw TCP connection                                                                   |
-| tls_passthrough.py       | Use conditional TLS interception based on a user-defined strategy.                            |

examples/complex/dns_spoofing.py

@@ -1,62 +0,0 @@
-"""
-This script makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
-connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
-Host header of the HTTP request.
-Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
-know the actual target and cannot construct a certificate that looks valid.
-Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
-Using transparent mode is the better option most of the time.
-
-Usage:
-    mitmproxy
-        -p 443
-        -s dns_spoofing.py
-        # Used as the target location if neither SNI nor host header are present.
-        -R http://example.com/
-    mitmdump
-        -p 80
-        -R http://localhost:443/
-
-    (Setting up a single proxy instance and using iptables to redirect to it
-    works as well)
-"""
-import re
-
-# This regex extracts splits the host header into host and port.
-# Handles the edge case of IPv6 addresses containing colons.
-# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
-parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
-
-
-class Rerouter:
-    def requestheaders(self, flow):
-        """
-        The original host header is retrieved early
-        before flow.request is replaced by mitmproxy new outgoing request
-        """
-        flow.metadata["original_host"] = flow.request.host_header
-
-    def request(self, flow):
-        if flow.client_conn.ssl_established:
-            flow.request.scheme = "https"
-            sni = flow.client_conn.connection.get_servername()
-            port = 443
-        else:
-            flow.request.scheme = "http"
-            sni = None
-            port = 80
-
-        host_header = flow.metadata["original_host"]
-        m = parse_host_header.match(host_header)
-        if m:
-            host_header = m.group("host").strip("[]")
-            if m.group("port"):
-                port = int(m.group("port"))
-
-        flow.request.host_header = host_header
-        flow.request.host = sni or host_header
-        flow.request.port = port
-
-
-def start():
-    return Rerouter()

examples/complex/nonblocking.py

@@ -1,11 +0,0 @@
-import time
-
-from mitmproxy.script import concurrent
-
-
-@concurrent  # Remove this and see what happens
-def request(flow):
-    # You don't want to use mitmproxy.ctx from a different thread
-    print("handle request: %s%s" % (flow.request.host, flow.request.path))
-    time.sleep(5)
-    print("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/custom_contentviews.py

@@ -0,0 +1,70 @@
+import string
+import lxml.html
+import lxml.etree
+from mitmproxy import contentviews
+from netlib import strutils
+
+
+class ViewPigLatin(contentviews.View):
+    name = "pig_latin_HTML"
+    prompt = ("pig latin HTML", "l")
+    content_types = ["text/html"]
+
+    def __call__(self, data, **metadata):
+        if strutils.is_xml(data):
+            parser = lxml.etree.HTMLParser(
+                strip_cdata=True,
+                remove_blank_text=True
+            )
+            d = lxml.html.fromstring(data, parser=parser)
+            docinfo = d.getroottree().docinfo
+
+            def piglify(src):
+                words = src.split()
+                ret = ''
+                for word in words:
+                    idx = -1
+                    while word[idx] in string.punctuation and (idx * -1) != len(word):
+                        idx -= 1
+                    if word[0].lower() in 'aeiou':
+                        if idx == -1:
+                            ret += word[0:] + "hay"
+                        else:
+                            ret += word[0:len(word) + idx + 1] + "hay" + word[idx + 1:]
+                    else:
+                        if idx == -1:
+                            ret += word[1:] + word[0] + "ay"
+                        else:
+                            ret += word[1:len(word) + idx + 1] + word[0] + "ay" + word[idx + 1:]
+                    ret += ' '
+                return ret.strip()
+
+            def recurse(root):
+                if hasattr(root, 'text') and root.text:
+                    root.text = piglify(root.text)
+                if hasattr(root, 'tail') and root.tail:
+                    root.tail = piglify(root.tail)
+
+                if len(root):
+                    for child in root:
+                        recurse(child)
+
+            recurse(d)
+
+            s = lxml.etree.tostring(
+                d,
+                pretty_print=True,
+                doctype=docinfo.doctype
+            )
+            return "HTML", contentviews.format_text(s)
+
+
+pig_view = ViewPigLatin()
+
+
+def start():
+    contentviews.add(pig_view)
+
+
+def done():
+    contentviews.remove(pig_view)

examples/dns_spoofing.py

@@ -0,0 +1,49 @@
+"""
+This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
+connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
+Host header of the HTTP request.
+Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
+know the actual target and cannot construct a certificate that looks valid.
+Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
+Using transparent mode is the better option most of the time.
+
+Usage:
+    mitmproxy
+        -p 443
+        -s dns_spoofing.py
+        # Used as the target location if neither SNI nor host header are present.
+        -R http://example.com/
+    mitmdump
+        -p 80
+        -R http://localhost:443/
+
+    (Setting up a single proxy instance and using iptables to redirect to it
+    works as well)
+"""
+import re
+
+# This regex extracts splits the host header into host and port.
+# Handles the edge case of IPv6 addresses containing colons.
+# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
+parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
+
+
+def request(flow):
+    if flow.client_conn.ssl_established:
+        flow.request.scheme = "https"
+        sni = flow.client_conn.connection.get_servername()
+        port = 443
+    else:
+        flow.request.scheme = "http"
+        sni = None
+        port = 80
+
+    host_header = flow.request.pretty_host
+    m = parse_host_header.match(host_header)
+    if m:
+        host_header = m.group("host").strip("[]")
+        if m.group("port"):
+            port = int(m.group("port"))
+
+    flow.request.host = sni or host_header
+    flow.request.port = port

examples/dup_and_replay.py

@@ -0,0 +1,7 @@
+from mitmproxy import master
+
+
+def request(flow):
+    f = master.duplicate_flow(flow)
+    f.request.path = "/changed"
+    master.replay_request(f, block=True, run_scripthooks=False)

examples/fail_with_500.py

@@ -0,0 +1,3 @@
+def response(flow):
+    flow.response.status_code = 500
+    flow.response.content = b""

examples/flowbasic

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 """
     This example shows how to build a proxy based on mitmproxy's Flow
     primitives.
@@ -8,14 +8,14 @@
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
-from mitmproxy import controller, options, master
+from mitmproxy import flow, controller, options
 from mitmproxy.proxy import ProxyServer, ProxyConfig
 
 
-class MyMaster(master.Master):
+class MyMaster(flow.FlowMaster):
     def run(self):
         try:
-            master.Master.run(self)
+            flow.FlowMaster.run(self)
         except KeyboardInterrupt:
             self.shutdown()
 
@@ -35,9 +35,9 @@ class MyMaster(master.Master):
     def log(self, l):
         print("log", l.msg)
 
-
 opts = options.Options(cadir="~/.mitmproxy/")
 config = ProxyConfig(opts)
+state = flow.State()
 server = ProxyServer(config)
-m = MyMaster(opts, server)
+m = MyMaster(opts, server, state)
 m.run()

examples/flowfilter.py

@@ -1,8 +1,6 @@
-"""
-This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
-Usage:
-    mitmdump -s "flowfilter.py FILTER"
-"""
+# This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
+# Usage: mitmdump -s "flowfilter.py FILTER"
+
 import sys
 from mitmproxy import flowfilter
 
@@ -12,7 +10,7 @@ class Filter:
         self.filter = flowfilter.parse(spec)
 
     def response(self, flow):
-        if flowfilter.match(self.filter, flow):
+        if flowfilter.match(flow, self.filter):
             print("Flow matches filter:")
             print(flow)
 

examples/flowwriter.py

@@ -1,13 +1,7 @@
-"""
-This script how to generate a mitmproxy dump file,
-as it would also be generated by passing `-w` to mitmproxy.
-In contrast to `-w`, this gives you full control over which
-flows should be saved and also allows you to rotate files or log
-to multiple files in parallel.
-"""
 import random
 import sys
-from mitmproxy import io
+
+from mitmproxy.flow import FlowWriter
 
 
 class Writer:
@@ -16,7 +10,7 @@ class Writer:
             f = sys.stdout
         else:
             f = open(path, "wb")
-        self.w = io.FlowWriter(f)
+        self.w = FlowWriter(f)
 
     def response(self, flow):
         if random.choice([True, False]):

examples/har_dump.py

@@ -3,20 +3,20 @@ This inline script can be used to dump flows as HAR files.
 """
 
 
+import pprint
 import json
 import sys
 import base64
 import zlib
-import os
 
 from datetime import datetime
 import pytz
 
 import mitmproxy
 
-from mitmproxy import version
-from mitmproxy.utils import strutils
-from mitmproxy.net.http import cookies
+from netlib import version
+from netlib import strutils
+from netlib.http import cookies
 
 HAR = {}
 
@@ -128,25 +128,22 @@ def response(flow):
         "timings": timings,
     }
 
-    # Store binary data as base64
+    # Store binay data as base64
     if strutils.is_mostly_bin(flow.response.content):
-        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
+        b64 = base64.b64encode(flow.response.content)
+        entry["response"]["content"]["text"] = b64.decode('ascii')
         entry["response"]["content"]["encoding"] = "base64"
     else:
-        entry["response"]["content"]["text"] = flow.response.get_text(strict=False)
+        entry["response"]["content"]["text"] = flow.response.text
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
-        params = [
-            {"name": a, "value": b}
-            for a, b in flow.request.urlencoded_form.items(multi=True)
-        ]
         entry["request"]["postData"] = {
-            "mimeType": flow.request.headers.get("Content-Type", ""),
-            "text": flow.request.get_text(strict=False),
-            "params": params
+            "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
+            "text": flow.request.content,
+            "params": name_value(flow.request.urlencoded_form)
         }
 
-    if flow.server_conn.connected():
+    if flow.server_conn:
         entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
 
     HAR["log"]["entries"].append(entry)
@@ -158,17 +155,16 @@ def done():
     """
     dump_file = sys.argv[1]
 
-    json_dump = json.dumps(HAR, indent=2)  # type: str
-
     if dump_file == '-':
-        mitmproxy.ctx.log(json_dump)
+        mitmproxy.ctx.log(pprint.pformat(HAR))
     else:
-        raw = json_dump.encode()  # type: bytes
-        if dump_file.endswith('.zhar'):
-            raw = zlib.compress(raw, 9)
+        json_dump = json.dumps(HAR, indent=2)
 
-        with open(os.path.expanduser(dump_file), "wb") as f:
-            f.write(raw)
+        if dump_file.endswith('.zhar'):
+            json_dump = zlib.compress(json_dump, 9)
+
+        with open(dump_file, "w") as f:
+            f.write(json_dump)
 
         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
 

examples/iframe_injector.py

@@ -11,7 +11,7 @@ class Injector:
     def response(self, flow):
         if flow.request.host in self.iframe_url:
             return
-        html = BeautifulSoup(flow.response.content, "html.parser")
+        html = BeautifulSoup(flow.response.content, "lxml")
         if html.body:
             iframe = html.new_tag(
                 "iframe",

examples/logging.py

@@ -0,0 +1,6 @@
+from mitmproxy import ctx
+
+
+def start():
+    ctx.log.info("This is some informative text.")
+    ctx.log.error("This is an error.")

examples/mitmproxywrapper.py

@@ -15,7 +15,7 @@ import os
 import sys
 
 
-class Wrapper:
+class Wrapper(object):
     def __init__(self, port, extra_arguments=None):
         self.port = port
         self.extra_arguments = extra_arguments

examples/modify_form.py

@@ -1,9 +1,7 @@
 def request(flow):
     if flow.request.urlencoded_form:
-        # If there's already a form, one can just add items to the dict:
         flow.request.urlencoded_form["mitmproxy"] = "rocks"
     else:
-        # One can also just pass new form data.
         # This sets the proper content type and overrides the body.
         flow.request.urlencoded_form = [
             ("foo", "bar")

examples/nonblocking.py

@@ -0,0 +1,10 @@
+import time
+import mitmproxy
+from mitmproxy.script import concurrent
+
+
+@concurrent  # Remove this and see what happens
+def request(flow):
+    mitmproxy.ctx.log("handle request: %s%s" % (flow.request.host, flow.request.path))
+    time.sleep(5)
+    mitmproxy.ctx.log("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/proxapp.py

@@ -4,7 +4,7 @@ instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
 a single simplest-possible page.
 """
 from flask import Flask
-from mitmproxy.addons import wsgiapp
+import mitmproxy
 
 app = Flask("proxapp")
 
@@ -14,12 +14,12 @@ def hello_world():
     return 'Hello World!'
 
 
+# Register the app using the magic domain "proxapp" on port 80. Requests to
+# this domain and port combination will now be routed to the WSGI app instance.
 def start():
-    # Host app at the magic domain "proxapp" on port 80. Requests to this
-    # domain and port combination will now be routed to the WSGI app instance.
-    return wsgiapp.WSGIApp(app, "proxapp", 80)
+    mitmproxy.ctx.master.apps.add(app, "proxapp", 80)
 
     # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
     # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
     # but won't send any data.
-    # mitmproxy.ctx.master.apps.add(app, "example.com", 443)
+    mitmproxy.ctx.master.apps.add(app, "example.com", 443)

examples/read_dumpfile

@@ -3,13 +3,13 @@
 # Simple script showing how to read a mitmproxy dump file
 #
 
-from mitmproxy import io
+from mitmproxy import flow
 from mitmproxy.exceptions import FlowReadException
 import pprint
 import sys
 
 with open(sys.argv[1], "rb") as logfile:
-    freader = io.FlowReader(logfile)
+    freader = flow.FlowReader(logfile)
     pp = pprint.PrettyPrinter(indent=4)
     try:
         for f in freader.stream():

examples/redirect_requests.py

@@ -0,0 +1,19 @@
+"""
+This example shows two ways to redirect flows to other destinations.
+"""
+from mitmproxy.models import HTTPResponse
+
+
+def request(flow):
+    # pretty_host takes the "Host" header of the request into account,
+    # which is useful in transparent mode where we usually only have the IP
+    # otherwise.
+
+    # Method 1: Answer with a locally generated response
+    if flow.request.pretty_host.endswith("example.com"):
+        resp = HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"})
+        flow.reply.send(resp)
+
+    # Method 2: Redirect the request to a different server
+    if flow.request.pretty_host.endswith("example.org"):
+        flow.request.host = "mitmproxy.org"

examples/simple/README.md

@@ -1,18 +0,0 @@
-## Simple Examples
-
-| Filename                     | Description                                                                |
-|:-----------------------------|:---------------------------------------------------------------------------|
-| add_header.py                | Simple script that just adds a header to every request.                    |
-| custom_contentview.py        | Add a custom content view to the mitmproxy UI.                             |
-| filter_flows.py              | This script demonstrates how to use mitmproxy's filter pattern in scripts. |
-| io_read_dumpfile.py          | Read a dumpfile generated by mitmproxy.                                    |
-| io_write_dumpfile.py         | Only write selected flows into a mitmproxy dumpfile.                       |
-| log_events.py                   | Use mitmproxy's logging API.                                            |
-| modify_body_inject_iframe.py | Inject configurable iframe into pages.                                     |
-| modify_form.py               | Modify HTTP form submissions.                                              |
-| modify_querystring.py        | Modify HTTP query strings.                                                 |
-| redirect_requests.py         | Redirect a request to a different server.                                  |
-| script_arguments.py          | Add arguments to a script.                                                 |
-| send_reply_from_proxy.py     | Send a HTTP response directly from the proxy.                              |
-| upsidedownternet.py          | Turn all images upside down.                                               |
-| wsgi_flask_app.py            | Embed a WSGI app into mitmproxy.                                           |

examples/simple/custom_contentview.py

@@ -1,28 +0,0 @@
-"""
-This example shows how one can add a custom contentview to mitmproxy.
-The content view API is explained in the mitmproxy.contentviews module.
-"""
-from mitmproxy import contentviews
-
-
-class ViewSwapCase(contentviews.View):
-    name = "swapcase"
-
-    # We don't have a good solution for the keyboard shortcut yet -
-    # you manually need to find a free letter. Contributions welcome :)
-    prompt = ("swap case text", "z")
-    content_types = ["text/plain"]
-
-    def __call__(self, data: bytes, **metadata):
-        return "case-swapped text", contentviews.format_text(data.swapcase())
-
-
-view = ViewSwapCase()
-
-
-def start():
-    contentviews.add(view)
-
-
-def done():
-    contentviews.remove(view)

examples/simple/log_events.py

@@ -1,12 +0,0 @@
-"""
-It is recommended to use `ctx.log` for logging within a script.
-This goes to the event log in mitmproxy and to stdout in mitmdump.
-
-If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 :-)
-"""
-from mitmproxy import ctx
-
-
-def start():
-    ctx.log.info("This is some informative text.")
-    ctx.log.error("This is an error.")

examples/simple/redirect_requests.py

@@ -1,11 +0,0 @@
-"""
-This example shows two ways to redirect flows to another server.
-"""
-
-
-def request(flow):
-    # pretty_host takes the "Host" header of the request into account,
-    # which is useful in transparent mode where we usually only have the IP
-    # otherwise.
-    if flow.request.pretty_host == "example.org":
-        flow.request.host = "mitmproxy.org"

examples/simple/send_reply_from_proxy.py

@@ -1,17 +0,0 @@
-"""
-This example shows how to send a reply from the proxy immediately
-without sending any data to the remote server.
-"""
-from mitmproxy import http
-
-
-def request(flow):
-    # pretty_url takes the "Host" header of the request into account, which
-    # is useful in transparent mode where we usually only have the IP otherwise.
-
-    if flow.request.pretty_url == "http://example.com/path":
-        flow.response = http.HTTPResponse.make(
-            200,  # (optional) status code
-            b"Hello World",  # (optional) content
-            {"Content-Type": "text/html"}  # (optional) headers
-        )

examples/simple/upsidedownternet.py

@@ -1,16 +0,0 @@
-"""
-This script rotates all images passing through the proxy by 180 degrees.
-"""
-import io
-
-from PIL import Image
-
-
-def response(flow):
-    if flow.response.headers.get("content-type", "").startswith("image"):
-        s = io.BytesIO(flow.response.content)
-        img = Image.open(s).rotate(180)
-        s2 = io.BytesIO()
-        img.save(s2, "png")
-        flow.response.content = s2.getvalue()
-        flow.response.headers["content-type"] = "image/png"

examples/sslstrip.py

@@ -1,9 +1,5 @@
-"""
-This script implements an sslstrip-like attack based on mitmproxy.
-https://moxie.org/software/sslstrip/
-"""
 import re
-import urllib
+from six.moves import urllib
 
 # set of SSL/TLS capable hosts
 secure_hosts = set()
@@ -21,18 +17,13 @@ def request(flow):
         flow.request.scheme = 'https'
         flow.request.port = 443
 
-        # We need to update the request destination to whatever is specified in the host header:
-        # Having no TLS Server Name Indication from the client and just an IP address as request.host
-        # in transparent mode, TLS server name certificate validation would fail.
-        flow.request.host = flow.request.pretty_host
-
 
 def response(flow):
     flow.response.headers.pop('Strict-Transport-Security', None)
     flow.response.headers.pop('Public-Key-Pins', None)
 
     # strip links in response body
-    flow.response.content = flow.response.content.replace(b'https://', b'http://')
+    flow.response.content = flow.response.content.replace('https://', 'http://')
 
     # strip meta tag upgrade-insecure-requests in response body
     csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>'

examples/stickycookies

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+"""
+This example builds on mitmproxy's base proxying infrastructure to
+implement functionality similar to the "sticky cookies" option.
+
+Heads Up: In the majority of cases, you want to use inline scripts.
+"""
+import os
+from mitmproxy import controller, proxy
+from mitmproxy.proxy.server import ProxyServer
+
+
+class StickyMaster(controller.Master):
+    def __init__(self, server):
+        controller.Master.__init__(self, server)
+        self.stickyhosts = {}
+
+    def run(self):
+        try:
+            return controller.Master.run(self)
+        except KeyboardInterrupt:
+            self.shutdown()
+
+    @controller.handler
+    def request(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if "cookie" in flow.request.headers:
+            self.stickyhosts[hid] = flow.request.headers.get_all("cookie")
+        elif hid in self.stickyhosts:
+            flow.request.headers.set_all("cookie", self.stickyhosts[hid])
+
+    @controller.handler
+    def response(self, flow):
+        hid = (flow.request.host, flow.request.port)
+        if "set-cookie" in flow.response.headers:
+            self.stickyhosts[hid] = flow.response.headers.get_all("set-cookie")
+
+
+config = proxy.ProxyConfig(port=8080)
+server = ProxyServer(config)
+m = StickyMaster(server)
+m.run()

examples/stream.py

@@ -1,6 +1,5 @@
 def responseheaders(flow):
     """
     Enables streaming for all responses.
-    This is equivalent to passing `--stream 0` to mitmproxy.
     """
     flow.response.stream = True

examples/stub.py

@@ -0,0 +1,87 @@
+import mitmproxy
+"""
+    This is a script stub, with definitions for all events.
+"""
+
+
+def start():
+    """
+        Called once on script startup before any other events
+    """
+    mitmproxy.ctx.log("start")
+
+
+def configure(options, updated):
+    """
+        Called once on script startup before any other events, and whenever options changes.
+    """
+    mitmproxy.ctx.log("configure")
+
+
+def clientconnect(root_layer):
+    """
+        Called when a client initiates a connection to the proxy. Note that a
+        connection can correspond to multiple HTTP requests
+    """
+    mitmproxy.ctx.log("clientconnect")
+
+
+def request(flow):
+    """
+        Called when a client request has been received.
+    """
+    mitmproxy.ctx.log("request")
+
+
+def serverconnect(server_conn):
+    """
+        Called when the proxy initiates a connection to the target server. Note that a
+        connection can correspond to multiple HTTP requests
+    """
+    mitmproxy.ctx.log("serverconnect")
+
+
+def responseheaders(flow):
+    """
+        Called when the response headers for a server response have been received,
+        but the response body has not been processed yet. Can be used to tell mitmproxy
+        to stream the response.
+    """
+    mitmproxy.ctx.log("responseheaders")
+
+
+def response(flow):
+    """
+       Called when a server response has been received.
+    """
+    mitmproxy.ctx.log("response")
+
+
+def error(flow):
+    """
+        Called when a flow error has occured, e.g. invalid server responses, or
+        interrupted connections. This is distinct from a valid server HTTP error
+        response, which is simply a response with an HTTP error code.
+    """
+    mitmproxy.ctx.log("error")
+
+
+def serverdisconnect(server_conn):
+    """
+        Called when the proxy closes the connection to the target server.
+    """
+    mitmproxy.ctx.log("serverdisconnect")
+
+
+def clientdisconnect(root_layer):
+    """
+        Called when a client disconnects from the proxy.
+    """
+    mitmproxy.ctx.log("clientdisconnect")
+
+
+def done():
+    """
+        Called once on script shutdown, after any other events.
+    """
+    mitmproxy.ctx.log("done")

examples/tcp_message.py

@@ -8,7 +8,7 @@ tcp_message Inline Script Hook API Demonstration
 example cmdline invocation:
 mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py
 """
-from mitmproxy.utils import strutils
+from netlib import strutils
 
 
 def tcp_message(tcp_msg):

examples/tls_passthrough.py

@@ -20,6 +20,7 @@ Example:
 
 Authors: Maximilian Hils, Matthew Tuusberg
 """
+from __future__ import absolute_import, print_function, division
 import collections
 import random
 
@@ -28,7 +29,7 @@ from enum import Enum
 
 import mitmproxy
 from mitmproxy.exceptions import TlsProtocolException
-from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
+from mitmproxy.protocol import TlsLayer, RawTCPLayer
 
 
 class InterceptionResult(Enum):
@@ -37,7 +38,7 @@ class InterceptionResult(Enum):
     skipped = None
 
 
-class _TlsStrategy:
+class _TlsStrategy(object):
     """
     Abstract base class for interception strategies.
     """

examples/upsidedownternet.py

@@ -0,0 +1,15 @@
+from six.moves import cStringIO as StringIO
+from PIL import Image
+
+
+def response(flow):
+    if flow.response.headers.get("content-type", "").startswith("image"):
+        try:
+            s = StringIO(flow.response.content)
+            img = Image.open(s).rotate(180)
+            s2 = StringIO()
+            img.save(s2, "png")
+            flow.response.content = s2.getvalue()
+            flow.response.headers["content-type"] = "image/png"
+        except:  # Unknown image types etc.
+            pass

issue_template.md

@@ -1,20 +1,22 @@
 ##### Steps to reproduce the problem:
 
-1.
-2.
-3.
+1. 
+2. 
+3. 
+
+##### What is the expected behavior?
+
+
+##### What went wrong?
 
 
 ##### Any other comments? What have you tried so far?
 
 
+---
 
-##### System information
+Mitmproxy Version:
+Operating System:
 
 
-<!--
-    Cut and paste the output of "mitmproxy --version".
-
-    If you're using an older version if mitmproxy, please specify the version
-    and OS.
--->
+<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) -->

mitmproxy/__init__.py

@@ -1,3 +0,0 @@
-# https://github.com/mitmproxy/mitmproxy/issues/1809
-# import script here so that pyinstaller registers it.
-from . import script  # noqa

mitmproxy/addons.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import, print_function, division
 from mitmproxy import exceptions
 import pprint
 
@@ -6,19 +7,12 @@ def _get_name(itm):
     return getattr(itm, "name", itm.__class__.__name__.lower())
 
 
-class AddonManager:
+class Addons(object):
     def __init__(self, master):
         self.chain = []
         self.master = master
         master.options.changed.connect(self._options_update)
 
-    def clear(self):
-        """
-            Remove all addons.
-        """
-        self.done()
-        self.chain = []
-
     def get(self, name):
         """
             Retrieve an addon by name. Addon names are equal to the .name
@@ -32,7 +26,7 @@ class AddonManager:
     def _options_update(self, options, updated):
         for i in self.chain:
             with self.master.handlecontext():
-                self.invoke_with_context(i, "configure", options, updated)
+                i.configure(options, updated)
 
     def startup(self, s):
         """
@@ -50,6 +44,8 @@ class AddonManager:
         """
             Add addons to the end of the chain, and run their startup events.
         """
+        if not addons:
+            raise ValueError("No addons specified.")
         self.chain.extend(addons)
         for i in addons:
             self.startup(i)
@@ -86,7 +82,4 @@ class AddonManager:
 
     def __call__(self, name, *args, **kwargs):
         for i in self.chain:
-            try:
-                self.invoke(i, name, *args, **kwargs)
-            except exceptions.AddonHalt:
-                return
+            self.invoke(i, name, *args, **kwargs)

mitmproxy/addons/__init__.py

@@ -1,40 +0,0 @@
-from mitmproxy.addons import anticache
-from mitmproxy.addons import anticomp
-from mitmproxy.addons import check_alpn
-from mitmproxy.addons import check_ca
-from mitmproxy.addons import clientplayback
-from mitmproxy.addons import disable_h2c_upgrade
-from mitmproxy.addons import onboarding
-from mitmproxy.addons import proxyauth
-from mitmproxy.addons import replace
-from mitmproxy.addons import script
-from mitmproxy.addons import serverplayback
-from mitmproxy.addons import setheaders
-from mitmproxy.addons import stickyauth
-from mitmproxy.addons import stickycookie
-from mitmproxy.addons import streambodies
-from mitmproxy.addons import streamfile
-from mitmproxy.addons import upstream_auth
-
-
-def default_addons():
-    return [
-        anticache.AntiCache(),
-        anticomp.AntiComp(),
-        check_alpn.CheckALPN(),
-        check_ca.CheckCA(),
-        clientplayback.ClientPlayback(),
-        disable_h2c_upgrade.DisableH2CleartextUpgrade(),
-        onboarding.Onboarding(),
-        proxyauth.ProxyAuth(),
-        replace.Replace(),
-        replace.ReplaceFile(),
-        script.ScriptLoader(),
-        serverplayback.ServerPlayback(),
-        setheaders.SetHeaders(),
-        stickyauth.StickyAuth(),
-        stickycookie.StickyCookie(),
-        streambodies.StreamBodies(),
-        streamfile.StreamFile(),
-        upstream_auth.UpstreamAuth(),
-    ]

mitmproxy/addons/check_alpn.py

@@ -1,17 +0,0 @@
-import mitmproxy
-from mitmproxy.net import tcp
-
-
-class CheckALPN:
-    def __init__(self):
-        self.failed = False
-
-    def configure(self, options, updated):
-        self.failed = mitmproxy.ctx.master.options.http2 and not tcp.HAS_ALPN
-        if self.failed:
-            mitmproxy.ctx.master.add_log(
-                "HTTP/2 is disabled because ALPN support missing!\n"
-                "OpenSSL 1.0.2+ required to support HTTP/2 connections.\n"
-                "Use --no-http2 to silence this warning.",
-                "warn",
-            )

mitmproxy/addons/check_ca.py

@@ -1,24 +0,0 @@
-import mitmproxy
-
-
-class CheckCA:
-    def __init__(self):
-        self.failed = False
-
-    def configure(self, options, updated):
-        has_ca = (
-            mitmproxy.ctx.master.server and
-            mitmproxy.ctx.master.server.config and
-            mitmproxy.ctx.master.server.config.certstore and
-            mitmproxy.ctx.master.server.config.certstore.default_ca
-        )
-        if has_ca:
-            self.failed = mitmproxy.ctx.master.server.config.certstore.default_ca.has_expired()
-            if self.failed:
-                mitmproxy.ctx.master.add_log(
-                    "The mitmproxy certificate authority has expired!\n"
-                    "Please delete all CA-related files in your ~/.mitmproxy folder.\n"
-                    "The CA will be regenerated automatically after restarting mitmproxy.\n"
-                    "Then make sure all your clients have the new CA installed.",
-                    "warn",
-                )

mitmproxy/addons/disable_h2c_upgrade.py

@@ -1,21 +0,0 @@
-class DisableH2CleartextUpgrade:
-
-    """
-    We currently only support HTTP/2 over a TLS connection. Some clients try
-    to upgrade a connection from HTTP/1.1 to h2c, so we need to remove those
-    headers to avoid protocol errors if one endpoints suddenly starts sending
-    HTTP/2 frames.
-    """
-
-    def process_flow(self, f):
-        if f.request.headers.get('upgrade', '') == 'h2c':
-            del f.request.headers['upgrade']
-            if 'connection' in f.request.headers:
-                del f.request.headers['connection']
-            if 'http2-settings' in f.request.headers:
-                del f.request.headers['http2-settings']
-
-    # Handlers
-
-    def request(self, f):
-        self.process_flow(f)

mitmproxy/addons/eventstore.py

@@ -1,19 +0,0 @@
-from typing import List  # noqa
-
-import blinker
-from mitmproxy.log import LogEntry
-
-
-class EventStore:
-    def __init__(self):
-        self.data = []  # type: List[LogEntry]
-        self.sig_add = blinker.Signal()
-        self.sig_refresh = blinker.Signal()
-
-    def log(self, entry: LogEntry):
-        self.data.append(entry)
-        self.sig_add.send(self, entry=entry)
-
-    def clear(self):
-        self.data.clear()
-        self.sig_refresh.send(self)

mitmproxy/addons/intercept.py

@@ -1,35 +0,0 @@
-from mitmproxy import flowfilter
-from mitmproxy import exceptions
-
-
-class Intercept:
-    def __init__(self):
-        self.filt = None
-
-    def configure(self, opts, updated):
-        if "intercept" in updated:
-            if not opts.intercept:
-                self.filt = None
-                return
-            self.filt = flowfilter.parse(opts.intercept)
-            if not self.filt:
-                raise exceptions.OptionsError(
-                    "Invalid interception filter: %s" % opts.intercept
-                )
-
-    def process_flow(self, f):
-        if self.filt:
-            should_intercept = all([
-                self.filt(f),
-                not f.request.is_replay,
-            ])
-            if should_intercept:
-                f.intercept()
-
-    # Handlers
-
-    def request(self, f):
-        self.process_flow(f)
-
-    def response(self, f):
-        self.process_flow(f)

mitmproxy/addons/onboarding.py

@@ -1,17 +0,0 @@
-from mitmproxy.addons import wsgiapp
-from mitmproxy.addons.onboardingapp import app
-
-
-class Onboarding(wsgiapp.WSGIApp):
-    def __init__(self):
-        super().__init__(app.Adapter(app.application), None, None)
-        self.enabled = False
-
-    def configure(self, options, updated):
-        self.host = options.onboarding_host
-        self.port = options.onboarding_port
-        self.enabled = options.onboarding
-
-    def request(self, f):
-        if self.enabled:
-            super().request(f)

mitmproxy/addons/proxyauth.py

@@ -1,165 +0,0 @@
-import binascii
-import weakref
-from typing import Optional
-from typing import Set  # noqa
-from typing import Tuple
-
-import passlib.apache
-
-import mitmproxy.net.http
-from mitmproxy import connections  # noqa
-from mitmproxy import exceptions
-from mitmproxy import http
-from mitmproxy.net.http import status_codes
-
-REALM = "mitmproxy"
-
-
-def mkauth(username: str, password: str, scheme: str = "basic") -> str:
-    """
-    Craft a basic auth string
-    """
-    v = binascii.b2a_base64(
-        (username + ":" + password).encode("utf8")
-    ).decode("ascii")
-    return scheme + " " + v
-
-
-def parse_http_basic_auth(s: str) -> Tuple[str, str, str]:
-    """
-    Parse a basic auth header.
-    Raises a ValueError if the input is invalid.
-    """
-    scheme, authinfo = s.split()
-    if scheme.lower() != "basic":
-        raise ValueError("Unknown scheme")
-    try:
-        user, password = binascii.a2b_base64(authinfo.encode()).decode("utf8", "replace").split(":")
-    except binascii.Error as e:
-        raise ValueError(str(e))
-    return scheme, user, password
-
-
-class ProxyAuth:
-    def __init__(self):
-        self.nonanonymous = False
-        self.htpasswd = None
-        self.singleuser = None
-        self.mode = None
-        self.authenticated = weakref.WeakSet()  # type: Set[connections.ClientConnection]
-        """Contains all connections that are permanently authenticated after an HTTP CONNECT"""
-
-    def enabled(self) -> bool:
-        return any([self.nonanonymous, self.htpasswd, self.singleuser])
-
-    def is_proxy_auth(self) -> bool:
-        """
-        Returns:
-            - True, if authentication is done as if mitmproxy is a proxy
-            - False, if authentication is done as if mitmproxy is a HTTP server
-        """
-        return self.mode in ("regular", "upstream")
-
-    def which_auth_header(self) -> str:
-        if self.is_proxy_auth():
-            return 'Proxy-Authorization'
-        else:
-            return 'Authorization'
-
-    def auth_required_response(self) -> http.HTTPResponse:
-        if self.is_proxy_auth():
-            return http.make_error_response(
-                status_codes.PROXY_AUTH_REQUIRED,
-                headers=mitmproxy.net.http.Headers(Proxy_Authenticate='Basic realm="{}"'.format(REALM)),
-            )
-        else:
-            return http.make_error_response(
-                status_codes.UNAUTHORIZED,
-                headers=mitmproxy.net.http.Headers(WWW_Authenticate='Basic realm="{}"'.format(REALM)),
-            )
-
-    def check(self, f: http.HTTPFlow) -> Optional[Tuple[str, str]]:
-        """
-        Check if a request is correctly authenticated.
-        Returns:
-            - a (username, password) tuple if successful,
-            - None, otherwise.
-        """
-        auth_value = f.request.headers.get(self.which_auth_header(), "")
-        try:
-            scheme, username, password = parse_http_basic_auth(auth_value)
-        except ValueError:
-            return None
-
-        if self.nonanonymous:
-            return username, password
-        elif self.singleuser:
-            if self.singleuser == [username, password]:
-                return username, password
-        elif self.htpasswd:
-            if self.htpasswd.check_password(username, password):
-                return username, password
-
-        return None
-
-    def authenticate(self, f: http.HTTPFlow) -> bool:
-        valid_credentials = self.check(f)
-        if valid_credentials:
-            f.metadata["proxyauth"] = valid_credentials
-            del f.request.headers[self.which_auth_header()]
-            return True
-        else:
-            f.response = self.auth_required_response()
-            return False
-
-    # Handlers
-    def configure(self, options, updated):
-        if "auth_nonanonymous" in updated:
-            self.nonanonymous = options.auth_nonanonymous
-        if "auth_singleuser" in updated:
-            if options.auth_singleuser:
-                parts = options.auth_singleuser.split(':')
-                if len(parts) != 2:
-                    raise exceptions.OptionsError(
-                        "Invalid single-user auth specification."
-                    )
-                self.singleuser = parts
-            else:
-                self.singleuser = None
-        if "auth_htpasswd" in updated:
-            if options.auth_htpasswd:
-                try:
-                    self.htpasswd = passlib.apache.HtpasswdFile(
-                        options.auth_htpasswd
-                    )
-                except (ValueError, OSError) as v:
-                    raise exceptions.OptionsError(
-                        "Could not open htpasswd file: %s" % v
-                    )
-            else:
-                self.htpasswd = None
-        if "mode" in updated:
-            self.mode = options.mode
-        if self.enabled():
-            if options.mode == "transparent":
-                raise exceptions.OptionsError(
-                    "Proxy Authentication not supported in transparent mode."
-                )
-            if options.mode == "socks5":
-                raise exceptions.OptionsError(
-                    "Proxy Authentication not supported in SOCKS mode. "
-                    "https://github.com/mitmproxy/mitmproxy/issues/738"
-                )
-                # TODO: check for multiple auth options
-
-    def http_connect(self, f: http.HTTPFlow) -> None:
-        if self.enabled():
-            if self.authenticate(f):
-                self.authenticated.add(f.client_conn)
-
-    def requestheaders(self, f: http.HTTPFlow) -> None:
-        if self.enabled():
-            # Is this connection authenticated by a previous HTTP CONNECT?
-            if f.client_conn in self.authenticated:
-                return
-            self.authenticate(f)

mitmproxy/addons/replace.py

@@ -1,112 +0,0 @@
-import re
-
-from mitmproxy import exceptions
-from mitmproxy import flowfilter
-from mitmproxy import ctx
-
-
-def parse_hook(s):
-    """
-        Returns a (pattern, regex, replacement) tuple.
-
-        The general form for a replacement hook is as follows:
-
-            /patt/regex/replacement
-
-        The first character specifies the separator. Example:
-
-            :~q:foo:bar
-
-        If only two clauses are specified, the pattern is set to match
-        universally (i.e. ".*"). Example:
-
-            /foo/bar/
-
-        Clauses are parsed from left to right. Extra separators are taken to be
-        part of the final clause. For instance, the replacement clause below is
-        "foo/bar/":
-
-            /one/two/foo/bar/
-    """
-    sep, rem = s[0], s[1:]
-    parts = rem.split(sep, 2)
-    if len(parts) == 2:
-        patt = ".*"
-        a, b = parts
-    elif len(parts) == 3:
-        patt, a, b = parts
-    else:
-        raise exceptions.OptionsError(
-            "Invalid replacement specifier: %s" % s
-        )
-    return patt, a, b
-
-
-class _ReplaceBase:
-    def __init__(self):
-        self.lst = []
-
-    def configure(self, options, updated):
-        """
-            .replacements is a list of tuples (fpat, rex, s):
-
-            fpatt: a string specifying a filter pattern.
-            rex: a regular expression, as bytes.
-            s: the replacement string, as bytes
-        """
-        if self.optionName in updated:
-            lst = []
-            for rep in getattr(options, self.optionName):
-                if isinstance(rep, str):
-                    fpatt, rex, s = parse_hook(rep)
-                else:
-                    fpatt, rex, s = rep
-
-                flt = flowfilter.parse(fpatt)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "Invalid filter pattern: %s" % fpatt
-                    )
-                try:
-                    re.compile(rex)
-                except re.error as e:
-                    raise exceptions.OptionsError(
-                        "Invalid regular expression: %s - %s" % (rex, str(e))
-                    )
-                lst.append((rex, s, flt))
-            self.lst = lst
-
-    def execute(self, f):
-        for rex, s, flt in self.lst:
-            if flt(f):
-                if f.response:
-                    self.replace(f.response, rex, s)
-                else:
-                    self.replace(f.request, rex, s)
-
-    def request(self, flow):
-        if not flow.reply.has_message:
-            self.execute(flow)
-
-    def response(self, flow):
-        if not flow.reply.has_message:
-            self.execute(flow)
-
-
-class Replace(_ReplaceBase):
-    optionName = "replacements"
-
-    def replace(self, obj, rex, s):
-        obj.replace(rex, s, flags=re.DOTALL)
-
-
-class ReplaceFile(_ReplaceBase):
-    optionName = "replacement_files"
-
-    def replace(self, obj, rex, s):
-        try:
-            v = open(s, "rb").read()
-        except IOError as e:
-            ctx.log.warn("Could not read replacement file: %s" % s)
-            return
-        obj.replace(rex, v, flags=re.DOTALL)

mitmproxy/addons/setheaders.py

@@ -1,83 +0,0 @@
-from mitmproxy import exceptions
-from mitmproxy import flowfilter
-
-
-def parse_setheader(s):
-    """
-        Returns a (pattern, regex, replacement) tuple.
-
-        The general form for a replacement hook is as follows:
-
-            /patt/regex/replacement
-
-        The first character specifies the separator. Example:
-
-            :~q:foo:bar
-
-        If only two clauses are specified, the pattern is set to match
-        universally (i.e. ".*"). Example:
-
-            /foo/bar/
-
-        Clauses are parsed from left to right. Extra separators are taken to be
-        part of the final clause. For instance, the replacement clause below is
-        "foo/bar/":
-
-            /one/two/foo/bar/
-    """
-    sep, rem = s[0], s[1:]
-    parts = rem.split(sep, 2)
-    if len(parts) == 2:
-        patt = ".*"
-        a, b = parts
-    elif len(parts) == 3:
-        patt, a, b = parts
-    else:
-        raise exceptions.OptionsError(
-            "Invalid replacement specifier: %s" % s
-        )
-    return patt, a, b
-
-
-class SetHeaders:
-    def __init__(self):
-        self.lst = []
-
-    def configure(self, options, updated):
-        """
-            options.setheaders is a tuple of (fpatt, header, value)
-
-            fpatt: String specifying a filter pattern.
-            header: Header name.
-            value: Header value string
-        """
-        if "setheaders" in updated:
-            self.lst = []
-            for shead in options.setheaders:
-                if isinstance(shead, str):
-                    fpatt, header, value = parse_setheader(shead)
-                else:
-                    fpatt, header, value = shead
-
-                flt = flowfilter.parse(fpatt)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "Invalid setheader filter pattern %s" % fpatt
-                    )
-                self.lst.append((fpatt, header, value, flt))
-
-    def run(self, f, hdrs):
-        for _, header, value, flt in self.lst:
-            if flt(f):
-                hdrs.pop(header, None)
-        for _, header, value, flt in self.lst:
-            if flt(f):
-                hdrs.add(header, value)
-
-    def request(self, flow):
-        if not flow.reply.has_message:
-            self.run(flow, flow.request.headers)
-
-    def response(self, flow):
-        if not flow.reply.has_message:
-            self.run(flow, flow.response.headers)

mitmproxy/addons/stickyauth.py

@@ -1,29 +0,0 @@
-from mitmproxy import exceptions
-from mitmproxy import flowfilter
-
-
-class StickyAuth:
-    def __init__(self):
-        self.flt = None
-        self.hosts = {}
-
-    def configure(self, options, updated):
-        if "stickyauth" in updated:
-            if options.stickyauth:
-                flt = flowfilter.parse(options.stickyauth)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "stickyauth: invalid filter expression: %s" % options.stickyauth
-                    )
-                self.flt = flt
-            else:
-                self.flt = None
-
-    def request(self, flow):
-        if self.flt:
-            host = flow.request.host
-            if "authorization" in flow.request.headers:
-                self.hosts[host] = flow.request.headers["authorization"]
-            elif flowfilter.match(self.flt, flow):
-                if host in self.hosts:
-                    flow.request.headers["authorization"] = self.hosts[host]

mitmproxy/addons/streambodies.py

@@ -1,34 +0,0 @@
-from mitmproxy.net.http import http1
-from mitmproxy import exceptions
-from mitmproxy import ctx
-
-
-class StreamBodies:
-    def __init__(self):
-        self.max_size = None
-
-    def configure(self, options, updated):
-        self.max_size = options.stream_large_bodies
-
-    def run(self, f, is_request):
-        if self.max_size:
-            r = f.request if is_request else f.response
-            try:
-                expected_size = http1.expected_http_body_size(
-                    f.request, f.response if not is_request else None
-                )
-            except exceptions.HttpException:
-                f.reply.kill()
-                return
-            if expected_size and not r.raw_content and not (0 <= expected_size <= self.max_size):
-                # r.stream may already be a callable, which we want to preserve.
-                r.stream = r.stream or True
-                # FIXME: make message generic when we add rquest streaming
-                ctx.log.info("Streaming response from %s" % f.request.host)
-
-    # FIXME! Request streaming doesn't work at the moment.
-    def requestheaders(self, f):
-        self.run(f, True)
-
-    def responseheaders(self, f):
-        self.run(f, False)

mitmproxy/addons/upstream_auth.py

@@ -1,53 +0,0 @@
-import re
-import base64
-
-from mitmproxy import exceptions
-from mitmproxy.utils import strutils
-
-
-def parse_upstream_auth(auth):
-    pattern = re.compile(".+:")
-    if pattern.search(auth) is None:
-        raise exceptions.OptionsError(
-            "Invalid upstream auth specification: %s" % auth
-        )
-    return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
-
-
-class UpstreamAuth():
-    """
-        This addon handles authentication to systems upstream from us for the
-        upstream proxy and reverse proxy mode. There are 3 cases:
-
-        - Upstream proxy CONNECT requests should have authentication added, and
-          subsequent already connected requests should not.
-        - Upstream proxy regular requests
-        - Reverse proxy regular requests (CONNECT is invalid in this mode)
-    """
-    def __init__(self):
-        self.auth = None
-        self.root_mode = None
-
-    def configure(self, options, updated):
-        # FIXME: We're doing this because our proxy core is terminally confused
-        # at the moment. Ideally, we should be able to check if we're in
-        # reverse proxy mode at the HTTP layer, so that scripts can put the
-        # proxy in reverse proxy mode for specific reuests.
-        if "mode" in updated:
-            self.root_mode = options.mode
-        if "upstream_auth" in updated:
-            if options.upstream_auth is None:
-                self.auth = None
-            else:
-                self.auth = parse_upstream_auth(options.upstream_auth)
-
-    def http_connect(self, f):
-        if self.auth and f.mode == "upstream":
-            f.request.headers["Proxy-Authorization"] = self.auth
-
-    def requestheaders(self, f):
-        if self.auth:
-            if f.mode == "upstream" and not f.server_conn.via:
-                f.request.headers["Proxy-Authorization"] = self.auth
-            elif self.root_mode == "reverse":
-                f.request.headers["Proxy-Authorization"] = self.auth

mitmproxy/addons/view.py

@@ -1,429 +0,0 @@
-"""
-The View:
-
-- Keeps track of a store of flows
-- Maintains a filtered, ordered view onto that list of flows
-- Exposes a number of signals so the view can be monitored
-- Tracks focus within the view
-- Exposes a settings store for flows that automatically expires if the flow is
-  removed from the store.
-"""
-import collections
-import typing
-import datetime
-
-import blinker
-import sortedcontainers
-
-import mitmproxy.flow
-from mitmproxy import flowfilter
-from mitmproxy import exceptions
-
-# The underlying sorted list implementation expects the sort key to be stable
-# for the lifetime of the object. However, if we sort by size, for instance,
-# the sort order changes as the flow progresses through its lifecycle. We
-# address this through two means:
-#
-# - Let order keys cache the sort value by flow ID.
-#
-# - Add a facility to refresh items in the list by removing and re-adding them
-# when they are updated.
-
-
-class _OrderKey:
-    def __init__(self, view):
-        self.view = view
-
-    def generate(self, f: mitmproxy.flow.Flow) -> typing.Any:  # pragma: no cover
-        pass
-
-    def refresh(self, f):
-        k = self._key()
-        old = self.view.settings[f][k]
-        new = self.generate(f)
-        if old != new:
-            self.view._view.remove(f)
-            self.view.settings[f][k] = new
-            self.view._view.add(f)
-            self.view.sig_view_refresh.send(self.view)
-
-    def _key(self):
-        return "_order_%s" % id(self)
-
-    def __call__(self, f):
-        if f.id in self.view._store:
-            k = self._key()
-            s = self.view.settings[f]
-            if k in s:
-                return s[k]
-            val = self.generate(f)
-            s[k] = val
-            return val
-        else:
-            return self.generate(f)
-
-
-class OrderRequestStart(_OrderKey):
-    def generate(self, f: mitmproxy.flow.Flow) -> datetime.datetime:
-        return f.request.timestamp_start or 0
-
-
-class OrderRequestMethod(_OrderKey):
-    def generate(self, f: mitmproxy.flow.Flow) -> str:
-        return f.request.method
-
-
-class OrderRequestURL(_OrderKey):
-    def generate(self, f: mitmproxy.flow.Flow) -> str:
-        return f.request.url
-
-
-class OrderKeySize(_OrderKey):
-    def generate(self, f: mitmproxy.flow.Flow) -> int:
-        s = 0
-        if f.request.raw_content:
-            s += len(f.request.raw_content)
-        if f.response and f.response.raw_content:
-            s += len(f.response.raw_content)
-        return s
-
-
-matchall = flowfilter.parse(".")
-
-
-orders = [
-    ("t", "time"),
-    ("m", "method"),
-    ("u", "url"),
-    ("z", "size"),
-]
-
-
-class View(collections.Sequence):
-    def __init__(self):
-        super().__init__()
-        self._store = collections.OrderedDict()
-        self.filter = matchall
-        # Should we show only marked flows?
-        self.show_marked = False
-
-        self.default_order = OrderRequestStart(self)
-        self.orders = dict(
-            time = self.default_order,
-            method = OrderRequestMethod(self),
-            url = OrderRequestURL(self),
-            size = OrderKeySize(self),
-        )
-        self.order_key = self.default_order
-        self.order_reversed = False
-        self.focus_follow = False
-
-        self._view = sortedcontainers.SortedListWithKey(key = self.order_key)
-
-        # The sig_view* signals broadcast events that affect the view. That is,
-        # an update to a flow in the store but not in the view does not trigger
-        # a signal. All signals are called after the view has been updated.
-        self.sig_view_update = blinker.Signal()
-        self.sig_view_add = blinker.Signal()
-        self.sig_view_remove = blinker.Signal()
-        # Signals that the view should be refreshed completely
-        self.sig_view_refresh = blinker.Signal()
-
-        # The sig_store* signals broadcast events that affect the underlying
-        # store. If a flow is removed from just the view, sig_view_remove is
-        # triggered. If it is removed from the store while it is also in the
-        # view, both sig_store_remove and sig_view_remove are triggered.
-        self.sig_store_remove = blinker.Signal()
-        # Signals that the store should be refreshed completely
-        self.sig_store_refresh = blinker.Signal()
-
-        self.focus = Focus(self)
-        self.settings = Settings(self)
-
-    def store_count(self):
-        return len(self._store)
-
-    def inbounds(self, index: int) -> bool:
-        """
-            Is this 0 <= index < len(self)
-        """
-        return 0 <= index < len(self)
-
-    def _rev(self, idx: int) -> int:
-        """
-            Reverses an index, if needed
-        """
-        if self.order_reversed:
-            if idx < 0:
-                idx = -idx - 1
-            else:
-                idx = len(self._view) - idx - 1
-                if idx < 0:
-                    raise IndexError
-        return idx
-
-    def __len__(self):
-        return len(self._view)
-
-    def __getitem__(self, offset) -> mitmproxy.flow.Flow:
-        return self._view[self._rev(offset)]
-
-    # Reflect some methods to the efficient underlying implementation
-
-    def _bisect(self, f: mitmproxy.flow.Flow) -> int:
-        v = self._view.bisect_right(f)
-        return self._rev(v - 1) + 1
-
-    def index(self, f: mitmproxy.flow.Flow, start: int = 0, stop: typing.Optional[int] = None) -> int:
-        return self._rev(self._view.index(f, start, stop))
-
-    def __contains__(self, f: mitmproxy.flow.Flow) -> bool:
-        return self._view.__contains__(f)
-
-    def _order_key_name(self):
-        return "_order_%s" % id(self.order_key)
-
-    def _base_add(self, f):
-        self.settings[f][self._order_key_name()] = self.order_key(f)
-        self._view.add(f)
-
-    def _refilter(self):
-        self._view.clear()
-        for i in self._store.values():
-            if self.show_marked and not i.marked:
-                continue
-            if self.filter(i):
-                self._base_add(i)
-        self.sig_view_refresh.send(self)
-
-    # API
-    def toggle_marked(self):
-        self.show_marked = not self.show_marked
-        self._refilter()
-
-    def set_reversed(self, value: bool):
-        self.order_reversed = value
-        self.sig_view_refresh.send(self)
-
-    def set_order(self, order_key: typing.Callable):
-        """
-            Sets the current view order.
-        """
-        self.order_key = order_key
-        newview = sortedcontainers.SortedListWithKey(key=order_key)
-        newview.update(self._view)
-        self._view = newview
-
-    def set_filter(self, flt: typing.Optional[flowfilter.TFilter]):
-        """
-            Sets the current view filter.
-        """
-        self.filter = flt or matchall
-        self._refilter()
-
-    def clear(self):
-        """
-            Clears both the store and view.
-        """
-        self._store.clear()
-        self._view.clear()
-        self.sig_view_refresh.send(self)
-        self.sig_store_refresh.send(self)
-
-    def clear_not_marked(self):
-        """
-            Clears only the unmarked flows.
-        """
-        for flow in self._store.copy().values():
-            if not flow.marked:
-                self._store.pop(flow.id)
-
-        self._refilter()
-        self.sig_store_refresh.send(self)
-
-    def add(self, f: mitmproxy.flow.Flow) -> bool:
-        """
-            Adds a flow to the state. If the flow already exists, it is
-            ignored.
-        """
-        if f.id not in self._store:
-            self._store[f.id] = f
-            if self.filter(f):
-                self._base_add(f)
-                if self.focus_follow:
-                    self.focus.flow = f
-                self.sig_view_add.send(self, flow=f)
-
-    def remove(self, f: mitmproxy.flow.Flow):
-        """
-            Removes the flow from the underlying store and the view.
-        """
-        if f.id in self._store:
-            if f in self._view:
-                self._view.remove(f)
-                self.sig_view_remove.send(self, flow=f)
-            del self._store[f.id]
-            self.sig_store_remove.send(self, flow=f)
-
-    def update(self, f: mitmproxy.flow.Flow):
-        """
-            Updates a flow. If the flow is not in the state, it's ignored.
-        """
-        if f.id in self._store:
-            if self.filter(f):
-                if f not in self._view:
-                    self._base_add(f)
-                    if self.focus_follow:
-                        self.focus.flow = f
-                    self.sig_view_add.send(self, flow=f)
-                else:
-                    # This is a tad complicated. The sortedcontainers
-                    # implementation assumes that the order key is stable. If
-                    # it changes mid-way Very Bad Things happen. We detect when
-                    # this happens, and re-fresh the item.
-                    self.order_key.refresh(f)
-                    self.sig_view_update.send(self, flow=f)
-            else:
-                try:
-                    self._view.remove(f)
-                    self.sig_view_remove.send(self, flow=f)
-                except ValueError:
-                    # The value was not in the view
-                    pass
-
-    def get_by_id(self, flow_id: str) -> typing.Optional[mitmproxy.flow.Flow]:
-        """
-        Get flow with the given id from the store.
-        Returns None if the flow is not found.
-        """
-        return self._store.get(flow_id)
-
-    # Event handlers
-    def configure(self, opts, updated):
-        if "filter" in updated:
-            filt = None
-            if opts.filter:
-                filt = flowfilter.parse(opts.filter)
-                if not filt:
-                    raise exceptions.OptionsError(
-                        "Invalid interception filter: %s" % opts.filter
-                    )
-            self.set_filter(filt)
-        if "console_order" in updated:
-            if opts.console_order is None:
-                self.set_order(self.default_order)
-            else:
-                if opts.console_order not in self.orders:
-                    raise exceptions.OptionsError(
-                        "Unknown flow order: %s" % opts.console_order
-                    )
-                self.set_order(self.orders[opts.console_order])
-        if "console_order_reversed" in updated:
-            self.set_reversed(opts.console_order_reversed)
-        if "console_focus_follow" in updated:
-            self.focus_follow = opts.console_focus_follow
-
-    def request(self, f):
-        self.add(f)
-
-    def error(self, f):
-        self.update(f)
-
-    def response(self, f):
-        self.update(f)
-
-    def intercept(self, f):
-        self.update(f)
-
-    def resume(self, f):
-        self.update(f)
-
-    def kill(self, f):
-        self.update(f)
-
-
-class Focus:
-    """
-        Tracks a focus element within a View.
-    """
-    def __init__(self, v: View) -> None:
-        self.view = v
-        self._flow = None  # type: mitmproxy.flow.Flow
-        self.sig_change = blinker.Signal()
-        if len(self.view):
-            self.flow = self.view[0]
-        v.sig_view_add.connect(self._sig_view_add)
-        v.sig_view_remove.connect(self._sig_view_remove)
-        v.sig_view_refresh.connect(self._sig_view_refresh)
-
-    @property
-    def flow(self) -> typing.Optional[mitmproxy.flow.Flow]:
-        return self._flow
-
-    @flow.setter
-    def flow(self, f: typing.Optional[mitmproxy.flow.Flow]):
-        if f is not None and f not in self.view:
-            raise ValueError("Attempt to set focus to flow not in view")
-        self._flow = f
-        self.sig_change.send(self)
-
-    @property
-    def index(self) -> typing.Optional[int]:
-        if self.flow:
-            return self.view.index(self.flow)
-
-    @index.setter
-    def index(self, idx):
-        if idx < 0 or idx > len(self.view) - 1:
-            raise ValueError("Index out of view bounds")
-        self.flow = self.view[idx]
-
-    def _nearest(self, f, v):
-        return min(v._bisect(f), len(v) - 1)
-
-    def _sig_view_remove(self, view, flow):
-        if len(view) == 0:
-            self.flow = None
-        elif flow is self.flow:
-            self.flow = view[self._nearest(self.flow, view)]
-
-    def _sig_view_refresh(self, view):
-        if len(view) == 0:
-            self.flow = None
-        elif self.flow is None:
-            self.flow = view[0]
-        elif self.flow not in view:
-            self.flow = view[self._nearest(self.flow, view)]
-
-    def _sig_view_add(self, view, flow):
-        # We only have to act if we don't have a focus element
-        if not self.flow:
-            self.flow = flow
-
-
-class Settings(collections.Mapping):
-    def __init__(self, view: View) -> None:
-        self.view = view
-        self._values = {}  # type: typing.MutableMapping[str, mitmproxy.flow.Flow]
-        view.sig_store_remove.connect(self._sig_store_remove)
-        view.sig_store_refresh.connect(self._sig_store_refresh)
-
-    def __iter__(self) -> typing.Iterator:
-        return iter(self._values)
-
-    def __len__(self) -> int:
-        return len(self._values)
-
-    def __getitem__(self, f: mitmproxy.flow.Flow) -> dict:
-        if f.id not in self.view._store:
-            raise KeyError
-        return self._values.setdefault(f.id, {})
-
-    def _sig_store_remove(self, view, flow):
-        if flow.id in self._values:
-            del self._values[flow.id]
-
-    def _sig_store_refresh(self, view):
-        for fid in list(self._values.keys()):
-            if fid not in view._store:
-                del self._values[fid]

mitmproxy/addons/wsgiapp.py

@@ -1,38 +0,0 @@
-from mitmproxy import ctx
-from mitmproxy import exceptions
-
-from mitmproxy.net import wsgi
-from mitmproxy import version
-
-
-class WSGIApp:
-    """
-        An addon that hosts a WSGI app withing mitproxy, at a specified
-        hostname and port.
-    """
-    def __init__(self, app, host, port):
-        self.app, self.host, self.port = app, host, port
-
-    def serve(self, app, flow):
-        """
-            Serves app on flow, and prevents further handling of the flow.
-        """
-        app = wsgi.WSGIAdaptor(
-            app,
-            flow.request.pretty_host,
-            flow.request.port,
-            version.MITMPROXY
-        )
-        err = app.serve(
-            flow,
-            flow.client_conn.wfile,
-            **{"mitmproxy.master": ctx.master}
-        )
-        if err:
-            ctx.log.error("Error in wsgi app. %s" % err)
-            raise exceptions.AddonHalt()
-        flow.reply.kill()
-
-    def request(self, f):
-        if (f.request.pretty_host, f.request.port) == (self.host, self.port):
-            self.serve(self.app, f)

mitmproxy/builtins/__init__.py

@@ -0,0 +1,27 @@
+from __future__ import absolute_import, print_function, division
+
+from mitmproxy.builtins import anticache
+from mitmproxy.builtins import anticomp
+from mitmproxy.builtins import filestreamer
+from mitmproxy.builtins import stickyauth
+from mitmproxy.builtins import stickycookie
+from mitmproxy.builtins import script
+from mitmproxy.builtins import replace
+from mitmproxy.builtins import setheaders
+from mitmproxy.builtins import serverplayback
+from mitmproxy.builtins import clientplayback
+
+
+def default_addons():
+    return [
+        anticache.AntiCache(),
+        anticomp.AntiComp(),
+        stickyauth.StickyAuth(),
+        stickycookie.StickyCookie(),
+        script.ScriptLoader(),
+        filestreamer.FileStreamer(),
+        replace.Replace(),
+        setheaders.SetHeaders(),
+        serverplayback.ServerPlayback(),
+        clientplayback.ClientPlayback(),
+    ]

mitmproxy/builtins/anticache.py

@@ -1,3 +1,6 @@
+from __future__ import absolute_import, print_function, division
+
+
 class AntiCache:
     def __init__(self):
         self.enabled = False

mitmproxy/builtins/anticomp.py

@@ -1,3 +1,6 @@
+from __future__ import absolute_import, print_function, division
+
+
 class AntiComp:
     def __init__(self):
         self.enabled = False