Update certinstall.rst

Added LDAP Auth

.appveyor.yml

@@ -7,8 +7,9 @@ environment:
   matrix:
     - PYTHON: "C:\\Python35"
       TOXENV: "py35"
-    - PYTHON: "C:\\Python27"
-      TOXENV: "py27"
+    # TODO: ENABLE WHEN AVAILABLE
+    # - PYTHON: "C:\\Python36"
+    #   TOXENV: "py36"
 
   SNAPSHOT_HOST:
     secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
@@ -18,6 +19,8 @@ environment:
     secure: 6yBwmO5gv4vAwoFYII8qjQ==
   SNAPSHOT_PASS:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
+  RTOOL_KEY:
+    secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -25,21 +28,48 @@ install:
   - "pip install -U tox"
 
 test_script:
-  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v"
+  - ps: "tox -- --verbose --cov-report=term"
+  - ps: |
+      $Env:VERSION = $(python mitmproxy/version.py)
+      $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
+      tox -e wheel
+      tox -e rtool -- bdist
+
+  - ps: |
+      if(
+        ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
+        (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+      ) {
+        tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
+        if (!(Test-Path "C:\projects\mitmproxy\release\installbuilder-installer.exe")) {
+          "Download InstallBuilder..."
+          (New-Object System.Net.WebClient).DownloadFile(
+            "https://installbuilder.bitrock.com/installbuilder-enterprise-17.1.0-windows-installer.exe",
+            "C:\projects\mitmproxy\release\installbuilder-installer.exe"
+          )
+        }
+        Start-Process "C:\projects\mitmproxy\release\installbuilder-installer.exe" "--mode unattended --unattendedmodeui none" -Wait
+        & 'C:\Program Files (x86)\BitRock InstallBuilder Enterprise 17.1.0\bin\builder-cli.exe' `
+          build `
+          .\release\installbuilder\mitmproxy.xml `
+          windows `
+          --license .\release\installbuilder\license.xml `
+          --setvars project.version=$Env:VERSION `
+          --verbose
+      }
 
 deploy_script:
+  # we build binaries on every run, but we only upload them for master snapshots or tags.
   ps: |
     if(
       ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-      pip install -U virtualenv
-      .\dev.ps1
-      cmd /c "python -u .\release\rtool.py bdist 2>&1"
-      python -u .\release\rtool.py upload-snapshot --bdist --wheel
+      (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+    ) {
+      tox -e rtool -- upload-snapshot --bdist --wheel --installer
     }
 
 cache:
+  - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml
   - C:\Users\appveyor\AppData\Local\pip\cache
 
 notifications:

.env

@@ -1,6 +0,0 @@
-DIR="$( dirname "${BASH_SOURCE[0]}" )"
-ACTIVATE_DIR="$(if [ -f "$DIR/venv/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)"
-if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/venv/$ACTIVATE_DIR/activate" ]; then
-    echo "Activating mitmproxy virtualenv..."
-    source "$DIR/venv/$ACTIVATE_DIR/activate"
-fi

.gitattributes

@@ -1,2 +1,2 @@
-mitmproxy/web/static/**/* -diff
+mitmproxy/tools/web/static/**/* -diff linguist-vendored
 web/src/js/filt/filt.js -diff

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 MANIFEST
-*/tmp
+**/tmp
 /venv*
 *.py[cdo]
 *.swp
@@ -11,6 +11,7 @@ MANIFEST
 .cache/
 .tox*/
 build/
+mitmproxy/contrib/kaitaistruct/*.ksy
 
 # UI
 
@@ -19,3 +20,6 @@ bower_components
 *.map
 sslkeylogfile.log
 .tox/
+.python-version
+coverage.xml
+web/coverage/

.travis.yml

@@ -1,19 +1,12 @@
 sudo: false
 language: python
 
-addons:
-  apt:
-    sources:
-      # Debian sid currently holds OpenSSL 1.0.2
-      # change this with future releases!
-      - debian-sid
-    packages:
-      - libssl-dev
-
 env:
   global:
     - CI_DEPS=codecov>=2.0.5
     - CI_COMMANDS=codecov
+git:
+  depth: 10000
 
 matrix:
   fast_finish: true
@@ -25,26 +18,55 @@ matrix:
       language: generic
       env: TOXENV=py35 BDIST=1
     - python: 3.5
-      env: TOXENV=py35 BDIST=1
+      env: TOXENV=py35 OPENSSL_OLD
+      addons:
+        apt:
+          packages:
+            - libssl-dev
     - python: 3.5
-      env: TOXENV=py35 NO_ALPN=1
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 2.7
-      env: TOXENV=py27 NO_ALPN=1
+      env: TOXENV=py35 BDIST=1 OPENSSL_ALPN
+      addons:
+        apt:
+          sources:
+            # Debian sid currently holds OpenSSL 1.1.0
+            # change this with future releases!
+            - debian-sid
+          packages:
+            - libssl-dev
+    - python: 3.6
+      env: TOXENV=py36 OPENSSL_ALPN
+      addons:
+        apt:
+          sources:
+            # Debian sid currently holds OpenSSL 1.1.0
+            # change this with future releases!
+            - debian-sid
+          packages:
+            - libssl-dev
+    - python: 3.5
+      env: TOXENV=individual_coverage
     - python: 3.5
       env: TOXENV=docs
-  allow_failures:
-    - python: pypy
+    - language: node_js
+      node_js: "node"
+      before_install:
+          - curl -o- -L https://yarnpkg.com/install.sh | bash
+          - export PATH=$HOME/.yarn/bin:$PATH
+      install:
+          - cd web && yarn
+          - yarn global add codecov
+      script: npm test && codecov
+      cache:
+          yarn: true
+          directories:
+              - web/node_modules
 
 install:
   - |
     if [[ $TRAVIS_OS_NAME == "osx" ]]
     then
-      brew update || brew update # try again if it fails
-      brew upgrade
-      brew reinstall openssl
-      brew reinstall pyenv
+      brew update || brew update
+      brew outdated pyenv || brew upgrade pyenv
       eval "$(pyenv init -)"
       env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.2
       pyenv global 3.5.2
@@ -53,18 +75,21 @@ install:
     fi
   - pip install tox
 
-script: tox -- --cov netlib --cov mitmproxy --cov pathod -v
+script:
+  - tox -- --verbose --cov-report=term
+  - |
+    if [[ $BDIST == "1" ]]
+    then
+        git fetch --unshallow --tags
+        tox -e rtool -- bdist
+    fi
 
 after_success:
+  # we build binaries on every run, but we only upload them for master snapshots or tags.
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
-        git fetch --unshallow
-        ./dev.sh 3.5
-        source venv3.5/bin/activate
-        pip install -e ./release
-        python -u ./release/rtool.py bdist
-        python -u ./release/rtool.py upload-snapshot --bdist
+        tox -e rtool -- upload-snapshot --bdist
     fi
 
 notifications:
@@ -79,3 +104,4 @@ cache:
   directories:
     - $HOME/.pyenv
     - $HOME/.cache/pip
+    # - $HOME/build/mitmproxy/mitmproxy/.tox

CHANGELOG

@@ -1,3 +1,72 @@
+28 April 2017: mitmproxy 2.0.2
+
+    * Fix mitmweb's Content-Security-Policy to work with Chrome 58+
+
+    * HTTP/2: actually use header normalization from hyper-h2
+
+
+15 March 2017: mitmproxy 2.0.1
+
+    * bump cryptography dependency
+
+    * bump pyparsing dependency
+
+    * HTTP/2: use header normalization from hyper-h2
+
+
+21 February 2017: mitmproxy 2.0
+
+    * HTTP/2 is now enabled by default.
+
+    * Image ContentView: Parse images with Kaitai Struct (kaitai.io) instead of Pillow. 
+      This simplifies installation, reduces binary size, and allows parsing in pure Python.
+
+    * Web: Add missing flow filters.
+
+    * Add transparent proxy support for OpenBSD.
+
+    * Check the mitmproxy CA for expiration and warn the user to regenerate it if necessary.
+
+    * Testing: Tremendous improvements, enforced 100% coverage for large parts of the 
+      codebase, increased overall coverage.
+
+    * Enforce individual coverage: one source file -> one test file with 100% coverage.
+
+    * A myriad of other small improvements throughout the project.
+
+    * Numerous bugfixes.
+
+
+26 December 2016: mitmproxy 1.0
+
+    * All mitmproxy tools are now Python 3 only! We plan to support Python 3.5 and higher.
+
+    * Web-Based User Interface: Mitmproxy now offically has a web-based user interface
+      called mitmweb. We consider it stable for all features currently exposed 
+      in the UI, but it still misses a lot of mitmproxy’s options.
+      
+    * Windows Compatibility: With mitmweb, mitmproxy is now useable on Windows. 
+      We are also introducing an installer (kindly sponsored by BitRock) that 
+      simplifies setup.
+      
+    * Configuration: The config file format is now a single YAML file. In most cases,
+      converting to the new format should be trivial - please see the docs for
+      more information.
+
+    * Console: Significant UI improvements - including sorting of flows by
+      size, type and url, status bar improvements, much faster indentation for
+      HTTP views, and more.
+
+    * HTTP/2: Significant improvements, but is temporarily disabled by default
+      due to wide-spread protocol implementation errors on some large website
+
+    * WebSocket: The protocol implementation is now mature, and is enabled by
+      default. Complete UI support is coming in the next release. Hooks for 
+      message interception and manipulation are available.
+
+    * A myriad of other small improvements throughout the project.
+
+
 16 October 2016: mitmproxy 0.18
 
     * Python 3 Compatibility for mitmproxy and pathod (Shadab Zafar, GSoC 2016)

CONTRIBUTORS

@@ -1,6 +1,6 @@
-  2184	Aldo Cortesi
-  1745	Maximilian Hils
-   507	Thomas Kriechbaumer
+  2407	Aldo Cortesi
+  1873	Maximilian Hils
+   556	Thomas Kriechbaumer
    258	Shadab Zafar
     97	Jason
     83	Marcelo Glezer
@@ -11,85 +11,91 @@
     14	Pedro Worcel
     14	David Weinstein
     13	Thomas Roth
-    11	Stephen Altamirano
     11	Jake Drahos
+    11	Stephen Altamirano
     11	arjun23496
     11	Justus Wingert
-    10	Sandor Nemes
-    10	Zohar Lorberbaum
     10	András Veres-Szentkirályi
-    10	Chris Czub
+    10	Zohar Lorberbaum
     10	smill
+    10	Chris Czub
+    10	Sandor Nemes
+    10	Doug Freed
      9	ikoz
-     9	Kyle Morton
      9	Legend Tang
      9	Rouli
+     9	Kyle Morton
      8	Jason A. Novak
      8	Chandler Abraham
-     7	Alexis Hildebrandt
      7	Matthias Urlichs
      7	Brad Peabody
      7	dufferzafar
+     7	Alexis Hildebrandt
      6	Felix Yan
-     5	Tomaz Muraus
-     5	elitest
-     5	iroiro123
-     5	Sam Cleveland
-     5	Choongwoo Han
      5	Will Coster
-     4	root
-     4	Clemens Brunner
+     5	Sam Cleveland
+     5	iroiro123
+     5	elitest
+     5	Tomaz Muraus
+     5	Choongwoo Han
      4	Schamper
-     4	Valtteri Virtanen
-     4	Wade 524
      4	Youhei Sakurai
      4	Bryan Bishop
+     4	root
+     4	Valtteri Virtanen
+     4	Clemens Brunner
      4	Marc Liyanage
-     4	Michael J. Bazzinotti
+     4	Wade 524
+     4	chhsiao90
      4	yonder
-     3	Eli Shvartsman
-     3	Chris Neasbitt
-     3	Guillem Anguera
-     3	MatthewShao
+     4	Michael J. Bazzinotti
      3	Ryan Welton
-     3	smill@cuckoo.sh
-     3	Manish Kumar
-     3	Benjamin Lee
      3	Ryan Laughlin
-     3	Zack B
      3	Kyle Manna
+     3	Eli Shvartsman
+     3	Vincent Haupert
+     3	Manish Kumar
+     3	Zack B
+     3	MatthewShao
      3	redfast00
      3	requires.io
+     3	Guillem Anguera
+     3	smill@cuckoo.sh
+     3	Chris Neasbitt
+     3	Benjamin Lee
+     2	Steven Van Acker
+     2	Slobodan Mišković
+     2	Jim Lloyd
      2	isra17
      2	israel
-     2	Colin Bendell
+     2	Sean Coates
+     2	Sachin Kelkar
      2	jpkrause
-     2	Paul
+     2	Bennett Blodinger
      2	lilydjwg
      2	Michael Frister
-     2	依云
-     2	Jaime Soriano Pastor
-     2	Nick Badger
-     2	Rob Wills
-     2	Heikki Hannikainen
-     2	Vincent Haupert
-     2	strohu
-     2	Wade Catron
-     2	Krzysztof Bielicki
-     2	Sachin Kelkar
      2	Israel Nir
-     2	Anant
-     2	alts
-     2	Doug Freed
-     2	Niko Kommenda
-     2	Terry Long
-     2	Mark E. Haase
-     2	Steven Van Acker
-     2	Jim Lloyd
-     2	Bennett Blodinger
-     2	Sean Coates
      2	Cory Benfield
-     1	Sergey Chipiga
+     2	phackt
+     2	Anant
+     2	Jaime Soriano Pastor
+     2	Paul
+     2	Colin Bendell
+     2	依云
+     2	Heikki Hannikainen
+     2	Rob Wills
+     2	Niko Kommenda
+     2	Naveen Pai
+     2	strohu
+     2	alts
+     2	Yoginski
+     2	Mark E. Haase
+     2	Wade Catron
+     2	Terry Long
+     2	Krzysztof Bielicki
+     2	Nick Badger
+     1	Nicolas Esteves
+     1	Andrew Orr
      1	Andrey Plotnikov
      1	Andy Smith
      1	Angelo Agatino Nicolosi
@@ -97,6 +103,7 @@
      1	BSalita
      1	Ben Lerner
      1	Bradley Baetz
+     1	Brady Law
      1	Brett Randall
      1	Chris Hamant
      1	Christian Frichot
@@ -105,6 +112,7 @@
      1	David Shaw
      1	Doug Lethin
      1	Drake Caraker
+     1	Edgar Boda-Majer
      1	Eric Entzel
      1	Felix Wolfsteller
      1	FreeArtMan
@@ -128,22 +136,25 @@
      1	Mathieu Mitchell
      1	Michael Bisbjerg
      1	Mike C
+     1	Mike Fotinakis
      1	Mikhail Korobov
      1	Morton Fox
      1	Nick HS
      1	Nick Raptis
-     1	Nicolas Esteves
+     1	Aditya
      1	Oleksandr Sheremet
      1	Parth Ganatra
      1	Pritam Baral
+     1	Quentin Pradet
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
      1	Ryo Onodera
+     1	Sahil Chelaramani
      1	Sahn Lam
      1	Sanchit Sokhey
      1	Seppo Yli-Olli
-     1	Aditya
+     1	Sergey Chipiga
      1	Stefan Wärting
      1	Steve Phillips
      1	Steven Noble
@@ -158,10 +169,8 @@
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
      1	Wes Turner
-     1	Yoginski
      1	Yuangxuan Wang
      1	capt8bit
-     1	chhsiao90
      1	cle1000
      1	davidpshaw
      1	deployable
@@ -172,7 +181,6 @@
      1	meeee
      1	michaeljau
      1	peralta
-     1	phackt
      1	phil plante
      1	sentient07
      1	sethp-jive

MANIFEST.in

@@ -1,4 +1,3 @@
 graft mitmproxy
 graft pathod
-graft netlib
-recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
+recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

README.rst

@@ -3,14 +3,15 @@ mitmproxy
 
 |travis| |appveyor| |coverage| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects, as well as
-their shared networking library, **netlib**.
+This repository contains the **mitmproxy** and **pathod** projects.
 
 ``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
 interface.
 
 ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
+``mitmweb`` is a web-based interface for mitmproxy.
+
 ``pathoc`` and ``pathod`` are perverse HTTP client and server applications
 designed to let you craft almost any conceivable HTTP request, including ones
 that creatively violate the standards.
@@ -23,8 +24,7 @@ Documentation & Help
 General information, tutorials, and precompiled binaries can be found on the mitmproxy
 and pathod websites.
 
-|mitmproxy_site| |pathod_site|
-
+|mitmproxy_site|
 
 The latest documentation for mitmproxy is also available on ReadTheDocs.
 
@@ -37,7 +37,7 @@ each other solve problems, and come up with new ideas for the project.
 |mitmproxy_discourse|
 
 
-Join our developer chat on Slack if you would like to hack on mitmproxy itself.
+Join our developer chat on Slack if you would like to contribute to mitmproxy itself.
 
 |slack|
 
@@ -45,79 +45,67 @@ Join our developer chat on Slack if you would like to hack on mitmproxy itself.
 Installation
 ------------
 
-The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`_.
+The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`__.
 If you want to contribute changes, keep on reading.
 
+Contributing
+------------
 
-Hacking
--------
+As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project forward,
+please consider contributing in the following areas:
 
-To get started hacking on mitmproxy, make sure you have Python_ 3.5.x or above with
-virtualenv_ installed (you can find installation instructions for virtualenv
-`here <http://virtualenv.readthedocs.org/en/latest/>`_). Then do the following:
+- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users in our forums_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported.
+- **Code Contributions:** We actively mark issues that we consider are `good first contributions`_. If you intend to work on a larger contribution to the project, please come talk to us first.
 
-.. code-block:: text
+Development Setup
+-----------------
+
+To get started hacking on mitmproxy, please follow the `advanced installation`_ steps to install mitmproxy from source, but stop right before running ``pip3 install mitmproxy``. Instead, do the following:
+
+.. code-block:: bash
 
     git clone https://github.com/mitmproxy/mitmproxy.git
     cd mitmproxy
-    ./dev.sh  # powershell .\dev.ps1 on Windows
+    ./dev.sh  # "powershell .\dev.ps1" on Windows
 
 
-The *dev* script will create a virtualenv environment in a directory called
-"venv", and install all mandatory and optional dependencies into it. The
-primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
+The *dev* script will create a `virtualenv`_ environment in a directory called "venv"
+and install all mandatory and optional dependencies into it. The primary
+mitmproxy components - mitmproxy and pathod - are installed as
 "editable", so any changes to the source in the repository will be reflected
 live in the virtualenv.
 
-To confirm that you're up and running, activate the virtualenv, and run the
-mitmproxy test suite:
-
-.. code-block:: text
-
-    . venv/bin/activate  # venv\Scripts\activate on Windows
-    py.test
-
-Note that the main executables for the project - ``mitmdump``, ``mitmproxy``,
+The main executables for the project - ``mitmdump``, ``mitmproxy``,
 ``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the
 virtualenv. After activating the virtualenv, they will be on your $PATH, and
 you can run them like any other command:
 
-.. code-block:: text
+.. code-block:: bash
 
+    . venv/bin/activate  # "venv\Scripts\activate" on Windows
     mitmdump --version
 
-For convenience, the project includes an autoenv_ file (`.env`_) that
-auto-activates the virtualenv when you cd into the mitmproxy directory.
-
-
 Testing
 -------
 
 If you've followed the procedure above, you already have all the development
-requirements installed, and you can simply run the test suite:
+requirements installed, and you can run the full test suite (including tests for code style and documentation) with tox_:
 
-.. code-block:: text
+.. code-block:: bash
 
-    py.test
+    tox
+
+For speedier testing, we recommend you run `pytest`_ directly on individual test files or folders:
+
+.. code-block:: bash
+
+    cd test/mitmproxy/addons
+    pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py
+
+As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes.
 
 Please ensure that all patches are accompanied by matching changes in the test
-suite. The project tries to maintain 100% test coverage.
-
-You can also use `tox` to run a full suite of tests in Python 2.7 and 3.5,
-including a quick test to check documentation and code linting.
-
-The following tox environments are relevant for local testing:
-
-.. code-block:: text
-
-    tox -e py27  # runs all tests with Python 2.7
-    tox -e py35  # runs all tests with Python 3.5
-    tox -e docs  # runs a does-it-compile check on the documentation
-    tox -e lint  # runs the linter for coding style checks
-
-We support Python 2.7 and 3.5, so please make sure all tests pass in both
-environments. Running `tox` ensures all necessary tests are executed.
-
+suite. The project tries to maintain 100% test coverage and enforces this strictly for some parts of the codebase.
 
 Documentation
 -------------
@@ -126,7 +114,7 @@ The mitmproxy documentation is build using Sphinx_, which is installed
 automatically if you set up a development environment as described above. After
 installation, you can render the documentation like this:
 
-.. code-block:: text
+.. code-block:: bash
 
     cd docs
     make clean
@@ -136,8 +124,8 @@ installation, you can render the documentation like this:
 The last command invokes `sphinx-autobuild`_, which watches the Sphinx directory and rebuilds
 the documentation when a change is detected.
 
-Style
------
+Code Style
+----------
 
 Keeping to a consistent code style throughout the project makes it easier to
 contribute and collaborate. Please stick to the guidelines in
@@ -145,22 +133,19 @@ contribute and collaborate. Please stick to the guidelines in
 good reason not to.
 
 This is automatically enforced on every PR. If we detect a linting error, the
-PR checks will fail and block merging. We are using this command to check for style compliance:
+PR checks will fail and block merging. You can run our lint checks yourself
+with the following command:
 
-.. code-block:: text
+.. code-block:: bash
 
-    flake8 --jobs 8 --count mitmproxy netlib pathod examples test
+    tox -e lint
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg
     :target: https://mitmproxy.org/
     :alt: mitmproxy.org
 
-.. |pathod_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-pathod.net-blue.svg
-    :target: https://pathod.net/
-    :alt: pathod.net
-
-.. |mitmproxy_docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
+.. |mitmproxy_docs| image:: https://shields.mitmproxy.org/api/docs-latest-brightgreen.svg
     :target: http://docs.mitmproxy.org/en/latest/
     :alt: mitmproxy documentation
 
@@ -172,15 +157,15 @@ PR checks will fail and block merging. We are using this command to check for st
     :target: http://slack.mitmproxy.org/
     :alt: Slack Developer Chat
 
-.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=Travis%20build
+.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=travis%20ci
     :target: https://travis-ci.org/mitmproxy/mitmproxy
     :alt: Travis Build Status
 
-.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mhils/mitmproxy/master.svg?label=Appveyor%20build
+.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mhils/mitmproxy/master.svg?label=appveyor%20ci
     :target: https://ci.appveyor.com/project/mhils/mitmproxy
     :alt: Appveyor Build Status
 
-.. |coverage| image:: https://codecov.io/gh/mitmproxy/mitmproxy/branch/master/graph/badge.svg
+.. |coverage| image:: https://shields.mitmproxy.org/codecov/c/github/mitmproxy/mitmproxy/master.svg?label=codecov
     :target: https://codecov.io/gh/mitmproxy/mitmproxy
     :alt: Coverage Status
 
@@ -192,12 +177,13 @@ PR checks will fail and block merging. We are using this command to check for st
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Supported Python versions
 
-.. _Python: https://www.python.org/
-.. _virtualenv: http://virtualenv.readthedocs.org/en/latest/
-.. _autoenv: https://github.com/kennethreitz/autoenv
-.. _.env: https://github.com/mitmproxy/mitmproxy/blob/master/.env
+.. _`advanced installation`: http://docs.mitmproxy.org/en/latest/install.html#advanced-installation
+.. _virtualenv: https://virtualenv.pypa.io/
+.. _`pytest`: http://pytest.org/
+.. _tox: https://tox.readthedocs.io/
 .. _Sphinx: http://sphinx-doc.org/
 .. _sphinx-autobuild: https://pypi.python.org/pypi/sphinx-autobuild
-.. _issue_tracker: https://github.com/mitmproxy/mitmproxy/issues
 .. _PEP8: https://www.python.org/dev/peps/pep-0008
-.. _Google Style Guide: https://google.github.io/styleguide/pyguide.html
+.. _`Google Style Guide`: https://google.github.io/styleguide/pyguide.html
+.. _forums: https://discourse.mitmproxy.org/
+.. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3Agood-first-contribution

dev.ps1

@@ -1,15 +1,19 @@
 $ErrorActionPreference = "Stop"
-$VENV = ".\venv"
 
-virtualenv $VENV --always-copy
-& $VENV\Scripts\activate.ps1
+$pyver = python --version
+if($pyver -notmatch "3\.[5-9]") {
+    Write-Warning "Unexpected Python version, expected Python 3.5 or above: $pyver"
+}
+
+python -m venv .\venv --copies
+& .\venv\Scripts\activate.ps1
 
 python -m pip install --disable-pip-version-check -U pip
 cmd /c "pip install -r requirements.txt 2>&1"
 
 echo @"
 
-  * Created virtualenv environment in $VENV.
+  * Created virtualenv environment in .\venv.
   * Installed all dependencies into the virtualenv.
   * Activated virtualenv environment.
 

dev.sh

@@ -2,16 +2,14 @@
 set -e
 set -x
 
-PYVERSION=$1
-VENV="venv$1"
+echo "Creating dev environment in ./venv..."
 
-echo "Creating dev environment in $VENV using Python $PYVERSION"
-
-python$PYVERSION -m virtualenv "$VENV" --always-copy
-. "$VENV/bin/activate"
-pip$PYVERSION install -U pip setuptools
-pip$PYVERSION install -r requirements.txt
+python3 -m venv venv
+. venv/bin/activate
+pip3 install -U pip setuptools
+pip3 install -r requirements.txt
 
 echo ""
-echo "* Virtualenv created in $VENV and all dependencies installed."
-echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV/bin/activate\`"
+echo "  * Created virtualenv environment in ./venv."
+echo "  * Installed all dependencies into the virtualenv."
+echo "  * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`"

docs/certinstall.rst

@@ -24,6 +24,9 @@ something like this:
 Click on the relevant icon, follow the setup instructions for the platform
 you're on and you are good to go.
 
+For iOS version 10.3 or up, you need to make sure ``mitmproxy`` is enabled in 
+``Certificate Trust Settings``, you can check it by going to 
+``Settings > General > About > Certificate Trust Settings``.
 
 Installing the mitmproxy CA certificate manually
 ------------------------------------------------
@@ -40,7 +43,9 @@ start of mitmproxy.
 iOS
 ^^^
 
-http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
+See http://jasdev.me/intercepting-ios-traffic
+
+and https://web.archive.org/web/20150920082614/http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
 
 iOS Simulator
 ^^^^^^^^^^^^^
@@ -50,7 +55,7 @@ See https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore
 Java
 ^^^^
 
-See http://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html
+See https://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html
 
 Android/Android Simulator
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -60,7 +65,7 @@ See http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets
 Windows
 ^^^^^^^
 
-See http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7
+See https://web.archive.org/web/20160612045445/http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7
 
 Windows (automated)
 ^^^^^^^^^^^^^^^^^^^
@@ -77,7 +82,7 @@ See https://support.apple.com/kb/PH7297?locale=en_US
 Ubuntu/Debian
 ^^^^^^^^^^^^^
 
-See http://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861
+See https://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861
 
 Mozilla Firefox
 ^^^^^^^^^^^^^^^
@@ -87,7 +92,7 @@ See https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox
 Chrome on Linux
 ^^^^^^^^^^^^^^^
 
-See https://code.google.com/p/chromium/wiki/LinuxCertManagement
+See https://stackoverflow.com/a/15076602/198996
 
 
 The mitmproxy certificate authority
@@ -130,7 +135,7 @@ mitmproxy-ca-cert.cer Same file as .pem, but with an extension expected by some
 Using a custom certificate
 --------------------------
 
-You can use your own certificate by passing the ``--cert [domain=]path_to_certificate`` option to
+You can use your own (leaf) certificate by passing the ``--cert [domain=]path_to_certificate`` option to
 mitmproxy. Mitmproxy then uses the provided certificate for interception of the
 specified domain instead of generating a certificate signed by its own CA.
 
@@ -203,4 +208,4 @@ directory and uses this as the client cert.
 
 
 
-.. _Certificate Pinning: http://security.stackexchange.com/questions/29988/what-is-certificate-pinning/
+.. _Certificate Pinning: https://security.stackexchange.com/questions/29988/what-is-certificate-pinning/

docs/conf.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 
 sys.path.insert(0, os.path.abspath('..'))
-import netlib.version
+from mitmproxy import version as mversion
 
 
 extensions = [
@@ -47,9 +47,9 @@ author = u'The mitmproxy project'
 # built documents.
 #
 # The short X.Y version.
-version = netlib.version.VERSION
+version = mversion.VERSION
 # The full version, including alpha/beta/rc tags.
-release = netlib.version.VERSION
+release = mversion.VERSION
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -231,10 +231,7 @@ def linkcode_resolve(domain, info):
         _, line = inspect.getsourcelines(obj)
     except (TypeError, IOError):
         return None
-    if spath.rfind("netlib") > -1:
-        off = spath.rfind("netlib")
-        mpath = spath[off:]
-    elif spath.rfind("mitmproxy") > -1:
+    if spath.rfind("mitmproxy") > -1:
         off = spath.rfind("mitmproxy")
         mpath = spath[off:]
     else:

docs/config.rst

@@ -3,84 +3,11 @@
 Configuration
 =============
 
-Mitmproxy is configured through a set of files in the users ~/.mitmproxy
-directory.
+Mitmproxy is configured with a YAML_ file, located at
+``~/.mitmproxy/config.yaml``. We'll have complete documentation for all
+supported options in the next release in the meantime, please consult the
+source_ for a complete list of options and types.
 
-mitmproxy.conf
-    Settings for the :program:`mitmproxy`. This file can contain any options supported by
-    mitmproxy.
 
-mitmdump.conf
-    Settings for the :program:`mitmdump`. This file can contain any options supported by mitmdump.
-
-common.conf
-    Settings shared between all command-line tools. Settings in this file are over-ridden by those
-    in the tool-specific files. Only options shared by mitmproxy and mitmdump should be used in
-    this file.
-
-Syntax
-------
-
-Comments
-^^^^^^^^
-
-.. code-block:: none
-
-    # this is a comment
-    ; this is also a comment (.ini style)
-    --- and this is a comment too (yaml style)
-
-Key/Value pairs
-^^^^^^^^^^^^^^^
-
-- Keys and values are case-sensitive
-- Whitespace is ignored
-- Lists are comma-delimited, and enclosed in square brackets
-
-.. code-block:: none
-
-    name = value   # (.ini style)
-    name: value    # (yaml style)
-    --name value   # (command-line option style)
-
-    fruit = [apple, orange, lemon]
-    indexes = [1, 12, 35 , 40]
-
-Flags
-^^^^^
-
-These are boolean options that take no value but true/false.
-
-.. code-block:: none
-
-    name = true    # (.ini style)
-    name
-    --name 	 	   # (command-line option style)
-
-Options
--------
-
-The options available in the config files are precisely those available as
-command-line flags, with the key being the option's long name. To get a
-complete list of these, use the ``--help`` option on each of the tools. Be
-careful to only specify common options in the **common.conf** file -
-unsupported options in this file will be detected as an error on startup.
-
-Examples
---------
-
-common.conf
-^^^^^^^^^^^
-
-Note that ``--port`` is an option supported by all tools.
-
-.. code-block:: none
-
-    port = 8080
-
-mitmproxy.conf
-^^^^^^^^^^^^^^
-
-.. code-block:: none
-
-    palette = light
+.. _YAML: http://www.yaml.org/start.html
+.. _source: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/options.py

docs/dev/architecture.rst

@@ -1,14 +0,0 @@
-.. _architecture:
-
-Architecture
-============
-
-To give you a better understanding of how mitmproxy works, mitmproxy's
-high-level architecture is detailed in the following graphic:
-
-.. image:: ../schematics/architecture.png
-
-:download:`architecture.pdf <../schematics/architecture.pdf>`
-
-Please don't refrain from asking any further
-questions on the mailing list, the Slack channel or the GitHub issue tracker.

docs/dev/contributing.rst

@@ -0,0 +1,11 @@
+.. _contributing:
+
+Contributing
+============
+
+As an open source project, **mitmproxy** welcomes contributions of all forms. 
+
+Please head over to the README_ to get started! 😃
+
+
+.. _README: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst

docs/dev/testing.rst

@@ -1,47 +0,0 @@
-.. _testing:
-
-Testing
-=======
-
-All the mitmproxy projects strive to maintain 100% code coverage. In general,
-patches and pull requests will be declined unless they're accompanied by a
-suitable extension to the test suite.
-
-Our tests are written for the `py.test`_ or nose_ test frameworks.
-At the point where you send your pull request, a command like this:
-
->>> py.test --cov mitmproxy --cov netlib
-
-Should give output something like this:
-
-.. code-block:: none
-
-    > ---------- coverage: platform darwin, python 2.7.2-final-0 --
-    > Name                   Stmts   Miss  Cover   Missing
-    > ----------------------------------------------------
-    > mitmproxy/__init__         0      0   100%
-    > mitmproxy/app              4      0   100%
-    > mitmproxy/cmdline        100      0   100%
-    > mitmproxy/controller      69      0   100%
-    > mitmproxy/dump           150      0   100%
-    > mitmproxy/encoding        39      0   100%
-    > mitmproxy/flowfilter     201      0   100%
-    > mitmproxy/flow           891      0   100%
-    > mitmproxy/proxy          427      0   100%
-    > mitmproxy/script          27      0   100%
-    > mitmproxy/utils          133      0   100%
-    > mitmproxy/version          4      0   100%
-    > ----------------------------------------------------
-    > TOTAL                   2045      0   100%
-    > ----------------------------------------------------
-    > Ran 251 tests in 11.864s
-
-
-There are exceptions to the coverage requirement - for instance, much of the
-console interface code can't sensibly be unit tested. These portions are
-excluded from coverage analysis either in the **.coveragerc** file, or using
-**#pragma no-cover** directives. To keep our coverage analysis relevant, we use
-these measures as sparingly as possible.
-
-.. _nose: https://nose.readthedocs.org/en/latest/
-.. _py.test: https://pytest.org/

docs/features/anticache.rst

@@ -11,5 +11,5 @@ sure you capture an HTTP exchange in its totality. It's also often used during
 
 ================== ======================
 command-line       ``--anticache``
-mitmproxy shortcut :kbd:`o` then :kbd:`a`
+mitmproxy shortcut :kbd:`O` then :kbd:`a`
 ================== ======================

docs/features/passthrough.rst

@@ -23,7 +23,7 @@ How it works
 
 ================== ======================
 command-line       ``--ignore regex``
-mitmproxy shortcut :kbd:`o` then :kbd:`I`
+mitmproxy shortcut :kbd:`O` then :kbd:`I`
 ================== ======================
 
 

docs/features/replacements.rst

@@ -48,25 +48,24 @@ In practice, it's pretty common for the replacement literal to be long and
 complex. For instance, it might be an XSS exploit that weighs in at hundreds or
 thousands of characters. To cope with this, there's a variation of the
 replacement hook specifier that lets you load the replacement text from a file.
-So, you might start **mitmdump** as follows:
+To specify a file as replacement, prefix the file path with ``@``.
+You might start **mitmdump** as follows:
 
->>> mitmdump --replace-from-file :~q:foo:~/xss-exploit
+>>> mitmdump --replacements :~q:foo:@~/xss-exploit
 
 This will load the replacement text from the file ``~/xss-exploit``.
 
-Both the ``--replace`` and ``--replace-from-file`` flags can be passed multiple
-times.
+The ``--replacements`` flag can be passed multiple times.
 
 
 Interactively
 -------------
 
-The :kbd:`R` shortcut key in the mitmproxy options menu (:kbd:`o`) lets you add and edit
+The :kbd:`R` shortcut key in the mitmproxy options menu (:kbd:`O`) lets you add and edit
 replacement hooks using a built-in editor. The context-sensitive help (:kbd:`?`) has
 complete usage information.
 
 ================== =======================
-command-line       ``--replace``,
-                   ``--replace-from-file``
-mitmproxy shortcut :kbd:`o` then :kbd:`R`
+command-line       ``--replacements``
+mitmproxy shortcut :kbd:`O` then :kbd:`R`
 ================== =======================

docs/features/responsestreaming.rst

@@ -40,8 +40,8 @@ You can also use a script to customize exactly which responses are streamed.
 Responses that should be tagged for streaming by setting their ``.stream``
 attribute to ``True``:
 
-.. literalinclude:: ../../examples/stream.py
-   :caption: examples/stream.py
+.. literalinclude:: ../../examples/complex/stream.py
+   :caption: examples/complex/stream.py
    :language: python
 
 Implementation Details
@@ -59,8 +59,8 @@ Modifying streamed data
 If the ``.stream`` attribute is callable, ``.stream`` will wrap the generator that yields all
 chunks.
 
-.. literalinclude:: ../../examples/stream_modify.py
-   :caption: examples/stream_modify.py
+.. literalinclude:: ../../examples/complex/stream_modify.py
+   :caption: examples/complex/stream_modify.py
    :language: python
 
 .. seealso::

docs/features/reverseproxy.rst

@@ -31,7 +31,8 @@ Host Header
 
 In reverse proxy mode, mitmproxy automatically rewrites the Host header to match the
 upstream server. This allows mitmproxy to easily connect to existing endpoints on the
-open web (e.g. ``mitmproxy -R https://example.com``).
+open web (e.g. ``mitmproxy -R https://example.com``). You can disable this behaviour
+by passing ``--keep-host-header`` on the console.
 
 However, keep in mind that absolute URLs within the returned document or HTTP redirects will
 NOT be rewritten by mitmproxy. This means that if you click on a link for "http://example.com"
@@ -39,4 +40,4 @@ in the returned web page, you will be taken directly to that URL, bypassing mitm
 
 One possible way to address this is to modify the hosts file of your OS so that "example.com"
 resolves to your proxy's IP, and then access the proxy by going directly to example.com.
-Make sure that your proxy can still resolve the original IP, or specify an IP in mitmproxy.
+Make sure that your proxy can still resolve the original IP, or specify an IP in mitmproxy.

docs/features/serverreplay.rst

@@ -31,7 +31,20 @@ in the past at the time of replay, and vice versa. Cookie expiry times are
 updated in a similar way.
 
 You can turn off response refreshing using the ``--norefresh`` argument, or using
-the :kbd:`o` options shortcut within :program:`mitmproxy`.
+the :kbd:`O` options shortcut within :program:`mitmproxy`.
+
+
+Replaying a session recorded in Reverse-proxy Mode
+--------------------------------------------------
+
+If you have captured the session in reverse proxy mode, in order to replay it you 
+still have to specify the server URL, otherwise you may get the error: 
+'HTTP protocol error in client request: Invalid HTTP request form (expected authority or absolute...)'.
+
+During replay, when the client's requests match previously recorded requests, then the
+respective recorded responses are simply replayed  by mitmproxy. 
+Otherwise, the unmatched requests is forwarded to the upstream server.
+If forwarding is not desired, you can use the --kill (-k) switch to prevent that.
 
 ================== ===========
 command-line       ``-S path``

docs/features/setheaders.rst

@@ -15,5 +15,5 @@ Example: Set the **Host** header to "example.com" for all requests.
 
 ================== =======================
 command-line       ``--setheader PATTERN``
-mitmproxy shortcut :kbd:`o` then :kbd:`H`
+mitmproxy shortcut :kbd:`O` then :kbd:`H`
 ================== =======================

docs/features/sticky.rst

@@ -22,7 +22,7 @@ to interact with the secured resources.
 
 ================== ======================
 command-line       ``-t FILTER``
-mitmproxy shortcut :kbd:`o` then :kbd:`t`
+mitmproxy shortcut :kbd:`O` then :kbd:`t`
 ================== ======================
 
 
@@ -37,5 +37,5 @@ replay of HTTP Digest authentication.
 
 ================== ======================
 command-line       ``-u FILTER``
-mitmproxy shortcut :kbd:`o` then :kbd:`A`
+mitmproxy shortcut :kbd:`O` then :kbd:`A`
 ================== ======================

docs/features/tcpproxy.rst

@@ -3,7 +3,7 @@
 TCP Proxy
 =========
 
-WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt
+In case mitmproxy does not handle a specific protocol, you can exempt
 hostnames from processing, so that mitmproxy acts as a generic TCP forwarder.
 This feature is closely related to the :ref:`passthrough` functionality,
 but differs in two important aspects:
@@ -19,7 +19,7 @@ How it works
 
 ================== ======================
 command-line       ``--tcp HOST``
-mitmproxy shortcut :kbd:`o` then :kbd:`T`
+mitmproxy shortcut :kbd:`O` then :kbd:`T`
 ================== ======================
 
 For a detailed description how the hostname pattern works, please look at the :ref:`passthrough`

docs/features/upstreamcerts.rst

@@ -19,5 +19,5 @@ Upstream cert sniffing is on by default, and can optionally be turned off.
 
 ================== ======================
 command-line       ``--no-upstream-cert``
-mitmproxy shortcut :kbd:`o` then :kbd:`U`
+mitmproxy shortcut :kbd:`O` then :kbd:`U`
 ================== ======================

docs/howmitmproxy.rst

@@ -43,7 +43,7 @@ client connects to the proxy and makes a request that looks like this:
 
     CONNECT example.com:443 HTTP/1.1
 
-A conventional proxy can neither view nor manipulate an TLS-encrypted data
+A conventional proxy can neither view nor manipulate a TLS-encrypted data
 stream, so a CONNECT request simply asks the proxy to open a pipe between the
 client and server. The proxy here is just a facilitator - it blindly forwards
 data in both directions without knowing anything about the contents. The
@@ -63,7 +63,7 @@ exactly this attack, by allowing a trusted third-party to cryptographically sign
 a server's certificates to verify that they are legit. If this signature doesn't
 match or is from a non-trusted party, a secure client will simply drop the
 connection and refuse to proceed. Despite the many shortcomings of the CA system
-as it exists today, this is usually fatal to attempts to MITM an TLS connection
+as it exists today, this is usually fatal to attempts to MITM a TLS connection
 for analysis. Our answer to this conundrum is to become a trusted Certificate
 Authority ourselves. Mitmproxy includes a full CA implementation that generates
 interception certificates on the fly. To get the client to trust these
@@ -143,7 +143,7 @@ Lets put all of this together into the complete explicitly proxied HTTPS flow.
 2. Mitmproxy responds with a ``200 Connection Established``, as if it has set up the CONNECT pipe.
 3. The client believes it's talking to the remote server, and initiates the TLS connection.
    It uses SNI to indicate the hostname it is connecting to.
-4. Mitmproxy connects to the server, and establishes an TLS connection using the SNI hostname
+4. Mitmproxy connects to the server, and establishes a TLS connection using the SNI hostname
    indicated by the client.
 5. The server responds with the matching certificate, which contains the CN and SAN values
    needed to generate the interception certificate.
@@ -217,7 +217,7 @@ explicit HTTPS connections to establish the CN and SANs, and cope with SNI.
    destination was.
 3. The client believes it's talking to the remote server, and initiates the TLS connection.
    It uses SNI to indicate the hostname it is connecting to.
-4. Mitmproxy connects to the server, and establishes an TLS connection using the SNI hostname
+4. Mitmproxy connects to the server, and establishes a TLS connection using the SNI hostname
    indicated by the client.
 5. The server responds with the matching certificate, which contains the CN and SAN values
    needed to generate the interception certificate.

docs/index.rst

@@ -17,6 +17,7 @@
 
     mitmproxy
     mitmdump
+    mitmweb
     config
 
 .. toctree::
@@ -46,6 +47,7 @@
     transparent
     transparent/linux
     transparent/osx
+    transparent/openbsd
 
 .. toctree::
     :hidden:
@@ -77,10 +79,9 @@
 
 .. toctree::
     :hidden:
-    :caption: Hacking
+    :caption: Development
 
-    dev/architecture
-    dev/testing
+    dev/contributing
     dev/sslkeylogfile
 
 ..  Indices and tables

docs/install.rst

@@ -3,130 +3,164 @@
 Installation
 ============
 
-.. _install-ubuntu:
+Please follow the steps for your operating system.
 
-Installation On Ubuntu
-----------------------
+Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or
+:ref:`mitmweb` from a terminal.
 
-Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Ubuntu 14.04.
+
+.. _install-macos:
+
+Installation on macOS
+---------------------
+
+You can use Homebrew to install everything:
 
 .. code:: bash
 
-   sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev g++
-   sudo pip install mitmproxy  # or pip install --user mitmproxy
+    brew install mitmproxy
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+Or you can download the pre-built binary packages from our `releases`_.
 
-On **Ubuntu 12.04** (and other systems with an outdated version of pip),
-you may need to update pip using ``pip install -U pip`` before installing mitmproxy.
 
-Installation From Source (Ubuntu)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. _install-windows:
 
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, install the dependencies as you would for a regular
-mitmproxy installation (see :ref:`install-ubuntu`).
-Then see the Hacking_ section of the README on GitHub.
+Installation on Windows
+-----------------------
 
-.. _install-fedora:
+The recommended way to install mitmproxy on Windows is to use the installer
+provided at `mitmproxy.org`_. After installation, you'll find shortcuts for
+:ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu.
+Both executables are  added to your PATH and can be invoked from the command
+line.
 
-Installation On Fedora
-----------------------
+.. note::
+    Mitmproxy's console interface is not supported on Windows, but you can use
+    mitmweb (the web-based interface) and mitmdump.
 
-Fedora comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Fedora 23.
+.. _install-linux:
 
-.. code:: bash
+Installation on Linux
+---------------------
 
-   sudo dnf install -y python-pip python-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libpng-devel libjpeg-devel
-   sudo pip install mitmproxy  # or pip install --user mitmproxy
+The recommended way to run mitmproxy on Linux is to use the pre-built binaries
+provided at `releases`_.
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+Our pre-built binaries provide you with the latest version of mitmproxy, a
+self-contained Python 3.5 environment and a recent version of OpenSSL that
+supports HTTP/2. Of course, you can also install mitmproxy from source if you
+prefer that (see :ref:`install-advanced`).
+
+.. _install-advanced:
+
+Advanced Installation
+---------------------
+
+.. _install-docker:
+
+Docker Images
+^^^^^^^^^^^^^
+
+You can also use the official mitmproxy images from `DockerHub`_. That being
+said, our portable binaries are just as easy to install and even easier to use. 😊
 
 
 .. _install-arch:
 
-Installation On Arch Linux
---------------------------
+Installation on Arch Linux
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 mitmproxy has been added into the [community] repository. Use pacman to install it:
 
 >>> sudo pacman -S mitmproxy
 
 
+.. _install-source-ubuntu:
 
-Installation On Mac OS X
-------------------------
-
-The easiest way to get up and running on OSX is to download the pre-built binary packages from
-`mitmproxy.org`_.
-
-There are a few bits of customization you might want to do to make mitmproxy comfortable to use on
-OSX. The default color scheme is optimized for a dark background terminal, but you can select a
-palette for a light terminal background with the ``--palette`` option.
-You can use the OSX **open** program to create a simple and effective ``~/.mailcap`` file to view
-request and response bodies:
-
-.. code-block:: none
-
-    application/*; /usr/bin/open -Wn %s
-    audio/*; /usr/bin/open -Wn %s
-    image/*; /usr/bin/open -Wn %s
-    video/*; /usr/bin/open -Wn %s
-
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
-
-
-Installation From Source (Mac OS X)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, there are a few OS X specific things to keep in mind.
-
-- Make sure that XCode is installed from the App Store, and that the command-line tools have been
-  downloaded (XCode/Preferences/Downloads).
-- If you're running a Python interpreter installed with homebrew (or similar), you may have to
-  install some dependencies by hand.
-
-Then see the Hacking_ section of the README on GitHub.
-
-Installation On Windows
------------------------
-
-.. note::
-    Please note that mitmdump is the only component of mitmproxy that is supported on Windows at
-    the moment.
-
-    **There is no interactive user interface on Windows.**
-
-
-First, install the latest version of Python 2.7 from the `Python website`_.
-If you already have an older version of Python 2.7 installed, make sure to install pip_
-(pip is included in Python 2.7.9+ by default). If pip aborts with an error, make sure you are using the current version of pip.
-
->>> python -m pip install --upgrade pip
-
-Next, add Python and the Python Scripts directory to your **PATH** variable.
-You can do this easily by running the following in powershell:
-
->>> [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User")
-
-Now, you can install mitmproxy by running
-
->>> pip install mitmproxy
-
-Once the installation is complete, you can run :ref:`mitmdump` from a command prompt.
-
-Installation From Source (Windows)
+Installation from Source on Ubuntu
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, install Python as outlined above, then see the
-Hacking_ section of the README on GitHub.
+Ubuntu comes with Python but we need to install pip3, python3-dev and several
+libraries. This was tested on a fully patched installation of Ubuntu 16.04.
+
+.. code:: bash
+
+   sudo apt-get install python3-dev python3-pip libffi-dev libssl-dev
+   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
+
+On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install
+a newer version of Python. mitmproxy requires Python 3.5 or higher. Please take
+a look at pyenv_. Make sure to have an up-to-date version of pip by running
+``pip3 install -U pip``.
 
 
-.. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst#hacking
+.. _install-source-fedora:
+
+Installation from Source on Fedora
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Fedora comes with Python but we need to install pip3, python3-dev and several
+libraries. This was tested on a fully patched installation of Fedora 24.
+
+.. code:: bash
+
+   sudo dnf install make gcc redhat-rpm-config python3-devel python3-pip libffi-devel openssl-devel
+   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
+
+Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
+
+
+.. _install-source-opensuse:
+
+Installation from Source on openSUSE
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This was tested on a fully patched installation of openSUSE Tumbleweed.
+Please note that openSUSE Leap 42.2 only comes with Python 3.4.x, whereas mitmproxy requires Python 3.5 or above.
+You can check you Python version by running ``python3 --version``.
+
+.. code:: bash
+
+   sudo zypper install python3-pip python3-devel libffi-devel openssl-devel gcc-c++
+   sudo pip3 install mitmproxy
+
+
+.. _install-source-windows:
+
+Installation from Source on Windows
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. note::
+    Mitmproxy's console interface is not supported on Windows, but you can use
+    mitmweb (the web-based interface) and mitmdump.
+
+First, install the latest version of Python 3.5 or later from the `Python
+website`_. During installation, make sure to select `Add Python to PATH`.
+
+Mitmproxy has no other dependencies on Windows. You can now install mitmproxy by running
+
+.. code:: powershell
+
+    pip3 install mitmproxy
+
+
+
+.. _install-dev-version:
+
+Latest Development Version
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you would like to install mitmproxy directly from the master branch on GitHub
+or would like to get set up to contribute to the project, install the
+dependencies as you would for a regular installation from source. Then see the
+project's README_ on GitHub. You can check your system information
+by running: ``mitmproxy --version``
+
+
+.. _README: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst
+.. _releases: https://github.com/mitmproxy/mitmproxy/releases
 .. _mitmproxy.org: https://mitmproxy.org/
 .. _`Python website`: https://www.python.org/downloads/windows/
 .. _pip: https://pip.pypa.io/en/latest/installing.html
+.. _pyenv: https://github.com/yyuu/pyenv
+.. _DockerHub: https://hub.docker.com/r/mitmproxy/mitmproxy/

docs/introduction.rst

@@ -6,6 +6,8 @@ with a console interface.
 
 **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
+**mitmweb** is a web-based interface for mitmproxy.
+
 Documentation, tutorials and distribution packages can be found on the
 mitmproxy website: `mitmproxy.org <https://mitmproxy.org/>`_
 

docs/mitmproxy.rst

@@ -66,7 +66,7 @@ At the moment, the Grid Editor is used in four parts of mitmproxy:
   - Editing request or response headers (:kbd:`e` for edit, then :kbd:`h` for headers in flow view)
   - Editing a query string (:kbd:`e` for edit, then :kbd:`q` for query in flow view)
   - Editing a URL-encoded form (:kbd:`e` for edit, then :kbd:`f` for form in flow view)
-  - Editing replacement patterns (:kbd:`o` for options, then :kbd:`R` for Replacement Patterns)
+  - Editing replacement patterns (:kbd:`O` for options, then :kbd:`R` for Replacement Patterns)
 
 If there is is no data, an empty editor will be started to let you add some.
 Here is the editor showing the headers from a request:

docs/mitmweb.rst

@@ -0,0 +1,18 @@
+.. _mitmweb:
+.. program:: mitmweb
+
+mitmweb
+=======
+
+**mitmweb** is mitmproxy's web-based user interface that allows interactive
+examination and modification of HTTP traffic. Like mitmproxy, it differs from
+mitmdump in that all flows are kept in memory, which means that it's intended
+for taking and manipulating small-ish samples.
+
+.. warning::
+
+  Mitmweb is currently in beta. We consider it stable for all features currently
+  exposed in the UI, but it still misses a lot of mitmproxy's features.
+
+
+.. image:: screenshots/mitmweb.png

docs/modd.conf

@@ -1,6 +1,6 @@
 @build = ./_build
 
-** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py {
+** !_build/** ../mitmproxy/**/*.py {
     prep: sphinx-build -W -d @build/doctrees -b html . @build/html
     daemon: devd -m @build/html
 }

docs/pathod/language.rst

@@ -23,7 +23,7 @@ HTTP Request
         mode if the server responds correctly. Apart from that, websocket
         requests are just like any other, and all aspects of the request
         can be over-ridden.
-    * - h\:\ :ref:`VALUE`\ =\ :ref:`VALUE`\
+    * - h\ :ref:`VALUE`\ =\ :ref:`VALUE`\
       - Set a header.
     * - r
       - Set the **raw** flag on this response. Pathod will not calculate a
@@ -73,7 +73,7 @@ HTTP Response
     * - m\ :ref:`VALUE`
       - HTTP Reason message. Automatically chosen according to the response
         code if not specified. (HTTP/1 only)
-    * - h\:\ :ref:`VALUE`\ =\ :ref:`VALUE`\
+    * - h\ :ref:`VALUE`\ =\ :ref:`VALUE`\
       - Set a header.
     * - r
       - Set the **raw** flag on this response. Pathod will not calculate a

docs/pathod/test.rst

@@ -14,7 +14,7 @@ The canonical docs can be accessed using pydoc:
 >>> pydoc pathod.test
 
 The remainder of this page demonstrates some common interaction patterns using
-<a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are
+`Nose`_. These examples are
 also applicable with only minor modification to most commonly used Python testing
 engines.
 
@@ -33,3 +33,6 @@ One instance per test
 .. literalinclude:: ../../examples/pathod/test_setup.py
    :caption: examples/pathod/test_setup.py
    :language: python
+
+
+.. _Nose: https://nose.readthedocs.org/en/latest/

docs/scripting/api.rst

@@ -5,36 +5,38 @@ API
 ===
 
 - Errors
-    - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_
+    - `mitmproxy.flow.Error <#mitmproxy.flow.Error>`_
 - HTTP
-    - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_
-    - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_
-    - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_
+    - `mitmproxy.http.HTTPRequest <#mitmproxy.http.HTTPRequest>`_
+    - `mitmproxy.http.HTTPResponse <#mitmproxy.http.HTTPResponse>`_
+    - `mitmproxy.http.HTTPFlow <#mitmproxy.http.HTTPFlow>`_
 - Logging
-    - `mitmproxy.controller.Log <#mitmproxy.controller.Log>`_
-    - `mitmproxy.controller.LogEntry <#mitmproxy.controller.LogEntry>`_
+    - `mitmproxy.log.Log <#mitmproxy.controller.Log>`_
+    - `mitmproxy.log.LogEntry <#mitmproxy.controller.LogEntry>`_
 
 
 Errors
 ------
 
-.. autoclass:: mitmproxy.models.flow.Error
+.. autoclass:: mitmproxy.flow.Error
     :inherited-members:
 
 HTTP
 ----
 
-.. autoclass:: mitmproxy.models.http.HTTPRequest
+.. autoclass:: mitmproxy.http.HTTPRequest
     :inherited-members:
 
-.. autoclass:: mitmproxy.models.http.HTTPResponse
+.. autoclass:: mitmproxy.http.HTTPResponse
     :inherited-members:
 
-.. autoclass:: mitmproxy.models.http.HTTPFlow
+.. autoclass:: mitmproxy.http.HTTPFlow
     :inherited-members:
 
 Logging
 --------
 
-.. autoclass:: mitmproxy.controller.Log
+.. autoclass:: mitmproxy.log.Log
+    :inherited-members:
+.. autoclass:: mitmproxy.log.LogEntry
     :inherited-members:

docs/scripting/events.rst

@@ -56,7 +56,7 @@ Connection
           connection can correspond to multiple HTTP requests.
 
           *root_layer*
-            The root layer (see `mitmproxy.protocol` for an explanation what
+            The root layer (see `mitmproxy.proxy.protocol` for an explanation what
             the root layer is), provides transparent access to all attributes
             of the :py:class:`~mitmproxy.proxy.RootContext`. For example,
             ``root_layer.client_conn.address`` gives the remote address of the
@@ -98,6 +98,18 @@ HTTP Events
     :widths: 40 60
     :header-rows: 0
 
+    *   - .. py:function:: http_connect(flow)
+        - Called when we receive an HTTP CONNECT request. Setting a non 2xx
+          response on the flow will return the response to the client abort the
+          connection. CONNECT requests and responses do not generate the usual
+          HTTP handler events. CONNECT requests are only valid in regular and
+          upstream proxy modes.
+
+          *flow*
+            A ``models.HTTPFlow`` object. The flow is guaranteed to have
+            non-None ``request``  and ``requestheaders`` attributes.
+
+
     *   - .. py:function:: request(flow)
         - Called when a client request has been received.
 
@@ -146,21 +158,54 @@ HTTP Events
 WebSocket Events
 -----------------
 
+These events are called only after a connection made an HTTP upgrade with
+"101 Switching Protocols". No further HTTP-related events after the handshake
+are issued, only new WebSocket messages are called.
+
 .. list-table::
     :widths: 40 60
     :header-rows: 0
 
-    *   - .. py:function:: websockets_handshake(flow)
-
-        - Called when a client wants to establish a WebSockets connection. The
-          WebSockets-specific headers can be manipulated to manipulate the
+    *   - .. py:function:: websocket_handshake(flow)
+        - Called when a client wants to establish a WebSocket connection. The
+          WebSocket-specific headers can be manipulated to alter the
           handshake. The ``flow`` object is guaranteed to have a non-None
           ``request`` attribute.
 
           *flow*
-            The flow containing the HTTP websocket handshake request. The
+            The flow containing the HTTP WebSocket handshake request. The
             object is guaranteed to have a non-None ``request`` attribute.
 
+    *   - .. py:function:: websocket_start(flow)
+        - Called when WebSocket connection is established after a successful
+          handshake.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_message(flow)
+
+        - Called when a WebSocket message is received from the client or server. The
+          sender and receiver are identifiable. The most recent message will be
+          ``flow.messages[-1]``. The message is user-modifiable. Currently there are
+          two types of messages, corresponding to the BINARY and TEXT frame types.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_end(flow)
+        - Called when WebSocket connection ends.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
+    *   - .. py:function:: websocket_error(flow)
+        - Called when a WebSocket error occurs - e.g. the connection closing
+          unexpectedly.
+
+          *flow*
+            A ``models.WebSocketFlow`` object.
+
 
 TCP Events
 ----------
@@ -173,6 +218,22 @@ connections.
     :widths: 40 60
     :header-rows: 0
 
+
+    *   - .. py:function:: tcp_start(flow)
+        - Called when TCP streaming starts.
+
+          *flow*
+            A ``models.TCPFlow`` object.
+
+    *   - .. py:function:: tcp_message(flow)
+
+        - Called when a TCP payload is received from the client or server. The
+          sender and receiver are identifiable. The most recent message will be
+          ``flow.messages[-1]``. The message is user-modifiable.
+
+          *flow*
+            A ``models.TCPFlow`` object.
+
     *   - .. py:function:: tcp_end(flow)
         - Called when TCP streaming ends.
 
@@ -185,18 +246,3 @@ connections.
 
           *flow*
             A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_message(flow)
-
-        - Called a TCP payload is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_start(flow)
-        - Called when TCP streaming starts.
-
-          *flow*
-            A ``models.TCPFlow`` object.

docs/scripting/overview.rst

@@ -6,7 +6,7 @@ Overview
 Mitmproxy has a powerful scripting API that allows you to control almost any
 aspect of traffic being proxied. In fact, much of mitmproxy's own core
 functionality is implemented using the exact same API exposed to scripters (see
-:src:`mitmproxy/builtins`).
+:src:`mitmproxy/addons`).
 
 
 A simple example
@@ -17,8 +17,8 @@ appropriate points of mitmproxy's operation. Here's a complete mitmproxy script
 that adds a new header to every HTTP response before it is returned to the
 client:
 
-.. literalinclude:: ../../examples/add_header.py
-   :caption: :src:`examples/add_header.py`
+.. literalinclude:: ../../examples/simple/add_header.py
+   :caption: :src:`examples/simple/add_header.py`
    :language: python
 
 All events that deal with an HTTP request get an instance of `HTTPFlow
@@ -29,6 +29,12 @@ will be added to all responses passing through the proxy:
 >>> mitmdump -s add_header.py
 
 
+Examples
+--------
+
+A collection of addons that demonstrate popular features can be found at :src:`examples/simple`.
+
+
 Using classes
 -------------
 
@@ -42,8 +48,8 @@ called before anything else happens. You can replace the current script object
 by returning it from this handler. Here's how this looks when applied to the
 example above:
 
-.. literalinclude:: ../../examples/classes.py
-   :caption: :src:`examples/classes.py`
+.. literalinclude:: ../../examples/simple/add_header_class.py
+   :caption: :src:`examples/simple/add_header_class.py`
    :language: python
 
 So here, we're using a module-level script to "boot up" into a class instance.
@@ -54,39 +60,23 @@ and is replaced by the class instance.
 Handling arguments
 ------------------
 
-Scripts can handle their own command-line arguments, just like any other Python
-program. Let's build on the example above to do something slightly more
-sophisticated - replace one value with another in all responses. Mitmproxy's
-`HTTPRequest <api.html#mitmproxy.models.http.HTTPRequest>`_ and `HTTPResponse
-<api.html#mitmproxy.models.http.HTTPResponse>`_ objects have a handy `replace
-<api.html#mitmproxy.models.http.HTTPResponse.replace>`_ method that takes care
-of all the details for us.
 
-.. literalinclude:: ../../examples/arguments.py
-   :caption: :src:`examples/arguments.py`
-   :language: python
-
-We can now call this script on the command-line like this:
-
->>> mitmdump -dd -s "./arguments.py html faketml"
-
-Whenever a handler is called, mitpmroxy rewrites the script environment so that
-it sees its own arguments as if it was invoked from the command-line.
+FIXME
 
 
 Logging and the context
 -----------------------
 
 Scripts should not output straight to stderr or stdout. Instead, the `log
-<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` contexzt module
+<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` context module
 should be used, so that the mitmproxy host program can handle output
-appropriately. So, mitmdump can print colorised sript output to the terminal,
+appropriately. So, mitmdump can print colorised script output to the terminal,
 and mitmproxy console can place script output in the event buffer.
 
 Here's how this looks:
 
-.. literalinclude:: ../../examples/logging.py
-   :caption: :src:`examples/logging.py`
+.. literalinclude:: ../../examples/simple/log_events.py
+   :caption: :src:`examples/simple/log_events.py`
    :language: python
 
 The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
@@ -126,15 +116,32 @@ It's possible to implement a concurrent mechanism on top of the blocking
 framework, and mitmproxy includes a handy example of this that is fit for most
 purposes. You can use it as follows:
 
-.. literalinclude:: ../../examples/nonblocking.py
-   :caption: :src:`examples/nonblocking.py`
+.. literalinclude:: ../../examples/complex/nonblocking.py
+   :caption: :src:`examples/complex/nonblocking.py`
+   :language: python
+
+
+Testing
+-------
+
+Mitmproxy includes a number of helpers for testing addons. The
+``mitmproxy.test.taddons`` module contains a context helper that takes care of
+setting up and tearing down the addon event context. The
+``mitmproxy.test.tflow`` module contains helpers for quickly creating test
+flows. Pydoc is the canonical reference for these modules, and mitmproxy's own
+test suite is an excellent source of examples of usage. Here, for instance, is
+the mitmproxy unit tests for the `anticache` option, demonstrating a good
+cross-section of the test helpers:
+
+.. literalinclude:: ../../test/mitmproxy/addons/test_anticache.py
+   :caption: :src:`test/mitmproxy/addons/test_anticache.py`
    :language: python
 
 
 Developing scripts
 ------------------
 
-Mitmprxoy monitors scripts for modifications, and reloads them on change. When
+Mitmproxy monitors scripts for modifications, and reloads them on change. When
 this happens, the script is shut down (the `done <events.html#done>`_  event is
 called), and the new instance is started up as if the script had just been
 loaded (the `start <events.html#start>`_ and `configure

docs/transparent.rst

@@ -27,7 +27,7 @@ Fully transparent mode
 By default mitmproxy will use its own local ip address for its server-side connections.
 In case this isn't desired, the --spoof-source-address argument can be used to
 use the client's ip address for server-side connections. The following config is
-required for this mode to work:
+required for this mode to work::
 
     CLIENT_NET=192.168.1.0/24
     TABLE_ID=100
@@ -42,9 +42,9 @@ required for this mode to work:
 
 This mode does require root privileges though. There's a wrapper in the examples directory
 called 'mitmproxy_shim.c', which will enable you to use this mode with dropped priviliges.
-It can be used as follows:
+It can be used as follows::
 
-    gcc examples/mitmproxy_shim.c -o mitmproxy_shim -lcap
+    gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap
     sudo chown root:root mitmproxy_shim
     sudo chmod u+s mitmproxy_shim
     ./mitmproxy_shim $(which mitmproxy) -T --spoof-source-address

docs/transparent/openbsd.rst

@@ -0,0 +1,53 @@
+.. _openbsd:
+
+OpenBSD
+=======
+
+ 1. :ref:`Install the mitmproxy certificate on the test device <certinstall>`
+
+ 2. Enable IP forwarding:
+
+    >>> sudo sysctl -w net.inet.ip.forwarding=1
+
+ 3. Place the following two lines in **/etc/pf.conf**:
+
+    .. code-block:: none
+
+        mitm_if = "re2"
+        pass in quick proto tcp from $mitm_if to port { 80, 443 } divert-to 127.0.0.1 port 8080
+
+    These rules tell pf to divert all traffic from ``$mitm_if`` destined for
+    port 80 or 443 to the local mitmproxy instance running on port 8080. You
+    should replace ``$mitm_if`` value with the interface on which your test
+    device will appear.
+
+ 4. Configure pf with the rules:
+
+    >>> doas pfctl -f /etc/pf.conf
+
+ 5. And now enable it:
+
+    >>> doas pfctl -e
+
+ 6. Fire up mitmproxy. You probably want a command like this:
+
+    >>> mitmproxy -T --host
+
+    The ``-T`` flag turns on transparent mode, and the ``--host``
+    argument tells mitmproxy to use the value of the Host header for URL display.
+
+ 7. Finally, configure your test device to use the host on which mitmproxy is
+    running as the default gateway.
+
+.. note::
+
+    Note that the **divert-to** rules in the pf.conf given above only apply to
+    inbound traffic. **This means that they will NOT redirect traffic coming
+    from the box running pf itself.** We can't distinguish between an outbound
+    connection from a non-mitmproxy app, and an outbound connection from
+    mitmproxy itself - if you want to intercept your traffic, you should use an
+    external host to run mitmproxy. Nonetheless, pf is flexible to cater for a
+    range of creative possibilities, like intercepting traffic emanating from
+    VMs.  See the **pf.conf** man page for more.
+
+.. _pf: http://man.openbsd.org/OpenBSD-current/man5/pf.conf.5

docs/transparent/osx.rst

@@ -63,7 +63,7 @@ Note that this means we don't support transparent mode for earlier versions of O
     running pf itself.** We can't distinguish between an outbound connection from a
     non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
     want to intercept your OSX traffic, you should use an external host to run
-    mitmproxy. None the less, pf is flexible to cater for a range of creative
+    mitmproxy. Nonetheless, pf is flexible to cater for a range of creative
     possibilities, like intercepting traffic emanating from VMs.  See the
     **pf.conf** man page for more.
 

docs/tutorials/transparent-dhcp.rst

@@ -38,8 +38,14 @@ DHCP and TFTP) services to a small-scale network.
     **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default
     `[1] <https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/>`_. For our use case, this needs
     to be disabled by changing ``dns=dnsmasq`` to ``#dns=dnsmasq`` in
-    **/etc/NetworkManager/NetworkManager.conf** and running
-
+    **/etc/NetworkManager/NetworkManager.conf** and
+    
+    if on Ubuntu 16.04 or newer running:
+    
+    >>> sudo systemctl restart NetworkManager
+    
+    if on Ubuntu 12.04 or 14.04 running:
+    
     >>> sudo restart network-manager
 
     afterwards.
@@ -61,6 +67,12 @@ DHCP and TFTP) services to a small-scale network.
 
     Apply changes:
 
+    if on Ubuntu 16.04 or newer:
+    
+    >>> sudo systemctl restart dnsmasq
+    
+    if on Ubuntu 12.04 or 14.04:
+    
     >>> sudo service dnsmasq restart
 
     Your **proxied machine** in the internal virtual network should now receive an IP address via DHCP:
@@ -74,8 +86,8 @@ To redirect traffic to mitmproxy, we need to add two iptables rules:
 
 .. code-block:: none
 
-    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
-    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
+    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
+    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
 
 4. Run mitmproxy
 ----------------

examples/README

@@ -1,31 +0,0 @@
-Some inline scripts may require additional dependencies, which can be installed using
-`pip install mitmproxy[examples]`.
-
-
-# inline script examples
-add_header.py             Simple script that just adds a header to every request.
-change_upstream_proxy.py  Dynamically change the upstream proxy
-dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
-dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
-fail_with_500.py          Turn every response into an Internal Server Error.
-filt.py                   Use mitmproxy's filter expressions in your script.
-flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
-iframe_injector.py        Inject configurable iframe into pages.
-modify_form.py            Modify all form submissions to add a parameter.
-modify_querystring.py     Modify all query strings to add a parameters.
-modify_response_body.py   Replace arbitrary strings in all responses
-nonblocking.py            Demonstrate parallel processing with a blocking script.
-proxapp.py                How to embed a WSGI app in a mitmproxy server
-redirect_requests.py      Redirect requests or directly reply to them.
-stub.py                   Script stub with a method definition for every event.
-upsidedownternet.py       Rewrites traffic to turn images upside down.
-
-
-# mitmproxy examples
-flowbasic                 Basic use of mitmproxy as a library.
-stickycookies             An example of writing a custom proxy with mitmproxy.
-
-
-# misc
-read_dumpfile             Read a dumpfile generated by mitmproxy.
-mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/README.md

@@ -0,0 +1,15 @@
+# Mitmproxy Scripting API
+
+Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being 
+proxied. In fact, much of mitmproxy’s own core functionality is implemented using the exact same API 
+exposed to scripters (see [mitmproxy/addons](../mitmproxy/addons)).
+
+This directory contains some examples of the scripting API. We recommend to start with the
+ones in [simple/](./simple).
+
+|  :warning: | If you are browsing this on GitHub, make sure to select the git tag matching your mitmproxy version. |
+|------------|------------------------------------------------------------------------------------------------------|
+
+
+Some inline scripts may require additional dependencies, which can be installed using
+`pip install mitmproxy[examples]`.

examples/add_header.py

@@ -1,2 +0,0 @@
-def response(flow):
-    flow.response.headers["newheader"] = "foo"

examples/arguments.py

@@ -1,17 +0,0 @@
-import argparse
-
-
-class Replacer:
-    def __init__(self, src, dst):
-        self.src, self.dst = src, dst
-
-    def response(self, flow):
-        flow.response.replace(self.src, self.dst)
-
-
-def start():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("src", type=str)
-    parser.add_argument("dst", type=str)
-    args = parser.parse_args()
-    return Replacer(args.src, args.dst)

examples/classes.py

@@ -1,7 +0,0 @@
-class AddHeader:
-    def response(self, flow):
-        flow.response.headers["newheader"] = "foo"
-
-
-def start():
-    return AddHeader()

examples/complex/README.md

@@ -0,0 +1,18 @@
+## Complex Examples
+
+| Filename                 | Description                                                                                   |
+|:-------------------------|:----------------------------------------------------------------------------------------------|
+| change_upstream_proxy.py | Dynamically change the upstream proxy.                                                        |
+| dns_spoofing.py          | Use mitmproxy in a DNS spoofing scenario.                                                     |
+| dup_and_replay.py        | Duplicates each request, changes it, and then replays the modified request.                   |
+| full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. |
+| har_dump.py              | Dump flows as HAR files.                                                                      |
+| mitmproxywrapper.py      | Bracket mitmproxy run with proxy enable/disable on OS X                                       |
+| nonblocking.py           | Demonstrate parallel processing with a blocking script                                        |
+| remote_debug.py          | This script enables remote debugging of the mitmproxy _UI_ with PyCharm.                      |
+| sslstrip.py              | sslstrip-like funtionality implemented with mitmproxy                                         |
+| stream.py                | Enable streaming for all responses.                                                           |
+| stream_modify.py         | Modify a streamed response body.                                                              |
+| tcp_message.py           | Modify a raw TCP connection                                                                   |
+| tls_passthrough.py       | Use conditional TLS interception based on a user-defined strategy.                            |
+| xss_scanner.py           | Scan all visited webpages.                                                                    |

examples/complex/dns_spoofing.py

@@ -0,0 +1,57 @@
+"""
+This script makes it possible to use mitmproxy in scenarios where IP spoofing
+has been used to redirect connections to mitmproxy. The way this works is that
+we rely on either the TLS Server Name Indication (SNI) or the Host header of the
+HTTP request. Of course, this is not foolproof - if an HTTPS connection comes
+without SNI, we don't know the actual target and cannot construct a certificate
+that looks valid. Similarly, if there's no Host header or a spoofed Host header,
+we're out of luck as well. Using transparent mode is the better option most of
+the time.
+
+Usage:
+    mitmproxy
+        -p 443
+        -s dns_spoofing.py
+        # Used as the target location if neither SNI nor host header are present.
+        -R http://example.com/
+        # To avoid auto rewriting of host header by the reverse proxy target.
+        --keep-host-header
+    mitmdump
+        -p 80
+        -R http://localhost:443/
+
+    (Setting up a single proxy instance and using iptables to redirect to it
+    works as well)
+"""
+import re
+
+# This regex extracts splits the host header into host and port.
+# Handles the edge case of IPv6 addresses containing colons.
+# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
+parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
+
+
+class Rerouter:
+    def request(self, flow):
+        if flow.client_conn.ssl_established:
+            flow.request.scheme = "https"
+            sni = flow.client_conn.connection.get_servername()
+            port = 443
+        else:
+            flow.request.scheme = "http"
+            sni = None
+            port = 80
+
+        host_header = flow.request.host_header
+        m = parse_host_header.match(host_header)
+        if m:
+            host_header = m.group("host").strip("[]")
+            if m.group("port"):
+                port = int(m.group("port"))
+
+        flow.request.host_header = host_header
+        flow.request.host = sni or host_header
+        flow.request.port = port
+
+
+addons = [Rerouter()]

examples/complex/dup_and_replay.py

@@ -2,5 +2,7 @@ from mitmproxy import ctx
 
 
 def request(flow):
-    f = ctx.master.duplicate_flow(flow)
+    f = flow.copy()
+    ctx.master.view.add(f)
+    f.request.path = "/changed"
     ctx.master.replay_request(f, block=True)

examples/complex/har_dump.py

@@ -3,20 +3,20 @@ This inline script can be used to dump flows as HAR files.
 """
 
 
-import pprint
 import json
-import sys
 import base64
 import zlib
+import os
 
 from datetime import datetime
-import pytz
+from datetime import timezone
 
 import mitmproxy
 
-from netlib import version
-from netlib import strutils
-from netlib.http import cookies
+from mitmproxy import version
+from mitmproxy import ctx
+from mitmproxy.utils import strutils
+from mitmproxy.net.http import cookies
 
 HAR = {}
 
@@ -25,17 +25,13 @@ HAR = {}
 SERVERS_SEEN = set()
 
 
-def start():
-    """
-        Called once on script startup before any other events.
-    """
-    if len(sys.argv) != 2:
-        raise ValueError(
-            'Usage: -s "har_dump.py filename" '
-            '(- will output to stdout, filenames ending with .zhar '
-            'will result in compressed har)'
-        )
+def load(l):
+    l.add_option(
+        "hardump", str, "", "HAR dump path.",
+    )
 
+
+def configure(updated):
     HAR.update({
         "log": {
             "version": "1.2",
@@ -89,7 +85,7 @@ def response(flow):
     # Timings set to -1 will be ignored as per spec.
     full_time = sum(v for v in timings.values() if v > -1)
 
-    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))
+    started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, timezone.utc).isoformat()
 
     # Response body size and encoding
     response_body_size = len(flow.response.raw_content)
@@ -128,23 +124,26 @@ def response(flow):
         "timings": timings,
     }
 
-    # Store binay data as base64
+    # Store binary data as base64
     if strutils.is_mostly_bin(flow.response.content):
-        b64 = base64.b64encode(flow.response.content)
-        entry["response"]["content"]["text"] = b64.decode('ascii')
+        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
         entry["response"]["content"]["encoding"] = "base64"
     else:
-        entry["response"]["content"]["text"] = flow.response.text
+        entry["response"]["content"]["text"] = flow.response.get_text(strict=False)
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
+        params = [
+            {"name": a, "value": b}
+            for a, b in flow.request.urlencoded_form.items(multi=True)
+        ]
         entry["request"]["postData"] = {
-            "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
-            "text": flow.request.content,
-            "params": name_value(flow.request.urlencoded_form)
+            "mimeType": flow.request.headers.get("Content-Type", ""),
+            "text": flow.request.get_text(strict=False),
+            "params": params
         }
 
-    if flow.server_conn:
-        entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
+    if flow.server_conn.connected():
+        entry["serverIPAddress"] = str(flow.server_conn.ip_address[0])
 
     HAR["log"]["entries"].append(entry)
 
@@ -153,24 +152,20 @@ def done():
     """
         Called once on script shutdown, after any other events.
     """
-    dump_file = sys.argv[1]
+    if ctx.options.hardump:
+        json_dump = json.dumps(HAR, indent=2)  # type: str
 
-    if dump_file == '-':
-        mitmproxy.ctx.log(pprint.pformat(HAR))
-    else:
-        json_dump = json.dumps(HAR, indent=2)
+        if ctx.options.hardump == '-':
+            mitmproxy.ctx.log(json_dump)
+        else:
+            raw = json_dump.encode()  # type: bytes
+            if ctx.options.hardump.endswith('.zhar'):
+                raw = zlib.compress(raw, 9)
 
-        if dump_file.endswith('.zhar'):
-            json_dump = zlib.compress(json_dump, 9)
+            with open(os.path.expanduser(ctx.options.hardump), "wb") as f:
+                f.write(raw)
 
-        with open(dump_file, "w") as f:
-            f.write(json_dump)
-
-        mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
-
-
-def format_datetime(dt):
-    return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat()
+            mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
 
 
 def format_cookies(cookie_list):
@@ -194,7 +189,7 @@ def format_cookies(cookie_list):
         # Expiration time needs to be formatted
         expire_ts = cookies.get_expiration_ts(attrs)
         if expire_ts is not None:
-            cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts))
+            cookie_har["expires"] = datetime.fromtimestamp(expire_ts, timezone.utc).isoformat()
 
         rv.append(cookie_har)
 
@@ -206,7 +201,7 @@ def format_request_cookies(fields):
 
 
 def format_response_cookies(fields):
-    return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)
+    return format_cookies((c[0], c[1][0], c[1][1]) for c in fields)
 
 
 def name_value(obj):

examples/complex/mitmproxywrapper.py

@@ -15,7 +15,7 @@ import os
 import sys
 
 
-class Wrapper(object):
+class Wrapper:
     def __init__(self, port, extra_arguments=None):
         self.port = port
         self.extra_arguments = extra_arguments

examples/complex/nonblocking.py

@@ -0,0 +1,11 @@
+import time
+
+from mitmproxy.script import concurrent
+
+
+@concurrent  # Remove this and see what happens
+def request(flow):
+    # You don't want to use mitmproxy.ctx from a different thread
+    print("handle request: %s%s" % (flow.request.host, flow.request.path))
+    time.sleep(5)
+    print("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/complex/remote_debug.py

@@ -14,6 +14,6 @@ Usage:
 """
 
 
-def start():
+def load(l):
     import pydevd
     pydevd.settrace("localhost", port=5678, stdoutToServer=True, stderrToServer=True)

examples/complex/sslstrip.py

@@ -1,5 +1,9 @@
+"""
+This script implements an sslstrip-like attack based on mitmproxy.
+https://moxie.org/software/sslstrip/
+"""
 import re
-from six.moves import urllib
+import urllib
 
 # set of SSL/TLS capable hosts
 secure_hosts = set()
@@ -17,13 +21,18 @@ def request(flow):
         flow.request.scheme = 'https'
         flow.request.port = 443
 
+        # We need to update the request destination to whatever is specified in the host header:
+        # Having no TLS Server Name Indication from the client and just an IP address as request.host
+        # in transparent mode, TLS server name certificate validation would fail.
+        flow.request.host = flow.request.pretty_host
+
 
 def response(flow):
     flow.response.headers.pop('Strict-Transport-Security', None)
     flow.response.headers.pop('Public-Key-Pins', None)
 
     # strip links in response body
-    flow.response.content = flow.response.content.replace('https://', 'http://')
+    flow.response.content = flow.response.content.replace(b'https://', b'http://')
 
     # strip meta tag upgrade-insecure-requests in response body
     csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>'

examples/complex/stream.py

@@ -1,5 +1,6 @@
 def responseheaders(flow):
     """
     Enables streaming for all responses.
+    This is equivalent to passing `--stream 0` to mitmproxy.
     """
     flow.response.stream = True

examples/complex/tcp_message.py

@@ -8,7 +8,7 @@ tcp_message Inline Script Hook API Demonstration
 example cmdline invocation:
 mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py
 """
-from netlib import strutils
+from mitmproxy.utils import strutils
 
 
 def tcp_message(tcp_msg):

examples/complex/tls_passthrough.py

@@ -20,16 +20,15 @@ Example:
 
 Authors: Maximilian Hils, Matthew Tuusberg
 """
-from __future__ import absolute_import, print_function, division
 import collections
 import random
 
-import sys
 from enum import Enum
 
 import mitmproxy
+from mitmproxy import ctx
 from mitmproxy.exceptions import TlsProtocolException
-from mitmproxy.protocol import TlsLayer, RawTCPLayer
+from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
 
 
 class InterceptionResult(Enum):
@@ -38,7 +37,7 @@ class InterceptionResult(Enum):
     skipped = None
 
 
-class _TlsStrategy(object):
+class _TlsStrategy:
     """
     Abstract base class for interception strategies.
     """
@@ -113,10 +112,16 @@ class TlsFeedback(TlsLayer):
 tls_strategy = None
 
 
-def start():
+def load(l):
+    l.add_option(
+        "tlsstrat", int, 0, "TLS passthrough strategy (0-100)",
+    )
+
+
+def configure(updated):
     global tls_strategy
-    if len(sys.argv) == 2:
-        tls_strategy = ProbabilisticStrategy(float(sys.argv[1]))
+    if ctx.options.tlsstrat > 0:
+        tls_strategy = ProbabilisticStrategy(float(ctx.options.tlsstrat) / 100.0)
     else:
         tls_strategy = ConservativeStrategy()
 

examples/complex/xss_scanner.py

@@ -0,0 +1,407 @@
+"""
+
+ __   __ _____ _____     _____
+ \ \ / // ____/ ____|   / ____|
+  \ V /| (___| (___    | (___   ___ __ _ _ __  _ __   ___ _ __
+   > <  \___ \\___ \    \___ \ / __/ _` | '_ \| '_ \ / _ \ '__|
+  / . \ ____) |___) |   ____) | (_| (_| | | | | | | |  __/ |
+ /_/ \_\_____/_____/   |_____/ \___\__,_|_| |_|_| |_|\___|_|
+
+
+This script automatically scans all visited webpages for XSS and SQLi vulnerabilities.
+
+Usage: mitmproxy -s xss_scanner.py
+
+This script scans for vulnerabilities by injecting a fuzzing payload (see PAYLOAD below) into 4 different places
+and examining the HTML to look for XSS and SQLi injection vulnerabilities. The XSS scanning functionality works by
+looking to see whether it is possible to inject HTML based off of of where the payload appears in the page and what
+characters are escaped. In addition, it also looks for any script tags that load javascript from unclaimed domains.
+The SQLi scanning functionality works by using regular expressions to look for errors from a number of different
+common databases. Since it is only looking for errors, it will not find blind SQLi vulnerabilities.
+
+The 4 places it injects the payload into are:
+1. URLs         (e.g. https://example.com/ -> https://example.com/PAYLOAD/)
+2. Queries      (e.g. https://example.com/index.html?a=b -> https://example.com/index.html?a=PAYLOAD)
+3. Referers     (e.g. The referer changes from https://example.com to PAYLOAD)
+4. User Agents  (e.g. The UA changes from Chrome to PAYLOAD)
+
+Reports from this script show up in the event log (viewable by pressing e) and formatted like:
+
+===== XSS Found ====
+XSS URL: http://daviddworken.com/vulnerableUA.php
+Injection Point: User Agent
+Suggested Exploit: <script>alert(0)</script>
+Line: 1029zxcs'd"ao<ac>so[sb]po(pc)se;sl/bsl\eq=3847asd
+
+"""
+
+from mitmproxy import ctx
+from socket import gaierror, gethostbyname
+from urllib.parse import urlparse
+import requests
+import re
+from html.parser import HTMLParser
+from mitmproxy import http
+from typing import Dict, Union, Tuple, Optional, List, NamedTuple
+
+# The actual payload is put between a frontWall and a backWall to make it easy
+# to locate the payload with regular expressions
+FRONT_WALL = b"1029zxc"
+BACK_WALL = b"3847asd"
+PAYLOAD = b"""s'd"ao<ac>so[sb]po(pc)se;sl/bsl\\eq="""
+FULL_PAYLOAD = FRONT_WALL + PAYLOAD + BACK_WALL
+
+# A XSSData is a named tuple with the following fields:
+#   - url -> str
+#   - injection_point -> str
+#   - exploit -> str
+#   - line -> str
+XSSData = NamedTuple('XSSData', [('url', str),
+                                 ('injection_point', str),
+                                 ('exploit', str),
+                                 ('line', str)])
+
+# A SQLiData is named tuple with the following fields:
+#   - url -> str
+#   - injection_point -> str
+#   - regex -> str
+#   - dbms -> str
+SQLiData = NamedTuple('SQLiData', [('url', str),
+                                   ('injection_point', str),
+                                   ('regex', str),
+                                   ('dbms', str)])
+
+
+VulnData = Tuple[Optional[XSSData], Optional[SQLiData]]
+Cookies = Dict[str, str]
+
+
+def get_cookies(flow: http.HTTPFlow) -> Cookies:
+    """ Return a dict going from cookie names to cookie values
+          - Note that it includes both the cookies sent in the original request and
+            the cookies sent by the server """
+    return {name: value for name, value in flow.request.cookies.fields}
+
+
+def find_unclaimed_URLs(body: Union[str, bytes], requestUrl: bytes) -> None:
+    """ Look for unclaimed URLs in script tags and log them if found"""
+    class ScriptURLExtractor(HTMLParser):
+        script_URLs = []
+
+        def handle_starttag(self, tag, attrs):
+            if tag == "script" and "src" in [name for name, value in attrs]:
+                for name, value in attrs:
+                    if name == "src":
+                        self.script_URLs.append(value)
+
+    parser = ScriptURLExtractor()
+    try:
+        parser.feed(body)
+    except TypeError:
+        parser.feed(body.decode('utf-8'))
+    for url in parser.script_URLs:
+        parser = urlparse(url)
+        domain = parser.netloc
+        try:
+            gethostbyname(domain)
+        except gaierror:
+            ctx.log.error("XSS found in %s due to unclaimed URL \"%s\" in script tag." % (requestUrl, url))
+
+
+def test_end_of_URL_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData:
+    """ Test the given URL for XSS via injection onto the end of the URL and
+        log the XSS if found """
+    parsed_URL = urlparse(request_URL)
+    path = parsed_URL.path
+    if path != "" and path[-1] != "/":  # ensure the path ends in a /
+        path += "/"
+    path += FULL_PAYLOAD.decode('utf-8')  # the path must be a string while the payload is bytes
+    url = parsed_URL._replace(path=path).geturl()
+    body = requests.get(url, cookies=cookies).text.lower()
+    xss_info = get_XSS_data(body, url, "End of URL")
+    sqli_info = get_SQLi_data(body, original_body, url, "End of URL")
+    return xss_info, sqli_info
+
+
+def test_referer_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData:
+    """ Test the given URL for XSS via injection into the referer and
+        log the XSS if found """
+    body = requests.get(request_URL, headers={'referer': FULL_PAYLOAD}, cookies=cookies).text.lower()
+    xss_info = get_XSS_data(body, request_URL, "Referer")
+    sqli_info = get_SQLi_data(body, original_body, request_URL, "Referer")
+    return xss_info, sqli_info
+
+
+def test_user_agent_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData:
+    """ Test the given URL for XSS via injection into the user agent and
+        log the XSS if found """
+    body = requests.get(request_URL, headers={'User-Agent': FULL_PAYLOAD}, cookies=cookies).text.lower()
+    xss_info = get_XSS_data(body, request_URL, "User Agent")
+    sqli_info = get_SQLi_data(body, original_body, request_URL, "User Agent")
+    return xss_info, sqli_info
+
+
+def test_query_injection(original_body: str, request_URL: str, cookies: Cookies):
+    """ Test the given URL for XSS via injection into URL queries and
+        log the XSS if found """
+    parsed_URL = urlparse(request_URL)
+    query_string = parsed_URL.query
+    # queries is a list of parameters where each parameter is set to the payload
+    queries = [query.split("=")[0] + "=" + FULL_PAYLOAD.decode('utf-8') for query in query_string.split("&")]
+    new_query_string = "&".join(queries)
+    new_URL = parsed_URL._replace(query=new_query_string).geturl()
+    body = requests.get(new_URL, cookies=cookies).text.lower()
+    xss_info = get_XSS_data(body, new_URL, "Query")
+    sqli_info = get_SQLi_data(body, original_body, new_URL, "Query")
+    return xss_info, sqli_info
+
+
+def log_XSS_data(xss_info: Optional[XSSData]) -> None:
+    """ Log information about the given XSS to mitmproxy """
+    # If it is None, then there is no info to log
+    if not xss_info:
+        return
+    ctx.log.error("===== XSS Found ====")
+    ctx.log.error("XSS URL: %s" % xss_info.url)
+    ctx.log.error("Injection Point: %s" % xss_info.injection_point)
+    ctx.log.error("Suggested Exploit: %s" % xss_info.exploit)
+    ctx.log.error("Line: %s" % xss_info.line)
+
+
+def log_SQLi_data(sqli_info: Optional[SQLiData]) -> None:
+    """ Log information about the given SQLi to mitmproxy """
+    if not sqli_info:
+        return
+    ctx.log.error("===== SQLi Found =====")
+    ctx.log.error("SQLi URL: %s" % sqli_info.url.decode('utf-8'))
+    ctx.log.error("Injection Point: %s" % sqli_info.injection_point.decode('utf-8'))
+    ctx.log.error("Regex used: %s" % sqli_info.regex.decode('utf-8'))
+    ctx.log.error("Suspected DBMS: %s" % sqli_info.dbms.decode('utf-8'))
+
+
+def get_SQLi_data(new_body: str, original_body: str, request_URL: str, injection_point: str) -> Optional[SQLiData]:
+    """ Return a SQLiDict if there is a SQLi otherwise return None
+        String String URL String -> (SQLiDict or None) """
+    # Regexes taken from Damn Small SQLi Scanner: https://github.com/stamparm/DSSS/blob/master/dsss.py#L17
+    DBMS_ERRORS = {
+        "MySQL": (r"SQL syntax.*MySQL", r"Warning.*mysql_.*", r"valid MySQL result", r"MySqlClient\."),
+        "PostgreSQL": (r"PostgreSQL.*ERROR", r"Warning.*\Wpg_.*", r"valid PostgreSQL result", r"Npgsql\."),
+        "Microsoft SQL Server": (r"Driver.* SQL[\-\_\ ]*Server", r"OLE DB.* SQL Server", r"(\W|\A)SQL Server.*Driver",
+                                 r"Warning.*mssql_.*", r"(\W|\A)SQL Server.*[0-9a-fA-F]{8}",
+                                 r"(?s)Exception.*\WSystem\.Data\.SqlClient\.", r"(?s)Exception.*\WRoadhouse\.Cms\."),
+        "Microsoft Access": (r"Microsoft Access Driver", r"JET Database Engine", r"Access Database Engine"),
+        "Oracle": (r"\bORA-[0-9][0-9][0-9][0-9]", r"Oracle error", r"Oracle.*Driver", r"Warning.*\Woci_.*", r"Warning.*\Wora_.*"),
+        "IBM DB2": (r"CLI Driver.*DB2", r"DB2 SQL error", r"\bdb2_\w+\("),
+        "SQLite": (r"SQLite/JDBCDriver", r"SQLite.Exception", r"System.Data.SQLite.SQLiteException", r"Warning.*sqlite_.*",
+                   r"Warning.*SQLite3::", r"\[SQLITE_ERROR\]"),
+        "Sybase": (r"(?i)Warning.*sybase.*", r"Sybase message", r"Sybase.*Server message.*"),
+    }
+    for dbms, regexes in DBMS_ERRORS.items():
+        for regex in regexes:
+            if re.search(regex, new_body) and not re.search(regex, original_body):
+                return SQLiData(request_URL,
+                                injection_point,
+                                regex,
+                                dbms)
+
+
+# A qc is either ' or "
+def inside_quote(qc: str, substring: bytes, text_index: int, body: bytes) -> bool:
+    """ Whether the Numberth occurence of the first string in the second
+        string is inside quotes as defined by the supplied QuoteChar """
+    substring = substring.decode('utf-8')
+    body = body.decode('utf-8')
+    num_substrings_found = 0
+    in_quote = False
+    for index, char in enumerate(body):
+        # Whether the next chunk of len(substring) chars is the substring
+        next_part_is_substring = (
+            (not (index + len(substring) > len(body))) and
+            (body[index:index + len(substring)] == substring)
+        )
+        # Whether this char is escaped with a \
+        is_not_escaped = (
+            (index - 1 < 0 or index - 1 > len(body)) or
+            (body[index - 1] != "\\")
+        )
+        if char == qc and is_not_escaped:
+            in_quote = not in_quote
+        if next_part_is_substring:
+            if num_substrings_found == text_index:
+                return in_quote
+            num_substrings_found += 1
+    return False
+
+
+def paths_to_text(html: str, str: str) -> List[str]:
+    """ Return list of Paths to a given str in the given HTML tree
+          - Note that it does a BFS """
+
+    def remove_last_occurence_of_sub_string(str: str, substr: str):
+        """ Delete the last occurence of substr from str
+        String String -> String
+        """
+        index = str.rfind(substr)
+        return str[:index] + str[index + len(substr):]
+
+    class PathHTMLParser(HTMLParser):
+        currentPath = ""
+        paths = []
+
+        def handle_starttag(self, tag, attrs):
+            self.currentPath += ("/" + tag)
+
+        def handle_endtag(self, tag):
+            self.currentPath = remove_last_occurence_of_sub_string(self.currentPath, "/" + tag)
+
+        def handle_data(self, data):
+            if str in data:
+                self.paths.append(self.currentPath)
+
+    parser = PathHTMLParser()
+    parser.feed(html)
+    return parser.paths
+
+
+def get_XSS_data(body: str, request_URL: str, injection_point: str) -> Optional[XSSData]:
+    """ Return a XSSDict if there is a XSS otherwise return None """
+    def in_script(text, index, body) -> bool:
+        """ Whether the Numberth occurence of the first string in the second
+            string is inside a script tag """
+        paths = paths_to_text(body.decode('utf-8'), text.decode("utf-8"))
+        try:
+            path = paths[index]
+            return "script" in path
+        except IndexError:
+            return False
+
+    def in_HTML(text: bytes, index: int, body: bytes) -> bool:
+        """ Whether the Numberth occurence of the first string in the second
+            string is inside the HTML but not inside a script tag or part of
+            a HTML attribute"""
+        # if there is a < then lxml will interpret that as a tag, so only search for the stuff before it
+        text = text.split(b"<")[0]
+        paths = paths_to_text(body.decode('utf-8'), text.decode("utf-8"))
+        try:
+            path = paths[index]
+            return "script" not in path
+        except IndexError:
+            return False
+
+    def inject_javascript_handler(html: str) -> bool:
+        """ Whether you can inject a Javascript:alert(0) as a link """
+        class injectJSHandlerHTMLParser(HTMLParser):
+            injectJSHandler = False
+
+            def handle_starttag(self, tag, attrs):
+                for name, value in attrs:
+                    if name == "href" and value.startswith(FRONT_WALL.decode('utf-8')):
+                        self.injectJSHandler = True
+
+        parser = injectJSHandlerHTMLParser()
+        parser.feed(html)
+        return parser.injectJSHandler
+    # Only convert the body to bytes if needed
+    if isinstance(body, str):
+        body = bytes(body, 'utf-8')
+    # Regex for between 24 and 72 (aka 24*3) characters encapsulated by the walls
+    regex = re.compile(b"""%s.{24,72}?%s""" % (FRONT_WALL, BACK_WALL))
+    matches = regex.findall(body)
+    for index, match in enumerate(matches):
+        # Where the string is injected into the HTML
+        in_script = in_script(match, index, body)
+        in_HTML = in_HTML(match, index, body)
+        in_tag = not in_script and not in_HTML
+        in_single_quotes = inside_quote("'", match, index, body)
+        in_double_quotes = inside_quote('"', match, index, body)
+        # Whether you can inject:
+        inject_open_angle = b"ao<ac" in match  # open angle brackets
+        inject_close_angle = b"ac>so" in match  # close angle brackets
+        inject_single_quotes = b"s'd" in match  # single quotes
+        inject_double_quotes = b'd"ao' in match  # double quotes
+        inject_slash = b"sl/bsl" in match  # forward slashes
+        inject_semi = b"se;sl" in match  # semicolons
+        inject_equals = b"eq=" in match  # equals sign
+        if in_script and inject_slash and inject_open_angle and inject_close_angle:  # e.g. <script>PAYLOAD</script>
+            return XSSData(request_URL,
+                           injection_point,
+                           '</script><script>alert(0)</script><script>',
+                           match.decode('utf-8'))
+        elif in_script and in_single_quotes and inject_single_quotes and inject_semi:  # e.g. <script>t='PAYLOAD';</script>
+            return XSSData(request_URL,
+                           injection_point,
+                           "';alert(0);g='",
+                           match.decode('utf-8'))
+        elif in_script and in_double_quotes and inject_double_quotes and inject_semi:  # e.g. <script>t="PAYLOAD";</script>
+            return XSSData(request_URL,
+                           injection_point,
+                           '";alert(0);g="',
+                           match.decode('utf-8'))
+        elif in_tag and in_single_quotes and inject_single_quotes and inject_open_angle and inject_close_angle and inject_slash:
+            # e.g. <a href='PAYLOAD'>Test</a>
+            return XSSData(request_URL,
+                           injection_point,
+                           "'><script>alert(0)</script>",
+                           match.decode('utf-8'))
+        elif in_tag and in_double_quotes and inject_double_quotes and inject_open_angle and inject_close_angle and inject_slash:
+            # e.g. <a href="PAYLOAD">Test</a>
+            return XSSData(request_URL,
+                           injection_point,
+                           '"><script>alert(0)</script>',
+                           match.decode('utf-8'))
+        elif in_tag and not in_double_quotes and not in_single_quotes and inject_open_angle and inject_close_angle and inject_slash:
+            # e.g. <a href=PAYLOAD>Test</a>
+            return XSSData(request_URL,
+                           injection_point,
+                           '><script>alert(0)</script>',
+                           match.decode('utf-8'))
+        elif inject_javascript_handler(body.decode('utf-8')):  # e.g. <html><a href=PAYLOAD>Test</a>
+            return XSSData(request_URL,
+                           injection_point,
+                           'Javascript:alert(0)',
+                           match.decode('utf-8'))
+        elif in_tag and in_double_quotes and inject_double_quotes and inject_equals:  # e.g. <a href="PAYLOAD">Test</a>
+            return XSSData(request_URL,
+                           injection_point,
+                           '" onmouseover="alert(0)" t="',
+                           match.decode('utf-8'))
+        elif in_tag and in_single_quotes and inject_single_quotes and inject_equals:  # e.g. <a href='PAYLOAD'>Test</a>
+            return XSSData(request_URL,
+                           injection_point,
+                           "' onmouseover='alert(0)' t='",
+                           match.decode('utf-8'))
+        elif in_tag and not in_single_quotes and not in_double_quotes and inject_equals:  # e.g. <a href=PAYLOAD>Test</a>
+            return XSSData(request_URL,
+                           injection_point,
+                           " onmouseover=alert(0) t=",
+                           match.decode('utf-8'))
+        elif in_HTML and not in_script and inject_open_angle and inject_close_angle and inject_slash:  # e.g. <html>PAYLOAD</html>
+            return XSSData(request_URL,
+                           injection_point,
+                           '<script>alert(0)</script>',
+                           match.decode('utf-8'))
+        else:
+            return None
+
+
+# response is mitmproxy's entry point
+def response(flow: http.HTTPFlow) -> None:
+    cookiesDict = get_cookies(flow)
+    # Example: http://xss.guru/unclaimedScriptTag.html
+    find_unclaimed_URLs(flow.response.content, flow.request.url)
+    results = test_end_of_URL_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict)
+    log_XSS_data(results[0])
+    log_SQLi_data(results[1])
+    # Example: https://daviddworken.com/vulnerableReferer.php
+    results = test_referer_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict)
+    log_XSS_data(results[0])
+    log_SQLi_data(results[1])
+    # Example: https://daviddworken.com/vulnerableUA.php
+    results = test_user_agent_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict)
+    log_XSS_data(results[0])
+    log_SQLi_data(results[1])
+    if "?" in flow.request.url:
+        # Example: https://daviddworken.com/vulnerable.php?name=
+        results = test_query_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict)
+        log_XSS_data(results[0])
+        log_SQLi_data(results[1])

examples/custom_contentviews.py

@@ -1,70 +0,0 @@
-import string
-import lxml.html
-import lxml.etree
-from mitmproxy import contentviews
-from netlib import strutils
-
-
-class ViewPigLatin(contentviews.View):
-    name = "pig_latin_HTML"
-    prompt = ("pig latin HTML", "l")
-    content_types = ["text/html"]
-
-    def __call__(self, data, **metadata):
-        if strutils.is_xml(data):
-            parser = lxml.etree.HTMLParser(
-                strip_cdata=True,
-                remove_blank_text=True
-            )
-            d = lxml.html.fromstring(data, parser=parser)
-            docinfo = d.getroottree().docinfo
-
-            def piglify(src):
-                words = src.split()
-                ret = ''
-                for word in words:
-                    idx = -1
-                    while word[idx] in string.punctuation and (idx * -1) != len(word):
-                        idx -= 1
-                    if word[0].lower() in 'aeiou':
-                        if idx == -1:
-                            ret += word[0:] + "hay"
-                        else:
-                            ret += word[0:len(word) + idx + 1] + "hay" + word[idx + 1:]
-                    else:
-                        if idx == -1:
-                            ret += word[1:] + word[0] + "ay"
-                        else:
-                            ret += word[1:len(word) + idx + 1] + word[0] + "ay" + word[idx + 1:]
-                    ret += ' '
-                return ret.strip()
-
-            def recurse(root):
-                if hasattr(root, 'text') and root.text:
-                    root.text = piglify(root.text)
-                if hasattr(root, 'tail') and root.tail:
-                    root.tail = piglify(root.tail)
-
-                if len(root):
-                    for child in root:
-                        recurse(child)
-
-            recurse(d)
-
-            s = lxml.etree.tostring(
-                d,
-                pretty_print=True,
-                doctype=docinfo.doctype
-            )
-            return "HTML", contentviews.format_text(s)
-
-
-pig_view = ViewPigLatin()
-
-
-def start():
-    contentviews.add(pig_view)
-
-
-def done():
-    contentviews.remove(pig_view)

examples/dns_spoofing.py

@@ -1,49 +0,0 @@
-"""
-This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
-connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
-Host header of the HTTP request.
-Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
-know the actual target and cannot construct a certificate that looks valid.
-Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
-Using transparent mode is the better option most of the time.
-
-Usage:
-    mitmproxy
-        -p 443
-        -s dns_spoofing.py
-        # Used as the target location if neither SNI nor host header are present.
-        -R http://example.com/
-    mitmdump
-        -p 80
-        -R http://localhost:443/
-
-    (Setting up a single proxy instance and using iptables to redirect to it
-    works as well)
-"""
-import re
-
-# This regex extracts splits the host header into host and port.
-# Handles the edge case of IPv6 addresses containing colons.
-# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
-parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
-
-
-def request(flow):
-    if flow.client_conn.ssl_established:
-        flow.request.scheme = "https"
-        sni = flow.client_conn.connection.get_servername()
-        port = 443
-    else:
-        flow.request.scheme = "http"
-        sni = None
-        port = 80
-
-    host_header = flow.request.pretty_host
-    m = parse_host_header.match(host_header)
-    if m:
-        host_header = m.group("host").strip("[]")
-        if m.group("port"):
-            port = int(m.group("port"))
-
-    flow.request.host = sni or host_header
-    flow.request.port = port

examples/dup_and_replay.py

@@ -1,7 +0,0 @@
-from mitmproxy import master
-
-
-def request(flow):
-    f = master.duplicate_flow(flow)
-    f.request.path = "/changed"
-    master.replay_request(f, block=True, run_scripthooks=False)

examples/fail_with_500.py

@@ -1,3 +0,0 @@
-def response(flow):
-    flow.response.status_code = 500
-    flow.response.content = b""

examples/flowbasic

@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-"""
-    This example shows how to build a proxy based on mitmproxy's Flow
-    primitives.
-
-    Heads Up: In the majority of cases, you want to use inline scripts.
-
-    Note that request and response messages are not automatically replied to,
-    so we need to implement handlers to do this.
-"""
-from mitmproxy import flow, controller, options
-from mitmproxy.proxy import ProxyServer, ProxyConfig
-
-
-class MyMaster(flow.FlowMaster):
-    def run(self):
-        try:
-            flow.FlowMaster.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    @controller.handler
-    def request(self, f):
-        print("request", f)
-
-    @controller.handler
-    def response(self, f):
-        print("response", f)
-
-    @controller.handler
-    def error(self, f):
-        print("error", f)
-
-    @controller.handler
-    def log(self, l):
-        print("log", l.msg)
-
-opts = options.Options(cadir="~/.mitmproxy/")
-config = ProxyConfig(opts)
-state = flow.State()
-server = ProxyServer(config)
-m = MyMaster(opts, server, state)
-m.run()

examples/flowfilter.py

@@ -1,21 +0,0 @@
-# This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
-# Usage: mitmdump -s "flowfilter.py FILTER"
-
-import sys
-from mitmproxy import flowfilter
-
-
-class Filter:
-    def __init__(self, spec):
-        self.filter = flowfilter.parse(spec)
-
-    def response(self, flow):
-        if flowfilter.match(flow, self.filter):
-            print("Flow matches filter:")
-            print(flow)
-
-
-def start():
-    if len(sys.argv) != 2:
-        raise ValueError("Usage: -s 'filt.py FILTER'")
-    return Filter(sys.argv[1])

examples/flowwriter.py

@@ -1,23 +0,0 @@
-import random
-import sys
-
-from mitmproxy.flow import FlowWriter
-
-
-class Writer:
-    def __init__(self, path):
-        if path == "-":
-            f = sys.stdout
-        else:
-            f = open(path, "wb")
-        self.w = FlowWriter(f)
-
-    def response(self, flow):
-        if random.choice([True, False]):
-            self.w.add(flow)
-
-
-def start():
-    if len(sys.argv) != 2:
-        raise ValueError('Usage: -s "flowriter.py filename"')
-    return Writer(sys.argv[1])

examples/iframe_injector.py

@@ -1,29 +0,0 @@
-# Usage: mitmdump -s "iframe_injector.py url"
-# (this script works best with --anticache)
-import sys
-from bs4 import BeautifulSoup
-
-
-class Injector:
-    def __init__(self, iframe_url):
-        self.iframe_url = iframe_url
-
-    def response(self, flow):
-        if flow.request.host in self.iframe_url:
-            return
-        html = BeautifulSoup(flow.response.content, "lxml")
-        if html.body:
-            iframe = html.new_tag(
-                "iframe",
-                src=self.iframe_url,
-                frameborder=0,
-                height=0,
-                width=0)
-            html.body.insert(0, iframe)
-            flow.response.content = str(html).encode("utf8")
-
-
-def start():
-    if len(sys.argv) != 2:
-        raise ValueError('Usage: -s "iframe_injector.py url"')
-    return Injector(sys.argv[1])

examples/logging.py

@@ -1,6 +0,0 @@
-from mitmproxy import ctx
-
-
-def start():
-    ctx.log.info("This is some informative text.")
-    ctx.log.error("This is an error.")

examples/modify_querystring.py

@@ -1,2 +0,0 @@
-def request(flow):
-    flow.request.query["mitmproxy"] = "rocks"

examples/nonblocking.py

@@ -1,10 +0,0 @@
-import time
-import mitmproxy
-from mitmproxy.script import concurrent
-
-
-@concurrent  # Remove this and see what happens
-def request(flow):
-    mitmproxy.ctx.log("handle request: %s%s" % (flow.request.host, flow.request.path))
-    time.sleep(5)
-    mitmproxy.ctx.log("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/redirect_requests.py

@@ -1,19 +0,0 @@
-"""
-This example shows two ways to redirect flows to other destinations.
-"""
-from mitmproxy.models import HTTPResponse
-
-
-def request(flow):
-    # pretty_host takes the "Host" header of the request into account,
-    # which is useful in transparent mode where we usually only have the IP
-    # otherwise.
-
-    # Method 1: Answer with a locally generated response
-    if flow.request.pretty_host.endswith("example.com"):
-        resp = HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"})
-        flow.reply.send(resp)
-
-    # Method 2: Redirect the request to a different server
-    if flow.request.pretty_host.endswith("example.org"):
-        flow.request.host = "mitmproxy.org"

examples/simple/README.md

@@ -0,0 +1,18 @@
+## Simple Examples
+
+| Filename                     | Description                                                                |
+|:-----------------------------|:---------------------------------------------------------------------------|
+| add_header.py                | Simple script that just adds a header to every request.                    |
+| custom_contentview.py        | Add a custom content view to the mitmproxy UI.                             |
+| filter_flows.py              | This script demonstrates how to use mitmproxy's filter pattern in scripts. |
+| io_read_dumpfile.py          | Read a dumpfile generated by mitmproxy.                                    |
+| io_write_dumpfile.py         | Only write selected flows into a mitmproxy dumpfile.                       |
+| log_events.py                   | Use mitmproxy's logging API.                                            |
+| modify_body_inject_iframe.py | Inject configurable iframe into pages.                                     |
+| modify_form.py               | Modify HTTP form submissions.                                              |
+| modify_querystring.py        | Modify HTTP query strings.                                                 |
+| redirect_requests.py         | Redirect a request to a different server.                                  |
+| script_arguments.py          | Add arguments to a script.                                                 |
+| send_reply_from_proxy.py     | Send a HTTP response directly from the proxy.                              |
+| upsidedownternet.py          | Turn all images upside down.                                               |
+| wsgi_flask_app.py            | Embed a WSGI app into mitmproxy.                                           |

examples/simple/add_header.py

@@ -0,0 +1,5 @@
+from mitmproxy import http
+
+
+def response(flow: http.HTTPFlow) -> None:
+    flow.response.headers["newheader"] = "foo"

examples/simple/add_header_class.py

@@ -0,0 +1,9 @@
+from mitmproxy import http
+
+
+class AddHeader:
+    def response(self, flow: http.HTTPFlow) -> None:
+        flow.response.headers["newheader"] = "foo"
+
+
+addons = [AddHeader()]

examples/simple/custom_contentview.py

@@ -0,0 +1,32 @@
+"""
+This example shows how one can add a custom contentview to mitmproxy.
+The content view API is explained in the mitmproxy.contentviews module.
+"""
+from mitmproxy import contentviews
+import typing
+
+
+CVIEWSWAPCASE = typing.Tuple[str, typing.Iterable[typing.List[typing.Tuple[str, typing.AnyStr]]]]
+
+
+class ViewSwapCase(contentviews.View):
+    name = "swapcase"
+
+    # We don't have a good solution for the keyboard shortcut yet -
+    # you manually need to find a free letter. Contributions welcome :)
+    prompt = ("swap case text", "z")
+    content_types = ["text/plain"]
+
+    def __call__(self, data: typing.AnyStr, **metadata) -> CVIEWSWAPCASE:
+        return "case-swapped text", contentviews.format_text(data.swapcase())
+
+
+view = ViewSwapCase()
+
+
+def load(l):
+    contentviews.add(view)
+
+
+def done():
+    contentviews.remove(view)

examples/simple/custom_option.py

@@ -0,0 +1,11 @@
+from mitmproxy import ctx
+
+
+def load(l):
+    ctx.log.info("Registering option 'custom'")
+    l.add_option("custom", bool, False, "A custom option")
+
+
+def configure(updated):
+    if "custom" in updated:
+        ctx.log.info("custom option value: %s" % ctx.options.custom)

examples/simple/filter_flows.py

@@ -0,0 +1,26 @@
+"""
+This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
+"""
+from mitmproxy import flowfilter
+from mitmproxy import ctx, http
+
+
+class Filter:
+    def __init__(self):
+        self.filter = None  # type: flowfilter.TFilter
+
+    def configure(self, updated):
+        self.filter = flowfilter.parse(ctx.options.flowfilter)
+
+    def load(self, l):
+        l.add_option(
+            "flowfilter", str, "", "Check that flow matches filter."
+        )
+
+    def response(self, flow: http.HTTPFlow) -> None:
+        if flowfilter.match(self.filter, flow):
+            print("Flow matches filter:")
+            print(flow)
+
+
+addons = [Filter()]

examples/simple/io_read_dumpfile.py

@@ -1,15 +1,17 @@
 #!/usr/bin/env python
+
+# type: ignore
 #
 # Simple script showing how to read a mitmproxy dump file
 #
-
-from mitmproxy import flow
+from mitmproxy import io
 from mitmproxy.exceptions import FlowReadException
 import pprint
 import sys
 
+
 with open(sys.argv[1], "rb") as logfile:
-    freader = flow.FlowReader(logfile)
+    freader = io.FlowReader(logfile)
     pp = pprint.PrettyPrinter(indent=4)
     try:
         for f in freader.stream():

examples/simple/io_write_dumpfile.py

@@ -0,0 +1,27 @@
+"""
+This script how to generate a mitmproxy dump file,
+as it would also be generated by passing `-w` to mitmproxy.
+In contrast to `-w`, this gives you full control over which
+flows should be saved and also allows you to rotate files or log
+to multiple files in parallel.
+"""
+import random
+import sys
+from mitmproxy import io, http
+import typing  # noqa
+
+
+class Writer:
+    def __init__(self, path: str) -> None:
+        if path == "-":
+            f = sys.stdout  # type: typing.IO[typing.Any]
+        else:
+            f = open(path, "wb")
+        self.w = io.FlowWriter(f)
+
+    def response(self, flow: http.HTTPFlow) -> None:
+        if random.choice([True, False]):
+            self.w.add(flow)
+
+
+addons = [Writer(sys.argv[1])]

examples/simple/log_events.py

@@ -0,0 +1,12 @@
+"""
+It is recommended to use `ctx.log` for logging within a script.
+This goes to the event log in mitmproxy and to stdout in mitmdump.
+
+If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 :-)
+"""
+from mitmproxy import ctx
+
+
+def load(l):
+    ctx.log.info("This is some informative text.")
+    ctx.log.error("This is an error.")

examples/simple/modify_body_inject_iframe.py

@@ -0,0 +1,26 @@
+# (this script works best with --anticache)
+from bs4 import BeautifulSoup
+from mitmproxy import ctx, http
+
+
+class Injector:
+    def load(self, loader):
+        loader.add_option(
+            "iframe", str, "", "IFrame to inject"
+        )
+
+    def response(self, flow: http.HTTPFlow) -> None:
+        if ctx.options.iframe:
+            html = BeautifulSoup(flow.response.content, "html.parser")
+            if html.body:
+                iframe = html.new_tag(
+                    "iframe",
+                    src=ctx.options.iframe,
+                    frameborder=0,
+                    height=0,
+                    width=0)
+                html.body.insert(0, iframe)
+                flow.response.content = str(html).encode("utf8")
+
+
+addons = [Injector()]

examples/simple/modify_form.py

@@ -1,7 +1,12 @@
-def request(flow):
+from mitmproxy import http
+
+
+def request(flow: http.HTTPFlow) -> None:
     if flow.request.urlencoded_form:
+        # If there's already a form, one can just add items to the dict:
         flow.request.urlencoded_form["mitmproxy"] = "rocks"
     else:
+        # One can also just pass new form data.
         # This sets the proper content type and overrides the body.
         flow.request.urlencoded_form = [
             ("foo", "bar")

examples/simple/modify_querystring.py

@@ -0,0 +1,5 @@
+from mitmproxy import http
+
+
+def request(flow: http.HTTPFlow) -> None:
+    flow.request.query["mitmproxy"] = "rocks"

examples/simple/redirect_requests.py

@@ -0,0 +1,12 @@
+"""
+This example shows two ways to redirect flows to another server.
+"""
+from mitmproxy import http
+
+
+def request(flow: http.HTTPFlow) -> None:
+    # pretty_host takes the "Host" header of the request into account,
+    # which is useful in transparent mode where we usually only have the IP
+    # otherwise.
+    if flow.request.pretty_host == "example.org":
+        flow.request.host = "mitmproxy.org"

examples/simple/send_reply_from_proxy.py

@@ -0,0 +1,17 @@
+"""
+This example shows how to send a reply from the proxy immediately
+without sending any data to the remote server.
+"""
+from mitmproxy import http
+
+
+def request(flow: http.HTTPFlow) -> None:
+    # pretty_url takes the "Host" header of the request into account, which
+    # is useful in transparent mode where we usually only have the IP otherwise.
+
+    if flow.request.pretty_url == "http://example.com/path":
+        flow.response = http.HTTPResponse.make(
+            200,  # (optional) status code
+            b"Hello World",  # (optional) content
+            {"Content-Type": "text/html"}  # (optional) headers
+        )

examples/simple/upsidedownternet.py

@@ -0,0 +1,16 @@
+"""
+This script rotates all images passing through the proxy by 180 degrees.
+"""
+import io
+from PIL import Image
+from mitmproxy import http
+
+
+def response(flow: http.HTTPFlow) -> None:
+    if flow.response.headers.get("content-type", "").startswith("image"):
+        s = io.BytesIO(flow.response.content)
+        img = Image.open(s).rotate(180)
+        s2 = io.BytesIO()
+        img.save(s2, "png")
+        flow.response.content = s2.getvalue()
+        flow.response.headers["content-type"] = "image/png"

examples/simple/wsgi_flask_app.py

@@ -4,22 +4,22 @@ instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
 a single simplest-possible page.
 """
 from flask import Flask
-import mitmproxy
+from mitmproxy.addons import wsgiapp
 
 app = Flask("proxapp")
 
 
 @app.route('/')
-def hello_world():
+def hello_world() -> str:
     return 'Hello World!'
 
 
-# Register the app using the magic domain "proxapp" on port 80. Requests to
-# this domain and port combination will now be routed to the WSGI app instance.
-def start():
-    mitmproxy.ctx.master.apps.add(app, "proxapp", 80)
+def load(l):
+    # Host app at the magic domain "proxapp.local" on port 80. Requests to this
+    # domain and port combination will now be routed to the WSGI app instance.
+    return wsgiapp.WSGIApp(app, "proxapp.local", 80)
 
     # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
     # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
     # but won't send any data.
-    mitmproxy.ctx.master.apps.add(app, "example.com", 443)
+    # mitmproxy.ctx.master.apps.add(app, "example.com", 443)

examples/stickycookies

@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-"""
-This example builds on mitmproxy's base proxying infrastructure to
-implement functionality similar to the "sticky cookies" option.
-
-Heads Up: In the majority of cases, you want to use inline scripts.
-"""
-import os
-from mitmproxy import controller, proxy
-from mitmproxy.proxy.server import ProxyServer
-
-
-class StickyMaster(controller.Master):
-    def __init__(self, server):
-        controller.Master.__init__(self, server)
-        self.stickyhosts = {}
-
-    def run(self):
-        try:
-            return controller.Master.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    @controller.handler
-    def request(self, flow):
-        hid = (flow.request.host, flow.request.port)
-        if "cookie" in flow.request.headers:
-            self.stickyhosts[hid] = flow.request.headers.get_all("cookie")
-        elif hid in self.stickyhosts:
-            flow.request.headers.set_all("cookie", self.stickyhosts[hid])
-
-    @controller.handler
-    def response(self, flow):
-        hid = (flow.request.host, flow.request.port)
-        if "set-cookie" in flow.response.headers:
-            self.stickyhosts[hid] = flow.response.headers.get_all("set-cookie")
-
-
-config = proxy.ProxyConfig(port=8080)
-server = ProxyServer(config)
-m = StickyMaster(server)
-m.run()

examples/stub.py

@@ -1,87 +0,0 @@
-import mitmproxy
-"""
-    This is a script stub, with definitions for all events.
-"""
-
-
-def start():
-    """
-        Called once on script startup before any other events
-    """
-    mitmproxy.ctx.log("start")
-
-
-def configure(options, updated):
-    """
-        Called once on script startup before any other events, and whenever options changes.
-    """
-    mitmproxy.ctx.log("configure")
-
-
-def clientconnect(root_layer):
-    """
-        Called when a client initiates a connection to the proxy. Note that a
-        connection can correspond to multiple HTTP requests
-    """
-    mitmproxy.ctx.log("clientconnect")
-
-
-def request(flow):
-    """
-        Called when a client request has been received.
-    """
-    mitmproxy.ctx.log("request")
-
-
-def serverconnect(server_conn):
-    """
-        Called when the proxy initiates a connection to the target server. Note that a
-        connection can correspond to multiple HTTP requests
-    """
-    mitmproxy.ctx.log("serverconnect")
-
-
-def responseheaders(flow):
-    """
-        Called when the response headers for a server response have been received,
-        but the response body has not been processed yet. Can be used to tell mitmproxy
-        to stream the response.
-    """
-    mitmproxy.ctx.log("responseheaders")
-
-
-def response(flow):
-    """
-       Called when a server response has been received.
-    """
-    mitmproxy.ctx.log("response")
-
-
-def error(flow):
-    """
-        Called when a flow error has occured, e.g. invalid server responses, or
-        interrupted connections. This is distinct from a valid server HTTP error
-        response, which is simply a response with an HTTP error code.
-    """
-    mitmproxy.ctx.log("error")
-
-
-def serverdisconnect(server_conn):
-    """
-        Called when the proxy closes the connection to the target server.
-    """
-    mitmproxy.ctx.log("serverdisconnect")
-
-
-def clientdisconnect(root_layer):
-    """
-        Called when a client disconnects from the proxy.
-    """
-    mitmproxy.ctx.log("clientdisconnect")
-
-
-def done():
-    """
-        Called once on script shutdown, after any other events.
-    """
-    mitmproxy.ctx.log("done")

examples/upsidedownternet.py

@@ -1,15 +0,0 @@
-from six.moves import cStringIO as StringIO
-from PIL import Image
-
-
-def response(flow):
-    if flow.response.headers.get("content-type", "").startswith("image"):
-        try:
-            s = StringIO(flow.response.content)
-            img = Image.open(s).rotate(180)
-            s2 = StringIO()
-            img.save(s2, "png")
-            flow.response.content = s2.getvalue()
-            flow.response.headers["content-type"] = "image/png"
-        except:  # Unknown image types etc.
-            pass

issue_template.md

@@ -1,22 +1,20 @@
 ##### Steps to reproduce the problem:
 
-1. 
-2. 
-3. 
-
-##### What is the expected behavior?
-
-
-##### What went wrong?
+1.
+2.
+3.
 
 
 ##### Any other comments? What have you tried so far?
 
 
----
 
-Mitmproxy Version:
-Operating System:
+##### System information
 
 
-<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) -->
+<!--
+    Cut and paste the output of "mitmproxy --version".
+
+    If you're using an older version if mitmproxy, please specify the version
+    and OS.
+-->

mitmproxy/__init__.py

@@ -0,0 +1,3 @@
+# https://github.com/mitmproxy/mitmproxy/issues/1809
+# import script here so that pyinstaller registers it.
+from . import script  # noqa