bump version

.appveyor.yml

@@ -1,16 +1,9 @@
 version: '{build}'
 build: off  # Not a C# project
-
 environment:
-  CI_DEPS: codecov>=2.0.5
-  CI_COMMANDS: codecov
   matrix:
-    - PYTHON: "C:\\Python35"
-      TOXENV: "py35"
-    # TODO: ENABLE WHEN AVAILABLE
-    # - PYTHON: "C:\\Python36"
-    #   TOXENV: "py36"
-
+    - PYTHON: "C:\\Python27"
+  PATH: "%APPDATA%\\Python\\Scripts;C:\\Python27;C:\\Python27\\Scripts;%PATH%"
   SNAPSHOT_HOST:
     secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
   SNAPSHOT_PORT:
@@ -19,59 +12,20 @@ environment:
     secure: 6yBwmO5gv4vAwoFYII8qjQ==
   SNAPSHOT_PASS:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
-  RTOOL_KEY:
-    secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
-
 install:
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "python -m pip install --disable-pip-version-check -U pip"
-  - "pip install -U tox"
-
+  - "pip install --user -U virtualenv"
+  - "dev.bat"
+  - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
 test_script:
-  - ps: "tox -- --verbose --cov-report=term"
-  - ps: |
-      $Env:VERSION = $(python mitmproxy/version.py)
-      $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
-      tox -e wheel
-      tox -e rtool -- bdist
-
-  - ps: |
-      if(
-        ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
-        (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-        tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
-        if (!(Test-Path "C:\projects\mitmproxy\release\installbuilder-installer.exe")) {
-          "Download InstallBuilder..."
-          (New-Object System.Net.WebClient).DownloadFile(
-            "https://installbuilder.bitrock.com/installbuilder-enterprise-17.1.0-windows-installer.exe",
-            "C:\projects\mitmproxy\release\installbuilder-installer.exe"
-          )
-        }
-        Start-Process "C:\projects\mitmproxy\release\installbuilder-installer.exe" "--mode unattended --unattendedmodeui none" -Wait
-        & 'C:\Program Files (x86)\BitRock InstallBuilder Enterprise 17.1.0\bin\builder-cli.exe' `
-          build `
-          .\release\installbuilder\mitmproxy.xml `
-          windows `
-          --license .\release\installbuilder\license.xml `
-          --setvars project.version=$Env:VERSION `
-          --verbose
-      }
-
-deploy_script:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
-  ps: |
-    if(
-      ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-    ) {
-      tox -e rtool -- upload-snapshot --bdist --wheel --installer
-    }
-
+  - "py.test --cov netlib --cov mitmproxy --cov pathod"
 cache:
-  - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml
   - C:\Users\appveyor\AppData\Local\pip\cache
-
+deploy_script:
+  ps: |
+    if(($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true")) {
+    python .\release\rtool.py bdist
+    python .\release\rtool.py upload-snapshot --bdist
+    }
 notifications:
-  - provider: Slack
-    incoming_webhook: https://hooks.slack.com/services/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
+- provider: Slack
+  incoming_webhook: https://hooks.slack.com/services/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

.codecov.yml

@@ -1 +0,0 @@
-comment: off

.dockerignore

@@ -0,0 +1 @@
+.git

.env

@@ -0,0 +1,6 @@
+DIR="$( dirname "${BASH_SOURCE[0]}" )"
+ACTIVATE_DIR="$(if [ -f "$DIR/venv/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)"
+if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/venv/$ACTIVATE_DIR/activate" ]; then
+    echo "Activating mitmproxy virtualenv..."
+    source "$DIR/venv/$ACTIVATE_DIR/activate"
+fi

.gitattributes

@@ -1,2 +1,2 @@
-mitmproxy/tools/web/static/**/* -diff linguist-vendored
+mitmproxy/web/static/**/* -diff
 web/src/js/filt/filt.js -diff

.gitignore

@@ -1,15 +1,14 @@
 .DS_Store
 MANIFEST
-**/tmp
-/venv*
+*/tmp
+/venv
 *.py[cdo]
 *.swp
 *.swo
 *.egg-info/
-.coverage*
+.coverage
 .idea
 .cache/
-.tox*/
 build/
 
 # UI
@@ -17,7 +16,3 @@ build/
 node_modules
 bower_components
 *.map
-sslkeylogfile.log
-.tox/
-.python-version
-coverage.xml

.sources/bootswatch.less

@@ -0,0 +1,171 @@
+// Bootswatch.less
+// Swatch: Journal
+// Version: 2.0.4
+// -----------------------------------------------------
+
+// TYPOGRAPHY
+// -----------------------------------------------------
+
+@import url('https://fonts.googleapis.com/css?family=Open+Sans:400,700');
+
+h1, h2, h3, h4, h5, h6, .navbar .brand {
+	font-weight: 700;
+}
+
+// SCAFFOLDING
+// -----------------------------------------------------
+
+a {
+	text-decoration: none;
+}
+
+.nav a, .navbar .brand, .subnav a, a.btn, .dropdown-menu a {
+	text-decoration: none;
+}
+
+// NAVBAR
+// -----------------------------------------------------
+
+.navbar {
+
+	.navbar-inner {
+		@shadow: 0 2px 4px rgba(0,0,0,.25), inset 0 -1px 0 rgba(0,0,0,.1);
+		.box-shadow(@shadow);
+		border-top: 1px solid #E5E5E5;
+		.border-radius(0);
+	}
+
+	.brand {
+		text-shadow: none;
+
+		&:hover {
+			background-color: #EEEEEE;
+		}
+	}
+
+	.navbar-text {
+		line-height: 68px;
+	}
+
+	.nav > li > a {
+		text-shadow: none;
+	}
+
+	.dropdown-menu {
+		.border-radius(0);
+	}
+
+	.nav li.dropdown.active > .dropdown-toggle,
+	.nav li.dropdown.active > .dropdown-toggle:hover,
+	.nav li.dropdown.open > .dropdown-toggle,
+	.nav li.dropdown.active.open > .dropdown-toggle,
+	.nav li.dropdown.active.open > .dropdown-toggle:hover {
+		background-color: @grayLighter;
+		color: @linkColor;
+	}
+
+	.nav li.dropdown .dropdown-toggle .caret,
+	.nav .open .caret,
+	.nav .open .dropdown-toggle:hover .caret {
+		border-top-color: @black;
+		opacity: 1;
+	}
+
+	.nav-collapse.in .nav li > a:hover {
+		background-color: @grayLighter;
+	}
+
+	.nav-collapse .nav li > a {
+		color: @textColor;
+		text-decoration: none;
+		font-weight: normal;
+	}
+
+	.nav-collapse .navbar-form,
+	.nav-collapse .navbar-search {
+		border-color: transparent;
+	}
+
+	.navbar-search .search-query,
+	.navbar-search .search-query:hover {
+		border: 1px solid @grayLighter;
+		color: @textColor;
+		.placeholder(@gray);
+	}
+}
+
+div.subnav {
+	background-color: @bodyBackground;
+	background-image: none;
+	@shadow: 0 1px 2px rgba(0,0,0,.25);
+	.box-shadow(@shadow);
+	.border-radius(0);
+
+	&.subnav-fixed {
+		top: @navbarHeight;
+	}
+
+	.nav > li > a:hover,
+	.nav > .active > a,
+	.nav > .active > a:hover {
+		color: @textColor;
+		text-decoration: none;
+		font-weight: normal;
+	}
+
+	.nav > li:first-child > a,
+	.nav > li:first-child > a:hover {
+		.border-radius(0);
+	}
+}
+
+// BUTTONS
+// -----------------------------------------------------
+
+.btn-primary {
+  .buttonBackground(lighten(@linkColor, 5%), @linkColor);
+}
+
+[class^="icon-"], [class*=" icon-"] {
+	vertical-align: -2px;
+}
+
+// MODALS
+// -----------------------------------------------------
+
+.modal {
+ 	.border-radius(0px);
+	background: @bodyBackground;
+}
+
+.modal-header {
+	border-bottom: none;
+}
+
+.modal-header .close {
+	text-decoration: none;
+}
+
+.modal-footer {
+	background: transparent;
+	.box-shadow(none);
+	border-top: none;
+}
+
+
+// MISC
+// -----------------------------------------------------
+
+code, pre, pre.prettyprint, .well {
+	background-color: @grayLighter;
+}
+
+.hero-unit {
+    .box-shadow(inset 0 1px 1px rgba(0,0,0,.05));
+	border: 1px solid rgba(0,0,0,.05);
+	.border-radius(0);
+}
+
+.table-bordered, .well, .prettyprint {
+	.border-radius(0);
+}

.sources/make

@@ -0,0 +1,5 @@
+#!/bin/sh
+pygmentize -f html ../examples/test_context.py > ../pathod/templates/examples_context.html
+pygmentize -f html ../examples/test_setup.py > ../pathod/templates/examples_setup.html
+pygmentize -f html ../examples/test_setupall.py > ../pathod/templates/examples_setupall.html
+pygmentize -f html ../examples/pathod_pathoc.py > ../pathod/templates/pathod_pathoc.html

.sources/variables.less

@@ -0,0 +1,208 @@
+// Variables.less
+// Variables to customize the look and feel of Bootstrap
+// Swatch: Journal
+// Version: 2.0.4
+// -----------------------------------------------------
+
+// GLOBAL VALUES
+// --------------------------------------------------
+
+
+// Grays
+// -------------------------
+@black:                 #000;
+@grayDarker:            #222;
+@grayDark:              #333;
+@gray:                  #888;
+@grayLight:             #999;
+@grayLighter:           #eee;
+@white:                 #fff;
+
+
+// Accent colors
+// -------------------------
+@blue:                  #4380D3;
+@blueDark:              darken(@blue, 15%);
+@green:                 #22B24C;
+@red:                   #C00;
+@yellow:                #FCFADB;
+@orange:                #FF7F00;
+@pink:                  #CC99CC;
+@purple:                #7a43b6;
+@tan:                   #FFCA73;
+
+
+
+// Scaffolding
+// -------------------------
+@bodyBackground:        #FCFBFD;
+@textColor:             @grayDarker;
+
+
+// Links
+// -------------------------
+@linkColor:             @blue;
+@linkColorHover:        @red;
+
+
+// Typography
+// -------------------------
+@sansFontFamily:        'Open Sans', "Helvetica Neue", Helvetica, Arial, sans-serif;
+@serifFontFamily:       Georgia, "Times New Roman", Times, serif;
+@monoFontFamily:        Menlo, Monaco, Consolas, "Courier New", monospace;
+
+@baseFontSize:          14px;
+@baseFontFamily:        @sansFontFamily;
+@baseLineHeight:        18px;
+@altFontFamily:         @serifFontFamily;
+
+@headingsFontFamily:    inherit; // empty to use BS default, @baseFontFamily
+@headingsFontWeight:    bold;    // instead of browser default, bold
+@headingsColor:         inherit; // empty to use BS default, @textColor
+
+
+// Tables
+// -------------------------
+@tableBackground:                   transparent; // overall background-color
+@tableBackgroundAccent:             @grayLighter; // for striping
+@tableBackgroundHover:              #f5f5f5; // for hover
+@tableBorder:                       #ddd; // table and cell border
+
+
+// Buttons
+// -------------------------
+@btnBackground:                     @white;
+@btnBackgroundHighlight:            darken(@white, 10%);
+@btnBorder:                         darken(@white, 20%);
+
+@btnPrimaryBackground:              @linkColor;
+@btnPrimaryBackgroundHighlight:     spin(@btnPrimaryBackground, 15%);
+
+@btnInfoBackground:                 #5bc0de;
+@btnInfoBackgroundHighlight:        #2f96b4;
+
+@btnSuccessBackground:              #62c462;
+@btnSuccessBackgroundHighlight:     #51a351;
+
+@btnWarningBackground:              lighten(@orange, 10%);
+@btnWarningBackgroundHighlight:     @orange;
+
+@btnDangerBackground:               #ee5f5b;
+@btnDangerBackgroundHighlight:      #bd362f;
+
+@btnInverseBackground:              @linkColor;
+@btnInverseBackgroundHighlight:     darken(@linkColor, 5%);
+
+
+// Forms
+// -------------------------
+@inputBackground:               @white;
+@inputBorder:                   #ccc;
+@inputBorderRadius:             3px;
+@inputDisabledBackground:       @grayLighter;
+@formActionsBackground:         @grayLighter;
+
+// Dropdowns
+// -------------------------
+@dropdownBackground:            @bodyBackground;
+@dropdownBorder:                rgba(0,0,0,.2);
+@dropdownLinkColor:             @textColor;
+@dropdownLinkColorHover:        @textColor;
+@dropdownLinkBackgroundHover:   #eee;
+@dropdownDividerTop:            #e5e5e5;
+@dropdownDividerBottom:         @white;
+
+
+
+// COMPONENT VARIABLES
+// --------------------------------------------------
+
+// Z-index master list
+// -------------------------
+// Used for a bird's eye view of components dependent on the z-axis
+// Try to avoid customizing these :)
+@zindexDropdown:          1000;
+@zindexPopover:           1010;
+@zindexTooltip:           1020;
+@zindexFixedNavbar:       1030;
+@zindexModalBackdrop:     1040;
+@zindexModal:             1050;
+
+
+// Sprite icons path
+// -------------------------
+@iconSpritePath:          "../img/glyphicons-halflings.png";
+@iconWhiteSpritePath:     "../img/glyphicons-halflings-white.png";
+
+
+// Input placeholder text color
+// -------------------------
+@placeholderText:         @grayLight;
+
+
+// Hr border color
+// -------------------------
+@hrBorder:                @grayLighter;
+
+
+// Navbar
+// -------------------------
+@navbarHeight:                    50px;
+@navbarBackground:                @bodyBackground;
+@navbarBackgroundHighlight:       @bodyBackground;
+
+@navbarText:                      @textColor;
+@navbarLinkColor:                 @linkColor;
+@navbarLinkColorHover:            @linkColor;
+@navbarLinkColorActive:           @navbarLinkColorHover;
+@navbarLinkBackgroundHover:       @grayLighter;
+@navbarLinkBackgroundActive:      @grayLighter;
+
+@navbarSearchBackground:          lighten(@navbarBackground, 25%);
+@navbarSearchBackgroundFocus:     @white;
+@navbarSearchBorder:              darken(@navbarSearchBackground, 30%);
+@navbarSearchPlaceholderColor:    #ccc;
+@navbarBrandColor:                @blue;
+
+
+// Hero unit
+// -------------------------
+@heroUnitBackground:              @grayLighter;
+@heroUnitHeadingColor:            inherit;
+@heroUnitLeadColor:               inherit;
+
+
+// Form states and alerts
+// -------------------------
+@warningText:             #c09853;
+@warningBackground:       #fcf8e3;
+@warningBorder:           darken(spin(@warningBackground, -10), 3%);
+
+@errorText:               #b94a48;
+@errorBackground:         #f2dede;
+@errorBorder:             darken(spin(@errorBackground, -10), 3%);
+
+@successText:             #468847;
+@successBackground:       #dff0d8;
+@successBorder:           darken(spin(@successBackground, -10), 5%);
+
+@infoText:                #3a87ad;
+@infoBackground:          #d9edf7;
+@infoBorder:              darken(spin(@infoBackground, -10), 7%);
+
+
+
+// GRID
+// --------------------------------------------------
+
+// Default 940px grid
+// -------------------------
+@gridColumns:             12;
+@gridColumnWidth:         60px;
+@gridGutterWidth:         20px;
+@gridRowWidth:            (@gridColumns * @gridColumnWidth) + (@gridGutterWidth * (@gridColumns - 1));
+
+// Fluid grid
+// -------------------------
+@fluidGridColumnWidth:    6.382978723%;
+@fluidGridGutterWidth:    2.127659574%;

.travis.yml

@@ -1,94 +1,79 @@
 sudo: false
 language: python
 
-env:
-  global:
-    - CI_DEPS=codecov>=2.0.5
-    - CI_COMMANDS=codecov
-git:
-  depth: 10000
+addons:
+  apt:
+    sources:
+      # Debian sid currently holds OpenSSL 1.0.2
+      # change this with future releases!
+      - debian-sid
+    packages:
+      - libssl-dev
 
 matrix:
   fast_finish: true
   include:
+    - python: 2.7
+    - python: 2.7
+      env: NO_ALPN=1
+    - language: generic
+      os: osx
+      osx_image: xcode7.1
+      git:
+        depth: 9999999
     - python: 3.5
-      env: TOXENV=lint
-    - os: osx
-      osx_image: xcode7.3
-      language: generic
-      env: TOXENV=py35 BDIST=1
+      env: SCOPE="netlib ./test/mitmproxy/script"
     - python: 3.5
-      env: TOXENV=py35 OPENSSL_OLD
-      addons:
-        apt:
-          packages:
-            - libssl-dev
-    - python: 3.5
-      env: TOXENV=py35 BDIST=1 OPENSSL_ALPN
-      addons:
-        apt:
-          sources:
-            # Debian sid currently holds OpenSSL 1.1.0
-            # change this with future releases!
-            - debian-sid
-          packages:
-            - libssl-dev
-    - python: 3.6
-      env: TOXENV=py36 OPENSSL_ALPN
-      addons:
-        apt:
-          sources:
-            # Debian sid currently holds OpenSSL 1.1.0
-            # change this with future releases!
-            - debian-sid
-          packages:
-            - libssl-dev
-    - python: 3.5
-      env: TOXENV=individual_coverage
-    - python: 3.5
-      env: TOXENV=docs
+      env: SCOPE="netlib ./test/mitmproxy/script" NO_ALPN=1
+    - python: 2.7
+      env: DOCS=1
+      script: 'cd docs && make html'
+  allow_failures:
+    - python: pypy
 
 install:
   - |
     if [[ $TRAVIS_OS_NAME == "osx" ]]
     then
-      brew update || brew update
-      brew outdated pyenv || brew upgrade pyenv
-      eval "$(pyenv init -)"
-      env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.2
-      pyenv global 3.5.2
-      pyenv shell 3.5.2
-      pip install -U pip setuptools wheel virtualenv
+      brew update || brew update # try again if it fails
+      brew outdated openssl || brew upgrade openssl
+      brew install python
     fi
-  - pip install tox
+  - pip install -U virtualenv
+  - ./dev.sh
+  - source ./venv/bin/activate
+
+before_script:
+  - "openssl version -a"
+  - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
 
 script:
-  - tox -- --verbose --cov-report=term
-  - |
-    if [[ $BDIST == "1" ]]
-    then
-        git fetch --unshallow --tags
-        tox -e rtool -- bdist
-    fi
+  - "py.test --cov netlib --cov mitmproxy --cov pathod ./test/$SCOPE"
 
 after_success:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
+  - coveralls
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
-        tox -e rtool -- upload-snapshot --bdist
+        pip install -e ./release
+        python ./release/rtool.py bdist
+        python ./release/rtool.py upload-snapshot --bdist --wheel
     fi
 
 notifications:
+  irc:
+    channels:
+      - "irc.oftc.net#mitmproxy"
+    on_success: change
+    on_failure: always
   slack:
-      -rooms:
-          mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu
-      on_success: change
-      on_failure: change
-      on_start: never
+    rooms:
+      - mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu#ci
+    on_success: always
+    on_failure: always
 
 cache:
   directories:
-    - $HOME/.pyenv
     - $HOME/.cache/pip
-    # - $HOME/build/mitmproxy/mitmproxy/.tox
+    - $HOME/.pyenv
+    - $HOME/Library/Caches/pip

CHANGELOG

@@ -1,104 +1,3 @@
-21 February 2017: mitmproxy 2.0
-
-    * HTTP/2 is now enabled by default.
-
-    * Image ContentView: Parse images with Kaitai Struct (kaitai.io) instead of Pillow. 
-      This simplifies installation, reduces binary size, and allows parsing in pure Python.
-
-    * Web: Add missing flow filters.
-
-    * Add transparent proxy support for OpenBSD.
-
-    * Check the mitmproxy CA for expiration and warn the user to regenerate it if necessary.
-
-    * Testing: Tremendous improvements, enforced 100% coverage for large parts of the 
-      codebase, increased overall coverage.
-
-    * Enforce individual coverage: one source file -> one test file with 100% coverage.
-
-    * A myriad of other small improvements throughout the project.
-
-    * Numerous bugfixes.
-
-
-26 December 2016: mitmproxy 1.0
-
-    * All mitmproxy tools are now Python 3 only! We plan to support Python 3.5 and higher.
-
-    * Web-Based User Interface: Mitmproxy now offically has a web-based user interface
-      called mitmweb. We consider it stable for all features currently exposed 
-      in the UI, but it still misses a lot of mitmproxy’s options.
-      
-    * Windows Compatibility: With mitmweb, mitmproxy is now useable on Windows. 
-      We are also introducing an installer (kindly sponsored by BitRock) that 
-      simplifies setup.
-      
-    * Configuration: The config file format is now a single YAML file. In most cases,
-      converting to the new format should be trivial - please see the docs for
-      more information.
-
-    * Console: Significant UI improvements - including sorting of flows by
-      size, type and url, status bar improvements, much faster indentation for
-      HTTP views, and more.
-
-    * HTTP/2: Significant improvements, but is temporarily disabled by default
-      due to wide-spread protocol implementation errors on some large website
-
-    * WebSocket: The protocol implementation is now mature, and is enabled by
-      default. Complete UI support is coming in the next release. Hooks for 
-      message interception and manipulation are available.
-
-    * A myriad of other small improvements throughout the project.
-
-
-16 October 2016: mitmproxy 0.18
-
-    * Python 3 Compatibility for mitmproxy and pathod (Shadab Zafar, GSoC 2016)
-
-    * Major improvements to mitmweb (Clemens Brunner & Jason Hao, GSoC 2016)
-
-    * Internal Core Refactor: Separation of most features into isolated Addons
-
-    * Initial Support for WebSockets
-
-    * Improved HTTP/2 Support
-
-    * Reverse Proxy Mode now automatically adjusts host headers and TLS Server Name Indication
-
-    * Improved HAR export
-
-    * Improved export functionality for curl, python code, raw http etc.
-
-    * Flow URLs are now truncated in the console for better visibility
-
-    * New filters for TCP, HTTP and marked flows.
-
-    * Mitmproxy now handles comma-separated Cookie headers
-
-    * Merge mitmproxy and pathod documentation
-
-    * Mitmdump now sanitizes its console output to not include control characters
-
-    * Improved message body handling for HTTP messages:
-      .raw_content provides the message body as seen on the wire
-      .content provides the decompressed body (e.g. un-gzipped)
-      .text provides the body decompressed and decoded body
-
-    * New HTTP Message getters/setters for cookies and form contents.
-
-    * Add ability to view only marked flows in mitmproxy
-
-    * Improved Script Reloader (Always use polling, watch for whole directory)
-
-    * Use tox for testing
-
-    * Unicode support for tnetstrings
-
-    * Add dumpfile converters for mitmproxy versions 0.11 and 0.12
-
-    * Numerous bugfixes
-
-
 9 April 2016: mitmproxy 0.17
 
     * Simplify repository and release structure. mitmproxy now comes as a single package, including netlib and pathod.

CONTRIBUTORS

@@ -1,177 +1,124 @@
-  2407	Aldo Cortesi
-  1873	Maximilian Hils
-   556	Thomas Kriechbaumer
-   258	Shadab Zafar
-    97	Jason
+  1813	Aldo Cortesi
+  1228	Maximilian Hils
+   282	Thomas Kriechbaumer
     83	Marcelo Glezer
-    68	Clemens
     28	Jim Shaver
     18	Henrik Nordstrom
-    16	Matthew Shao
-    14	Pedro Worcel
+    17	Shadab Zafar
     14	David Weinstein
+    14	Pedro Worcel
     13	Thomas Roth
     11	Jake Drahos
-    11	Stephen Altamirano
-    11	arjun23496
     11	Justus Wingert
+    11	Stephen Altamirano
     10	András Veres-Szentkirályi
-    10	Zohar Lorberbaum
-    10	smill
     10	Chris Czub
     10	Sandor Nemes
-    10	Doug Freed
-     9	ikoz
-     9	Legend Tang
-     9	Rouli
      9	Kyle Morton
-     8	Jason A. Novak
+     9	Legend Tang
+     9	Matthew Shao
+     9	Rouli
      8	Chandler Abraham
-     7	Matthias Urlichs
-     7	Brad Peabody
-     7	dufferzafar
+     8	Jason A. Novak
      7	Alexis Hildebrandt
-     6	Felix Yan
-     5	Will Coster
-     5	Sam Cleveland
-     5	iroiro123
-     5	elitest
-     5	Tomaz Muraus
+     7	Brad Peabody
+     7	Matthias Urlichs
      5	Choongwoo Han
-     4	Schamper
-     4	Youhei Sakurai
+     5	Sam Cleveland
+     5	Tomaz Muraus
+     5	elitest
+     5	iroiro123
      4	Bryan Bishop
-     4	root
-     4	Valtteri Virtanen
-     4	Clemens Brunner
      4	Marc Liyanage
-     4	Wade 524
-     4	chhsiao90
-     4	yonder
      4	Michael J. Bazzinotti
-     3	Ryan Welton
-     3	Ryan Laughlin
-     3	Kyle Manna
-     3	Eli Shvartsman
-     3	Vincent Haupert
-     3	Manish Kumar
-     3	Zack B
-     3	MatthewShao
-     3	redfast00
-     3	requires.io
-     3	Guillem Anguera
-     3	smill@cuckoo.sh
-     3	Chris Neasbitt
+     4	Valtteri Virtanen
+     4	Wade 524
+     4	Youhei Sakurai
+     4	root
      3	Benjamin Lee
-     2	Steven Van Acker
-     2	Slobodan Mišković
+     3	Chris Neasbitt
+     3	Eli Shvartsman
+     3	Felix Yan
+     3	Guillem Anguera
+     3	Kyle Manna
+     3	MatthewShao
+     3	Ryan Welton
+     3	Zack B
+     2	Anant
+     2	Bennett Blodinger
+     2	Colin Bendell
+     2	Heikki Hannikainen
+     2	Israel Nir
+     2	Jaime Soriano Pastor
      2	Jim Lloyd
+     2	Krzysztof Bielicki
+     2	Mark E. Haase
+     2	Michael Frister
+     2	Nick Badger
+     2	Niko Kommenda
+     2	Paul
+     2	Rob Wills
+     2	Sean Coates
+     2	Terry Long
+     2	Wade Catron
+     2	alts
      2	isra17
      2	israel
-     2	Sean Coates
-     2	Sachin Kelkar
-     2	jpkrause
-     2	Bennett Blodinger
-     2	lilydjwg
-     2	Michael Frister
-     2	Israel Nir
-     2	Cory Benfield
-     2	phackt
-     2	Anant
-     2	Jaime Soriano Pastor
-     2	Paul
-     2	Colin Bendell
-     2	依云
-     2	Heikki Hannikainen
-     2	Rob Wills
-     2	Niko Kommenda
-     2	Naveen Pai
-     2	strohu
-     2	alts
-     2	Yoginski
-     2	Mark E. Haase
-     2	Wade Catron
-     2	Terry Long
-     2	Krzysztof Bielicki
-     2	Nick Badger
-     1	Nicolas Esteves
-     1	Andrew Orr
+     2	requires.io
      1	Andrey Plotnikov
      1	Andy Smith
-     1	Angelo Agatino Nicolosi
-     1	Anthony Zhang
-     1	BSalita
      1	Ben Lerner
      1	Bradley Baetz
-     1	Brady Law
-     1	Brett Randall
      1	Chris Hamant
-     1	Christian Frichot
      1	Dan Wilbraham
      1	David Dworken
      1	David Shaw
      1	Doug Lethin
-     1	Drake Caraker
-     1	Edgar Boda-Majer
      1	Eric Entzel
      1	Felix Wolfsteller
      1	FreeArtMan
      1	Gabriel Kirkpatrick
      1	Henrik Nordström
-     1	Israel Blancas
      1	Ivaylo Popov
      1	JC
      1	Jakub Nawalaniec
      1	Jakub Wilk
      1	James Billingham
-     1	Jason Pepas
      1	Jean Regisser
-     1	Jonathan Jones
      1	Jorge Villacorta
      1	Kit Randel
-     1	Kostya Esmukov
-     1	Linmiao Xu
      1	Lucas Cimon
      1	M. Utku Altinkaya
      1	Mathieu Mitchell
      1	Michael Bisbjerg
      1	Mike C
-     1	Mike Fotinakis
      1	Mikhail Korobov
      1	Morton Fox
      1	Nick HS
      1	Nick Raptis
-     1	Aditya
+     1	Nicolas Esteves
      1	Oleksandr Sheremet
-     1	Parth Ganatra
      1	Pritam Baral
-     1	Quentin Pradet
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
      1	Ryo Onodera
-     1	Sahil Chelaramani
      1	Sahn Lam
-     1	Sanchit Sokhey
      1	Seppo Yli-Olli
      1	Sergey Chipiga
      1	Stefan Wärting
      1	Steve Phillips
-     1	Steven Noble
+     1	Steven Van Acker
      1	Suyash
-     1	Tai Dickerson
      1	Tarashish Mishra
      1	TearsDontFalls
-     1	Thiago Arrais
      1	Tim Becker
      1	Timothy Elliott
-     1	Tyler St. Onge
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
-     1	Wes Turner
+     1	Will Coster
      1	Yuangxuan Wang
      1	capt8bit
-     1	cle1000
      1	davidpshaw
      1	deployable
      1	gecko655
@@ -185,5 +132,5 @@
      1	sentient07
      1	sethp-jive
      1	starenka
-     1	vulnminer
      1	vzvu3k6k
+     1	依云

Dockerfile

@@ -0,0 +1,4 @@
+FROM mitmproxy/base:latest-onbuild
+EXPOSE 8080
+EXPOSE 8081
+VOLUME /certs

MANIFEST.in

@@ -1,3 +1,4 @@
 graft mitmproxy
 graft pathod
-recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
+graft netlib
+recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

README.rst

@@ -1,118 +1,93 @@
 mitmproxy
 ^^^^^^^^^
 
-|travis| |appveyor| |coverage| |latest_release| |python_versions|
+|travis| |coveralls| |downloads| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects.
+This repository contains the **mitmproxy** and **pathod** projects, as well as their shared networking library, **netlib**.
 
-``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
-interface.
+``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console interface.
 
 ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
-``mitmweb`` is a web-based interface for mitmproxy.
-
-``pathoc`` and ``pathod`` are perverse HTTP client and server applications
-designed to let you craft almost any conceivable HTTP request, including ones
-that creatively violate the standards.
+``pathoc`` and ``pathod`` are perverse HTTP client and server applications designed to let you craft almost any conceivable HTTP request, including ones that creatively violate the standards.
 
 
 Documentation & Help
 --------------------
 
+Documentation, tutorials and precompiled binaries can be found on the mitmproxy and pathod websites.
 
-General information, tutorials, and precompiled binaries can be found on the mitmproxy
-and pathod websites.
-
-|mitmproxy_site|
+|mitmproxy_site| |pathod_site|
 
 The latest documentation for mitmproxy is also available on ReadTheDocs.
 
 |mitmproxy_docs|
 
-
-Join our discussion forum on Discourse to ask questions, help
-each other solve problems, and come up with new ideas for the project.
-
-|mitmproxy_discourse|
-
-
-Join our developer chat on Slack if you would like to contribute to mitmproxy itself.
+You can join our developer chat on Slack.
 
 |slack|
 
 
-Installation
-------------
+Hacking
+-------
 
-The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`__.
-If you want to contribute changes, keep on reading.
-
-Contributing
-------------
-
-As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project forward,
-please consider contributing in the following areas:
-
-- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users in our forums_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported.
-- **Code Contributions:** We actively mark issues that we consider are `good first contributions`_. If you intend to work on a larger contribution to the project, please come talk to us first.
-
-Development Setup
------------------
-
-To get started hacking on mitmproxy, please follow the `advanced installation`_ steps to install mitmproxy from source, but stop right before running ``pip3 install mitmproxy``. Instead, do the following:
+To get started hacking on mitmproxy, make sure you have Python_ 2.7.x. with
+virtualenv_ installed (you can find installation instructions for virtualenv here_).
+Then do the following:
 
 .. code-block:: text
 
     git clone https://github.com/mitmproxy/mitmproxy.git
     cd mitmproxy
-    ./dev.sh  # "powershell .\dev.ps1" on Windows
+    ./dev
 
 
-The *dev* script will create a `virtualenv`_ environment in a directory called "venv"
-and install all mandatory and optional dependencies into it. The primary
-mitmproxy components - mitmproxy and pathod - are installed as
-"editable", so any changes to the source in the repository will be reflected
-live in the virtualenv.
+The *dev* script will create a virtualenv environment in a directory called "venv",
+and install all mandatory and optional dependencies into it.
+The primary mitmproxy components - mitmproxy, netlib and pathod - are installed as "editable",
+so any changes to the source in the repository will be reflected live in the virtualenv.
 
-The main executables for the project - ``mitmdump``, ``mitmproxy``,
-``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the
-virtualenv. After activating the virtualenv, they will be on your $PATH, and
-you can run them like any other command:
+To confirm that you're up and running, activate the virtualenv, and run the
+mitmproxy test suite:
+
+.. code-block:: text
+
+    . venv/bin/activate # venv\Scripts\activate.bat on Windows
+    py.test
+
+Note that the main executables for the project - ``mitmdump``, ``mitmproxy``,
+``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the virtualenv. After activating the
+virtualenv, they will be on your $PATH, and you can run them like any other
+command:
 
 .. code-block:: text
 
-    . venv/bin/activate  # "venv\Scripts\activate" on Windows
     mitmdump --version
 
+For convenience, the project includes an autoenv_ file (`.env`_) that
+auto-activates the virtualenv when you cd into the mitmproxy directory.
+
+
 Testing
 -------
 
 If you've followed the procedure above, you already have all the development
-requirements installed, and you can run the full test suite (including tests for code style and documentation) with tox_:
+requirements installed, and you can simply run the test suite:
 
 .. code-block:: text
 
-    tox
-
-For speedier testing, we recommend you run `pytest`_ directly on individual test files or folders:
-
-.. code-block:: text
-
-    cd test/mitmproxy/addons
-    pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py
-
-As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes.
+    py.test
 
 Please ensure that all patches are accompanied by matching changes in the test
-suite. The project tries to maintain 100% test coverage and enforces this strictly for some parts of the codebase.
+suite. The project tries to maintain 100% test coverage.
+
 
 Documentation
--------------
+----
 
-The mitmproxy documentation is build using Sphinx_, which is installed
-automatically if you set up a development environment as described above. After
-installation, you can render the documentation like this:
+The mitmproxy documentation is build using Sphinx_, which is installed automatically if you set up a development
+environment as described above.
+After installation, you can render the documentation like this:
 
 .. code-block:: text
 
@@ -124,51 +99,36 @@ installation, you can render the documentation like this:
 The last command invokes `sphinx-autobuild`_, which watches the Sphinx directory and rebuilds
 the documentation when a change is detected.
 
-Code Style
-----------
-
-Keeping to a consistent code style throughout the project makes it easier to
-contribute and collaborate. Please stick to the guidelines in
-`PEP8`_ and the `Google Style Guide`_ unless there's a very
-good reason not to.
-
-This is automatically enforced on every PR. If we detect a linting error, the
-PR checks will fail and block merging. You can run our lint checks yourself
-with the following command:
-
-.. code-block:: text
-
-    tox -e lint
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg
     :target: https://mitmproxy.org/
     :alt: mitmproxy.org
 
-.. |mitmproxy_docs| image:: https://shields.mitmproxy.org/api/docs-latest-brightgreen.svg
+.. |pathod_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-pathod.net-blue.svg
+    :target: https://pathod.net/
+    :alt: pathod.net
+
+.. |mitmproxy_docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
     :target: http://docs.mitmproxy.org/en/latest/
     :alt: mitmproxy documentation
 
-.. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg
-    :target: https://discourse.mitmproxy.org
-    :alt: Discourse: mitmproxy
-
 .. |slack| image:: http://slack.mitmproxy.org/badge.svg
     :target: http://slack.mitmproxy.org/
     :alt: Slack Developer Chat
 
-.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=travis%20ci
+.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg
     :target: https://travis-ci.org/mitmproxy/mitmproxy
-    :alt: Travis Build Status
+    :alt: Build Status
 
-.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mhils/mitmproxy/master.svg?label=appveyor%20ci
-    :target: https://ci.appveyor.com/project/mhils/mitmproxy
-    :alt: Appveyor Build Status
-
-.. |coverage| image:: https://shields.mitmproxy.org/codecov/c/github/mitmproxy/mitmproxy/master.svg?label=codecov
-    :target: https://codecov.io/gh/mitmproxy/mitmproxy
+.. |coveralls| image:: https://shields.mitmproxy.org/coveralls/mitmproxy/mitmproxy/master.svg
+    :target: https://coveralls.io/r/mitmproxy/mitmproxy
     :alt: Coverage Status
 
+.. |downloads| image:: https://shields.mitmproxy.org/pypi/dm/mitmproxy.svg?color=orange
+    :target: https://pypi.python.org/pypi/mitmproxy
+    :alt: Downloads
+
 .. |latest_release| image:: https://shields.mitmproxy.org/pypi/v/mitmproxy.svg
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Latest Version
@@ -177,13 +137,11 @@ with the following command:
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Supported Python versions
 
-.. _`advanced installation`: http://docs.mitmproxy.org/en/latest/install.html#advanced-installation
-.. _virtualenv: https://virtualenv.pypa.io/
-.. _`pytest`: http://pytest.org/
-.. _tox: https://tox.readthedocs.io/
+.. _Python: https://www.python.org/
+.. _virtualenv: http://virtualenv.readthedocs.org/en/latest/
+.. _here: http://virtualenv.readthedocs.org/en/latest/installation.html
+.. _autoenv: https://github.com/kennethreitz/autoenv
+.. _.env: https://github.com/mitmproxy/mitmproxy/blob/master/.env
 .. _Sphinx: http://sphinx-doc.org/
 .. _sphinx-autobuild: https://pypi.python.org/pypi/sphinx-autobuild
-.. _PEP8: https://www.python.org/dev/peps/pep-0008
-.. _`Google Style Guide`: https://google.github.io/styleguide/pyguide.html
-.. _forums: https://discourse.mitmproxy.org/
-.. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3Agood-first-contribution
+.. _issue_tracker: https://github.com/mitmproxy/mitmproxy/issues

dev.bat

@@ -0,0 +1,14 @@
+@echo off
+set VENV=.\venv
+
+virtualenv %VENV% --always-copy
+if %errorlevel% neq 0 exit /b %errorlevel%
+call %VENV%\Scripts\activate.bat
+if %errorlevel% neq 0 exit /b %errorlevel%
+pip install -r requirements.txt
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo.
+echo * Created virtualenv environment in %VENV%.
+echo * Installed all dependencies into the virtualenv.
+echo * Activated virtualenv environment.

dev.ps1

@@ -1,20 +0,0 @@
-$ErrorActionPreference = "Stop"
-
-$pyver = python --version
-if($pyver -notmatch "3\.[5-9]") {
-    Write-Warning "Unexpected Python version, expected Python 3.5 or above: $pyver"
-}
-
-python -m venv .\venv --copies
-& .\venv\Scripts\activate.ps1
-
-python -m pip install --disable-pip-version-check -U pip
-cmd /c "pip install -r requirements.txt 2>&1"
-
-echo @"
-
-  * Created virtualenv environment in .\venv.
-  * Installed all dependencies into the virtualenv.
-  * Activated virtualenv environment.
-
-"@

dev.sh

@@ -1,15 +1,13 @@
-#!/bin/sh
+#!/bin/bash
 set -e
-set -x
+VENV=./venv
 
-echo "Creating dev environment in ./venv..."
-
-python3 -m venv venv
-. venv/bin/activate
-pip3 install -U pip setuptools
-pip3 install -r requirements.txt
+python -m virtualenv $VENV --always-copy
+. $VENV/bin/activate
+pip install -U pip setuptools
+pip install -r requirements.txt
 
 echo ""
-echo "  * Created virtualenv environment in ./venv."
-echo "  * Installed all dependencies into the virtualenv."
-echo "  * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`"
+echo "* Created virtualenv environment in $VENV."
+echo "* Installed all dependencies into the virtualenv."
+echo "* You can now activate the virtualenv: \`. $VENV/bin/activate\`"

docs/Makefile

@@ -192,4 +192,4 @@ pseudoxml:
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
 
 livehtml:
-	sphinx-autobuild -b html -z '../mitmproxy' -r '___jb_(old|bak|tmp)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	sphinx-autobuild -b html -z '../mitmproxy' -z '../../netlib/netlib' -r '___jb_(old|bak)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html

docs/_static/theme_overrides.css

@@ -1,44 +0,0 @@
-
-/* override table width restrictions */
-.wy-table-responsive table td, .wy-table-responsive table th {
-    white-space: normal;
-}
-
-.wy-table-responsive > table > tbody > tr > td {
-    vertical-align: top !important;
-}
-
-.wy-table-responsive {
-    margin-bottom: 24px;
-    max-width: 100%;
-    overflow: visible;
-}
-
-.wy-menu-vertical header, .wy-menu-vertical p.caption {
-    color: #e0e0e0;
-}
-
-.code-block-caption {
-    height: 1.5em;
-}
-
-.code-block-caption .caption-text {
-    font-size: 0.8em;
-    float: right;
-}
-
-.code-block-caption .headerlink {
-    display: none !important;
-}
-
-.function .headerlink {
-    display: none !important;
-}
-
-dl .reference.internal {
-    display: none !important;
-}
-
-dl .headerlink {
-    display: none !important;
-}

docs/certinstall.rst

@@ -40,9 +40,7 @@ start of mitmproxy.
 iOS
 ^^^
 
-See http://jasdev.me/intercepting-ios-traffic
-
-and http://web.archive.org/web/20150920082614/http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
+http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
 
 iOS Simulator
 ^^^^^^^^^^^^^
@@ -67,7 +65,7 @@ See http://windows.microsoft.com/en-ca/windows/import-export-certificates-privat
 Windows (automated)
 ^^^^^^^^^^^^^^^^^^^
 
->>> certutil.exe -importpfx Root mitmproxy-ca-cert.p12
+>>> certutil.exe -importpfx mitmproxy-ca-cert.p12
 
 See also: https://technet.microsoft.com/en-us/library/cc732443.aspx
 
@@ -132,12 +130,12 @@ mitmproxy-ca-cert.cer Same file as .pem, but with an extension expected by some
 Using a custom certificate
 --------------------------
 
-You can use your own certificate by passing the ``--cert [domain=]path_to_certificate`` option to
+You can use your own certificate by passing the ``--cert`` option to
 mitmproxy. Mitmproxy then uses the provided certificate for interception of the
-specified domain instead of generating a certificate signed by its own CA.
+specified domains instead of generating a certificate signed by its own CA.
 
 The certificate file is expected to be in the PEM format.  You can include
-intermediary certificates right below your leaf certificate, so that your PEM
+intermediary certificates right below your leaf certificate, so that you PEM
 file roughly looks like this:
 
 .. code-block:: none
@@ -160,18 +158,7 @@ For example, you can generate a certificate in this format using these instructi
 >>> openssl req -new -x509 -key cert.key -out cert.crt
     (Specify the mitm domain as Common Name, e.g. *.google.com)
 >>> cat cert.key cert.crt > cert.pem
-
-Now, you can run mitmproxy with the generated certificate:
-
-**For all domain names**
-
-``>>>mitmproxy --cert *=cert.pem``
-
-**For specific domain names**
-
-``>>>mitmproxy --cert *.example.com=cert.pem``
-
-**Note:** ``*.example.com`` is for all the subdomains. You can also use ``www.example.com`` for a particular subdomain.
+>>> mitmproxy --cert=cert.pem
 
 
 Using a custom certificate authority
@@ -205,4 +192,4 @@ directory and uses this as the client cert.
 
 
 
-.. _Certificate Pinning: http://security.stackexchange.com/questions/29988/what-is-certificate-pinning/
+.. _Certificate Pinning: http://security.stackexchange.com/questions/29988/what-is-certificate-pinning/

docs/conf.py

@@ -1,18 +1,40 @@
-import importlib
-import inspect
-import os
-import subprocess
+# -*- coding: utf-8 -*-
+#
+# mitmproxy documentation build configuration file, created by
+# sphinx-quickstart on Thu Sep 03 14:04:13 2015.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
 import sys
+import os
+import shlex
 
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('..'))
-from mitmproxy import version as mversion
 
+import mitmproxy.version
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
 
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
-    'sphinx.ext.extlinks',
-    'sphinx.ext.linkcode',
     'sphinx.ext.viewcode',
     'sphinx.ext.napoleon',
     'sphinxcontrib.documentedlist'
@@ -39,7 +61,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mitmproxy docs'
-copyright = u'2016, the mitmproxy project'
+copyright = u'2015, the mitmproxy project'
 author = u'The mitmproxy project'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -47,9 +69,9 @@ author = u'The mitmproxy project'
 # built documents.
 #
 # The short X.Y version.
-version = mversion.VERSION
+version = mitmproxy.version.VERSION
 # The full version, including alpha/beta/rc tags.
-release = mversion.VERSION
+release = mitmproxy.version.VERSION
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -131,7 +153,7 @@ html_favicon = "favicon.ico"
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
@@ -163,7 +185,7 @@ html_static_path = ['_static']
 #html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-html_show_sourcelink = False
+#html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
 #html_show_sphinx = True
@@ -194,50 +216,4 @@ html_show_sourcelink = False
 #html_search_scorer = 'scorer.js'
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'mitmproxydoc'
-
-last_tag, tag_dist, commit = (
-    subprocess.check_output(["git", "describe", "--tags", "--long"])
-        .decode()
-        .strip()
-        .rsplit("-", 2)
-)
-tag_dist = int(tag_dist)
-if tag_dist == 0:
-    tag = last_tag
-else:
-    tag = "master"
-
-SRCBASE = "https://github.com/mitmproxy/mitmproxy/blob/{}".format(tag)
-
-extlinks = dict(
-    src = (SRCBASE + r"/%s", '')
-)
-
-
-def linkcode_resolve(domain, info):
-    if domain != 'py':
-        return None
-    module, fullname = info['module'], info['fullname']
-    if not module:
-        return None
-    obj = importlib.import_module(module)
-    for item in fullname.split('.'):
-        obj = getattr(obj, item, None)
-    if obj is None:
-        return None
-    try:
-        spath = inspect.getsourcefile(obj)
-        _, line = inspect.getsourcelines(obj)
-    except (TypeError, IOError):
-        return None
-    if spath.rfind("mitmproxy") > -1:
-        off = spath.rfind("mitmproxy")
-        mpath = spath[off:]
-    else:
-        return None
-    return SRCBASE + "/%s#L%s" % (mpath, line)
-
-
-def setup(app):
-    app.add_stylesheet('theme_overrides.css')
+htmlhelp_basename = 'mitmproxydoc'

docs/config.rst

@@ -3,11 +3,84 @@
 Configuration
 =============
 
-Mitmproxy is configured with a YAML_ file, located at
-``~/.mitmproxy/config.yaml``. We'll have complete documentation for all
-supported options in the next release in the meantime, please consult the
-source_ for a complete list of options and types.
+Mitmproxy is configured through a set of files in the users ~/.mitmproxy
+directory.
 
+mitmproxy.conf
+    Settings for the :program:`mitmproxy`. This file can contain any options supported by
+    mitmproxy.
 
-.. _YAML: http://www.yaml.org/start.html
-.. _source: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/options.py
+mitmdump.conf
+    Settings for the :program:`mitmdump`. This file can contain any options supported by mitmdump.
+
+common.conf
+    Settings shared between all command-line tools. Settings in this file are over-ridden by those
+    in the tool-specific files. Only options shared by mitmproxy and mitmdump should be used in
+    this file.
+
+Syntax
+------
+
+Comments
+^^^^^^^^
+
+.. code-block:: none
+
+    # this is a comment
+    ; this is also a comment (.ini style)
+    --- and this is a comment too (yaml style)
+
+Key/Value pairs
+^^^^^^^^^^^^^^^
+
+- Keys and values are case-sensitive
+- Whitespace is ignored
+- Lists are comma-delimited, and enclosed in square brackets
+
+.. code-block:: none
+
+    name = value   # (.ini style)
+    name: value    # (yaml style)
+    --name value   # (command-line option style)
+
+    fruit = [apple, orange, lemon]
+    indexes = [1, 12, 35 , 40]
+
+Flags
+^^^^^
+
+These are boolean options that take no value but true/false.
+
+.. code-block:: none
+
+    name = true    # (.ini style)
+    name
+    --name 	 	   # (command-line option style)
+
+Options
+-------
+
+The options available in the config files are precisely those available as
+command-line flags, with the key being the option's long name. To get a
+complete list of these, use the :option:`--help` option on each of the tools. Be
+careful to only specify common options in the **common.conf** file -
+unsupported options in this file will be detected as an error on startup.
+
+Examples
+--------
+
+common.conf
+^^^^^^^^^^^
+
+Note that :option:`--port` is an option supported by all tools.
+
+.. code-block:: none
+
+    port = 8080
+
+mitmproxy.conf
+^^^^^^^^^^^^^^
+
+.. code-block:: none
+
+    palette = light

docs/dev/addingviews.html

@@ -18,22 +18,22 @@ __prompt__, and __content\_types__ and a function named __\_\_call\_\___.
 Adding a new content viewer to parse a data type is as simple as writing a new
 View class.  Your new content viewer View class should have the same properties
 as the other View classes: __name__, __prompt__, and __content\_types__ and a
-__\_\_call\_\___ function to parse the content of the request/response.
+__\_\_call\_\___ function to parse the content of the request/response. 
 
-* The __name__ property should be a string describing the contents and new content viewer;
+* The __name__ property should be a string describing the contents and new content viewer; 
 * The __prompt__ property should be a two item tuple:
 
   - __1__: A string that will be used to display the new content viewer's type; and
-  - __2__: A one character string that will be the hotkey used to select the new content viewer from the Flow View screen;
+  - __2__: A one character string that will be the hotkey used to select the new content viewer from the Flow View screen; 
 
-* The __content\_types__ property should be a list of strings of HTTP Content\-Types that the new content viewer can parse.
+* The __content\_types__ property should be a list of strings of HTTP Content\-Types that the new content viewer can parse.  
   * Note that mitmproxy will use the content\_types to try and heuristically show a friendly view of content and that you can override the built-in views by populating content\_types with values for content\_types that are already parsed -- e.g. "image/png".
 
 After defining the __name__, __prompt__, and __content\_types__ properties of
 the class, you should write the __\_\_call\_\___ function, which will parse the
 request/response data and provide a friendly view of the data.  The
 __\_\_call\_\___ function should take the following arguments: __self__,
-__hdrs__, __content__, __limit__; __hdrs__ is a MultiDict object containing
+__hdrs__, __content__, __limit__; __hdrs__ is a ODictCaseless object containing
 the headers of the request/response; __content__ is the content of the
 request/response, and __limit__ is an integer representing the amount of data
 to display in the view window.
@@ -46,7 +46,7 @@ Alternatively, you can display content as a series of key-value pairs; to do
 so, prepare a list of lists, where each list item is a two item list -- a key
 that describes the data, and then the data itself; after preparing the list of
 lists, use the __common.format\_keyvals__ function on it to prepare it as text
-for display.
+for display.  
 
 If the new content viewer fails or throws an exception, mitmproxy will default
 to a __raw__ view.

docs/dev/architecture.rst

@@ -0,0 +1,14 @@
+.. _architecture:
+
+Architecture
+============
+
+To give you a better understanding of how mitmproxy works, mitmproxy's
+high-level architecture is detailed in the following graphic:
+
+.. image:: ../schematics/architecture.png
+
+:download:`architecture.pdf <../schematics/architecture.pdf>`
+
+Please don't refrain from asking any further
+questions on the mailing list, the Slack channel or the GitHub issue tracker.

docs/dev/contributing.rst

@@ -1,11 +0,0 @@
-.. _contributing:
-
-Contributing
-============
-
-As an open source project, **mitmproxy** welcomes contributions of all forms. 
-
-Please head over to the README_ to get started! 😃
-
-
-.. _README: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst

docs/dev/exceptions.rst

@@ -0,0 +1,9 @@
+.. _exceptions:
+
+Exceptions
+==========
+
+.. automodule:: mitmproxy.exceptions
+    :show-inheritance:
+    :members:
+    :undoc-members:

docs/dev/models.rst

@@ -0,0 +1,59 @@
+.. _models:
+
+Models
+======
+
+.. automodule:: netlib.http
+
+    .. autoclass:: Request
+
+        .. rubric:: Data
+        .. autoattribute:: first_line_format
+        .. autoattribute:: method
+        .. autoattribute:: scheme
+        .. autoattribute:: host
+        .. autoattribute:: port
+        .. autoattribute:: path
+        .. autoattribute:: http_version
+        .. autoattribute:: headers
+        .. autoattribute:: content
+        .. autoattribute:: timestamp_start
+        .. autoattribute:: timestamp_end
+        .. rubric:: Computed Properties and Convenience Methods
+        .. autoattribute:: text
+        .. autoattribute:: url
+        .. autoattribute:: pretty_host
+        .. autoattribute:: pretty_url
+        .. autoattribute:: query
+        .. autoattribute:: cookies
+        .. autoattribute:: path_components
+        .. automethod:: anticache
+        .. automethod:: anticomp
+        .. automethod:: constrain_encoding
+        .. autoattribute:: urlencoded_form
+        .. autoattribute:: multipart_form
+
+    .. autoclass:: Response
+
+        .. rubric:: Data
+        .. autoattribute:: http_version
+        .. autoattribute:: status_code
+        .. autoattribute:: reason
+        .. autoattribute:: headers
+        .. autoattribute:: content
+        .. autoattribute:: timestamp_start
+        .. autoattribute:: timestamp_end
+        .. rubric:: Computed Properties and Convenience Methods
+        .. autoattribute:: text
+        .. autoattribute:: cookies
+
+    .. autoclass:: Headers
+        :members:
+        :special-members:
+        :no-undoc-members:
+
+    .. autoclass:: decoded
+
+.. automodule:: mitmproxy.models
+    :show-inheritance:
+    :members: HTTPFlow, Error, ClientConnection, ServerConnection

docs/dev/protocols.rst

@@ -0,0 +1,15 @@
+.. _protocols:
+
+Protocols
+=========
+
+.. automodule:: mitmproxy.protocol
+
+    .. autoclass:: Layer
+        :members:
+        :special-members:
+
+    .. autoclass:: ServerConnectionMixin
+        :members:
+
+    .. autoexception:: Kill

docs/dev/proxy.rst

@@ -0,0 +1,12 @@
+.. _proxy:
+
+Proxy Server
+============
+
+.. automodule:: mitmproxy.proxy
+
+    .. autoclass:: ProxyServer
+    .. autoclass:: DummyServer
+    .. autoclass:: ProxyConfig
+    .. autoclass:: RootContext
+        :members:

docs/dev/testing.rst

@@ -0,0 +1,47 @@
+.. _testing:
+
+Testing
+=======
+
+All the mitmproxy projects strive to maintain 100% code coverage. In general,
+patches and pull requests will be declined unless they're accompanied by a
+suitable extension to the test suite.
+
+Our tests are written for the `py.test`_ or nose_ test frameworks.
+At the point where you send your pull request, a command like this:
+
+>>> py.test -n 4 --cov mitmproxy
+
+Should give output something like this:
+
+.. code-block:: none
+
+    > ---------- coverage: platform darwin, python 2.7.2-final-0 --
+    > Name                   Stmts   Miss  Cover   Missing
+    > ----------------------------------------------------
+    > mitmproxy/__init__         0      0   100%
+    > mitmproxy/app              4      0   100%
+    > mitmproxy/cmdline        100      0   100%
+    > mitmproxy/controller      69      0   100%
+    > mitmproxy/dump           150      0   100%
+    > mitmproxy/encoding        39      0   100%
+    > mitmproxy/filt           201      0   100%
+    > mitmproxy/flow           891      0   100%
+    > mitmproxy/proxy          427      0   100%
+    > mitmproxy/script          27      0   100%
+    > mitmproxy/utils          133      0   100%
+    > mitmproxy/version          4      0   100%
+    > ----------------------------------------------------
+    > TOTAL                   2045      0   100%
+    > ----------------------------------------------------
+    > Ran 251 tests in 11.864s
+
+
+There are exceptions to the coverage requirement - for instance, much of the
+console interface code can't sensibly be unit tested. These portions are
+excluded from coverage analysis either in the **.coveragerc** file, or using
+**#pragma no-cover** directives. To keep our coverage analysis relevant, we use
+these measures as sparingly as possible.
+
+.. _nose: https://nose.readthedocs.org/en/latest/
+.. _py.test: https://pytest.org/

docs/features/anticache.rst

@@ -2,7 +2,7 @@
 
 Anticache
 =========
-When the ``--anticache`` option is passed to mitmproxy, it removes headers
+When the :option:`--anticache` option is passed to mitmproxy, it removes headers
 (``if-none-match`` and ``if-modified-since``) that might elicit a
 ``304 not modified`` response from the server. This is useful when you want to make
 sure you capture an HTTP exchange in its totality. It's also often used during
@@ -10,6 +10,6 @@ sure you capture an HTTP exchange in its totality. It's also often used during
 
 
 ================== ======================
-command-line       ``--anticache``
+command-line       :option:`--anticache`
 mitmproxy shortcut :kbd:`o` then :kbd:`a`
 ================== ======================

docs/features/clientreplay.rst

@@ -12,7 +12,7 @@ conversation, where requests may have been made concurrently.
 You may want to use client-side replay in conjunction with the
 :ref:`anticache` option, to make sure the server responds with complete data.
 
-================== ===========
-command-line       ``-c path``
-mitmproxy shortcut :kbd:`R` then :kbd:`c`
-================== ===========
+================== =================
+command-line       :option:`-c path`
+mitmproxy shortcut :kbd:`c`
+================== =================

docs/features/filters.rst

@@ -8,7 +8,7 @@ Filter expressions consist of the following operators:
 
 .. documentedlist::
     :header: "Expression" "Description"
-    :listobject: mitmproxy.flowfilter.help
+    :listobject: mitmproxy.filt.help
 
 - Regexes are Python-style
 - Regexes can be specified as quoted strings
@@ -36,3 +36,4 @@ Anything but requests with a text/html content type:
 .. code-block:: none
 
     !(~q & ~t "text/html")
+

docs/features/passthrough.rst

@@ -10,8 +10,7 @@ mechanism:
   mitmproxy's interception leads to errors. For example, the Twitter app, Windows Update or
   the Apple App Store fail to work if mitmproxy is active.
 - **Convenience:** You really don't care about some parts of the traffic and just want them to go
-  away. Note that mitmproxy's "Limit" option is often the better alternative here, as it is
-  not affected by the limitations listed below.
+  away.
 
 If you want to peek into (SSL-protected) non-HTTP connections, check out the :ref:`tcpproxy`
 feature.
@@ -21,24 +20,21 @@ take a look at the :ref:`responsestreaming` feature.
 How it works
 ------------
 
-================== ======================
-command-line       ``--ignore regex``
+================== =============================
+command-line       :option:`--ignore regex`
 mitmproxy shortcut :kbd:`o` then :kbd:`I`
-================== ======================
+================== =============================
 
 
 mitmproxy allows you to specify a regex which is matched against a ``host:port`` string
 (e.g. "example.com:443") to determine hosts that should be excluded.
 
-Limitations
------------
-
 There are two important quirks to consider:
 
 - **In transparent mode, the ignore pattern is matched against the IP and ClientHello SNI host.** While we usually infer the
-  hostname from the Host header if the ``--host`` argument is passed to mitmproxy, we do not
+  hostname from the Host header if the :option:`--host` argument is passed to mitmproxy, we do not
   have access to this information before the SSL handshake. If the client uses SNI however, then we treat the SNI host as an ignore target.
-- **In regular mode, explicit HTTP requests are never ignored.** [#explicithttp]_ The ignore pattern is
+- In regular mode, explicit HTTP requests are never ignored. [#explicithttp]_ The ignore pattern is
   applied on CONNECT requests, which initiate HTTPS or clear-text WebSocket connections.
 
 Tutorial
@@ -46,7 +42,7 @@ Tutorial
 
 If you just want to ignore one specific domain, there's usually a bulletproof method to do so:
 
-1. Run mitmproxy or mitmdump in verbose mode (``-v``) and observe the ``host:port``
+1. Run mitmproxy or mitmdump in verbose mode (:option:`-v`) and observe the ``host:port``
    information in the serverconnect messages. mitmproxy will filter on these.
 2. Take the ``host:port`` string, surround it with ^ and $, escape all dots (. becomes \\.)
    and use this as your ignore pattern:
@@ -90,7 +86,6 @@ Here are some other examples for ignore patterns:
 
     - :ref:`tcpproxy`
     - :ref:`responsestreaming`
-    - mitmproxy's "Limit" feature
 
 .. rubric:: Footnotes
 

docs/features/proxyauth.rst

@@ -10,8 +10,8 @@ upstream servers. For now, only HTTP Basic authentication is supported. The
 proxy auth options are not compatible with the transparent, socks or reverse proxy
 mode.
 
-================== ======================
-command-line       ``--nonanonymous``,
-                   ``--singleuser USER``,
-                   ``--htpasswd PATH``
-================== ======================
+================== =============================
+command-line       :option:`--nonanonymous`,
+                   :option:`--singleuser USER`,
+                   :option:`--htpasswd PATH`
+================== =============================

docs/features/replacements.rst

@@ -54,7 +54,7 @@ So, you might start **mitmdump** as follows:
 
 This will load the replacement text from the file ``~/xss-exploit``.
 
-Both the ``--replace`` and ``--replace-from-file`` flags can be passed multiple
+Both the :option:`--replace` and :option:`--replace-from-file` flags can be passed multiple
 times.
 
 
@@ -65,8 +65,8 @@ The :kbd:`R` shortcut key in the mitmproxy options menu (:kbd:`o`) lets you add
 replacement hooks using a built-in editor. The context-sensitive help (:kbd:`?`) has
 complete usage information.
 
-================== =======================
-command-line       ``--replace``,
-                   ``--replace-from-file``
+================== =============================
+command-line       :option:`--replace`,
+                   :option:`--replace-from-file`
 mitmproxy shortcut :kbd:`o` then :kbd:`R`
-================== =======================
+================== =============================

docs/features/responsestreaming.rst

@@ -19,9 +19,9 @@ On the command-line
 Streaming can be enabled on the command line for all response bodies exceeding a certain size.
 The SIZE argument understands k/m/g suffixes, e.g. 3m for 3 megabytes.
 
-================== =================
-command-line       ``--stream SIZE``
-================== =================
+================== =============================
+command-line       :option:`--stream SIZE`
+================== =============================
 
 .. warning::
 
@@ -35,13 +35,13 @@ command-line       ``--stream SIZE``
 Customizing Response Streaming
 ------------------------------
 
-You can also use a script to customize exactly which responses are streamed.
+You can also use an :ref:`inlinescripts` to customize exactly
+which responses are streamed.
 
-Responses that should be tagged for streaming by setting their ``.stream``
-attribute to ``True``:
+Responses that should be tagged for streaming by setting their ``.stream`` attribute to ``True``:
 
-.. literalinclude:: ../../examples/complex/stream.py
-   :caption: examples/complex/stream.py
+.. literalinclude:: ../../examples/stream.py
+   :caption: examples/stream.py
    :language: python
 
 Implementation Details
@@ -59,8 +59,8 @@ Modifying streamed data
 If the ``.stream`` attribute is callable, ``.stream`` will wrap the generator that yields all
 chunks.
 
-.. literalinclude:: ../../examples/complex/stream_modify.py
-   :caption: examples/complex/stream_modify.py
+.. literalinclude:: ../../examples/stream_modify.py
+   :caption: examples/stream_modify.py
    :language: python
 
 .. seealso::

docs/features/reverseproxy.rst

@@ -7,9 +7,9 @@ In reverse proxy mode, mitmproxy accepts standard HTTP(S) requests and forwards
 them to the specified upstream server. This is in contrast to :ref:`upstreamproxy`, in which
 mitmproxy forwards HTTP(S) proxy requests to an upstream proxy server.
 
-================== ================================
-command-line       ``-R http[s]://hostname[:port]``
-================== ================================
+================== =====================================
+command-line       :option:`-R http[s]://hostname[:port]`
+================== =====================================
 
 Here, **http[s]** signifies if the proxy should use TLS to connect to the server.
 mitmproxy always accepts both encrypted and unencrypted requests and transforms
@@ -29,14 +29,29 @@ them to what the server expects.
 Host Header
 -----------
 
-In reverse proxy mode, mitmproxy automatically rewrites the Host header to match the
-upstream server. This allows mitmproxy to easily connect to existing endpoints on the
-open web (e.g. ``mitmproxy -R https://example.com``).
+In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this
+may lead to issues with public web servers. For example, consider the following scenario:
 
-However, keep in mind that absolute URLs within the returned document or HTTP redirects will
-NOT be rewritten by mitmproxy. This means that if you click on a link for "http://example.com"
-in the returned web page, you will be taken directly to that URL, bypassing mitmproxy.
+.. code-block:: none
+    :emphasize-lines: 5
 
-One possible way to address this is to modify the hosts file of your OS so that "example.com"
-resolves to your proxy's IP, and then access the proxy by going directly to example.com.
-Make sure that your proxy can still resolve the original IP, or specify an IP in mitmproxy.
+    >>> mitmdump -d -R http://example.com/
+    >>> curl http://localhost:8080/
+
+    >> GET https://example.com/
+        Host: localhost:8080
+        User-Agent: curl/7.35.0
+        [...]
+
+    << 404 Not Found 345B
+
+Since the Host header doesn't match "example.com", an error is returned.
+There are two ways to solve this:
+
+1. Modify the hosts file of your OS so that "example.com" resolves to your proxy's IP.
+   Then, access example.com directly. Make sure that your proxy can still resolve the original IP
+   or specify an IP in mitmproxy.
+2. Use mitmproxy's :ref:`setheaders` feature to rewrite the host header:
+   ``--setheader :~q:Host:example.com``.
+   However, keep in mind that absolute URLs within the returned document or HTTP redirects will
+   cause the client application to bypass the proxy.

docs/features/serverreplay.rst

@@ -13,7 +13,7 @@ By default, :program:`mitmproxy` excludes request headers when matching incoming
 requests with responses from the replay file. This works in most circumstances,
 and makes it possible to replay server responses in situations where request
 headers would naturally vary, e.g. using a different user agent.
-The ``--rheader headername`` command-line option allows you to override
+The :option:`--rheader headername` command-line option allows you to override
 this behaviour by specifying individual headers that should be included in matching.
 
 
@@ -30,23 +30,10 @@ recording. So, if they were in the past at the time of recording, they will be
 in the past at the time of replay, and vice versa. Cookie expiry times are
 updated in a similar way.
 
-You can turn off response refreshing using the ``--norefresh`` argument, or using
+You can turn off response refreshing using the :option:`--norefresh` argument, or using
 the :kbd:`o` options shortcut within :program:`mitmproxy`.
 
-
-Replaying a session recorded in Reverse-proxy Mode
---------------------------------------------------
-
-If you have captured the session in reverse proxy mode, in order to replay it you 
-still have to specify the server URL, otherwise you may get the error: 
-'HTTP protocol error in client request: Invalid HTTP request form (expected authority or absolute...)'.
-
-During replay, when the client's requests match previously recorded requests, then the
-respective recorded responses are simply replayed  by mitmproxy. 
-Otherwise, the unmatched requests is forwarded to the upstream server.
-If forwarding is not desired, you can use the --kill (-k) switch to prevent that.
-
-================== ===========
-command-line       ``-S path``
-mitmproxy shortcut :kbd:`R` then :kbd:`s`
-================== ===========
+================== =================
+command-line       :option:`-S path`
+mitmproxy shortcut :kbd:`S`
+================== =================

docs/features/setheaders.rst

@@ -13,7 +13,7 @@ Example: Set the **Host** header to "example.com" for all requests.
 
     mitmdump -R http://example.com --setheader :~q:Host:example.com
 
-================== =======================
-command-line       ``--setheader PATTERN``
+================== =============================
+command-line       :option:`--setheader PATTERN`
 mitmproxy shortcut :kbd:`o` then :kbd:`H`
-================== =======================
+================== =============================

docs/features/socksproxy.rst

@@ -5,6 +5,6 @@ SOCKS Mode
 
 In this mode, mitmproxy acts as a SOCKS5 proxy server.
 
-================== ===========
-command-line       ``--socks``
-================== ===========
+================== =================
+command-line       :option:`--socks`
+================== =================

docs/features/sticky.rst

@@ -21,7 +21,7 @@ record the authentication process once, and simply replay it on startup every ti
 to interact with the secured resources.
 
 ================== ======================
-command-line       ``-t FILTER``
+command-line       :option:`-t FILTER`
 mitmproxy shortcut :kbd:`o` then :kbd:`t`
 ================== ======================
 
@@ -36,6 +36,6 @@ authentication through the proxy. Note that :program:`mitmproxy` doesn't (yet) s
 replay of HTTP Digest authentication.
 
 ================== ======================
-command-line       ``-u FILTER``
+command-line       :option:`-u FILTER`
 mitmproxy shortcut :kbd:`o` then :kbd:`A`
 ================== ======================

docs/features/tcpproxy.rst

@@ -3,7 +3,7 @@
 TCP Proxy
 =========
 
-In case mitmproxy does not handle a specific protocol, you can exempt
+WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt
 hostnames from processing, so that mitmproxy acts as a generic TCP forwarder.
 This feature is closely related to the :ref:`passthrough` functionality,
 but differs in two important aspects:
@@ -18,7 +18,7 @@ How it works
 ------------
 
 ================== ======================
-command-line       ``--tcp HOST``
+command-line       :option:`--tcp HOST`
 mitmproxy shortcut :kbd:`o` then :kbd:`T`
 ================== ======================
 

docs/features/upstreamcerts.rst

@@ -17,7 +17,7 @@ certs in transparent mode.
 
 Upstream cert sniffing is on by default, and can optionally be turned off.
 
-================== ======================
-command-line       ``--no-upstream-cert``
+================== =============================
+command-line       :option:`--no-upstream-cert`
 mitmproxy shortcut :kbd:`o` then :kbd:`U`
-================== ======================
+================== =============================

docs/features/upstreamproxy.rst

@@ -7,6 +7,6 @@ In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
 requests to a specified upstream proxy server. This is in contrast to :ref:`reverseproxy`,
 in which mitmproxy forwards ordinary HTTP requests to an upstream server.
 
-================== =============================
-command-line       ``-U http://hostname[:port]``
-================== =============================
+================== ===================================
+command-line       :option:`-U http://hostname[:port]`
+================== ===================================

docs/howmitmproxy.rst

@@ -6,17 +6,17 @@ process works will help you deploy it creatively, and take into account its
 fundamental assumptions and how to work around them. This document explains
 mitmproxy's proxy mechanism in detail, starting with the simplest unencrypted
 explicit proxying, and working up to the most complicated interaction -
-transparent proxying of TLS-protected traffic [#tls]_ in the presence of `Server
-Name Indication`_.
+transparent proxying of SSL-protected traffic [#ssl]_ in the presence of `Server Name Indication`_.
 
 Explicit HTTP
 -------------
 
-Configuring the client to use mitmproxy as an explicit proxy is the simplest and
-most reliable way to intercept traffic. The proxy protocol is codified in the
-`HTTP RFC`_, so the behaviour of both the client and the server is well defined,
-and usually reliable. In the simplest possible interaction with mitmproxy, a
-client connects directly to the proxy, and makes a request that looks like this:
+Configuring the client to use mitmproxy as an explicit proxy is the simplest
+and most reliable way to intercept traffic. The proxy protocol is codified in the
+`HTTP RFC`_, so the behaviour of both
+the client and the server is well defined, and usually reliable. In the
+simplest possible interaction with mitmproxy, a client connects directly to the
+proxy, and makes a request that looks like this:
 
 .. code-block:: none
 
@@ -43,11 +43,11 @@ client connects to the proxy and makes a request that looks like this:
 
     CONNECT example.com:443 HTTP/1.1
 
-A conventional proxy can neither view nor manipulate an TLS-encrypted data
+A conventional proxy can neither view nor manipulate an SSL-encrypted data
 stream, so a CONNECT request simply asks the proxy to open a pipe between the
 client and server. The proxy here is just a facilitator - it blindly forwards
 data in both directions without knowing anything about the contents. The
-negotiation of the TLS connection happens over this pipe, and the subsequent
+negotiation of the SSL connection happens over this pipe, and the subsequent
 flow of requests and responses are completely opaque to the proxy.
 
 The MITM in mitmproxy
@@ -58,17 +58,17 @@ name stands for Man-In-The-Middle - a reference to the process we use to
 intercept and interfere with these theoretically opaque data streams. The basic
 idea is to pretend to be the server to the client, and pretend to be the client
 to the server, while we sit in the middle decoding traffic from both sides. The
-tricky part is that the `Certificate Authority`_ system is designed to prevent
-exactly this attack, by allowing a trusted third-party to cryptographically sign
-a server's certificates to verify that they are legit. If this signature doesn't
-match or is from a non-trusted party, a secure client will simply drop the
-connection and refuse to proceed. Despite the many shortcomings of the CA system
-as it exists today, this is usually fatal to attempts to MITM an TLS connection
-for analysis. Our answer to this conundrum is to become a trusted Certificate
-Authority ourselves. Mitmproxy includes a full CA implementation that generates
-interception certificates on the fly. To get the client to trust these
-certificates, we :ref:`register mitmproxy as a trusted CA with the device
-manually <certinstall>`.
+tricky part is that the `Certificate Authority`_ system is
+designed to prevent exactly this attack, by allowing a trusted third-party to
+cryptographically sign a server's SSL certificates to verify that they are
+legit. If this signature doesn't match or is from a non-trusted party, a secure
+client will simply drop the connection and refuse to proceed. Despite the many
+shortcomings of the CA system as it exists today, this is usually fatal to
+attempts to MITM an SSL connection for analysis. Our answer to this conundrum
+is to become a trusted Certificate Authority ourselves. Mitmproxy includes a
+full CA implementation that generates interception certificates on the fly. To
+get the client to trust these certificates, we :ref:`register mitmproxy as a trusted
+CA with the device manually <certinstall>`.
 
 Complication 1: What's the remote hostname?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -89,12 +89,13 @@ information to initiate the pipe, even though it doesn't reveal the remote
 hostname.
 
 Mitmproxy has a cunning mechanism that smooths this over - :ref:`upstream
-certificate sniffing <upstreamcerts>`. As soon as we see the CONNECT request, we
-pause the client part of the conversation, and initiate a simultaneous
-connection to the server. We complete the TLS handshake with the server, and
-inspect the certificates it used. Now, we use the Common Name in the upstream
-certificates to generate the dummy certificate for the client. Voila, we have
-the correct hostname to present to the client, even if it was never specified.
+certificate sniffing <upstreamcerts>`. As soon as we
+see the CONNECT request, we pause the client part of the conversation, and
+initiate a simultaneous connection to the server. We complete the SSL handshake
+with the server, and inspect the certificates it used. Now, we use the Common
+Name in the upstream SSL certificates to generate the dummy certificate for the
+client. Voila, we have the correct hostname to present to the client, even if
+it was never specified.
 
 
 Complication 2: Subject Alternative Name
@@ -102,31 +103,31 @@ Complication 2: Subject Alternative Name
 
 Enter the next complication. Sometimes, the certificate Common Name is not, in
 fact, the hostname that the client is connecting to. This is because of the
-optional `Subject Alternative Name`_ field in the certificate that allows an
-arbitrary number of alternative domains to be specified. If the expected domain
-matches any of these, the client will proceed, even though the domain doesn't
-match the certificate CN. The answer here is simple: when we extract the CN from
-the upstream cert, we also extract the SANs, and add them to the generated dummy
-certificate.
+optional `Subject Alternative Name`_ field in the SSL certificate
+that allows an arbitrary number of alternative domains to be specified. If the
+expected domain matches any of these, the client will proceed, even though the
+domain doesn't match the certificate Common Name. The answer here is simple:
+when we extract the CN from the upstream cert, we also extract the SANs, and
+add them to the generated dummy certificate.
 
 
 Complication 3: Server Name Indication
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-One of the big limitations of vanilla TLS is that each certificate requires its
+One of the big limitations of vanilla SSL is that each certificate requires its
 own IP address. This means that you couldn't do virtual hosting where multiple
-domains with independent certificates share the same IP address. In a world with
-a rapidly shrinking IPv4 address pool this is a problem, and we have a solution
-in the form of the `Server Name Indication`_ extension to the TLS protocols.
-This lets the client specify the remote server name at the start of the TLS
-handshake, which then lets the server select the right certificate to complete
-the process.
+domains with independent certificates share the same IP address. In a world
+with a rapidly shrinking IPv4 address pool this is a problem, and we have a
+solution in the form of the `Server Name Indication`_ extension to
+the SSL and TLS protocols. This lets the client specify the remote server name
+at the start of the SSL handshake, which then lets the server select the right
+certificate to complete the process.
 
 SNI breaks our upstream certificate sniffing process, because when we connect
 without using SNI, we get served a default certificate that may have nothing to
 do with the certificate expected by the client. The solution is another tricky
 complication to the client connection process. After the client connects, we
-allow the TLS handshake to continue until just **after** the SNI value has been
+allow the SSL handshake to continue until just _after_ the SNI value has been
 passed to us. Now we can pause the conversation, and initiate an upstream
 connection using the correct SNI value, which then serves us the correct
 upstream certificate, from which we can extract the expected CN and SANs.
@@ -141,31 +142,32 @@ Lets put all of this together into the complete explicitly proxied HTTPS flow.
 
 1. The client makes a connection to mitmproxy, and issues an HTTP CONNECT request.
 2. Mitmproxy responds with a ``200 Connection Established``, as if it has set up the CONNECT pipe.
-3. The client believes it's talking to the remote server, and initiates the TLS connection.
+3. The client believes it's talking to the remote server, and initiates the SSL connection.
    It uses SNI to indicate the hostname it is connecting to.
-4. Mitmproxy connects to the server, and establishes an TLS connection using the SNI hostname
+4. Mitmproxy connects to the server, and establishes an SSL connection using the SNI hostname
    indicated by the client.
-5. The server responds with the matching certificate, which contains the CN and SAN values
+5. The server responds with the matching SSL certificate, which contains the CN and SAN values
    needed to generate the interception certificate.
 6. Mitmproxy generates the interception cert, and continues the
-   client TLS handshake paused in step 3.
-7. The client sends the request over the established TLS connection.
-8. Mitmproxy passes the request on to the server over the TLS connection initiated in step 4.
+   client SSL handshake paused in step 3.
+7. The client sends the request over the established SSL connection.
+8. Mitmproxy passes the request on to the server over the SSL connection initiated in step 4.
 
 Transparent HTTP
 ----------------
 
-When a transparent proxy is used, the connection is redirected into a proxy at
-the network layer, without any client configuration being required. This makes
-transparent proxying ideal for those situations where you can't change client
-behaviour - proxy-oblivious Android applications being a common example.
+When a transparent proxy is used, the HTTP/S connection is redirected into a
+proxy at the network layer, without any client configuration being required.
+This makes transparent proxying ideal for those situations where you can't
+change client behaviour - proxy-oblivious Android applications being a common
+example.
 
 To achieve this, we need to introduce two extra components. The first is a
 redirection mechanism that transparently reroutes a TCP connection destined for
 a server on the Internet to a listening proxy server. This usually takes the
-form of a firewall on the same host as the proxy server - `iptables`_ on Linux
-or pf_ on OSX. Once the client has initiated the connection, it makes a vanilla
-HTTP request, which might look something like this:
+form of a firewall on the same host as the proxy server - `iptables`_ on Linux or
+pf_ on OSX. Once the client has initiated the connection, it makes a vanilla HTTP request,
+which might look something like this:
 
 .. code-block:: none
 
@@ -173,35 +175,32 @@ HTTP request, which might look something like this:
 
 Note that this request differs from the explicit proxy variation, in that it
 omits the scheme and hostname. How, then, do we know which upstream host to
-forward the request to? The routing mechanism that has performed the redirection
-keeps track of the original destination for us.  Each routing mechanism has a
-different way of exposing this data, so this introduces the second component
-required for working transparent proxying: a host module that knows how to
-retrieve the original destination address from the router. In mitmproxy, this
-takes the form of a built-in set of modules_ that know how to talk to each
-platform's redirection mechanism. Once we have this information, the process is
-fairly straight-forward.
+forward the request to? The routing mechanism that has performed the
+redirection keeps track of the original destination for us.  Each routing
+mechanism has a different way of exposing this data, so this introduces the
+second component required for working transparent proxying: a host module that
+knows how to retrieve the original destination address from the router. In
+mitmproxy, this takes the form of a built-in set of
+modules_ that know how to talk to each platform's redirection mechanism.
+Once we have this information, the process is fairly straight-forward.
 
 .. image:: schematics/how-mitmproxy-works-transparent.png
     :align: center
 
 1. The client makes a connection to the server.
-2. The router redirects the connection to mitmproxy, which is typically
-   listening on a local port of the same host. Mitmproxy then consults the
-   routing mechanism to establish what the original destination was.
+2. The router redirects the connection to mitmproxy, which is typically listening on a local port
+   of the same host. Mitmproxy then consults the routing mechanism to establish what the original
+   destination was.
 3. Now, we simply read the client's request...
 4. ... and forward it upstream.
 
 Transparent HTTPS
 -----------------
 
-The first step is to determine whether we should treat an incoming connection as
-HTTPS. The mechanism for doing this is simple - we use the routing mechanism to
-find out what the original destination port is. All incoming connections pass
-through different layers which can determin the actual protocol to use.
-Automatic TLS detection works for SSLv3, TLS 1.0, TLS 1.1, and TLS 1.2 by
-looking for a *ClientHello* message at the beginning of each connection. This
-works independently of the used TCP port.
+The first step is to determine whether we should treat an incoming connection
+as HTTPS. The mechanism for doing this is simple - we use the routing mechanism
+to find out what the original destination port is. By default, we treat all
+traffic destined for ports 443 and 8443 as SSL.
 
 From here, the process is a merger of the methods we've described for
 transparently proxying HTTP, and explicitly proxying HTTPS. We use the routing
@@ -215,21 +214,21 @@ explicit HTTPS connections to establish the CN and SANs, and cope with SNI.
 2. The router redirects the connection to mitmproxy, which is typically listening on a local port
    of the same host. Mitmproxy then consults the routing mechanism to establish what the original
    destination was.
-3. The client believes it's talking to the remote server, and initiates the TLS connection.
+3. The client believes it's talking to the remote server, and initiates the SSL connection.
    It uses SNI to indicate the hostname it is connecting to.
-4. Mitmproxy connects to the server, and establishes an TLS connection using the SNI hostname
+4. Mitmproxy connects to the server, and establishes an SSL connection using the SNI hostname
    indicated by the client.
-5. The server responds with the matching certificate, which contains the CN and SAN values
+5. The server responds with the matching SSL certificate, which contains the CN and SAN values
    needed to generate the interception certificate.
-6. Mitmproxy generates the interception cert, and continues the client TLS handshake paused in
+6. Mitmproxy generates the interception cert, and continues the client SSL handshake paused in
    step 3.
-7. The client sends the request over the established TLS connection.
-8. Mitmproxy passes the request on to the server over the TLS connection initiated in step 4.
+7. The client sends the request over the established SSL connection.
+8. Mitmproxy passes the request on to the server over the SSL connection initiated in step 4.
 
 .. rubric:: Footnotes
 
-.. [#tls] The use of "TLS" refers to both SSL (outdated and insecure) and TLS
-          (1.0 and up) in the generic sense, unless otherwise specified.
+.. [#ssl] I use "SSL" to refer to both SSL and TLS in the generic sense, unless otherwise
+          specified.
 
 .. _Server Name Indication: https://en.wikipedia.org/wiki/Server_Name_Indication
 .. _HTTP RFC: https://tools.ietf.org/html/rfc7230

docs/index.rst

@@ -17,7 +17,6 @@
 
     mitmproxy
     mitmdump
-    mitmweb
     config
 
 .. toctree::
@@ -47,15 +46,13 @@
     transparent
     transparent/linux
     transparent/osx
-    transparent/openbsd
 
 .. toctree::
     :hidden:
     :caption: Scripting
 
-    scripting/overview
-    scripting/events
-    scripting/api
+    scripting/inlinescripts
+    scripting/mitmproxy
 
 
 .. toctree::
@@ -66,26 +63,21 @@
     tutorials/gamecenter
     tutorials/transparent-dhcp
 
-
 .. toctree::
     :hidden:
-    :caption: Pathod & Pathoc
+    :caption: Hacking
 
-    pathod/intro
-    pathod/language
-    pathod/library
-    pathod/test
-
-
-.. toctree::
-    :hidden:
-    :caption: Development
-
-    dev/contributing
+    dev/architecture
+    dev/testing
     dev/sslkeylogfile
+    dev/protocols
+    dev/proxy
+    dev/exceptions
+    dev/models
 
 ..  Indices and tables
     ==================
 
     * :ref:`genindex`
     * :ref:`modindex`
+

docs/install.rst

@@ -3,150 +3,101 @@
 Installation
 ============
 
-Please follow the steps for your operating system.
+.. _install-ubuntu:
 
-Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or
-:ref:`mitmweb` from a terminal.
+Installation On Ubuntu
+----------------------
+
+Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
+This was tested on a fully patched installation of Ubuntu 14.04.
+
+>>> sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev
+>>> sudo pip install mitmproxy
+
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+
+On **Ubuntu 12.04** (and other systems with an outdated version of pip),
+you may need to update pip using ``pip install -U pip`` before installing mitmproxy.
+
+Installation From Source (Ubuntu)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, install the dependencies as you would for a regular
+mitmproxy installation (see :ref:`install-ubuntu`).
+Then see the Hacking_ section of the README on GitHub.
 
 
-.. _install-macos:
 
-Installation on macOS
----------------------
+Installation On Mac OS X
+------------------------
 
-You can use Homebrew to install everything:
+The easiest way to get up and running on OSX is to download the pre-built binary packages from
+`mitmproxy.org`_.
 
-.. code:: bash
+There are a few bits of customization you might want to do to make mitmproxy comfortable to use on
+OSX. The default color scheme is optimized for a dark background terminal, but you can select a
+palette for a light terminal background with the ``--palette`` option.
+You can use the OSX **open** program to create a simple and effective ``~/.mailcap`` file to view
+request and response bodies:
 
-    brew install mitmproxy
+.. code-block:: none
 
-Or you can download the pre-built binary packages from our `releases`_.
+    application/*; /usr/bin/open -Wn %s
+    audio/*; /usr/bin/open -Wn %s
+    image/*; /usr/bin/open -Wn %s
+    video/*; /usr/bin/open -Wn %s
+
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
 
-.. _install-windows:
+Installation From Source (Mac OS X)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Installation on Windows
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, there are a few OS X specific things to keep in mind.
+
+- Make sure that XCode is installed from the App Store, and that the command-line tools have been
+  downloaded (XCode/Preferences/Downloads).
+- If you're running a Python interpreter installed with homebrew (or similar), you may have to
+  install some dependencies by hand.
+
+Then see the Hacking_ section of the README on GitHub.
+
+Installation On Windows
 -----------------------
 
-The recommended way to install mitmproxy on Windows is to use the installer
-provided at `mitmproxy.org`_. After installation, you'll find shortcuts for
-:ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu.
-Both executables are  added to your PATH and can be invoked from the command
-line.
-
 .. note::
-    Mitmproxy's console interface is not supported on Windows, but you can use
-    mitmweb (the web-based interface) and mitmdump.
+    Please note that mitmdump is the only component of mitmproxy that is supported on Windows at
+    the moment.
 
-.. _install-linux:
-
-Installation on Linux
----------------------
-
-The recommended way to run mitmproxy on Linux is to use the pre-built binaries
-provided at `releases`_.
-
-Our pre-built binaries provide you with the latest version of mitmproxy, a
-self-contained Python 3.5 environment and a recent version of OpenSSL that
-supports HTTP/2. Of course, you can also install mitmproxy from source if you
-prefer that (see :ref:`install-advanced`).
-
-.. _install-advanced:
-
-Advanced Installation
----------------------
-
-.. _install-docker:
-
-Docker Images
-^^^^^^^^^^^^^
-
-You can also use the official mitmproxy images from `DockerHub`_. That being
-said, our portable binaries are just as easy to install and even easier to use. 😊
+    **There is no interactive user interface on Windows.**
 
 
-.. _install-arch:
+First, install the latest version of Python 2.7 from the `Python website`_.
+If you already have an older version of Python 2.7 installed, make sure to install pip_
+(pip is included in Python 2.7.9+ by default).
 
-Installation on Arch Linux
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+Next, add Python and the Python Scripts directory to your **PATH** variable.
+You can do this easily by running the following in powershell:
 
-mitmproxy has been added into the [community] repository. Use pacman to install it:
+>>> [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User")
 
->>> sudo pacman -S mitmproxy
+Now, you can install mitmproxy by running
 
+>>> pip install mitmproxy
 
-.. _install-source-ubuntu:
+Once the installation is complete, you can run :ref:`mitmdump` from a command prompt.
 
-Installation from Source on Ubuntu
+Installation From Source (Windows)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Ubuntu comes with Python but we need to install pip3, python3-dev and several
-libraries. This was tested on a fully patched installation of Ubuntu 16.04.
-
-.. code:: bash
-
-   sudo apt-get install python3-dev python3-pip libffi-dev libssl-dev
-   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
-
-On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install
-a newer version of Python. mitmproxy requires Python 3.5 or higher. Please take
-a look at pyenv_. Make sure to have an up-to-date version of pip by running
-``pip3 install -U pip``.
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to
+get set up to contribute to the project, install Python as outlined above, then see the
+Hacking_ section of the README on GitHub.
 
 
-.. _install-source-fedora:
-
-Installation from Source on Fedora
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Fedora comes with Python but we need to install pip3, python3-dev and several
-libraries. This was tested on a fully patched installation of Fedora 24.
-
-.. code:: bash
-
-   sudo dnf install make gcc redhat-rpm-config python3-devel python3-pip libffi-devel openssl-devel
-   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
-
-Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
-
-
-
-.. _install-source-windows:
-
-🐱💻 Installation from Source on Windows
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. note::
-    Mitmproxy's console interface is not supported on Windows, but you can use
-    mitmweb (the web-based interface) and mitmdump.
-
-First, install the latest version of Python 3.5 or later from the `Python
-website`_. During installation, make sure to select `Add Python to PATH`.
-
-Mitmproxy has no other dependencies on Windows. You can now install mitmproxy by running
-
-.. code:: powershell
-
-    pip3 install mitmproxy
-
-
-
-.. _install-dev-version:
-
-Latest Development Version
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-If you would like to install mitmproxy directly from the master branch on GitHub
-or would like to get set up to contribute to the project, install the
-dependencies as you would for a regular installation from source. Then see the
-project's README_ on GitHub. You can check your system information
-by running: ``mitmproxy --version``
-
-
-.. _README: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst
-.. _releases: https://github.com/mitmproxy/mitmproxy/releases
+.. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst#hacking
 .. _mitmproxy.org: https://mitmproxy.org/
 .. _`Python website`: https://www.python.org/downloads/windows/
 .. _pip: https://pip.pypa.io/en/latest/installing.html
-.. _pyenv: https://github.com/yyuu/pyenv
-.. _DockerHub: https://hub.docker.com/r/mitmproxy/mitmproxy/

docs/introduction.rst

@@ -1,25 +1,24 @@
 Introduction
 ============
 
-**mitmproxy** is an interactive man-in-the-middle proxy for HTTP and HTTPS
+**mitmproxy** is an interactive, SSL-capable man-in-the-middle proxy for HTTP
 with a console interface.
 
 **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
-**mitmweb** is a web-based interface for mitmproxy.
-
 Documentation, tutorials and distribution packages can be found on the
 mitmproxy website: `mitmproxy.org <https://mitmproxy.org/>`_
 
 
 .. rubric:: Features
 
-- Intercept HTTP & HTTPS requests and responses and modify them on the fly
-- Save complete HTTP conversations for later replay and analysis
-- Replay the client-side of an HTTP conversations
-- Replay HTTP responses of a previously recorded server
-- Reverse proxy mode to forward traffic to a specified server
-- Transparent proxy mode on OSX and Linux
-- Make scripted changes to HTTP traffic using Python
-- SSL/TLS certificates for interception are generated on the fly
-- And much, much more...
+
+- Intercept HTTP requests and responses and modify them on the fly.
+- Save complete HTTP conversations for later replay and analysis.
+- Replay the client-side of an HTTP conversations.
+- Replay HTTP responses of a previously recorded server.
+- Reverse proxy mode to forward traffic to a specified server.
+- Transparent proxy mode on OSX and Linux.
+- Make scripted changes to HTTP traffic using Python.
+- SSL certificates for interception are generated on the fly.
+- And much, much more.

docs/mitmdump.rst

@@ -7,7 +7,7 @@ mitmdump
 
 **mitmdump** is the command-line companion to mitmproxy. It provides
 tcpdump-like functionality to let you view, record, and programmatically
-transform HTTP traffic. See the ``--help`` flag output for complete
+transform HTTP traffic. See the :option:`--help` flag output for complete
 documentation.
 
 
@@ -28,7 +28,7 @@ Filtering saved traffic
 
 >>> mitmdump -nr infile -w outfile "~m post"
 
-Start mitmdump without binding to the proxy port (``-n``), read all flows from
+Start mitmdump without binding to the proxy port (:option:`-n`), read all flows from
 infile, apply the specified filter expression (only match POSTs), and write to
 outfile.
 
@@ -38,8 +38,8 @@ Client replay
 
 >>> mitmdump -nc outfile
 
-Start mitmdump without binding to the proxy port (``-n``), then replay all
-requests from outfile (``-c filename``). Flags combine in the obvious way, so
+Start mitmdump without binding to the proxy port (:option:`-n`), then replay all
+requests from outfile (:option:`-c filename`). Flags combine in the obvious way, so
 you can replay requests from one file, and write the resulting flows to
 another:
 

docs/mitmweb.rst

@@ -1,18 +0,0 @@
-.. _mitmweb:
-.. program:: mitmweb
-
-mitmweb
-=======
-
-**mitmweb** is mitmproxy's web-based user interface that allows interactive
-examination and modification of HTTP traffic. Like mitmproxy, it differs from
-mitmdump in that all flows are kept in memory, which means that it's intended
-for taking and manipulating small-ish samples.
-
-.. warning::
-
-  Mitmweb is currently in beta. We consider it stable for all features currently
-  exposed in the UI, but it still misses a lot of mitmproxy's features.
-
-
-.. image:: screenshots/mitmweb.png

docs/modd.conf

@@ -1,6 +1,6 @@
 @build = ./_build
 
-** !_build/** ../mitmproxy/**/*.py {
+** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py {
     prep: sphinx-build -W -d @build/doctrees -b html . @build/html
     daemon: devd -m @build/html
 }

docs/pathod/intro.rst

@@ -1,307 +0,0 @@
-.. _intro:
-
-Pathology 101
-=============
-
-
-pathod
-------
-
-Pathod is a pathological HTTP daemon designed to let you craft almost any
-conceivable HTTP response, including ones that creatively violate the
-standards. HTTP responses are specified using a :ref:`small, terse language
-<language>` which pathod shares with its evil twin :ref:`pathoc`. To start
-playing with pathod, fire up the daemon:
-
->>> pathod
-
-By default, the service listens on port 9999 of localhost, and the default
-crafting anchor point is the path **/p/**. Anything after this URL prefix is
-treated as a response specifier. So, hitting the following URL will generate an
-HTTP 200 response with 100 bytes of random data:
-
-    http://localhost:9999/p/200:b@100
-
-See the :ref:`language documentation <language>` to get (much) fancier. The
-pathod daemon also takes a range of configuration options. To view those, use
-the command-line help:
-
->>> pathod --help
-
-Mimicing a proxy
-^^^^^^^^^^^^^^^^
-
-Pathod automatically responds to both straight HTTP and proxy requests. For
-proxy requests, the upstream host is ignored, and the path portion of the URL
-is used to match anchors. This lets you test software that supports a proxy
-configuration by spoofing responses from upstream servers.
-
-By default, we treat all proxy CONNECT requests as HTTPS traffic, serving the
-response using either pathod's built-in certificates, or the cert/key pair
-specified by the user. You can over-ride this behaviour if you're testing a
-client that makes a non-SSL CONNECT request using the **-C** command-line
-option.
-
-Anchors
-^^^^^^^
-
-Anchors provide an alternative to specifying the response in the URL. Instead,
-you attach a response to a pre-configured anchor point, specified with a regex.
-When a URL matching the regex is requested, the specified response is served.
-
->>> pathod -a "/foo=200"
-
-Here, "/foo" is the regex specifying the anchor path, and the part after the "="
-is a response specifier.
-
-
-File Access
-^^^^^^^^^^^
-
-There are two operators in the :ref:`language <language>` that load contents
-from file - the **+** operator to load an entire request specification from
-file, and the **>** value specifier. In pathod, both of these operators are
-restricted to a directory specified at startup, or disabled if no directory is
-specified:
-
->>> pathod -d ~/staticdir"
-
-
-Internal Error Responses
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-Pathod uses the non-standard 800 response code to indicate internal errors, to
-distinguish them from crafted responses. For example, a request to:
-
-    http://localhost:9999/p/foo
-
-... will return an 800 response because "foo" is not a valid page specifier.
-
-
-
-
-
-.. _pathoc:
-
-
-pathoc
-------
-
-Pathoc is a perverse HTTP daemon designed to let you craft almost any
-conceivable HTTP request, including ones that creatively violate the standards.
-HTTP requests are specified using a :ref:`small, terse language <language>`,
-which pathod shares with its server-side twin pathod. To view pathoc's complete
-range of options, use the command-line help:
-
->>> pathoc --help
-
-
-Getting Started
-^^^^^^^^^^^^^^^
-
-The basic pattern for pathoc commands is as follows:
-
-    pathoc hostname request [request ...]
-
-That is, we specify the hostname to connect to, followed by one or more
-requests. Lets start with a simple example::
-
-    > pathoc google.com get:/
-    07-06-16 12:13:43: >> 'GET':/
-    << 302 Found: 261 bytes
-
-Here, we make a GET request to the path / on port 80 of google.com. Pathoc's
-output tells us that the server responded with a 302 redirection. We can tell
-pathoc to connect using SSL, in which case the default port is changed to 443
-(you can over-ride the default port with the **-p** command-line option)::
-
-    > pathoc -s www.google.com get:/
-    07-06-16 12:14:56: >> 'GET':/
-    << 302 Found: 262 bytes
-
-
-Multiple Requests
-^^^^^^^^^^^^^^^^^
-
-There are two ways to tell pathoc to issue multiple requests. The first is to specify
-them on the command-line, like so::
-
-    > pathoc google.com get:/ get:/
-    07-06-16 12:21:04: >> 'GET':/
-    << 302 Found: 261 bytes
-    07-06-16 12:21:04: >> 'GET':/
-    << 302 Found: 261 bytes
-
-In this case, pathoc issues the specified requests over the same TCP connection -
-so in the above example only one connection is made to google.com
-
-The other way to issue multiple requests is to use the **-n** flag::
-
-    > pathoc -n 2 google.com get:/
-    07-06-16 12:21:04: >> 'GET':/
-    << 302 Found: 261 bytes
-    07-06-16 12:21:04: >> 'GET':/
-    << 302 Found: 261 bytes
-
-The output is identical, but two separate TCP connections are made to the
-upstream server. These two specification styles can be combined::
-
-    pathoc -n 2 google.com get:/ get:/
-
-
-Here, two distinct TCP connections are made, with two requests issued over
-each.
-
-
-
-Basic Fuzzing
-^^^^^^^^^^^^^
-
-The combination of pathoc's powerful request specification language and a few
-of its command-line options makes for quite a powerful basic fuzzer. Here's an
-example::
-
-    pathoc -e -I 200 -t 2 -n 1000 localhost get:/:b@10:ir,@1
-
-The request specified here is a valid GET with a body consisting of 10 random bytes,
-but with 1 random byte inserted in a random place. This could be in the headers,
-in the initial request line, or in the body itself. There are a few things
-to note here:
-
-- Corrupting the request in this way will often make the server enter a state where
-  it's awaiting more input from the client. This is where the
-  **-t** option comes in, which sets a timeout that causes pathoc to
-  disconnect after two seconds.
-- The **-n** option tells pathoc to repeat the request 1000 times.
-- The **-I** option tells pathoc to ignore HTTP 200 response codes.
-  You can use this to fine-tune what pathoc considers to be an exceptional
-  condition, and therefore log-worthy.
-- The **-e** option tells pathoc to print an explanation of each logged
-  request, in the form of an expanded pathoc specification with all random
-  portions and automatic header additions resolved. This lets you precisely
-  replay a request that triggered an error.
-
-
-Interacting with Proxies
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-Pathoc has a reasonably sophisticated suite of features for interacting with
-proxies. The proxy request syntax very closely mirrors that of straight HTTP,
-which means that it is possible to make proxy-style requests using pathoc
-without any additional syntax, by simply specifying a full URL instead of a
-simple path:
-
->>> pathoc -p 8080 localhost "get:'http://google.com'"
-
-Another common use case is to use an HTTP CONNECT request to probe remote
-servers via a proxy. This is done with the **-c** command-line option, which
-allows you to specify a remote host and port pair:
-
->>> pathoc -c google.com:80 -p 8080 localhost get:/
-
-Note that pathoc does **not** negotiate SSL without being explictly instructed
-to do so. If you're making a CONNECT request to an SSL-protected resource, you
-must also pass the **-s** flag:
-
->>> pathoc -sc google.com:443 -p 8080 localhost get:/
-
-
-
-Embedded response specification
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-One interesting feature of the Request specification language is that you can
-embed a response specification in it, which is then added to the request path.
-Here's an example:
-
->>> pathoc localhost:9999 "get:/p/:s'401:ir,@1'"
-
-This crafts a request that connects to the pathod server, and which then crafts
-a response that generates a 401, with one random byte embedded at a random
-point. The response specification is parsed and expanded by pathoc, so you see
-syntax errors immediately. This really becomes handy when combined with the
-**-e** flag to show the expanded request::
-
-    07-06-16 12:32:01: >> 'GET':/p/:s'401:i35,\x27\\x1b\x27:h\x27Content-Length\x27=\x270\x27:h\x27Content-Length\x27=\x270\x27':h'Host'='localhost'
-    << 401 Unauthorized: 0 bytes
-
-Note that the embedded response has been resolved *before* being sent to
-the server, so that "ir,@1" (embed a random byte at a random location) has
-become "i15,\'o\'" (embed the character "o" at offset 15). You now have a
-pathoc request specification that is precisely reproducible, even with random
-components. This feature comes in terribly handy when testing a proxy, since
-you can now drive the server response completely from the client, and have a
-complete log of reproducible requests to analyze afterwards.
-
-
-Request Examples
-----------------
-
-.. list-table::
-    :widths: 50 50
-    :header-rows: 0
-
-    * - get:/
-      - Get path /
-
-    * - get:/:b@100
-      - 100 random bytes as the body
-
-    * - get:/:h"Etag"="&;drop table browsers;"
-      - Add a header
-
-    * - get:/:u"&;drop table browsers;"
-      - Add a User-Agent header
-
-    * - get:/:b@100:dr
-      - Drop the connection randomly
-
-    * - get:/:b@100,ascii:ir,@1
-      - 100 ASCII bytes as the body, and randomly inject a random byte
-
-    * - ws:/
-      - Initiate a websocket handshake.
-
-
-Response Examples
------------------
-
-.. list-table::
-    :widths: 50 50
-    :header-rows: 0
-
-
-    * - 200
-      - A basic HTTP 200 response.
-
-    * - 200:r
-      - A basic HTTP 200 response with no Content-Length header. This will hang.
-
-    * - 200:da
-      - Server-side disconnect after all content has been sent.
-
-    * - 200:b\@100
-      - 100 random bytes as the body. A Content-Length header is added, so the disconnect
-        is no longer needed.
-
-    * - 200:b\@100:h"Etag"="';drop table servers;"
-      - Add a Server header
-
-    * - 200:b\@100:dr
-      - Drop the connection randomly
-
-    * - 200:b\@100,ascii:ir,@1
-      - 100 ASCII bytes as the body, and randomly inject a random byte
-
-    * - 200:b\@1k:c"text/json"
-      - 1k of random bytes, with a text/json content type
-
-    * - 200:b\@1k:p50,120
-      - 1k of random bytes, pause for 120 seconds after 50 bytes
-
-    * - 200:b\@1k:pr,f
-      - 1k of random bytes, but hang forever at a random location
-
-    * - 200:b\@100:h\@1k,ascii_letters='foo'
-      - 100 ASCII bytes as the body, randomly generated 100k header name, with the value
-        'foo'.

docs/pathod/language.rst

@@ -1,257 +0,0 @@
-.. _language:
-
-language spec
-=============
-
-************
-HTTP Request
-************
-
-    **method:path:[colon-separated list of features]**
-
-.. list-table::
-    :widths: 20 80
-    :header-rows: 0
-
-    * - method
-      - A :ref:`VALUE` specifying the HTTP method to
-        use. Standard methods do not need to be enclosed in quotes, while
-        non-standard methods can be specified as quoted strings.
-
-        The special method **ws** creates a valid websocket upgrade
-        GET request, and signals to pathoc to switch to websocket recieve
-        mode if the server responds correctly. Apart from that, websocket
-        requests are just like any other, and all aspects of the request
-        can be over-ridden.
-    * - h\:\ :ref:`VALUE`\ =\ :ref:`VALUE`\
-      - Set a header.
-    * - r
-      - Set the **raw** flag on this response. Pathod will not calculate a
-        *Content-Length* header if a body is set.
-    * - c\ :ref:`VALUE`
-      - A shortcut for setting the Content-Type header. Equivalent to
-        ``h"Content-Type"=VALUE``
-    * - u\ :ref:`VALUE`
-        uSHORTCUT
-      - Set a User-Agent header on this request. You can specify either a
-        complete :ref:`VALUE`, or a User-Agent shortcut: **android**,
-        **blackberry**, **bingbot**, **chrome**, **firefox**, **googlebot**,
-        **ie9**, **ipad**, **iphone**, **safari**.
-    * - b\ :ref:`VALUE`
-      - Set the body. The appropriate Content-Length header is added
-        automatically unless the **r** flag is set.
-    * - s\ :ref:`VALUE`
-      - An embedded Response specification, appended to the path of the request.
-    * - x\ :ref:`INTEGER`
-      - Repeat this message N times.
-    * - d\ :ref:`OFFSET`
-      - Disconnect after OFFSET bytes (HTTP/1 only).
-    * - i\ :ref:`OFFSET`,\ :ref:`VALUE`
-      - Inject the specified value at the offset (HTTP/1 only)
-    * - p\ :ref:`OFFSET`,SECONDS
-      - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer
-        or "f" to pause forever (HTTP/1 only)
-
-
-*************
-HTTP Response
-*************
-
-    **code:[colon-separated list of features]**
-
-.. list-table::
-    :widths: 20 80
-    :header-rows: 0
-
-    * - code
-      - An integer specifying the HTTP response code.
-
-        The special method **ws** creates a valid websocket upgrade
-        response (code 101), and moves pathod to websocket mode. Apart
-        from that, websocket responses are just like any other, and all
-        aspects of the response can be over-ridden.
-    * - m\ :ref:`VALUE`
-      - HTTP Reason message. Automatically chosen according to the response
-        code if not specified. (HTTP/1 only)
-    * - h\:\ :ref:`VALUE`\ =\ :ref:`VALUE`\
-      - Set a header.
-    * - r
-      - Set the **raw** flag on this response. Pathod will not calculate a
-        *Content-Length* header if a body is set.
-    * - l\ :ref:`VALUE`
-      - A shortcut for setting the Location header. Equivalent to
-        ``h"Location"=VALUE``
-    * - c\ :ref:`VALUE`
-      - A shortcut for setting the Content-Type header. Equivalent to
-        ``h"Content-Type"=VALUE``
-    * - b\ :ref:`VALUE`
-      - Set the body. The appropriate Content-Length header is added
-        automatically unless the **r** flag is set.
-    * - d\ :ref:`OFFSET`
-      - Disconnect after OFFSET bytes (HTTP/1 only).
-    * - i\ :ref:`OFFSET`,\ :ref:`VALUE`
-      - Inject the specified value at the offset (HTTP/1 only)
-    * - p\ :ref:`OFFSET`,SECONDS
-      - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer
-        or "f" to pause forever (HTTP/1 only)
-
-***************
-Websocket Frame
-***************
-
-    **wf:[colon-separated list of features]**
-
-.. list-table::
-    :widths: 20 80
-    :header-rows: 0
-
-    * - b\ :ref:`VALUE`
-      - Set the frame payload. If a masking key is present, the value is
-        encoded automatically.
-    * - c\ :ref:`INTEGER`
-      - Set the op code. This can either be an integer from 0-15, or be one of
-        the following opcode names: **text** (the default), **continue**,
-        **binary**, **close**, **ping**, **pong**.
-    * - d\ :ref:`OFFSET`
-      - Disconnect after OFFSET bytes
-    * - i\ :ref:`OFFSET`,\ :ref:`VALUE`
-      - Inject the specified value at the offset
-    * - p\ :ref:`OFFSET`,SECONDS
-      - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer
-        or "f" to pause forever
-    * - x\ :ref:`INTEGER`
-      - Repeat this message N times.
-    * - [-]fin
-      - Set or un-set the **fin** bit.
-    * - k\ :ref:`VALUE`
-      - Set the masking key. The resulting value must be exactly 4 bytes long.
-        The special form **knone** specifies that no key should be set, even if
-        the mask bit is on.
-    * - l\ :ref:`INTEGER`
-      - Set the payload length in the frame header, regardless of the actual
-        body length.
-    * - [-]mask
-      - Set or un-set the <b>mask</b> bit.
-    * - r\ :ref:`VALUE`
-      - Set the raw frame payload. This disables masking, even if the key is present.
-    * - [-]rsv1
-      - Set or un-set the **rsv1** bit.
-    * - [-]rsv2
-      - Set or un-set the **rsv2** bit.
-    * - [-]rsv2
-      - Set or un-set the **rsv2** bit.
-
-
-
-**********
-Data types
-**********
-
-.. _INTEGER:
-
-INTEGER
-^^^^^^^
-
-.. _OFFSET:
-
-OFFSET
-^^^^^^
-
-Offsets are calculated relative to the base message, before any injections or
-other transforms are applied. They have 3 flavors:
-
-=======                 ==========================
-integer                 An integer byte offset
-**r**                   A random location
-**a**                   The end of the message
-=======                 ==========================
-
-
-.. _VALUE:
-
-VALUE
-^^^^^
-
-Literals
-""""""""
-
-Literal values are specified as a quoted strings::
-
-    "foo"
-
-Either single or double quotes are accepted, and quotes can be escaped with
-backslashes within the string::
-
-    'fo\'o'
-
-Literal values can contain Python-style backslash escape sequences::
-
-    'foo\r\nbar'
-
-
-
-Generated
-"""""""""
-
-An @-symbol lead-in specifies that generated data should be used. There are two
-components to a generator specification - a size, and a data type. By default
-pathod assumes a data type of "bytes".
-
-Here's a value specifier for generating 100 bytes::
-
-    @100
-
-You can use standard suffixes to indicate larger values. Here, for instance, is
-a specifier for generating 100 megabytes:
-
-    @100m
-
-Data is generated and served efficiently - if you really want to send a
-terabyte of data to a client, pathod can do it. The supported suffixes are:
-
-==========          ====================
-b                   1024**0 (bytes)
-k                   1024**1 (kilobytes)
-m                   1024**2 (megabytes)
-g                   1024**3 (gigabytes)
-t                   1024**4 (terabytes)
-==========          ====================
-
-Data types are separated from the size specification by a comma. This specification
-generates 100mb of ASCII::
-
-    @100m,ascii
-
-Supported data types are:
-
-=================          ==============================================
-ascii                      All ASCII characters
-ascii_letters              A-Za-z
-ascii_lowercase            a-z
-ascii_uppercase            A-Z
-bytes                      All 256 byte values
-digits                     0-9
-hexdigits                  0-f
-octdigits                  0-7
-punctuation                !"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ and space
-whitespace                 \\t \\n \\x0b \\x0c \\r and space
-=================          ==============================================
-
-
-
-Files
-"""""
-
-You can load a value from a specified file path. To do so, you have to specify a
-_staticdir_ option to pathod on the command-line, like so:
-
->>> pathod -d ~/myassets
-
-All paths are relative paths under this directory. File loads are indicated by
-starting the value specifier with the left angle bracket::
-
-    <my/path
-
-The path value can also be a quoted string, with the same syntax as literals::
-
-    <"my/path"

docs/pathod/library.rst

@@ -1,14 +0,0 @@
-.. _library:
-
-pathod library
-==============
-
-Behind the pathod and pathoc command-line tools lurks the **pathod** library, a
-powerful way to manipulate and serve HTTP requests and responses from code. The
-canonical documentation for the library is in the code, and can be accessed
-using pydoc.
-
-
-.. literalinclude:: ../../examples/pathod/libpathod_pathoc.py
-   :caption: examples/pathod/libpathod_pathoc.py
-   :language: python

docs/pathod/test.rst

@@ -1,38 +0,0 @@
-.. _test:
-
-pathod.test
-===========
-
-The **pathod.test** module is a light, flexible testing layer for HTTP clients.
-It works by firing up a Pathod instance in a separate thread, letting you use
-Pathod's full abilities to generate responses, and then query Pathod's internal
-logs to establish what happened. All the mechanics of startup, shutdown, finding
-free ports and so forth are taken care of for you.
-
-The canonical docs can be accessed using pydoc:
-
->>> pydoc pathod.test
-
-The remainder of this page demonstrates some common interaction patterns using
-`Nose`_. These examples are
-also applicable with only minor modification to most commonly used Python testing
-engines.
-
-
-Context Manager
----------------
-
-.. literalinclude:: ../../examples/pathod/test_context.py
-   :caption: examples/pathod/test_context.py
-   :language: python
-
-
-One instance per test
----------------------
-
-.. literalinclude:: ../../examples/pathod/test_setup.py
-   :caption: examples/pathod/test_setup.py
-   :language: python
-
-
-.. _Nose: https://nose.readthedocs.org/en/latest/

docs/scripting/api.rst

@@ -1,42 +0,0 @@
-.. _api:
-
-
-API
-===
-
-- Errors
-    - `mitmproxy.flow.Error <#mitmproxy.flow.Error>`_
-- HTTP
-    - `mitmproxy.http.HTTPRequest <#mitmproxy.http.HTTPRequest>`_
-    - `mitmproxy.http.HTTPResponse <#mitmproxy.http.HTTPResponse>`_
-    - `mitmproxy.http.HTTPFlow <#mitmproxy.http.HTTPFlow>`_
-- Logging
-    - `mitmproxy.log.Log <#mitmproxy.controller.Log>`_
-    - `mitmproxy.log.LogEntry <#mitmproxy.controller.LogEntry>`_
-
-
-Errors
-------
-
-.. autoclass:: mitmproxy.flow.Error
-    :inherited-members:
-
-HTTP
-----
-
-.. autoclass:: mitmproxy.http.HTTPRequest
-    :inherited-members:
-
-.. autoclass:: mitmproxy.http.HTTPResponse
-    :inherited-members:
-
-.. autoclass:: mitmproxy.http.HTTPFlow
-    :inherited-members:
-
-Logging
---------
-
-.. autoclass:: mitmproxy.log.Log
-    :inherited-members:
-.. autoclass:: mitmproxy.log.LogEntry
-    :inherited-members:

docs/scripting/events.rst

@@ -1,248 +0,0 @@
-.. _events:
-
-Events
-=======
-
-General
--------
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: configure(options, updated)
-        - Called once on startup, and whenever options change.
-
-          *options*
-            An ``options.Options`` object with the total current configuration
-            state of mitmproxy.
-          *updated*
-            A set of strings indicating which configuration options have been
-            updated. This contains all options when *configure* is called on
-            startup, and only changed options subsequently.
-
-    *   - .. py:function:: done()
-        - Called once when the script shuts down, either because it's been
-          unloaded, or because the proxy itself is shutting down.
-
-    *   - .. py:function:: log(entry)
-        - Called whenever an event log is added.
-
-          *entry*
-            An ``controller.LogEntry`` object - ``entry.msg`` is the log text,
-            and ``entry.level`` is the urgency level ("debug", "info", "warn",
-            "error").
-
-    *   - .. py:function:: start()
-        - Called once on startup, before any other events. If you return a
-          value  from this event, it will replace the current addon. This
-          allows you to, "boot into" an addon implemented as a class instance
-          from the module level.
-
-    *   - .. py:function:: tick()
-        - Called at a regular sub-second interval as long as the addon is
-          executing.
-
-
-Connection
-----------
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: clientconnect(root_layer)
-        - Called when a client initiates a connection to the proxy. Note that a
-          connection can correspond to multiple HTTP requests.
-
-          *root_layer*
-            The root layer (see `mitmproxy.proxy.protocol` for an explanation what
-            the root layer is), provides transparent access to all attributes
-            of the :py:class:`~mitmproxy.proxy.RootContext`. For example,
-            ``root_layer.client_conn.address`` gives the remote address of the
-            connecting client.
-
-    *   - .. py:function:: clientdisconnect(root_layer)
-        - Called when a client disconnects from the proxy.
-
-          *root_layer*
-            The root layer object.
-
-    *   - .. py:function:: next_layer(layer)
-
-        - Called whenever layers are switched. You may change which layer will
-          be used by returning a new layer object from this event.
-
-          *layer*
-            The next layer, as determined by mitmpmroxy.
-
-    *   - .. py:function:: serverconnect(server_conn)
-        - Called before the proxy initiates a connection to the target server.
-          Note that a connection can correspond to multiple HTTP requests.
-
-          *server_conn*
-            A ``ServerConnection`` object. It is guaranteed to have a non-None
-            ``address`` attribute.
-
-    *   - .. py:function:: serverdisconnect(server_conn)
-        - Called when the proxy has closed the server connection.
-
-          *server_conn*
-            A ``ServerConnection`` object.
-
-
-HTTP Events
------------
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: http_connect(flow)
-        - Called when we receive an HTTP CONNECT request. Setting a non 2xx
-          response on the flow will return the response to the client abort the
-          connection. CONNECT requests and responses do not generate the usual
-          HTTP handler events. CONNECT requests are only valid in regular and
-          upstream proxy modes.
-
-          *flow*
-            A ``models.HTTPFlow`` object. The flow is guaranteed to have
-            non-None ``request``  and ``requestheaders`` attributes.
-
-
-    *   - .. py:function:: request(flow)
-        - Called when a client request has been received.
-
-          *flow*
-            A ``models.HTTPFlow`` object. At this point, the flow is
-            guaranteed to have a non-None ``request`` attribute.
-
-    *   - .. py:function:: requestheaders(flow)
-        - Called when the headers of a client request have been received, but
-          before the request body is read.
-
-          *flow*
-            A ``models.HTTPFlow`` object. At this point, the flow is
-            guaranteed to have a non-None ``request`` attribute.
-
-    *   - .. py:function:: responseheaders(flow)
-
-        - Called when the headers of a server response have been received, but
-          before the response body is read.
-
-          *flow*
-            A ``models.HTTPFlow`` object. At this point, the flow is
-            guaranteed to have a non-none ``request`` and ``response``
-            attributes, however the response will have no content.
-
-    *   - .. py:function:: response(flow)
-
-        - Called when a server response has been received.
-
-          *flow*
-            A ``models.HTTPFlow`` object. At this point, the flow is
-            guaranteed to have a non-none ``request`` and ``response``
-            attributes. The raw response body will be in ``response.body``,
-            unless response streaming has been enabled.
-
-    *   - .. py:function:: error(flow)
-        - Called when a flow error has occurred, e.g. invalid server responses,
-          or interrupted connections. This is distinct from a valid server HTTP
-          error response, which is simply a response with an HTTP error code.
-
-          *flow*
-            The flow containing the error. It is guaranteed to have
-            non-None ``error`` attribute.
-
-
-WebSocket Events
------------------
-
-These events are called only after a connection made an HTTP upgrade with
-"101 Switching Protocols". No further HTTP-related events after the handshake
-are issued, only new WebSocket messages are called.
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: websocket_handshake(flow)
-        - Called when a client wants to establish a WebSocket connection. The
-          WebSocket-specific headers can be manipulated to alter the
-          handshake. The ``flow`` object is guaranteed to have a non-None
-          ``request`` attribute.
-
-          *flow*
-            The flow containing the HTTP WebSocket handshake request. The
-            object is guaranteed to have a non-None ``request`` attribute.
-
-    *   - .. py:function:: websocket_start(flow)
-        - Called when WebSocket connection is established after a successful
-          handshake.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_message(flow)
-
-        - Called when a WebSocket message is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable. Currently there are
-          two types of messages, corresponding to the BINARY and TEXT frame types.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_end(flow)
-        - Called when WebSocket connection ends.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-    *   - .. py:function:: websocket_error(flow)
-        - Called when a WebSocket error occurs - e.g. the connection closing
-          unexpectedly.
-
-          *flow*
-            A ``models.WebSocketFlow`` object.
-
-
-TCP Events
-----------
-
-These events are called only if the connection is in :ref:`TCP mode
-<tcpproxy>`. So, for instance, TCP events are not called for ordinary HTTP/S
-connections.
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-
-    *   - .. py:function:: tcp_start(flow)
-        - Called when TCP streaming starts.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_message(flow)
-
-        - Called when a TCP payload is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_end(flow)
-        - Called when TCP streaming ends.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_error(flow)
-        - Called when a TCP error occurs - e.g. the connection closing
-          unexpectedly.
-
-          *flow*
-            A ``models.TCPFlow`` object.

docs/scripting/inlinescripts.rst

@@ -0,0 +1,231 @@
+.. _inlinescripts:
+
+Inline Scripts
+==============
+
+**mitmproxy** has a powerful scripting API that allows you to modify flows
+on-the-fly or rewrite previously saved flows locally.
+
+The mitmproxy scripting API is event driven - a script is simply a Python
+module that exposes a set of event methods. Here's a complete mitmproxy script
+that adds a new header to every HTTP response before it is returned to the
+client:
+
+.. literalinclude:: ../../examples/add_header.py
+   :caption: examples/add_header.py
+   :language: python
+
+The first argument to each event method is an instance of
+:py:class:`~mitmproxy.script.ScriptContext` that lets the script interact with the global mitmproxy
+state. The **response** event also gets an instance of :py:class:`~mitmproxy.script.ScriptContext`,
+which we can use to manipulate the response itself.
+
+We can now run this script using mitmdump or mitmproxy as follows:
+
+>>> mitmdump -s add_header.py
+
+The new header will be added to all responses passing through the proxy.
+
+Examples
+--------
+
+mitmproxy comes with a variety of example inline scripts, which demonstrate many basic tasks.
+We encourage you to either browse them locally or on `GitHub`_.
+
+
+Events
+------
+
+The ``context`` argument passed to each event method is always a
+:py:class:`~mitmproxy.script.ScriptContext` instance. It is guaranteed to be the same object
+for the scripts lifetime and is not shared between multiple inline scripts. You can safely use it
+to store any form of state you require.
+
+Script Lifecycle Events
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. py:function:: start(context, argv)
+
+    Called once on startup, before any other events.
+
+    :param List[str] argv: The inline scripts' arguments.
+        For example, ``mitmproxy -s 'example.py --foo 42'`` sets argv to ``["--foo", "42"]``.
+
+.. py:function:: done(context)
+
+    Called once on script shutdown, after any other events.
+
+Connection Events
+^^^^^^^^^^^^^^^^^
+
+.. py:function:: clientconnect(context, root_layer)
+
+    Called when a client initiates a connection to the proxy. Note that
+    a connection can correspond to multiple HTTP requests.
+
+    .. versionchanged:: 0.14
+
+    :param Layer root_layer: The root layer (see :ref:`protocols` for an explanation what the root
+        layer is), which provides transparent access to all attributes of the
+        :py:class:`~mitmproxy.proxy.RootContext`. For example, ``root_layer.client_conn.address``
+        gives the remote address of the connecting client.
+
+.. py:function:: clientdisconnect(context, root_layer)
+
+    Called when a client disconnects from the proxy.
+
+    .. versionchanged:: 0.14
+
+    :param Layer root_layer: see :py:func:`clientconnect`
+
+.. py:function:: serverconnect(context, server_conn)
+
+    Called before the proxy initiates a connection to the target server. Note that
+    a connection can correspond to multiple HTTP requests.
+
+    :param ServerConnection server_conn: The server connection object. It is guaranteed to have a
+        non-None ``address`` attribute.
+
+.. py:function:: serverdisconnect(context, server_conn)
+
+    Called when the proxy has closed the server connection.
+
+    .. versionadded:: 0.14
+
+    :param ServerConnection server_conn: see :py:func:`serverconnect`
+
+HTTP Events
+^^^^^^^^^^^
+
+.. py:function:: request(context, flow)
+
+    Called when a client request has been received. The ``flow`` object is
+    guaranteed to have a non-None ``request`` attribute.
+
+    :param HTTPFlow flow: The flow containing the request which has been received.
+        The object is guaranteed to have a non-None ``request`` attribute.
+
+.. py:function:: responseheaders(context, flow)
+
+    Called when the headers of a server response have been received.
+    This will always be called before the response hook.
+
+    :param HTTPFlow flow: The flow containing the request and response.
+        The object is guaranteed to have non-None ``request`` and
+        ``response`` attributes. ``response.content`` will be ``None``,
+        as the response body has not been read yet.
+
+.. py:function:: response(context, flow)
+
+    Called when a server response has been received.
+
+    :param HTTPFlow flow: The flow containing the request and response.
+        The object is guaranteed to have non-None ``request`` and
+        ``response`` attributes. ``response.body`` will contain the raw response body,
+        unless response streaming has been enabled.
+
+.. py:function:: error(context, flow)
+
+    Called when a flow error has occurred, e.g. invalid server responses, or
+    interrupted connections. This is distinct from a valid server HTTP error
+    response, which is simply a response with an HTTP error code.
+
+    :param HTTPFlow flow: The flow containing the error.
+        It is guaranteed to have non-None ``error`` attribute.
+
+TCP Events
+^^^^^^^^^^
+
+.. py:function:: tcp_message(context, tcp_msg)
+
+    .. warning::  API is subject to change
+
+    If the proxy is in :ref:`TCP mode <tcpproxy>`, this event is called when it
+    receives a TCP payload from the client or server.
+
+    The sender and receiver are identifiable. The message is user-modifiable.
+
+    :param TcpMessage tcp_msg: see *examples/tcp_message.py*
+
+API
+---
+
+The canonical API documentation is the code, which you can browse here, locally or on `GitHub`_.
+*Use the Source, Luke!*
+
+The main classes you will deal with in writing mitmproxy scripts are:
+
+:py:class:`~mitmproxy.script.ScriptContext`
+    - A handle for interacting with mitmproxy's Flow Master from within scripts.
+:py:class:`~mitmproxy.models.ClientConnection`
+    - Describes a client connection.
+:py:class:`~mitmproxy.models.ServerConnection`
+    - Describes a server connection.
+:py:class:`~mitmproxy.models.HTTPFlow`
+    - A collection of objects representing a single HTTP transaction.
+:py:class:`~mitmproxy.models.HTTPRequest`
+    - An HTTP request.
+:py:class:`~mitmproxy.models.HTTPResponse`
+    - An HTTP response.
+:py:class:`~mitmproxy.models.Error`
+    - A communications error.
+:py:class:`netlib.http.Headers`
+    - A dictionary-like object for managing HTTP headers.
+:py:class:`netlib.certutils.SSLCert`
+    - Exposes information SSL certificates.
+:py:class:`mitmproxy.flow.FlowMaster`
+    - The "heart" of mitmproxy, usually subclassed as :py:class:`mitmproxy.dump.DumpMaster` or
+      :py:class:`mitmproxy.console.ConsoleMaster`.
+
+Script Context
+--------------
+
+.. autoclass:: mitmproxy.script.ScriptContext
+    :members:
+    :undoc-members:
+
+Running scripts in parallel
+---------------------------
+
+We have a single flow primitive, so when a script is blocking, other requests are not processed.
+While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the
+:py:obj:`mitmproxy.script.concurrent` decorator.
+**If your script does not block, you should avoid the overhead of the decorator.**
+
+.. literalinclude:: ../../examples/nonblocking.py
+   :caption: examples/nonblocking.py
+   :language: python
+
+Make scripts configurable with arguments
+----------------------------------------
+
+Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by
+surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``.
+The arguments are then exposed in the start event:
+
+.. literalinclude:: ../../examples/modify_response_body.py
+   :caption: examples/modify_response_body.py
+   :language: python
+
+Running scripts on saved flows
+------------------------------
+
+Sometimes, we want to run a script on :py:class:`~mitmproxy.models.Flow` objects that are already
+complete.  This happens when you start a script, and then load a saved set of flows from a file
+(see the "scripted data transformation" example `here <https://mitmproxy.org/doc/mitmdump.html>`_).
+It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut
+in mitmproxy.
+
+In this case, there are no client connections, and the events are run in the following order:
+**start**, **request**, **responseheaders**, **response**, **error**, **done**.
+If the flow doesn't have a **response** or **error** associated with it, the matching events will
+be skipped.
+
+Spaces in the script path
+-------------------------
+
+By default, spaces are interpreted as a separator between the inline script and its arguments
+(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of
+quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``.
+
+.. _GitHub: https://github.com/mitmproxy/mitmproxy

docs/scripting/mitmproxy.rst

@@ -0,0 +1,26 @@
+
+mitmproxy
+=========
+
+.. note::
+
+    We strongly encourage you to use :ref:`inlinescripts` rather than mitmproxy.
+        - Inline Scripts are equally powerful and provide an easier syntax.
+        - Most examples are written as inline scripts.
+        - Multiple inline scripts can be used together.
+        - Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI.
+
+
+All of mitmproxy's basic functionality is exposed through the **mitmproxy**
+library. The example below shows a simple implementation of the "sticky cookie"
+functionality included in the interactive mitmproxy program. Traffic is
+monitored for ``Cookie`` and ``Set-Cookie`` headers, and requests are rewritten
+to include a previously seen cookie if they don't already have one. In effect,
+this lets you log in to a site using your browser, and then make subsequent
+requests using a tool like curl, which will then seem to be part of the
+authenticated session.
+
+
+.. literalinclude:: ../../examples/stickycookies
+   :caption: examples/stickycookies
+   :language: python

docs/scripting/overview.rst

@@ -1,158 +0,0 @@
-.. _overview:
-
-Overview
-========
-
-Mitmproxy has a powerful scripting API that allows you to control almost any
-aspect of traffic being proxied. In fact, much of mitmproxy's own core
-functionality is implemented using the exact same API exposed to scripters (see
-:src:`mitmproxy/addons`).
-
-
-A simple example
-----------------
-
-Scripting is event driven, with named handlers on the script object called at
-appropriate points of mitmproxy's operation. Here's a complete mitmproxy script
-that adds a new header to every HTTP response before it is returned to the
-client:
-
-.. literalinclude:: ../../examples/simple/add_header.py
-   :caption: :src:`examples/simple/add_header.py`
-   :language: python
-
-All events that deal with an HTTP request get an instance of `HTTPFlow
-<api.html#mitmproxy.models.http.HTTPFlow>`_, which we can use to manipulate the
-response itself. We can now run this script using mitmdump, and the new header
-will be added to all responses passing through the proxy:
-
->>> mitmdump -s add_header.py
-
-
-Using classes
--------------
-
-In the example above, the script object is the ``add_header`` module itself.
-That is, the handlers are declared at the global level of the script. This is
-great for quick hacks, but soon becomes limiting as scripts become more
-sophisticated.
-
-When a script first starts up, the `start <events.html#start>`_, event is
-called before anything else happens. You can replace the current script object
-by returning it from this handler. Here's how this looks when applied to the
-example above:
-
-.. literalinclude:: ../../examples/simple/add_header_class.py
-   :caption: :src:`examples/simple/add_header_class.py`
-   :language: python
-
-So here, we're using a module-level script to "boot up" into a class instance.
-From this point on, the module-level script is removed from the handler chain,
-and is replaced by the class instance.
-
-
-Handling arguments
-------------------
-
-Scripts can handle their own command-line arguments, just like any other Python
-program. Let's build on the example above to do something slightly more
-sophisticated - replace one value with another in all responses. Mitmproxy's
-`HTTPRequest <api.html#mitmproxy.models.http.HTTPRequest>`_ and `HTTPResponse
-<api.html#mitmproxy.models.http.HTTPResponse>`_ objects have a handy `replace
-<api.html#mitmproxy.models.http.HTTPResponse.replace>`_ method that takes care
-of all the details for us.
-
-.. literalinclude:: ../../examples/simple/script_arguments.py
-   :caption: :src:`examples/simple/script_arguments.py`
-   :language: python
-
-We can now call this script on the command-line like this:
-
->>> mitmdump -dd -s "./script_arguments.py html faketml"
-
-Whenever a handler is called, mitpmroxy rewrites the script environment so that
-it sees its own arguments as if it was invoked from the command-line.
-
-
-Logging and the context
------------------------
-
-Scripts should not output straight to stderr or stdout. Instead, the `log
-<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` context module
-should be used, so that the mitmproxy host program can handle output
-appropriately. So, mitmdump can print colorised script output to the terminal,
-and mitmproxy console can place script output in the event buffer.
-
-Here's how this looks:
-
-.. literalinclude:: ../../examples/simple/log_events.py
-   :caption: :src:`examples/simple/log_events.py`
-   :language: python
-
-The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
-for advanced usage.
-
-
-Running scripts on saved flows
-------------------------------
-
-When a flow is loaded from disk, the sequence of events that the flow would
-have gone through on the wire is partially replayed. So, for instance, an HTTP
-flow loaded from disk will trigger `requestheaders
-<events.html#requestheaders>`_,  `request <events.html#request>`_,
-`responseheaders <events.html#responseheaders>`_ and  `response
-<events.html#response>`_ in order. We can use this behaviour to transform saved
-traffic using scripts. For example, we can invoke the replacer script from
-above on saved traffic as follows:
-
->>> mitmdump -dd -s "./arguments.py html fakehtml" -r saved -w changed
-
-This command starts the ``arguments`` script, reads all the flows from
-``saved`` transforming them in the process, then writes them all to
-``changed``.
-
-The mitmproxy console tool provides interactive ways to run transforming
-scripts on flows - for instance, you can run a one-shot script on a single flow
-through the ``|`` (pipe) shortcut.
-
-
-Concurrency
------------
-
-The mitmproxy script mechanism is single threaded, and the proxy blocks while
-script handlers execute. This hugely simplifies the most common case, where
-handlers are light-weight and the blocking doesn't have a performance impact.
-It's possible to implement a concurrent mechanism on top of the blocking
-framework, and mitmproxy includes a handy example of this that is fit for most
-purposes. You can use it as follows:
-
-.. literalinclude:: ../../examples/complex/nonblocking.py
-   :caption: :src:`examples/complex/nonblocking.py`
-   :language: python
-
-
-Testing
--------
-
-Mitmproxy includes a number of helpers for testing addons. The
-``mitmproxy.test.taddons`` module contains a context helper that takes care of
-setting up and tearing down the addon event context. The
-``mitmproxy.test.tflow`` module contains helpers for quickly creating test
-flows. Pydoc is the canonical reference for these modules, and mitmproxy's own
-test suite is an excellent source of examples of usage. Here, for instance, is
-the mitmproxy unit tests for the `anticache` option, demonstrating a good
-cross-section of the test helpers:
-
-.. literalinclude:: ../../test/mitmproxy/addons/test_anticache.py
-   :caption: :src:`test/mitmproxy/addons/test_anticache.py`
-   :language: python
-
-
-Developing scripts
-------------------
-
-Mitmproxy monitors scripts for modifications, and reloads them on change. When
-this happens, the script is shut down (the `done <events.html#done>`_  event is
-called), and the new instance is started up as if the script had just been
-loaded (the `start <events.html#start>`_ and `configure
-<events.html#configure>`_ events are called).

docs/transparent.rst

@@ -1,6 +1,5 @@
 .. _transparent:
 
-====================
 Transparent Proxying
 ====================
 
@@ -21,33 +20,5 @@ destination of the TCP connection.
 At the moment, mitmproxy supports transparent proxying on OSX Lion and above,
 and all current flavors of Linux.
 
-Fully transparent mode
-======================
-
-By default mitmproxy will use its own local ip address for its server-side connections.
-In case this isn't desired, the --spoof-source-address argument can be used to
-use the client's ip address for server-side connections. The following config is
-required for this mode to work::
-
-    CLIENT_NET=192.168.1.0/24
-    TABLE_ID=100
-    MARK=1
-
-    echo "$TABLE_ID     mitmproxy" >> /etc/iproute2/rt_tables
-    iptables -t mangle -A PREROUTING -d $CLIENT_NET -j MARK --set-mark $MARK
-    iptables -t nat -A PREROUTING -p tcp -s $CLIENT_NET --match multiport --dports 80,443 -j REDIRECT --to-port 8080
-
-    ip rule add fwmark $MARK lookup $TABLE_ID
-    ip route add local $CLIENT_NET dev lo table $TABLE_ID
-
-This mode does require root privileges though. There's a wrapper in the examples directory
-called 'mitmproxy_shim.c', which will enable you to use this mode with dropped priviliges.
-It can be used as follows::
-
-    gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap
-    sudo chown root:root mitmproxy_shim
-    sudo chmod u+s mitmproxy_shim
-    ./mitmproxy_shim $(which mitmproxy) -T --spoof-source-address
-
 .. _iptables: http://www.netfilter.org/
 .. _pf: https://en.wikipedia.org/wiki/PF_\(firewall\)

docs/transparent/linux.rst

@@ -35,7 +35,7 @@ achieve transparent mode.
 
     >>> mitmproxy -T --host
 
-    The ``-T`` flag turns on transparent mode, and the ``--host``
+    The :option:`-T` flag turns on transparent mode, and the :option:`--host`
     argument tells mitmproxy to use the value of the Host header for URL display.
 
  6. Finally, configure your test device to use the host on which mitmproxy is

docs/transparent/openbsd.rst

@@ -1,53 +0,0 @@
-.. _openbsd:
-
-OpenBSD
-=======
-
- 1. :ref:`Install the mitmproxy certificate on the test device <certinstall>`
-
- 2. Enable IP forwarding:
-
-    >>> sudo sysctl -w net.inet.ip.forwarding=1
-
- 3. Place the following two lines in **/etc/pf.conf**:
-
-    .. code-block:: none
-
-        mitm_if = "re2"
-        pass in quick proto tcp from $mitm_if to port { 80, 443 } divert-to 127.0.0.1 port 8080
-
-    These rules tell pf to divert all traffic from ``$mitm_if`` destined for
-    port 80 or 443 to the local mitmproxy instance running on port 8080. You
-    should replace ``$mitm_if`` value with the interface on which your test
-    device will appear.
-
- 4. Configure pf with the rules:
-
-    >>> doas pfctl -f /etc/pf.conf
-
- 5. And now enable it:
-
-    >>> doas pfctl -e
-
- 6. Fire up mitmproxy. You probably want a command like this:
-
-    >>> mitmproxy -T --host
-
-    The ``-T`` flag turns on transparent mode, and the ``--host``
-    argument tells mitmproxy to use the value of the Host header for URL display.
-
- 7. Finally, configure your test device to use the host on which mitmproxy is
-    running as the default gateway.
-
-.. note::
-
-    Note that the **divert-to** rules in the pf.conf given above only apply to
-    inbound traffic. **This means that they will NOT redirect traffic coming
-    from the box running pf itself.** We can't distinguish between an outbound
-    connection from a non-mitmproxy app, and an outbound connection from
-    mitmproxy itself - if you want to intercept your traffic, you should use an
-    external host to run mitmproxy. Nonetheless, pf is flexible to cater for a
-    range of creative possibilities, like intercepting traffic emanating from
-    VMs.  See the **pf.conf** man page for more.
-
-.. _pf: http://man.openbsd.org/OpenBSD-current/man5/pf.conf.5

docs/transparent/osx.rst

@@ -50,7 +50,7 @@ Note that this means we don't support transparent mode for earlier versions of O
 
     >>> mitmproxy -T --host
 
-    The ``-T`` flag turns on transparent mode, and the ``--host``
+    The :option:`-T` flag turns on transparent mode, and the :option:`--host`
     argument tells mitmproxy to use the value of the Host header for URL display.
 
  8. Finally, configure your test device to use the host on which mitmproxy is
@@ -63,7 +63,7 @@ Note that this means we don't support transparent mode for earlier versions of O
     running pf itself.** We can't distinguish between an outbound connection from a
     non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
     want to intercept your OSX traffic, you should use an external host to run
-    mitmproxy. Nonetheless, pf is flexible to cater for a range of creative
+    mitmproxy. None the less, pf is flexible to cater for a range of creative
     possibilities, like intercepting traffic emanating from VMs.  See the
     **pf.conf** man page for more.
 

docs/tutorials/gamecenter.rst

@@ -51,7 +51,7 @@ The contents of the submission are particularly interesting:
             <key>context</key>
             <integer>0</integer>
             <key>score-value</key>
-            <integer>55</integer>
+            <integer>0</integer>
             <key>timestamp</key>
             <integer>1363515361321</integer>
           </dict>

docs/tutorials/transparent-dhcp.rst

@@ -38,14 +38,8 @@ DHCP and TFTP) services to a small-scale network.
     **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default
     `[1] <https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/>`_. For our use case, this needs
     to be disabled by changing ``dns=dnsmasq`` to ``#dns=dnsmasq`` in
-    **/etc/NetworkManager/NetworkManager.conf** and
-    
-    if on Ubuntu 16.04 or newer running:
-    
-    >>> sudo systemctl restart NetworkManager
-    
-    if on Ubuntu 12.04 or 14.04 running:
-    
+    **/etc/NetworkManager/NetworkManager.conf** and running
+
     >>> sudo restart network-manager
 
     afterwards.
@@ -67,12 +61,6 @@ DHCP and TFTP) services to a small-scale network.
 
     Apply changes:
 
-    if on Ubuntu 16.04 or newer:
-    
-    >>> sudo systemctl restart dnsmasq
-    
-    if on Ubuntu 12.04 or 14.04:
-    
     >>> sudo service dnsmasq restart
 
     Your **proxied machine** in the internal virtual network should now receive an IP address via DHCP:
@@ -86,8 +74,8 @@ To redirect traffic to mitmproxy, we need to add two iptables rules:
 
 .. code-block:: none
 
-    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
-    sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080
 
 4. Run mitmproxy
 ----------------

examples/README

@@ -0,0 +1,30 @@
+Some inline scripts may require additional dependencies, which can be installed using
+`pip install mitmproxy[examples]`.
+
+
+# inline script examples
+add_header.py             Simple script that just adds a header to every request.
+change_upstream_proxy.py  Dynamically change the upstream proxy
+dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
+dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+filt.py                   Use mitmproxy's filter expressions in your script.
+flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
+iframe_injector.py        Inject configurable iframe into pages.
+modify_form.py            Modify all form submissions to add a parameter.
+modify_querystring.py     Modify all query strings to add a parameters.
+modify_response_body.py   Replace arbitrary strings in all responses
+nonblocking.py            Demonstrate parallel processing with a blocking script.
+proxapp.py                How to embed a WSGI app in a mitmproxy server
+redirect_requests.py      Redirect requests or directly reply to them.
+stub.py                   Script stub with a method definition for every event.
+upsidedownternet.py       Rewrites traffic to turn images upside down.
+
+
+# mitmproxy examples
+flowbasic                 Basic use of mitmproxy as a library.
+stickycookies             An example of writing a custom proxy with mitmproxy.
+
+
+# misc
+read_dumpfile             Read a dumpfile generated by mitmproxy.
+mitmproxywrapper.py       Bracket mitmproxy run with proxy enable/disable on OS X

examples/README.md

@@ -1,15 +0,0 @@
-# Mitmproxy Scripting API
-
-Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being 
-proxied. In fact, much of mitmproxy’s own core functionality is implemented using the exact same API 
-exposed to scripters (see [mitmproxy/addons](../mitmproxy/addons)).
-
-This directory contains some examples of the scripting API. We recommend to start with the
-ones in [simple/](./simple).
-
-|  :warning: | If you are browsing this on GitHub, make sure to select the git tag matching your mitmproxy version. |
-|------------|------------------------------------------------------------------------------------------------------|
-
-
-Some inline scripts may require additional dependencies, which can be installed using
-`pip install mitmproxy[examples]`.

examples/add_header.py

@@ -1,2 +1,2 @@
-def response(flow):
+def response(context, flow):
     flow.response.headers["newheader"] = "foo"

examples/change_upstream_proxy.py

@@ -14,11 +14,11 @@ def proxy_address(flow):
         return ("localhost", 8081)
 
 
-def request(flow):
+def request(context, flow):
     if flow.request.method == "CONNECT":
         # If the decision is done by domain, one could also modify the server address here.
         # We do it after CONNECT here to have the request data available as well.
         return
     address = proxy_address(flow)
     if flow.live:
-        flow.live.change_upstream_proxy_server(address)
+        flow.live.change_upstream_proxy_server(address)

examples/complex/README.md

@@ -1,18 +0,0 @@
-## Complex Examples
-
-| Filename                 | Description                                                                                   |
-|:-------------------------|:----------------------------------------------------------------------------------------------|
-| change_upstream_proxy.py | Dynamically change the upstream proxy.                                                        |
-| dns_spoofing.py          | Use mitmproxy in a DNS spoofing scenario.                                                     |
-| dup_and_replay.py        | Duplicates each request, changes it, and then replays the modified request.                   |
-| flowbasic.py             | Basic use of mitmproxy's FlowMaster directly.                                                 |
-| full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. |
-| har_dump.py              | Dump flows as HAR files.                                                                      |
-| mitmproxywrapper.py      | Bracket mitmproxy run with proxy enable/disable on OS X                                       |
-| nonblocking.py           | Demonstrate parallel processing with a blocking script                                        |
-| remote_debug.py          | This script enables remote debugging of the mitmproxy _UI_ with PyCharm.                      |
-| sslstrip.py              | sslstrip-like funtionality implemented with mitmproxy                                         |
-| stream.py                | Enable streaming for all responses.                                                           |
-| stream_modify.py         | Modify a streamed response body.                                                              |
-| tcp_message.py           | Modify a raw TCP connection                                                                   |
-| tls_passthrough.py       | Use conditional TLS interception based on a user-defined strategy.                            |

examples/complex/dns_spoofing.py

@@ -1,62 +0,0 @@
-"""
-This script makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
-connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
-Host header of the HTTP request.
-Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
-know the actual target and cannot construct a certificate that looks valid.
-Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
-Using transparent mode is the better option most of the time.
-
-Usage:
-    mitmproxy
-        -p 443
-        -s dns_spoofing.py
-        # Used as the target location if neither SNI nor host header are present.
-        -R http://example.com/
-    mitmdump
-        -p 80
-        -R http://localhost:443/
-
-    (Setting up a single proxy instance and using iptables to redirect to it
-    works as well)
-"""
-import re
-
-# This regex extracts splits the host header into host and port.
-# Handles the edge case of IPv6 addresses containing colons.
-# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
-parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
-
-
-class Rerouter:
-    def requestheaders(self, flow):
-        """
-        The original host header is retrieved early
-        before flow.request is replaced by mitmproxy new outgoing request
-        """
-        flow.metadata["original_host"] = flow.request.host_header
-
-    def request(self, flow):
-        if flow.client_conn.ssl_established:
-            flow.request.scheme = "https"
-            sni = flow.client_conn.connection.get_servername()
-            port = 443
-        else:
-            flow.request.scheme = "http"
-            sni = None
-            port = 80
-
-        host_header = flow.metadata["original_host"]
-        m = parse_host_header.match(host_header)
-        if m:
-            host_header = m.group("host").strip("[]")
-            if m.group("port"):
-                port = int(m.group("port"))
-
-        flow.request.host_header = host_header
-        flow.request.host = sni or host_header
-        flow.request.port = port
-
-
-def start():
-    return Rerouter()

examples/complex/dup_and_replay.py

@@ -1,8 +0,0 @@
-from mitmproxy import ctx
-
-
-def request(flow):
-    f = flow.copy()
-    ctx.master.view.add(f)
-    f.request.path = "/changed"
-    ctx.master.replay_request(f, block=True)

examples/complex/flowbasic.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-"""
-    This example shows how to build a proxy based on mitmproxy's Flow
-    primitives.
-
-    Heads Up: In the majority of cases, you want to use inline scripts.
-
-    Note that request and response messages are not automatically replied to,
-    so we need to implement handlers to do this.
-"""
-from mitmproxy import controller, options, master
-from mitmproxy.proxy import ProxyServer, ProxyConfig
-
-
-class MyMaster(master.Master):
-    def run(self):
-        try:
-            master.Master.run(self)
-        except KeyboardInterrupt:
-            self.shutdown()
-
-    @controller.handler
-    def request(self, f):
-        print("request", f)
-
-    @controller.handler
-    def response(self, f):
-        print("response", f)
-
-    @controller.handler
-    def error(self, f):
-        print("error", f)
-
-    @controller.handler
-    def log(self, l):
-        print("log", l.msg)
-
-
-opts = options.Options(cadir="~/.mitmproxy/")
-config = ProxyConfig(opts)
-server = ProxyServer(config)
-m = MyMaster(opts, server)
-m.run()

examples/complex/full_transparency_shim.c

@@ -1,87 +0,0 @@
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <string.h>
-#include <sys/prctl.h>
-#include <sys/types.h>
-#include <sys/capability.h>
-#include <unistd.h>
-#include <errno.h>
-
-/* This setuid wrapper can be used to run mitmproxy in full transparency mode, as a normal user.
- * It will set the required capabilities (CAP_NET_RAW), drop privileges, and will then run argv[1]
- * with the same capabilities.
- *
- * It can be compiled as follows:
- * gcc examples/mitmproxy_shim.c -o mitmproxy_shim -lcap
-*/
-
-int set_caps(cap_t cap_struct, cap_value_t *cap_list, size_t bufsize) {
-	int cap_count = bufsize / sizeof(cap_list[0]);
-
-	if (cap_set_flag(cap_struct, CAP_PERMITTED, cap_count, cap_list, CAP_SET) ||
-            cap_set_flag(cap_struct, CAP_EFFECTIVE, cap_count, cap_list, CAP_SET) ||
-            cap_set_flag(cap_struct, CAP_INHERITABLE, cap_count, cap_list, CAP_SET)) {
-		if (cap_count < 2) {
-			fprintf(stderr, "Cannot manipulate capability data structure as user: %s.\n", strerror(errno));
-		} else {
-			fprintf(stderr, "Cannot manipulate capability data structure as root: %s.\n", strerror(errno));
-		}
-		return -1;
-	}
-
-	if (cap_count < 2) {
-		if (prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_RAISE, CAP_NET_RAW, 0, 0)) {
-			fprintf(stderr, "Failed to add CAP_NET_RAW to the ambient set: %s.\n", strerror(errno));
-			return -2;
-		}
-	}
-
-	if (cap_set_proc(cap_struct)) {
-		if (cap_count < 2) {
-			fprintf(stderr, "Cannot set capabilities as user: %s.\n", strerror(errno)); 
-		} else {
-			fprintf(stderr, "Cannot set capabilities as root: %s.\n", strerror(errno));
-		}
-		return -3;
-	}
-
-	if (cap_count > 1) {
-		if (prctl(PR_SET_KEEPCAPS, 1L)) {
-			fprintf(stderr, "Cannot keep capabilities after dropping privileges: %s.\n", strerror(errno));
-			return -4;
-		}
-		if (cap_clear(cap_struct)) {
-			fprintf(stderr, "Cannot clear capability data structure: %s.\n", strerror(errno));
-			return -5;
-		}
-	}
-}
-
-int main(int argc, char **argv, char **envp) {
-	cap_t cap_struct = cap_init();
-	cap_value_t root_caps[2] = { CAP_NET_RAW, CAP_SETUID };
-	cap_value_t user_caps[1] = { CAP_NET_RAW };
-	uid_t user = getuid();
-	int res;
-
-	if (setresuid(0, 0, 0)) {
-		fprintf(stderr, "Cannot switch to root: %s.\n", strerror(errno));
-		return 1;
-	}
-
-	if (res = set_caps(cap_struct, root_caps, sizeof(root_caps)))
-		return res;
-
-	if (setresuid(user, user, user)) {
-		fprintf(stderr, "Cannot drop root privileges: %s.\n", strerror(errno));
-		return 2;
-	}
-
-	if (res = set_caps(cap_struct, user_caps, sizeof(user_caps)))
-		return res;
-
-	if (execve(argv[1], argv + 1, envp)) {
-		fprintf(stderr, "Failed to execute %s: %s\n", argv[1], strerror(errno));
-		return 3;
-	}
-}

examples/complex/har_dump.py

@@ -1,220 +0,0 @@
-"""
-This inline script can be used to dump flows as HAR files.
-"""
-
-
-import json
-import sys
-import base64
-import zlib
-import os
-
-from datetime import datetime
-import pytz
-
-import mitmproxy
-
-from mitmproxy import version
-from mitmproxy.utils import strutils
-from mitmproxy.net.http import cookies
-
-HAR = {}
-
-# A list of server seen till now is maintained so we can avoid
-# using 'connect' time for entries that use an existing connection.
-SERVERS_SEEN = set()
-
-
-def start():
-    """
-        Called once on script startup before any other events.
-    """
-    if len(sys.argv) != 2:
-        raise ValueError(
-            'Usage: -s "har_dump.py filename" '
-            '(- will output to stdout, filenames ending with .zhar '
-            'will result in compressed har)'
-        )
-
-    HAR.update({
-        "log": {
-            "version": "1.2",
-            "creator": {
-                "name": "mitmproxy har_dump",
-                "version": "0.1",
-                "comment": "mitmproxy version %s" % version.MITMPROXY
-            },
-            "entries": []
-        }
-    })
-
-
-def response(flow):
-    """
-       Called when a server response has been received.
-    """
-
-    # -1 indicates that these values do not apply to current request
-    ssl_time = -1
-    connect_time = -1
-
-    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:
-        connect_time = (flow.server_conn.timestamp_tcp_setup -
-                        flow.server_conn.timestamp_start)
-
-        if flow.server_conn.timestamp_ssl_setup is not None:
-            ssl_time = (flow.server_conn.timestamp_ssl_setup -
-                        flow.server_conn.timestamp_tcp_setup)
-
-        SERVERS_SEEN.add(flow.server_conn)
-
-    # Calculate raw timings from timestamps. DNS timings can not be calculated
-    # for lack of a way to measure it. The same goes for HAR blocked.
-    # mitmproxy will open a server connection as soon as it receives the host
-    # and port from the client connection. So, the time spent waiting is actually
-    # spent waiting between request.timestamp_end and response.timestamp_start
-    # thus it correlates to HAR wait instead.
-    timings_raw = {
-        'send': flow.request.timestamp_end - flow.request.timestamp_start,
-        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
-        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
-        'connect': connect_time,
-        'ssl': ssl_time,
-    }
-
-    # HAR timings are integers in ms, so we re-encode the raw timings to that format.
-    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])
-
-    # full_time is the sum of all timings.
-    # Timings set to -1 will be ignored as per spec.
-    full_time = sum(v for v in timings.values() if v > -1)
-
-    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))
-
-    # Response body size and encoding
-    response_body_size = len(flow.response.raw_content)
-    response_body_decoded_size = len(flow.response.content)
-    response_body_compression = response_body_decoded_size - response_body_size
-
-    entry = {
-        "startedDateTime": started_date_time,
-        "time": full_time,
-        "request": {
-            "method": flow.request.method,
-            "url": flow.request.url,
-            "httpVersion": flow.request.http_version,
-            "cookies": format_request_cookies(flow.request.cookies.fields),
-            "headers": name_value(flow.request.headers),
-            "queryString": name_value(flow.request.query or {}),
-            "headersSize": len(str(flow.request.headers)),
-            "bodySize": len(flow.request.content),
-        },
-        "response": {
-            "status": flow.response.status_code,
-            "statusText": flow.response.reason,
-            "httpVersion": flow.response.http_version,
-            "cookies": format_response_cookies(flow.response.cookies.fields),
-            "headers": name_value(flow.response.headers),
-            "content": {
-                "size": response_body_size,
-                "compression": response_body_compression,
-                "mimeType": flow.response.headers.get('Content-Type', '')
-            },
-            "redirectURL": flow.response.headers.get('Location', ''),
-            "headersSize": len(str(flow.response.headers)),
-            "bodySize": response_body_size,
-        },
-        "cache": {},
-        "timings": timings,
-    }
-
-    # Store binary data as base64
-    if strutils.is_mostly_bin(flow.response.content):
-        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
-        entry["response"]["content"]["encoding"] = "base64"
-    else:
-        entry["response"]["content"]["text"] = flow.response.get_text(strict=False)
-
-    if flow.request.method in ["POST", "PUT", "PATCH"]:
-        params = [
-            {"name": a, "value": b}
-            for a, b in flow.request.urlencoded_form.items(multi=True)
-        ]
-        entry["request"]["postData"] = {
-            "mimeType": flow.request.headers.get("Content-Type", ""),
-            "text": flow.request.get_text(strict=False),
-            "params": params
-        }
-
-    if flow.server_conn.connected():
-        entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
-
-    HAR["log"]["entries"].append(entry)
-
-
-def done():
-    """
-        Called once on script shutdown, after any other events.
-    """
-    dump_file = sys.argv[1]
-
-    json_dump = json.dumps(HAR, indent=2)  # type: str
-
-    if dump_file == '-':
-        mitmproxy.ctx.log(json_dump)
-    else:
-        raw = json_dump.encode()  # type: bytes
-        if dump_file.endswith('.zhar'):
-            raw = zlib.compress(raw, 9)
-
-        with open(os.path.expanduser(dump_file), "wb") as f:
-            f.write(raw)
-
-        mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
-
-
-def format_datetime(dt):
-    return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat()
-
-
-def format_cookies(cookie_list):
-    rv = []
-
-    for name, value, attrs in cookie_list:
-        cookie_har = {
-            "name": name,
-            "value": value,
-        }
-
-        # HAR only needs some attributes
-        for key in ["path", "domain", "comment"]:
-            if key in attrs:
-                cookie_har[key] = attrs[key]
-
-        # These keys need to be boolean!
-        for key in ["httpOnly", "secure"]:
-            cookie_har[key] = bool(key in attrs)
-
-        # Expiration time needs to be formatted
-        expire_ts = cookies.get_expiration_ts(attrs)
-        if expire_ts is not None:
-            cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts))
-
-        rv.append(cookie_har)
-
-    return rv
-
-
-def format_request_cookies(fields):
-    return format_cookies(cookies.group_cookies(fields))
-
-
-def format_response_cookies(fields):
-    return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)
-
-
-def name_value(obj):
-    """
-        Convert (key, value) pairs to HAR format.
-    """
-    return [{"name": k, "value": v} for k, v in obj.items()]

examples/complex/remote_debug.py

@@ -1,19 +0,0 @@
-"""
-This script enables remote debugging of the mitmproxy *UI* with PyCharm.
-For general debugging purposes, it is easier to just debug mitmdump within PyCharm.
-
-Usage:
-    - pip install pydevd on the mitmproxy machine
-    - Open the Run/Debug Configuration dialog box in PyCharm, and select the Python Remote Debug configuration type.
-    - Debugging works in the way that mitmproxy connects to the debug server on startup.
-      Specify host and port that mitmproxy can use to reach your PyCharm instance on startup.
-    - Adjust this inline script accordingly.
-    - Start debug server in PyCharm
-    - Set breakpoints
-    - Start mitmproxy -s remote_debug.py
-"""
-
-
-def start():
-    import pydevd
-    pydevd.settrace("localhost", port=5678, stdoutToServer=True, stderrToServer=True)

examples/complex/sslstrip.py

@@ -1,57 +0,0 @@
-"""
-This script implements an sslstrip-like attack based on mitmproxy.
-https://moxie.org/software/sslstrip/
-"""
-import re
-import urllib
-
-# set of SSL/TLS capable hosts
-secure_hosts = set()
-
-
-def request(flow):
-    flow.request.headers.pop('If-Modified-Since', None)
-    flow.request.headers.pop('Cache-Control', None)
-
-    # do not force https redirection
-    flow.request.headers.pop('Upgrade-Insecure-Requests', None)
-
-    # proxy connections to SSL-enabled hosts
-    if flow.request.pretty_host in secure_hosts:
-        flow.request.scheme = 'https'
-        flow.request.port = 443
-
-        # We need to update the request destination to whatever is specified in the host header:
-        # Having no TLS Server Name Indication from the client and just an IP address as request.host
-        # in transparent mode, TLS server name certificate validation would fail.
-        flow.request.host = flow.request.pretty_host
-
-
-def response(flow):
-    flow.response.headers.pop('Strict-Transport-Security', None)
-    flow.response.headers.pop('Public-Key-Pins', None)
-
-    # strip links in response body
-    flow.response.content = flow.response.content.replace(b'https://', b'http://')
-
-    # strip meta tag upgrade-insecure-requests in response body
-    csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>'
-    flow.response.content = re.sub(csp_meta_tag_pattern, b'', flow.response.content, flags=re.IGNORECASE)
-
-    # strip links in 'Location' header
-    if flow.response.headers.get('Location', '').startswith('https://'):
-        location = flow.response.headers['Location']
-        hostname = urllib.parse.urlparse(location).hostname
-        if hostname:
-            secure_hosts.add(hostname)
-        flow.response.headers['Location'] = location.replace('https://', 'http://', 1)
-
-    # strip upgrade-insecure-requests in Content-Security-Policy header
-    if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE):
-        csp = flow.response.headers['Content-Security-Policy']
-        flow.response.headers['Content-Security-Policy'] = re.sub('upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE)
-
-    # strip secure flag from 'Set-Cookie' headers
-    cookies = flow.response.headers.get_all('Set-Cookie')
-    cookies = [re.sub(r';\s*secure\s*', '', s) for s in cookies]
-    flow.response.headers.set_all('Set-Cookie', cookies)

examples/complex/tcp_message.py

@@ -1,27 +0,0 @@
-"""
-tcp_message Inline Script Hook API Demonstration
-------------------------------------------------
-
-* modifies packets containing "foo" to "bar"
-* prints various details for each packet.
-
-example cmdline invocation:
-mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py
-"""
-from mitmproxy.utils import strutils
-
-
-def tcp_message(tcp_msg):
-    modified_msg = tcp_msg.message.replace("foo", "bar")
-
-    is_modified = False if modified_msg == tcp_msg.message else True
-    tcp_msg.message = modified_msg
-
-    print(
-        "[tcp_message{}] from {} {} to {} {}:\r\n{}".format(
-            " (modified)" if is_modified else "",
-            "client" if tcp_msg.sender == tcp_msg.client_conn else "server",
-            tcp_msg.sender.address,
-            "server" if tcp_msg.receiver == tcp_msg.server_conn else "client",
-            tcp_msg.receiver.address, strutils.bytes_to_escaped_str(tcp_msg.message))
-    )

examples/custom_contentviews.py

@@ -0,0 +1,68 @@
+import string
+import lxml.html
+import lxml.etree
+from mitmproxy import utils, contentviews
+
+
+class ViewPigLatin(contentviews.View):
+    name = "pig_latin_HTML"
+    prompt = ("pig latin HTML", "l")
+    content_types = ["text/html"]
+
+    def __call__(self, data, **metadata):
+        if utils.isXML(data):
+            parser = lxml.etree.HTMLParser(
+                strip_cdata=True,
+                remove_blank_text=True
+            )
+            d = lxml.html.fromstring(data, parser=parser)
+            docinfo = d.getroottree().docinfo
+
+            def piglify(src):
+                words = string.split(src)
+                ret = ''
+                for word in words:
+                    idx = -1
+                    while word[idx] in string.punctuation and (idx * -1) != len(word): idx -= 1
+                    if word[0].lower() in 'aeiou':
+                        if idx == -1:
+                            ret += word[0:] + "hay"
+                        else:
+                            ret += word[0:len(word) + idx + 1] + "hay" + word[idx + 1:]
+                    else:
+                        if idx == -1:
+                            ret += word[1:] + word[0] + "ay"
+                        else:
+                            ret += word[1:len(word) + idx + 1] + word[0] + "ay" + word[idx + 1:]
+                    ret += ' '
+                return ret.strip()
+
+            def recurse(root):
+                if hasattr(root, 'text') and root.text:
+                    root.text = piglify(root.text)
+                if hasattr(root, 'tail') and root.tail:
+                    root.tail = piglify(root.tail)
+
+                if len(root):
+                    for child in root:
+                        recurse(child)
+
+            recurse(d)
+
+            s = lxml.etree.tostring(
+                d,
+                pretty_print=True,
+                doctype=docinfo.doctype
+            )
+            return "HTML", contentviews.format_text(s)
+
+
+pig_view = ViewPigLatin()
+
+
+def start(context, argv):
+    context.add_contentview(pig_view)
+
+
+def done(context):
+    context.remove_contentview(pig_view)

examples/dns_spoofing.py

@@ -0,0 +1,50 @@
+"""
+This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
+connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
+Host header of the HTTP request.
+Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
+know the actual target and cannot construct a certificate that looks valid.
+Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
+Using transparent mode is the better option most of the time.
+
+Usage:
+    mitmproxy
+        -p 443
+        -s dns_spoofing.py
+        # Used as the target location if neither SNI nor host header are present.
+        -R http://example.com/
+    mitmdump
+        -p 80
+        -R http://localhost:443/
+
+    (Setting up a single proxy instance and using iptables to redirect to it
+    works as well)
+"""
+import re
+
+
+# This regex extracts splits the host header into host and port.
+# Handles the edge case of IPv6 addresses containing colons.
+# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
+parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
+
+
+def request(context, flow):
+    if flow.client_conn.ssl_established:
+        flow.request.scheme = "https"
+        sni = flow.client_conn.connection.get_servername()
+        port = 443
+    else:
+        flow.request.scheme = "http"
+        sni = None
+        port = 80
+
+    host_header = flow.request.pretty_host
+    m = parse_host_header.match(host_header)
+    if m:
+        host_header = m.group("host").strip("[]")
+        if m.group("port"):
+            port = int(m.group("port"))
+
+    flow.request.host = sni or host_header
+    flow.request.port = port

examples/dup_and_replay.py

@@ -0,0 +1,4 @@
+def request(context, flow):
+    f = context.duplicate_flow(flow)
+    f.request.path = "/changed"
+    context.replay_request(f)

examples/filt.py

@@ -0,0 +1,16 @@
+# This scripts demonstrates how to use mitmproxy's filter pattern in inline scripts.
+# Usage: mitmdump -s "filt.py FILTER"
+
+from mitmproxy import filt
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError("Usage: -s 'filt.py FILTER'")
+    context.filter = filt.parse(argv[1])
+
+
+def response(context, flow):
+    if flow.match(context.filter):
+        print("Flow matches filter:")
+        print(flow)

examples/flowbasic

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+"""
+    This example shows how to build a proxy based on mitmproxy's Flow
+    primitives.
+
+    Heads Up: In the majority of cases, you want to use inline scripts.
+
+    Note that request and response messages are not automatically replied to,
+    so we need to implement handlers to do this.
+"""
+from mitmproxy import flow
+from mitmproxy.proxy import ProxyServer, ProxyConfig
+
+
+class MyMaster(flow.FlowMaster):
+    def run(self):
+        try:
+            flow.FlowMaster.run(self)
+        except KeyboardInterrupt:
+            self.shutdown()
+
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
+        if f:
+            f.reply()
+        return f
+
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
+        if f:
+            f.reply()
+        print(f)
+        return f
+
+
+config = ProxyConfig(
+    port=8080,
+    # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
+    cadir="~/.mitmproxy/"
+)
+state = flow.State()
+server = ProxyServer(config)
+m = MyMaster(server, state)
+m.run()

examples/flowwriter.py

@@ -0,0 +1,20 @@
+import random
+import sys
+
+from mitmproxy.flow import FlowWriter
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "flowriter.py filename"')
+
+    if argv[1] == "-":
+        f = sys.stdout
+    else:
+        f = open(argv[1], "wb")
+    context.flow_writer = FlowWriter(f)
+
+
+def response(context, flow):
+    if random.choice([True, False]):
+        context.flow_writer.add(flow)

examples/har_extractor.py

@@ -0,0 +1,243 @@
+"""
+    This inline script utilizes harparser.HAR from
+    https://github.com/JustusW/harparser to generate a HAR log object.
+"""
+import six
+from harparser import HAR
+
+from datetime import datetime
+
+
+class _HARLog(HAR.log):
+    # The attributes need to be registered here for them to actually be
+    # available later via self. This is due to HAREncodable linking __getattr__
+    # to __getitem__. Anything that is set only in __init__ will just be added
+    # as key/value pair to self.__classes__.
+    __page_list__ = []
+    __page_count__ = 0
+    __page_ref__ = {}
+
+    def __init__(self, page_list=[]):
+        self.__page_list__ = page_list
+        self.__page_count__ = 0
+        self.__page_ref__ = {}
+
+        HAR.log.__init__(self, {"version": "1.2",
+                                "creator": {"name": "MITMPROXY HARExtractor",
+                                            "version": "0.1",
+                                            "comment": ""},
+                                "pages": [],
+                                "entries": []})
+
+    def reset(self):
+        self.__init__(self.__page_list__)
+
+    def add(self, obj):
+        if isinstance(obj, HAR.pages):
+            self['pages'].append(obj)
+        if isinstance(obj, HAR.entries):
+            self['entries'].append(obj)
+
+    def create_page_id(self):
+        self.__page_count__ += 1
+        return "autopage_%s" % str(self.__page_count__)
+
+    def set_page_ref(self, page, ref):
+        self.__page_ref__[page] = ref
+
+    def get_page_ref(self, page):
+        return self.__page_ref__.get(page, None)
+
+    def get_page_list(self):
+        return self.__page_list__
+
+
+def start(context, argv):
+    """
+        On start we create a HARLog instance. You will have to adapt this to
+        suit your actual needs of HAR generation. As it will probably be
+        necessary to cluster logs by IPs or reset them from time to time.
+    """
+    context.dump_file = None
+    if len(argv) > 1:
+        context.dump_file = argv[1]
+    else:
+        raise ValueError(
+            'Usage: -s "har_extractor.py filename" '
+            '(- will output to stdout, filenames ending with .zhar '
+            'will result in compressed har)'
+        )
+    context.HARLog = _HARLog()
+    context.seen_server = set()
+
+
+def response(context, flow):
+    """
+       Called when a server response has been received. At the time of this
+       message both a request and a response are present and completely done.
+    """
+    # Values are converted from float seconds to int milliseconds later.
+    ssl_time = -.001
+    connect_time = -.001
+    if flow.server_conn not in context.seen_server:
+        # Calculate the connect_time for this server_conn. Afterwards add it to
+        # seen list, in order to avoid the connect_time being present in entries
+        # that use an existing connection.
+        connect_time = (flow.server_conn.timestamp_tcp_setup -
+                        flow.server_conn.timestamp_start)
+        context.seen_server.add(flow.server_conn)
+
+        if flow.server_conn.timestamp_ssl_setup is not None:
+            # Get the ssl_time for this server_conn as the difference between
+            # the start of the successful tcp setup and the successful ssl
+            # setup. If no ssl setup has been made it is left as -1 since it
+            # doesn't apply to this connection.
+            ssl_time = (flow.server_conn.timestamp_ssl_setup -
+                        flow.server_conn.timestamp_tcp_setup)
+
+    # Calculate the raw timings from the different timestamps present in the
+    # request and response object. For lack of a way to measure it dns timings
+    # can not be calculated. The same goes for HAR blocked: MITMProxy will open
+    # a server connection as soon as it receives the host and port from the
+    # client connection. So the time spent waiting is actually spent waiting
+    # between request.timestamp_end and response.timestamp_start thus it
+    # correlates to HAR wait instead.
+    timings_raw = {
+        'send': flow.request.timestamp_end - flow.request.timestamp_start,
+        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
+        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
+        'connect': connect_time,
+        'ssl': ssl_time
+    }
+
+    # HAR timings are integers in ms, so we have to re-encode the raw timings to
+    # that format.
+    timings = dict([(k, int(1000 * v)) for k, v in six.iteritems(timings_raw)])
+
+    # The full_time is the sum of all timings.
+    # Timings set to -1 will be ignored as per spec.
+    full_time = sum(v for v in timings.values() if v > -1)
+
+    started_date_time = datetime.utcfromtimestamp(
+        flow.request.timestamp_start).isoformat()
+
+    request_query_string = [{"name": k, "value": v}
+                            for k, v in flow.request.query or {}]
+
+    response_body_size = len(flow.response.content)
+    response_body_decoded_size = len(flow.response.get_decoded_content())
+    response_body_compression = response_body_decoded_size - response_body_size
+
+    entry = HAR.entries({
+        "startedDateTime": started_date_time,
+        "time": full_time,
+        "request": {
+            "method": flow.request.method,
+            "url": flow.request.url,
+            "httpVersion": flow.request.http_version,
+            "cookies": format_cookies(flow.request.cookies),
+            "headers": format_headers(flow.request.headers),
+            "queryString": request_query_string,
+            "headersSize": len(str(flow.request.headers)),
+            "bodySize": len(flow.request.content),
+        },
+        "response": {
+            "status": flow.response.status_code,
+            "statusText": flow.response.reason,
+            "httpVersion": flow.response.http_version,
+            "cookies": format_cookies(flow.response.cookies),
+            "headers": format_headers(flow.response.headers),
+            "content": {
+                "size": response_body_size,
+                "compression": response_body_compression,
+                "mimeType": flow.response.headers.get('Content-Type', '')
+            },
+            "redirectURL": flow.response.headers.get('Location', ''),
+            "headersSize": len(str(flow.response.headers)),
+            "bodySize": response_body_size,
+        },
+        "cache": {},
+        "timings": timings,
+    })
+
+    # If the current url is in the page list of context.HARLog or
+    # does not have a referrer, we add it as a new pages object.
+    if (flow.request.url in context.HARLog.get_page_list() or
+            flow.request.headers.get('Referer') is None):
+        page_id = context.HARLog.create_page_id()
+        context.HARLog.add(
+            HAR.pages({
+                "startedDateTime": entry['startedDateTime'],
+                "id": page_id,
+                "title": flow.request.url,
+            })
+        )
+        context.HARLog.set_page_ref(flow.request.url, page_id)
+        entry['pageref'] = page_id
+
+    # Lookup the referer in the page_ref of context.HARLog to point this entries
+    # pageref attribute to the right pages object, then set it as a new
+    # reference to build a reference tree.
+    elif context.HARLog.get_page_ref(flow.request.headers.get('Referer')) is not None:
+        entry['pageref'] = context.HARLog.get_page_ref(
+            flow.request.headers['Referer']
+        )
+        context.HARLog.set_page_ref(
+            flow.request.headers['Referer'], entry['pageref']
+        )
+
+    context.HARLog.add(entry)
+
+
+def done(context):
+    """
+        Called once on script shutdown, after any other events.
+    """
+    import pprint
+    import json
+
+    json_dump = context.HARLog.json()
+    compressed_json_dump = context.HARLog.compress()
+
+    if context.dump_file == '-':
+        context.log(pprint.pformat(json.loads(json_dump)))
+    elif context.dump_file.endswith('.zhar'):
+        file(context.dump_file, "w").write(compressed_json_dump)
+    else:
+        file(context.dump_file, "w").write(json_dump)
+    context.log(
+        "HAR log finished with %s bytes (%s bytes compressed)" % (
+            len(json_dump), len(compressed_json_dump)
+        )
+    )
+    context.log(
+        "Compression rate is %s%%" % str(
+            100. * len(compressed_json_dump) / len(json_dump)
+        )
+    )
+
+
+def format_cookies(obj):
+    if obj:
+        return [{"name": k.strip(), "value": v[0]} for k, v in obj.items()]
+    return ""
+
+
+def format_headers(obj):
+    if obj:
+        return [{"name": k, "value": v} for k, v in obj.fields]
+    return ""
+
+
+def print_attributes(obj, filter_string=None, hide_privates=False):
+    """
+        Useful helper method to quickly get all attributes of an object and its
+        values.
+    """
+    for attr in dir(obj):
+        if hide_privates and "__" in attr:
+            continue
+        if filter_string is not None and filter_string not in attr:
+            continue
+        value = getattr(obj, attr)
+        print("%s.%s" % ('obj', attr), value, type(value))

examples/iframe_injector.py

@@ -0,0 +1,27 @@
+# Usage: mitmdump -s "iframe_injector.py url"
+# (this script works best with --anticache)
+from bs4 import BeautifulSoup
+from mitmproxy.models import decoded
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "iframe_injector.py url"')
+    context.iframe_url = argv[1]
+
+
+def response(context, flow):
+    if flow.request.host in context.iframe_url:
+        return
+    with decoded(flow.response):  # Remove content encoding (gzip, ...)
+        html = BeautifulSoup(flow.response.content, "lxml")
+        if html.body:
+            iframe = html.new_tag(
+                "iframe",
+                src=context.iframe_url,
+                frameborder=0,
+                height=0,
+                width=0)
+            html.body.insert(0, iframe)
+            flow.response.content = str(html)
+            context.log("Iframe inserted.")

examples/mitmproxywrapper.py

@@ -15,7 +15,8 @@ import os
 import sys
 
 
-class Wrapper:
+class Wrapper(object):
+
     def __init__(self, port, extra_arguments=None):
         self.port = port
         self.extra_arguments = extra_arguments
@@ -141,7 +142,7 @@ class Wrapper:
             '--toggle',
             action='store_true',
             help='just toggle the proxy configuration')
-        # parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
+#         parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
         parser.add_argument(
             '-p',
             '--port',
@@ -154,8 +155,8 @@ class Wrapper:
 
         if args.toggle:
             wrapper.toggle_proxy()
-        # elif args.honeyproxy:
-        #     wrapper.wrap_honeyproxy()
+#         elif args.honeyproxy:
+#             wrapper.wrap_honeyproxy()
         else:
             wrapper.wrap_mitmproxy()
 

examples/modify_form.py

@@ -0,0 +1,5 @@
+def request(context, flow):
+    form = flow.request.urlencoded_form
+    if form is not None:
+        form["mitmproxy"] = ["rocks"]
+        flow.request.urlencoded_form = form

examples/modify_querystring.py

@@ -0,0 +1,5 @@
+def request(context, flow):
+    q = flow.request.query
+    if q:
+        q["mitmproxy"] = ["rocks"]
+        flow.request.query = q

examples/modify_response_body.py

@@ -0,0 +1,18 @@
+# Usage: mitmdump -s "modify_response_body.py mitmproxy bananas"
+# (this script works best with --anticache)
+from mitmproxy.models import decoded
+
+
+def start(context, argv):
+    if len(argv) != 3:
+        raise ValueError('Usage: -s "modify_response_body.py old new"')
+    # You may want to use Python's argparse for more sophisticated argument
+    # parsing.
+    context.old, context.new = argv[1], argv[2]
+
+
+def response(context, flow):
+    with decoded(flow.response):  # automatically decode gzipped responses.
+        flow.response.content = flow.response.content.replace(
+            context.old,
+            context.new)

examples/nonblocking.py

@@ -1,11 +1,9 @@
 import time
-
 from mitmproxy.script import concurrent
 
 
 @concurrent  # Remove this and see what happens
-def request(flow):
-    # You don't want to use mitmproxy.ctx from a different thread
+def request(context, flow):
     print("handle request: %s%s" % (flow.request.host, flow.request.path))
     time.sleep(5)
     print("start  request: %s%s" % (flow.request.host, flow.request.path))