bump version

.appveyor.yml

@@ -1,15 +1,9 @@
 version: '{build}'
 build: off  # Not a C# project
-
 environment:
-  CI_DEPS: codecov>=2.0.5
-  CI_COMMANDS: codecov
   matrix:
-    - PYTHON: "C:\\Python35"
-      TOXENV: "py35"
     - PYTHON: "C:\\Python27"
-      TOXENV: "py27"
-
+  PATH: "%APPDATA%\\Python\\Scripts;C:\\Python27;C:\\Python27\\Scripts;%PATH%"
   SNAPSHOT_HOST:
     secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
   SNAPSHOT_PORT:
@@ -18,30 +12,20 @@ environment:
     secure: 6yBwmO5gv4vAwoFYII8qjQ==
   SNAPSHOT_PASS:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
-
 install:
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "python -m pip install --disable-pip-version-check -U pip"
-  - "pip install -U tox"
-
+  - "pip install --user -U virtualenv"
+  - "dev.bat"
+  - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
 test_script:
-  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v"
-
-deploy_script:
-  ps: |
-    if(
-      ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-      pip install -U virtualenv
-      .\dev.ps1
-      cmd /c "python -u .\release\rtool.py bdist 2>&1"
-      python -u .\release\rtool.py upload-snapshot --bdist --wheel
-    }
-
+  - "py.test --cov netlib --cov mitmproxy --cov pathod"
 cache:
   - C:\Users\appveyor\AppData\Local\pip\cache
-
+deploy_script:
+  ps: |
+    if(($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true")) {
+    python .\release\rtool.py bdist
+    python .\release\rtool.py upload-snapshot --bdist
+    }
 notifications:
-  - provider: Slack
-    incoming_webhook: https://hooks.slack.com/services/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
+- provider: Slack
+  incoming_webhook: https://hooks.slack.com/services/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

.codecov.yml

@@ -1 +0,0 @@
-comment: off

.dockerignore

@@ -0,0 +1 @@
+.git

.gitignore

@@ -1,15 +1,14 @@
 .DS_Store
 MANIFEST
 */tmp
-/venv*
+/venv
 *.py[cdo]
 *.swp
 *.swo
 *.egg-info/
-.coverage*
+.coverage
 .idea
 .cache/
-.tox*/
 build/
 
 # UI
@@ -17,5 +16,3 @@ build/
 node_modules
 bower_components
 *.map
-sslkeylogfile.log
-.tox/

.sources/bootswatch.less

@@ -0,0 +1,171 @@
+// Bootswatch.less
+// Swatch: Journal
+// Version: 2.0.4
+// -----------------------------------------------------
+
+// TYPOGRAPHY
+// -----------------------------------------------------
+
+@import url('https://fonts.googleapis.com/css?family=Open+Sans:400,700');
+
+h1, h2, h3, h4, h5, h6, .navbar .brand {
+	font-weight: 700;
+}
+
+// SCAFFOLDING
+// -----------------------------------------------------
+
+a {
+	text-decoration: none;
+}
+
+.nav a, .navbar .brand, .subnav a, a.btn, .dropdown-menu a {
+	text-decoration: none;
+}
+
+// NAVBAR
+// -----------------------------------------------------
+
+.navbar {
+
+	.navbar-inner {
+		@shadow: 0 2px 4px rgba(0,0,0,.25), inset 0 -1px 0 rgba(0,0,0,.1);
+		.box-shadow(@shadow);
+		border-top: 1px solid #E5E5E5;
+		.border-radius(0);
+	}
+
+	.brand {
+		text-shadow: none;
+
+		&:hover {
+			background-color: #EEEEEE;
+		}
+	}
+
+	.navbar-text {
+		line-height: 68px;
+	}
+
+	.nav > li > a {
+		text-shadow: none;
+	}
+
+	.dropdown-menu {
+		.border-radius(0);
+	}
+
+	.nav li.dropdown.active > .dropdown-toggle,
+	.nav li.dropdown.active > .dropdown-toggle:hover,
+	.nav li.dropdown.open > .dropdown-toggle,
+	.nav li.dropdown.active.open > .dropdown-toggle,
+	.nav li.dropdown.active.open > .dropdown-toggle:hover {
+		background-color: @grayLighter;
+		color: @linkColor;
+	}
+
+	.nav li.dropdown .dropdown-toggle .caret,
+	.nav .open .caret,
+	.nav .open .dropdown-toggle:hover .caret {
+		border-top-color: @black;
+		opacity: 1;
+	}
+
+	.nav-collapse.in .nav li > a:hover {
+		background-color: @grayLighter;
+	}
+
+	.nav-collapse .nav li > a {
+		color: @textColor;
+		text-decoration: none;
+		font-weight: normal;
+	}
+
+	.nav-collapse .navbar-form,
+	.nav-collapse .navbar-search {
+		border-color: transparent;
+	}
+
+	.navbar-search .search-query,
+	.navbar-search .search-query:hover {
+		border: 1px solid @grayLighter;
+		color: @textColor;
+		.placeholder(@gray);
+	}
+}
+
+div.subnav {
+	background-color: @bodyBackground;
+	background-image: none;
+	@shadow: 0 1px 2px rgba(0,0,0,.25);
+	.box-shadow(@shadow);
+	.border-radius(0);
+
+	&.subnav-fixed {
+		top: @navbarHeight;
+	}
+
+	.nav > li > a:hover,
+	.nav > .active > a,
+	.nav > .active > a:hover {
+		color: @textColor;
+		text-decoration: none;
+		font-weight: normal;
+	}
+
+	.nav > li:first-child > a,
+	.nav > li:first-child > a:hover {
+		.border-radius(0);
+	}
+}
+
+// BUTTONS
+// -----------------------------------------------------
+
+.btn-primary {
+  .buttonBackground(lighten(@linkColor, 5%), @linkColor);
+}
+
+[class^="icon-"], [class*=" icon-"] {
+	vertical-align: -2px;
+}
+
+// MODALS
+// -----------------------------------------------------
+
+.modal {
+ 	.border-radius(0px);
+	background: @bodyBackground;
+}
+
+.modal-header {
+	border-bottom: none;
+}
+
+.modal-header .close {
+	text-decoration: none;
+}
+
+.modal-footer {
+	background: transparent;
+	.box-shadow(none);
+	border-top: none;
+}
+
+
+// MISC
+// -----------------------------------------------------
+
+code, pre, pre.prettyprint, .well {
+	background-color: @grayLighter;
+}
+
+.hero-unit {
+    .box-shadow(inset 0 1px 1px rgba(0,0,0,.05));
+	border: 1px solid rgba(0,0,0,.05);
+	.border-radius(0);
+}
+
+.table-bordered, .well, .prettyprint {
+	.border-radius(0);
+}

.sources/make

@@ -0,0 +1,5 @@
+#!/bin/sh
+pygmentize -f html ../examples/test_context.py > ../pathod/templates/examples_context.html
+pygmentize -f html ../examples/test_setup.py > ../pathod/templates/examples_setup.html
+pygmentize -f html ../examples/test_setupall.py > ../pathod/templates/examples_setupall.html
+pygmentize -f html ../examples/pathod_pathoc.py > ../pathod/templates/pathod_pathoc.html

.sources/variables.less

@@ -0,0 +1,208 @@
+// Variables.less
+// Variables to customize the look and feel of Bootstrap
+// Swatch: Journal
+// Version: 2.0.4
+// -----------------------------------------------------
+
+// GLOBAL VALUES
+// --------------------------------------------------
+
+
+// Grays
+// -------------------------
+@black:                 #000;
+@grayDarker:            #222;
+@grayDark:              #333;
+@gray:                  #888;
+@grayLight:             #999;
+@grayLighter:           #eee;
+@white:                 #fff;
+
+
+// Accent colors
+// -------------------------
+@blue:                  #4380D3;
+@blueDark:              darken(@blue, 15%);
+@green:                 #22B24C;
+@red:                   #C00;
+@yellow:                #FCFADB;
+@orange:                #FF7F00;
+@pink:                  #CC99CC;
+@purple:                #7a43b6;
+@tan:                   #FFCA73;
+
+
+
+// Scaffolding
+// -------------------------
+@bodyBackground:        #FCFBFD;
+@textColor:             @grayDarker;
+
+
+// Links
+// -------------------------
+@linkColor:             @blue;
+@linkColorHover:        @red;
+
+
+// Typography
+// -------------------------
+@sansFontFamily:        'Open Sans', "Helvetica Neue", Helvetica, Arial, sans-serif;
+@serifFontFamily:       Georgia, "Times New Roman", Times, serif;
+@monoFontFamily:        Menlo, Monaco, Consolas, "Courier New", monospace;
+
+@baseFontSize:          14px;
+@baseFontFamily:        @sansFontFamily;
+@baseLineHeight:        18px;
+@altFontFamily:         @serifFontFamily;
+
+@headingsFontFamily:    inherit; // empty to use BS default, @baseFontFamily
+@headingsFontWeight:    bold;    // instead of browser default, bold
+@headingsColor:         inherit; // empty to use BS default, @textColor
+
+
+// Tables
+// -------------------------
+@tableBackground:                   transparent; // overall background-color
+@tableBackgroundAccent:             @grayLighter; // for striping
+@tableBackgroundHover:              #f5f5f5; // for hover
+@tableBorder:                       #ddd; // table and cell border
+
+
+// Buttons
+// -------------------------
+@btnBackground:                     @white;
+@btnBackgroundHighlight:            darken(@white, 10%);
+@btnBorder:                         darken(@white, 20%);
+
+@btnPrimaryBackground:              @linkColor;
+@btnPrimaryBackgroundHighlight:     spin(@btnPrimaryBackground, 15%);
+
+@btnInfoBackground:                 #5bc0de;
+@btnInfoBackgroundHighlight:        #2f96b4;
+
+@btnSuccessBackground:              #62c462;
+@btnSuccessBackgroundHighlight:     #51a351;
+
+@btnWarningBackground:              lighten(@orange, 10%);
+@btnWarningBackgroundHighlight:     @orange;
+
+@btnDangerBackground:               #ee5f5b;
+@btnDangerBackgroundHighlight:      #bd362f;
+
+@btnInverseBackground:              @linkColor;
+@btnInverseBackgroundHighlight:     darken(@linkColor, 5%);
+
+
+// Forms
+// -------------------------
+@inputBackground:               @white;
+@inputBorder:                   #ccc;
+@inputBorderRadius:             3px;
+@inputDisabledBackground:       @grayLighter;
+@formActionsBackground:         @grayLighter;
+
+// Dropdowns
+// -------------------------
+@dropdownBackground:            @bodyBackground;
+@dropdownBorder:                rgba(0,0,0,.2);
+@dropdownLinkColor:             @textColor;
+@dropdownLinkColorHover:        @textColor;
+@dropdownLinkBackgroundHover:   #eee;
+@dropdownDividerTop:            #e5e5e5;
+@dropdownDividerBottom:         @white;
+
+
+
+// COMPONENT VARIABLES
+// --------------------------------------------------
+
+// Z-index master list
+// -------------------------
+// Used for a bird's eye view of components dependent on the z-axis
+// Try to avoid customizing these :)
+@zindexDropdown:          1000;
+@zindexPopover:           1010;
+@zindexTooltip:           1020;
+@zindexFixedNavbar:       1030;
+@zindexModalBackdrop:     1040;
+@zindexModal:             1050;
+
+
+// Sprite icons path
+// -------------------------
+@iconSpritePath:          "../img/glyphicons-halflings.png";
+@iconWhiteSpritePath:     "../img/glyphicons-halflings-white.png";
+
+
+// Input placeholder text color
+// -------------------------
+@placeholderText:         @grayLight;
+
+
+// Hr border color
+// -------------------------
+@hrBorder:                @grayLighter;
+
+
+// Navbar
+// -------------------------
+@navbarHeight:                    50px;
+@navbarBackground:                @bodyBackground;
+@navbarBackgroundHighlight:       @bodyBackground;
+
+@navbarText:                      @textColor;
+@navbarLinkColor:                 @linkColor;
+@navbarLinkColorHover:            @linkColor;
+@navbarLinkColorActive:           @navbarLinkColorHover;
+@navbarLinkBackgroundHover:       @grayLighter;
+@navbarLinkBackgroundActive:      @grayLighter;
+
+@navbarSearchBackground:          lighten(@navbarBackground, 25%);
+@navbarSearchBackgroundFocus:     @white;
+@navbarSearchBorder:              darken(@navbarSearchBackground, 30%);
+@navbarSearchPlaceholderColor:    #ccc;
+@navbarBrandColor:                @blue;
+
+
+// Hero unit
+// -------------------------
+@heroUnitBackground:              @grayLighter;
+@heroUnitHeadingColor:            inherit;
+@heroUnitLeadColor:               inherit;
+
+
+// Form states and alerts
+// -------------------------
+@warningText:             #c09853;
+@warningBackground:       #fcf8e3;
+@warningBorder:           darken(spin(@warningBackground, -10), 3%);
+
+@errorText:               #b94a48;
+@errorBackground:         #f2dede;
+@errorBorder:             darken(spin(@errorBackground, -10), 3%);
+
+@successText:             #468847;
+@successBackground:       #dff0d8;
+@successBorder:           darken(spin(@successBackground, -10), 5%);
+
+@infoText:                #3a87ad;
+@infoBackground:          #d9edf7;
+@infoBorder:              darken(spin(@infoBackground, -10), 7%);
+
+
+
+// GRID
+// --------------------------------------------------
+
+// Default 940px grid
+// -------------------------
+@gridColumns:             12;
+@gridColumnWidth:         60px;
+@gridGutterWidth:         20px;
+@gridRowWidth:            (@gridColumns * @gridColumnWidth) + (@gridGutterWidth * (@gridColumns - 1));
+
+// Fluid grid
+// -------------------------
+@fluidGridColumnWidth:    6.382978723%;
+@fluidGridGutterWidth:    2.127659574%;

.travis.yml

@@ -10,30 +10,24 @@ addons:
     packages:
       - libssl-dev
 
-env:
-  global:
-    - CI_DEPS=codecov>=2.0.5
-    - CI_COMMANDS=codecov
-
 matrix:
   fast_finish: true
   include:
-    - python: 3.5
-      env: TOXENV=lint
-    - os: osx
-      osx_image: xcode7.3
-      language: generic
-      env: TOXENV=py35 BDIST=1
-    - python: 3.5
-      env: TOXENV=py35 BDIST=1
-    - python: 3.5
-      env: TOXENV=py35 NO_ALPN=1
     - python: 2.7
-      env: TOXENV=py27
     - python: 2.7
-      env: TOXENV=py27 NO_ALPN=1
+      env: NO_ALPN=1
+    - language: generic
+      os: osx
+      osx_image: xcode7.1
+      git:
+        depth: 9999999
     - python: 3.5
-      env: TOXENV=docs
+      env: SCOPE="netlib ./test/mitmproxy/script"
+    - python: 3.5
+      env: SCOPE="netlib ./test/mitmproxy/script" NO_ALPN=1
+    - python: 2.7
+      env: DOCS=1
+      script: 'cd docs && make html'
   allow_failures:
     - python: pypy
 
@@ -42,40 +36,44 @@ install:
     if [[ $TRAVIS_OS_NAME == "osx" ]]
     then
       brew update || brew update # try again if it fails
-      brew upgrade
-      brew reinstall openssl
-      brew reinstall pyenv
-      eval "$(pyenv init -)"
-      env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.2
-      pyenv global 3.5.2
-      pyenv shell 3.5.2
-      pip install -U pip setuptools wheel virtualenv
+      brew outdated openssl || brew upgrade openssl
+      brew install python
     fi
-  - pip install tox
+  - pip install -U virtualenv
+  - ./dev.sh
+  - source ./venv/bin/activate
 
-script: tox -- --cov netlib --cov mitmproxy --cov pathod -v
+before_script:
+  - "openssl version -a"
+  - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
+
+script:
+  - "py.test --cov netlib --cov mitmproxy --cov pathod ./test/$SCOPE"
 
 after_success:
+  - coveralls
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
-        git fetch --unshallow
-        ./dev.sh 3.5
-        source venv3.5/bin/activate
         pip install -e ./release
-        python -u ./release/rtool.py bdist
-        python -u ./release/rtool.py upload-snapshot --bdist
+        python ./release/rtool.py bdist
+        python ./release/rtool.py upload-snapshot --bdist --wheel
     fi
 
 notifications:
+  irc:
+    channels:
+      - "irc.oftc.net#mitmproxy"
+    on_success: change
+    on_failure: always
   slack:
-      -rooms:
-          mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu
-      on_success: change
-      on_failure: change
-      on_start: never
+    rooms:
+      - mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu#ci
+    on_success: always
+    on_failure: always
 
 cache:
   directories:
-    - $HOME/.pyenv
     - $HOME/.cache/pip
+    - $HOME/.pyenv
+    - $HOME/Library/Caches/pip

CHANGELOG

@@ -1,51 +1,3 @@
-16 October 2016: mitmproxy 0.18
-
-    * Python 3 Compatibility for mitmproxy and pathod (Shadab Zafar, GSoC 2016)
-
-    * Major improvements to mitmweb (Clemens Brunner & Jason Hao, GSoC 2016)
-
-    * Internal Core Refactor: Separation of most features into isolated Addons
-
-    * Initial Support for WebSockets
-
-    * Improved HTTP/2 Support
-
-    * Reverse Proxy Mode now automatically adjusts host headers and TLS Server Name Indication
-
-    * Improved HAR export
-
-    * Improved export functionality for curl, python code, raw http etc.
-
-    * Flow URLs are now truncated in the console for better visibility
-
-    * New filters for TCP, HTTP and marked flows.
-
-    * Mitmproxy now handles comma-separated Cookie headers
-
-    * Merge mitmproxy and pathod documentation
-
-    * Mitmdump now sanitizes its console output to not include control characters
-
-    * Improved message body handling for HTTP messages:
-      .raw_content provides the message body as seen on the wire
-      .content provides the decompressed body (e.g. un-gzipped)
-      .text provides the body decompressed and decoded body
-
-    * New HTTP Message getters/setters for cookies and form contents.
-
-    * Add ability to view only marked flows in mitmproxy
-
-    * Improved Script Reloader (Always use polling, watch for whole directory)
-
-    * Use tox for testing
-
-    * Unicode support for tnetstrings
-
-    * Add dumpfile converters for mitmproxy versions 0.11 and 0.12
-
-    * Numerous bugfixes
-
-
 9 April 2016: mitmproxy 0.17
 
     * Simplify repository and release structure. mitmproxy now comes as a single package, including netlib and pathod.

CONTRIBUTORS

@@ -1,128 +1,92 @@
-  2184	Aldo Cortesi
-  1745	Maximilian Hils
-   507	Thomas Kriechbaumer
-   258	Shadab Zafar
-    97	Jason
+  1813	Aldo Cortesi
+  1228	Maximilian Hils
+   282	Thomas Kriechbaumer
     83	Marcelo Glezer
-    68	Clemens
     28	Jim Shaver
     18	Henrik Nordstrom
-    16	Matthew Shao
-    14	Pedro Worcel
+    17	Shadab Zafar
     14	David Weinstein
+    14	Pedro Worcel
     13	Thomas Roth
-    11	Stephen Altamirano
     11	Jake Drahos
-    11	arjun23496
     11	Justus Wingert
-    10	Sandor Nemes
-    10	Zohar Lorberbaum
+    11	Stephen Altamirano
     10	András Veres-Szentkirályi
     10	Chris Czub
-    10	smill
-     9	ikoz
+    10	Sandor Nemes
      9	Kyle Morton
      9	Legend Tang
+     9	Matthew Shao
      9	Rouli
-     8	Jason A. Novak
      8	Chandler Abraham
+     8	Jason A. Novak
      7	Alexis Hildebrandt
-     7	Matthias Urlichs
      7	Brad Peabody
-     7	dufferzafar
-     6	Felix Yan
+     7	Matthias Urlichs
+     5	Choongwoo Han
+     5	Sam Cleveland
      5	Tomaz Muraus
      5	elitest
      5	iroiro123
-     5	Sam Cleveland
-     5	Choongwoo Han
-     5	Will Coster
-     4	root
-     4	Clemens Brunner
-     4	Schamper
-     4	Valtteri Virtanen
-     4	Wade 524
-     4	Youhei Sakurai
      4	Bryan Bishop
      4	Marc Liyanage
      4	Michael J. Bazzinotti
-     4	yonder
-     3	Eli Shvartsman
+     4	Valtteri Virtanen
+     4	Wade 524
+     4	Youhei Sakurai
+     4	root
+     3	Benjamin Lee
      3	Chris Neasbitt
+     3	Eli Shvartsman
+     3	Felix Yan
      3	Guillem Anguera
+     3	Kyle Manna
      3	MatthewShao
      3	Ryan Welton
-     3	smill@cuckoo.sh
-     3	Manish Kumar
-     3	Benjamin Lee
-     3	Ryan Laughlin
      3	Zack B
-     3	Kyle Manna
-     3	redfast00
-     3	requires.io
+     2	Anant
+     2	Bennett Blodinger
+     2	Colin Bendell
+     2	Heikki Hannikainen
+     2	Israel Nir
+     2	Jaime Soriano Pastor
+     2	Jim Lloyd
+     2	Krzysztof Bielicki
+     2	Mark E. Haase
+     2	Michael Frister
+     2	Nick Badger
+     2	Niko Kommenda
+     2	Paul
+     2	Rob Wills
+     2	Sean Coates
+     2	Terry Long
+     2	Wade Catron
+     2	alts
      2	isra17
      2	israel
-     2	Colin Bendell
-     2	jpkrause
-     2	Paul
-     2	lilydjwg
-     2	Michael Frister
-     2	依云
-     2	Jaime Soriano Pastor
-     2	Nick Badger
-     2	Rob Wills
-     2	Heikki Hannikainen
-     2	Vincent Haupert
-     2	strohu
-     2	Wade Catron
-     2	Krzysztof Bielicki
-     2	Sachin Kelkar
-     2	Israel Nir
-     2	Anant
-     2	alts
-     2	Doug Freed
-     2	Niko Kommenda
-     2	Terry Long
-     2	Mark E. Haase
-     2	Steven Van Acker
-     2	Jim Lloyd
-     2	Bennett Blodinger
-     2	Sean Coates
-     2	Cory Benfield
-     1	Sergey Chipiga
+     2	requires.io
      1	Andrey Plotnikov
      1	Andy Smith
-     1	Angelo Agatino Nicolosi
-     1	Anthony Zhang
-     1	BSalita
      1	Ben Lerner
      1	Bradley Baetz
-     1	Brett Randall
      1	Chris Hamant
-     1	Christian Frichot
      1	Dan Wilbraham
      1	David Dworken
      1	David Shaw
      1	Doug Lethin
-     1	Drake Caraker
      1	Eric Entzel
      1	Felix Wolfsteller
      1	FreeArtMan
      1	Gabriel Kirkpatrick
      1	Henrik Nordström
-     1	Israel Blancas
      1	Ivaylo Popov
      1	JC
      1	Jakub Nawalaniec
      1	Jakub Wilk
      1	James Billingham
-     1	Jason Pepas
      1	Jean Regisser
-     1	Jonathan Jones
      1	Jorge Villacorta
      1	Kit Randel
-     1	Kostya Esmukov
-     1	Linmiao Xu
      1	Lucas Cimon
      1	M. Utku Altinkaya
      1	Mathieu Mitchell
@@ -134,35 +98,27 @@
      1	Nick Raptis
      1	Nicolas Esteves
      1	Oleksandr Sheremet
-     1	Parth Ganatra
      1	Pritam Baral
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
      1	Ryo Onodera
      1	Sahn Lam
-     1	Sanchit Sokhey
      1	Seppo Yli-Olli
-     1	Aditya
+     1	Sergey Chipiga
      1	Stefan Wärting
      1	Steve Phillips
-     1	Steven Noble
+     1	Steven Van Acker
      1	Suyash
-     1	Tai Dickerson
      1	Tarashish Mishra
      1	TearsDontFalls
-     1	Thiago Arrais
      1	Tim Becker
      1	Timothy Elliott
-     1	Tyler St. Onge
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
-     1	Wes Turner
-     1	Yoginski
+     1	Will Coster
      1	Yuangxuan Wang
      1	capt8bit
-     1	chhsiao90
-     1	cle1000
      1	davidpshaw
      1	deployable
      1	gecko655
@@ -172,10 +128,9 @@
      1	meeee
      1	michaeljau
      1	peralta
-     1	phackt
      1	phil plante
      1	sentient07
      1	sethp-jive
      1	starenka
-     1	vulnminer
      1	vzvu3k6k
+     1	依云

Dockerfile

@@ -0,0 +1,4 @@
+FROM mitmproxy/base:latest-onbuild
+EXPOSE 8080
+EXPOSE 8081
+VOLUME /certs

README.rst

@@ -1,86 +1,64 @@
 mitmproxy
 ^^^^^^^^^
 
-|travis| |appveyor| |coverage| |latest_release| |python_versions|
+|travis| |coveralls| |downloads| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects, as well as
-their shared networking library, **netlib**.
+This repository contains the **mitmproxy** and **pathod** projects, as well as their shared networking library, **netlib**.
 
-``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
-interface.
+``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console interface.
 
 ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
-``pathoc`` and ``pathod`` are perverse HTTP client and server applications
-designed to let you craft almost any conceivable HTTP request, including ones
-that creatively violate the standards.
+``pathoc`` and ``pathod`` are perverse HTTP client and server applications designed to let you craft almost any conceivable HTTP request, including ones that creatively violate the standards.
 
 
 Documentation & Help
 --------------------
 
-
-General information, tutorials, and precompiled binaries can be found on the mitmproxy
-and pathod websites.
+Documentation, tutorials and precompiled binaries can be found on the mitmproxy and pathod websites.
 
 |mitmproxy_site| |pathod_site|
 
-
 The latest documentation for mitmproxy is also available on ReadTheDocs.
 
 |mitmproxy_docs|
 
-
-Join our discussion forum on Discourse to ask questions, help
-each other solve problems, and come up with new ideas for the project.
-
-|mitmproxy_discourse|
-
-
-Join our developer chat on Slack if you would like to hack on mitmproxy itself.
+You can join our developer chat on Slack.
 
 |slack|
 
 
-Installation
-------------
-
-The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`_.
-If you want to contribute changes, keep on reading.
-
-
 Hacking
 -------
 
-To get started hacking on mitmproxy, make sure you have Python_ 3.5.x or above with
-virtualenv_ installed (you can find installation instructions for virtualenv
-`here <http://virtualenv.readthedocs.org/en/latest/>`_). Then do the following:
+To get started hacking on mitmproxy, make sure you have Python_ 2.7.x. with
+virtualenv_ installed (you can find installation instructions for virtualenv here_).
+Then do the following:
 
 .. code-block:: text
 
     git clone https://github.com/mitmproxy/mitmproxy.git
     cd mitmproxy
-    ./dev.sh  # powershell .\dev.ps1 on Windows
+    ./dev
 
 
-The *dev* script will create a virtualenv environment in a directory called
-"venv", and install all mandatory and optional dependencies into it. The
-primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
-"editable", so any changes to the source in the repository will be reflected
-live in the virtualenv.
+The *dev* script will create a virtualenv environment in a directory called "venv",
+and install all mandatory and optional dependencies into it.
+The primary mitmproxy components - mitmproxy, netlib and pathod - are installed as "editable",
+so any changes to the source in the repository will be reflected live in the virtualenv.
 
 To confirm that you're up and running, activate the virtualenv, and run the
 mitmproxy test suite:
 
 .. code-block:: text
 
-    . venv/bin/activate  # venv\Scripts\activate on Windows
+    . venv/bin/activate # venv\Scripts\activate.bat on Windows
     py.test
 
 Note that the main executables for the project - ``mitmdump``, ``mitmproxy``,
-``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the
-virtualenv. After activating the virtualenv, they will be on your $PATH, and
-you can run them like any other command:
+``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the virtualenv. After activating the
+virtualenv, they will be on your $PATH, and you can run them like any other
+command:
 
 .. code-block:: text
 
@@ -103,28 +81,13 @@ requirements installed, and you can simply run the test suite:
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project tries to maintain 100% test coverage.
 
-You can also use `tox` to run a full suite of tests in Python 2.7 and 3.5,
-including a quick test to check documentation and code linting.
-
-The following tox environments are relevant for local testing:
-
-.. code-block:: text
-
-    tox -e py27  # runs all tests with Python 2.7
-    tox -e py35  # runs all tests with Python 3.5
-    tox -e docs  # runs a does-it-compile check on the documentation
-    tox -e lint  # runs the linter for coding style checks
-
-We support Python 2.7 and 3.5, so please make sure all tests pass in both
-environments. Running `tox` ensures all necessary tests are executed.
-
 
 Documentation
--------------
+----
 
-The mitmproxy documentation is build using Sphinx_, which is installed
-automatically if you set up a development environment as described above. After
-installation, you can render the documentation like this:
+The mitmproxy documentation is build using Sphinx_, which is installed automatically if you set up a development
+environment as described above.
+After installation, you can render the documentation like this:
 
 .. code-block:: text
 
@@ -136,20 +99,6 @@ installation, you can render the documentation like this:
 The last command invokes `sphinx-autobuild`_, which watches the Sphinx directory and rebuilds
 the documentation when a change is detected.
 
-Style
------
-
-Keeping to a consistent code style throughout the project makes it easier to
-contribute and collaborate. Please stick to the guidelines in
-`PEP8`_ and the `Google Style Guide`_ unless there's a very
-good reason not to.
-
-This is automatically enforced on every PR. If we detect a linting error, the
-PR checks will fail and block merging. We are using this command to check for style compliance:
-
-.. code-block:: text
-
-    flake8 --jobs 8 --count mitmproxy netlib pathod examples test
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg
@@ -164,26 +113,22 @@ PR checks will fail and block merging. We are using this command to check for st
     :target: http://docs.mitmproxy.org/en/latest/
     :alt: mitmproxy documentation
 
-.. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg
-    :target: https://discourse.mitmproxy.org
-    :alt: Discourse: mitmproxy
-
 .. |slack| image:: http://slack.mitmproxy.org/badge.svg
     :target: http://slack.mitmproxy.org/
     :alt: Slack Developer Chat
 
-.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=Travis%20build
+.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg
     :target: https://travis-ci.org/mitmproxy/mitmproxy
-    :alt: Travis Build Status
+    :alt: Build Status
 
-.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mhils/mitmproxy/master.svg?label=Appveyor%20build
-    :target: https://ci.appveyor.com/project/mhils/mitmproxy
-    :alt: Appveyor Build Status
-
-.. |coverage| image:: https://codecov.io/gh/mitmproxy/mitmproxy/branch/master/graph/badge.svg
-    :target: https://codecov.io/gh/mitmproxy/mitmproxy
+.. |coveralls| image:: https://shields.mitmproxy.org/coveralls/mitmproxy/mitmproxy/master.svg
+    :target: https://coveralls.io/r/mitmproxy/mitmproxy
     :alt: Coverage Status
 
+.. |downloads| image:: https://shields.mitmproxy.org/pypi/dm/mitmproxy.svg?color=orange
+    :target: https://pypi.python.org/pypi/mitmproxy
+    :alt: Downloads
+
 .. |latest_release| image:: https://shields.mitmproxy.org/pypi/v/mitmproxy.svg
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Latest Version
@@ -194,10 +139,9 @@ PR checks will fail and block merging. We are using this command to check for st
 
 .. _Python: https://www.python.org/
 .. _virtualenv: http://virtualenv.readthedocs.org/en/latest/
+.. _here: http://virtualenv.readthedocs.org/en/latest/installation.html
 .. _autoenv: https://github.com/kennethreitz/autoenv
 .. _.env: https://github.com/mitmproxy/mitmproxy/blob/master/.env
 .. _Sphinx: http://sphinx-doc.org/
 .. _sphinx-autobuild: https://pypi.python.org/pypi/sphinx-autobuild
 .. _issue_tracker: https://github.com/mitmproxy/mitmproxy/issues
-.. _PEP8: https://www.python.org/dev/peps/pep-0008
-.. _Google Style Guide: https://google.github.io/styleguide/pyguide.html

dev.bat

@@ -0,0 +1,14 @@
+@echo off
+set VENV=.\venv
+
+virtualenv %VENV% --always-copy
+if %errorlevel% neq 0 exit /b %errorlevel%
+call %VENV%\Scripts\activate.bat
+if %errorlevel% neq 0 exit /b %errorlevel%
+pip install -r requirements.txt
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+echo.
+echo * Created virtualenv environment in %VENV%.
+echo * Installed all dependencies into the virtualenv.
+echo * Activated virtualenv environment.

dev.ps1

@@ -1,16 +0,0 @@
-$ErrorActionPreference = "Stop"
-$VENV = ".\venv"
-
-virtualenv $VENV --always-copy
-& $VENV\Scripts\activate.ps1
-
-python -m pip install --disable-pip-version-check -U pip
-cmd /c "pip install -r requirements.txt 2>&1"
-
-echo @"
-
-  * Created virtualenv environment in $VENV.
-  * Installed all dependencies into the virtualenv.
-  * Activated virtualenv environment.
-
-"@

dev.sh

@@ -1,17 +1,13 @@
-#!/bin/sh
+#!/bin/bash
 set -e
-set -x
+VENV=./venv
 
-PYVERSION=$1
-VENV="venv$1"
-
-echo "Creating dev environment in $VENV using Python $PYVERSION"
-
-python$PYVERSION -m virtualenv "$VENV" --always-copy
-. "$VENV/bin/activate"
-pip$PYVERSION install -U pip setuptools
-pip$PYVERSION install -r requirements.txt
+python -m virtualenv $VENV --always-copy
+. $VENV/bin/activate
+pip install -U pip setuptools
+pip install -r requirements.txt
 
 echo ""
-echo "* Virtualenv created in $VENV and all dependencies installed."
-echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV/bin/activate\`"
+echo "* Created virtualenv environment in $VENV."
+echo "* Installed all dependencies into the virtualenv."
+echo "* You can now activate the virtualenv: \`. $VENV/bin/activate\`"

docs/Makefile

@@ -192,4 +192,4 @@ pseudoxml:
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
 
 livehtml:
-	sphinx-autobuild -b html -z '../mitmproxy' -r '___jb_(old|bak|tmp)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	sphinx-autobuild -b html -z '../mitmproxy' -z '../../netlib/netlib' -r '___jb_(old|bak)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html

docs/_static/theme_overrides.css

@@ -1,44 +0,0 @@
-
-/* override table width restrictions */
-.wy-table-responsive table td, .wy-table-responsive table th {
-    white-space: normal;
-}
-
-.wy-table-responsive > table > tbody > tr > td {
-    vertical-align: top !important;
-}
-
-.wy-table-responsive {
-    margin-bottom: 24px;
-    max-width: 100%;
-    overflow: visible;
-}
-
-.wy-menu-vertical header, .wy-menu-vertical p.caption {
-    color: #e0e0e0;
-}
-
-.code-block-caption {
-    height: 1.5em;
-}
-
-.code-block-caption .caption-text {
-    font-size: 0.8em;
-    float: right;
-}
-
-.code-block-caption .headerlink {
-    display: none !important;
-}
-
-.function .headerlink {
-    display: none !important;
-}
-
-dl .reference.internal {
-    display: none !important;
-}
-
-dl .headerlink {
-    display: none !important;
-}

docs/certinstall.rst

@@ -65,7 +65,7 @@ See http://windows.microsoft.com/en-ca/windows/import-export-certificates-privat
 Windows (automated)
 ^^^^^^^^^^^^^^^^^^^
 
->>> certutil.exe -importpfx Root mitmproxy-ca-cert.p12
+>>> certutil.exe -importpfx mitmproxy-ca-cert.p12
 
 See also: https://technet.microsoft.com/en-us/library/cc732443.aspx
 
@@ -130,12 +130,12 @@ mitmproxy-ca-cert.cer Same file as .pem, but with an extension expected by some
 Using a custom certificate
 --------------------------
 
-You can use your own certificate by passing the ``--cert [domain=]path_to_certificate`` option to
+You can use your own certificate by passing the ``--cert`` option to
 mitmproxy. Mitmproxy then uses the provided certificate for interception of the
-specified domain instead of generating a certificate signed by its own CA.
+specified domains instead of generating a certificate signed by its own CA.
 
 The certificate file is expected to be in the PEM format.  You can include
-intermediary certificates right below your leaf certificate, so that your PEM
+intermediary certificates right below your leaf certificate, so that you PEM
 file roughly looks like this:
 
 .. code-block:: none
@@ -158,18 +158,7 @@ For example, you can generate a certificate in this format using these instructi
 >>> openssl req -new -x509 -key cert.key -out cert.crt
     (Specify the mitm domain as Common Name, e.g. *.google.com)
 >>> cat cert.key cert.crt > cert.pem
-
-Now, you can run mitmproxy with the generated certificate:
-
-**For all domain names**
-
-``>>>mitmproxy --cert *=cert.pem``
-
-**For specific domain names**
-
-``>>>mitmproxy --cert *.example.com=cert.pem``
-
-**Note:** ``*.example.com`` is for all the subdomains. You can also use ``www.example.com`` for a particular subdomain.
+>>> mitmproxy --cert=cert.pem
 
 
 Using a custom certificate authority
@@ -203,4 +192,4 @@ directory and uses this as the client cert.
 
 
 
-.. _Certificate Pinning: http://security.stackexchange.com/questions/29988/what-is-certificate-pinning/
+.. _Certificate Pinning: http://security.stackexchange.com/questions/29988/what-is-certificate-pinning/

docs/conf.py

@@ -1,18 +1,40 @@
-import importlib
-import inspect
-import os
-import subprocess
+# -*- coding: utf-8 -*-
+#
+# mitmproxy documentation build configuration file, created by
+# sphinx-quickstart on Thu Sep 03 14:04:13 2015.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
 import sys
+import os
+import shlex
 
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('..'))
-import netlib.version
 
+import mitmproxy.version
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
 
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
-    'sphinx.ext.extlinks',
-    'sphinx.ext.linkcode',
     'sphinx.ext.viewcode',
     'sphinx.ext.napoleon',
     'sphinxcontrib.documentedlist'
@@ -39,7 +61,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mitmproxy docs'
-copyright = u'2016, the mitmproxy project'
+copyright = u'2015, the mitmproxy project'
 author = u'The mitmproxy project'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -47,9 +69,9 @@ author = u'The mitmproxy project'
 # built documents.
 #
 # The short X.Y version.
-version = netlib.version.VERSION
+version = mitmproxy.version.VERSION
 # The full version, including alpha/beta/rc tags.
-release = netlib.version.VERSION
+release = mitmproxy.version.VERSION
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -131,7 +153,7 @@ html_favicon = "favicon.ico"
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
@@ -163,7 +185,7 @@ html_static_path = ['_static']
 #html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-html_show_sourcelink = False
+#html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
 #html_show_sphinx = True
@@ -194,53 +216,4 @@ html_show_sourcelink = False
 #html_search_scorer = 'scorer.js'
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'mitmproxydoc'
-
-last_tag, tag_dist, commit = (
-    subprocess.check_output(["git", "describe", "--tags", "--long"])
-        .decode()
-        .strip()
-        .rsplit("-", 2)
-)
-tag_dist = int(tag_dist)
-if tag_dist == 0:
-    tag = last_tag
-else:
-    tag = "master"
-
-SRCBASE = "https://github.com/mitmproxy/mitmproxy/blob/{}".format(tag)
-
-extlinks = dict(
-    src = (SRCBASE + r"/%s", '')
-)
-
-
-def linkcode_resolve(domain, info):
-    if domain != 'py':
-        return None
-    module, fullname = info['module'], info['fullname']
-    if not module:
-        return None
-    obj = importlib.import_module(module)
-    for item in fullname.split('.'):
-        obj = getattr(obj, item, None)
-    if obj is None:
-        return None
-    try:
-        spath = inspect.getsourcefile(obj)
-        _, line = inspect.getsourcelines(obj)
-    except (TypeError, IOError):
-        return None
-    if spath.rfind("netlib") > -1:
-        off = spath.rfind("netlib")
-        mpath = spath[off:]
-    elif spath.rfind("mitmproxy") > -1:
-        off = spath.rfind("mitmproxy")
-        mpath = spath[off:]
-    else:
-        return None
-    return SRCBASE + "/%s#L%s" % (mpath, line)
-
-
-def setup(app):
-    app.add_stylesheet('theme_overrides.css')
+htmlhelp_basename = 'mitmproxydoc'

docs/config.rst

@@ -62,7 +62,7 @@ Options
 
 The options available in the config files are precisely those available as
 command-line flags, with the key being the option's long name. To get a
-complete list of these, use the ``--help`` option on each of the tools. Be
+complete list of these, use the :option:`--help` option on each of the tools. Be
 careful to only specify common options in the **common.conf** file -
 unsupported options in this file will be detected as an error on startup.
 
@@ -72,7 +72,7 @@ Examples
 common.conf
 ^^^^^^^^^^^
 
-Note that ``--port`` is an option supported by all tools.
+Note that :option:`--port` is an option supported by all tools.
 
 .. code-block:: none
 

docs/dev/addingviews.html

@@ -18,22 +18,22 @@ __prompt__, and __content\_types__ and a function named __\_\_call\_\___.
 Adding a new content viewer to parse a data type is as simple as writing a new
 View class.  Your new content viewer View class should have the same properties
 as the other View classes: __name__, __prompt__, and __content\_types__ and a
-__\_\_call\_\___ function to parse the content of the request/response.
+__\_\_call\_\___ function to parse the content of the request/response. 
 
-* The __name__ property should be a string describing the contents and new content viewer;
+* The __name__ property should be a string describing the contents and new content viewer; 
 * The __prompt__ property should be a two item tuple:
 
   - __1__: A string that will be used to display the new content viewer's type; and
-  - __2__: A one character string that will be the hotkey used to select the new content viewer from the Flow View screen;
+  - __2__: A one character string that will be the hotkey used to select the new content viewer from the Flow View screen; 
 
-* The __content\_types__ property should be a list of strings of HTTP Content\-Types that the new content viewer can parse.
+* The __content\_types__ property should be a list of strings of HTTP Content\-Types that the new content viewer can parse.  
   * Note that mitmproxy will use the content\_types to try and heuristically show a friendly view of content and that you can override the built-in views by populating content\_types with values for content\_types that are already parsed -- e.g. "image/png".
 
 After defining the __name__, __prompt__, and __content\_types__ properties of
 the class, you should write the __\_\_call\_\___ function, which will parse the
 request/response data and provide a friendly view of the data.  The
 __\_\_call\_\___ function should take the following arguments: __self__,
-__hdrs__, __content__, __limit__; __hdrs__ is a MultiDict object containing
+__hdrs__, __content__, __limit__; __hdrs__ is a ODictCaseless object containing
 the headers of the request/response; __content__ is the content of the
 request/response, and __limit__ is an integer representing the amount of data
 to display in the view window.
@@ -46,7 +46,7 @@ Alternatively, you can display content as a series of key-value pairs; to do
 so, prepare a list of lists, where each list item is a two item list -- a key
 that describes the data, and then the data itself; after preparing the list of
 lists, use the __common.format\_keyvals__ function on it to prepare it as text
-for display.
+for display.  
 
 If the new content viewer fails or throws an exception, mitmproxy will default
 to a __raw__ view.

docs/dev/exceptions.rst

@@ -0,0 +1,9 @@
+.. _exceptions:
+
+Exceptions
+==========
+
+.. automodule:: mitmproxy.exceptions
+    :show-inheritance:
+    :members:
+    :undoc-members:

docs/dev/models.rst

@@ -0,0 +1,59 @@
+.. _models:
+
+Models
+======
+
+.. automodule:: netlib.http
+
+    .. autoclass:: Request
+
+        .. rubric:: Data
+        .. autoattribute:: first_line_format
+        .. autoattribute:: method
+        .. autoattribute:: scheme
+        .. autoattribute:: host
+        .. autoattribute:: port
+        .. autoattribute:: path
+        .. autoattribute:: http_version
+        .. autoattribute:: headers
+        .. autoattribute:: content
+        .. autoattribute:: timestamp_start
+        .. autoattribute:: timestamp_end
+        .. rubric:: Computed Properties and Convenience Methods
+        .. autoattribute:: text
+        .. autoattribute:: url
+        .. autoattribute:: pretty_host
+        .. autoattribute:: pretty_url
+        .. autoattribute:: query
+        .. autoattribute:: cookies
+        .. autoattribute:: path_components
+        .. automethod:: anticache
+        .. automethod:: anticomp
+        .. automethod:: constrain_encoding
+        .. autoattribute:: urlencoded_form
+        .. autoattribute:: multipart_form
+
+    .. autoclass:: Response
+
+        .. rubric:: Data
+        .. autoattribute:: http_version
+        .. autoattribute:: status_code
+        .. autoattribute:: reason
+        .. autoattribute:: headers
+        .. autoattribute:: content
+        .. autoattribute:: timestamp_start
+        .. autoattribute:: timestamp_end
+        .. rubric:: Computed Properties and Convenience Methods
+        .. autoattribute:: text
+        .. autoattribute:: cookies
+
+    .. autoclass:: Headers
+        :members:
+        :special-members:
+        :no-undoc-members:
+
+    .. autoclass:: decoded
+
+.. automodule:: mitmproxy.models
+    :show-inheritance:
+    :members: HTTPFlow, Error, ClientConnection, ServerConnection

docs/dev/protocols.rst

@@ -0,0 +1,15 @@
+.. _protocols:
+
+Protocols
+=========
+
+.. automodule:: mitmproxy.protocol
+
+    .. autoclass:: Layer
+        :members:
+        :special-members:
+
+    .. autoclass:: ServerConnectionMixin
+        :members:
+
+    .. autoexception:: Kill

docs/dev/proxy.rst

@@ -0,0 +1,12 @@
+.. _proxy:
+
+Proxy Server
+============
+
+.. automodule:: mitmproxy.proxy
+
+    .. autoclass:: ProxyServer
+    .. autoclass:: DummyServer
+    .. autoclass:: ProxyConfig
+    .. autoclass:: RootContext
+        :members:

docs/dev/testing.rst

@@ -10,7 +10,7 @@ suitable extension to the test suite.
 Our tests are written for the `py.test`_ or nose_ test frameworks.
 At the point where you send your pull request, a command like this:
 
->>> py.test --cov mitmproxy --cov netlib
+>>> py.test -n 4 --cov mitmproxy
 
 Should give output something like this:
 
@@ -25,7 +25,7 @@ Should give output something like this:
     > mitmproxy/controller      69      0   100%
     > mitmproxy/dump           150      0   100%
     > mitmproxy/encoding        39      0   100%
-    > mitmproxy/flowfilter     201      0   100%
+    > mitmproxy/filt           201      0   100%
     > mitmproxy/flow           891      0   100%
     > mitmproxy/proxy          427      0   100%
     > mitmproxy/script          27      0   100%

docs/features/anticache.rst

@@ -2,7 +2,7 @@
 
 Anticache
 =========
-When the ``--anticache`` option is passed to mitmproxy, it removes headers
+When the :option:`--anticache` option is passed to mitmproxy, it removes headers
 (``if-none-match`` and ``if-modified-since``) that might elicit a
 ``304 not modified`` response from the server. This is useful when you want to make
 sure you capture an HTTP exchange in its totality. It's also often used during
@@ -10,6 +10,6 @@ sure you capture an HTTP exchange in its totality. It's also often used during
 
 
 ================== ======================
-command-line       ``--anticache``
+command-line       :option:`--anticache`
 mitmproxy shortcut :kbd:`o` then :kbd:`a`
 ================== ======================

docs/features/clientreplay.rst

@@ -12,7 +12,7 @@ conversation, where requests may have been made concurrently.
 You may want to use client-side replay in conjunction with the
 :ref:`anticache` option, to make sure the server responds with complete data.
 
-================== ===========
-command-line       ``-c path``
-mitmproxy shortcut :kbd:`R` then :kbd:`c`
-================== ===========
+================== =================
+command-line       :option:`-c path`
+mitmproxy shortcut :kbd:`c`
+================== =================

docs/features/filters.rst

@@ -8,7 +8,7 @@ Filter expressions consist of the following operators:
 
 .. documentedlist::
     :header: "Expression" "Description"
-    :listobject: mitmproxy.flowfilter.help
+    :listobject: mitmproxy.filt.help
 
 - Regexes are Python-style
 - Regexes can be specified as quoted strings
@@ -36,3 +36,4 @@ Anything but requests with a text/html content type:
 .. code-block:: none
 
     !(~q & ~t "text/html")
+

docs/features/passthrough.rst

@@ -10,8 +10,7 @@ mechanism:
   mitmproxy's interception leads to errors. For example, the Twitter app, Windows Update or
   the Apple App Store fail to work if mitmproxy is active.
 - **Convenience:** You really don't care about some parts of the traffic and just want them to go
-  away. Note that mitmproxy's "Limit" option is often the better alternative here, as it is
-  not affected by the limitations listed below.
+  away.
 
 If you want to peek into (SSL-protected) non-HTTP connections, check out the :ref:`tcpproxy`
 feature.
@@ -21,24 +20,21 @@ take a look at the :ref:`responsestreaming` feature.
 How it works
 ------------
 
-================== ======================
-command-line       ``--ignore regex``
+================== =============================
+command-line       :option:`--ignore regex`
 mitmproxy shortcut :kbd:`o` then :kbd:`I`
-================== ======================
+================== =============================
 
 
 mitmproxy allows you to specify a regex which is matched against a ``host:port`` string
 (e.g. "example.com:443") to determine hosts that should be excluded.
 
-Limitations
------------
-
 There are two important quirks to consider:
 
 - **In transparent mode, the ignore pattern is matched against the IP and ClientHello SNI host.** While we usually infer the
-  hostname from the Host header if the ``--host`` argument is passed to mitmproxy, we do not
+  hostname from the Host header if the :option:`--host` argument is passed to mitmproxy, we do not
   have access to this information before the SSL handshake. If the client uses SNI however, then we treat the SNI host as an ignore target.
-- **In regular mode, explicit HTTP requests are never ignored.** [#explicithttp]_ The ignore pattern is
+- In regular mode, explicit HTTP requests are never ignored. [#explicithttp]_ The ignore pattern is
   applied on CONNECT requests, which initiate HTTPS or clear-text WebSocket connections.
 
 Tutorial
@@ -46,7 +42,7 @@ Tutorial
 
 If you just want to ignore one specific domain, there's usually a bulletproof method to do so:
 
-1. Run mitmproxy or mitmdump in verbose mode (``-v``) and observe the ``host:port``
+1. Run mitmproxy or mitmdump in verbose mode (:option:`-v`) and observe the ``host:port``
    information in the serverconnect messages. mitmproxy will filter on these.
 2. Take the ``host:port`` string, surround it with ^ and $, escape all dots (. becomes \\.)
    and use this as your ignore pattern:
@@ -90,7 +86,6 @@ Here are some other examples for ignore patterns:
 
     - :ref:`tcpproxy`
     - :ref:`responsestreaming`
-    - mitmproxy's "Limit" feature
 
 .. rubric:: Footnotes
 

docs/features/proxyauth.rst

@@ -10,8 +10,8 @@ upstream servers. For now, only HTTP Basic authentication is supported. The
 proxy auth options are not compatible with the transparent, socks or reverse proxy
 mode.
 
-================== ======================
-command-line       ``--nonanonymous``,
-                   ``--singleuser USER``,
-                   ``--htpasswd PATH``
-================== ======================
+================== =============================
+command-line       :option:`--nonanonymous`,
+                   :option:`--singleuser USER`,
+                   :option:`--htpasswd PATH`
+================== =============================

docs/features/replacements.rst

@@ -54,7 +54,7 @@ So, you might start **mitmdump** as follows:
 
 This will load the replacement text from the file ``~/xss-exploit``.
 
-Both the ``--replace`` and ``--replace-from-file`` flags can be passed multiple
+Both the :option:`--replace` and :option:`--replace-from-file` flags can be passed multiple
 times.
 
 
@@ -65,8 +65,8 @@ The :kbd:`R` shortcut key in the mitmproxy options menu (:kbd:`o`) lets you add
 replacement hooks using a built-in editor. The context-sensitive help (:kbd:`?`) has
 complete usage information.
 
-================== =======================
-command-line       ``--replace``,
-                   ``--replace-from-file``
+================== =============================
+command-line       :option:`--replace`,
+                   :option:`--replace-from-file`
 mitmproxy shortcut :kbd:`o` then :kbd:`R`
-================== =======================
+================== =============================

docs/features/responsestreaming.rst

@@ -19,9 +19,9 @@ On the command-line
 Streaming can be enabled on the command line for all response bodies exceeding a certain size.
 The SIZE argument understands k/m/g suffixes, e.g. 3m for 3 megabytes.
 
-================== =================
-command-line       ``--stream SIZE``
-================== =================
+================== =============================
+command-line       :option:`--stream SIZE`
+================== =============================
 
 .. warning::
 
@@ -35,10 +35,10 @@ command-line       ``--stream SIZE``
 Customizing Response Streaming
 ------------------------------
 
-You can also use a script to customize exactly which responses are streamed.
+You can also use an :ref:`inlinescripts` to customize exactly
+which responses are streamed.
 
-Responses that should be tagged for streaming by setting their ``.stream``
-attribute to ``True``:
+Responses that should be tagged for streaming by setting their ``.stream`` attribute to ``True``:
 
 .. literalinclude:: ../../examples/stream.py
    :caption: examples/stream.py

docs/features/reverseproxy.rst

@@ -7,9 +7,9 @@ In reverse proxy mode, mitmproxy accepts standard HTTP(S) requests and forwards
 them to the specified upstream server. This is in contrast to :ref:`upstreamproxy`, in which
 mitmproxy forwards HTTP(S) proxy requests to an upstream proxy server.
 
-================== ================================
-command-line       ``-R http[s]://hostname[:port]``
-================== ================================
+================== =====================================
+command-line       :option:`-R http[s]://hostname[:port]`
+================== =====================================
 
 Here, **http[s]** signifies if the proxy should use TLS to connect to the server.
 mitmproxy always accepts both encrypted and unencrypted requests and transforms
@@ -29,14 +29,29 @@ them to what the server expects.
 Host Header
 -----------
 
-In reverse proxy mode, mitmproxy automatically rewrites the Host header to match the
-upstream server. This allows mitmproxy to easily connect to existing endpoints on the
-open web (e.g. ``mitmproxy -R https://example.com``).
+In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this
+may lead to issues with public web servers. For example, consider the following scenario:
 
-However, keep in mind that absolute URLs within the returned document or HTTP redirects will
-NOT be rewritten by mitmproxy. This means that if you click on a link for "http://example.com"
-in the returned web page, you will be taken directly to that URL, bypassing mitmproxy.
+.. code-block:: none
+    :emphasize-lines: 5
 
-One possible way to address this is to modify the hosts file of your OS so that "example.com"
-resolves to your proxy's IP, and then access the proxy by going directly to example.com.
-Make sure that your proxy can still resolve the original IP, or specify an IP in mitmproxy.
+    >>> mitmdump -d -R http://example.com/
+    >>> curl http://localhost:8080/
+
+    >> GET https://example.com/
+        Host: localhost:8080
+        User-Agent: curl/7.35.0
+        [...]
+
+    << 404 Not Found 345B
+
+Since the Host header doesn't match "example.com", an error is returned.
+There are two ways to solve this:
+
+1. Modify the hosts file of your OS so that "example.com" resolves to your proxy's IP.
+   Then, access example.com directly. Make sure that your proxy can still resolve the original IP
+   or specify an IP in mitmproxy.
+2. Use mitmproxy's :ref:`setheaders` feature to rewrite the host header:
+   ``--setheader :~q:Host:example.com``.
+   However, keep in mind that absolute URLs within the returned document or HTTP redirects will
+   cause the client application to bypass the proxy.

docs/features/serverreplay.rst

@@ -13,7 +13,7 @@ By default, :program:`mitmproxy` excludes request headers when matching incoming
 requests with responses from the replay file. This works in most circumstances,
 and makes it possible to replay server responses in situations where request
 headers would naturally vary, e.g. using a different user agent.
-The ``--rheader headername`` command-line option allows you to override
+The :option:`--rheader headername` command-line option allows you to override
 this behaviour by specifying individual headers that should be included in matching.
 
 
@@ -30,10 +30,10 @@ recording. So, if they were in the past at the time of recording, they will be
 in the past at the time of replay, and vice versa. Cookie expiry times are
 updated in a similar way.
 
-You can turn off response refreshing using the ``--norefresh`` argument, or using
+You can turn off response refreshing using the :option:`--norefresh` argument, or using
 the :kbd:`o` options shortcut within :program:`mitmproxy`.
 
-================== ===========
-command-line       ``-S path``
-mitmproxy shortcut :kbd:`R` then :kbd:`s`
-================== ===========
+================== =================
+command-line       :option:`-S path`
+mitmproxy shortcut :kbd:`S`
+================== =================

docs/features/setheaders.rst

@@ -13,7 +13,7 @@ Example: Set the **Host** header to "example.com" for all requests.
 
     mitmdump -R http://example.com --setheader :~q:Host:example.com
 
-================== =======================
-command-line       ``--setheader PATTERN``
+================== =============================
+command-line       :option:`--setheader PATTERN`
 mitmproxy shortcut :kbd:`o` then :kbd:`H`
-================== =======================
+================== =============================

docs/features/socksproxy.rst

@@ -5,6 +5,6 @@ SOCKS Mode
 
 In this mode, mitmproxy acts as a SOCKS5 proxy server.
 
-================== ===========
-command-line       ``--socks``
-================== ===========
+================== =================
+command-line       :option:`--socks`
+================== =================

docs/features/sticky.rst

@@ -21,7 +21,7 @@ record the authentication process once, and simply replay it on startup every ti
 to interact with the secured resources.
 
 ================== ======================
-command-line       ``-t FILTER``
+command-line       :option:`-t FILTER`
 mitmproxy shortcut :kbd:`o` then :kbd:`t`
 ================== ======================
 
@@ -36,6 +36,6 @@ authentication through the proxy. Note that :program:`mitmproxy` doesn't (yet) s
 replay of HTTP Digest authentication.
 
 ================== ======================
-command-line       ``-u FILTER``
+command-line       :option:`-u FILTER`
 mitmproxy shortcut :kbd:`o` then :kbd:`A`
 ================== ======================

docs/features/tcpproxy.rst

@@ -18,7 +18,7 @@ How it works
 ------------
 
 ================== ======================
-command-line       ``--tcp HOST``
+command-line       :option:`--tcp HOST`
 mitmproxy shortcut :kbd:`o` then :kbd:`T`
 ================== ======================
 

docs/features/upstreamcerts.rst

@@ -17,7 +17,7 @@ certs in transparent mode.
 
 Upstream cert sniffing is on by default, and can optionally be turned off.
 
-================== ======================
-command-line       ``--no-upstream-cert``
+================== =============================
+command-line       :option:`--no-upstream-cert`
 mitmproxy shortcut :kbd:`o` then :kbd:`U`
-================== ======================
+================== =============================

docs/features/upstreamproxy.rst

@@ -7,6 +7,6 @@ In this mode, mitmproxy accepts proxy requests and unconditionally forwards all
 requests to a specified upstream proxy server. This is in contrast to :ref:`reverseproxy`,
 in which mitmproxy forwards ordinary HTTP requests to an upstream server.
 
-================== =============================
-command-line       ``-U http://hostname[:port]``
-================== =============================
+================== ===================================
+command-line       :option:`-U http://hostname[:port]`
+================== ===================================

docs/howmitmproxy.rst

@@ -6,17 +6,17 @@ process works will help you deploy it creatively, and take into account its
 fundamental assumptions and how to work around them. This document explains
 mitmproxy's proxy mechanism in detail, starting with the simplest unencrypted
 explicit proxying, and working up to the most complicated interaction -
-transparent proxying of TLS-protected traffic [#tls]_ in the presence of `Server
-Name Indication`_.
+transparent proxying of SSL-protected traffic [#ssl]_ in the presence of `Server Name Indication`_.
 
 Explicit HTTP
 -------------
 
-Configuring the client to use mitmproxy as an explicit proxy is the simplest and
-most reliable way to intercept traffic. The proxy protocol is codified in the
-`HTTP RFC`_, so the behaviour of both the client and the server is well defined,
-and usually reliable. In the simplest possible interaction with mitmproxy, a
-client connects directly to the proxy, and makes a request that looks like this:
+Configuring the client to use mitmproxy as an explicit proxy is the simplest
+and most reliable way to intercept traffic. The proxy protocol is codified in the
+`HTTP RFC`_, so the behaviour of both
+the client and the server is well defined, and usually reliable. In the
+simplest possible interaction with mitmproxy, a client connects directly to the
+proxy, and makes a request that looks like this:
 
 .. code-block:: none
 
@@ -43,11 +43,11 @@ client connects to the proxy and makes a request that looks like this:
 
     CONNECT example.com:443 HTTP/1.1
 
-A conventional proxy can neither view nor manipulate an TLS-encrypted data
+A conventional proxy can neither view nor manipulate an SSL-encrypted data
 stream, so a CONNECT request simply asks the proxy to open a pipe between the
 client and server. The proxy here is just a facilitator - it blindly forwards
 data in both directions without knowing anything about the contents. The
-negotiation of the TLS connection happens over this pipe, and the subsequent
+negotiation of the SSL connection happens over this pipe, and the subsequent
 flow of requests and responses are completely opaque to the proxy.
 
 The MITM in mitmproxy
@@ -58,17 +58,17 @@ name stands for Man-In-The-Middle - a reference to the process we use to
 intercept and interfere with these theoretically opaque data streams. The basic
 idea is to pretend to be the server to the client, and pretend to be the client
 to the server, while we sit in the middle decoding traffic from both sides. The
-tricky part is that the `Certificate Authority`_ system is designed to prevent
-exactly this attack, by allowing a trusted third-party to cryptographically sign
-a server's certificates to verify that they are legit. If this signature doesn't
-match or is from a non-trusted party, a secure client will simply drop the
-connection and refuse to proceed. Despite the many shortcomings of the CA system
-as it exists today, this is usually fatal to attempts to MITM an TLS connection
-for analysis. Our answer to this conundrum is to become a trusted Certificate
-Authority ourselves. Mitmproxy includes a full CA implementation that generates
-interception certificates on the fly. To get the client to trust these
-certificates, we :ref:`register mitmproxy as a trusted CA with the device
-manually <certinstall>`.
+tricky part is that the `Certificate Authority`_ system is
+designed to prevent exactly this attack, by allowing a trusted third-party to
+cryptographically sign a server's SSL certificates to verify that they are
+legit. If this signature doesn't match or is from a non-trusted party, a secure
+client will simply drop the connection and refuse to proceed. Despite the many
+shortcomings of the CA system as it exists today, this is usually fatal to
+attempts to MITM an SSL connection for analysis. Our answer to this conundrum
+is to become a trusted Certificate Authority ourselves. Mitmproxy includes a
+full CA implementation that generates interception certificates on the fly. To
+get the client to trust these certificates, we :ref:`register mitmproxy as a trusted
+CA with the device manually <certinstall>`.
 
 Complication 1: What's the remote hostname?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -89,12 +89,13 @@ information to initiate the pipe, even though it doesn't reveal the remote
 hostname.
 
 Mitmproxy has a cunning mechanism that smooths this over - :ref:`upstream
-certificate sniffing <upstreamcerts>`. As soon as we see the CONNECT request, we
-pause the client part of the conversation, and initiate a simultaneous
-connection to the server. We complete the TLS handshake with the server, and
-inspect the certificates it used. Now, we use the Common Name in the upstream
-certificates to generate the dummy certificate for the client. Voila, we have
-the correct hostname to present to the client, even if it was never specified.
+certificate sniffing <upstreamcerts>`. As soon as we
+see the CONNECT request, we pause the client part of the conversation, and
+initiate a simultaneous connection to the server. We complete the SSL handshake
+with the server, and inspect the certificates it used. Now, we use the Common
+Name in the upstream SSL certificates to generate the dummy certificate for the
+client. Voila, we have the correct hostname to present to the client, even if
+it was never specified.
 
 
 Complication 2: Subject Alternative Name
@@ -102,31 +103,31 @@ Complication 2: Subject Alternative Name
 
 Enter the next complication. Sometimes, the certificate Common Name is not, in
 fact, the hostname that the client is connecting to. This is because of the
-optional `Subject Alternative Name`_ field in the certificate that allows an
-arbitrary number of alternative domains to be specified. If the expected domain
-matches any of these, the client will proceed, even though the domain doesn't
-match the certificate CN. The answer here is simple: when we extract the CN from
-the upstream cert, we also extract the SANs, and add them to the generated dummy
-certificate.
+optional `Subject Alternative Name`_ field in the SSL certificate
+that allows an arbitrary number of alternative domains to be specified. If the
+expected domain matches any of these, the client will proceed, even though the
+domain doesn't match the certificate Common Name. The answer here is simple:
+when we extract the CN from the upstream cert, we also extract the SANs, and
+add them to the generated dummy certificate.
 
 
 Complication 3: Server Name Indication
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-One of the big limitations of vanilla TLS is that each certificate requires its
+One of the big limitations of vanilla SSL is that each certificate requires its
 own IP address. This means that you couldn't do virtual hosting where multiple
-domains with independent certificates share the same IP address. In a world with
-a rapidly shrinking IPv4 address pool this is a problem, and we have a solution
-in the form of the `Server Name Indication`_ extension to the TLS protocols.
-This lets the client specify the remote server name at the start of the TLS
-handshake, which then lets the server select the right certificate to complete
-the process.
+domains with independent certificates share the same IP address. In a world
+with a rapidly shrinking IPv4 address pool this is a problem, and we have a
+solution in the form of the `Server Name Indication`_ extension to
+the SSL and TLS protocols. This lets the client specify the remote server name
+at the start of the SSL handshake, which then lets the server select the right
+certificate to complete the process.
 
 SNI breaks our upstream certificate sniffing process, because when we connect
 without using SNI, we get served a default certificate that may have nothing to
 do with the certificate expected by the client. The solution is another tricky
 complication to the client connection process. After the client connects, we
-allow the TLS handshake to continue until just **after** the SNI value has been
+allow the SSL handshake to continue until just _after_ the SNI value has been
 passed to us. Now we can pause the conversation, and initiate an upstream
 connection using the correct SNI value, which then serves us the correct
 upstream certificate, from which we can extract the expected CN and SANs.
@@ -141,31 +142,32 @@ Lets put all of this together into the complete explicitly proxied HTTPS flow.
 
 1. The client makes a connection to mitmproxy, and issues an HTTP CONNECT request.
 2. Mitmproxy responds with a ``200 Connection Established``, as if it has set up the CONNECT pipe.
-3. The client believes it's talking to the remote server, and initiates the TLS connection.
+3. The client believes it's talking to the remote server, and initiates the SSL connection.
    It uses SNI to indicate the hostname it is connecting to.
-4. Mitmproxy connects to the server, and establishes an TLS connection using the SNI hostname
+4. Mitmproxy connects to the server, and establishes an SSL connection using the SNI hostname
    indicated by the client.
-5. The server responds with the matching certificate, which contains the CN and SAN values
+5. The server responds with the matching SSL certificate, which contains the CN and SAN values
    needed to generate the interception certificate.
 6. Mitmproxy generates the interception cert, and continues the
-   client TLS handshake paused in step 3.
-7. The client sends the request over the established TLS connection.
-8. Mitmproxy passes the request on to the server over the TLS connection initiated in step 4.
+   client SSL handshake paused in step 3.
+7. The client sends the request over the established SSL connection.
+8. Mitmproxy passes the request on to the server over the SSL connection initiated in step 4.
 
 Transparent HTTP
 ----------------
 
-When a transparent proxy is used, the connection is redirected into a proxy at
-the network layer, without any client configuration being required. This makes
-transparent proxying ideal for those situations where you can't change client
-behaviour - proxy-oblivious Android applications being a common example.
+When a transparent proxy is used, the HTTP/S connection is redirected into a
+proxy at the network layer, without any client configuration being required.
+This makes transparent proxying ideal for those situations where you can't
+change client behaviour - proxy-oblivious Android applications being a common
+example.
 
 To achieve this, we need to introduce two extra components. The first is a
 redirection mechanism that transparently reroutes a TCP connection destined for
 a server on the Internet to a listening proxy server. This usually takes the
-form of a firewall on the same host as the proxy server - `iptables`_ on Linux
-or pf_ on OSX. Once the client has initiated the connection, it makes a vanilla
-HTTP request, which might look something like this:
+form of a firewall on the same host as the proxy server - `iptables`_ on Linux or
+pf_ on OSX. Once the client has initiated the connection, it makes a vanilla HTTP request,
+which might look something like this:
 
 .. code-block:: none
 
@@ -173,35 +175,32 @@ HTTP request, which might look something like this:
 
 Note that this request differs from the explicit proxy variation, in that it
 omits the scheme and hostname. How, then, do we know which upstream host to
-forward the request to? The routing mechanism that has performed the redirection
-keeps track of the original destination for us.  Each routing mechanism has a
-different way of exposing this data, so this introduces the second component
-required for working transparent proxying: a host module that knows how to
-retrieve the original destination address from the router. In mitmproxy, this
-takes the form of a built-in set of modules_ that know how to talk to each
-platform's redirection mechanism. Once we have this information, the process is
-fairly straight-forward.
+forward the request to? The routing mechanism that has performed the
+redirection keeps track of the original destination for us.  Each routing
+mechanism has a different way of exposing this data, so this introduces the
+second component required for working transparent proxying: a host module that
+knows how to retrieve the original destination address from the router. In
+mitmproxy, this takes the form of a built-in set of
+modules_ that know how to talk to each platform's redirection mechanism.
+Once we have this information, the process is fairly straight-forward.
 
 .. image:: schematics/how-mitmproxy-works-transparent.png
     :align: center
 
 1. The client makes a connection to the server.
-2. The router redirects the connection to mitmproxy, which is typically
-   listening on a local port of the same host. Mitmproxy then consults the
-   routing mechanism to establish what the original destination was.
+2. The router redirects the connection to mitmproxy, which is typically listening on a local port
+   of the same host. Mitmproxy then consults the routing mechanism to establish what the original
+   destination was.
 3. Now, we simply read the client's request...
 4. ... and forward it upstream.
 
 Transparent HTTPS
 -----------------
 
-The first step is to determine whether we should treat an incoming connection as
-HTTPS. The mechanism for doing this is simple - we use the routing mechanism to
-find out what the original destination port is. All incoming connections pass
-through different layers which can determin the actual protocol to use.
-Automatic TLS detection works for SSLv3, TLS 1.0, TLS 1.1, and TLS 1.2 by
-looking for a *ClientHello* message at the beginning of each connection. This
-works independently of the used TCP port.
+The first step is to determine whether we should treat an incoming connection
+as HTTPS. The mechanism for doing this is simple - we use the routing mechanism
+to find out what the original destination port is. By default, we treat all
+traffic destined for ports 443 and 8443 as SSL.
 
 From here, the process is a merger of the methods we've described for
 transparently proxying HTTP, and explicitly proxying HTTPS. We use the routing
@@ -215,21 +214,21 @@ explicit HTTPS connections to establish the CN and SANs, and cope with SNI.
 2. The router redirects the connection to mitmproxy, which is typically listening on a local port
    of the same host. Mitmproxy then consults the routing mechanism to establish what the original
    destination was.
-3. The client believes it's talking to the remote server, and initiates the TLS connection.
+3. The client believes it's talking to the remote server, and initiates the SSL connection.
    It uses SNI to indicate the hostname it is connecting to.
-4. Mitmproxy connects to the server, and establishes an TLS connection using the SNI hostname
+4. Mitmproxy connects to the server, and establishes an SSL connection using the SNI hostname
    indicated by the client.
-5. The server responds with the matching certificate, which contains the CN and SAN values
+5. The server responds with the matching SSL certificate, which contains the CN and SAN values
    needed to generate the interception certificate.
-6. Mitmproxy generates the interception cert, and continues the client TLS handshake paused in
+6. Mitmproxy generates the interception cert, and continues the client SSL handshake paused in
    step 3.
-7. The client sends the request over the established TLS connection.
-8. Mitmproxy passes the request on to the server over the TLS connection initiated in step 4.
+7. The client sends the request over the established SSL connection.
+8. Mitmproxy passes the request on to the server over the SSL connection initiated in step 4.
 
 .. rubric:: Footnotes
 
-.. [#tls] The use of "TLS" refers to both SSL (outdated and insecure) and TLS
-          (1.0 and up) in the generic sense, unless otherwise specified.
+.. [#ssl] I use "SSL" to refer to both SSL and TLS in the generic sense, unless otherwise
+          specified.
 
 .. _Server Name Indication: https://en.wikipedia.org/wiki/Server_Name_Indication
 .. _HTTP RFC: https://tools.ietf.org/html/rfc7230

docs/index.rst

@@ -51,9 +51,8 @@
     :hidden:
     :caption: Scripting
 
-    scripting/overview
-    scripting/events
-    scripting/api
+    scripting/inlinescripts
+    scripting/mitmproxy
 
 
 .. toctree::
@@ -64,17 +63,6 @@
     tutorials/gamecenter
     tutorials/transparent-dhcp
 
-
-.. toctree::
-    :hidden:
-    :caption: Pathod & Pathoc
-
-    pathod/intro
-    pathod/language
-    pathod/library
-    pathod/test
-
-
 .. toctree::
     :hidden:
     :caption: Hacking
@@ -82,9 +70,14 @@
     dev/architecture
     dev/testing
     dev/sslkeylogfile
+    dev/protocols
+    dev/proxy
+    dev/exceptions
+    dev/models
 
 ..  Indices and tables
     ==================
 
     * :ref:`genindex`
     * :ref:`modindex`
+

docs/install.rst

@@ -11,10 +11,8 @@ Installation On Ubuntu
 Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
 This was tested on a fully patched installation of Ubuntu 14.04.
 
-.. code:: bash
-
-   sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev g++
-   sudo pip install mitmproxy  # or pip install --user mitmproxy
+>>> sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev
+>>> sudo pip install mitmproxy
 
 Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
@@ -29,31 +27,6 @@ get set up to contribute to the project, install the dependencies as you would f
 mitmproxy installation (see :ref:`install-ubuntu`).
 Then see the Hacking_ section of the README on GitHub.
 
-.. _install-fedora:
-
-Installation On Fedora
-----------------------
-
-Fedora comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Fedora 23.
-
-.. code:: bash
-
-   sudo dnf install -y python-pip python-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libpng-devel libjpeg-devel
-   sudo pip install mitmproxy  # or pip install --user mitmproxy
-
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
-
-
-.. _install-arch:
-
-Installation On Arch Linux
---------------------------
-
-mitmproxy has been added into the [community] repository. Use pacman to install it:
-
->>> sudo pacman -S mitmproxy
-
 
 
 Installation On Mac OS X
@@ -103,9 +76,7 @@ Installation On Windows
 
 First, install the latest version of Python 2.7 from the `Python website`_.
 If you already have an older version of Python 2.7 installed, make sure to install pip_
-(pip is included in Python 2.7.9+ by default). If pip aborts with an error, make sure you are using the current version of pip.
-
->>> python -m pip install --upgrade pip
+(pip is included in Python 2.7.9+ by default).
 
 Next, add Python and the Python Scripts directory to your **PATH** variable.
 You can do this easily by running the following in powershell:

docs/introduction.rst

@@ -1,7 +1,7 @@
 Introduction
 ============
 
-**mitmproxy** is an interactive man-in-the-middle proxy for HTTP and HTTPS
+**mitmproxy** is an interactive, SSL-capable man-in-the-middle proxy for HTTP
 with a console interface.
 
 **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP.
@@ -12,12 +12,13 @@ mitmproxy website: `mitmproxy.org <https://mitmproxy.org/>`_
 
 .. rubric:: Features
 
-- Intercept HTTP & HTTPS requests and responses and modify them on the fly
-- Save complete HTTP conversations for later replay and analysis
-- Replay the client-side of an HTTP conversations
-- Replay HTTP responses of a previously recorded server
-- Reverse proxy mode to forward traffic to a specified server
-- Transparent proxy mode on OSX and Linux
-- Make scripted changes to HTTP traffic using Python
-- SSL/TLS certificates for interception are generated on the fly
-- And much, much more...
+
+- Intercept HTTP requests and responses and modify them on the fly.
+- Save complete HTTP conversations for later replay and analysis.
+- Replay the client-side of an HTTP conversations.
+- Replay HTTP responses of a previously recorded server.
+- Reverse proxy mode to forward traffic to a specified server.
+- Transparent proxy mode on OSX and Linux.
+- Make scripted changes to HTTP traffic using Python.
+- SSL certificates for interception are generated on the fly.
+- And much, much more.

docs/mitmdump.rst

@@ -7,7 +7,7 @@ mitmdump
 
 **mitmdump** is the command-line companion to mitmproxy. It provides
 tcpdump-like functionality to let you view, record, and programmatically
-transform HTTP traffic. See the ``--help`` flag output for complete
+transform HTTP traffic. See the :option:`--help` flag output for complete
 documentation.
 
 
@@ -28,7 +28,7 @@ Filtering saved traffic
 
 >>> mitmdump -nr infile -w outfile "~m post"
 
-Start mitmdump without binding to the proxy port (``-n``), read all flows from
+Start mitmdump without binding to the proxy port (:option:`-n`), read all flows from
 infile, apply the specified filter expression (only match POSTs), and write to
 outfile.
 
@@ -38,8 +38,8 @@ Client replay
 
 >>> mitmdump -nc outfile
 
-Start mitmdump without binding to the proxy port (``-n``), then replay all
-requests from outfile (``-c filename``). Flags combine in the obvious way, so
+Start mitmdump without binding to the proxy port (:option:`-n`), then replay all
+requests from outfile (:option:`-c filename`). Flags combine in the obvious way, so
 you can replay requests from one file, and write the resulting flows to
 another:
 

docs/pathod/intro.rst

@@ -1,307 +0,0 @@
-.. _intro:
-
-Pathology 101
-=============
-
-
-pathod
-------
-
-Pathod is a pathological HTTP daemon designed to let you craft almost any
-conceivable HTTP response, including ones that creatively violate the
-standards. HTTP responses are specified using a :ref:`small, terse language
-<language>` which pathod shares with its evil twin :ref:`pathoc`. To start
-playing with pathod, fire up the daemon:
-
->>> pathod
-
-By default, the service listens on port 9999 of localhost, and the default
-crafting anchor point is the path **/p/**. Anything after this URL prefix is
-treated as a response specifier. So, hitting the following URL will generate an
-HTTP 200 response with 100 bytes of random data:
-
-    http://localhost:9999/p/200:b@100
-
-See the :ref:`language documentation <language>` to get (much) fancier. The
-pathod daemon also takes a range of configuration options. To view those, use
-the command-line help:
-
->>> pathod --help
-
-Mimicing a proxy
-^^^^^^^^^^^^^^^^
-
-Pathod automatically responds to both straight HTTP and proxy requests. For
-proxy requests, the upstream host is ignored, and the path portion of the URL
-is used to match anchors. This lets you test software that supports a proxy
-configuration by spoofing responses from upstream servers.
-
-By default, we treat all proxy CONNECT requests as HTTPS traffic, serving the
-response using either pathod's built-in certificates, or the cert/key pair
-specified by the user. You can over-ride this behaviour if you're testing a
-client that makes a non-SSL CONNECT request using the **-C** command-line
-option.
-
-Anchors
-^^^^^^^
-
-Anchors provide an alternative to specifying the response in the URL. Instead,
-you attach a response to a pre-configured anchor point, specified with a regex.
-When a URL matching the regex is requested, the specified response is served.
-
->>> pathod -a "/foo=200"
-
-Here, "/foo" is the regex specifying the anchor path, and the part after the "="
-is a response specifier.
-
-
-File Access
-^^^^^^^^^^^
-
-There are two operators in the :ref:`language <language>` that load contents
-from file - the **+** operator to load an entire request specification from
-file, and the **>** value specifier. In pathod, both of these operators are
-restricted to a directory specified at startup, or disabled if no directory is
-specified:
-
->>> pathod -d ~/staticdir"
-
-
-Internal Error Responses
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-Pathod uses the non-standard 800 response code to indicate internal errors, to
-distinguish them from crafted responses. For example, a request to:
-
-    http://localhost:9999/p/foo
-
-... will return an 800 response because "foo" is not a valid page specifier.
-
-
-
-
-
-.. _pathoc:
-
-
-pathoc
-------
-
-Pathoc is a perverse HTTP daemon designed to let you craft almost any
-conceivable HTTP request, including ones that creatively violate the standards.
-HTTP requests are specified using a :ref:`small, terse language <language>`,
-which pathod shares with its server-side twin pathod. To view pathoc's complete
-range of options, use the command-line help:
-
->>> pathoc --help
-
-
-Getting Started
-^^^^^^^^^^^^^^^
-
-The basic pattern for pathoc commands is as follows:
-
-    pathoc hostname request [request ...]
-
-That is, we specify the hostname to connect to, followed by one or more
-requests. Lets start with a simple example::
-
-    > pathoc google.com get:/
-    07-06-16 12:13:43: >> 'GET':/
-    << 302 Found: 261 bytes
-
-Here, we make a GET request to the path / on port 80 of google.com. Pathoc's
-output tells us that the server responded with a 302 redirection. We can tell
-pathoc to connect using SSL, in which case the default port is changed to 443
-(you can over-ride the default port with the **-p** command-line option)::
-
-    > pathoc -s www.google.com get:/
-    07-06-16 12:14:56: >> 'GET':/
-    << 302 Found: 262 bytes
-
-
-Multiple Requests
-^^^^^^^^^^^^^^^^^
-
-There are two ways to tell pathoc to issue multiple requests. The first is to specify
-them on the command-line, like so::
-
-    > pathoc google.com get:/ get:/
-    07-06-16 12:21:04: >> 'GET':/
-    << 302 Found: 261 bytes
-    07-06-16 12:21:04: >> 'GET':/
-    << 302 Found: 261 bytes
-
-In this case, pathoc issues the specified requests over the same TCP connection -
-so in the above example only one connection is made to google.com
-
-The other way to issue multiple requests is to use the **-n** flag::
-
-    > pathoc -n 2 google.com get:/
-    07-06-16 12:21:04: >> 'GET':/
-    << 302 Found: 261 bytes
-    07-06-16 12:21:04: >> 'GET':/
-    << 302 Found: 261 bytes
-
-The output is identical, but two separate TCP connections are made to the
-upstream server. These two specification styles can be combined::
-
-    pathoc -n 2 google.com get:/ get:/
-
-
-Here, two distinct TCP connections are made, with two requests issued over
-each.
-
-
-
-Basic Fuzzing
-^^^^^^^^^^^^^
-
-The combination of pathoc's powerful request specification language and a few
-of its command-line options makes for quite a powerful basic fuzzer. Here's an
-example::
-
-    pathoc -e -I 200 -t 2 -n 1000 localhost get:/:b@10:ir,@1
-
-The request specified here is a valid GET with a body consisting of 10 random bytes,
-but with 1 random byte inserted in a random place. This could be in the headers,
-in the initial request line, or in the body itself. There are a few things
-to note here:
-
-- Corrupting the request in this way will often make the server enter a state where
-  it's awaiting more input from the client. This is where the
-  **-t** option comes in, which sets a timeout that causes pathoc to
-  disconnect after two seconds.
-- The **-n** option tells pathoc to repeat the request 1000 times.
-- The **-I** option tells pathoc to ignore HTTP 200 response codes.
-  You can use this to fine-tune what pathoc considers to be an exceptional
-  condition, and therefore log-worthy.
-- The **-e** option tells pathoc to print an explanation of each logged
-  request, in the form of an expanded pathoc specification with all random
-  portions and automatic header additions resolved. This lets you precisely
-  replay a request that triggered an error.
-
-
-Interacting with Proxies
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-Pathoc has a reasonably sophisticated suite of features for interacting with
-proxies. The proxy request syntax very closely mirrors that of straight HTTP,
-which means that it is possible to make proxy-style requests using pathoc
-without any additional syntax, by simply specifying a full URL instead of a
-simple path:
-
->>> pathoc -p 8080 localhost "get:'http://google.com'"
-
-Another common use case is to use an HTTP CONNECT request to probe remote
-servers via a proxy. This is done with the **-c** command-line option, which
-allows you to specify a remote host and port pair:
-
->>> pathoc -c google.com:80 -p 8080 localhost get:/
-
-Note that pathoc does **not** negotiate SSL without being explictly instructed
-to do so. If you're making a CONNECT request to an SSL-protected resource, you
-must also pass the **-s** flag:
-
->>> pathoc -sc google.com:443 -p 8080 localhost get:/
-
-
-
-Embedded response specification
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-One interesting feature of the Request specification language is that you can
-embed a response specification in it, which is then added to the request path.
-Here's an example:
-
->>> pathoc localhost:9999 "get:/p/:s'401:ir,@1'"
-
-This crafts a request that connects to the pathod server, and which then crafts
-a response that generates a 401, with one random byte embedded at a random
-point. The response specification is parsed and expanded by pathoc, so you see
-syntax errors immediately. This really becomes handy when combined with the
-**-e** flag to show the expanded request::
-
-    07-06-16 12:32:01: >> 'GET':/p/:s'401:i35,\x27\\x1b\x27:h\x27Content-Length\x27=\x270\x27:h\x27Content-Length\x27=\x270\x27':h'Host'='localhost'
-    << 401 Unauthorized: 0 bytes
-
-Note that the embedded response has been resolved *before* being sent to
-the server, so that "ir,@1" (embed a random byte at a random location) has
-become "i15,\'o\'" (embed the character "o" at offset 15). You now have a
-pathoc request specification that is precisely reproducible, even with random
-components. This feature comes in terribly handy when testing a proxy, since
-you can now drive the server response completely from the client, and have a
-complete log of reproducible requests to analyze afterwards.
-
-
-Request Examples
-----------------
-
-.. list-table::
-    :widths: 50 50
-    :header-rows: 0
-
-    * - get:/
-      - Get path /
-
-    * - get:/:b@100
-      - 100 random bytes as the body
-
-    * - get:/:h"Etag"="&;drop table browsers;"
-      - Add a header
-
-    * - get:/:u"&;drop table browsers;"
-      - Add a User-Agent header
-
-    * - get:/:b@100:dr
-      - Drop the connection randomly
-
-    * - get:/:b@100,ascii:ir,@1
-      - 100 ASCII bytes as the body, and randomly inject a random byte
-
-    * - ws:/
-      - Initiate a websocket handshake.
-
-
-Response Examples
------------------
-
-.. list-table::
-    :widths: 50 50
-    :header-rows: 0
-
-
-    * - 200
-      - A basic HTTP 200 response.
-
-    * - 200:r
-      - A basic HTTP 200 response with no Content-Length header. This will hang.
-
-    * - 200:da
-      - Server-side disconnect after all content has been sent.
-
-    * - 200:b\@100
-      - 100 random bytes as the body. A Content-Length header is added, so the disconnect
-        is no longer needed.
-
-    * - 200:b\@100:h"Etag"="';drop table servers;"
-      - Add a Server header
-
-    * - 200:b\@100:dr
-      - Drop the connection randomly
-
-    * - 200:b\@100,ascii:ir,@1
-      - 100 ASCII bytes as the body, and randomly inject a random byte
-
-    * - 200:b\@1k:c"text/json"
-      - 1k of random bytes, with a text/json content type
-
-    * - 200:b\@1k:p50,120
-      - 1k of random bytes, pause for 120 seconds after 50 bytes
-
-    * - 200:b\@1k:pr,f
-      - 1k of random bytes, but hang forever at a random location
-
-    * - 200:b\@100:h\@1k,ascii_letters='foo'
-      - 100 ASCII bytes as the body, randomly generated 100k header name, with the value
-        'foo'.

docs/pathod/language.rst

@@ -1,257 +0,0 @@
-.. _language:
-
-language spec
-=============
-
-************
-HTTP Request
-************
-
-    **method:path:[colon-separated list of features]**
-
-.. list-table::
-    :widths: 20 80
-    :header-rows: 0
-
-    * - method
-      - A :ref:`VALUE` specifying the HTTP method to
-        use. Standard methods do not need to be enclosed in quotes, while
-        non-standard methods can be specified as quoted strings.
-
-        The special method **ws** creates a valid websocket upgrade
-        GET request, and signals to pathoc to switch to websocket recieve
-        mode if the server responds correctly. Apart from that, websocket
-        requests are just like any other, and all aspects of the request
-        can be over-ridden.
-    * - h\:\ :ref:`VALUE`\ =\ :ref:`VALUE`\
-      - Set a header.
-    * - r
-      - Set the **raw** flag on this response. Pathod will not calculate a
-        *Content-Length* header if a body is set.
-    * - c\ :ref:`VALUE`
-      - A shortcut for setting the Content-Type header. Equivalent to
-        ``h"Content-Type"=VALUE``
-    * - u\ :ref:`VALUE`
-        uSHORTCUT
-      - Set a User-Agent header on this request. You can specify either a
-        complete :ref:`VALUE`, or a User-Agent shortcut: **android**,
-        **blackberry**, **bingbot**, **chrome**, **firefox**, **googlebot**,
-        **ie9**, **ipad**, **iphone**, **safari**.
-    * - b\ :ref:`VALUE`
-      - Set the body. The appropriate Content-Length header is added
-        automatically unless the **r** flag is set.
-    * - s\ :ref:`VALUE`
-      - An embedded Response specification, appended to the path of the request.
-    * - x\ :ref:`INTEGER`
-      - Repeat this message N times.
-    * - d\ :ref:`OFFSET`
-      - Disconnect after OFFSET bytes (HTTP/1 only).
-    * - i\ :ref:`OFFSET`,\ :ref:`VALUE`
-      - Inject the specified value at the offset (HTTP/1 only)
-    * - p\ :ref:`OFFSET`,SECONDS
-      - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer
-        or "f" to pause forever (HTTP/1 only)
-
-
-*************
-HTTP Response
-*************
-
-    **code:[colon-separated list of features]**
-
-.. list-table::
-    :widths: 20 80
-    :header-rows: 0
-
-    * - code
-      - An integer specifying the HTTP response code.
-
-        The special method **ws** creates a valid websocket upgrade
-        response (code 101), and moves pathod to websocket mode. Apart
-        from that, websocket responses are just like any other, and all
-        aspects of the response can be over-ridden.
-    * - m\ :ref:`VALUE`
-      - HTTP Reason message. Automatically chosen according to the response
-        code if not specified. (HTTP/1 only)
-    * - h\:\ :ref:`VALUE`\ =\ :ref:`VALUE`\
-      - Set a header.
-    * - r
-      - Set the **raw** flag on this response. Pathod will not calculate a
-        *Content-Length* header if a body is set.
-    * - l\ :ref:`VALUE`
-      - A shortcut for setting the Location header. Equivalent to
-        ``h"Location"=VALUE``
-    * - c\ :ref:`VALUE`
-      - A shortcut for setting the Content-Type header. Equivalent to
-        ``h"Content-Type"=VALUE``
-    * - b\ :ref:`VALUE`
-      - Set the body. The appropriate Content-Length header is added
-        automatically unless the **r** flag is set.
-    * - d\ :ref:`OFFSET`
-      - Disconnect after OFFSET bytes (HTTP/1 only).
-    * - i\ :ref:`OFFSET`,\ :ref:`VALUE`
-      - Inject the specified value at the offset (HTTP/1 only)
-    * - p\ :ref:`OFFSET`,SECONDS
-      - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer
-        or "f" to pause forever (HTTP/1 only)
-
-***************
-Websocket Frame
-***************
-
-    **wf:[colon-separated list of features]**
-
-.. list-table::
-    :widths: 20 80
-    :header-rows: 0
-
-    * - b\ :ref:`VALUE`
-      - Set the frame payload. If a masking key is present, the value is
-        encoded automatically.
-    * - c\ :ref:`INTEGER`
-      - Set the op code. This can either be an integer from 0-15, or be one of
-        the following opcode names: **text** (the default), **continue**,
-        **binary**, **close**, **ping**, **pong**.
-    * - d\ :ref:`OFFSET`
-      - Disconnect after OFFSET bytes
-    * - i\ :ref:`OFFSET`,\ :ref:`VALUE`
-      - Inject the specified value at the offset
-    * - p\ :ref:`OFFSET`,SECONDS
-      - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer
-        or "f" to pause forever
-    * - x\ :ref:`INTEGER`
-      - Repeat this message N times.
-    * - [-]fin
-      - Set or un-set the **fin** bit.
-    * - k\ :ref:`VALUE`
-      - Set the masking key. The resulting value must be exactly 4 bytes long.
-        The special form **knone** specifies that no key should be set, even if
-        the mask bit is on.
-    * - l\ :ref:`INTEGER`
-      - Set the payload length in the frame header, regardless of the actual
-        body length.
-    * - [-]mask
-      - Set or un-set the <b>mask</b> bit.
-    * - r\ :ref:`VALUE`
-      - Set the raw frame payload. This disables masking, even if the key is present.
-    * - [-]rsv1
-      - Set or un-set the **rsv1** bit.
-    * - [-]rsv2
-      - Set or un-set the **rsv2** bit.
-    * - [-]rsv2
-      - Set or un-set the **rsv2** bit.
-
-
-
-**********
-Data types
-**********
-
-.. _INTEGER:
-
-INTEGER
-^^^^^^^
-
-.. _OFFSET:
-
-OFFSET
-^^^^^^
-
-Offsets are calculated relative to the base message, before any injections or
-other transforms are applied. They have 3 flavors:
-
-=======                 ==========================
-integer                 An integer byte offset
-**r**                   A random location
-**a**                   The end of the message
-=======                 ==========================
-
-
-.. _VALUE:
-
-VALUE
-^^^^^
-
-Literals
-""""""""
-
-Literal values are specified as a quoted strings::
-
-    "foo"
-
-Either single or double quotes are accepted, and quotes can be escaped with
-backslashes within the string::
-
-    'fo\'o'
-
-Literal values can contain Python-style backslash escape sequences::
-
-    'foo\r\nbar'
-
-
-
-Generated
-"""""""""
-
-An @-symbol lead-in specifies that generated data should be used. There are two
-components to a generator specification - a size, and a data type. By default
-pathod assumes a data type of "bytes".
-
-Here's a value specifier for generating 100 bytes::
-
-    @100
-
-You can use standard suffixes to indicate larger values. Here, for instance, is
-a specifier for generating 100 megabytes:
-
-    @100m
-
-Data is generated and served efficiently - if you really want to send a
-terabyte of data to a client, pathod can do it. The supported suffixes are:
-
-==========          ====================
-b                   1024**0 (bytes)
-k                   1024**1 (kilobytes)
-m                   1024**2 (megabytes)
-g                   1024**3 (gigabytes)
-t                   1024**4 (terabytes)
-==========          ====================
-
-Data types are separated from the size specification by a comma. This specification
-generates 100mb of ASCII::
-
-    @100m,ascii
-
-Supported data types are:
-
-=================          ==============================================
-ascii                      All ASCII characters
-ascii_letters              A-Za-z
-ascii_lowercase            a-z
-ascii_uppercase            A-Z
-bytes                      All 256 byte values
-digits                     0-9
-hexdigits                  0-f
-octdigits                  0-7
-punctuation                !"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ and space
-whitespace                 \\t \\n \\x0b \\x0c \\r and space
-=================          ==============================================
-
-
-
-Files
-"""""
-
-You can load a value from a specified file path. To do so, you have to specify a
-_staticdir_ option to pathod on the command-line, like so:
-
->>> pathod -d ~/myassets
-
-All paths are relative paths under this directory. File loads are indicated by
-starting the value specifier with the left angle bracket::
-
-    <my/path
-
-The path value can also be a quoted string, with the same syntax as literals::
-
-    <"my/path"

docs/pathod/library.rst

@@ -1,14 +0,0 @@
-.. _library:
-
-pathod library
-==============
-
-Behind the pathod and pathoc command-line tools lurks the **pathod** library, a
-powerful way to manipulate and serve HTTP requests and responses from code. The
-canonical documentation for the library is in the code, and can be accessed
-using pydoc.
-
-
-.. literalinclude:: ../../examples/pathod/libpathod_pathoc.py
-   :caption: examples/pathod/libpathod_pathoc.py
-   :language: python

docs/pathod/test.rst

@@ -1,35 +0,0 @@
-.. _test:
-
-pathod.test
-===========
-
-The **pathod.test** module is a light, flexible testing layer for HTTP clients.
-It works by firing up a Pathod instance in a separate thread, letting you use
-Pathod's full abilities to generate responses, and then query Pathod's internal
-logs to establish what happened. All the mechanics of startup, shutdown, finding
-free ports and so forth are taken care of for you.
-
-The canonical docs can be accessed using pydoc:
-
->>> pydoc pathod.test
-
-The remainder of this page demonstrates some common interaction patterns using
-<a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are
-also applicable with only minor modification to most commonly used Python testing
-engines.
-
-
-Context Manager
----------------
-
-.. literalinclude:: ../../examples/pathod/test_context.py
-   :caption: examples/pathod/test_context.py
-   :language: python
-
-
-One instance per test
----------------------
-
-.. literalinclude:: ../../examples/pathod/test_setup.py
-   :caption: examples/pathod/test_setup.py
-   :language: python

docs/scripting/api.rst

@@ -1,40 +0,0 @@
-.. _api:
-
-
-API
-===
-
-- Errors
-    - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_
-- HTTP
-    - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_
-    - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_
-    - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_
-- Logging
-    - `mitmproxy.controller.Log <#mitmproxy.controller.Log>`_
-    - `mitmproxy.controller.LogEntry <#mitmproxy.controller.LogEntry>`_
-
-
-Errors
-------
-
-.. autoclass:: mitmproxy.models.flow.Error
-    :inherited-members:
-
-HTTP
-----
-
-.. autoclass:: mitmproxy.models.http.HTTPRequest
-    :inherited-members:
-
-.. autoclass:: mitmproxy.models.http.HTTPResponse
-    :inherited-members:
-
-.. autoclass:: mitmproxy.models.http.HTTPFlow
-    :inherited-members:
-
-Logging
---------
-
-.. autoclass:: mitmproxy.controller.Log
-    :inherited-members:

docs/scripting/events.rst

@@ -1,202 +0,0 @@
-.. _events:
-
-Events
-=======
-
-General
--------
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: configure(options, updated)
-        - Called once on startup, and whenever options change.
-
-          *options*
-            An ``options.Options`` object with the total current configuration
-            state of mitmproxy.
-          *updated*
-            A set of strings indicating which configuration options have been
-            updated. This contains all options when *configure* is called on
-            startup, and only changed options subsequently.
-
-    *   - .. py:function:: done()
-        - Called once when the script shuts down, either because it's been
-          unloaded, or because the proxy itself is shutting down.
-
-    *   - .. py:function:: log(entry)
-        - Called whenever an event log is added.
-
-          *entry*
-            An ``controller.LogEntry`` object - ``entry.msg`` is the log text,
-            and ``entry.level`` is the urgency level ("debug", "info", "warn",
-            "error").
-
-    *   - .. py:function:: start()
-        - Called once on startup, before any other events. If you return a
-          value  from this event, it will replace the current addon. This
-          allows you to, "boot into" an addon implemented as a class instance
-          from the module level.
-
-    *   - .. py:function:: tick()
-        - Called at a regular sub-second interval as long as the addon is
-          executing.
-
-
-Connection
-----------
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: clientconnect(root_layer)
-        - Called when a client initiates a connection to the proxy. Note that a
-          connection can correspond to multiple HTTP requests.
-
-          *root_layer*
-            The root layer (see `mitmproxy.protocol` for an explanation what
-            the root layer is), provides transparent access to all attributes
-            of the :py:class:`~mitmproxy.proxy.RootContext`. For example,
-            ``root_layer.client_conn.address`` gives the remote address of the
-            connecting client.
-
-    *   - .. py:function:: clientdisconnect(root_layer)
-        - Called when a client disconnects from the proxy.
-
-          *root_layer*
-            The root layer object.
-
-    *   - .. py:function:: next_layer(layer)
-
-        - Called whenever layers are switched. You may change which layer will
-          be used by returning a new layer object from this event.
-
-          *layer*
-            The next layer, as determined by mitmpmroxy.
-
-    *   - .. py:function:: serverconnect(server_conn)
-        - Called before the proxy initiates a connection to the target server.
-          Note that a connection can correspond to multiple HTTP requests.
-
-          *server_conn*
-            A ``ServerConnection`` object. It is guaranteed to have a non-None
-            ``address`` attribute.
-
-    *   - .. py:function:: serverdisconnect(server_conn)
-        - Called when the proxy has closed the server connection.
-
-          *server_conn*
-            A ``ServerConnection`` object.
-
-
-HTTP Events
------------
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: request(flow)
-        - Called when a client request has been received.
-
-          *flow*
-            A ``models.HTTPFlow`` object. At this point, the flow is
-            guaranteed to have a non-None ``request`` attribute.
-
-    *   - .. py:function:: requestheaders(flow)
-        - Called when the headers of a client request have been received, but
-          before the request body is read.
-
-          *flow*
-            A ``models.HTTPFlow`` object. At this point, the flow is
-            guaranteed to have a non-None ``request`` attribute.
-
-    *   - .. py:function:: responseheaders(flow)
-
-        - Called when the headers of a server response have been received, but
-          before the response body is read.
-
-          *flow*
-            A ``models.HTTPFlow`` object. At this point, the flow is
-            guaranteed to have a non-none ``request`` and ``response``
-            attributes, however the response will have no content.
-
-    *   - .. py:function:: response(flow)
-
-        - Called when a server response has been received.
-
-          *flow*
-            A ``models.HTTPFlow`` object. At this point, the flow is
-            guaranteed to have a non-none ``request`` and ``response``
-            attributes. The raw response body will be in ``response.body``,
-            unless response streaming has been enabled.
-
-    *   - .. py:function:: error(flow)
-        - Called when a flow error has occurred, e.g. invalid server responses,
-          or interrupted connections. This is distinct from a valid server HTTP
-          error response, which is simply a response with an HTTP error code.
-
-          *flow*
-            The flow containing the error. It is guaranteed to have
-            non-None ``error`` attribute.
-
-
-WebSocket Events
------------------
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: websockets_handshake(flow)
-
-        - Called when a client wants to establish a WebSockets connection. The
-          WebSockets-specific headers can be manipulated to manipulate the
-          handshake. The ``flow`` object is guaranteed to have a non-None
-          ``request`` attribute.
-
-          *flow*
-            The flow containing the HTTP websocket handshake request. The
-            object is guaranteed to have a non-None ``request`` attribute.
-
-
-TCP Events
-----------
-
-These events are called only if the connection is in :ref:`TCP mode
-<tcpproxy>`. So, for instance, TCP events are not called for ordinary HTTP/S
-connections.
-
-.. list-table::
-    :widths: 40 60
-    :header-rows: 0
-
-    *   - .. py:function:: tcp_end(flow)
-        - Called when TCP streaming ends.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_error(flow)
-        - Called when a TCP error occurs - e.g. the connection closing
-          unexpectedly.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_message(flow)
-
-        - Called a TCP payload is received from the client or server. The
-          sender and receiver are identifiable. The most recent message will be
-          ``flow.messages[-1]``. The message is user-modifiable.
-
-          *flow*
-            A ``models.TCPFlow`` object.
-
-    *   - .. py:function:: tcp_start(flow)
-        - Called when TCP streaming starts.
-
-          *flow*
-            A ``models.TCPFlow`` object.

docs/scripting/inlinescripts.rst

@@ -0,0 +1,231 @@
+.. _inlinescripts:
+
+Inline Scripts
+==============
+
+**mitmproxy** has a powerful scripting API that allows you to modify flows
+on-the-fly or rewrite previously saved flows locally.
+
+The mitmproxy scripting API is event driven - a script is simply a Python
+module that exposes a set of event methods. Here's a complete mitmproxy script
+that adds a new header to every HTTP response before it is returned to the
+client:
+
+.. literalinclude:: ../../examples/add_header.py
+   :caption: examples/add_header.py
+   :language: python
+
+The first argument to each event method is an instance of
+:py:class:`~mitmproxy.script.ScriptContext` that lets the script interact with the global mitmproxy
+state. The **response** event also gets an instance of :py:class:`~mitmproxy.script.ScriptContext`,
+which we can use to manipulate the response itself.
+
+We can now run this script using mitmdump or mitmproxy as follows:
+
+>>> mitmdump -s add_header.py
+
+The new header will be added to all responses passing through the proxy.
+
+Examples
+--------
+
+mitmproxy comes with a variety of example inline scripts, which demonstrate many basic tasks.
+We encourage you to either browse them locally or on `GitHub`_.
+
+
+Events
+------
+
+The ``context`` argument passed to each event method is always a
+:py:class:`~mitmproxy.script.ScriptContext` instance. It is guaranteed to be the same object
+for the scripts lifetime and is not shared between multiple inline scripts. You can safely use it
+to store any form of state you require.
+
+Script Lifecycle Events
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. py:function:: start(context, argv)
+
+    Called once on startup, before any other events.
+
+    :param List[str] argv: The inline scripts' arguments.
+        For example, ``mitmproxy -s 'example.py --foo 42'`` sets argv to ``["--foo", "42"]``.
+
+.. py:function:: done(context)
+
+    Called once on script shutdown, after any other events.
+
+Connection Events
+^^^^^^^^^^^^^^^^^
+
+.. py:function:: clientconnect(context, root_layer)
+
+    Called when a client initiates a connection to the proxy. Note that
+    a connection can correspond to multiple HTTP requests.
+
+    .. versionchanged:: 0.14
+
+    :param Layer root_layer: The root layer (see :ref:`protocols` for an explanation what the root
+        layer is), which provides transparent access to all attributes of the
+        :py:class:`~mitmproxy.proxy.RootContext`. For example, ``root_layer.client_conn.address``
+        gives the remote address of the connecting client.
+
+.. py:function:: clientdisconnect(context, root_layer)
+
+    Called when a client disconnects from the proxy.
+
+    .. versionchanged:: 0.14
+
+    :param Layer root_layer: see :py:func:`clientconnect`
+
+.. py:function:: serverconnect(context, server_conn)
+
+    Called before the proxy initiates a connection to the target server. Note that
+    a connection can correspond to multiple HTTP requests.
+
+    :param ServerConnection server_conn: The server connection object. It is guaranteed to have a
+        non-None ``address`` attribute.
+
+.. py:function:: serverdisconnect(context, server_conn)
+
+    Called when the proxy has closed the server connection.
+
+    .. versionadded:: 0.14
+
+    :param ServerConnection server_conn: see :py:func:`serverconnect`
+
+HTTP Events
+^^^^^^^^^^^
+
+.. py:function:: request(context, flow)
+
+    Called when a client request has been received. The ``flow`` object is
+    guaranteed to have a non-None ``request`` attribute.
+
+    :param HTTPFlow flow: The flow containing the request which has been received.
+        The object is guaranteed to have a non-None ``request`` attribute.
+
+.. py:function:: responseheaders(context, flow)
+
+    Called when the headers of a server response have been received.
+    This will always be called before the response hook.
+
+    :param HTTPFlow flow: The flow containing the request and response.
+        The object is guaranteed to have non-None ``request`` and
+        ``response`` attributes. ``response.content`` will be ``None``,
+        as the response body has not been read yet.
+
+.. py:function:: response(context, flow)
+
+    Called when a server response has been received.
+
+    :param HTTPFlow flow: The flow containing the request and response.
+        The object is guaranteed to have non-None ``request`` and
+        ``response`` attributes. ``response.body`` will contain the raw response body,
+        unless response streaming has been enabled.
+
+.. py:function:: error(context, flow)
+
+    Called when a flow error has occurred, e.g. invalid server responses, or
+    interrupted connections. This is distinct from a valid server HTTP error
+    response, which is simply a response with an HTTP error code.
+
+    :param HTTPFlow flow: The flow containing the error.
+        It is guaranteed to have non-None ``error`` attribute.
+
+TCP Events
+^^^^^^^^^^
+
+.. py:function:: tcp_message(context, tcp_msg)
+
+    .. warning::  API is subject to change
+
+    If the proxy is in :ref:`TCP mode <tcpproxy>`, this event is called when it
+    receives a TCP payload from the client or server.
+
+    The sender and receiver are identifiable. The message is user-modifiable.
+
+    :param TcpMessage tcp_msg: see *examples/tcp_message.py*
+
+API
+---
+
+The canonical API documentation is the code, which you can browse here, locally or on `GitHub`_.
+*Use the Source, Luke!*
+
+The main classes you will deal with in writing mitmproxy scripts are:
+
+:py:class:`~mitmproxy.script.ScriptContext`
+    - A handle for interacting with mitmproxy's Flow Master from within scripts.
+:py:class:`~mitmproxy.models.ClientConnection`
+    - Describes a client connection.
+:py:class:`~mitmproxy.models.ServerConnection`
+    - Describes a server connection.
+:py:class:`~mitmproxy.models.HTTPFlow`
+    - A collection of objects representing a single HTTP transaction.
+:py:class:`~mitmproxy.models.HTTPRequest`
+    - An HTTP request.
+:py:class:`~mitmproxy.models.HTTPResponse`
+    - An HTTP response.
+:py:class:`~mitmproxy.models.Error`
+    - A communications error.
+:py:class:`netlib.http.Headers`
+    - A dictionary-like object for managing HTTP headers.
+:py:class:`netlib.certutils.SSLCert`
+    - Exposes information SSL certificates.
+:py:class:`mitmproxy.flow.FlowMaster`
+    - The "heart" of mitmproxy, usually subclassed as :py:class:`mitmproxy.dump.DumpMaster` or
+      :py:class:`mitmproxy.console.ConsoleMaster`.
+
+Script Context
+--------------
+
+.. autoclass:: mitmproxy.script.ScriptContext
+    :members:
+    :undoc-members:
+
+Running scripts in parallel
+---------------------------
+
+We have a single flow primitive, so when a script is blocking, other requests are not processed.
+While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the
+:py:obj:`mitmproxy.script.concurrent` decorator.
+**If your script does not block, you should avoid the overhead of the decorator.**
+
+.. literalinclude:: ../../examples/nonblocking.py
+   :caption: examples/nonblocking.py
+   :language: python
+
+Make scripts configurable with arguments
+----------------------------------------
+
+Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by
+surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``.
+The arguments are then exposed in the start event:
+
+.. literalinclude:: ../../examples/modify_response_body.py
+   :caption: examples/modify_response_body.py
+   :language: python
+
+Running scripts on saved flows
+------------------------------
+
+Sometimes, we want to run a script on :py:class:`~mitmproxy.models.Flow` objects that are already
+complete.  This happens when you start a script, and then load a saved set of flows from a file
+(see the "scripted data transformation" example `here <https://mitmproxy.org/doc/mitmdump.html>`_).
+It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut
+in mitmproxy.
+
+In this case, there are no client connections, and the events are run in the following order:
+**start**, **request**, **responseheaders**, **response**, **error**, **done**.
+If the flow doesn't have a **response** or **error** associated with it, the matching events will
+be skipped.
+
+Spaces in the script path
+-------------------------
+
+By default, spaces are interpreted as a separator between the inline script and its arguments
+(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of
+quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``.
+
+.. _GitHub: https://github.com/mitmproxy/mitmproxy

docs/scripting/mitmproxy.rst

@@ -0,0 +1,26 @@
+
+mitmproxy
+=========
+
+.. note::
+
+    We strongly encourage you to use :ref:`inlinescripts` rather than mitmproxy.
+        - Inline Scripts are equally powerful and provide an easier syntax.
+        - Most examples are written as inline scripts.
+        - Multiple inline scripts can be used together.
+        - Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI.
+
+
+All of mitmproxy's basic functionality is exposed through the **mitmproxy**
+library. The example below shows a simple implementation of the "sticky cookie"
+functionality included in the interactive mitmproxy program. Traffic is
+monitored for ``Cookie`` and ``Set-Cookie`` headers, and requests are rewritten
+to include a previously seen cookie if they don't already have one. In effect,
+this lets you log in to a site using your browser, and then make subsequent
+requests using a tool like curl, which will then seem to be part of the
+authenticated session.
+
+
+.. literalinclude:: ../../examples/stickycookies
+   :caption: examples/stickycookies
+   :language: python

docs/scripting/overview.rst

@@ -1,141 +0,0 @@
-.. _overview:
-
-Overview
-========
-
-Mitmproxy has a powerful scripting API that allows you to control almost any
-aspect of traffic being proxied. In fact, much of mitmproxy's own core
-functionality is implemented using the exact same API exposed to scripters (see
-:src:`mitmproxy/builtins`).
-
-
-A simple example
-----------------
-
-Scripting is event driven, with named handlers on the script object called at
-appropriate points of mitmproxy's operation. Here's a complete mitmproxy script
-that adds a new header to every HTTP response before it is returned to the
-client:
-
-.. literalinclude:: ../../examples/add_header.py
-   :caption: :src:`examples/add_header.py`
-   :language: python
-
-All events that deal with an HTTP request get an instance of `HTTPFlow
-<api.html#mitmproxy.models.http.HTTPFlow>`_, which we can use to manipulate the
-response itself. We can now run this script using mitmdump, and the new header
-will be added to all responses passing through the proxy:
-
->>> mitmdump -s add_header.py
-
-
-Using classes
--------------
-
-In the example above, the script object is the ``add_header`` module itself.
-That is, the handlers are declared at the global level of the script. This is
-great for quick hacks, but soon becomes limiting as scripts become more
-sophisticated.
-
-When a script first starts up, the `start <events.html#start>`_, event is
-called before anything else happens. You can replace the current script object
-by returning it from this handler. Here's how this looks when applied to the
-example above:
-
-.. literalinclude:: ../../examples/classes.py
-   :caption: :src:`examples/classes.py`
-   :language: python
-
-So here, we're using a module-level script to "boot up" into a class instance.
-From this point on, the module-level script is removed from the handler chain,
-and is replaced by the class instance.
-
-
-Handling arguments
-------------------
-
-Scripts can handle their own command-line arguments, just like any other Python
-program. Let's build on the example above to do something slightly more
-sophisticated - replace one value with another in all responses. Mitmproxy's
-`HTTPRequest <api.html#mitmproxy.models.http.HTTPRequest>`_ and `HTTPResponse
-<api.html#mitmproxy.models.http.HTTPResponse>`_ objects have a handy `replace
-<api.html#mitmproxy.models.http.HTTPResponse.replace>`_ method that takes care
-of all the details for us.
-
-.. literalinclude:: ../../examples/arguments.py
-   :caption: :src:`examples/arguments.py`
-   :language: python
-
-We can now call this script on the command-line like this:
-
->>> mitmdump -dd -s "./arguments.py html faketml"
-
-Whenever a handler is called, mitpmroxy rewrites the script environment so that
-it sees its own arguments as if it was invoked from the command-line.
-
-
-Logging and the context
------------------------
-
-Scripts should not output straight to stderr or stdout. Instead, the `log
-<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` contexzt module
-should be used, so that the mitmproxy host program can handle output
-appropriately. So, mitmdump can print colorised sript output to the terminal,
-and mitmproxy console can place script output in the event buffer.
-
-Here's how this looks:
-
-.. literalinclude:: ../../examples/logging.py
-   :caption: :src:`examples/logging.py`
-   :language: python
-
-The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
-for advanced usage.
-
-
-Running scripts on saved flows
-------------------------------
-
-When a flow is loaded from disk, the sequence of events that the flow would
-have gone through on the wire is partially replayed. So, for instance, an HTTP
-flow loaded from disk will trigger `requestheaders
-<events.html#requestheaders>`_,  `request <events.html#request>`_,
-`responseheaders <events.html#responseheaders>`_ and  `response
-<events.html#response>`_ in order. We can use this behaviour to transform saved
-traffic using scripts. For example, we can invoke the replacer script from
-above on saved traffic as follows:
-
->>> mitmdump -dd -s "./arguments.py html fakehtml" -r saved -w changed
-
-This command starts the ``arguments`` script, reads all the flows from
-``saved`` transforming them in the process, then writes them all to
-``changed``.
-
-The mitmproxy console tool provides interactive ways to run transforming
-scripts on flows - for instance, you can run a one-shot script on a single flow
-through the ``|`` (pipe) shortcut.
-
-
-Concurrency
------------
-
-The mitmproxy script mechanism is single threaded, and the proxy blocks while
-script handlers execute. This hugely simplifies the most common case, where
-handlers are light-weight and the blocking doesn't have a performance impact.
-It's possible to implement a concurrent mechanism on top of the blocking
-framework, and mitmproxy includes a handy example of this that is fit for most
-purposes. You can use it as follows:
-
-.. literalinclude:: ../../examples/nonblocking.py
-   :caption: :src:`examples/nonblocking.py`
-   :language: python
-
-
-Developing scripts
-------------------
-
-Mitmprxoy monitors scripts for modifications, and reloads them on change. When
-this happens, the script is shut down (the `done <events.html#done>`_  event is
-called), and the new instance is started up as if the script had just been
-loaded (the `start <events.html#start>`_ and `configure
-<events.html#configure>`_ events are called).

docs/transparent.rst

@@ -1,6 +1,5 @@
 .. _transparent:
 
-====================
 Transparent Proxying
 ====================
 
@@ -21,33 +20,5 @@ destination of the TCP connection.
 At the moment, mitmproxy supports transparent proxying on OSX Lion and above,
 and all current flavors of Linux.
 
-Fully transparent mode
-======================
-
-By default mitmproxy will use its own local ip address for its server-side connections.
-In case this isn't desired, the --spoof-source-address argument can be used to
-use the client's ip address for server-side connections. The following config is
-required for this mode to work:
-
-    CLIENT_NET=192.168.1.0/24
-    TABLE_ID=100
-    MARK=1
-
-    echo "$TABLE_ID     mitmproxy" >> /etc/iproute2/rt_tables
-    iptables -t mangle -A PREROUTING -d $CLIENT_NET -j MARK --set-mark $MARK
-    iptables -t nat -A PREROUTING -p tcp -s $CLIENT_NET --match multiport --dports 80,443 -j REDIRECT --to-port 8080
-
-    ip rule add fwmark $MARK lookup $TABLE_ID
-    ip route add local $CLIENT_NET dev lo table $TABLE_ID
-
-This mode does require root privileges though. There's a wrapper in the examples directory
-called 'mitmproxy_shim.c', which will enable you to use this mode with dropped priviliges.
-It can be used as follows:
-
-    gcc examples/mitmproxy_shim.c -o mitmproxy_shim -lcap
-    sudo chown root:root mitmproxy_shim
-    sudo chmod u+s mitmproxy_shim
-    ./mitmproxy_shim $(which mitmproxy) -T --spoof-source-address
-
 .. _iptables: http://www.netfilter.org/
 .. _pf: https://en.wikipedia.org/wiki/PF_\(firewall\)

docs/transparent/linux.rst

@@ -35,7 +35,7 @@ achieve transparent mode.
 
     >>> mitmproxy -T --host
 
-    The ``-T`` flag turns on transparent mode, and the ``--host``
+    The :option:`-T` flag turns on transparent mode, and the :option:`--host`
     argument tells mitmproxy to use the value of the Host header for URL display.
 
  6. Finally, configure your test device to use the host on which mitmproxy is

docs/transparent/osx.rst

@@ -50,7 +50,7 @@ Note that this means we don't support transparent mode for earlier versions of O
 
     >>> mitmproxy -T --host
 
-    The ``-T`` flag turns on transparent mode, and the ``--host``
+    The :option:`-T` flag turns on transparent mode, and the :option:`--host`
     argument tells mitmproxy to use the value of the Host header for URL display.
 
  8. Finally, configure your test device to use the host on which mitmproxy is

docs/tutorials/gamecenter.rst

@@ -51,7 +51,7 @@ The contents of the submission are particularly interesting:
             <key>context</key>
             <integer>0</integer>
             <key>score-value</key>
-            <integer>55</integer>
+            <integer>0</integer>
             <key>timestamp</key>
             <integer>1363515361321</integer>
           </dict>

examples/README

@@ -7,7 +7,6 @@ add_header.py             Simple script that just adds a header to every request
 change_upstream_proxy.py  Dynamically change the upstream proxy
 dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
 dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
-fail_with_500.py          Turn every response into an Internal Server Error.
 filt.py                   Use mitmproxy's filter expressions in your script.
 flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
 iframe_injector.py        Inject configurable iframe into pages.

examples/add_header.py

@@ -1,2 +1,2 @@
-def response(flow):
+def response(context, flow):
     flow.response.headers["newheader"] = "foo"

examples/arguments.py

@@ -1,17 +0,0 @@
-import argparse
-
-
-class Replacer:
-    def __init__(self, src, dst):
-        self.src, self.dst = src, dst
-
-    def response(self, flow):
-        flow.response.replace(self.src, self.dst)
-
-
-def start():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("src", type=str)
-    parser.add_argument("dst", type=str)
-    args = parser.parse_args()
-    return Replacer(args.src, args.dst)

examples/change_upstream_proxy.py

@@ -14,11 +14,11 @@ def proxy_address(flow):
         return ("localhost", 8081)
 
 
-def request(flow):
+def request(context, flow):
     if flow.request.method == "CONNECT":
         # If the decision is done by domain, one could also modify the server address here.
         # We do it after CONNECT here to have the request data available as well.
         return
     address = proxy_address(flow)
     if flow.live:
-        flow.live.change_upstream_proxy_server(address)
+        flow.live.change_upstream_proxy_server(address)

examples/classes.py

@@ -1,7 +0,0 @@
-class AddHeader:
-    def response(self, flow):
-        flow.response.headers["newheader"] = "foo"
-
-
-def start():
-    return AddHeader()

examples/custom_contentviews.py

@@ -1,8 +1,7 @@
 import string
 import lxml.html
 import lxml.etree
-from mitmproxy import contentviews
-from netlib import strutils
+from mitmproxy import utils, contentviews
 
 
 class ViewPigLatin(contentviews.View):
@@ -11,7 +10,7 @@ class ViewPigLatin(contentviews.View):
     content_types = ["text/html"]
 
     def __call__(self, data, **metadata):
-        if strutils.is_xml(data):
+        if utils.isXML(data):
             parser = lxml.etree.HTMLParser(
                 strip_cdata=True,
                 remove_blank_text=True
@@ -20,12 +19,11 @@ class ViewPigLatin(contentviews.View):
             docinfo = d.getroottree().docinfo
 
             def piglify(src):
-                words = src.split()
+                words = string.split(src)
                 ret = ''
                 for word in words:
                     idx = -1
-                    while word[idx] in string.punctuation and (idx * -1) != len(word):
-                        idx -= 1
+                    while word[idx] in string.punctuation and (idx * -1) != len(word): idx -= 1
                     if word[0].lower() in 'aeiou':
                         if idx == -1:
                             ret += word[0:] + "hay"
@@ -62,9 +60,9 @@ class ViewPigLatin(contentviews.View):
 pig_view = ViewPigLatin()
 
 
-def start():
-    contentviews.add(pig_view)
+def start(context, argv):
+    context.add_contentview(pig_view)
 
 
-def done():
-    contentviews.remove(pig_view)
+def done(context):
+    context.remove_contentview(pig_view)

examples/dns_spoofing.py

@@ -22,13 +22,14 @@ Usage:
 """
 import re
 
+
 # This regex extracts splits the host header into host and port.
 # Handles the edge case of IPv6 addresses containing colons.
 # https://bugzilla.mozilla.org/show_bug.cgi?id=45891
 parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
 
 
-def request(flow):
+def request(context, flow):
     if flow.client_conn.ssl_established:
         flow.request.scheme = "https"
         sni = flow.client_conn.connection.get_servername()
@@ -46,4 +47,4 @@ def request(flow):
             port = int(m.group("port"))
 
     flow.request.host = sni or host_header
-    flow.request.port = port
+    flow.request.port = port

examples/dup_and_replay.py

@@ -1,7 +1,4 @@
-from mitmproxy import master
-
-
-def request(flow):
-    f = master.duplicate_flow(flow)
+def request(context, flow):
+    f = context.duplicate_flow(flow)
     f.request.path = "/changed"
-    master.replay_request(f, block=True, run_scripthooks=False)
+    context.replay_request(f)

examples/fail_with_500.py

@@ -1,3 +0,0 @@
-def response(flow):
-    flow.response.status_code = 500
-    flow.response.content = b""

examples/filt.py

@@ -0,0 +1,16 @@
+# This scripts demonstrates how to use mitmproxy's filter pattern in inline scripts.
+# Usage: mitmdump -s "filt.py FILTER"
+
+from mitmproxy import filt
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError("Usage: -s 'filt.py FILTER'")
+    context.filter = filt.parse(argv[1])
+
+
+def response(context, flow):
+    if flow.match(context.filter):
+        print("Flow matches filter:")
+        print(flow)

examples/flowbasic

@@ -8,7 +8,7 @@
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
-from mitmproxy import flow, controller, options
+from mitmproxy import flow
 from mitmproxy.proxy import ProxyServer, ProxyConfig
 
 
@@ -19,25 +19,26 @@ class MyMaster(flow.FlowMaster):
         except KeyboardInterrupt:
             self.shutdown()
 
-    @controller.handler
-    def request(self, f):
-        print("request", f)
+    def handle_request(self, f):
+        f = flow.FlowMaster.handle_request(self, f)
+        if f:
+            f.reply()
+        return f
 
-    @controller.handler
-    def response(self, f):
-        print("response", f)
+    def handle_response(self, f):
+        f = flow.FlowMaster.handle_response(self, f)
+        if f:
+            f.reply()
+        print(f)
+        return f
 
-    @controller.handler
-    def error(self, f):
-        print("error", f)
 
-    @controller.handler
-    def log(self, l):
-        print("log", l.msg)
-
-opts = options.Options(cadir="~/.mitmproxy/")
-config = ProxyConfig(opts)
+config = ProxyConfig(
+    port=8080,
+    # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
+    cadir="~/.mitmproxy/"
+)
 state = flow.State()
 server = ProxyServer(config)
-m = MyMaster(opts, server, state)
+m = MyMaster(server, state)
 m.run()

examples/flowfilter.py

@@ -1,21 +0,0 @@
-# This scripts demonstrates how to use mitmproxy's filter pattern in scripts.
-# Usage: mitmdump -s "flowfilter.py FILTER"
-
-import sys
-from mitmproxy import flowfilter
-
-
-class Filter:
-    def __init__(self, spec):
-        self.filter = flowfilter.parse(spec)
-
-    def response(self, flow):
-        if flowfilter.match(flow, self.filter):
-            print("Flow matches filter:")
-            print(flow)
-
-
-def start():
-    if len(sys.argv) != 2:
-        raise ValueError("Usage: -s 'filt.py FILTER'")
-    return Filter(sys.argv[1])

examples/flowwriter.py

@@ -4,20 +4,17 @@ import sys
 from mitmproxy.flow import FlowWriter
 
 
-class Writer:
-    def __init__(self, path):
-        if path == "-":
-            f = sys.stdout
-        else:
-            f = open(path, "wb")
-        self.w = FlowWriter(f)
-
-    def response(self, flow):
-        if random.choice([True, False]):
-            self.w.add(flow)
-
-
-def start():
-    if len(sys.argv) != 2:
+def start(context, argv):
+    if len(argv) != 2:
         raise ValueError('Usage: -s "flowriter.py filename"')
-    return Writer(sys.argv[1])
+
+    if argv[1] == "-":
+        f = sys.stdout
+    else:
+        f = open(argv[1], "wb")
+    context.flow_writer = FlowWriter(f)
+
+
+def response(context, flow):
+    if random.choice([True, False]):
+        context.flow_writer.add(flow)

examples/full_transparency_shim.c

@@ -1,87 +0,0 @@
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <string.h>
-#include <sys/prctl.h>
-#include <sys/types.h>
-#include <sys/capability.h>
-#include <unistd.h>
-#include <errno.h>
-
-/* This setuid wrapper can be used to run mitmproxy in full transparency mode, as a normal user.
- * It will set the required capabilities (CAP_NET_RAW), drop privileges, and will then run argv[1]
- * with the same capabilities.
- *
- * It can be compiled as follows:
- * gcc examples/mitmproxy_shim.c -o mitmproxy_shim -lcap
-*/
-
-int set_caps(cap_t cap_struct, cap_value_t *cap_list, size_t bufsize) {
-	int cap_count = bufsize / sizeof(cap_list[0]);
-
-	if (cap_set_flag(cap_struct, CAP_PERMITTED, cap_count, cap_list, CAP_SET) ||
-            cap_set_flag(cap_struct, CAP_EFFECTIVE, cap_count, cap_list, CAP_SET) ||
-            cap_set_flag(cap_struct, CAP_INHERITABLE, cap_count, cap_list, CAP_SET)) {
-		if (cap_count < 2) {
-			fprintf(stderr, "Cannot manipulate capability data structure as user: %s.\n", strerror(errno));
-		} else {
-			fprintf(stderr, "Cannot manipulate capability data structure as root: %s.\n", strerror(errno));
-		}
-		return -1;
-	}
-
-	if (cap_count < 2) {
-		if (prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_RAISE, CAP_NET_RAW, 0, 0)) {
-			fprintf(stderr, "Failed to add CAP_NET_RAW to the ambient set: %s.\n", strerror(errno));
-			return -2;
-		}
-	}
-
-	if (cap_set_proc(cap_struct)) {
-		if (cap_count < 2) {
-			fprintf(stderr, "Cannot set capabilities as user: %s.\n", strerror(errno)); 
-		} else {
-			fprintf(stderr, "Cannot set capabilities as root: %s.\n", strerror(errno));
-		}
-		return -3;
-	}
-
-	if (cap_count > 1) {
-		if (prctl(PR_SET_KEEPCAPS, 1L)) {
-			fprintf(stderr, "Cannot keep capabilities after dropping privileges: %s.\n", strerror(errno));
-			return -4;
-		}
-		if (cap_clear(cap_struct)) {
-			fprintf(stderr, "Cannot clear capability data structure: %s.\n", strerror(errno));
-			return -5;
-		}
-	}
-}
-
-int main(int argc, char **argv, char **envp) {
-	cap_t cap_struct = cap_init();
-	cap_value_t root_caps[2] = { CAP_NET_RAW, CAP_SETUID };
-	cap_value_t user_caps[1] = { CAP_NET_RAW };
-	uid_t user = getuid();
-	int res;
-
-	if (setresuid(0, 0, 0)) {
-		fprintf(stderr, "Cannot switch to root: %s.\n", strerror(errno));
-		return 1;
-	}
-
-	if (res = set_caps(cap_struct, root_caps, sizeof(root_caps)))
-		return res;
-
-	if (setresuid(user, user, user)) {
-		fprintf(stderr, "Cannot drop root privileges: %s.\n", strerror(errno));
-		return 2;
-	}
-
-	if (res = set_caps(cap_struct, user_caps, sizeof(user_caps)))
-		return res;
-
-	if (execve(argv[1], argv + 1, envp)) {
-		fprintf(stderr, "Failed to execute %s: %s\n", argv[1], strerror(errno));
-		return 3;
-	}
-}

examples/har_dump.py

@@ -1,216 +0,0 @@
-"""
-This inline script can be used to dump flows as HAR files.
-"""
-
-
-import pprint
-import json
-import sys
-import base64
-import zlib
-
-from datetime import datetime
-import pytz
-
-import mitmproxy
-
-from netlib import version
-from netlib import strutils
-from netlib.http import cookies
-
-HAR = {}
-
-# A list of server seen till now is maintained so we can avoid
-# using 'connect' time for entries that use an existing connection.
-SERVERS_SEEN = set()
-
-
-def start():
-    """
-        Called once on script startup before any other events.
-    """
-    if len(sys.argv) != 2:
-        raise ValueError(
-            'Usage: -s "har_dump.py filename" '
-            '(- will output to stdout, filenames ending with .zhar '
-            'will result in compressed har)'
-        )
-
-    HAR.update({
-        "log": {
-            "version": "1.2",
-            "creator": {
-                "name": "mitmproxy har_dump",
-                "version": "0.1",
-                "comment": "mitmproxy version %s" % version.MITMPROXY
-            },
-            "entries": []
-        }
-    })
-
-
-def response(flow):
-    """
-       Called when a server response has been received.
-    """
-
-    # -1 indicates that these values do not apply to current request
-    ssl_time = -1
-    connect_time = -1
-
-    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:
-        connect_time = (flow.server_conn.timestamp_tcp_setup -
-                        flow.server_conn.timestamp_start)
-
-        if flow.server_conn.timestamp_ssl_setup is not None:
-            ssl_time = (flow.server_conn.timestamp_ssl_setup -
-                        flow.server_conn.timestamp_tcp_setup)
-
-        SERVERS_SEEN.add(flow.server_conn)
-
-    # Calculate raw timings from timestamps. DNS timings can not be calculated
-    # for lack of a way to measure it. The same goes for HAR blocked.
-    # mitmproxy will open a server connection as soon as it receives the host
-    # and port from the client connection. So, the time spent waiting is actually
-    # spent waiting between request.timestamp_end and response.timestamp_start
-    # thus it correlates to HAR wait instead.
-    timings_raw = {
-        'send': flow.request.timestamp_end - flow.request.timestamp_start,
-        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
-        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
-        'connect': connect_time,
-        'ssl': ssl_time,
-    }
-
-    # HAR timings are integers in ms, so we re-encode the raw timings to that format.
-    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])
-
-    # full_time is the sum of all timings.
-    # Timings set to -1 will be ignored as per spec.
-    full_time = sum(v for v in timings.values() if v > -1)
-
-    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))
-
-    # Response body size and encoding
-    response_body_size = len(flow.response.raw_content)
-    response_body_decoded_size = len(flow.response.content)
-    response_body_compression = response_body_decoded_size - response_body_size
-
-    entry = {
-        "startedDateTime": started_date_time,
-        "time": full_time,
-        "request": {
-            "method": flow.request.method,
-            "url": flow.request.url,
-            "httpVersion": flow.request.http_version,
-            "cookies": format_request_cookies(flow.request.cookies.fields),
-            "headers": name_value(flow.request.headers),
-            "queryString": name_value(flow.request.query or {}),
-            "headersSize": len(str(flow.request.headers)),
-            "bodySize": len(flow.request.content),
-        },
-        "response": {
-            "status": flow.response.status_code,
-            "statusText": flow.response.reason,
-            "httpVersion": flow.response.http_version,
-            "cookies": format_response_cookies(flow.response.cookies.fields),
-            "headers": name_value(flow.response.headers),
-            "content": {
-                "size": response_body_size,
-                "compression": response_body_compression,
-                "mimeType": flow.response.headers.get('Content-Type', '')
-            },
-            "redirectURL": flow.response.headers.get('Location', ''),
-            "headersSize": len(str(flow.response.headers)),
-            "bodySize": response_body_size,
-        },
-        "cache": {},
-        "timings": timings,
-    }
-
-    # Store binay data as base64
-    if strutils.is_mostly_bin(flow.response.content):
-        b64 = base64.b64encode(flow.response.content)
-        entry["response"]["content"]["text"] = b64.decode('ascii')
-        entry["response"]["content"]["encoding"] = "base64"
-    else:
-        entry["response"]["content"]["text"] = flow.response.text
-
-    if flow.request.method in ["POST", "PUT", "PATCH"]:
-        entry["request"]["postData"] = {
-            "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
-            "text": flow.request.content,
-            "params": name_value(flow.request.urlencoded_form)
-        }
-
-    if flow.server_conn:
-        entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
-
-    HAR["log"]["entries"].append(entry)
-
-
-def done():
-    """
-        Called once on script shutdown, after any other events.
-    """
-    dump_file = sys.argv[1]
-
-    if dump_file == '-':
-        mitmproxy.ctx.log(pprint.pformat(HAR))
-    else:
-        json_dump = json.dumps(HAR, indent=2)
-
-        if dump_file.endswith('.zhar'):
-            json_dump = zlib.compress(json_dump, 9)
-
-        with open(dump_file, "w") as f:
-            f.write(json_dump)
-
-        mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
-
-
-def format_datetime(dt):
-    return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat()
-
-
-def format_cookies(cookie_list):
-    rv = []
-
-    for name, value, attrs in cookie_list:
-        cookie_har = {
-            "name": name,
-            "value": value,
-        }
-
-        # HAR only needs some attributes
-        for key in ["path", "domain", "comment"]:
-            if key in attrs:
-                cookie_har[key] = attrs[key]
-
-        # These keys need to be boolean!
-        for key in ["httpOnly", "secure"]:
-            cookie_har[key] = bool(key in attrs)
-
-        # Expiration time needs to be formatted
-        expire_ts = cookies.get_expiration_ts(attrs)
-        if expire_ts is not None:
-            cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts))
-
-        rv.append(cookie_har)
-
-    return rv
-
-
-def format_request_cookies(fields):
-    return format_cookies(cookies.group_cookies(fields))
-
-
-def format_response_cookies(fields):
-    return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)
-
-
-def name_value(obj):
-    """
-        Convert (key, value) pairs to HAR format.
-    """
-    return [{"name": k, "value": v} for k, v in obj.items()]

examples/har_extractor.py

@@ -0,0 +1,243 @@
+"""
+    This inline script utilizes harparser.HAR from
+    https://github.com/JustusW/harparser to generate a HAR log object.
+"""
+import six
+from harparser import HAR
+
+from datetime import datetime
+
+
+class _HARLog(HAR.log):
+    # The attributes need to be registered here for them to actually be
+    # available later via self. This is due to HAREncodable linking __getattr__
+    # to __getitem__. Anything that is set only in __init__ will just be added
+    # as key/value pair to self.__classes__.
+    __page_list__ = []
+    __page_count__ = 0
+    __page_ref__ = {}
+
+    def __init__(self, page_list=[]):
+        self.__page_list__ = page_list
+        self.__page_count__ = 0
+        self.__page_ref__ = {}
+
+        HAR.log.__init__(self, {"version": "1.2",
+                                "creator": {"name": "MITMPROXY HARExtractor",
+                                            "version": "0.1",
+                                            "comment": ""},
+                                "pages": [],
+                                "entries": []})
+
+    def reset(self):
+        self.__init__(self.__page_list__)
+
+    def add(self, obj):
+        if isinstance(obj, HAR.pages):
+            self['pages'].append(obj)
+        if isinstance(obj, HAR.entries):
+            self['entries'].append(obj)
+
+    def create_page_id(self):
+        self.__page_count__ += 1
+        return "autopage_%s" % str(self.__page_count__)
+
+    def set_page_ref(self, page, ref):
+        self.__page_ref__[page] = ref
+
+    def get_page_ref(self, page):
+        return self.__page_ref__.get(page, None)
+
+    def get_page_list(self):
+        return self.__page_list__
+
+
+def start(context, argv):
+    """
+        On start we create a HARLog instance. You will have to adapt this to
+        suit your actual needs of HAR generation. As it will probably be
+        necessary to cluster logs by IPs or reset them from time to time.
+    """
+    context.dump_file = None
+    if len(argv) > 1:
+        context.dump_file = argv[1]
+    else:
+        raise ValueError(
+            'Usage: -s "har_extractor.py filename" '
+            '(- will output to stdout, filenames ending with .zhar '
+            'will result in compressed har)'
+        )
+    context.HARLog = _HARLog()
+    context.seen_server = set()
+
+
+def response(context, flow):
+    """
+       Called when a server response has been received. At the time of this
+       message both a request and a response are present and completely done.
+    """
+    # Values are converted from float seconds to int milliseconds later.
+    ssl_time = -.001
+    connect_time = -.001
+    if flow.server_conn not in context.seen_server:
+        # Calculate the connect_time for this server_conn. Afterwards add it to
+        # seen list, in order to avoid the connect_time being present in entries
+        # that use an existing connection.
+        connect_time = (flow.server_conn.timestamp_tcp_setup -
+                        flow.server_conn.timestamp_start)
+        context.seen_server.add(flow.server_conn)
+
+        if flow.server_conn.timestamp_ssl_setup is not None:
+            # Get the ssl_time for this server_conn as the difference between
+            # the start of the successful tcp setup and the successful ssl
+            # setup. If no ssl setup has been made it is left as -1 since it
+            # doesn't apply to this connection.
+            ssl_time = (flow.server_conn.timestamp_ssl_setup -
+                        flow.server_conn.timestamp_tcp_setup)
+
+    # Calculate the raw timings from the different timestamps present in the
+    # request and response object. For lack of a way to measure it dns timings
+    # can not be calculated. The same goes for HAR blocked: MITMProxy will open
+    # a server connection as soon as it receives the host and port from the
+    # client connection. So the time spent waiting is actually spent waiting
+    # between request.timestamp_end and response.timestamp_start thus it
+    # correlates to HAR wait instead.
+    timings_raw = {
+        'send': flow.request.timestamp_end - flow.request.timestamp_start,
+        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
+        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
+        'connect': connect_time,
+        'ssl': ssl_time
+    }
+
+    # HAR timings are integers in ms, so we have to re-encode the raw timings to
+    # that format.
+    timings = dict([(k, int(1000 * v)) for k, v in six.iteritems(timings_raw)])
+
+    # The full_time is the sum of all timings.
+    # Timings set to -1 will be ignored as per spec.
+    full_time = sum(v for v in timings.values() if v > -1)
+
+    started_date_time = datetime.utcfromtimestamp(
+        flow.request.timestamp_start).isoformat()
+
+    request_query_string = [{"name": k, "value": v}
+                            for k, v in flow.request.query or {}]
+
+    response_body_size = len(flow.response.content)
+    response_body_decoded_size = len(flow.response.get_decoded_content())
+    response_body_compression = response_body_decoded_size - response_body_size
+
+    entry = HAR.entries({
+        "startedDateTime": started_date_time,
+        "time": full_time,
+        "request": {
+            "method": flow.request.method,
+            "url": flow.request.url,
+            "httpVersion": flow.request.http_version,
+            "cookies": format_cookies(flow.request.cookies),
+            "headers": format_headers(flow.request.headers),
+            "queryString": request_query_string,
+            "headersSize": len(str(flow.request.headers)),
+            "bodySize": len(flow.request.content),
+        },
+        "response": {
+            "status": flow.response.status_code,
+            "statusText": flow.response.reason,
+            "httpVersion": flow.response.http_version,
+            "cookies": format_cookies(flow.response.cookies),
+            "headers": format_headers(flow.response.headers),
+            "content": {
+                "size": response_body_size,
+                "compression": response_body_compression,
+                "mimeType": flow.response.headers.get('Content-Type', '')
+            },
+            "redirectURL": flow.response.headers.get('Location', ''),
+            "headersSize": len(str(flow.response.headers)),
+            "bodySize": response_body_size,
+        },
+        "cache": {},
+        "timings": timings,
+    })
+
+    # If the current url is in the page list of context.HARLog or
+    # does not have a referrer, we add it as a new pages object.
+    if (flow.request.url in context.HARLog.get_page_list() or
+            flow.request.headers.get('Referer') is None):
+        page_id = context.HARLog.create_page_id()
+        context.HARLog.add(
+            HAR.pages({
+                "startedDateTime": entry['startedDateTime'],
+                "id": page_id,
+                "title": flow.request.url,
+            })
+        )
+        context.HARLog.set_page_ref(flow.request.url, page_id)
+        entry['pageref'] = page_id
+
+    # Lookup the referer in the page_ref of context.HARLog to point this entries
+    # pageref attribute to the right pages object, then set it as a new
+    # reference to build a reference tree.
+    elif context.HARLog.get_page_ref(flow.request.headers.get('Referer')) is not None:
+        entry['pageref'] = context.HARLog.get_page_ref(
+            flow.request.headers['Referer']
+        )
+        context.HARLog.set_page_ref(
+            flow.request.headers['Referer'], entry['pageref']
+        )
+
+    context.HARLog.add(entry)
+
+
+def done(context):
+    """
+        Called once on script shutdown, after any other events.
+    """
+    import pprint
+    import json
+
+    json_dump = context.HARLog.json()
+    compressed_json_dump = context.HARLog.compress()
+
+    if context.dump_file == '-':
+        context.log(pprint.pformat(json.loads(json_dump)))
+    elif context.dump_file.endswith('.zhar'):
+        file(context.dump_file, "w").write(compressed_json_dump)
+    else:
+        file(context.dump_file, "w").write(json_dump)
+    context.log(
+        "HAR log finished with %s bytes (%s bytes compressed)" % (
+            len(json_dump), len(compressed_json_dump)
+        )
+    )
+    context.log(
+        "Compression rate is %s%%" % str(
+            100. * len(compressed_json_dump) / len(json_dump)
+        )
+    )
+
+
+def format_cookies(obj):
+    if obj:
+        return [{"name": k.strip(), "value": v[0]} for k, v in obj.items()]
+    return ""
+
+
+def format_headers(obj):
+    if obj:
+        return [{"name": k, "value": v} for k, v in obj.fields]
+    return ""
+
+
+def print_attributes(obj, filter_string=None, hide_privates=False):
+    """
+        Useful helper method to quickly get all attributes of an object and its
+        values.
+    """
+    for attr in dir(obj):
+        if hide_privates and "__" in attr:
+            continue
+        if filter_string is not None and filter_string not in attr:
+            continue
+        value = getattr(obj, attr)
+        print("%s.%s" % ('obj', attr), value, type(value))

examples/iframe_injector.py

@@ -1,29 +1,27 @@
 # Usage: mitmdump -s "iframe_injector.py url"
 # (this script works best with --anticache)
-import sys
 from bs4 import BeautifulSoup
+from mitmproxy.models import decoded
 
 
-class Injector:
-    def __init__(self, iframe_url):
-        self.iframe_url = iframe_url
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "iframe_injector.py url"')
+    context.iframe_url = argv[1]
 
-    def response(self, flow):
-        if flow.request.host in self.iframe_url:
-            return
+
+def response(context, flow):
+    if flow.request.host in context.iframe_url:
+        return
+    with decoded(flow.response):  # Remove content encoding (gzip, ...)
         html = BeautifulSoup(flow.response.content, "lxml")
         if html.body:
             iframe = html.new_tag(
                 "iframe",
-                src=self.iframe_url,
+                src=context.iframe_url,
                 frameborder=0,
                 height=0,
                 width=0)
             html.body.insert(0, iframe)
-            flow.response.content = str(html).encode("utf8")
-
-
-def start():
-    if len(sys.argv) != 2:
-        raise ValueError('Usage: -s "iframe_injector.py url"')
-    return Injector(sys.argv[1])
+            flow.response.content = str(html)
+            context.log("Iframe inserted.")

examples/logging.py

@@ -1,6 +0,0 @@
-from mitmproxy import ctx
-
-
-def start():
-    ctx.log.info("This is some informative text.")
-    ctx.log.error("This is an error.")

examples/mitmproxywrapper.py

@@ -16,6 +16,7 @@ import sys
 
 
 class Wrapper(object):
+
     def __init__(self, port, extra_arguments=None):
         self.port = port
         self.extra_arguments = extra_arguments
@@ -141,7 +142,7 @@ class Wrapper(object):
             '--toggle',
             action='store_true',
             help='just toggle the proxy configuration')
-        # parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
+#         parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
         parser.add_argument(
             '-p',
             '--port',
@@ -154,8 +155,8 @@ class Wrapper(object):
 
         if args.toggle:
             wrapper.toggle_proxy()
-        # elif args.honeyproxy:
-        #     wrapper.wrap_honeyproxy()
+#         elif args.honeyproxy:
+#             wrapper.wrap_honeyproxy()
         else:
             wrapper.wrap_mitmproxy()
 

examples/modify_form.py

@@ -1,8 +1,5 @@
-def request(flow):
-    if flow.request.urlencoded_form:
-        flow.request.urlencoded_form["mitmproxy"] = "rocks"
-    else:
-        # This sets the proper content type and overrides the body.
-        flow.request.urlencoded_form = [
-            ("foo", "bar")
-        ]
+def request(context, flow):
+    form = flow.request.urlencoded_form
+    if form is not None:
+        form["mitmproxy"] = ["rocks"]
+        flow.request.urlencoded_form = form

examples/modify_querystring.py

@@ -1,2 +1,5 @@
-def request(flow):
-    flow.request.query["mitmproxy"] = "rocks"
+def request(context, flow):
+    q = flow.request.query
+    if q:
+        q["mitmproxy"] = ["rocks"]
+        flow.request.query = q

examples/modify_response_body.py

@@ -0,0 +1,18 @@
+# Usage: mitmdump -s "modify_response_body.py mitmproxy bananas"
+# (this script works best with --anticache)
+from mitmproxy.models import decoded
+
+
+def start(context, argv):
+    if len(argv) != 3:
+        raise ValueError('Usage: -s "modify_response_body.py old new"')
+    # You may want to use Python's argparse for more sophisticated argument
+    # parsing.
+    context.old, context.new = argv[1], argv[2]
+
+
+def response(context, flow):
+    with decoded(flow.response):  # automatically decode gzipped responses.
+        flow.response.content = flow.response.content.replace(
+            context.old,
+            context.new)

examples/nonblocking.py

@@ -1,10 +1,9 @@
 import time
-import mitmproxy
 from mitmproxy.script import concurrent
 
 
 @concurrent  # Remove this and see what happens
-def request(flow):
-    mitmproxy.ctx.log("handle request: %s%s" % (flow.request.host, flow.request.path))
+def request(context, flow):
+    print("handle request: %s%s" % (flow.request.host, flow.request.path))
     time.sleep(5)
-    mitmproxy.ctx.log("start  request: %s%s" % (flow.request.host, flow.request.path))
+    print("start  request: %s%s" % (flow.request.host, flow.request.path))

examples/pathod/libpathod_pathoc.py

@@ -3,5 +3,5 @@ from pathod import pathoc
 
 p = pathoc.Pathoc(("google.com", 80))
 p.connect()
-print(p.request("get:/"))
-print(p.request("get:/foo"))
+print p.request("get:/")
+print p.request("get:/foo")

examples/pathod/test_setup.py

@@ -3,6 +3,7 @@ from pathod import test
 
 
 class Test:
+
     """
         Testing the requests module with
         a pathod instance started for

examples/pathod/test_setupall.py

@@ -3,12 +3,12 @@ from pathod import test
 
 
 class Test:
+
     """
         Testing the requests module with
         a single pathod instance started
         for the test suite.
     """
-
     @classmethod
     def setup_class(cls):
         cls.d = test.Daemon()

examples/proxapp.py

@@ -4,7 +4,6 @@ instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
 a single simplest-possible page.
 """
 from flask import Flask
-import mitmproxy
 
 app = Flask("proxapp")
 
@@ -16,10 +15,10 @@ def hello_world():
 
 # Register the app using the magic domain "proxapp" on port 80. Requests to
 # this domain and port combination will now be routed to the WSGI app instance.
-def start():
-    mitmproxy.ctx.master.apps.add(app, "proxapp", 80)
+def start(context, argv):
+    context.app_registry.add(app, "proxapp", 80)
 
     # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
     # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
     # but won't send any data.
-    mitmproxy.ctx.master.apps.add(app, "example.com", 443)
+    context.app_registry.add(app, "example.com", 443)

examples/read_dumpfile

@@ -4,7 +4,6 @@
 #
 
 from mitmproxy import flow
-from mitmproxy.exceptions import FlowReadException
 import pprint
 import sys
 
@@ -17,5 +16,5 @@ with open(sys.argv[1], "rb") as logfile:
             print(f.request.host)
             pp.pprint(f.get_state())
             print("")
-    except FlowReadException as e:
-        print("Flow file corrupted: {}".format(e))
+    except flow.FlowReadError as v:
+        print "Flow file corrupted. Stopped loading."

examples/redirect_requests.py

@@ -2,17 +2,20 @@
 This example shows two ways to redirect flows to other destinations.
 """
 from mitmproxy.models import HTTPResponse
+from netlib.http import Headers
 
-
-def request(flow):
+def request(context, flow):
     # pretty_host takes the "Host" header of the request into account,
     # which is useful in transparent mode where we usually only have the IP
     # otherwise.
 
     # Method 1: Answer with a locally generated response
     if flow.request.pretty_host.endswith("example.com"):
-        resp = HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"})
-        flow.reply.send(resp)
+        resp = HTTPResponse(
+            "HTTP/1.1", 200, "OK",
+            Headers(Content_Type="text/html"),
+            "helloworld")
+        flow.reply(resp)
 
     # Method 2: Redirect the request to a different server
     if flow.request.pretty_host.endswith("example.org"):

examples/remote_debug.py

@@ -1,19 +0,0 @@
-"""
-This script enables remote debugging of the mitmproxy *UI* with PyCharm.
-For general debugging purposes, it is easier to just debug mitmdump within PyCharm.
-
-Usage:
-    - pip install pydevd on the mitmproxy machine
-    - Open the Run/Debug Configuration dialog box in PyCharm, and select the Python Remote Debug configuration type.
-    - Debugging works in the way that mitmproxy connects to the debug server on startup.
-      Specify host and port that mitmproxy can use to reach your PyCharm instance on startup.
-    - Adjust this inline script accordingly.
-    - Start debug server in PyCharm
-    - Set breakpoints
-    - Start mitmproxy -s remote_debug.py
-"""
-
-
-def start():
-    import pydevd
-    pydevd.settrace("localhost", port=5678, stdoutToServer=True, stderrToServer=True)

examples/sslstrip.py

@@ -1,48 +1,40 @@
+from netlib.http import decoded
 import re
 from six.moves import urllib
 
-# set of SSL/TLS capable hosts
-secure_hosts = set()
+def start(context, argv) :
 
+    #set of SSL/TLS capable hosts
+    context.secure_hosts = set()
+
+def request(context, flow) :
 
-def request(flow):
     flow.request.headers.pop('If-Modified-Since', None)
     flow.request.headers.pop('Cache-Control', None)
 
-    # do not force https redirection
-    flow.request.headers.pop('Upgrade-Insecure-Requests', None)
-
-    # proxy connections to SSL-enabled hosts
-    if flow.request.pretty_host in secure_hosts:
+    #proxy connections to SSL-enabled hosts
+    if flow.request.pretty_host in context.secure_hosts :
         flow.request.scheme = 'https'
         flow.request.port = 443
 
+def response(context, flow) :
 
-def response(flow):
-    flow.response.headers.pop('Strict-Transport-Security', None)
-    flow.response.headers.pop('Public-Key-Pins', None)
+    with decoded(flow.response) :
+        flow.request.headers.pop('Strict-Transport-Security', None)
+        flow.request.headers.pop('Public-Key-Pins', None)
 
-    # strip links in response body
-    flow.response.content = flow.response.content.replace('https://', 'http://')
+        #strip links in response body
+        flow.response.content = flow.response.content.replace('https://', 'http://')
 
-    # strip meta tag upgrade-insecure-requests in response body
-    csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>'
-    flow.response.content = re.sub(csp_meta_tag_pattern, b'', flow.response.content, flags=re.IGNORECASE)
+        #strip links in 'Location' header
+        if flow.response.headers.get('Location','').startswith('https://'):
+            location = flow.response.headers['Location']
+            hostname = urllib.parse.urlparse(location).hostname
+            if hostname:
+                context.secure_hosts.add(hostname)
+            flow.response.headers['Location'] = location.replace('https://', 'http://', 1)
 
-    # strip links in 'Location' header
-    if flow.response.headers.get('Location', '').startswith('https://'):
-        location = flow.response.headers['Location']
-        hostname = urllib.parse.urlparse(location).hostname
-        if hostname:
-            secure_hosts.add(hostname)
-        flow.response.headers['Location'] = location.replace('https://', 'http://', 1)
-
-    # strip upgrade-insecure-requests in Content-Security-Policy header
-    if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE):
-        csp = flow.response.headers['Content-Security-Policy']
-        flow.response.headers['Content-Security-Policy'] = re.sub('upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE)
-
-    # strip secure flag from 'Set-Cookie' headers
-    cookies = flow.response.headers.get_all('Set-Cookie')
-    cookies = [re.sub(r';\s*secure\s*', '', s) for s in cookies]
-    flow.response.headers.set_all('Set-Cookie', cookies)
+        #strip secure flag from 'Set-Cookie' headers
+        cookies = flow.response.headers.get_all('Set-Cookie')
+        cookies = [re.sub(r';\s*secure\s*', '', s) for s in cookies]
+        flow.response.headers.set_all('Set-Cookie', cookies)

examples/stickycookies

@@ -21,19 +21,19 @@ class StickyMaster(controller.Master):
         except KeyboardInterrupt:
             self.shutdown()
 
-    @controller.handler
-    def request(self, flow):
+    def handle_request(self, flow):
         hid = (flow.request.host, flow.request.port)
         if "cookie" in flow.request.headers:
             self.stickyhosts[hid] = flow.request.headers.get_all("cookie")
         elif hid in self.stickyhosts:
             flow.request.headers.set_all("cookie", self.stickyhosts[hid])
+        flow.reply()
 
-    @controller.handler
-    def response(self, flow):
+    def handle_response(self, flow):
         hid = (flow.request.host, flow.request.port)
         if "set-cookie" in flow.response.headers:
             self.stickyhosts[hid] = flow.response.headers.get_all("set-cookie")
+        flow.reply()
 
 
 config = proxy.ProxyConfig(port=8080)

examples/stream.py

@@ -1,4 +1,4 @@
-def responseheaders(flow):
+def responseheaders(context, flow):
     """
     Enables streaming for all responses.
     """

examples/stream_modify.py

@@ -16,5 +16,5 @@ def modify(chunks):
         yield chunk.replace("foo", "bar")
 
 
-def responseheaders(flow):
+def responseheaders(context, flow):
     flow.response.stream = modify

examples/stub.py

@@ -1,87 +1,79 @@
-import mitmproxy
 """
     This is a script stub, with definitions for all events.
 """
 
 
-def start():
+def start(context, argv):
     """
-        Called once on script startup before any other events
+        Called once on script startup, before any other events.
     """
-    mitmproxy.ctx.log("start")
+    context.log("start")
 
 
-def configure(options, updated):
-    """
-        Called once on script startup before any other events, and whenever options changes.
-    """
-    mitmproxy.ctx.log("configure")
-
-
-def clientconnect(root_layer):
+def clientconnect(context, root_layer):
     """
         Called when a client initiates a connection to the proxy. Note that a
         connection can correspond to multiple HTTP requests
     """
-    mitmproxy.ctx.log("clientconnect")
+    context.log("clientconnect")
 
 
-def request(flow):
+def request(context, flow):
     """
         Called when a client request has been received.
     """
-    mitmproxy.ctx.log("request")
+    context.log("request")
 
 
-def serverconnect(server_conn):
+def serverconnect(context, server_conn):
     """
         Called when the proxy initiates a connection to the target server. Note that a
         connection can correspond to multiple HTTP requests
     """
-    mitmproxy.ctx.log("serverconnect")
+    context.log("serverconnect")
 
 
-def responseheaders(flow):
+def responseheaders(context, flow):
     """
         Called when the response headers for a server response have been received,
         but the response body has not been processed yet. Can be used to tell mitmproxy
         to stream the response.
     """
-    mitmproxy.ctx.log("responseheaders")
+    context.log("responseheaders")
 
 
-def response(flow):
+def response(context, flow):
     """
        Called when a server response has been received.
     """
-    mitmproxy.ctx.log("response")
+    context.log("response")
 
 
-def error(flow):
+def error(context, flow):
     """
         Called when a flow error has occured, e.g. invalid server responses, or
         interrupted connections. This is distinct from a valid server HTTP error
         response, which is simply a response with an HTTP error code.
     """
-    mitmproxy.ctx.log("error")
+    context.log("error")
 
 
-def serverdisconnect(server_conn):
+def serverdisconnect(context, server_conn):
     """
         Called when the proxy closes the connection to the target server.
     """
-    mitmproxy.ctx.log("serverdisconnect")
+    context.log("serverdisconnect")
 
 
-def clientdisconnect(root_layer):
+def clientdisconnect(context, root_layer):
     """
         Called when a client disconnects from the proxy.
     """
-    mitmproxy.ctx.log("clientdisconnect")
+    context.log("clientdisconnect")
 
 
-def done():
+def done(context):
     """
         Called once on script shutdown, after any other events.
     """
-    mitmproxy.ctx.log("done")
+    context.log("done")

examples/tcp_message.py

@@ -1,4 +1,4 @@
-"""
+'''
 tcp_message Inline Script Hook API Demonstration
 ------------------------------------------------
 
@@ -7,21 +7,18 @@ tcp_message Inline Script Hook API Demonstration
 
 example cmdline invocation:
 mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py
-"""
-from netlib import strutils
+'''
+from netlib.utils import clean_bin
 
-
-def tcp_message(tcp_msg):
+def tcp_message(ctx, tcp_msg):
     modified_msg = tcp_msg.message.replace("foo", "bar")
 
     is_modified = False if modified_msg == tcp_msg.message else True
     tcp_msg.message = modified_msg
 
-    print(
-        "[tcp_message{}] from {} {} to {} {}:\r\n{}".format(
-            " (modified)" if is_modified else "",
-            "client" if tcp_msg.sender == tcp_msg.client_conn else "server",
-            tcp_msg.sender.address,
-            "server" if tcp_msg.receiver == tcp_msg.server_conn else "client",
-            tcp_msg.receiver.address, strutils.bytes_to_escaped_str(tcp_msg.message))
-    )
+    print("[tcp_message{}] from {} {} to {} {}:\r\n{}".format(
+        " (modified)" if is_modified else "",
+        "client" if tcp_msg.sender == tcp_msg.client_conn else "server",
+        tcp_msg.sender.address,
+        "server" if tcp_msg.receiver == tcp_msg.server_conn else "client",
+        tcp_msg.receiver.address, clean_bin(tcp_msg.message)))

examples/tls_passthrough.py

@@ -20,14 +20,12 @@ Example:
 
 Authors: Maximilian Hils, Matthew Tuusberg
 """
-from __future__ import absolute_import, print_function, division
+from __future__ import (absolute_import, print_function, division)
 import collections
 import random
 
-import sys
 from enum import Enum
 
-import mitmproxy
 from mitmproxy.exceptions import TlsProtocolException
 from mitmproxy.protocol import TlsLayer, RawTCPLayer
 
@@ -42,7 +40,6 @@ class _TlsStrategy(object):
     """
     Abstract base class for interception strategies.
     """
-
     def __init__(self):
         # A server_address -> interception results mapping
         self.history = collections.defaultdict(lambda: collections.deque(maxlen=200))
@@ -81,7 +78,6 @@ class ProbabilisticStrategy(_TlsStrategy):
     """
     Fixed probability that we intercept a given connection.
     """
-
     def __init__(self, p):
         self.p = p
         super(ProbabilisticStrategy, self).__init__()
@@ -98,6 +94,7 @@ class TlsFeedback(TlsLayer):
 
     def _establish_tls_with_client(self):
         server_address = self.server_conn.address
+        tls_strategy = self.script_context.tls_strategy
 
         try:
             super(TlsFeedback, self)._establish_tls_with_client()
@@ -110,18 +107,15 @@ class TlsFeedback(TlsLayer):
 
 # inline script hooks below.
 
-tls_strategy = None
 
-
-def start():
-    global tls_strategy
-    if len(sys.argv) == 2:
-        tls_strategy = ProbabilisticStrategy(float(sys.argv[1]))
+def start(context, argv):
+    if len(argv) == 2:
+        context.tls_strategy = ProbabilisticStrategy(float(argv[1]))
     else:
-        tls_strategy = ConservativeStrategy()
+        context.tls_strategy = ConservativeStrategy()
 
 
-def next_layer(next_layer):
+def next_layer(context, next_layer):
     """
     This hook does the actual magic - if the next layer is planned to be a TLS layer,
     we check if we want to enter pass-through mode instead.
@@ -129,13 +123,14 @@ def next_layer(next_layer):
     if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
         server_address = next_layer.server_conn.address
 
-        if tls_strategy.should_intercept(server_address):
+        if context.tls_strategy.should_intercept(server_address):
             # We try to intercept.
             # Monkey-Patch the layer to get feedback from the TLSLayer if interception worked.
             next_layer.__class__ = TlsFeedback
+            next_layer.script_context = context
         else:
             # We don't intercept - reply with a pass-through layer and add a "skipped" entry.
-            mitmproxy.ctx.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info")
-            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
-            next_layer.reply.send(next_layer_replacement)
-            tls_strategy.record_skipped(server_address)
+            context.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info")
+            next_layer_replacement = RawTCPLayer(next_layer.ctx, logging=False)
+            next_layer.reply(next_layer_replacement)
+            context.tls_strategy.record_skipped(server_address)

examples/upsidedownternet.py

@@ -1,15 +1,17 @@
 from six.moves import cStringIO as StringIO
 from PIL import Image
+from mitmproxy.models import decoded
 
 
-def response(flow):
+def response(context, flow):
     if flow.response.headers.get("content-type", "").startswith("image"):
-        try:
-            s = StringIO(flow.response.content)
-            img = Image.open(s).rotate(180)
-            s2 = StringIO()
-            img.save(s2, "png")
-            flow.response.content = s2.getvalue()
-            flow.response.headers["content-type"] = "image/png"
-        except:  # Unknown image types etc.
-            pass
+        with decoded(flow.response):  # automatically decode gzipped responses.
+            try:
+                s = StringIO(flow.response.content)
+                img = Image.open(s).rotate(180)
+                s2 = StringIO()
+                img.save(s2, "png")
+                flow.response.content = s2.getvalue()
+                flow.response.headers["content-type"] = "image/png"
+            except:  # Unknown image types etc.
+                pass

issue_template.md

@@ -10,13 +10,10 @@
 ##### What went wrong?
 
 
-##### Any other comments? What have you tried so far?
+##### Any other comments?
 
 
 ---
 
 Mitmproxy Version:
-Operating System:
-
-
-<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) -->
+Operating System: