kylefang/mitmproxy
1
0
Fork 0
mirror of https://github.com/zhigang1992/mitmproxy.git synced 2026-03-26 08:54:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1720 from cortesi/proxyrefactor

Browse Source 
proxy.protocol.http-related refactoring
This commit is contained in:
Aldo Cortesi
2016-11-14 08:03:10 +13:00
committed by GitHub
parent afa124a9f6 9b08279c7c
commit b636e4353a
24 changed files with 860 additions and 768 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
docs/scripting/events.rst
View File

@@ -98,6 +98,18 @@ HTTP Events
:widths: 40 60
:header-rows: 0
* - .. py:function:: http_connect(flow)
- Called when we receive an HTTP CONNECT request. Setting a non 2xx
response on the flow will return the response to the client abort the
connection. CONNECT requests and responses do not generate the usual
HTTP handler events. CONNECT requests are only valid in regular and
upstream proxy modes.
*flow*
A ``models.HTTPFlow`` object. The flow is guaranteed to have
non-None ``request`` and ``requestheaders`` attributes.
* - .. py:function:: request(flow)
- Called when a client request has been received.

4
mitmproxy/addons/__init__.py
View File

@@ -3,6 +3,7 @@ from mitmproxy.addons import anticomp
from mitmproxy.addons import clientplayback
from mitmproxy.addons import streamfile
from mitmproxy.addons import onboarding
from mitmproxy.addons import proxyauth
from mitmproxy.addons import replace
from mitmproxy.addons import script
from mitmproxy.addons import setheaders
@@ -10,11 +11,13 @@ from mitmproxy.addons import serverplayback
from mitmproxy.addons import stickyauth
from mitmproxy.addons import stickycookie
from mitmproxy.addons import streambodies
from mitmproxy.addons import upstream_auth
def default_addons():
return [
onboarding.Onboarding(),
proxyauth.ProxyAuth(),
anticache.AntiCache(),
anticomp.AntiComp(),
stickyauth.StickyAuth(),
@@ -26,4 +29,5 @@ def default_addons():
setheaders.SetHeaders(),
serverplayback.ServerPlayback(),
clientplayback.ClientPlayback(),
upstream_auth.UpstreamAuth(),
]

148
mitmproxy/addons/proxyauth.py Normal file
View File

@@ -0,0 +1,148 @@
import binascii
import passlib.apache
from mitmproxy import exceptions
from mitmproxy import http
import mitmproxy.net.http
REALM = "mitmproxy"
def mkauth(username, password, scheme="basic"):
v = binascii.b2a_base64(
(username + ":" + password).encode("utf8")
).decode("ascii")
return scheme + " " + v
def parse_http_basic_auth(s):
words = s.split()
if len(words) != 2:
return None
scheme = words[0]
try:
user = binascii.a2b_base64(words[1]).decode("utf8", "replace")
except binascii.Error:
return None
parts = user.split(':')
if len(parts) != 2:
return None
return scheme, parts[0], parts[1]
class ProxyAuth:
def __init__(self):
self.nonanonymous = False
self.htpasswd = None
self.singleuser = None
def enabled(self):
return any([self.nonanonymous, self.htpasswd, self.singleuser])
def which_auth_header(self, f):
if f.mode == "regular":
return 'Proxy-Authorization'
else:
return 'Authorization'
def auth_required_response(self, f):
if f.mode == "regular":
hdrname = 'Proxy-Authenticate'
else:
hdrname = 'WWW-Authenticate'
headers = mitmproxy.net.http.Headers()
headers[hdrname] = 'Basic realm="%s"' % REALM
if f.mode == "transparent":
return http.make_error_response(
401,
"Authentication Required",
headers
)
else:
return http.make_error_response(
407,
"Proxy Authentication Required",
headers,
)
def check(self, f):
auth_value = f.request.headers.get(self.which_auth_header(f), None)
if not auth_value:
return False
parts = parse_http_basic_auth(auth_value)
if not parts:
return False
scheme, username, password = parts
if scheme.lower() != 'basic':
return False
if self.nonanonymous:
pass
elif self.singleuser:
if [username, password] != self.singleuser:
return False
elif self.htpasswd:
if not self.htpasswd.check_password(username, password):
return False
else:
raise NotImplementedError("Should never happen.")
return True
def authenticate(self, f):
if self.check(f):
del f.request.headers[self.which_auth_header(f)]
else:
f.response = self.auth_required_response(f)
# Handlers
def configure(self, options, updated):
if "auth_nonanonymous" in updated:
self.nonanonymous = options.auth_nonanonymous
if "auth_singleuser" in updated:
if options.auth_singleuser:
parts = options.auth_singleuser.split(':')
if len(parts) != 2:
raise exceptions.OptionsError(
"Invalid single-user auth specification."
)
self.singleuser = parts
else:
self.singleuser = None
if "auth_htpasswd" in updated:
if options.auth_htpasswd:
try:
self.htpasswd = passlib.apache.HtpasswdFile(
options.auth_htpasswd
)
except (ValueError, OSError) as v:
raise exceptions.OptionsError(
"Could not open htpasswd file: %s" % v
)
else:
self.htpasswd = None
if self.enabled():
if options.mode == "transparent":
raise exceptions.OptionsError(
"Proxy Authentication not supported in transparent mode."
)
elif options.mode == "socks5":
raise exceptions.OptionsError(
"Proxy Authentication not supported in SOCKS mode. "
"https://github.com/mitmproxy/mitmproxy/issues/738"
)
# TODO: check for multiple auth options
def http_connect(self, f):
if self.enabled() and f.mode == "regular":
self.authenticate(f)
def requestheaders(self, f):
if self.enabled():
# Are we already authenticated in CONNECT?
if not (f.mode == "regular" and f.server_conn.via):
self.authenticate(f)

53
mitmproxy/addons/upstream_auth.py Normal file
View File

@@ -0,0 +1,53 @@
import re
import base64
from mitmproxy import exceptions
from mitmproxy.utils import strutils
def parse_upstream_auth(auth):
pattern = re.compile(".+:")
if pattern.search(auth) is None:
raise exceptions.OptionsError(
"Invalid upstream auth specification: %s" % auth
)
return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
class UpstreamAuth():
"""
This addon handles authentication to systems upstream from us for the
upstream proxy and reverse proxy mode. There are 3 cases:
- Upstream proxy CONNECT requests should have authentication added, and
subsequent already connected requests should not.
- Upstream proxy regular requests
- Reverse proxy regular requests (CONNECT is invalid in this mode)
"""
def __init__(self):
self.auth = None
self.root_mode = None
def configure(self, options, updated):
# FIXME: We're doing this because our proxy core is terminally confused
# at the moment. Ideally, we should be able to check if we're in
# reverse proxy mode at the HTTP layer, so that scripts can put the
# proxy in reverse proxy mode for specific reuests.
if "mode" in updated:
self.root_mode = options.mode
if "upstream_auth" in updated:
if options.upstream_auth is None:
self.auth = None
else:
self.auth = parse_upstream_auth(options.upstream_auth)
def http_connect(self, f):
if self.auth and f.mode == "upstream":
f.request.headers["Proxy-Authorization"] = self.auth
def requestheaders(self, f):
if self.auth:
if f.mode == "upstream" and not f.server_conn.via:
f.request.headers["Proxy-Authorization"] = self.auth
elif self.root_mode == "reverse":
f.request.headers["Proxy-Authorization"] = self.auth

2
mitmproxy/connections.py
View File

@@ -42,7 +42,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
self.timestamp_start = time.time()
self.timestamp_end = None
self.timestamp_ssl_setup = None
self.protocol = None
self.sni = None
self.cipher_name = None
self.tls_version = None
@@ -144,7 +143,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
self.timestamp_end = None
self.timestamp_tcp_setup = None
self.timestamp_ssl_setup = None
self.protocol = None
def connected(self):
return bool(self.connection) and not self.finished

1
mitmproxy/events.py
View File

@@ -13,6 +13,7 @@ Events = frozenset([
"tcp_error",
"tcp_end",
"http_connect",
"request",
"requestheaders",
"response",

9
mitmproxy/http.py
View File

@@ -53,7 +53,7 @@ class HTTPRequest(http.Request):
def get_state(self):
state = super().get_state()
state.update(
is_replay=self.is_replay,
is_replay=self.is_replay
)
return state
@@ -143,7 +143,7 @@ class HTTPFlow(flow.Flow):
transaction.
"""
def __init__(self, client_conn, server_conn, live=None):
def __init__(self, client_conn, server_conn, live=None, mode="regular"):
super().__init__("http", client_conn, server_conn, live)
self.request = None # type: HTTPRequest
@@ -163,11 +163,14 @@ class HTTPFlow(flow.Flow):
""":py:class:`ClientConnection` object """
self.intercepted = False # type: bool
""" Is this flow currently being intercepted? """
self.mode = mode
""" What mode was the proxy layer in when receiving this request? """
_stateobject_attributes = flow.Flow._stateobject_attributes.copy()
_stateobject_attributes.update(
request=HTTPRequest,
response=HTTPResponse
response=HTTPResponse,
mode=str
)
def __repr__(self):

1
mitmproxy/io_compat.py
View File

@@ -69,6 +69,7 @@ def convert_018_019(data):
data["client_conn"]["sni"] = None
data["client_conn"]["cipher_name"] = None
data["client_conn"]["tls_version"] = None
data["mode"] = "regular"
data["metadata"] = dict()
return data

4
mitmproxy/master.py
View File

@@ -255,6 +255,10 @@ class Master:
def next_layer(self, top_layer):
pass
@controller.handler
def http_connect(self, f):
pass
@controller.handler
def error(self, f):
pass

176
mitmproxy/net/http/authentication.py
View File

@@ -1,176 +0,0 @@
import argparse
import binascii
def parse_http_basic_auth(s):
words = s.split()
if len(words) != 2:
return None
scheme = words[0]
try:
user = binascii.a2b_base64(words[1]).decode("utf8", "replace")
except binascii.Error:
return None
parts = user.split(':')
if len(parts) != 2:
return None
return scheme, parts[0], parts[1]
def assemble_http_basic_auth(scheme, username, password):
v = binascii.b2a_base64((username + ":" + password).encode("utf8")).decode("ascii")
return scheme + " " + v
class NullProxyAuth:
"""
No proxy auth at all (returns empty challange headers)
"""
def __init__(self, password_manager):
self.password_manager = password_manager
def clean(self, headers_):
"""
Clean up authentication headers, so they're not passed upstream.
"""
def authenticate(self, headers_):
"""
Tests that the user is allowed to use the proxy
"""
return True
def auth_challenge_headers(self):
"""
Returns a dictionary containing the headers require to challenge the user
"""
return {}
class BasicAuth(NullProxyAuth):
CHALLENGE_HEADER = None
AUTH_HEADER = None
def __init__(self, password_manager, realm):
NullProxyAuth.__init__(self, password_manager)
self.realm = realm
def clean(self, headers):
del headers[self.AUTH_HEADER]
def authenticate(self, headers):
auth_value = headers.get(self.AUTH_HEADER)
if not auth_value:
return False
parts = parse_http_basic_auth(auth_value)
if not parts:
return False
scheme, username, password = parts
if scheme.lower() != 'basic':
return False
if not self.password_manager.test(username, password):
return False
self.username = username
return True
def auth_challenge_headers(self):
return {self.CHALLENGE_HEADER: 'Basic realm="%s"' % self.realm}
class BasicWebsiteAuth(BasicAuth):
CHALLENGE_HEADER = 'WWW-Authenticate'
AUTH_HEADER = 'Authorization'
class BasicProxyAuth(BasicAuth):
CHALLENGE_HEADER = 'Proxy-Authenticate'
AUTH_HEADER = 'Proxy-Authorization'
class PassMan:
def test(self, username_, password_token_):
return False
class PassManNonAnon(PassMan):
"""
Ensure the user specifies a username, accept any password.
"""
def test(self, username, password_token_):
if username:
return True
return False
class PassManHtpasswd(PassMan):
"""
Read usernames and passwords from an htpasswd file
"""
def __init__(self, path):
"""
Raises ValueError if htpasswd file is invalid.
"""
import passlib.apache
self.htpasswd = passlib.apache.HtpasswdFile(path)
def test(self, username, password_token):
return bool(self.htpasswd.check_password(username, password_token))
class PassManSingleUser(PassMan):
def __init__(self, username, password):
self.username, self.password = username, password
def test(self, username, password_token):
return self.username == username and self.password == password_token
class AuthAction(argparse.Action):
"""
Helper class to allow seamless integration int argparse. Example usage:
parser.add_argument(
"--nonanonymous",
action=NonanonymousAuthAction, nargs=0,
help="Allow access to any user long as a credentials are specified."
)
"""
def __call__(self, parser, namespace, values, option_string=None):
passman = self.getPasswordManager(values)
authenticator = BasicProxyAuth(passman, "mitmproxy")
setattr(namespace, self.dest, authenticator)
def getPasswordManager(self, s): # pragma: no cover
raise NotImplementedError()
class SingleuserAuthAction(AuthAction):
def getPasswordManager(self, s):
if len(s.split(':')) != 2:
raise argparse.ArgumentTypeError(
"Invalid single-user specification. Please use the format username:password"
)
username, password = s.split(':')
return PassManSingleUser(username, password)
class NonanonymousAuthAction(AuthAction):
def getPasswordManager(self, s):
return PassManNonAnon()
class HtpasswdAuthAction(AuthAction):
def getPasswordManager(self, s):
return PassManHtpasswd(s)

63
mitmproxy/proxy/config.py
View File

@@ -1,18 +1,14 @@
import base64
import collections
import os
import re
from typing import Any
from mitmproxy.utils import strutils
from OpenSSL import SSL, crypto
from mitmproxy import exceptions
from mitmproxy import options as moptions
from mitmproxy import certs
from mitmproxy.net import tcp
from mitmproxy.net.http import authentication
from mitmproxy.net.http import url
CONF_BASENAME = "mitmproxy"
@@ -56,21 +52,11 @@ def parse_server_spec(spec):
return ServerSpec(scheme, address)
def parse_upstream_auth(auth):
pattern = re.compile(".+:")
if pattern.search(auth) is None:
raise exceptions.OptionsError(
"Invalid upstream auth specification: %s" % auth
)
return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
class ProxyConfig:
def __init__(self, options: moptions.Options) -> None:
self.options = options
self.authenticator = None
self.check_ignore = None
self.check_tcp = None
self.certstore = None
@@ -134,54 +120,5 @@ class ProxyConfig:
)
self.upstream_server = None
self.upstream_auth = None
if options.upstream_server:
self.upstream_server = parse_server_spec(options.upstream_server)
if options.upstream_auth:
self.upstream_auth = parse_upstream_auth(options.upstream_auth)
self.authenticator = authentication.NullProxyAuth(None)
needsauth = any(
[
options.auth_nonanonymous,
options.auth_singleuser,
options.auth_htpasswd
]
)
if needsauth:
if options.mode == "transparent":
raise exceptions.OptionsError(
"Proxy Authentication not supported in transparent mode."
)
elif options.mode == "socks5":
raise exceptions.OptionsError(
"Proxy Authentication not supported in SOCKS mode. "
"https://github.com/mitmproxy/mitmproxy/issues/738"
)
elif options.auth_singleuser:
parts = options.auth_singleuser.split(':')
if len(parts) != 2:
raise exceptions.OptionsError(
"Invalid single-user specification. "
"Please use the format username:password"
)
password_manager = authentication.PassManSingleUser(*parts)
elif options.auth_nonanonymous:
password_manager = authentication.PassManNonAnon()
elif options.auth_htpasswd:
try:
password_manager = authentication.PassManHtpasswd(
options.auth_htpasswd
)
except ValueError as v:
raise exceptions.OptionsError(str(v))
if options.mode == "reverse":
self.authenticator = authentication.BasicWebsiteAuth(
password_manager,
self.upstream_server.address
)
else:
self.authenticator = authentication.BasicProxyAuth(
password_manager,
"mitmproxy"
)

3
mitmproxy/proxy/modes/socks_proxy.py
View File

@@ -5,9 +5,6 @@ from mitmproxy.net import socks
class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin):
def __init__(self, ctx):
super().__init__(ctx)
def __call__(self):
try:
# Parse Client Greeting

478
mitmproxy/proxy/protocol/http.py
View File

@@ -1,12 +1,13 @@
import h2.exceptions
import time
import traceback
import enum
from mitmproxy import exceptions
from mitmproxy import http
from mitmproxy import flow
from mitmproxy.proxy.protocol import base
from mitmproxy.proxy.protocol import websockets as pwebsockets
import mitmproxy.net.http
from mitmproxy.net import tcp
from mitmproxy.net import websockets
@@ -18,14 +19,6 @@ class _HttpTransmissionLayer(base.Layer):
def read_request_body(self, request):
raise NotImplementedError()
def read_request(self, f):
request = self.read_request_headers(f)
request.data.content = b"".join(
self.read_request_body(request)
)
request.timestamp_end = time.time()
return request
def send_request(self, request):
raise NotImplementedError()
@@ -120,12 +113,42 @@ class UpstreamConnectLayer(base.Layer):
self.server_conn.address = address
def is_ok(status):
return 200 <= status < 300
class HTTPMode(enum.Enum):
regular = 1
transparent = 2
upstream = 3
# At this point, we see only a subset of the proxy modes
MODE_REQUEST_FORMS = {
HTTPMode.regular: ("authority", "absolute"),
HTTPMode.transparent: ("relative"),
HTTPMode.upstream: ("authority", "absolute"),
}
def validate_request_form(mode, request):
if request.first_line_format == "absolute" and request.scheme != "http":
raise exceptions.HttpException(
"Invalid request scheme: %s" % request.scheme
)
allowed_request_forms = MODE_REQUEST_FORMS[mode]
if request.first_line_format not in allowed_request_forms:
err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
" or ".join(allowed_request_forms), request.first_line_format
)
raise exceptions.HttpException(err_message)
class HttpLayer(base.Layer):
def __init__(self, ctx, mode):
super().__init__(ctx)
self.mode = mode
self.flow = None # type: http.HTTPFlow
self.__initial_server_conn = None
"Contains the original destination in transparent mode, which needs to be restored"
"if an inline script modified the target server for a single http request"
@@ -133,25 +156,108 @@ class HttpLayer(base.Layer):
# see https://github.com/mitmproxy/mitmproxy/issues/925
self.__initial_server_tls = None
# Requests happening after CONNECT do not need Proxy-Authorization headers.
self.http_authenticated = False
self.connect_request = False
def __call__(self):
if self.mode == "transparent":
if self.mode == HTTPMode.transparent:
self.__initial_server_tls = self.server_tls
self.__initial_server_conn = self.server_conn
while True:
self.flow = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
if not self._process_flow(self.flow):
flow = http.HTTPFlow(
self.client_conn,
self.server_conn,
live=self,
mode=self.mode.name
)
if not self._process_flow(flow):
return
def handle_regular_connect(self, f):
self.connect_request = True
try:
self.set_server((f.request.host, f.request.port))
except (
exceptions.ProtocolException, exceptions.NetlibException
) as e:
# HTTPS tasting means that ordinary errors like resolution
# and connection errors can happen here.
self.send_error_response(502, repr(e))
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return False
if f.response:
resp = f.response
else:
resp = http.make_connect_response(f.request.data.http_version)
self.send_response(resp)
if is_ok(resp.status_code):
layer = self.ctx.next_layer(self)
layer()
return False
def handle_upstream_connect(self, f):
self.establish_server_connection(
f.request.host,
f.request.port,
f.request.scheme
)
self.send_request(f.request)
f.response = self.read_response_headers()
f.response.data.content = b"".join(
self.read_response_body(f.request, f.response)
)
self.send_response(f.response)
if is_ok(f.response.status_code):
layer = UpstreamConnectLayer(self, f.request)
return layer()
return False
def _process_flow(self, f):
try:
request = self.get_request_from_client(f)
# Make sure that the incoming request matches our expectations
self.validate_request(request)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return False
try:
request = self.read_request_headers(f)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen
# before/between requests.
return False
f.request = request
if request.first_line_format == "authority":
# The standards are silent on what we should do with a CONNECT
# request body, so although it's not common, it's allowed.
f.request.data.content = b"".join(
self.read_request_body(f.request)
)
f.request.timestamp_end = time.time()
self.channel.ask("http_connect", f)
if self.mode is HTTPMode.regular:
return self.handle_regular_connect(f)
elif self.mode is HTTPMode.upstream:
return self.handle_upstream_connect(f)
else:
msg = "Unexpected CONNECT request."
self.send_error_response(400, msg)
raise exceptions.ProtocolException(msg)
self.channel.ask("requestheaders", f)
if request.headers.get("expect", "").lower() == "100-continue":
# TODO: We may have to use send_response_headers for HTTP2
# here.
self.send_response(http.expect_continue_response)
request.headers.pop("expect")
request.data.content = b"".join(self.read_request_body(request))
request.timestamp_end = time.time()
validate_request_form(self.mode, request)
except exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
@@ -162,36 +268,25 @@ class HttpLayer(base.Layer):
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
# Proxy Authentication conceptually does not work in transparent mode.
# We catch this misconfiguration on startup. Here, we sort out requests
# after a successful CONNECT request (which do not need to be validated anymore)
if not (self.http_authenticated or self.authenticate(request)):
return False
f.request = request
try:
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return False
except (exceptions.ProtocolException, exceptions.NetlibException) as e:
# HTTPS tasting means that ordinary errors like resolution and
# connection errors can happen here.
self.send_error_response(502, repr(e))
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return False
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
f.request.headers["Host"] = self.config.upstream_server.address.host
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
f.request.headers["Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(f)
# Determine .scheme, .host and .port attributes for inline scripts. For
# absolute-form requests, they are directly given in the request. For
# authority-form requests, we only need to determine the request
# scheme. For relative-form requests, we need to determine host and
# port as well.
if self.mode is HTTPMode.transparent:
# Setting request.host also updates the host header, which we want
# to preserve
host_header = f.request.headers.get("host", None)
f.request.host = self.__initial_server_conn.address.host
f.request.port = self.__initial_server_conn.address.port
if host_header:
f.request.headers["host"] = host_header
f.request.scheme = "https" if self.__initial_server_tls else "http"
self.channel.ask("request", f)
try:
if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
@@ -205,7 +300,55 @@ class HttpLayer(base.Layer):
f.request.port,
f.request.scheme
)
self.get_response_from_server(f)
def get_response():
self.send_request(f.request)
f.response = self.read_response_headers()
try:
get_response()
except exceptions.NetlibException as e:
self.log(
"server communication error: %s" % repr(e),
level="debug"
)
# In any case, we try to reconnect at least once. This is
# necessary because it might be possible that we already
# initiated an upstream connection after clientconnect that
# has already been expired, e.g consider the following event
# log:
# > clientconnect (transparent mode destination known)
# > serverconnect (required for client tls handshake)
# > read n% of large request
# > server detects timeout, disconnects
# > read (100-n)% of large request
# > send large request upstream
if isinstance(e, exceptions.Http2ProtocolException):
# do not try to reconnect for HTTP2
raise exceptions.ProtocolException(
"First and only attempt to get response via HTTP2 failed."
)
self.disconnect()
self.connect()
get_response()
# call the appropriate script hook - this is an opportunity for
# an inline script to set f.stream = True
self.channel.ask("responseheaders", f)
if f.response.stream:
f.response.data.content = None
else:
f.response.data.content = b"".join(
self.read_response_body(f.request, f.response)
)
f.response.timestamp_end = time.time()
# no further manipulation of self.server_conn beyond this point
# we can safely set it as the final attribute value here.
f.server_conn = self.server_conn
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
@@ -213,20 +356,49 @@ class HttpLayer(base.Layer):
self.log("response", "debug", [repr(f.response)])
self.channel.ask("response", f)
self.send_response_to_client(f)
if not f.response.stream:
# no streaming:
# we already received the full response from the server and can
# send it to the client straight away.
self.send_response(f.response)
else:
# streaming:
# First send the headers and then transfer the response incrementally
self.send_response_headers(f.response)
chunks = self.read_response_body(
f.request,
f.response
)
if callable(f.response.stream):
chunks = f.response.stream(chunks)
self.send_response_body(f.response, chunks)
f.response.timestamp_end = time.time()
if self.check_close_connection(f):
return False
# Handle 101 Switching Protocols
if f.response.status_code == 101:
self.handle_101_switching_protocols(f)
return False # should never be reached
# Handle a successful HTTP 101 Switching Protocols Response,
# received after e.g. a WebSocket upgrade request.
# Check for WebSockets handshake
is_websockets = (
websockets.check_handshake(f.request.headers) and
websockets.check_handshake(f.response.headers)
)
if is_websockets and not self.config.options.websockets:
self.log(
"Client requested WebSocket connection, but the protocol is disabled.",
"info"
)
# Upstream Proxy Mode: Handle CONNECT
if f.request.first_line_format == "authority" and f.response.status_code == 200:
self.handle_upstream_mode_connect(f.request.copy())
return False
if is_websockets and self.config.options.websockets:
layer = pwebsockets.WebSocketsLayer(self, f)
else:
layer = self.ctx.next_layer(self)
layer()
return False # should never be reached
except (exceptions.ProtocolException, exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
@@ -244,135 +416,24 @@ class HttpLayer(base.Layer):
return True
def get_request_from_client(self, f):
request = self.read_request(f)
f.request = request
self.channel.ask("requestheaders", f)
if request.headers.get("expect", "").lower() == "100-continue":
# TODO: We may have to use send_response_headers for HTTP2 here.
self.send_response(http.expect_continue_response)
request.headers.pop("expect")
request.content = b"".join(self.read_request_body(request))
request.timestamp_end = time.time()
return request
def send_error_response(self, code, message, headers=None):
def send_error_response(self, code, message, headers=None) -> None:
try:
response = http.make_error_response(code, message, headers)
self.send_response(response)
except (exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException):
self.log(traceback.format_exc(), "debug")
def change_upstream_proxy_server(self, address):
def change_upstream_proxy_server(self, address) -> None:
# Make set_upstream_proxy_server always available,
# even if there's no UpstreamConnectLayer
if address != self.server_conn.address:
return self.set_server(address)
self.set_server(address)
def handle_regular_mode_connect(self, request):
self.http_authenticated = True
self.set_server((request.host, request.port))
self.send_response(http.make_connect_response(request.data.http_version))
layer = self.ctx.next_layer(self)
layer()
def handle_upstream_mode_connect(self, connect_request):
layer = UpstreamConnectLayer(self, connect_request)
layer()
def send_response_to_client(self, f):
if not f.response.stream:
# no streaming:
# we already received the full response from the server and can
# send it to the client straight away.
self.send_response(f.response)
else:
# streaming:
# First send the headers and then transfer the response incrementally
self.send_response_headers(f.response)
chunks = self.read_response_body(
f.request,
f.response
)
if callable(f.response.stream):
chunks = f.response.stream(chunks)
self.send_response_body(f.response, chunks)
f.response.timestamp_end = time.time()
def get_response_from_server(self, f):
def get_response():
self.send_request(f.request)
f.response = self.read_response_headers()
try:
get_response()
except exceptions.NetlibException as e:
self.log(
"server communication error: %s" % repr(e),
level="debug"
)
# In any case, we try to reconnect at least once. This is
# necessary because it might be possible that we already
# initiated an upstream connection after clientconnect that
# has already been expired, e.g consider the following event
# log:
# > clientconnect (transparent mode destination known)
# > serverconnect (required for client tls handshake)
# > read n% of large request
# > server detects timeout, disconnects
# > read (100-n)% of large request
# > send large request upstream
if isinstance(e, exceptions.Http2ProtocolException):
# do not try to reconnect for HTTP2
raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.")
self.disconnect()
self.connect()
get_response()
# call the appropriate script hook - this is an opportunity for an
# inline script to set f.stream = True
self.channel.ask("responseheaders", f)
if f.response.stream:
f.response.data.content = None
else:
f.response.data.content = b"".join(self.read_response_body(
f.request,
f.response
))
f.response.timestamp_end = time.time()
# no further manipulation of self.server_conn beyond this point
# we can safely set it as the final attribute value here.
f.server_conn = self.server_conn
def process_request_hook(self, f):
# Determine .scheme, .host and .port attributes for inline scripts.
# For absolute-form requests, they are directly given in the request.
# For authority-form requests, we only need to determine the request scheme.
# For relative-form requests, we need to determine host and port as
# well.
if self.mode == "regular":
pass # only absolute-form at this point, nothing to do here.
elif self.mode == "upstream":
pass
else:
# Setting request.host also updates the host header, which we want to preserve
host_header = f.request.headers.get("host", None)
f.request.host = self.__initial_server_conn.address.host
f.request.port = self.__initial_server_conn.address.port
if host_header:
f.request.headers["host"] = host_header
f.request.scheme = "https" if self.__initial_server_tls else "http"
self.channel.ask("request", f)
def establish_server_connection(self, host, port, scheme):
def establish_server_connection(self, host: str, port: int, scheme: str):
address = tcp.Address((host, port))
tls = (scheme == "https")
if self.mode == "regular" or self.mode == "transparent":
if self.mode is HTTPMode.regular or self.mode is HTTPMode.transparent:
# If there's an existing connection that doesn't match our expectations, kill it.
if address != self.server_conn.address or tls != self.server_tls:
self.set_server(address)
@@ -385,80 +446,3 @@ class HttpLayer(base.Layer):
self.connect()
if tls:
raise exceptions.HttpProtocolException("Cannot change scheme in upstream proxy mode.")
"""
# This is a very ugly (untested) workaround to solve a very ugly problem.
if self.server_conn and self.server_conn.tls_established and not ssl:
self.disconnect()
self.connect()
elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
if self.server_conn.tls_established:
self.disconnect()
self.connect()
self.send_request(make_connect_request(address))
tls_layer = TlsLayer(self, False, True)
tls_layer._establish_tls_with_server()
"""
def validate_request(self, request):
if request.first_line_format == "absolute" and request.scheme != "http":
raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
expected_request_forms = {
"regular": ("authority", "absolute",),
"upstream": ("authority", "absolute"),
"transparent": ("relative",)
}
allowed_request_forms = expected_request_forms[self.mode]
if request.first_line_format not in allowed_request_forms:
err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
" or ".join(allowed_request_forms), request.first_line_format
)
raise exceptions.HttpException(err_message)
if self.mode == "regular" and request.first_line_format == "absolute":
request.first_line_format = "relative"
def authenticate(self, request):
if self.config.authenticator:
if self.config.authenticator.authenticate(request.headers):
self.config.authenticator.clean(request.headers)
else:
if self.mode == "transparent":
self.send_response(http.make_error_response(
401,
"Authentication Required",
mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers())
))
else:
self.send_response(http.make_error_response(
407,
"Proxy Authentication Required",
mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers())
))
return False
return True
def handle_101_switching_protocols(self, f):
"""
Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request.
"""
# Check for WebSockets handshake
is_websockets = (
f and
websockets.check_handshake(f.request.headers) and
websockets.check_handshake(f.response.headers)
)
if is_websockets and not self.config.options.websockets:
self.log(
"Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
"info"
)
if is_websockets and self.config.options.websockets:
layer = pwebsockets.WebSocketsLayer(self, f)
else:
layer = self.ctx.next_layer(self)
layer()

29
mitmproxy/proxy/root_context.py
View File

@@ -2,6 +2,7 @@ from mitmproxy import log
from mitmproxy import exceptions
from mitmproxy.proxy import protocol
from mitmproxy.proxy import modes
from mitmproxy.proxy.protocol import http
class RootContext:
@@ -63,16 +64,21 @@ class RootContext:
# An inline script may upgrade from http to https,
# in which case we need some form of TLS layer.
if isinstance(top_layer, modes.ReverseProxy):
return protocol.TlsLayer(top_layer, client_tls, top_layer.server_tls, top_layer.server_conn.address.host)
return protocol.TlsLayer(
top_layer,
client_tls,
top_layer.server_tls,
top_layer.server_conn.address.host
)
if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer):
return protocol.TlsLayer(top_layer, client_tls, client_tls)
# 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed.
if isinstance(top_layer, protocol.TlsLayer):
if isinstance(top_layer.ctx, modes.HttpProxy):
return protocol.Http1Layer(top_layer, "regular")
return protocol.Http1Layer(top_layer, http.HTTPMode.regular)
if isinstance(top_layer.ctx, modes.HttpUpstreamProxy):
return protocol.Http1Layer(top_layer, "upstream")
return protocol.Http1Layer(top_layer, http.HTTPMode.upstream)
# 4. Check for other TLS cases (e.g. after CONNECT).
if client_tls:
@@ -86,21 +92,12 @@ class RootContext:
if isinstance(top_layer, protocol.TlsLayer):
alpn = top_layer.client_conn.get_alpn_proto_negotiated()
if alpn == b'h2':
return protocol.Http2Layer(top_layer, 'transparent')
return protocol.Http2Layer(top_layer, http.HTTPMode.transparent)
if alpn == b'http/1.1':
return protocol.Http1Layer(top_layer, 'transparent')
return protocol.Http1Layer(top_layer, http.HTTPMode.transparent)
# 6. Check for raw tcp mode
is_ascii = (
len(d) == 3 and
# expect A-Za-z
all(65 <= x <= 90 or 97 <= x <= 122 for x in d)
)
if self.config.options.rawtcp and not is_ascii:
return protocol.RawTCPLayer(top_layer)
# 7. Assume HTTP1 by default
return protocol.Http1Layer(top_layer, 'transparent')
# 6. Assume HTTP1 by default
return protocol.Http1Layer(top_layer, http.HTTPMode.transparent)
def log(self, msg, level, subs=()):
"""

4
mitmproxy/tools/cmdline.py
View File

@@ -463,8 +463,8 @@ def proxy_options(parser):
action="store", dest="upstream_auth", default=None,
type=str,
help="""
Proxy Authentication:
username:password
Add HTTP Basic authentcation to upstream proxy and reverse proxy
requests. Format: username:password
"""
)
rawtcp = group.add_mutually_exclusive_group()

42
mitmproxy/tools/web/master.py
View File

@@ -12,7 +12,6 @@ from mitmproxy.addons import intercept
from mitmproxy import options
from mitmproxy import master
from mitmproxy.tools.web import app
from mitmproxy.net.http import authentication
class Stop(Exception):
@@ -52,7 +51,7 @@ class Options(options.Options):
wdebug: bool = False,
wport: int = 8081,
wiface: str = "127.0.0.1",
wauthenticator: Optional[authentication.PassMan] = None,
# wauthenticator: Optional[authentication.PassMan] = None,
wsingleuser: Optional[str] = None,
whtpasswd: Optional[str] = None,
**kwargs
@@ -60,29 +59,30 @@ class Options(options.Options):
self.wdebug = wdebug
self.wport = wport
self.wiface = wiface
self.wauthenticator = wauthenticator
self.wsingleuser = wsingleuser
self.whtpasswd = whtpasswd
# self.wauthenticator = wauthenticator
# self.wsingleuser = wsingleuser
# self.whtpasswd = whtpasswd
self.intercept = intercept
super().__init__(**kwargs)
# TODO: This doesn't belong here.
def process_web_options(self, parser):
if self.wsingleuser or self.whtpasswd:
if self.wsingleuser:
if len(self.wsingleuser.split(':')) != 2:
return parser.error(
"Invalid single-user specification. Please use the format username:password"
)
username, password = self.wsingleuser.split(':')
self.wauthenticator = authentication.PassManSingleUser(username, password)
elif self.whtpasswd:
try:
self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
except ValueError as v:
return parser.error(v.message)
else:
self.wauthenticator = None
# if self.wsingleuser or self.whtpasswd:
# if self.wsingleuser:
# if len(self.wsingleuser.split(':')) != 2:
# return parser.error(
# "Invalid single-user specification. Please use the format username:password"
# )
# username, password = self.wsingleuser.split(':')
# # self.wauthenticator = authentication.PassManSingleUser(username, password)
# elif self.whtpasswd:
# try:
# self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
# except ValueError as v:
# return parser.error(v.message)
# else:
# self.wauthenticator = None
pass
class WebMaster(master.Master):
@@ -98,7 +98,7 @@ class WebMaster(master.Master):
self.addons.add(*addons.default_addons())
self.addons.add(self.view, intercept.Intercept())
self.app = app.Application(
self, self.options.wdebug, self.options.wauthenticator
self, self.options.wdebug, False
)
# This line is just for type hinting
self.options = self.options # type: Options

174
test/mitmproxy/addons/test_proxyauth.py Normal file
View File

@@ -0,0 +1,174 @@
import binascii
from mitmproxy import exceptions
from mitmproxy.test import taddons
from mitmproxy.test import tflow
from mitmproxy.test import tutils
from mitmproxy.addons import proxyauth
def test_parse_http_basic_auth():
assert proxyauth.parse_http_basic_auth(
proxyauth.mkauth("test", "test")
) == ("basic", "test", "test")
assert not proxyauth.parse_http_basic_auth("")
assert not proxyauth.parse_http_basic_auth("foo bar")
v = "basic " + binascii.b2a_base64(b"foo").decode("ascii")
assert not proxyauth.parse_http_basic_auth(v)
def test_configure():
up = proxyauth.ProxyAuth()
with taddons.context() as ctx:
tutils.raises(
exceptions.OptionsError,
ctx.configure, up, auth_singleuser="foo"
)
ctx.configure(up, auth_singleuser="foo:bar")
assert up.singleuser == ["foo", "bar"]
ctx.configure(up, auth_singleuser=None)
assert up.singleuser is None
ctx.configure(up, auth_nonanonymous=True)
assert up.nonanonymous
ctx.configure(up, auth_nonanonymous=False)
assert not up.nonanonymous
tutils.raises(
exceptions.OptionsError,
ctx.configure,
up,
auth_htpasswd = tutils.test_data.path(
"mitmproxy/net/data/server.crt"
)
)
tutils.raises(
exceptions.OptionsError,
ctx.configure,
up,
auth_htpasswd = "nonexistent"
)
ctx.configure(
up,
auth_htpasswd = tutils.test_data.path(
"mitmproxy/net/data/htpasswd"
)
)
assert up.htpasswd
assert up.htpasswd.check_password("test", "test")
assert not up.htpasswd.check_password("test", "foo")
ctx.configure(up, auth_htpasswd = None)
assert not up.htpasswd
tutils.raises(
exceptions.OptionsError,
ctx.configure,
up,
auth_nonanonymous = True,
mode = "transparent"
)
tutils.raises(
exceptions.OptionsError,
ctx.configure,
up,
auth_nonanonymous = True,
mode = "socks5"
)
def test_check():
up = proxyauth.ProxyAuth()
with taddons.context() as ctx:
ctx.configure(up, auth_nonanonymous=True)
f = tflow.tflow()
assert not up.check(f)
f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
"test", "test"
)
assert up.check(f)
f.request.headers["Proxy-Authorization"] = "invalid"
assert not up.check(f)
f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
"test", "test", scheme = "unknown"
)
assert not up.check(f)
ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:test")
f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
"test", "test"
)
assert up.check(f)
ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:foo")
assert not up.check(f)
ctx.configure(
up,
auth_singleuser = None,
auth_htpasswd = tutils.test_data.path(
"mitmproxy/net/data/htpasswd"
)
)
f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
"test", "test"
)
assert up.check(f)
f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
"test", "foo"
)
assert not up.check(f)
def test_authenticate():
up = proxyauth.ProxyAuth()
with taddons.context() as ctx:
ctx.configure(up, auth_nonanonymous=True)
f = tflow.tflow()
assert not f.response
up.authenticate(f)
assert f.response.status_code == 407
f = tflow.tflow()
f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
"test", "test"
)
up.authenticate(f)
assert not f.response
assert not f.request.headers.get("Proxy-Authorization")
f = tflow.tflow()
f.mode = "transparent"
assert not f.response
up.authenticate(f)
assert f.response.status_code == 401
f = tflow.tflow()
f.mode = "transparent"
f.request.headers["Authorization"] = proxyauth.mkauth(
"test", "test"
)
up.authenticate(f)
assert not f.response
assert not f.request.headers.get("Authorization")
def test_handlers():
up = proxyauth.ProxyAuth()
with taddons.context() as ctx:
ctx.configure(up, auth_nonanonymous=True)
f = tflow.tflow()
assert not f.response
up.requestheaders(f)
assert f.response.status_code == 407
f = tflow.tflow()
f.request.method = "CONNECT"
assert not f.response
up.http_connect(f)
assert f.response.status_code == 407

65
test/mitmproxy/addons/test_upstream_auth.py Normal file
View File

@@ -0,0 +1,65 @@
import base64
from mitmproxy import exceptions
from mitmproxy.test import taddons
from mitmproxy.test import tflow
from mitmproxy.test import tutils
from mitmproxy.addons import upstream_auth
def test_configure():
up = upstream_auth.UpstreamAuth()
with taddons.context() as tctx:
tctx.configure(up, upstream_auth="test:test")
assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:test")
tctx.configure(up, upstream_auth="test:")
assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:")
tctx.configure(up, upstream_auth=None)
assert not up.auth
tutils.raises(
exceptions.OptionsError,
tctx.configure,
up,
upstream_auth=""
)
tutils.raises(
exceptions.OptionsError,
tctx.configure,
up,
upstream_auth=":"
)
tutils.raises(
exceptions.OptionsError,
tctx.configure,
up,
upstream_auth=":test"
)
def test_simple():
up = upstream_auth.UpstreamAuth()
with taddons.context() as tctx:
tctx.configure(up, upstream_auth="foo:bar")
f = tflow.tflow()
f.mode = "upstream"
up.requestheaders(f)
assert "proxy-authorization" in f.request.headers
f = tflow.tflow()
up.requestheaders(f)
assert "proxy-authorization" not in f.request.headers
tctx.configure(up, mode="reverse")
f = tflow.tflow()
f.mode = "transparent"
up.requestheaders(f)
assert "proxy-authorization" in f.request.headers
f = tflow.tflow()
f.mode = "upstream"
up.http_connect(f)
assert "proxy-authorization" in f.request.headers

122
test/mitmproxy/net/http/test_authentication.py
View File

@@ -1,122 +0,0 @@
import binascii
from mitmproxy.test import tutils
from mitmproxy.net.http import authentication, Headers
def test_parse_http_basic_auth():
vals = ("basic", "foo", "bar")
assert authentication.parse_http_basic_auth(
authentication.assemble_http_basic_auth(*vals)
) == vals
assert not authentication.parse_http_basic_auth("")
assert not authentication.parse_http_basic_auth("foo bar")
v = "basic " + binascii.b2a_base64(b"foo").decode("ascii")
assert not authentication.parse_http_basic_auth(v)
class TestPassManNonAnon:
def test_simple(self):
p = authentication.PassManNonAnon()
assert not p.test("", "")
assert p.test("user", "")
class TestPassManHtpasswd:
def test_file_errors(self):
tutils.raises(
"malformed htpasswd file",
authentication.PassManHtpasswd,
tutils.test_data.path("mitmproxy/net/data/server.crt"))
def test_simple(self):
pm = authentication.PassManHtpasswd(tutils.test_data.path("mitmproxy/net/data/htpasswd"))
vals = ("basic", "test", "test")
authentication.assemble_http_basic_auth(*vals)
assert pm.test("test", "test")
assert not pm.test("test", "foo")
assert not pm.test("foo", "test")
assert not pm.test("test", "")
assert not pm.test("", "")
class TestPassManSingleUser:
def test_simple(self):
pm = authentication.PassManSingleUser("test", "test")
assert pm.test("test", "test")
assert not pm.test("test", "foo")
assert not pm.test("foo", "test")
class TestNullProxyAuth:
def test_simple(self):
na = authentication.NullProxyAuth(authentication.PassManNonAnon())
assert not na.auth_challenge_headers()
assert na.authenticate("foo")
na.clean({})
class TestBasicProxyAuth:
def test_simple(self):
ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(), "test")
headers = Headers()
assert ba.auth_challenge_headers()
assert not ba.authenticate(headers)
def test_authenticate_clean(self):
ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(), "test")
headers = Headers()
vals = ("basic", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
assert ba.authenticate(headers)
ba.clean(headers)
assert ba.AUTH_HEADER not in headers
headers[ba.AUTH_HEADER] = ""
assert not ba.authenticate(headers)
headers[ba.AUTH_HEADER] = "foo"
assert not ba.authenticate(headers)
vals = ("foo", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
assert not ba.authenticate(headers)
ba = authentication.BasicProxyAuth(authentication.PassMan(), "test")
vals = ("basic", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
assert not ba.authenticate(headers)
class Bunch:
pass
class TestAuthAction:
def test_nonanonymous(self):
m = Bunch()
aa = authentication.NonanonymousAuthAction(None, "authenticator")
aa(None, m, None, None)
assert m.authenticator
def test_singleuser(self):
m = Bunch()
aa = authentication.SingleuserAuthAction(None, "authenticator")
aa(None, m, "foo:bar", None)
assert m.authenticator
tutils.raises("invalid", aa, None, m, "foo", None)
def test_httppasswd(self):
m = Bunch()
aa = authentication.HtpasswdAuthAction(None, "authenticator")
aa(None, m, tutils.test_data.path("mitmproxy/net/data/htpasswd"), None)
assert m.authenticator

2
test/mitmproxy/protocol/test_http1.py
View File

@@ -20,7 +20,7 @@ class TestInvalidRequests(tservers.HTTPProxyTest):
with p.connect():
r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port))
assert r.status_code == 400
assert b"Invalid HTTP request form" in r.content
assert b"Unexpected CONNECT" in r.content
def test_relative_request(self):
p = self.pathoc_raw()

81
test/mitmproxy/test_eventsequence.py Normal file
View File

@@ -0,0 +1,81 @@
from mitmproxy import events
import contextlib
from . import tservers
class Eventer:
def __init__(self, **handlers):
self.failure = None
self.called = []
self.handlers = handlers
for i in events.Events - {"tick"}:
def mkprox():
evt = i
def prox(*args, **kwargs):
self.called.append(evt)
if evt in self.handlers:
try:
handlers[evt](*args, **kwargs)
except AssertionError as e:
self.failure = e
return prox
setattr(self, i, mkprox())
def fail(self):
pass
class SequenceTester:
@contextlib.contextmanager
def addon(self, addon):
self.master.addons.add(addon)
yield
self.master.addons.remove(addon)
if addon.failure:
raise addon.failure
class TestBasic(tservers.HTTPProxyTest, SequenceTester):
ssl = True
def test_requestheaders(self):
def hdrs(f):
assert f.request.headers
assert not f.request.content
def req(f):
assert f.request.headers
assert f.request.content
with self.addon(Eventer(requestheaders=hdrs, request=req)):
p = self.pathoc()
with p.connect():
assert p.request("get:'/p/200':b@10").status_code == 200
def test_100_continue_fail(self):
e = Eventer()
with self.addon(e):
p = self.pathoc()
with p.connect():
p.request(
"""
get:'/p/200'
h'expect'='100-continue'
h'content-length'='1000'
da
"""
)
assert "requestheaders" in e.called
assert "responseheaders" not in e.called
def test_connect(self):
e = Eventer()
with self.addon(e):
p = self.pathoc()
with p.connect():
p.request("get:'/p/200:b@1'")
assert "http_connect" in e.called
assert e.called.count("requestheaders") == 1
assert e.called.count("request") == 1

31
test/mitmproxy/test_proxy.py
View File

@@ -107,23 +107,12 @@ class TestProcessProxyOptions:
self.assert_noerr("-T")
self.assert_noerr("-U", "http://localhost")
self.assert_err("expected one argument", "-U")
self.assert_err("Invalid server specification", "-U", "upstream")
self.assert_noerr("--upstream-auth", "test:test")
self.assert_err("expected one argument", "--upstream-auth")
self.assert_err(
"Invalid upstream auth specification", "--upstream-auth", "test"
)
self.assert_err("mutually exclusive", "-R", "http://localhost", "-T")
def test_socks_auth(self):
self.assert_err(
"Proxy Authentication not supported in SOCKS mode.",
"--socks",
"--nonanonymous"
)
def test_client_certs(self):
with tutils.tmpdir() as cadir:
self.assert_noerr("--client-certs", cadir)
@@ -141,26 +130,6 @@ class TestProcessProxyOptions:
tutils.test_data.path("mitmproxy/data/testkey.pem"))
self.assert_err("does not exist", "--cert", "nonexistent")
def test_auth(self):
p = self.assert_noerr("--nonanonymous")
assert p.authenticator
p = self.assert_noerr(
"--htpasswd",
tutils.test_data.path("mitmproxy/data/htpasswd"))
assert p.authenticator
self.assert_err(
"malformed htpasswd file",
"--htpasswd",
tutils.test_data.path("mitmproxy/data/htpasswd.invalid"))
p = self.assert_noerr("--singleuser", "test:test")
assert p.authenticator
self.assert_err(
"invalid single-user specification",
"--singleuser",
"test")
def test_insecure(self):
p = self.assert_noerr("--insecure")
assert p.openssl_verification_mode_server == SSL.VERIFY_NONE

21
test/mitmproxy/test_proxy_config.py
View File

@@ -1,5 +1,4 @@
from mitmproxy.test import tutils
import base64
from mitmproxy.proxy import config
@@ -26,23 +25,3 @@ def test_parse_server_spec():
config.parse_server_spec,
"http://"
)
def test_parse_upstream_auth():
tutils.raises(
"Invalid upstream auth specification",
config.parse_upstream_auth,
""
)
tutils.raises(
"Invalid upstream auth specification",
config.parse_upstream_auth,
":"
)
tutils.raises(
"Invalid upstream auth specification",
config.parse_upstream_auth,
":test"
)
assert config.parse_upstream_auth("test:test") == b"Basic" + b" " + base64.b64encode(b"test:test")
assert config.parse_upstream_auth("test:") == b"Basic" + b" " + base64.b64encode(b"test:")

103
test/mitmproxy/test_server.py
View File

@@ -6,6 +6,7 @@ from mitmproxy.test import tutils
from mitmproxy import controller
from mitmproxy import options
from mitmproxy.addons import script
from mitmproxy.addons import proxyauth
from mitmproxy import http
from mitmproxy.proxy.config import HostMatcher, parse_server_spec
import mitmproxy.net.http
@@ -13,7 +14,6 @@ from mitmproxy.net import tcp
from mitmproxy.net import socks
from mitmproxy import certs
from mitmproxy import exceptions
from mitmproxy.net.http import authentication
from mitmproxy.net.http import http1
from mitmproxy.net.tcp import Address
from pathod import pathoc
@@ -50,10 +50,7 @@ class CommonMixin:
def test_replay(self):
assert self.pathod("304").status_code == 304
if isinstance(self, tservers.HTTPUpstreamProxyTest) and self.ssl:
assert len(self.master.state.flows) == 2
else:
assert len(self.master.state.flows) == 1
assert len(self.master.state.flows) == 1
l = self.master.state.flows[-1]
assert l.response.status_code == 304
l.request.path = "/p/305"
@@ -288,6 +285,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin):
class TestHTTPAuth(tservers.HTTPProxyTest):
def test_auth(self):
self.master.addons.add(proxyauth.ProxyAuth())
self.master.options.auth_singleuser = "test:test"
assert self.pathod("202").status_code == 407
p = self.pathoc()
@@ -298,14 +296,15 @@ class TestHTTPAuth(tservers.HTTPProxyTest):
h'%s'='%s'
""" % (
self.server.port,
mitmproxy.net.http.authentication.BasicProxyAuth.AUTH_HEADER,
authentication.assemble_http_basic_auth("basic", "test", "test")
"Proxy-Authorization",
proxyauth.mkauth("test", "test")
))
assert ret.status_code == 202
class TestHTTPReverseAuth(tservers.ReverseProxyTest):
def test_auth(self):
self.master.addons.add(proxyauth.ProxyAuth())
self.master.options.auth_singleuser = "test:test"
assert self.pathod("202").status_code == 401
p = self.pathoc()
@@ -315,8 +314,8 @@ class TestHTTPReverseAuth(tservers.ReverseProxyTest):
'/p/202'
h'%s'='%s'
""" % (
mitmproxy.net.http.authentication.BasicWebsiteAuth.AUTH_HEADER,
authentication.assemble_http_basic_auth("basic", "test", "test")
"Authorization",
proxyauth.mkauth("test", "test")
))
assert ret.status_code == 202
@@ -672,6 +671,13 @@ class TestProxySSL(tservers.HTTPProxyTest):
first_flow = self.master.state.flows[0]
assert first_flow.server_conn.timestamp_ssl_setup
def test_via(self):
# tests that the ssl timestamp is present when ssl is used
f = self.pathod("200:b@10")
assert f.status_code == 200
first_flow = self.master.state.flows[0]
assert not first_flow.server_conn.via
class MasterRedirectRequest(tservers.TestMaster):
redirect_port = None # Set by TestRedirectRequest
@@ -952,12 +958,15 @@ class TestUpstreamProxySSL(
assert req.status_code == 418
# CONNECT from pathoc to chain[0],
assert self.proxy.tmaster.state.flow_count() == 2
assert self.proxy.tmaster.state.flow_count() == 1
assert self.proxy.tmaster.state.flows[0].server_conn.via
# request from pathoc to chain[0]
# CONNECT from proxy to chain[1],
assert self.chain[0].tmaster.state.flow_count() == 2
assert self.chain[0].tmaster.state.flow_count() == 1
assert self.chain[0].tmaster.state.flows[0].server_conn.via
# request from proxy to chain[1]
# request from chain[0] (regular proxy doesn't store CONNECTs)
assert not self.chain[1].tmaster.state.flows[0].server_conn.via
assert self.chain[1].tmaster.state.flow_count() == 1
@@ -978,21 +987,12 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest):
def test_reconnect(self):
"""
Tests proper functionality of ConnectionHandler.server_reconnect mock.
If we have a disconnect on a secure connection that's transparently proxified to
an upstream http proxy, we need to send the CONNECT request again.
If we have a disconnect on a secure connection that's transparently
proxified to an upstream http proxy, we need to send the CONNECT
request again.
"""
self.chain[1].tmaster.addons.add(
RequestKiller([2])
)
self.chain[0].tmaster.addons.add(
RequestKiller(
[
1, # CONNECT
3, # reCONNECT
4 # request
]
)
)
self.chain[0].tmaster.addons.add(RequestKiller([1, 2]))
self.chain[1].tmaster.addons.add(RequestKiller([1]))
p = self.pathoc()
with p.connect():
@@ -1000,44 +1000,27 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest):
assert req.content == b"content"
assert req.status_code == 418
assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT and request
# CONNECT, failing request,
assert self.chain[0].tmaster.state.flow_count() == 4
# reCONNECT, request
# failing request, request
assert self.chain[1].tmaster.state.flow_count() == 2
# (doesn't store (repeated) CONNECTs from chain[0]
# as it is a regular proxy)
assert not self.chain[1].tmaster.state.flows[0].response # killed
assert self.chain[1].tmaster.state.flows[1].response
assert self.proxy.tmaster.state.flows[0].request.first_line_format == "authority"
assert self.proxy.tmaster.state.flows[1].request.first_line_format == "relative"
assert self.chain[0].tmaster.state.flows[
0].request.first_line_format == "authority"
assert self.chain[0].tmaster.state.flows[
1].request.first_line_format == "relative"
assert self.chain[0].tmaster.state.flows[
2].request.first_line_format == "authority"
assert self.chain[0].tmaster.state.flows[
3].request.first_line_format == "relative"
assert self.chain[1].tmaster.state.flows[
0].request.first_line_format == "relative"
assert self.chain[1].tmaster.state.flows[
1].request.first_line_format == "relative"
# First request goes through all three proxies exactly once
assert self.proxy.tmaster.state.flow_count() == 1
assert self.chain[0].tmaster.state.flow_count() == 1
assert self.chain[1].tmaster.state.flow_count() == 1
req = p.request("get:'/p/418:b\"content2\"'")
assert req.status_code == 502
assert self.proxy.tmaster.state.flow_count() == 3 # + new request
# + new request, repeated CONNECT from chain[1]
assert self.chain[0].tmaster.state.flow_count() == 6
# (both terminated)
# nothing happened here
assert self.chain[1].tmaster.state.flow_count() == 2
assert self.proxy.tmaster.state.flow_count() == 2
assert self.chain[0].tmaster.state.flow_count() == 2
# Upstream sees two requests due to reconnection attempt
assert self.chain[1].tmaster.state.flow_count() == 3
assert not self.chain[1].tmaster.state.flows[-1].response
assert not self.chain[1].tmaster.state.flows[-2].response
# Reconnection failed, so we're now disconnected
tutils.raises(
exceptions.HttpException,
p.request,
"get:'/p/418:b\"content3\"'"
)
class AddUpstreamCertsToClientChainMixin: