kylefang/mitmproxy
1
0
Fork 0
mirror of https://github.com/zhigang1992/mitmproxy.git synced 2026-04-28 12:15:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1379 from cortesi/proxyconfig

Browse Source 
Unify ProxyConfig and Options
This commit is contained in:
Aldo Cortesi
2016-07-19 16:59:23 +12:00
committed by GitHub
parent 5034a6232c 9c9d28d068
commit ecd01ce7c6
26 changed files with 622 additions and 530 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
mitmproxy/cmdline.py
View File

@@ -1,21 +1,32 @@
from __future__ import absolute_import, print_function, division
import base64
import os
import re
import configargparse
from mitmproxy import exceptions
from mitmproxy import filt
from mitmproxy.proxy import config
from mitmproxy import platform
from netlib import human
from netlib import strutils
from netlib import tcp
from netlib import version
from netlib.http import url
APP_HOST = "mitm.it"
APP_PORT = 80
CA_DIR = "~/.mitmproxy"
# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default.
# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old
DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" \
"ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" \
"ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" \
"ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" \
"DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" \
"DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" \
"AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" \
"HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" \
"!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"
class ParseException(Exception):
@@ -107,110 +118,164 @@ def parse_setheader(s):
return _parse_hook(s)
def parse_server_spec(spec):
try:
p = url.parse(spec)
if p[0] not in (b"http", b"https"):
raise ValueError()
except ValueError:
raise configargparse.ArgumentTypeError(
"Invalid server specification: %s" % spec
)
address = tcp.Address(p[1:3])
scheme = p[0].lower()
return config.ServerSpec(scheme, address)
def parse_upstream_auth(auth):
pattern = re.compile(".+:")
if pattern.search(auth) is None:
raise configargparse.ArgumentTypeError(
"Invalid upstream auth specification: %s" % auth
)
return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
def get_common_options(options):
def get_common_options(args):
stickycookie, stickyauth = None, None
if options.stickycookie_filt:
stickycookie = options.stickycookie_filt
if args.stickycookie_filt:
stickycookie = args.stickycookie_filt
if options.stickyauth_filt:
stickyauth = options.stickyauth_filt
if args.stickyauth_filt:
stickyauth = args.stickyauth_filt
stream_large_bodies = options.stream_large_bodies
stream_large_bodies = args.stream_large_bodies
if stream_large_bodies:
stream_large_bodies = human.parse_size(stream_large_bodies)
reps = []
for i in options.replace:
for i in args.replace:
try:
p = parse_replace_hook(i)
except ParseException as e:
raise configargparse.ArgumentTypeError(e)
raise exceptions.OptionsError(e)
reps.append(p)
for i in options.replace_file:
for i in args.replace_file:
try:
patt, rex, path = parse_replace_hook(i)
except ParseException as e:
raise configargparse.ArgumentTypeError(e)
raise exceptions.OptionsError(e)
try:
v = open(path, "rb").read()
except IOError as e:
raise configargparse.ArgumentTypeError(
raise exceptions.OptionsError(
"Could not read replace file: %s" % path
)
reps.append((patt, rex, v))
setheaders = []
for i in options.setheader:
for i in args.setheader:
try:
p = parse_setheader(i)
except ParseException as e:
raise configargparse.ArgumentTypeError(e)
raise exceptions.OptionsError(e)
setheaders.append(p)
if options.outfile and options.outfile[0] == options.rfile:
if options.outfile[1] == "wb":
raise configargparse.ArgumentTypeError(
if args.outfile and args.outfile[0] == args.rfile:
if args.outfile[1] == "wb":
raise exceptions.OptionsError(
"Cannot use '{}' for both reading and writing flows. "
"Are you looking for --afile?".format(options.rfile)
"Are you looking for --afile?".format(args.rfile)
)
else:
raise configargparse.ArgumentTypeError(
raise exceptions.OptionsError(
"Cannot use '{}' for both reading and appending flows. "
"That would trigger an infinite loop."
)
return dict(
app=options.app,
app_host=options.app_host,
app_port=options.app_port,
# Proxy config
certs = []
for i in args.certs:
parts = i.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
certs.append(parts)
anticache=options.anticache,
anticomp=options.anticomp,
client_replay=options.client_replay,
kill=options.kill,
no_server=options.no_server,
refresh_server_playback=not options.norefresh,
rheaders=options.rheaders,
rfile=options.rfile,
body_size_limit = args.body_size_limit
if body_size_limit:
try:
body_size_limit = human.parse_size(body_size_limit)
except ValueError as e:
raise exceptions.OptionsError(
"Invalid body size limit specification: %s" % body_size_limit
)
# Establish proxy mode
c = 0
mode, upstream_server = "regular", None
if args.transparent_proxy:
c += 1
if not platform.resolver:
raise exceptions.OptionsError(
"Transparent mode not supported on this platform."
)
mode = "transparent"
if args.socks_proxy:
c += 1
mode = "socks5"
if args.reverse_proxy:
c += 1
mode = "reverse"
upstream_server = args.reverse_proxy
if args.upstream_proxy:
c += 1
mode = "upstream"
upstream_server = args.upstream_proxy
if c > 1:
raise exceptions.OptionsError(
"Transparent, SOCKS5, reverse and upstream proxy mode "
"are mutually exclusive. Read the docs on proxy modes "
"to understand why."
)
if args.add_upstream_certs_to_client_chain and args.no_upstream_cert:
raise exceptions.OptionsError(
"The no-upstream-cert and add-upstream-certs-to-client-chain "
"options are mutually exclusive. If no-upstream-cert is enabled "
"then the upstream certificate is not retrieved before generating "
"the client certificate chain."
)
return dict(
app=args.app,
app_host=args.app_host,
app_port=args.app_port,
anticache=args.anticache,
anticomp=args.anticomp,
client_replay=args.client_replay,
kill=args.kill,
no_server=args.no_server,
refresh_server_playback=not args.norefresh,
rheaders=args.rheaders,
rfile=args.rfile,
replacements=reps,
setheaders=setheaders,
server_replay=options.server_replay,
scripts=options.scripts,
server_replay=args.server_replay,
scripts=args.scripts,
stickycookie=stickycookie,
stickyauth=stickyauth,
stream_large_bodies=stream_large_bodies,
showhost=options.showhost,
outfile=options.outfile,
verbosity=options.verbose,
nopop=options.nopop,
replay_ignore_content=options.replay_ignore_content,
replay_ignore_params=options.replay_ignore_params,
replay_ignore_payload_params=options.replay_ignore_payload_params,
replay_ignore_host=options.replay_ignore_host
showhost=args.showhost,
outfile=args.outfile,
verbosity=args.verbose,
nopop=args.nopop,
replay_ignore_content=args.replay_ignore_content,
replay_ignore_params=args.replay_ignore_params,
replay_ignore_payload_params=args.replay_ignore_payload_params,
replay_ignore_host=args.replay_ignore_host,
auth_nonanonymous = args.auth_nonanonymous,
auth_singleuser = args.auth_singleuser,
auth_htpasswd = args.auth_htpasswd,
add_upstream_certs_to_client_chain = args.add_upstream_certs_to_client_chain,
body_size_limit = body_size_limit,
cadir = args.cadir,
certs = certs,
ciphers_client = args.ciphers_client,
ciphers_server = args.ciphers_server,
clientcerts = args.clientcerts,
http2 = args.http2,
ignore_hosts = args.ignore_hosts,
listen_host = args.addr,
listen_port = args.port,
mode = mode,
no_upstream_cert = args.no_upstream_cert,
rawtcp = args.rawtcp,
upstream_server = upstream_server,
upstream_auth = args.upstream_auth,
ssl_version_client = args.ssl_version_client,
ssl_version_server = args.ssl_version_server,
ssl_verify_upstream_cert = args.ssl_verify_upstream_cert,
ssl_verify_upstream_trusted_cadir = args.ssl_verify_upstream_trusted_cadir,
ssl_verify_upstream_trusted_ca = args.ssl_verify_upstream_trusted_ca,
tcp_hosts = args.tcp_hosts,
)
@@ -242,8 +307,8 @@ def basic_options(parser):
)
parser.add_argument(
"--cadir",
action="store", type=str, dest="cadir", default=config.CA_DIR,
help="Location of the default mitmproxy CA files. (%s)" % config.CA_DIR
action="store", type=str, dest="cadir", default=CA_DIR,
help="Location of the default mitmproxy CA files. (%s)" % CA_DIR
)
parser.add_argument(
"--host",
@@ -327,7 +392,7 @@ def proxy_modes(parser):
group.add_argument(
"-R", "--reverse",
action="store",
type=parse_server_spec,
type=str,
dest="reverse_proxy",
default=None,
help="""
@@ -350,7 +415,7 @@ def proxy_modes(parser):
group.add_argument(
"-U", "--upstream",
action="store",
type=parse_server_spec,
type=str,
dest="upstream_proxy",
default=None,
help="Forward all requests to upstream proxy server: http://host[:port]"
@@ -408,7 +473,7 @@ def proxy_options(parser):
parser.add_argument(
"--upstream-auth",
action="store", dest="upstream_auth", default=None,
type=parse_upstream_auth,
type=str,
help="""
Proxy Authentication:
username:password
@@ -441,7 +506,7 @@ def proxy_ssl_options(parser):
'as the first entry. Can be passed multiple times.')
group.add_argument(
"--ciphers-client", action="store",
type=str, dest="ciphers_client", default=config.DEFAULT_CLIENT_CIPHERS,
type=str, dest="ciphers_client", default=DEFAULT_CLIENT_CIPHERS,
help="Set supported ciphers for client connections. (OpenSSL Syntax)"
)
group.add_argument(
@@ -696,8 +761,8 @@ def mitmproxy():
usage="%(prog)s [options]",
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(config.CA_DIR, "common.conf"),
os.path.join(config.CA_DIR, "mitmproxy.conf")
os.path.join(CA_DIR, "common.conf"),
os.path.join(CA_DIR, "mitmproxy.conf")
],
add_config_file_help=True,
add_env_var_help=True
@@ -751,8 +816,8 @@ def mitmdump():
usage="%(prog)s [options] [filter]",
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(config.CA_DIR, "common.conf"),
os.path.join(config.CA_DIR, "mitmdump.conf")
os.path.join(CA_DIR, "common.conf"),
os.path.join(CA_DIR, "mitmdump.conf")
],
add_config_file_help=True,
add_env_var_help=True
@@ -781,8 +846,8 @@ def mitmweb():
usage="%(prog)s [options]",
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(config.CA_DIR, "common.conf"),
os.path.join(config.CA_DIR, "mitmweb.conf")
os.path.join(CA_DIR, "common.conf"),
os.path.join(CA_DIR, "mitmweb.conf")
],
add_config_file_help=True,
add_env_var_help=True

2
mitmproxy/console/master.py
View File

@@ -476,7 +476,7 @@ class ConsoleMaster(flow.FlowMaster):
sys.exit(1)
self.loop.set_alarm_in(0.01, self.ticker)
if self.server.config.http2 and not tcp.HAS_ALPN: # pragma: no cover
if self.options.http2 and not tcp.HAS_ALPN: # pragma: no cover
def http2err(*args, **kwargs):
signals.status_message.send(
message = "HTTP/2 disabled - OpenSSL 1.0.2+ required."

54
mitmproxy/console/options.py
View File

@@ -42,8 +42,8 @@ class Options(urwid.WidgetWrap):
select.Option(
"Ignore Patterns",
"I",
lambda: master.server.config.check_ignore,
self.ignorepatterns
lambda: master.options.ignore_hosts,
self.ignore_hosts
),
select.Option(
"Replacement Patterns",
@@ -82,14 +82,14 @@ class Options(urwid.WidgetWrap):
select.Option(
"No Upstream Certs",
"U",
lambda: master.server.config.no_upstream_cert,
self.toggle_upstream_cert
lambda: master.options.no_upstream_cert,
master.options.toggler("no_upstream_cert")
),
select.Option(
"TCP Proxying",
"T",
lambda: master.server.config.check_tcp,
self.tcp_proxy
lambda: master.options.tcp_hosts,
self.tcp_hosts
),
select.Heading("Utility"),
@@ -152,21 +152,20 @@ class Options(urwid.WidgetWrap):
return super(self.__class__, self).keypress(size, key)
def clearall(self):
self.master.server.config.no_upstream_cert = False
self.master.set_ignore_filter([])
self.master.set_tcp_filter([])
self.master.options.update(
anticache = False,
anticomp = False,
ignore_hosts = (),
tcp_hosts = (),
kill = False,
no_upstream_cert = False,
refresh_server_playback = True,
replacements = [],
scripts = [],
setheaders = [],
showhost = False,
stickyauth = None,
stickycookie = None
stickycookie = None,
)
self.master.state.default_body_view = contentviews.get("Auto")
@@ -177,10 +176,6 @@ class Options(urwid.WidgetWrap):
expire = 1
)
def toggle_upstream_cert(self):
self.master.server.config.no_upstream_cert = not self.master.server.config.no_upstream_cert
signals.update_settings.send(self)
def setheaders(self):
self.master.view_grideditor(
grideditor.SetHeadersEditor(
@@ -190,14 +185,21 @@ class Options(urwid.WidgetWrap):
)
)
def ignorepatterns(self):
def _set(ignore):
self.master.set_ignore_filter(ignore)
def tcp_hosts(self):
self.master.view_grideditor(
grideditor.HostPatternEditor(
self.master,
self.master.get_ignore_filter(),
_set
self.master.options.tcp_hosts,
self.master.options.setter("tcp_hosts")
)
)
def ignore_hosts(self):
self.master.view_grideditor(
grideditor.HostPatternEditor(
self.master,
self.master.options.ignore_hosts,
self.master.options.setter("ignore_hosts")
)
)
@@ -229,18 +231,6 @@ class Options(urwid.WidgetWrap):
def has_default_displaymode(self):
return self.master.state.default_body_view.name != "Auto"
def tcp_proxy(self):
def _set(tcp):
self.master.set_tcp_filter(tcp)
signals.update_settings.send(self)
self.master.view_grideditor(
grideditor.HostPatternEditor(
self.master,
self.master.get_tcp_filter(),
_set
)
)
def sticky_auth(self):
signals.status_prompt.send(
prompt = "Sticky auth filter",

12
mitmproxy/console/statusbar.py
View File

@@ -156,14 +156,14 @@ class StatusBar(urwid.WidgetWrap):
r.append(":%s in file]" % self.master.server_playback.count())
else:
r.append(":%s to go]" % self.master.server_playback.count())
if self.master.get_ignore_filter():
if self.master.options.ignore_hosts:
r.append("[")
r.append(("heading_key", "I"))
r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
if self.master.get_tcp_filter():
r.append("gnore:%d]" % len(self.master.options.ignore_hosts))
if self.master.options.tcp_hosts:
r.append("[")
r.append(("heading_key", "T"))
r.append("CP:%d]" % len(self.master.get_tcp_filter()))
r.append("CP:%d]" % len(self.master.options.tcp_hosts))
if self.master.state.intercept_txt:
r.append("[")
r.append(("heading_key", "i"))
@@ -200,7 +200,7 @@ class StatusBar(urwid.WidgetWrap):
opts.append("norefresh")
if self.master.options.kill:
opts.append("killextra")
if self.master.server.config.no_upstream_cert:
if self.master.options.no_upstream_cert:
opts.append("no-upstream-cert")
if self.master.state.follow_focus:
opts.append("following")
@@ -214,7 +214,7 @@ class StatusBar(urwid.WidgetWrap):
if opts:
r.append("[%s]" % (":".join(opts)))
if self.master.server.config.mode in ["reverse", "upstream"]:
if self.master.options.mode in ["reverse", "upstream"]:
dst = self.master.server.config.upstream_server
r.append("[dest:%s]" % netlib.http.url.unparse(
dst.scheme,

2
mitmproxy/dump.py
View File

@@ -53,7 +53,7 @@ class DumpMaster(flow.FlowMaster):
self.set_stream_large_bodies(options.stream_large_bodies)
if self.server and self.server.config.http2 and not tcp.HAS_ALPN: # pragma: no cover
if self.server and self.options.http2 and not tcp.HAS_ALPN: # pragma: no cover
print("ALPN support missing (OpenSSL 1.0.2+ required)!\n"
"HTTP/2 is disabled. Use --no-http2 to silence this warning.",
file=sys.stderr)

15
mitmproxy/flow/master.py
View File

@@ -13,7 +13,6 @@ from mitmproxy.flow import io
from mitmproxy.flow import modules
from mitmproxy.onboarding import app
from mitmproxy.protocol import http_replay
from mitmproxy.proxy.config import HostMatcher
class FlowMaster(controller.Master):
@@ -48,18 +47,6 @@ class FlowMaster(controller.Master):
port
)
def get_ignore_filter(self):
return self.server.config.check_ignore.patterns
def set_ignore_filter(self, host_patterns):
self.server.config.check_ignore = HostMatcher(host_patterns)
def get_tcp_filter(self):
return self.server.config.check_tcp.patterns
def set_tcp_filter(self, host_patterns):
self.server.config.check_tcp = HostMatcher(host_patterns)
def set_stream_large_bodies(self, max_size):
if max_size is not None:
self.stream_large_bodies = modules.StreamLargeBodies(max_size)
@@ -191,7 +178,7 @@ class FlowMaster(controller.Master):
Loads a flow
"""
if isinstance(f, models.HTTPFlow):
if self.server and self.server.config.mode == "reverse":
if self.server and self.options.mode == "reverse":
f.request.host = self.server.config.upstream_server.address.host
f.request.port = self.server.config.upstream_server.address.port
f.request.scheme = self.server.config.upstream_server.scheme

55
mitmproxy/flow/options.py
View File

@@ -1,6 +1,7 @@
from __future__ import absolute_import, print_function, division
from mitmproxy import options
from typing import Tuple, Optional, Sequence # noqa
from mitmproxy import cmdline
APP_HOST = "mitm.it"
APP_PORT = 80
@@ -36,6 +37,33 @@ class Options(options.Options):
replay_ignore_params=(), # type: Sequence[str]
replay_ignore_payload_params=(), # type: Sequence[str]
replay_ignore_host=False, # type: bool
# Proxy options
auth_nonanonymous=False, # type: bool
auth_singleuser=None, # type: Optional[str]
auth_htpasswd=None, # type: Optional[str]
add_upstream_certs_to_client_chain=False, # type: bool
body_size_limit=None, # type: Optional[int]
cadir = cmdline.CA_DIR, # type: str
certs = (), # type: Sequence[Tuple[str, str]]
ciphers_client = cmdline.DEFAULT_CLIENT_CIPHERS, # type: str
ciphers_server = None, # type: Optional[str]
clientcerts = None, # type: Optional[str]
http2 = True, # type: bool
ignore_hosts = (), # type: Sequence[str]
listen_host = "", # type: str
listen_port = 8080, # type: int
mode = "regular", # type: str
no_upstream_cert = False, # type: bool
rawtcp = False, # type: bool
upstream_server = "", # type: str
upstream_auth = "", # type: str
ssl_version_client="secure", # type: str
ssl_version_server="secure", # type: str
ssl_verify_upstream_cert=False, # type: bool
ssl_verify_upstream_trusted_cadir=None, # type: str
ssl_verify_upstream_trusted_ca=None, # type: str
tcp_hosts = (), # type: Sequence[str]
):
# We could replace all assignments with clever metaprogramming,
# but type hints are a much more valueable asset.
@@ -66,4 +94,31 @@ class Options(options.Options):
self.replay_ignore_params = replay_ignore_params
self.replay_ignore_payload_params = replay_ignore_payload_params
self.replay_ignore_host = replay_ignore_host
# Proxy options
self.auth_nonanonymous = auth_nonanonymous
self.auth_singleuser = auth_singleuser
self.auth_htpasswd = auth_htpasswd
self.add_upstream_certs_to_client_chain = add_upstream_certs_to_client_chain
self.body_size_limit = body_size_limit
self.cadir = cadir
self.certs = certs
self.ciphers_client = ciphers_client
self.ciphers_server = ciphers_server
self.clientcerts = clientcerts
self.http2 = http2
self.ignore_hosts = ignore_hosts
self.listen_host = listen_host
self.listen_port = listen_port
self.mode = mode
self.no_upstream_cert = no_upstream_cert
self.rawtcp = rawtcp
self.upstream_server = upstream_server
self.upstream_auth = upstream_auth
self.ssl_version_client = ssl_version_client
self.ssl_version_server = ssl_version_server
self.ssl_verify_upstream_cert = ssl_verify_upstream_cert
self.ssl_verify_upstream_trusted_cadir = ssl_verify_upstream_trusted_cadir
self.ssl_verify_upstream_trusted_ca = ssl_verify_upstream_trusted_ca
self.tcp_hosts = tcp_hosts
super(Options, self).__init__()

82
mitmproxy/main.py
View File

@@ -41,14 +41,14 @@ def get_server(dummy_server, options):
sys.exit(1)
def process_options(parser, options):
if options.sysinfo:
def process_options(parser, options, args):
if args.sysinfo:
print(debug.sysinfo())
sys.exit(0)
if options.quiet:
options.verbose = 0
if args.quiet:
args.verbose = 0
debug.register_info_dumpers()
return config.process_proxy_options(parser, options)
return config.ProxyConfig(options)
def mitmproxy(args=None): # pragma: no cover
@@ -62,21 +62,22 @@ def mitmproxy(args=None): # pragma: no cover
assert_utf8_env()
parser = cmdline.mitmproxy()
options = parser.parse_args(args)
proxy_config = process_options(parser, options)
console_options = console.master.Options(**cmdline.get_common_options(options))
console_options.palette = options.palette
console_options.palette_transparent = options.palette_transparent
console_options.eventlog = options.eventlog
console_options.follow = options.follow
console_options.intercept = options.intercept
console_options.limit = options.limit
console_options.no_mouse = options.no_mouse
server = get_server(console_options.no_server, proxy_config)
args = parser.parse_args(args)
try:
console_options = console.master.Options(
**cmdline.get_common_options(args)
)
console_options.palette = args.palette
console_options.palette_transparent = args.palette_transparent
console_options.eventlog = args.eventlog
console_options.follow = args.follow
console_options.intercept = args.intercept
console_options.limit = args.limit
console_options.no_mouse = args.no_mouse
proxy_config = process_options(parser, console_options, args)
server = get_server(console_options.no_server, proxy_config)
m = console.master.ConsoleMaster(server, console_options)
except exceptions.OptionsError as e:
print("mitmproxy: %s" % e, file=sys.stderr)
@@ -93,19 +94,17 @@ def mitmdump(args=None): # pragma: no cover
version_check.check_pyopenssl_version()
parser = cmdline.mitmdump()
options = parser.parse_args(args)
proxy_config = process_options(parser, options)
if options.quiet:
options.flow_detail = 0
dump_options = dump.Options(**cmdline.get_common_options(options))
dump_options.flow_detail = options.flow_detail
dump_options.keepserving = options.keepserving
dump_options.filtstr = " ".join(options.args) if options.args else None
server = get_server(dump_options.no_server, proxy_config)
args = parser.parse_args(args)
if args.quiet:
args.flow_detail = 0
try:
dump_options = dump.Options(**cmdline.get_common_options(args))
dump_options.flow_detail = args.flow_detail
dump_options.keepserving = args.keepserving
dump_options.filtstr = " ".join(args.args) if args.args else None
proxy_config = process_options(parser, dump_options, args)
server = get_server(dump_options.no_server, proxy_config)
master = dump.DumpMaster(server, dump_options)
def cleankill(*args, **kwargs):
@@ -130,21 +129,20 @@ def mitmweb(args=None): # pragma: no cover
parser = cmdline.mitmweb()
options = parser.parse_args(args)
proxy_config = process_options(parser, options)
web_options = web.master.Options(**cmdline.get_common_options(options))
web_options.intercept = options.intercept
web_options.wdebug = options.wdebug
web_options.wiface = options.wiface
web_options.wport = options.wport
web_options.wsingleuser = options.wsingleuser
web_options.whtpasswd = options.whtpasswd
web_options.process_web_options(parser)
server = get_server(web_options.no_server, proxy_config)
args = parser.parse_args(args)
try:
web_options = web.master.Options(**cmdline.get_common_options(args))
web_options.intercept = args.intercept
web_options.wdebug = args.wdebug
web_options.wiface = args.wiface
web_options.wport = args.wport
web_options.wsingleuser = args.wsingleuser
web_options.whtpasswd = args.whtpasswd
web_options.process_web_options(parser)
proxy_config = process_options(parser, web_options, args)
server = get_server(web_options.no_server, proxy_config)
m = web.master.WebMaster(server, web_options)
except exceptions.OptionsError as e:
print("mitmweb: %s" % e, file=sys.stderr)

4
mitmproxy/onboarding/app.py
View File

@@ -47,7 +47,7 @@ class PEM(tornado.web.RequestHandler):
return config.CONF_BASENAME + "-ca-cert.pem"
def get(self):
p = os.path.join(self.request.master.server.config.cadir, self.filename)
p = os.path.join(self.request.master.options.cadir, self.filename)
self.set_header("Content-Type", "application/x-x509-ca-cert")
self.set_header(
"Content-Disposition",
@@ -65,7 +65,7 @@ class P12(tornado.web.RequestHandler):
return config.CONF_BASENAME + "-ca-cert.p12"
def get(self):
p = os.path.join(self.request.master.server.config.cadir, self.filename)
p = os.path.join(self.request.master.options.cadir, self.filename)
self.set_header("Content-Type", "application/x-pkcs12")
self.set_header(
"Content-Disposition",

2
mitmproxy/options.py
View File

@@ -52,7 +52,7 @@ class Options(object):
if attr in self._opts:
return self._opts[attr]
else:
raise AttributeError()
raise AttributeError("No such option: %s" % attr)
def __setattr__(self, attr, value):
if not self._initialized:

4
mitmproxy/protocol/base.py
View File

@@ -114,7 +114,7 @@ class ServerConnectionMixin(object):
def __init__(self, server_address=None):
super(ServerConnectionMixin, self).__init__()
self.server_conn = models.ServerConnection(server_address, (self.config.host, 0))
self.server_conn = models.ServerConnection(server_address, (self.config.options.listen_host, 0))
self.__check_self_connect()
def __check_self_connect(self):
@@ -125,7 +125,7 @@ class ServerConnectionMixin(object):
address = self.server_conn.address
if address:
self_connect = (
address.port == self.config.port and
address.port == self.config.options.listen_port and
address.host in ("localhost", "127.0.0.1", "::1")
)
if self_connect:

16
mitmproxy/protocol/http1.py
View File

@@ -12,12 +12,18 @@ class Http1Layer(http._HttpTransmissionLayer):
self.mode = mode
def read_request(self):
req = http1.read_request(self.client_conn.rfile, body_size_limit=self.config.body_size_limit)
req = http1.read_request(
self.client_conn.rfile, body_size_limit=self.config.options.body_size_limit
)
return models.HTTPRequest.wrap(req)
def read_request_body(self, request):
expected_size = http1.expected_http_body_size(request)
return http1.read_body(self.client_conn.rfile, expected_size, self.config.body_size_limit)
return http1.read_body(
self.client_conn.rfile,
expected_size,
self.config.options.body_size_limit
)
def send_request(self, request):
self.server_conn.wfile.write(http1.assemble_request(request))
@@ -29,7 +35,11 @@ class Http1Layer(http._HttpTransmissionLayer):
def read_response_body(self, request, response):
expected_size = http1.expected_http_body_size(request, response)
return http1.read_body(self.server_conn.rfile, expected_size, self.config.body_size_limit)
return http1.read_body(
self.server_conn.rfile,
expected_size,
self.config.options.body_size_limit
)
def send_response_headers(self, response):
raw = http1.assemble_response_head(response)

15
mitmproxy/protocol/http2.py
View File

@@ -183,14 +183,21 @@ class Http2Layer(base.Layer):
return True
def _handle_data_received(self, eid, event, source_conn):
if self.config.body_size_limit and self.streams[eid].queued_data_length > self.config.body_size_limit:
bsl = self.config.options.body_size_limit
if bsl and self.streams[eid].queued_data_length > bsl:
self.streams[eid].zombie = time.time()
source_conn.h2.safe_reset_stream(event.stream_id, h2.errors.REFUSED_STREAM)
self.log("HTTP body too large. Limit is {}.".format(self.config.body_size_limit), "info")
source_conn.h2.safe_reset_stream(
event.stream_id,
h2.errors.REFUSED_STREAM
)
self.log("HTTP body too large. Limit is {}.".format(bsl), "info")
else:
self.streams[eid].data_queue.put(event.data)
self.streams[eid].queued_data_length += len(event.data)
source_conn.h2.safe_increment_flow_control(event.stream_id, event.flow_controlled_length)
source_conn.h2.safe_increment_flow_control(
event.stream_id,
event.flow_controlled_length
)
return True
def _handle_stream_ended(self, eid):

10
mitmproxy/protocol/http_replay.py
View File

@@ -44,9 +44,9 @@ class RequestReplayThread(basethread.BaseThread):
if not self.flow.response:
# In all modes, we directly connect to the server displayed
if self.config.mode == "upstream":
if self.config.options.mode == "upstream":
server_address = self.config.upstream_server.address
server = models.ServerConnection(server_address, (self.config.host, 0))
server = models.ServerConnection(server_address, (self.config.options.listen_host, 0))
server.connect()
if r.scheme == "https":
connect_request = models.make_connect_request((r.data.host, r.port))
@@ -55,7 +55,7 @@ class RequestReplayThread(basethread.BaseThread):
resp = http1.read_response(
server.rfile,
connect_request,
body_size_limit=self.config.body_size_limit
body_size_limit=self.config.options.body_size_limit
)
if resp.status_code != 200:
raise exceptions.ReplayException("Upstream server refuses CONNECT request")
@@ -68,7 +68,7 @@ class RequestReplayThread(basethread.BaseThread):
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = models.ServerConnection(server_address, (self.config.host, 0))
server = models.ServerConnection(server_address, (self.config.options.listen_host, 0))
server.connect()
if r.scheme == "https":
server.establish_ssl(
@@ -83,7 +83,7 @@ class RequestReplayThread(basethread.BaseThread):
self.flow.response = models.HTTPResponse.wrap(http1.read_response(
server.rfile,
r,
body_size_limit=self.config.body_size_limit
body_size_limit=self.config.options.body_size_limit
))
if self.channel:
response_reply = self.channel.ask("response", self.flow)

18
mitmproxy/protocol/tls.py
View File

@@ -366,9 +366,9 @@ class TlsLayer(base.Layer):
# 2.5 The client did not sent a SNI value, we don't know the certificate subject.
client_tls_requires_server_connection = (
self._server_tls and
not self.config.no_upstream_cert and
not self.config.options.no_upstream_cert and
(
self.config.add_upstream_certs_to_client_chain or
self.config.options.add_upstream_certs_to_client_chain or
self._client_hello.alpn_protocols or
not self._client_hello.sni
)
@@ -473,7 +473,7 @@ class TlsLayer(base.Layer):
self.log("Establish TLS with client", "debug")
cert, key, chain_file = self._find_cert()
if self.config.add_upstream_certs_to_client_chain:
if self.config.options.add_upstream_certs_to_client_chain:
extra_certs = self.server_conn.server_certs
else:
extra_certs = None
@@ -483,7 +483,7 @@ class TlsLayer(base.Layer):
cert, key,
method=self.config.openssl_method_client,
options=self.config.openssl_options_client,
cipher_list=self.config.ciphers_client,
cipher_list=self.config.options.ciphers_client,
dhparams=self.config.certstore.dhparams,
chain_file=chain_file,
alpn_select_callback=self.__alpn_select_callback,
@@ -519,10 +519,10 @@ class TlsLayer(base.Layer):
alpn = [x for x in self._client_hello.alpn_protocols if not deprecated_http2_variant(x)]
else:
alpn = None
if alpn and b"h2" in alpn and not self.config.http2:
if alpn and b"h2" in alpn and not self.config.options.http2:
alpn.remove(b"h2")
ciphers_server = self.config.ciphers_server
ciphers_server = self.config.options.ciphers_server
if not ciphers_server:
ciphers_server = []
for id in self._client_hello.cipher_suites:
@@ -536,8 +536,8 @@ class TlsLayer(base.Layer):
method=self.config.openssl_method_server,
options=self.config.openssl_options_server,
verify_options=self.config.openssl_verification_mode_server,
ca_path=self.config.openssl_trusted_cadir_server,
ca_pemfile=self.config.openssl_trusted_ca_server,
ca_path=self.config.options.ssl_verify_upstream_trusted_cadir,
ca_pemfile=self.config.options.ssl_verify_upstream_trusted_ca,
cipher_list=ciphers_server,
alpn_protos=alpn,
)
@@ -595,7 +595,7 @@ class TlsLayer(base.Layer):
use_upstream_cert = (
self.server_conn and
self.server_conn.tls_established and
(not self.config.no_upstream_cert)
(not self.config.options.no_upstream_cert)
)
if use_upstream_cert:
upstream_cert = self.server_conn.cert

316
mitmproxy/proxy/config.py
View File

@@ -1,32 +1,21 @@
from __future__ import absolute_import, print_function, division
import base64
import collections
import os
import re
from netlib import strutils
import six
from OpenSSL import SSL
from OpenSSL import SSL, crypto
from mitmproxy import platform
from mitmproxy import exceptions
from netlib import certutils
from netlib import human
from netlib import tcp
from netlib.http import authentication
from netlib.http import url
CONF_BASENAME = "mitmproxy"
CA_DIR = "~/.mitmproxy"
# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default.
# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old
DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" \
"ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" \
"ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" \
"ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" \
"DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" \
"DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" \
"AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" \
"HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" \
"!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"
class HostMatcher(object):
@@ -55,187 +44,148 @@ class HostMatcher(object):
ServerSpec = collections.namedtuple("ServerSpec", "scheme address")
def parse_server_spec(spec):
try:
p = url.parse(spec)
if p[0] not in (b"http", b"https"):
raise ValueError()
except ValueError:
raise exceptions.OptionsError(
"Invalid server specification: %s" % spec
)
host, port = p[1:3]
address = tcp.Address((host.decode("ascii"), port))
scheme = p[0].decode("ascii").lower()
return ServerSpec(scheme, address)
def parse_upstream_auth(auth):
pattern = re.compile(".+:")
if pattern.search(auth) is None:
raise exceptions.OptionsError(
"Invalid upstream auth specification: %s" % auth
)
return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
class ProxyConfig:
def __init__(
self,
host='',
port=8080,
cadir=CA_DIR,
clientcerts=None,
no_upstream_cert=False,
body_size_limit=None,
mode="regular",
upstream_server=None,
upstream_auth=None,
authenticator=None,
ignore_hosts=tuple(),
tcp_hosts=tuple(),
http2=True,
rawtcp=False,
ciphers_client=DEFAULT_CLIENT_CIPHERS,
ciphers_server=None,
certs=tuple(),
ssl_version_client="secure",
ssl_version_server="secure",
ssl_verify_upstream_cert=False,
ssl_verify_upstream_trusted_cadir=None,
ssl_verify_upstream_trusted_ca=None,
add_upstream_certs_to_client_chain=False,
):
self.host = host
self.port = port
self.ciphers_client = ciphers_client
self.ciphers_server = ciphers_server
self.clientcerts = clientcerts
self.no_upstream_cert = no_upstream_cert
self.body_size_limit = body_size_limit
self.mode = mode
if upstream_server:
self.upstream_server = ServerSpec(upstream_server[0], tcp.Address.wrap(upstream_server[1]))
self.upstream_auth = upstream_auth
else:
self.upstream_server = None
self.upstream_auth = None
def __init__(self, options):
self.options = options
self.check_ignore = HostMatcher(ignore_hosts)
self.check_tcp = HostMatcher(tcp_hosts)
self.http2 = http2
self.rawtcp = rawtcp
self.authenticator = authenticator
self.cadir = os.path.expanduser(cadir)
self.certstore = certutils.CertStore.from_store(
self.cadir,
CONF_BASENAME
self.authenticator = None
self.check_ignore = None
self.check_tcp = None
self.certstore = None
self.clientcerts = None
self.openssl_verification_mode_server = None
self.configure(options)
options.changed.connect(self.configure)
def configure(self, options):
conflict = all(
[
options.add_upstream_certs_to_client_chain,
options.ssl_verify_upstream_cert
]
)
for spec, cert in certs:
self.certstore.add_cert_file(spec, cert)
if conflict:
raise exceptions.OptionsError(
"The verify-upstream-cert and add-upstream-certs-to-client-chain "
"options are mutually exclusive. If upstream certificates are verified "
"then extra upstream certificates are not available for inclusion "
"to the client chain."
)
self.openssl_method_client, self.openssl_options_client = \
tcp.sslversion_choices[ssl_version_client]
self.openssl_method_server, self.openssl_options_server = \
tcp.sslversion_choices[ssl_version_server]
if ssl_verify_upstream_cert:
if options.ssl_verify_upstream_cert:
self.openssl_verification_mode_server = SSL.VERIFY_PEER
else:
self.openssl_verification_mode_server = SSL.VERIFY_NONE
self.openssl_trusted_cadir_server = ssl_verify_upstream_trusted_cadir
self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca
self.add_upstream_certs_to_client_chain = add_upstream_certs_to_client_chain
self.check_ignore = HostMatcher(options.ignore_hosts)
self.check_tcp = HostMatcher(options.tcp_hosts)
def process_proxy_options(parser, options):
body_size_limit = options.body_size_limit
if body_size_limit:
body_size_limit = human.parse_size(body_size_limit)
self.openssl_method_client, self.openssl_options_client = \
tcp.sslversion_choices[options.ssl_version_client]
self.openssl_method_server, self.openssl_options_server = \
tcp.sslversion_choices[options.ssl_version_server]
c = 0
mode, upstream_server, upstream_auth = "regular", None, None
if options.transparent_proxy:
c += 1
if not platform.resolver:
return parser.error("Transparent mode not supported on this platform.")
mode = "transparent"
if options.socks_proxy:
c += 1
mode = "socks5"
if options.reverse_proxy:
c += 1
mode = "reverse"
upstream_server = options.reverse_proxy
if options.upstream_proxy:
c += 1
mode = "upstream"
upstream_server = options.upstream_proxy
upstream_auth = options.upstream_auth
if c > 1:
return parser.error(
"Transparent, SOCKS5, reverse and upstream proxy mode "
"are mutually exclusive. Read the docs on proxy modes to understand why."
)
if options.add_upstream_certs_to_client_chain and options.no_upstream_cert:
return parser.error(
"The no-upstream-cert and add-upstream-certs-to-client-chain "
"options are mutually exclusive. If no-upstream-cert is enabled "
"then the upstream certificate is not retrieved before generating "
"the client certificate chain."
)
if options.add_upstream_certs_to_client_chain and options.ssl_verify_upstream_cert:
return parser.error(
"The verify-upstream-cert and add-upstream-certs-to-client-chain "
"options are mutually exclusive. If upstream certificates are verified "
"then extra upstream certificates are not available for inclusion "
"to the client chain."
)
if options.clientcerts:
options.clientcerts = os.path.expanduser(options.clientcerts)
if not os.path.exists(options.clientcerts):
return parser.error(
"Client certificate path does not exist: %s" % options.clientcerts
certstore_path = os.path.expanduser(options.cadir)
if not os.path.exists(os.path.dirname(certstore_path)):
raise exceptions.OptionsError(
"Certificate Authority parent directory does not exist: %s" %
os.path.dirname(options.cadir)
)
if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd:
self.certstore = certutils.CertStore.from_store(
certstore_path,
CONF_BASENAME
)
if options.transparent_proxy:
return parser.error("Proxy Authentication not supported in transparent mode.")
if options.socks_proxy:
return parser.error(
"Proxy Authentication not supported in SOCKS mode. "
"https://github.com/mitmproxy/mitmproxy/issues/738"
)
if options.auth_singleuser:
if len(options.auth_singleuser.split(':')) != 2:
return parser.error(
"Invalid single-user specification. Please use the format username:password"
if options.clientcerts:
clientcerts = os.path.expanduser(options.clientcerts)
if not os.path.exists(clientcerts):
raise exceptions.OptionsError(
"Client certificate path does not exist: %s" %
options.clientcerts
)
self.clientcerts = clientcerts
for spec, cert in options.certs:
cert = os.path.expanduser(cert)
if not os.path.exists(cert):
raise exceptions.OptionsError(
"Certificate file does not exist: %s" % cert
)
username, password = options.auth_singleuser.split(':')
password_manager = authentication.PassManSingleUser(username, password)
elif options.auth_nonanonymous:
password_manager = authentication.PassManNonAnon()
elif options.auth_htpasswd:
try:
password_manager = authentication.PassManHtpasswd(
options.auth_htpasswd)
except ValueError as v:
return parser.error(v)
authenticator = authentication.BasicProxyAuth(password_manager, "mitmproxy")
else:
authenticator = authentication.NullProxyAuth(None)
self.certstore.add_cert_file(spec, cert)
except crypto.Error:
raise exceptions.OptionsError(
"Invalid certificate format: %s" % cert
)
certs = []
for i in options.certs:
parts = i.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
parts[1] = os.path.expanduser(parts[1])
if not os.path.exists(parts[1]):
parser.error("Certificate file does not exist: %s" % parts[1])
certs.append(parts)
self.upstream_server = None
self.upstream_auth = None
if options.upstream_server:
self.upstream_server = parse_server_spec(options.upstream_server)
if options.upstream_auth:
self.upstream_auth = parse_upstream_auth(options.upstream_auth)
return ProxyConfig(
host=options.addr,
port=options.port,
cadir=options.cadir,
clientcerts=options.clientcerts,
no_upstream_cert=options.no_upstream_cert,
body_size_limit=body_size_limit,
mode=mode,
upstream_server=upstream_server,
upstream_auth=upstream_auth,
ignore_hosts=options.ignore_hosts,
tcp_hosts=options.tcp_hosts,
http2=options.http2,
rawtcp=options.rawtcp,
authenticator=authenticator,
ciphers_client=options.ciphers_client,
ciphers_server=options.ciphers_server,
certs=tuple(certs),
ssl_version_client=options.ssl_version_client,
ssl_version_server=options.ssl_version_server,
ssl_verify_upstream_cert=options.ssl_verify_upstream_cert,
ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir,
ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca,
add_upstream_certs_to_client_chain=options.add_upstream_certs_to_client_chain,
)
self.authenticator = authentication.NullProxyAuth(None)
needsauth = any(
[
options.auth_nonanonymous,
options.auth_singleuser,
options.auth_htpasswd
]
)
if needsauth:
if options.mode == "transparent":
raise exceptions.OptionsError(
"Proxy Authentication not supported in transparent mode."
)
elif options.mode == "socks5":
raise exceptions.OptionsError(
"Proxy Authentication not supported in SOCKS mode. "
"https://github.com/mitmproxy/mitmproxy/issues/738"
)
elif options.auth_singleuser:
parts = options.auth_singleuser.split(':')
if len(parts) != 2:
raise exceptions.OptionsError(
"Invalid single-user specification. "
"Please use the format username:password"
)
password_manager = authentication.PassManSingleUser(*parts)
elif options.auth_nonanonymous:
password_manager = authentication.PassManNonAnon()
elif options.auth_htpasswd:
try:
password_manager = authentication.PassManHtpasswd(
options.auth_htpasswd
)
except ValueError as v:
raise exceptions.OptionsError(str(v))
self.authenticator = authentication.BasicProxyAuth(
password_manager,
"mitmproxy"
)

2
mitmproxy/proxy/root_context.py
View File

@@ -102,7 +102,7 @@ class RootContext(object):
# expect A-Za-z
all(65 <= x <= 90 or 97 <= x <= 122 for x in six.iterbytes(d))
)
if self.config.rawtcp and not is_ascii:
if self.config.options.rawtcp and not is_ascii:
return protocol.RawTCPLayer(top_layer)
# 7. Assume HTTP1 by default

6
mitmproxy/proxy/server.py
View File

@@ -41,7 +41,9 @@ class ProxyServer(tcp.TCPServer):
"""
self.config = config
try:
super(ProxyServer, self).__init__((config.host, config.port))
super(ProxyServer, self).__init__(
(config.options.listen_host, config.options.listen_port)
)
except socket.error as e:
six.reraise(
exceptions.ServerException,
@@ -83,7 +85,7 @@ class ConnectionHandler(object):
self.channel
)
mode = self.config.mode
mode = self.config.options.mode
if mode == "upstream":
return modes.HttpUpstreamProxy(
root_ctx,

14
mitmproxy/web/app.py
View File

@@ -336,12 +336,12 @@ class Settings(RequestHandler):
self.write(dict(
data=dict(
version=version.VERSION,
mode=str(self.master.server.config.mode),
mode=str(self.master.options.mode),
intercept=self.state.intercept_txt,
showhost=self.master.options.showhost,
no_upstream_cert=self.master.server.config.no_upstream_cert,
rawtcp=self.master.server.config.rawtcp,
http2=self.master.server.config.http2,
no_upstream_cert=self.master.options.no_upstream_cert,
rawtcp=self.master.options.rawtcp,
http2=self.master.options.http2,
anticache=self.master.options.anticache,
anticomp=self.master.options.anticomp,
stickyauth=self.master.options.stickyauth,
@@ -360,13 +360,13 @@ class Settings(RequestHandler):
self.master.options.showhost = v
update[k] = v
elif k == "no_upstream_cert":
self.master.server.config.no_upstream_cert = v
self.master.options.no_upstream_cert = v
update[k] = v
elif k == "rawtcp":
self.master.server.config.rawtcp = v
self.master.options.rawtcp = v
update[k] = v
elif k == "http2":
self.master.server.config.http2 = v
self.master.options.http2 = v
update[k] = v
elif k == "anticache":
self.master.options.anticache = v

29
test/mitmproxy/test_cmdline.py
View File

@@ -1,5 +1,4 @@
import argparse
import base64
from mitmproxy import cmdline
from . import tutils
@@ -36,34 +35,6 @@ def test_parse_replace_hook():
)
def test_parse_server_spec():
tutils.raises("Invalid server specification", cmdline.parse_server_spec, "")
assert cmdline.parse_server_spec(
"http://foo.com:88") == (b"http", (b"foo.com", 88))
assert cmdline.parse_server_spec(
"http://foo.com") == (b"http", (b"foo.com", 80))
assert cmdline.parse_server_spec(
"https://foo.com") == (b"https", (b"foo.com", 443))
tutils.raises(
"Invalid server specification",
cmdline.parse_server_spec,
"foo.com")
tutils.raises(
"Invalid server specification",
cmdline.parse_server_spec,
"http://")
def test_parse_upstream_auth():
tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, "")
tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, ":")
tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, ":test")
assert cmdline.parse_upstream_auth(
"test:test") == b"Basic" + b" " + base64.b64encode(b"test:test")
assert cmdline.parse_upstream_auth(
"test:") == b"Basic" + b" " + base64.b64encode(b"test:")
def test_parse_setheaders():
x = cmdline.parse_setheader("/foo/bar/voing")
assert x == ("foo", "bar", "voing")

19
test/mitmproxy/test_flow.py
View File

@@ -4,6 +4,7 @@ import io
import netlib.utils
from netlib.http import Headers
from mitmproxy import filt, controller, flow
from mitmproxy.flow import options
from mitmproxy.contrib import tnetstring
from mitmproxy.exceptions import FlowReadException
from mitmproxy.models import Error
@@ -11,7 +12,6 @@ from mitmproxy.models import Flow
from mitmproxy.models import HTTPFlow
from mitmproxy.models import HTTPRequest
from mitmproxy.models import HTTPResponse
from mitmproxy.proxy.config import HostMatcher
from mitmproxy.proxy import ProxyConfig
from mitmproxy.proxy.server import DummyServer
from mitmproxy.models.connections import ClientConnection
@@ -639,11 +639,12 @@ class TestSerialize:
def test_load_flows_reverse(self):
r = self._treader()
s = flow.State()
conf = ProxyConfig(
opts = options.Options(
mode="reverse",
upstream_server=("https", ("use-this-domain", 80))
upstream_server="https://use-this-domain"
)
fm = flow.FlowMaster(None, DummyServer(conf), s)
conf = ProxyConfig(opts)
fm = flow.FlowMaster(opts, DummyServer(conf), s)
fm.load_flows(r)
assert s.flows[0].request.host == "use-this-domain"
@@ -688,14 +689,6 @@ class TestSerialize:
class TestFlowMaster:
def test_getset_ignore(self):
p = mock.Mock()
p.config.check_ignore = HostMatcher()
fm = flow.FlowMaster(None, p, flow.State())
assert not fm.get_ignore_filter()
fm.set_ignore_filter(["^apple\.com:", ":443$"])
assert fm.get_ignore_filter()
def test_replay(self):
s = flow.State()
fm = flow.FlowMaster(None, None, s)
@@ -753,7 +746,7 @@ class TestFlowMaster:
pb = [tutils.tflow(resp=True), f]
fm = flow.FlowMaster(
flow.options.Options(),
DummyServer(ProxyConfig()),
DummyServer(ProxyConfig(options.Options())),
s
)
assert not fm.start_server_playback(

21
test/mitmproxy/test_protocol_http2.py
View File

@@ -9,6 +9,7 @@ import traceback
import h2
from mitmproxy.flow import options
from mitmproxy.proxy.config import ProxyConfig
from mitmproxy.cmdline import APP_HOST, APP_PORT
@@ -88,9 +89,11 @@ class _Http2TestBase(object):
@classmethod
def setup_class(cls):
cls.config = ProxyConfig(**cls.get_proxy_config())
cls.masteroptions = options.Options()
cnf, opts = cls.get_proxy_config()
cls.config = ProxyConfig(opts, **cnf)
tmaster = tservers.TestMaster(cls.config)
tmaster = tservers.TestMaster(opts, cls.config)
tmaster.start_app(APP_HOST, APP_PORT)
cls.proxy = tservers.ProxyThread(tmaster)
cls.proxy.start()
@@ -101,12 +104,10 @@ class _Http2TestBase(object):
@classmethod
def get_proxy_config(cls):
cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy")
return dict(
no_upstream_cert=False,
cadir=cls.cadir,
authenticator=None,
)
opts = options.Options(listen_port=0, no_upstream_cert=False)
opts.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy")
d = dict()
return d, opts
@property
def master(self):
@@ -118,8 +119,6 @@ class _Http2TestBase(object):
self.server.server.handle_server_event = self.handle_server_event
def _setup_connection(self):
self.config.http2 = True
client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port))
client.connect()
@@ -587,7 +586,7 @@ class TestBodySizeLimit(_Http2Test):
return True
def test_body_size_limit(self):
self.config.body_size_limit = 20
self.config.options.body_size_limit = 20
client, h2_conn = self._setup_connection()

56
test/mitmproxy/test_proxy.py
View File

@@ -4,9 +4,10 @@ from OpenSSL import SSL
from mitmproxy import cmdline
from mitmproxy.proxy import ProxyConfig
from mitmproxy.proxy.config import process_proxy_options
from mitmproxy.models.connections import ServerConnection
from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler
from mitmproxy.flow import options
from mitmproxy.proxy import config
from netlib.exceptions import TcpDisconnect
from pathod import test
from netlib.http import http1
@@ -58,8 +59,10 @@ class TestProcessProxyOptions:
def p(self, *args):
parser = tutils.MockParser()
cmdline.common_options(parser)
opts = parser.parse_args(args=args)
return parser, process_proxy_options(parser, opts)
args = parser.parse_args(args=args)
opts = cmdline.get_common_options(args)
pconf = config.ProxyConfig(options.Options(**opts))
return parser, pconf
def assert_err(self, err, *args):
tutils.raises(err, self.p, *args)
@@ -82,24 +85,29 @@ class TestProcessProxyOptions:
@mock.patch("mitmproxy.platform.resolver")
def test_modes(self, _):
self.assert_noerr("-R", "http://localhost")
self.assert_err("expected one argument", "-R")
self.assert_err("Invalid server specification", "-R", "reverse")
self.assert_noerr("-T")
self.assert_noerr("-U", "http://localhost")
self.assert_err("expected one argument", "-U")
self.assert_err("Invalid server specification", "-U", "upstream")
self.assert_noerr("--upstream-auth", "test:test")
self.assert_err("expected one argument", "--upstream-auth")
self.assert_err("Invalid upstream auth specification", "--upstream-auth", "test")
self.assert_err("mutually exclusive", "-R", "http://localhost", "-T")
# self.assert_noerr("-R", "http://localhost")
# self.assert_err("expected one argument", "-R")
# self.assert_err("Invalid server specification", "-R", "reverse")
#
# self.assert_noerr("-T")
#
# self.assert_noerr("-U", "http://localhost")
# self.assert_err("expected one argument", "-U")
# self.assert_err("Invalid server specification", "-U", "upstream")
#
# self.assert_noerr("--upstream-auth", "test:test")
# self.assert_err("expected one argument", "--upstream-auth")
self.assert_err(
"Invalid upstream auth specification", "--upstream-auth", "test"
)
# self.assert_err("mutually exclusive", "-R", "http://localhost", "-T")
def test_socks_auth(self):
self.assert_err("Proxy Authentication not supported in SOCKS mode.", "--socks", "--nonanonymous")
self.assert_err(
"Proxy Authentication not supported in SOCKS mode.",
"--socks",
"--nonanonymous"
)
def test_client_certs(self):
with tutils.tmpdir() as cadir:
@@ -145,12 +153,12 @@ class TestProcessProxyOptions:
def test_upstream_trusted_cadir(self):
expected_dir = "/path/to/a/ca/dir"
p = self.assert_noerr("--upstream-trusted-cadir", expected_dir)
assert p.openssl_trusted_cadir_server == expected_dir
assert p.options.ssl_verify_upstream_trusted_cadir == expected_dir
def test_upstream_trusted_ca(self):
expected_file = "/path/to/a/cert/file"
p = self.assert_noerr("--upstream-trusted-ca", expected_file)
assert p.openssl_trusted_ca_server == expected_file
assert p.options.ssl_verify_upstream_trusted_ca == expected_file
class TestProxyServer:
@@ -159,13 +167,13 @@ class TestProxyServer:
@tutils.skip_windows
def test_err(self):
conf = ProxyConfig(
port=1
options.Options(listen_port=1),
)
tutils.raises("error starting proxy server", ProxyServer, conf)
def test_err_2(self):
conf = ProxyConfig(
host="invalidhost"
options.Options(listen_host="invalidhost"),
)
tutils.raises("error starting proxy server", ProxyServer, conf)
@@ -184,7 +192,7 @@ class TestConnectionHandler:
config = mock.Mock()
root_layer = mock.Mock()
root_layer.side_effect = RuntimeError
config.mode.return_value = root_layer
config.options.mode.return_value = root_layer
channel = mock.Mock()
def ask(_, x):

48
test/mitmproxy/test_proxy_config.py Normal file
View File

@@ -0,0 +1,48 @@
from . import tutils
import base64
from mitmproxy.proxy import config
def test_parse_server_spec():
tutils.raises(
"Invalid server specification", config.parse_server_spec, ""
)
assert config.parse_server_spec("http://foo.com:88") == (
"http", ("foo.com", 88)
)
assert config.parse_server_spec("http://foo.com") == (
"http", ("foo.com", 80)
)
assert config.parse_server_spec("https://foo.com") == (
"https", ("foo.com", 443)
)
tutils.raises(
"Invalid server specification",
config.parse_server_spec,
"foo.com"
)
tutils.raises(
"Invalid server specification",
config.parse_server_spec,
"http://"
)
def test_parse_upstream_auth():
tutils.raises(
"Invalid upstream auth specification",
config.parse_upstream_auth,
""
)
tutils.raises(
"Invalid upstream auth specification",
config.parse_upstream_auth,
":"
)
tutils.raises(
"Invalid upstream auth specification",
config.parse_upstream_auth,
":test"
)
assert config.parse_upstream_auth("test:test") == b"Basic" + b" " + base64.b64encode(b"test:test")
assert config.parse_upstream_auth("test:") == b"Basic" + b" " + base64.b64encode(b"test:")

56
test/mitmproxy/test_server.py
View File

@@ -15,7 +15,7 @@ from pathod import pathoc, pathod
from mitmproxy.builtins import script
from mitmproxy import controller
from mitmproxy.proxy.config import HostMatcher
from mitmproxy.proxy.config import HostMatcher, parse_server_spec
from mitmproxy.models import Error, HTTPResponse, HTTPFlow
from . import tutils, tservers
@@ -298,13 +298,8 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
class TestHTTPAuth(tservers.HTTPProxyTest):
authenticator = http.authentication.BasicProxyAuth(
http.authentication.PassManSingleUser(
"test",
"test"),
"realm")
def test_auth(self):
self.master.options.auth_singleuser = "test:test"
assert self.pathod("202").status_code == 407
p = self.pathoc()
ret = p.request("""
@@ -368,15 +363,17 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest):
])
def test_verification_w_cadir(self):
self.config.openssl_verification_mode_server = SSL.VERIFY_PEER
self.config.openssl_trusted_cadir_server = tutils.test_data.path(
"data/trusted-cadir/")
self.config.options.update(
ssl_verify_upstream_cert = True,
ssl_verify_upstream_trusted_cadir = tutils.test_data.path(
"data/trusted-cadir/"
)
)
self.pathoc()
def test_verification_w_pemfile(self):
self.config.openssl_verification_mode_server = SSL.VERIFY_PEER
self.config.openssl_trusted_ca_server = tutils.test_data.path(
self.config.options.ssl_verify_upstream_trusted_ca = tutils.test_data.path(
"data/trusted-cadir/trusted-ca.pem")
self.pathoc()
@@ -401,23 +398,29 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest):
def test_default_verification_w_bad_cert(self):
"""Should use no verification."""
self.config.openssl_trusted_ca_server = tutils.test_data.path(
"data/trusted-cadir/trusted-ca.pem")
self.config.options.update(
ssl_verify_upstream_trusted_ca = tutils.test_data.path(
"data/trusted-cadir/trusted-ca.pem"
)
)
assert self._request().status_code == 242
def test_no_verification_w_bad_cert(self):
self.config.openssl_verification_mode_server = SSL.VERIFY_NONE
self.config.openssl_trusted_ca_server = tutils.test_data.path(
"data/trusted-cadir/trusted-ca.pem")
self.config.options.update(
ssl_verify_upstream_cert = False,
ssl_verify_upstream_trusted_ca = tutils.test_data.path(
"data/trusted-cadir/trusted-ca.pem"
)
)
assert self._request().status_code == 242
def test_verification_w_bad_cert(self):
self.config.openssl_verification_mode_server = SSL.VERIFY_PEER
self.config.openssl_trusted_ca_server = tutils.test_data.path(
"data/trusted-cadir/trusted-ca.pem")
self.config.options.update(
ssl_verify_upstream_cert = True,
ssl_verify_upstream_trusted_ca = tutils.test_data.path(
"data/trusted-cadir/trusted-ca.pem"
)
)
assert self._request().status_code == 502
@@ -484,9 +487,10 @@ class TestHttps2Http(tservers.ReverseProxyTest):
@classmethod
def get_proxy_config(cls):
d = super(TestHttps2Http, cls).get_proxy_config()
d["upstream_server"] = ("http", d["upstream_server"][1])
return d
d, opts = super(TestHttps2Http, cls).get_proxy_config()
s = parse_server_spec(opts.upstream_server)
opts.upstream_server = "http://%s" % s.address
return d, opts
def pathoc(self, ssl, sni=None):
"""

69
test/mitmproxy/tservers.py
View File

@@ -32,11 +32,10 @@ def errapp(environ, start_response):
class TestMaster(flow.FlowMaster):
def __init__(self, config):
config.port = 0
def __init__(self, opts, config):
s = ProxyServer(config)
state = flow.State()
flow.FlowMaster.__init__(self, options.Options(), s, state)
flow.FlowMaster.__init__(self, opts, s, state)
self.addons.add(*builtins.default_addons())
self.apps.add(testapp, "testapp", 80)
self.apps.add(errapp, "errapp", 80)
@@ -55,7 +54,8 @@ class ProxyThread(threading.Thread):
threading.Thread.__init__(self)
self.tmaster = tmaster
self.name = "ProxyThread (%s:%s)" % (
tmaster.server.address.host, tmaster.server.address.port)
tmaster.server.address.host, tmaster.server.address.port
)
controller.should_exit = False
@property
@@ -78,7 +78,6 @@ class ProxyTestBase(object):
ssl = None
ssloptions = False
no_upstream_cert = False
authenticator = None
masterclass = TestMaster
add_upstream_certs_to_client_chain = False
@@ -91,9 +90,9 @@ class ProxyTestBase(object):
ssl=cls.ssl,
ssloptions=cls.ssloptions)
cls.config = ProxyConfig(**cls.get_proxy_config())
tmaster = cls.masterclass(cls.config)
cnf, opts = cls.get_proxy_config()
cls.config = ProxyConfig(opts, **cnf)
tmaster = cls.masterclass(opts, cls.config)
tmaster.start_app(APP_HOST, APP_PORT)
cls.proxy = ProxyThread(tmaster)
cls.proxy.start()
@@ -120,11 +119,12 @@ class ProxyTestBase(object):
@classmethod
def get_proxy_config(cls):
cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy")
return dict(
cnf = dict()
return cnf, options.Options(
listen_port=0,
cadir=cls.cadir,
no_upstream_cert = cls.no_upstream_cert,
cadir = cls.cadir,
authenticator = cls.authenticator,
add_upstream_certs_to_client_chain = cls.add_upstream_certs_to_client_chain,
add_upstream_certs_to_client_chain=cls.add_upstream_certs_to_client_chain
)
@@ -199,9 +199,9 @@ class TransparentProxyTest(ProxyTestBase):
@classmethod
def get_proxy_config(cls):
d = ProxyTestBase.get_proxy_config()
d["mode"] = "transparent"
return d
d, opts = ProxyTestBase.get_proxy_config()
opts.mode = "transparent"
return d, opts
def pathod(self, spec, sni=None):
"""
@@ -231,13 +231,17 @@ class ReverseProxyTest(ProxyTestBase):
@classmethod
def get_proxy_config(cls):
d = ProxyTestBase.get_proxy_config()
d["upstream_server"] = (
"https" if cls.ssl else "http",
("127.0.0.1", cls.server.port)
d, opts = ProxyTestBase.get_proxy_config()
opts.upstream_server = "".join(
[
"https" if cls.ssl else "http",
"://",
"127.0.0.1:",
str(cls.server.port)
]
)
d["mode"] = "reverse"
return d
opts.mode = "reverse"
return d, opts
def pathoc(self, sni=None):
"""
@@ -266,9 +270,9 @@ class SocksModeTest(HTTPProxyTest):
@classmethod
def get_proxy_config(cls):
d = ProxyTestBase.get_proxy_config()
d["mode"] = "socks5"
return d
d, opts = ProxyTestBase.get_proxy_config()
opts.mode = "socks5"
return d, opts
class ChainProxyTest(ProxyTestBase):
@@ -287,15 +291,16 @@ class ChainProxyTest(ProxyTestBase):
cls.chain = []
super(ChainProxyTest, cls).setup_class()
for _ in range(cls.n):
config = ProxyConfig(**cls.get_proxy_config())
tmaster = cls.masterclass(config)
cnf, opts = cls.get_proxy_config()
config = ProxyConfig(opts, **cnf)
tmaster = cls.masterclass(opts, config)
proxy = ProxyThread(tmaster)
proxy.start()
cls.chain.insert(0, proxy)
# Patch the orginal proxy to upstream mode
cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(
**cls.get_proxy_config())
cnf, opts = cls.get_proxy_config()
cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(opts, **cnf)
@classmethod
def teardown_class(cls):
@@ -311,13 +316,13 @@ class ChainProxyTest(ProxyTestBase):
@classmethod
def get_proxy_config(cls):
d = super(ChainProxyTest, cls).get_proxy_config()
d, opts = super(ChainProxyTest, cls).get_proxy_config()
if cls.chain: # First proxy is in normal mode.
d.update(
opts.update(
mode="upstream",
upstream_server=("http", ("127.0.0.1", cls.chain[0].port))
upstream_server="http://127.0.0.1:%s" % cls.chain[0].port
)
return d
return d, opts
class HTTPUpstreamProxyTest(ChainProxyTest, HTTPProxyTest):