kylefang/mitmproxy
1
0
Fork 0
mirror of https://github.com/zhigang1992/mitmproxy.git synced 2026-04-28 20:24:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #342 from mitmproxy/server_change_api

Browse Source 
Server change api
This commit is contained in:
Aldo Cortesi
2014-09-07 12:59:35 +12:00
parent 0e0cff638c fdd7b2f108
commit 3d62e90dbf
34 changed files with 780 additions and 712 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
doc-src/scripting/inlinescripts.html
View File

@@ -29,46 +29,45 @@ The new header will be added to all responses passing through the proxy.
Called once on startup, before any other events.
### clientconnect(ScriptContext, ClientConnect)
### clientconnect(ScriptContext, ConnectionHandler)
Called when a client initiates a connection to the proxy. Note that
a connection can correspond to multiple HTTP requests.
### serverconnect(ScriptContext, ServerConnection)
### serverconnect(ScriptContext, ConnectionHandler)
Called when the proxy initiates a connection to the target server. Note that
a connection can correspond to multiple HTTP requests.
### request(ScriptContext, Flow)
### request(ScriptContext, HTTPFlow)
Called when a client request has been received. The __Flow__ object is
Called when a client request has been received. The __HTTPFlow__ object is
guaranteed to have a non-None __request__ attribute.
### responseheaders(ScriptContext, Flow)
### responseheaders(ScriptContext, HTTPFlow)
Called when the headers of a server response have been received.
This will always be called before the response hook.
The __Flow__ object is guaranteed to have non-None __request__ and
__response__ attributes. __response.content__ will not be valid,
The __HTTPFlow__ object is guaranteed to have non-None __request__ and
__response__ attributes. __response.content__ will be None,
as the response body has not been read yet.
### response(ScriptContext, Flow)
### response(ScriptContext, HTTPFlow)
Called when a server response has been received. The __Flow__ object is
Called when a server response has been received. The __HTTPFlow__ object is
guaranteed to have non-None __request__ and __response__ attributes.
Note that if response streaming is enabled for this response,
__response.content__ will not contain the response body.
### error(ScriptContext, Flow)
### error(ScriptContext, HTTPFlow)
Called when a flow error has occurred, e.g. invalid server responses, or
interrupted connections. This is distinct from a valid server HTTP error
response, which is simply a response with an HTTP error code. The __Flow__
response, which is simply a response with an HTTP error code. The __HTTPFlow__
object is guaranteed to have non-None __request__ and __error__ attributes.
### clientdisconnect(ScriptContext, ClientDisconnect)
### clientdisconnect(ScriptContext, ConnectionHandler)
Called when a client disconnects from the proxy.
@@ -96,22 +95,10 @@ The main classes you will deal with in writing mitmproxy scripts are:
<th>libmproxy.proxy.connection.ServerConnection</th>
<td>Describes a server connection.</td>
</tr>
<tr>
<th>libmproxy.protocol.primitives.Error</th>
<td>A communications error.</td>
</tr>
<tr>
<th>libmproxy.protocol.http.HTTPFlow</th>
<td>A collection of objects representing a single HTTP transaction.</td>
</tr>
<tr>
<th>libmproxy.flow.ODict</th>
<td>A dictionary-like object for managing sets of key/value data. There
is also a variant called CaselessODict that ignores key case for some
calls (used mainly for headers).
</td>
</tr>
<tr>
<th>libmproxy.protocol.http.HTTPResponse</th>
<td>An HTTP response.</td>
@@ -120,10 +107,22 @@ The main classes you will deal with in writing mitmproxy scripts are:
<th>libmproxy.protocol.http.HTTPRequest</th>
<td>An HTTP request.</td>
</tr>
<tr>
<th>libmproxy.protocol.primitives.Error</th>
<td>A communications error.</td>
</tr>
<tr>
<th>libmproxy.script.ScriptContext</th>
<td> A handle for interacting with mitmproxy's from within scripts.</td>
</tr>
<tr>
<th>libmproxy.flow.ODict</th>
<td>A dictionary-like object for managing sets of key/value data. There
is also a variant called CaselessODict that ignores key case for some
calls (used mainly for headers).
</td>
</tr>
<tr>
<th>libmproxy.certutils.SSLCert</th>
<td>Exposes information SSL certificates.</td>
@@ -161,9 +160,9 @@ flows from a file (see the "scripted data transformation" example on the
one-shot script on a single flow through the _|_ (pipe) shortcut in mitmproxy.
In this case, there are no client connections, and the events are run in the
following order: __start__, __request__, __response__, __error__, __done__. If
following order: __start__, __request__, __responseheaders__, __response__, __error__, __done__. If
the flow doesn't have a __response__ or __error__ associated with it, the
matching event will be skipped.
matching events will be skipped.
## Spaces in the script path
By default, spaces are interpreted as separator between the inline script and its arguments (e.g. <code>-s "foo.py

4
examples/add_header.py
View File

@@ -1,2 +1,2 @@
def response(context, flow):
flow.response.headers["newheader"] = ["foo"]
def response(ctx, flow):
flow.response.headers["newheader"] = ["foo"]

6
examples/dup_and_replay.py
View File

@@ -1,4 +1,4 @@
def request(ctx, flow):
f = ctx.duplicate_flow(flow)
f.request.path = "/changed"
ctx.replay_request(f)
f = ctx.duplicate_flow(flow)
f.request.path = "/changed"
ctx.replay_request(f)

9
examples/flowbasic
View File

@@ -3,11 +3,14 @@
This example shows how to build a proxy based on mitmproxy's Flow
primitives.
Heads Up: In the majority of cases, you want to use inline scripts.
Note that request and response messages are not automatically replied to,
so we need to implement handlers to do this.
"""
import os
from libmproxy import proxy, flow
from libmproxy import flow, proxy
from libmproxy.proxy.server import ProxyServer
class MyMaster(flow.FlowMaster):
def run(self):
@@ -31,9 +34,9 @@ class MyMaster(flow.FlowMaster):
config = proxy.ProxyConfig(
cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
ca_file = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
)
state = flow.State()
server = proxy.ProxyServer(config, 8080)
server = ProxyServer(config, 8080)
m = MyMaster(server, state)
m.run()

50
examples/iframe_injector
View File

@@ -1,50 +0,0 @@
#!/usr/bin/env python
"""
Zap encoding in requests and inject iframe after body tag in html responses.
Usage:
iframe_injector http://someurl/somefile.html
"""
from libmproxy import controller, proxy
import os
import sys
class InjectingMaster(controller.Master):
def __init__(self, server, iframe_url):
controller.Master.__init__(self, server)
self._iframe_url = iframe_url
def run(self):
try:
return controller.Master.run(self)
except KeyboardInterrupt:
self.shutdown()
def handle_request(self, msg):
if 'Accept-Encoding' in msg.headers:
msg.headers["Accept-Encoding"] = 'none'
msg.reply()
def handle_response(self, msg):
if msg.content:
c = msg.replace('<body>', '<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % self._iframe_url)
if c > 0:
print 'Iframe injected!'
msg.reply()
def main(argv):
if len(argv) != 2:
print "Usage: %s IFRAME_URL" % argv[0]
sys.exit(1)
iframe_url = argv[1]
config = proxy.ProxyConfig(
cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
)
server = proxy.ProxyServer(config, 8080)
print 'Starting proxy...'
m = InjectingMaster(server, iframe_url)
m.run()
if __name__ == '__main__':
main(sys.argv)

18
examples/iframe_injector.py Normal file
View File

@@ -0,0 +1,18 @@
# Usage: mitmdump -s "iframe_injector.py url"
# (this script works best with --anticache)
from libmproxy.protocol.http import decoded
def start(ctx, argv):
if len(argv) != 2:
raise ValueError('Usage: -s "iframe_injector.py url"')
ctx.iframe_url = argv[1]
def handle_response(ctx, flow):
with decoded(flow.response): # Remove content encoding (gzip, ...)
c = flow.response.replace(
'<body>',
'<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % ctx.iframe_url)
if c > 0:
ctx.log("Iframe injected!")

10
examples/modify_form.py
View File

@@ -1,8 +1,6 @@
def request(context, flow):
def request(ctx, flow):
if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
frm = flow.request.get_form_urlencoded()
frm["mitmproxy"] = ["rocks"]
flow.request.set_form_urlencoded(frm)
form = flow.request.get_form_urlencoded()
form["mitmproxy"] = ["rocks"]
flow.request.set_form_urlencoded(form)

5
examples/modify_querystring.py
View File

@@ -1,7 +1,6 @@
def request(context, flow):
def request(ctx, flow):
q = flow.request.get_query()
if q:
q["mitmproxy"] = ["rocks"]
flow.request.set_query(q)
flow.request.set_query(q)

3
examples/nonblocking.py
View File

@@ -1,8 +1,9 @@
import time
from libmproxy.script import concurrent
@concurrent
def request(context, flow):
print "handle request: %s%s" % (flow.request.host, flow.request.path)
time.sleep(5)
print "start request: %s%s" % (flow.request.host, flow.request.path)
print "start request: %s%s" % (flow.request.host, flow.request.path)

10
examples/redirect_requests.py
View File

@@ -6,13 +6,15 @@ This example shows two ways to redirect flows to other destinations.
"""
def request(context, flow):
if flow.request.get_host(hostheader=True).endswith("example.com"):
def request(ctx, flow):
# pretty_host(hostheader=True) takes the Host: header of the request into account,
# which is useful in transparent mode where we usually only have the IP otherwise.
if flow.request.pretty_host(hostheader=True).endswith("example.com"):
resp = HTTPResponse(
[1, 1], 200, "OK",
ODictCaseless([["Content-Type", "text/html"]]),
"helloworld")
flow.reply(resp)
if flow.request.get_host(hostheader=True).endswith("example.org"):
if flow.request.pretty_host(hostheader=True).endswith("example.org"):
flow.request.host = "mitmproxy.org"
flow.request.headers["Host"] = ["mitmproxy.org"]
flow.request.update_host_header()

8
examples/stickycookies
View File

@@ -5,8 +5,10 @@ implement functionality similar to the "sticky cookies" option. This is at
a lower level than the Flow mechanism, so we're dealing directly with
request and response objects.
"""
from libmproxy import controller, proxy
import os
from libmproxy import controller, proxy
from libmproxy.proxy.server import ProxyServer
class StickyMaster(controller.Master):
def __init__(self, server):
@@ -35,8 +37,8 @@ class StickyMaster(controller.Master):
config = proxy.ProxyConfig(
cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
ca_file = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
)
server = proxy.ProxyServer(config, 8080)
server = ProxyServer(config, 8080)
m = StickyMaster(server)
m.run()

6
examples/stub.py
View File

@@ -7,14 +7,14 @@ def start(ctx, argv):
"""
ctx.log("start")
def clientconnect(ctx, client_connect):
def clientconnect(ctx, conn_handler):
"""
Called when a client initiates a connection to the proxy. Note that a
connection can correspond to multiple HTTP requests
"""
ctx.log("clientconnect")
def serverconnect(ctx, server_connection):
def serverconnect(ctx, conn_handler):
"""
Called when the proxy initiates a connection to the target server. Note that a
connection can correspond to multiple HTTP requests
@@ -50,7 +50,7 @@ def error(ctx, flow):
"""
ctx.log("error")
def clientdisconnect(ctx, client_disconnect):
def clientdisconnect(ctx, conn_handler):
"""
Called when a client disconnects from the proxy.
"""

2
examples/upsidedownternet.py
View File

@@ -1,7 +1,7 @@
import cStringIO
from PIL import Image
def response(context, flow):
def response(ctx, flow):
if flow.response.headers["content-type"] == ["image/png"]:
s = cStringIO.StringIO(flow.response.content)
img = Image.open(s).rotate(180)

2
libmproxy/console/common.py
View File

@@ -177,7 +177,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
req_timestamp = f.request.timestamp_start,
req_is_replay = f.request.is_replay,
req_method = f.request.method,
req_url = f.request.get_url(hostheader=hostheader),
req_url = f.request.pretty_url(hostheader=hostheader),
err_msg = f.error.msg if f.error else None,
resp_code = f.response.code if f.response else None,

6
libmproxy/console/flowview.py
View File

@@ -528,7 +528,9 @@ class FlowView(common.WWrap):
def set_url(self, url):
request = self.flow.request
if not request.set_url(str(url)):
try:
request.url = str(url)
except ValueError:
return "Invalid URL."
self.master.refresh_flow(self.flow)
@@ -608,7 +610,7 @@ class FlowView(common.WWrap):
elif part == "q":
self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.get_query().lst, self.set_query, conn))
elif part == "u" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
self.master.prompt_edit("URL", conn.get_url(), self.set_url)
self.master.prompt_edit("URL", conn.url, self.set_url)
elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
self.master.prompt_onekey("Method", self.method_options, self.edit_method)
elif part == "c" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:

2
libmproxy/dump.py
View File

@@ -55,7 +55,7 @@ def str_request(f, showhost):
c = f.client_conn.address.host
else:
c = "[replay]"
r = "%s %s %s"%(c, f.request.method, f.request.get_url(showhost, f))
r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
if f.request.stickycookie:
r = "[stickycookie] " + r
return r

4
libmproxy/filt.py
View File

@@ -208,7 +208,7 @@ class FDomain(_Rex):
code = "d"
help = "Domain"
def __call__(self, f):
return bool(re.search(self.expr, f.request.get_host(False, f), re.IGNORECASE))
return bool(re.search(self.expr, f.request.host, re.IGNORECASE))
class FUrl(_Rex):
@@ -222,7 +222,7 @@ class FUrl(_Rex):
return klass(*toks)
def __call__(self, f):
return re.search(self.expr, f.request.get_url(False, f))
return re.search(self.expr, f.request.url)
class _Int(_Action):

15
libmproxy/flow.py
View File

@@ -259,8 +259,8 @@ class StickyCookieState:
Returns a (domain, port, path) tuple.
"""
return (
m["domain"] or f.request.get_host(False, f),
f.request.get_port(f),
m["domain"] or f.request.host,
f.request.port,
m["path"] or "/"
)
@@ -278,7 +278,7 @@ class StickyCookieState:
c = Cookie.SimpleCookie(str(i))
m = c.values()[0]
k = self.ckey(m, f)
if self.domain_match(f.request.get_host(False, f), k[0]):
if self.domain_match(f.request.host, k[0]):
self.jar[self.ckey(m, f)] = m
def handle_request(self, f):
@@ -286,8 +286,8 @@ class StickyCookieState:
if f.match(self.flt):
for i in self.jar.keys():
match = [
self.domain_match(f.request.get_host(False, f), i[0]),
f.request.get_port(f) == i[1],
self.domain_match(f.request.host, i[0]),
f.request.port == i[1],
f.request.path.startswith(i[2])
]
if all(match):
@@ -306,7 +306,7 @@ class StickyAuthState:
self.hosts = {}
def handle_request(self, f):
host = f.request.get_host(False, f)
host = f.request.host
if "authorization" in f.request.headers:
self.hosts[host] = f.request.headers["authorization"]
elif f.match(self.flt):
@@ -612,6 +612,7 @@ class FlowMaster(controller.Master):
if f.request:
self.handle_request(f)
if f.response:
self.handle_responseheaders(f)
self.handle_response(f)
if f.error:
self.handle_error(f)
@@ -668,7 +669,7 @@ class FlowMaster(controller.Master):
self.masterq,
self.should_exit
)
rt.start() # pragma: no cover
rt.start() # pragma: no cover
if block:
rt.join()

499
libmproxy/protocol/http.py
View File

@@ -26,16 +26,22 @@ def get_line(fp):
return line
def send_connect_request(conn, host, port):
def send_connect_request(conn, host, port, update_state=True):
upstream_request = HTTPRequest("authority", "CONNECT", None, host, port, None,
(1, 1), ODictCaseless(), "")
conn.send(upstream_request._assemble())
conn.send(upstream_request.assemble())
resp = HTTPResponse.from_stream(conn.rfile, upstream_request.method)
if resp.code != 200:
raise proxy.ProxyError(resp.code,
"Cannot establish SSL " +
"connection with upstream proxy: \r\n" +
str(resp._assemble()))
str(resp.assemble()))
if update_state:
conn.state.append(("http", {
"state": "connect",
"host": host,
"port": port}
))
return resp
@@ -67,6 +73,9 @@ class decoded(object):
class HTTPMessage(stateobject.SimpleStateObject):
"""
Base class for HTTPRequest and HTTPResponse
"""
def __init__(self, httpversion, headers, content, timestamp_start=None,
timestamp_end=None):
self.httpversion = httpversion
@@ -151,36 +160,29 @@ class HTTPMessage(stateobject.SimpleStateObject):
c += self.headers.replace(pattern, repl, *args, **kwargs)
return c
@classmethod
def from_stream(cls, rfile, include_body=True, body_size_limit=None):
"""
Parse an HTTP message from a file stream
"""
raise NotImplementedError # pragma: nocover
def _assemble_first_line(self):
"""
Returns the assembled request/response line
"""
raise NotImplementedError # pragma: nocover
raise NotImplementedError() # pragma: nocover
def _assemble_headers(self):
"""
Returns the assembled headers
"""
raise NotImplementedError # pragma: nocover
raise NotImplementedError() # pragma: nocover
def _assemble_head(self):
"""
Returns the assembled request/response line plus headers
"""
raise NotImplementedError # pragma: nocover
raise NotImplementedError() # pragma: nocover
def _assemble(self):
def assemble(self):
"""
Returns the assembled request/response
"""
raise NotImplementedError # pragma: nocover
raise NotImplementedError() # pragma: nocover
class HTTPRequest(HTTPMessage):
@@ -189,7 +191,17 @@ class HTTPRequest(HTTPMessage):
Exposes the following attributes:
flow: Flow object the request belongs to
method: HTTP method
scheme: URL scheme (http/https) (absolute-form only)
host: Host portion of the URL (absolute-form and authority-form only)
port: Destination port (absolute-form and authority-form only)
path: Path portion of the URL (not present in authority-form)
httpversion: HTTP version tuple, e.g. (1,1)
headers: ODictCaseless object
@@ -205,18 +217,6 @@ class HTTPRequest(HTTPMessage):
form_out: The request form which mitmproxy has send out to the destination
method: HTTP method
scheme: URL scheme (http/https) (absolute-form only)
host: Host portion of the URL (absolute-form and authority-form only)
port: Destination port (absolute-form and authority-form only)
path: Path portion of the URL (not present in authority-form)
httpversion: HTTP version tuple
timestamp_start: Timestamp indicating when request transmission started
timestamp_end: Timestamp indicating when request transmission ended
@@ -358,7 +358,7 @@ class HTTPRequest(HTTPMessage):
def _assemble_head(self, form=None):
return "%s\r\n%s\r\n" % (self._assemble_first_line(form), self._assemble_headers())
def _assemble(self, form=None):
def assemble(self, form=None):
"""
Assembles the request for transmission to the server. We make some
modifications to make sure interception works properly.
@@ -405,6 +405,12 @@ class HTTPRequest(HTTPMessage):
e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0]
)]
def update_host_header(self):
"""
Update the host header to reflect the current target.
"""
self.headers["Host"] = [self.host]
def get_form_urlencoded(self):
"""
Retrieves the URL-encoded form data, returning an ODict object.
@@ -426,16 +432,16 @@ class HTTPRequest(HTTPMessage):
self.headers["Content-Type"] = [HDR_FORM_URLENCODED]
self.content = utils.urlencode(odict.lst)
def get_path_components(self, f):
def get_path_components(self):
"""
Returns the path components of the URL as a list of strings.
Components are unquoted.
"""
_, _, path, _, _, _ = urlparse.urlparse(self.get_url(False, f))
_, _, path, _, _, _ = urlparse.urlparse(self.url)
return [urllib.unquote(i) for i in path.split("/") if i]
def set_path_components(self, lst, f):
def set_path_components(self, lst):
"""
Takes a list of strings, and sets the path component of the URL.
@@ -443,32 +449,32 @@ class HTTPRequest(HTTPMessage):
"""
lst = [urllib.quote(i, safe="") for i in lst]
path = "/" + "/".join(lst)
scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.get_url(False, f))
self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f)
scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.url)
self.url = urlparse.urlunparse([scheme, netloc, path, params, query, fragment])
def get_query(self, f):
def get_query(self):
"""
Gets the request query string. Returns an ODict object.
"""
_, _, _, _, query, _ = urlparse.urlparse(self.get_url(False, f))
_, _, _, _, query, _ = urlparse.urlparse(self.url)
if query:
return ODict(utils.urldecode(query))
return ODict([])
def set_query(self, odict, f):
def set_query(self, odict):
"""
Takes an ODict object, and sets the request query string.
"""
scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.get_url(False, f))
scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.url)
query = utils.urlencode(odict.lst)
self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f)
self.url = urlparse.urlunparse([scheme, netloc, path, params, query, fragment])
def get_host(self, hostheader, flow):
def pretty_host(self, hostheader):
"""
Heuristic to get the host of the request.
Note that get_host() does not always return the TCP destination of the request,
e.g. on a transparently intercepted request to an unrelated HTTP proxy.
Note that pretty_host() does not always return the TCP destination of the request,
e.g. if an upstream proxy is in place
If hostheader is set to True, the Host: header will be used as additional (and preferred) data source.
This is handy in transparent mode, where only the ip of the destination is known, but not the
@@ -478,54 +484,27 @@ class HTTPRequest(HTTPMessage):
if hostheader:
host = self.headers.get_first("host")
if not host:
if self.host:
host = self.host
else:
for s in flow.server_conn.state:
if s[0] == "http" and s[1]["state"] == "connect":
host = s[1]["host"]
break
if not host:
host = flow.server_conn.address.host
host = self.host
host = host.encode("idna")
return host
def get_scheme(self, flow):
"""
Returns the request port, either from the request itself or from the flow's server connection
"""
if self.scheme:
return self.scheme
if self.form_out == "authority": # On SSLed connections, the original CONNECT request is still unencrypted.
return "http"
return "https" if flow.server_conn.ssl_established else "http"
def get_port(self, flow):
"""
Returns the request port, either from the request itself or from the flow's server connection
"""
if self.port:
return self.port
for s in flow.server_conn.state:
if s[0] == "http" and s[1].get("state") == "connect":
return s[1]["port"]
return flow.server_conn.address.port
def get_url(self, hostheader, flow):
"""
Returns a URL string, constructed from the Request's URL components.
If hostheader is True, we use the value specified in the request
Host header to construct the URL.
"""
def pretty_url(self, hostheader):
if self.form_out == "authority": # upstream proxy mode
return "%s:%s" % (self.get_host(hostheader, flow), self.get_port(flow))
return utils.unparse_url(self.get_scheme(flow),
self.get_host(hostheader, flow),
self.get_port(flow),
return "%s:%s" % (self.pretty_host(hostheader), self.port)
return utils.unparse_url(self.scheme,
self.pretty_host(hostheader),
self.port,
self.path).encode('ascii')
def set_url(self, url, flow):
@property
def url(self):
"""
Returns a URL string, constructed from the Request's URL components.
"""
return self.pretty_url(False)
@url.setter
def url(self, url):
"""
Parses a URL specification, and updates the Request's information
accordingly.
@@ -534,30 +513,8 @@ class HTTPRequest(HTTPMessage):
"""
parts = http.parse_url(url)
if not parts:
return False
scheme, host, port, path = parts
is_ssl = (True if scheme == "https" else False)
self.path = path
if host != self.get_host(False, flow) or port != self.get_port(flow):
if flow.live:
flow.live.change_server((host, port), ssl=is_ssl)
else:
# There's not live server connection, we're just changing the attributes here.
flow.server_conn = ServerConnection((host, port),
proxy.AddressPriority.MANUALLY_CHANGED)
flow.server_conn.ssl_established = is_ssl
# If this is an absolute request, replace the attributes on the request object as well.
if self.host:
self.host = host
if self.port:
self.port = port
if self.scheme:
self.scheme = scheme
return True
raise ValueError("Invalid URL: %s" % url)
self.scheme, self.host, self.port, self.path = parts
def get_cookies(self):
cookie_headers = self.headers.get("cookie")
@@ -590,7 +547,7 @@ class HTTPResponse(HTTPMessage):
Exposes the following attributes:
flow: Flow object the request belongs to
httpversion: HTTP version tuple, e.g. (1,1)
code: HTTP response code
@@ -602,8 +559,6 @@ class HTTPResponse(HTTPMessage):
is content associated, but not present. CONTENT_MISSING evaluates
to False to make checking for the presence of content natural.
httpversion: HTTP version tuple
timestamp_start: Timestamp indicating when request transmission started
timestamp_end: Timestamp indicating when request transmission ended
@@ -682,7 +637,8 @@ class HTTPResponse(HTTPMessage):
if self.content:
headers["Content-Length"] = [str(len(self.content))]
elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers: # add content-length for chuncked transfer-encoding with no content
# add content-length for chuncked transfer-encoding with no content
elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers:
headers["Content-Length"] = ["0"]
return str(headers)
@@ -691,7 +647,7 @@ class HTTPResponse(HTTPMessage):
return '%s\r\n%s\r\n' % (
self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding))
def _assemble(self):
def assemble(self):
"""
Assembles the response for transmission to the client. We make some
modifications to make sure interception works properly.
@@ -774,12 +730,14 @@ class HTTPResponse(HTTPMessage):
class HTTPFlow(Flow):
"""
A Flow is a collection of objects representing a single HTTP
A HTTPFlow is a collection of objects representing a single HTTP
transaction. The main attributes are:
request: HTTPRequest object
response: HTTPResponse object
error: Error object
server_conn: ServerConnection object
client_conn: ClientConnection object
Note that it's possible for a Flow to have both a response and an error
object. This might happen, for instance, when a response was received
@@ -816,7 +774,7 @@ class HTTPFlow(Flow):
s = "<HTTPFlow"
for a in ("request", "response", "error", "client_conn", "server_conn"):
if getattr(self, a, False):
s += "\r\n %s = {flow.%s}" % (a,a)
s += "\r\n %s = {flow.%s}" % (a, a)
s += ">"
return s.format(flow=self)
@@ -895,6 +853,10 @@ class HttpAuthenticationError(Exception):
class HTTPHandler(ProtocolHandler):
"""
HTTPHandler implements mitmproxys understanding of the HTTP protocol.
"""
def __init__(self, c):
super(HTTPHandler, self).__init__(c)
self.expected_form_in = c.config.http_form_in
@@ -905,19 +867,21 @@ class HTTPHandler(ProtocolHandler):
while self.handle_flow():
pass
def get_response_from_server(self, request, include_body=True):
def get_response_from_server(self, flow):
self.c.establish_server_connection()
request_raw = request._assemble()
request_raw = flow.request.assemble()
for i in range(2):
for attempt in (0, 1):
try:
self.c.server_conn.send(request_raw)
res = HTTPResponse.from_stream(self.c.server_conn.rfile, request.method,
body_size_limit=self.c.config.body_size_limit, include_body=include_body)
return res
# Only get the headers at first...
flow.response = HTTPResponse.from_stream(self.c.server_conn.rfile, flow.request.method,
body_size_limit=self.c.config.body_size_limit,
include_body=False)
break
except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v:
self.c.log("error in server communication: %s" % repr(v), level="debug")
if i < 1:
if attempt == 0:
# In any case, we try to reconnect at least once.
# This is necessary because it might be possible that we already initiated an upstream connection
# after clientconnect that has already been expired, e.g consider the following event log:
@@ -931,13 +895,24 @@ class HTTPHandler(ProtocolHandler):
else:
raise
# call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
self.c.channel.ask("responseheaders", flow)
# now get the rest of the request body, if body still needs to be read but not streaming this response
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit,
flow.request.method, flow.response.code, False)
def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
try:
try:
req = HTTPRequest.from_stream(self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
except tcp.NetLibDisconnect: # specifically ignore disconnects that happen before/between requests.
except tcp.NetLibDisconnect: # don't throw an error for disconnects that happen before/between requests.
return False
self.c.log("request", "debug", [req._assemble_first_line(req.form_in)])
ret = self.process_request(flow, req)
@@ -951,8 +926,7 @@ class HTTPHandler(ProtocolHandler):
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow)
self.determine_server_address(flow, flow.request) # The inline script may have changed request.host
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
self.process_server_address(flow) # The inline script may have changed request.host
if request_reply is None or request_reply == KILL:
return False
@@ -960,20 +934,7 @@ class HTTPHandler(ProtocolHandler):
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
# read initially in "stream" mode, so we can get the headers separately
flow.response = self.get_response_from_server(flow.request, include_body=False)
# call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
self.c.channel.ask("responseheaders", flow)
# now get the rest of the request body, if body still needs to be read but not streaming this response
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit,
flow.request.method, flow.response.code, False)
self.get_response_from_server(flow)
# no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
@@ -984,72 +945,39 @@ class HTTPHandler(ProtocolHandler):
if response_reply is None or response_reply == KILL:
return False
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response._assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.response.timestamp_end = utils.timestamp()
self.send_response_to_client(flow)
flow.timestamp_end = utils.timestamp()
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return False
if self.check_close_connection(flow):
return False
# We sent a CONNECT request to an upstream proxy.
if flow.request.form_in == "authority" and flow.response.code == 200:
# TODO: Eventually add headers (space/usefulness tradeoff)
# Make sure to add state info before the actual upgrade happens.
# During the upgrade, we may receive an SNI indication from the client,
# TODO: Possibly add headers (memory consumption/usefulness tradeoff)
# Make sure to add state info before the actual processing of the CONNECT request happens.
# During an SSL upgrade, we may receive an SNI indication from the client,
# which resets the upstream connection. If this is the case, we must
# already re-issue the CONNECT request at this point.
self.c.server_conn.state.append(("http", {"state": "connect",
"host": flow.request.host,
"port": flow.request.port}))
if self.c.check_ignore_address((flow.request.host, flow.request.port)):
self.c.log("Ignore host: %s:%s" % self.c.server_conn.address(), "info")
TCPHandler(self.c).handle_messages()
if not self.process_connect_request((flow.request.host, flow.request.port)):
return False
else:
if flow.request.port in self.c.config.ssl_ports:
self.ssl_upgrade()
self.skip_authentication = True
# If the user has changed the target server on this connection,
# restore the original target server
flow.live.restore_server()
flow.live = None
return True
return True # Next flow please.
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
self.handle_error(e, flow)
finally:
flow.timestamp_end = utils.timestamp()
flow.live = None # Connection is not live anymore.
return False
def handle_server_reconnect(self, state):
if state["state"] == "connect":
send_connect_request(self.c.server_conn, state["host"], state["port"])
send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False)
else: # pragma: nocover
raise RuntimeError("Unknown State: %s" % state["state"])
@@ -1093,16 +1021,6 @@ class HTTPHandler(ProtocolHandler):
self.c.client_conn.wfile.write(html_content)
self.c.client_conn.wfile.flush()
def ssl_upgrade(self):
"""
Upgrade the connection to SSL after an authority (CONNECT) request has been made.
"""
self.c.log("Received CONNECT request. Upgrading to SSL...", "debug")
self.expected_form_in = "relative"
self.expected_form_out = "relative"
self.c.establish_ssl(server=True, client=True)
self.c.log("Upgrade to SSL completed.", "debug")
def process_request(self, flow, request):
"""
@returns:
@@ -1115,14 +1033,30 @@ class HTTPHandler(ProtocolHandler):
if not self.skip_authentication:
self.authenticate(request)
# Determine .scheme, .host and .port attributes
# For absolute-form requests, they are directly given in the request.
# For authority-form requests, we only need to determine the request scheme.
# For relative-form requests, we need to determine host and port as well.
if not request.scheme:
request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http"
if not request.host:
# Host/Port Complication: In upstream mode, use the server we CONNECTed to,
# not the upstream proxy.
if flow.server_conn:
for s in flow.server_conn.state:
if s[0] == "http" and s[1]["state"] == "connect":
request.host, request.port = s[1]["host"], s[1]["port"]
if not request.host and flow.server_conn:
request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port
# Now we can process the request.
if request.form_in == "authority":
if self.c.client_conn.ssl_established:
raise http.HttpError(400, "Must not CONNECT on already encrypted connection")
if self.expected_form_in == "absolute":
if not self.c.config.get_upstream_server:
self.c.set_server_address((request.host, request.port),
proxy.AddressPriority.FROM_PROTOCOL)
if not self.c.config.get_upstream_server: # Regular mode
self.c.set_server_address((request.host, request.port))
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
self.c.establish_server_connection()
self.c.client_conn.send(
@@ -1131,34 +1065,123 @@ class HTTPHandler(ProtocolHandler):
('Proxy-agent: %s\r\n' % self.c.server_version) +
'\r\n'
)
if self.c.check_ignore_address(self.c.server_conn.address):
self.c.log("Ignore host: %s:%s" % self.c.server_conn.address(), "info")
TCPHandler(self.c).handle_messages()
return False
else:
if self.c.server_conn.address.port in self.c.config.ssl_ports:
self.ssl_upgrade()
self.skip_authentication = True
return True
else:
return self.process_connect_request(self.c.server_conn.address)
else: # upstream proxy mode
return None
else:
pass # CONNECT should never occur if we don't expect absolute-form requests
elif request.form_in == self.expected_form_in:
request.form_out = self.expected_form_out
if request.form_in == "absolute":
if request.scheme != "http":
raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme)
self.determine_server_address(flow, request)
request.form_out = self.expected_form_out
if request.form_out == "relative":
self.c.set_server_address((request.host, request.port))
flow.server_conn = self.c.server_conn
return None
raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" %
(self.expected_form_in, request.form_in))
def determine_server_address(self, flow, request):
if request.form_in == "absolute":
self.c.set_server_address((request.host, request.port),
proxy.AddressPriority.FROM_PROTOCOL)
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
def process_server_address(self, flow):
# Depending on the proxy mode, server handling is entirely different
# We provide a mostly unified API to the user, which needs to be unfiddled here
# ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 )
address = netlib.tcp.Address((flow.request.host, flow.request.port))
ssl = (flow.request.scheme == "https")
if self.c.config.http_form_in == self.c.config.http_form_out == "absolute": # Upstream Proxy mode
# The connection to the upstream proxy may have a state we may need to take into account.
connected_to = None
for s in flow.server_conn.state:
if s[0] == "http" and s[1]["state"] == "connect":
connected_to = tcp.Address((s[1]["host"], s[1]["port"]))
# We need to reconnect if the current flow either requires a (possibly impossible)
# change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else.
needs_server_change = (
ssl != self.c.server_conn.ssl_established
or
(connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't.
)
if needs_server_change:
# force create new connection to the proxy server to reset state
self.live.change_server(self.c.server_conn.address, force=True)
if ssl:
send_connect_request(self.c.server_conn, address.host, address.port)
self.c.establish_ssl(server=True)
else:
# If we're not in upstream mode, we just want to update the host and possibly establish TLS.
self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match.
flow.server_conn = self.c.server_conn
def send_response_to_client(self, flow):
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response.assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.response.timestamp_end = utils.timestamp()
def check_close_connection(self, flow):
"""
Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
"""
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return True
return False
def process_connect_request(self, address):
"""
Process a CONNECT request.
Returns True if the CONNECT request has been processed successfully.
Returns False, if the connection should be closed immediately.
"""
address = tcp.Address.wrap(address)
if self.c.check_ignore_address(address):
self.c.log("Ignore host: %s:%s" % address(), "info")
TCPHandler(self.c).handle_messages()
return False
else:
self.expected_form_in = "relative"
self.expected_form_out = "relative"
self.skip_authentication = True
if address.port in self.c.config.ssl_ports:
self.c.log("Received CONNECT request to SSL port. Upgrading to SSL...", "debug")
self.c.establish_ssl(server=True, client=True)
self.c.log("Upgrade to SSL completed.", "debug")
return True
def authenticate(self, request):
if self.c.config.authenticator:
@@ -1178,33 +1201,29 @@ class RequestReplayThread(threading.Thread):
threading.Thread.__init__(self)
def run(self):
r = self.flow.request
form_out_backup = r.form_out
try:
r = self.flow.request
form_out_backup = r.form_out
r.form_out = self.config.http_form_out
server_address, server_ssl = False, False
if self.config.get_upstream_server:
try:
# this will fail in transparent mode
upstream_info = self.config.get_upstream_server(self.flow.client_conn)
server_ssl = upstream_info[1]
server_address = upstream_info[2:]
except proxy.ProxyError:
pass
if not server_address:
server_address = (r.get_host(False, self.flow), r.get_port(self.flow))
server = ServerConnection(server_address, None)
server.connect()
if server_ssl or r.get_scheme(self.flow) == "https":
if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode -> send CONNECT
send_connect_request(server, r.get_host(), r.get_port())
# In all modes, we directly connect to the server displayed
if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode
server_address = self.config.get_upstream_server(self.flow.client_conn)[2:]
server = ServerConnection(server_address)
server.connect()
if r.scheme == "https":
send_connect_request(server, r.host, r.port)
server.establish_ssl(self.config.clientcerts, sni=r.host)
r.form_out = "relative"
server.establish_ssl(self.config.clientcerts,
self.flow.server_conn.sni)
server.send(r._assemble())
else:
r.form_out = "absolute"
else:
server_address = (r.host, r.port)
server = ServerConnection(server_address)
server.connect()
if r.scheme == "https":
server.establish_ssl(self.config.clientcerts, sni=r.host)
r.form_out = "relative"
server.send(r.assemble())
self.flow.response = HTTPResponse.from_stream(server.rfile, r.method,
body_size_limit=self.config.body_size_limit)
self.channel.ask("response", self.flow)

72
libmproxy/protocol/primitives.py
View File

@@ -2,7 +2,6 @@ from __future__ import absolute_import
import copy
import netlib.tcp
from .. import stateobject, utils, version
from ..proxy.primitives import AddressPriority
from ..proxy.connection import ClientConnection, ServerConnection
@@ -13,9 +12,9 @@ class Error(stateobject.SimpleStateObject):
"""
An Error.
This is distinct from an HTTP error response (say, a code 500), which
is represented by a normal Response object. This class is responsible
for indicating errors that fall outside of normal HTTP communications,
This is distinct from an protocol error response (say, a HTTP code 500), which
is represented by a normal HTTPResponse object. This class is responsible
for indicating errors that fall outside of normal protocol communications,
like interrupted connections, timeouts, protocol errors.
Exposes the following attributes:
@@ -53,13 +52,17 @@ class Error(stateobject.SimpleStateObject):
class Flow(stateobject.SimpleStateObject):
"""
A Flow is a collection of objects representing a single transaction.
This class is usually subclassed for each protocol, e.g. HTTPFlow.
"""
def __init__(self, conntype, client_conn, server_conn, live=None):
self.conntype = conntype
self.client_conn = client_conn
"""@type: ClientConnection"""
self.server_conn = server_conn
"""@type: ServerConnection"""
self.live = live # Used by flow.request.set_url to change the server address
self.live = live
"""@type: LiveConnection"""
self.error = None
@@ -118,6 +121,10 @@ class Flow(stateobject.SimpleStateObject):
class ProtocolHandler(object):
"""
A ProtocolHandler implements an application-layer protocol, e.g. HTTP.
See: libmproxy.protocol.http.HTTPHandler
"""
def __init__(self, c):
self.c = c
"""@type: libmproxy.proxy.server.ConnectionHandler"""
@@ -149,48 +156,53 @@ class ProtocolHandler(object):
class LiveConnection(object):
"""
This facade allows protocol handlers to interface with a live connection,
without requiring the expose the ConnectionHandler.
This facade allows interested parties (FlowMaster, inline scripts) to interface with a live connection,
without requiring to expose the internals of the ConnectionHandler.
"""
def __init__(self, c):
self._c = c
self.c = c
"""@type: libmproxy.proxy.server.ConnectionHandler"""
self._backup_server_conn = None
"""@type: libmproxy.proxy.connection.ServerConnection"""
def change_server(self, address, ssl, persistent_change=False):
def change_server(self, address, ssl=False, force=False, persistent_change=False):
address = netlib.tcp.Address.wrap(address)
if address != self._c.server_conn.address:
if force or address != self.c.server_conn.address or ssl != self.c.server_conn.ssl_established:
self._c.log("Change server connection: %s:%s -> %s:%s" % (
self._c.server_conn.address.host,
self._c.server_conn.address.port,
self.c.log("Change server connection: %s:%s -> %s:%s [persistent: %s]" % (
self.c.server_conn.address.host,
self.c.server_conn.address.port,
address.host,
address.port
address.port,
persistent_change
), "debug")
if not hasattr(self, "_backup_server_conn"):
self._backup_server_conn = self._c.server_conn
self._c.server_conn = None
if not self._backup_server_conn:
self._backup_server_conn = self.c.server_conn
self.c.server_conn = None
else: # This is at least the second temporary change. We can kill the current connection.
self._c.del_server_connection()
self.c.del_server_connection()
self._c.set_server_address(address, AddressPriority.MANUALLY_CHANGED)
self._c.establish_server_connection(ask=False)
self.c.set_server_address(address)
self.c.establish_server_connection(ask=False)
if ssl:
self._c.establish_ssl(server=True)
if hasattr(self, "_backup_server_conn") and persistent_change:
del self._backup_server_conn
self.c.establish_ssl(server=True)
if persistent_change:
self._backup_server_conn = None
def restore_server(self):
if not hasattr(self, "_backup_server_conn"):
# TODO: Similar to _backup_server_conn, introduce _cache_server_conn, which keeps the changed connection open
# This may be beneficial if a user is rewriting all requests from http to https or similar.
if not self._backup_server_conn:
return
self._c.log("Restore original server connection: %s:%s -> %s:%s" % (
self._c.server_conn.address.host,
self._c.server_conn.address.port,
self.c.log("Restore original server connection: %s:%s -> %s:%s" % (
self.c.server_conn.address.host,
self.c.server_conn.address.port,
self._backup_server_conn.address.host,
self._backup_server_conn.address.port
), "debug")
self._c.del_server_connection()
self._c.server_conn = self._backup_server_conn
del self._backup_server_conn
self.c.del_server_connection()
self.c.server_conn = self._backup_server_conn
self._backup_server_conn = None

4
libmproxy/protocol/tcp.py
View File

@@ -59,11 +59,11 @@ class TCPHandler(ProtocolHandler):
# if one of the peers is over SSL, we need to send bytes/strings
if not src.ssl_established: # only ssl to dst, i.e. we revc'd into buf but need bytes/string now.
contents = buf[:size].tobytes()
# self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(contents)), "debug")
self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(contents)), "debug")
dst.connection.send(contents)
else:
# socket.socket.send supports raw bytearrays/memoryviews
# self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(buf.tobytes())), "debug")
self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(buf.tobytes())), "debug")
dst.connection.send(buf[:size])
except socket.error as e:
self.c.log("TCP connection closed unexpectedly.", "debug")

3
libmproxy/proxy/__init__.py
View File

@@ -1 +1,2 @@
from .primitives import *
from .primitives import *
from .config import ProxyConfig

6
libmproxy/proxy/config.py
View File

@@ -1,8 +1,8 @@
from __future__ import absolute_import
import os
from .. import utils, platform
import re
from netlib import http_auth, certutils
from .. import utils, platform
from .primitives import ConstUpstreamServerResolver, TransparentUpstreamServerResolver
TRANSPARENT_SSL_PORTS = [443, 8443]
@@ -11,7 +11,7 @@ CONF_DIR = "~/.mitmproxy"
class ProxyConfig:
def __init__(self, confdir=CONF_DIR, clientcerts=None,
def __init__(self, confdir=CONF_DIR, ca_file=None, clientcerts=None,
no_upstream_cert=False, body_size_limit=None,
mode=None, upstream_server=None, http_form_in=None, http_form_out=None,
authenticator=None, ignore=[],
@@ -44,7 +44,7 @@ class ProxyConfig:
self.ignore = [re.compile(i, re.IGNORECASE) for i in ignore]
self.authenticator = authenticator
self.confdir = os.path.expanduser(confdir)
self.ca_file = os.path.join(self.confdir, CONF_BASENAME + "-ca.pem")
self.ca_file = ca_file or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem")
self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME)
for spec, cert in certs:
self.certstore.add_cert_file(spec, cert)

18
libmproxy/proxy/connection.py
View File

@@ -72,13 +72,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.SimpleStateObject):
class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject):
def __init__(self, address, priority):
def __init__(self, address):
tcp.TCPClient.__init__(self, address)
self.priority = priority
self.state = [] # a list containing (conntype, state) tuples
self.peername = None
self.sockname = None
self.timestamp_start = None
self.timestamp_end = None
self.timestamp_tcp_setup = None
@@ -99,8 +96,6 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject):
_stateobject_attributes = dict(
state=list,
peername=tuple,
sockname=tuple,
timestamp_start=float,
timestamp_end=float,
timestamp_tcp_setup=float,
@@ -115,9 +110,10 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject):
def _get_state(self):
d = super(ServerConnection, self)._get_state()
d.update(
address={"address": self.address(), "use_ipv6": self.address.use_ipv6},
source_address= {"address": self.source_address(),
"use_ipv6": self.source_address.use_ipv6} if self.source_address else None,
address={"address": self.address(),
"use_ipv6": self.address.use_ipv6},
source_address= ({"address": self.source_address(),
"use_ipv6": self.source_address.use_ipv6} if self.source_address else None),
cert=self.cert.to_pem() if self.cert else None
)
return d
@@ -131,7 +127,7 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject):
@classmethod
def _from_state(cls, state):
f = cls(tuple(), None)
f = cls(tuple())
f._load_state(state)
return f
@@ -141,8 +137,6 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject):
def connect(self):
self.timestamp_start = utils.timestamp()
tcp.TCPClient.connect(self)
self.peername = self.connection.getpeername()
self.sockname = self.connection.getsockname()
self.timestamp_tcp_setup = utils.timestamp()
def send(self, message):

13
libmproxy/proxy/primitives.py
View File

@@ -45,19 +45,6 @@ class TransparentUpstreamServerResolver(UpstreamServerResolver):
return [ssl, ssl] + list(dst)
class AddressPriority(object):
"""
Enum that signifies the priority of the given address when choosing the destination host.
Higher is better (None < i)
"""
MANUALLY_CHANGED = 3
"""user changed the target address in the ui"""
FROM_SETTINGS = 2
"""upstream server from arguments (reverse proxy, upstream proxy or from transparent resolver)"""
FROM_PROTOCOL = 1
"""derived from protocol (e.g. absolute-form http requests)"""
class Log:
def __init__(self, msg, level="info"):
self.msg = msg

25
libmproxy/proxy/server.py
View File

@@ -1,11 +1,10 @@
from __future__ import absolute_import
import re
import socket
from OpenSSL import SSL
from netlib import tcp
from .primitives import ProxyServerError, Log, ProxyError, AddressPriority
from .primitives import ProxyServerError, Log, ProxyError
from .connection import ClientConnection, ServerConnection
from ..protocol.handle import protocol_handler
from .. import version
@@ -76,7 +75,7 @@ class ConnectionHandler:
client_ssl, server_ssl = False, False
if self.config.get_upstream_server:
upstream_info = self.config.get_upstream_server(self.client_conn.connection)
self.set_server_address(upstream_info[2:], AddressPriority.FROM_SETTINGS)
self.set_server_address(upstream_info[2:])
client_ssl, server_ssl = upstream_info[:2]
if self.check_ignore_address(self.server_conn.address):
self.log("Ignore host: %s:%s" % self.server_conn.address(), "info")
@@ -129,27 +128,22 @@ class ConnectionHandler:
else:
return False
def set_server_address(self, address, priority):
def set_server_address(self, address):
"""
Sets a new server address with the given priority.
Does not re-establish either connection or SSL handshake.
"""
address = tcp.Address.wrap(address)
if self.server_conn:
if self.server_conn.priority > priority:
self.log("Attempt to change server address, "
"but priority is too low (is: %s, got: %s)" % (
self.server_conn.priority, priority), "debug")
return
if self.server_conn.address == address:
self.server_conn.priority = priority # Possibly increase priority
return
# Don't reconnect to the same destination.
if self.server_conn and self.server_conn.address == address:
return
if self.server_conn:
self.del_server_connection()
self.log("Set new server address: %s:%s" % (address.host, address.port), "debug")
self.server_conn = ServerConnection(address, priority)
self.server_conn = ServerConnection(address)
def establish_server_connection(self, ask=True):
"""
@@ -212,12 +206,11 @@ class ConnectionHandler:
def server_reconnect(self):
address = self.server_conn.address
had_ssl = self.server_conn.ssl_established
priority = self.server_conn.priority
state = self.server_conn.state
sni = self.sni
self.log("(server reconnect follows)", "debug")
self.del_server_connection()
self.set_server_address(address, priority)
self.set_server_address(address)
self.establish_server_connection()
for s in state:

3
libmproxy/stateobject.py
View File

@@ -21,6 +21,9 @@ class StateObject(object):
except AttributeError: # we may compare with something that's not a StateObject
return False
def __ne__(self, other):
return not self.__eq__(other)
class SimpleStateObject(StateObject):
"""

2
test/test_examples.py
View File

@@ -12,6 +12,8 @@ def test_load_scripts():
tmaster = tservers.TestMaster(config.ProxyConfig())
for f in scripts:
if "iframe_injector" in f:
f += " foo" # one argument required
if "modify_response_body" in f:
f += " foo bar" # two arguments required
script.Script(f, tmaster) # Loads the script file.

96
test/test_flow.py
View File

@@ -481,7 +481,7 @@ class TestSerialize:
f2 = l[0]
assert f2._get_state() == f._get_state()
assert f2.request._assemble() == f.request._assemble()
assert f2.request.assemble() == f.request.assemble()
def test_load_flows(self):
r = self._treader()
@@ -753,63 +753,61 @@ class TestRequest:
def test_simple(self):
f = tutils.tflow()
r = f.request
u = r.get_url(False, f)
assert r.set_url(u, f)
assert not r.set_url("", f)
assert r.get_url(False, f) == u
assert r._assemble()
assert r.size() == len(r._assemble())
u = r.url
r.url = u
tutils.raises(ValueError, setattr, r, "url", "")
assert r.url == u
assert r.assemble()
assert r.size() == len(r.assemble())
r2 = r.copy()
assert r == r2
r.content = None
assert r._assemble()
assert r.size() == len(r._assemble())
assert r.assemble()
assert r.size() == len(r.assemble())
r.content = CONTENT_MISSING
tutils.raises("Cannot assemble flow with CONTENT_MISSING", r._assemble)
tutils.raises("Cannot assemble flow with CONTENT_MISSING", r.assemble)
def test_get_url(self):
f = tutils.tflow()
r = f.request
r = tutils.treq()
assert r.get_url(False, f) == "http://address:22/path"
assert r.url == "http://address:22/path"
r.scheme = "https"
assert r.get_url(False, f) == "https://address:22/path"
assert r.url == "https://address:22/path"
r.host = "host"
r.port = 42
assert r.get_url(False, f) == "https://host:42/path"
assert r.url == "https://host:42/path"
r.host = "address"
r.port = 22
assert r.get_url(False, f) == "https://address:22/path"
assert r.url== "https://address:22/path"
assert r.get_url(True, f) == "https://address:22/path"
assert r.pretty_url(True) == "https://address:22/path"
r.headers["Host"] = ["foo.com"]
assert r.get_url(False, f) == "https://address:22/path"
assert r.get_url(True, f) == "https://foo.com:22/path"
assert r.pretty_url(False) == "https://address:22/path"
assert r.pretty_url(True) == "https://foo.com:22/path"
def test_path_components(self):
f = tutils.tflow()
r = f.request
r = tutils.treq()
r.path = "/"
assert r.get_path_components(f) == []
assert r.get_path_components() == []
r.path = "/foo/bar"
assert r.get_path_components(f) == ["foo", "bar"]
assert r.get_path_components() == ["foo", "bar"]
q = flow.ODict()
q["test"] = ["123"]
r.set_query(q, f)
assert r.get_path_components(f) == ["foo", "bar"]
r.set_query(q)
assert r.get_path_components() == ["foo", "bar"]
r.set_path_components([], f)
assert r.get_path_components(f) == []
r.set_path_components(["foo"], f)
assert r.get_path_components(f) == ["foo"]
r.set_path_components(["/oo"], f)
assert r.get_path_components(f) == ["/oo"]
r.set_path_components([])
assert r.get_path_components() == []
r.set_path_components(["foo"])
assert r.get_path_components() == ["foo"]
r.set_path_components(["/oo"])
assert r.get_path_components() == ["/oo"]
assert "%2F" in r.path
def test_getset_form_urlencoded(self):
@@ -828,26 +826,26 @@ class TestRequest:
def test_getset_query(self):
h = flow.ODictCaseless()
f = tutils.tflow()
f.request.path = "/foo?x=y&a=b"
q = f.request.get_query(f)
r = tutils.treq()
r.path = "/foo?x=y&a=b"
q = r.get_query()
assert q.lst == [("x", "y"), ("a", "b")]
f.request.path = "/"
q = f.request.get_query(f)
r.path = "/"
q = r.get_query()
assert not q
f.request.path = "/?adsfa"
q = f.request.get_query(f)
r.path = "/?adsfa"
q = r.get_query()
assert q.lst == [("adsfa", "")]
f.request.path = "/foo?x=y&a=b"
assert f.request.get_query(f)
f.request.set_query(flow.ODict([]), f)
assert not f.request.get_query(f)
r.path = "/foo?x=y&a=b"
assert r.get_query()
r.set_query(flow.ODict([]))
assert not r.get_query()
qv = flow.ODict([("a", "b"), ("c", "d")])
f.request.set_query(qv, f)
assert f.request.get_query(f) == qv
r.set_query(qv)
assert r.get_query() == qv
def test_anticache(self):
h = flow.ODictCaseless()
@@ -968,18 +966,18 @@ class TestResponse:
def test_simple(self):
f = tutils.tflow(resp=True)
resp = f.response
assert resp._assemble()
assert resp.size() == len(resp._assemble())
assert resp.assemble()
assert resp.size() == len(resp.assemble())
resp2 = resp.copy()
assert resp2 == resp
resp.content = None
assert resp._assemble()
assert resp.size() == len(resp._assemble())
assert resp.assemble()
assert resp.size() == len(resp.assemble())
resp.content = CONTENT_MISSING
tutils.raises("Cannot assemble flow with CONTENT_MISSING", resp._assemble)
tutils.raises("Cannot assemble flow with CONTENT_MISSING", resp.assemble)
def test_refresh(self):
r = tutils.tresp()

173
test/test_protocol_http.py
View File

@@ -1,5 +1,4 @@
from libmproxy.protocol.http import *
from libmproxy.protocol import KILL
from cStringIO import StringIO
import tutils, tservers
@@ -32,11 +31,27 @@ class TestHTTPRequest:
f.request.host = f.server_conn.address.host
f.request.port = f.server_conn.address.port
f.request.scheme = "http"
assert f.request._assemble() == "OPTIONS * HTTP/1.1\r\nHost: address:22\r\n\r\n"
assert f.request.assemble() == "OPTIONS * HTTP/1.1\r\nHost: address:22\r\n\r\n"
def test_origin_form(self):
s = StringIO("GET /foo\xff HTTP/1.1")
tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s)
s = StringIO("GET /foo HTTP/1.1\r\nConnection: Upgrade\r\nUpgrade: h2c")
r = HTTPRequest.from_stream(s)
assert r.headers["Upgrade"] == ["h2c"]
raw = r._assemble_headers()
assert "Upgrade" not in raw
assert "Host" not in raw
r.url = "http://example.com/foo"
raw = r._assemble_headers()
assert "Host" in raw
assert not "Host" in r.headers
r.update_host_header()
assert "Host" in r.headers
def test_authority_form(self):
s = StringIO("CONNECT oops-no-port.com HTTP/1.1")
@@ -44,26 +59,31 @@ class TestHTTPRequest:
s = StringIO("CONNECT address:22 HTTP/1.1")
r = HTTPRequest.from_stream(s)
r.scheme, r.host, r.port = "http", "address", 22
assert r._assemble() == "CONNECT address:22 HTTP/1.1\r\nHost: address:22\r\n\r\n"
assert r.assemble() == "CONNECT address:22 HTTP/1.1\r\nHost: address:22\r\n\r\n"
assert r.pretty_url(False) == "address:22"
def test_absolute_form(self):
s = StringIO("GET oops-no-protocol.com HTTP/1.1")
tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s)
s = StringIO("GET http://address:22/ HTTP/1.1")
r = HTTPRequest.from_stream(s)
assert r._assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\n\r\n"
assert r.assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\n\r\n"
def test_assemble_unknown_form(self):
r = tutils.treq()
tutils.raises("Invalid request form", r._assemble, "antiauthority")
tutils.raises("Invalid request form", r.assemble, "antiauthority")
def test_set_url(self):
f = tutils.tflow(req=tutils.treq_absolute())
f.request.set_url("https://otheraddress:42/ORLY", f)
assert f.request.scheme == "https"
assert f.request.host == "otheraddress"
assert f.request.port == 42
assert f.request.path == "/ORLY"
r = tutils.treq_absolute()
r.url = "https://otheraddress:42/ORLY"
assert r.scheme == "https"
assert r.host == "otheraddress"
assert r.port == 42
assert r.path == "/ORLY"
def test_repr(self):
r = tutils.treq()
assert repr(r)
class TestHTTPResponse:
@@ -86,6 +106,19 @@ class TestHTTPResponse:
assert r.content == ""
tutils.raises("Invalid server response: 'content", HTTPResponse.from_stream, s, "GET")
def test_repr(self):
r = tutils.tresp()
assert "unknown content type" in repr(r)
r.headers["content-type"] = ["foo"]
assert "foo" in repr(r)
assert repr(tutils.tresp(content=CONTENT_MISSING))
class TestHTTPFlow(object):
def test_repr(self):
f = tutils.tflow(resp=True, err=True)
assert repr(f)
class TestInvalidRequests(tservers.HTTPProxTest):
ssl = True
@@ -100,120 +133,4 @@ class TestInvalidRequests(tservers.HTTPProxTest):
p.connect()
r = p.request("get:/p/200")
assert r.status_code == 400
assert "Invalid HTTP request form" in r.content
class TestProxyChaining(tservers.HTTPChainProxyTest):
def test_all(self):
self.chain[1].tmaster.replacehooks.add("~q", "foo", "bar") # replace in request
self.chain[0].tmaster.replacehooks.add("~q", "foo", "oh noes!")
self.proxy.tmaster.replacehooks.add("~q", "bar", "baz")
self.chain[0].tmaster.replacehooks.add("~s", "baz", "ORLY") # replace in response
p = self.pathoc()
req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase)
assert req.content == "ORLY"
assert req.status_code == 418
class TestProxyChainingSSL(tservers.HTTPChainProxyTest):
ssl = True
def test_simple(self):
p = self.pathoc()
req = p.request("get:'/p/418:b\"content\"'")
assert req.content == "content"
assert req.status_code == 418
assert self.chain[1].tmaster.state.flow_count() == 2 # CONNECT from pathoc to chain[0],
# request from pathoc to chain[0]
assert self.chain[0].tmaster.state.flow_count() == 2 # CONNECT from chain[1] to proxy,
# request from chain[1] to proxy
assert self.proxy.tmaster.state.flow_count() == 1 # request from chain[0] (regular proxy doesn't store CONNECTs)
def test_closing_connect_response(self):
"""
https://github.com/mitmproxy/mitmproxy/issues/313
"""
def handle_request(f):
f.request.httpversion = (1, 0)
del f.request.headers["Content-Length"]
f.reply()
_handle_request = self.chain[0].tmaster.handle_request
self.chain[0].tmaster.handle_request = handle_request
try:
assert self.pathoc().request("get:/p/418").status_code == 418
finally:
self.chain[0].tmaster.handle_request = _handle_request
def test_sni(self):
p = self.pathoc(sni="foo.com")
req = p.request("get:'/p/418:b\"content\"'")
assert req.content == "content"
assert req.status_code == 418
class TestProxyChainingSSLReconnect(tservers.HTTPChainProxyTest):
ssl = True
def test_reconnect(self):
"""
Tests proper functionality of ConnectionHandler.server_reconnect mock.
If we have a disconnect on a secure connection that's transparently proxified to
an upstream http proxy, we need to send the CONNECT request again.
"""
def kill_requests(master, attr, exclude):
k = [0] # variable scope workaround: put into array
_func = getattr(master, attr)
def handler(f):
k[0] += 1
if not (k[0] in exclude):
f.client_conn.finish()
f.error = Error("terminated")
f.reply(KILL)
return _func(f)
setattr(master, attr, handler)
kill_requests(self.proxy.tmaster, "handle_request",
exclude=[
# fail first request
2, # allow second request
])
kill_requests(self.chain[0].tmaster, "handle_request",
exclude=[
1, # CONNECT
# fail first request
3, # reCONNECT
4, # request
])
p = self.pathoc()
req = p.request("get:'/p/418:b\"content\"'")
assert self.chain[1].tmaster.state.flow_count() == 2 # CONNECT and request
assert self.chain[0].tmaster.state.flow_count() == 4 # CONNECT, failing request,
# reCONNECT, request
assert self.proxy.tmaster.state.flow_count() == 2 # failing request, request
# (doesn't store (repeated) CONNECTs from chain[0]
# as it is a regular proxy)
assert req.content == "content"
assert req.status_code == 418
assert not self.proxy.tmaster.state._flow_list[0].response # killed
assert self.proxy.tmaster.state._flow_list[1].response
assert self.chain[1].tmaster.state._flow_list[0].request.form_in == "authority"
assert self.chain[1].tmaster.state._flow_list[1].request.form_in == "relative"
assert self.chain[0].tmaster.state._flow_list[0].request.form_in == "authority"
assert self.chain[0].tmaster.state._flow_list[1].request.form_in == "relative"
assert self.chain[0].tmaster.state._flow_list[2].request.form_in == "authority"
assert self.chain[0].tmaster.state._flow_list[3].request.form_in == "relative"
assert self.proxy.tmaster.state._flow_list[0].request.form_in == "relative"
assert self.proxy.tmaster.state._flow_list[1].request.form_in == "relative"
req = p.request("get:'/p/418:b\"content2\"'")
assert req.status_code == 502
assert self.chain[1].tmaster.state.flow_count() == 3 # + new request
assert self.chain[0].tmaster.state.flow_count() == 6 # + new request, repeated CONNECT from chain[1]
# (both terminated)
assert self.proxy.tmaster.state.flow_count() == 2 # nothing happened here
assert "Invalid HTTP request form" in r.content

17
test/test_proxy.py
View File

@@ -23,25 +23,34 @@ class TestServerConnection:
self.d.shutdown()
def test_simple(self):
sc = ServerConnection((self.d.IFACE, self.d.port), None)
sc = ServerConnection((self.d.IFACE, self.d.port))
sc.connect()
f = tutils.tflow()
f.server_conn = sc
f.request.path = "/p/200:da"
sc.send(f.request._assemble())
sc.send(f.request.assemble())
assert http.read_response(sc.rfile, f.request.method, 1000)
assert self.d.last_log()
sc.finish()
def test_terminate_error(self):
sc = ServerConnection((self.d.IFACE, self.d.port), None)
sc = ServerConnection((self.d.IFACE, self.d.port))
sc.connect()
sc.connection = mock.Mock()
sc.connection.recv = mock.Mock(return_value=False)
sc.connection.flush = mock.Mock(side_effect=tcp.NetLibDisconnect)
sc.finish()
def test_repr(self):
sc = tutils.tserver_conn()
assert "address:22" in repr(sc)
assert "ssl" not in repr(sc)
sc.ssl_established = True
assert "ssl" in repr(sc)
sc.sni = "foo"
assert "foo" in repr(sc)
class TestProcessProxyOptions:
def p(self, *args):
@@ -112,7 +121,7 @@ class TestProcessProxyOptions:
class TestProxyServer:
@tutils.SkipWindows # binding to 0.0.0.0:1 works without special permissions on Windows
@tutils.SkipWindows # binding to 0.0.0.0:1 works without special permissions on Windows
def test_err(self):
parser = argparse.ArgumentParser()
cmdline.common_options(parser)

2
test/test_script.py
View File

@@ -99,7 +99,7 @@ class TestScript:
d = Dummy()
assert s.run(hook, d)[0]
d.reply()
while (time.time() - t_start) < 5 and m.call_count <= 5:
while (time.time() - t_start) < 20 and m.call_count <= 5:
if m.call_count == 5:
return
time.sleep(0.001)

239
test/test_server.py
View File

@@ -1,10 +1,10 @@
import socket, time
import mock
from libmproxy.proxy.config import ProxyConfig
from netlib import tcp, http_auth, http
from libpathod import pathoc, pathod
from netlib.certutils import SSLCert
import tutils, tservers
from libmproxy import flow
from libmproxy.protocol import KILL
from libmproxy.protocol import KILL, Error
from libmproxy.protocol.http import CONTENT_MISSING
"""
@@ -21,8 +21,11 @@ class CommonMixin:
def test_replay(self):
assert self.pathod("304").status_code == 304
assert len(self.master.state.view) == 1
l = self.master.state.view[0]
if isinstance(self, tservers.HTTPUpstreamProxTest) and self.ssl:
assert len(self.master.state.view) == 2
else:
assert len(self.master.state.view) == 1
l = self.master.state.view[-1]
assert l.response.code == 304
l.request.path = "/p/305"
rt = self.master.replay_request(l, block=True)
@@ -31,18 +34,28 @@ class CommonMixin:
# Disconnect error
l.request.path = "/p/305:d0"
rt = self.master.replay_request(l, block=True)
assert l.error
assert not rt
if isinstance(self, tservers.HTTPUpstreamProxTest):
assert l.response.code == 502
else:
assert l.error
# Port error
l.request.port = 1
self.master.replay_request(l, block=True)
assert l.error
# In upstream mode, we get a 502 response from the upstream proxy server.
# In upstream mode with ssl, the replay will fail as we cannot establish SSL with the upstream proxy.
rt = self.master.replay_request(l, block=True)
assert not rt
if isinstance(self, tservers.HTTPUpstreamProxTest) and not self.ssl:
assert l.response.code == 502
else:
assert l.error
def test_http(self):
f = self.pathod("304")
assert f.status_code == 304
l = self.master.state.view[0]
l = self.master.state.view[-1] # In Upstream mode with SSL, we may already have a previous CONNECT request.
assert l.client_conn.address
assert "host" in l.request.headers
assert l.response.code == 304
@@ -55,6 +68,51 @@ class CommonMixin:
line = t.rfile.readline()
assert ("Bad Request" in line) or ("Bad Gateway" in line)
def test_sni(self):
if not self.ssl:
return
f = self.pathod("304", sni="testserver.com")
assert f.status_code == 304
log = self.server.last_log()
assert log["request"]["sni"] == "testserver.com"
class TcpMixin:
def _ignore_on(self):
conf = ProxyConfig(ignore=[".+:%s" % self.server.port])
self.config.ignore.append(conf.ignore[0])
def _ignore_off(self):
self.config.ignore.pop()
def test_ignore(self):
spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"'
n = self.pathod(spec)
self._ignore_on()
i = self.pathod(spec)
i2 = self.pathod(spec)
self._ignore_off()
assert i.status_code == i2.status_code == n.status_code == 304
assert "Alternate-Protocol" in i.headers
assert "Alternate-Protocol" in i2.headers
assert "Alternate-Protocol" not in n.headers
# Test that we get the original SSL cert
if self.ssl:
i_cert = SSLCert(i.sslinfo.certchain[0])
i2_cert = SSLCert(i2.sslinfo.certchain[0])
n_cert = SSLCert(n.sslinfo.certchain[0])
assert i_cert == i2_cert
assert i_cert != n_cert
# Test Non-HTTP traffic
spec = "200:i0,@100:d0" # this results in just 100 random bytes
assert self.pathod(spec).status_code == 502 # mitmproxy responds with bad gateway
self._ignore_on()
tutils.raises("invalid server response", self.pathod, spec) # pathoc tries to parse answer as HTTP
self._ignore_off()
class AppMixin:
@@ -64,7 +122,6 @@ class AppMixin:
assert "mitmproxy" in ret.content
class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
def test_app_err(self):
p = self.pathoc()
@@ -175,7 +232,7 @@ class TestHTTPConnectSSLError(tservers.HTTPProxTest):
tutils.raises("502 - Bad Gateway", p.http_connect, dst)
class TestHTTPS(tservers.HTTPProxTest, CommonMixin):
class TestHTTPS(tservers.HTTPProxTest, CommonMixin, TcpMixin):
ssl = True
ssloptions = pathod.SSLOptions(request_client_cert=True)
clientcerts = True
@@ -184,12 +241,6 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin):
assert f.status_code == 304
assert self.server.last_log()["request"]["clientcert"]["keyinfo"]
def test_sni(self):
f = self.pathod("304", sni="testserver.com")
assert f.status_code == 304
l = self.server.last_log()
assert self.server.last_log()["request"]["sni"] == "testserver.com"
def test_error_post_connect(self):
p = self.pathoc()
assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400
@@ -217,21 +268,16 @@ class TestHTTPSNoCommonName(tservers.HTTPProxTest):
assert f.sslinfo.certchain[0].get_subject().CN == "127.0.0.1"
class TestReverse(tservers.ReverseProxTest, CommonMixin):
class TestReverse(tservers.ReverseProxTest, CommonMixin, TcpMixin):
reverse = True
class TestTransparent(tservers.TransparentProxTest, CommonMixin):
class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin):
ssl = False
class TestTransparentSSL(tservers.TransparentProxTest, CommonMixin):
class TestTransparentSSL(tservers.TransparentProxTest, CommonMixin, TcpMixin):
ssl = True
def test_sni(self):
f = self.pathod("304", sni="testserver.com")
assert f.status_code == 304
l = self.server.last_log()
assert l["request"]["sni"] == "testserver.com"
def test_sslerr(self):
p = pathoc.Pathoc(("localhost", self.proxy.port))
@@ -312,7 +358,7 @@ class TestProxy(tservers.HTTPProxTest):
f = self.pathod("200:b@100")
assert f.status_code == 200
f = self.master.state.view[0]
assert f.server_conn.peername == ("127.0.0.1", self.server.port)
assert f.server_conn.address == ("127.0.0.1", self.server.port)
class TestProxySSL(tservers.HTTPProxTest):
ssl=True
@@ -330,21 +376,19 @@ class MasterRedirectRequest(tservers.TestMaster):
def handle_request(self, f):
request = f.request
if request.path == "/p/201":
url = request.get_url(False, f)
url = request.url
new = "http://127.0.0.1:%s/p/201" % self.redirect_port
request.set_url(new, f)
request.set_url(new, f)
request.url = new
f.live.change_server(("127.0.0.1", self.redirect_port), False)
request.set_url(url, f)
request.url = url
tutils.raises("SSL handshake error", f.live.change_server, ("127.0.0.1", self.redirect_port), True)
request.set_url(new, f)
request.set_url(url, f)
request.set_url(new, f)
request.url = new
tservers.TestMaster.handle_request(self, f)
def handle_response(self, f):
f.response.content = str(f.client_conn.address.port)
f.response.headers["server-conn-id"] = [str(f.server_conn.source_address.port)]
tservers.TestMaster.handle_response(self, f)
@@ -377,7 +421,8 @@ class TestRedirectRequest(tservers.HTTPProxTest):
assert self.server.last_log()
assert not self.server2.last_log()
assert r3.content == r2.content == r1.content
assert r1.content == r2.content == r3.content
assert r1.headers.get_first("server-conn-id") == r3.headers.get_first("server-conn-id")
# Make sure that we actually use the same connection in this test case
class MasterStreamRequest(tservers.TestMaster):
@@ -502,6 +547,132 @@ class TestIncompleteResponse(tservers.HTTPProxTest):
class TestCertForward(tservers.HTTPProxTest):
certforward = True
ssl = True
def test_app_err(self):
tutils.raises("handshake error", self.pathod, "200:b@100")
class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin):
ssl = False
def test_order(self):
self.proxy.tmaster.replacehooks.add("~q", "foo", "bar") # replace in request
self.chain[0].tmaster.replacehooks.add("~q", "bar", "baz")
self.chain[1].tmaster.replacehooks.add("~q", "foo", "oh noes!")
self.chain[0].tmaster.replacehooks.add("~s", "baz", "ORLY") # replace in response
p = self.pathoc()
req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase)
assert req.content == "ORLY"
assert req.status_code == 418
class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin):
ssl = True
def _ignore_on(self):
super(TestUpstreamProxySSL, self)._ignore_on()
conf = ProxyConfig(ignore=[".+:%s" % self.server.port])
for proxy in self.chain:
proxy.tmaster.server.config.ignore.append(conf.ignore[0])
def _ignore_off(self):
super(TestUpstreamProxySSL, self)._ignore_off()
for proxy in self.chain:
proxy.tmaster.server.config.ignore.pop()
def test_simple(self):
p = self.pathoc()
req = p.request("get:'/p/418:b\"content\"'")
assert req.content == "content"
assert req.status_code == 418
assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT from pathoc to chain[0],
# request from pathoc to chain[0]
assert self.chain[0].tmaster.state.flow_count() == 2 # CONNECT from proxy to chain[1],
# request from proxy to chain[1]
assert self.chain[1].tmaster.state.flow_count() == 1 # request from chain[0] (regular proxy doesn't store CONNECTs)
def test_closing_connect_response(self):
"""
https://github.com/mitmproxy/mitmproxy/issues/313
"""
def handle_request(f):
f.request.httpversion = (1, 0)
del f.request.headers["Content-Length"]
f.reply()
_handle_request = self.chain[0].tmaster.handle_request
self.chain[0].tmaster.handle_request = handle_request
try:
assert self.pathoc().request("get:/p/418").status_code == 418
finally:
self.chain[0].tmaster.handle_request = _handle_request
class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest):
ssl = True
def test_reconnect(self):
"""
Tests proper functionality of ConnectionHandler.server_reconnect mock.
If we have a disconnect on a secure connection that's transparently proxified to
an upstream http proxy, we need to send the CONNECT request again.
"""
def kill_requests(master, attr, exclude):
k = [0] # variable scope workaround: put into array
_func = getattr(master, attr)
def handler(f):
k[0] += 1
if not (k[0] in exclude):
f.client_conn.finish()
f.error = Error("terminated")
f.reply(KILL)
return _func(f)
setattr(master, attr, handler)
kill_requests(self.chain[1].tmaster, "handle_request",
exclude=[
# fail first request
2, # allow second request
])
kill_requests(self.chain[0].tmaster, "handle_request",
exclude=[
1, # CONNECT
# fail first request
3, # reCONNECT
4, # request
])
p = self.pathoc()
req = p.request("get:'/p/418:b\"content\"'")
assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT and request
assert self.chain[0].tmaster.state.flow_count() == 4 # CONNECT, failing request,
# reCONNECT, request
assert self.chain[1].tmaster.state.flow_count() == 2 # failing request, request
# (doesn't store (repeated) CONNECTs from chain[0]
# as it is a regular proxy)
assert req.content == "content"
assert req.status_code == 418
assert not self.chain[1].tmaster.state._flow_list[0].response # killed
assert self.chain[1].tmaster.state._flow_list[1].response
assert self.proxy.tmaster.state._flow_list[0].request.form_in == "authority"
assert self.proxy.tmaster.state._flow_list[1].request.form_in == "relative"
assert self.chain[0].tmaster.state._flow_list[0].request.form_in == "authority"
assert self.chain[0].tmaster.state._flow_list[1].request.form_in == "relative"
assert self.chain[0].tmaster.state._flow_list[2].request.form_in == "authority"
assert self.chain[0].tmaster.state._flow_list[3].request.form_in == "relative"
assert self.chain[1].tmaster.state._flow_list[0].request.form_in == "relative"
assert self.chain[1].tmaster.state._flow_list[1].request.form_in == "relative"
req = p.request("get:'/p/418:b\"content2\"'")
assert req.status_code == 502
assert self.proxy.tmaster.state.flow_count() == 3 # + new request
assert self.chain[0].tmaster.state.flow_count() == 6 # + new request, repeated CONNECT from chain[1]
# (both terminated)
assert self.chain[1].tmaster.state.flow_count() == 2 # nothing happened here

107
test/tservers.py
View File

@@ -84,29 +84,19 @@ class ProxTestBase(object):
masterclass = TestMaster
externalapp = False
certforward = False
@classmethod
def setupAll(cls):
cls.server = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions)
cls.server2 = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions)
pconf = cls.get_proxy_config()
cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy")
cls.config = ProxyConfig(
no_upstream_cert = cls.no_upstream_cert,
confdir = cls.confdir,
authenticator = cls.authenticator,
certforward = cls.certforward,
ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []),
**pconf
)
cls.config = ProxyConfig(**cls.get_proxy_config())
tmaster = cls.masterclass(cls.config)
tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp)
cls.proxy = ProxyThread(tmaster)
cls.proxy.start()
@property
def master(cls):
return cls.proxy.tmaster
@classmethod
def teardownAll(cls):
shutil.rmtree(cls.confdir)
@@ -121,24 +111,20 @@ class ProxTestBase(object):
self.server2.clear_log()
@property
def scheme(self):
return "https" if self.ssl else "http"
@property
def proxies(self):
"""
The URL base for the server instance.
"""
return (
(self.scheme, ("127.0.0.1", self.proxy.port))
)
def master(self):
return self.proxy.tmaster
@classmethod
def get_proxy_config(cls):
d = dict()
if cls.clientcerts:
d["clientcerts"] = tutils.test_data.path("data/clientcert")
return d
cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy")
return dict(
no_upstream_cert = cls.no_upstream_cert,
confdir = cls.confdir,
authenticator = cls.authenticator,
certforward = cls.certforward,
ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []),
clientcerts = tutils.test_data.path("data/clientcert") if cls.clientcerts else None
)
class HTTPProxTest(ProxTestBase):
@@ -265,49 +251,50 @@ class ReverseProxTest(ProxTestBase):
class ChainProxTest(ProxTestBase):
"""
Chain n instances of mitmproxy in a row - because we can.
Chain three instances of mitmproxy in a row to test upstream mode.
Proxy order is cls.proxy -> cls.chain[0] -> cls.chain[1]
cls.proxy and cls.chain[0] are in upstream mode,
cls.chain[1] is in regular mode.
"""
chain = None
n = 2
chain_config = [lambda port, sslports: ProxyConfig(
upstream_server= (False, False, "127.0.0.1", port),
http_form_in = "absolute",
http_form_out = "absolute",
ssl_ports=sslports
)] * n
@classmethod
def setupAll(cls):
super(ChainProxTest, cls).setupAll()
cls.chain = []
for i in range(cls.n):
sslports = [cls.server.port, cls.server2.port]
config = cls.chain_config[i](cls.proxy.port if i == 0 else cls.chain[-1].port,
sslports)
super(ChainProxTest, cls).setupAll()
for _ in range(cls.n):
config = ProxyConfig(**cls.get_proxy_config())
tmaster = cls.masterclass(config)
tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp)
cls.chain.append(ProxyThread(tmaster))
cls.chain[-1].start()
proxy = ProxyThread(tmaster)
proxy.start()
cls.chain.insert(0, proxy)
# Patch the orginal proxy to upstream mode
cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(**cls.get_proxy_config())
@classmethod
def teardownAll(cls):
super(ChainProxTest, cls).teardownAll()
for p in cls.chain:
p.tmaster.shutdown()
for proxy in cls.chain:
proxy.shutdown()
def setUp(self):
super(ChainProxTest, self).setUp()
for p in self.chain:
p.tmaster.clear_log()
p.tmaster.state.clear()
for proxy in self.chain:
proxy.tmaster.clear_log()
proxy.tmaster.state.clear()
@classmethod
def get_proxy_config(cls):
d = super(ChainProxTest, cls).get_proxy_config()
if cls.chain: # First proxy is in normal mode.
d.update(
mode="upstream",
upstream_server=(False, False, "127.0.0.1", cls.chain[0].port)
)
return d
class HTTPChainProxyTest(ChainProxTest):
def pathoc(self, sni=None):
"""
Returns a connected Pathoc instance.
"""
p = libpathod.pathoc.Pathoc(("localhost", self.chain[-1].port), ssl=self.ssl, sni=sni)
if self.ssl:
p.connect(("127.0.0.1", self.server.port))
else:
p.connect()
return p
class HTTPUpstreamProxTest(ChainProxTest, HTTPProxTest):
pass