Move human-friendly format functions to netlib.human, remove redundant implementations

mitmproxy/cmdline.py

@@ -7,7 +7,8 @@ import configargparse
 
 from netlib.tcp import Address, sslversion_choices
 import netlib.http.url
-from . import filt, utils, version
+from netlib import human
+from . import filt, version
 from .proxy import config
 
 APP_HOST = "mitm.it"
@@ -135,7 +136,9 @@ def get_common_options(options):
     if options.stickyauth_filt:
         stickyauth = options.stickyauth_filt
 
-    stream_large_bodies = utils.parse_size(options.stream_large_bodies)
+    stream_large_bodies = options.stream_large_bodies
+    if stream_large_bodies:
+        stream_large_bodies = human.parse_size(stream_large_bodies)
 
     reps = []
     for i in options.replace:

mitmproxy/console/common.py

@@ -4,7 +4,8 @@ import urwid
 import urwid.util
 import os
 
-import netlib.utils
+import netlib
+from netlib import human
 
 from .. import utils
 from .. import flow
@@ -419,7 +420,7 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False):
     )
     if f.response:
         if f.response.content:
-            contentdesc = netlib.utils.pretty_size(len(f.response.content))
+            contentdesc = human.pretty_size(len(f.response.content))
         elif f.response.content is None:
             contentdesc = "[content missing]"
         else:
@@ -427,7 +428,7 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False):
         duration = 0
         if f.response.timestamp_end and f.request.timestamp_start:
             duration = f.response.timestamp_end - f.request.timestamp_start
-        roundtrip = utils.pretty_duration(duration)
+        roundtrip = human.pretty_duration(duration)
 
         d.update(dict(
             resp_code = f.response.status_code,

mitmproxy/console/statusbar.py

@@ -3,6 +3,7 @@ import os.path
 import urwid
 
 import netlib.utils
+from netlib import human
 from . import pathedit, signals, common
 
 
@@ -193,7 +194,7 @@ class StatusBar(urwid.WidgetWrap):
             opts.append("following")
         if self.master.stream_large_bodies:
             opts.append(
-                "stream:%s" % netlib.utils.pretty_size(
+                "stream:%s" % human.pretty_size(
                     self.master.stream_large_bodies.max_size
                 )
             )

mitmproxy/dump.py

@@ -4,8 +4,8 @@ import sys
 import click
 import itertools
 
-from netlib import tcp
-from netlib.utils import bytes_to_escaped_str, pretty_size
+from netlib import tcp, human
+from netlib.utils import bytes_to_escaped_str
 from . import flow, filt, contentviews, controller
 from .exceptions import ContentViewException, FlowReadException, ScriptException
 
@@ -287,7 +287,7 @@ class DumpMaster(flow.FlowMaster):
         if flow.response.content is None:
             size = "(content missing)"
         else:
-            size = pretty_size(len(flow.response.content))
+            size = human.pretty_size(len(flow.response.content))
         size = click.style(size, bold=True)
 
         arrows = click.style("<<", bold=True)

mitmproxy/proxy/config.py

@@ -6,11 +6,11 @@ import re
 import six
 from OpenSSL import SSL
 
-from netlib import certutils, tcp
+from netlib import certutils, tcp, human
 from netlib.http import authentication
 from netlib.tcp import Address, sslversion_choices
 
-from .. import utils, platform
+from .. import platform
 
 CONF_BASENAME = "mitmproxy"
 CA_DIR = "~/.mitmproxy"
@@ -125,7 +125,9 @@ class ProxyConfig:
 
 
 def process_proxy_options(parser, options):
-    body_size_limit = utils.parse_size(options.body_size_limit)
+    body_size_limit = options.body_size_limit
+    if body_size_limit:
+        body_size_limit = human.parse_size(body_size_limit)
 
     c = 0
     mode, upstream_server, upstream_auth = "regular", None, None

mitmproxy/utils.py

@@ -58,20 +58,6 @@ def pretty_json(s):
     return json.dumps(p, sort_keys=True, indent=4)
 
 
-def pretty_duration(secs):
-    formatters = [
-        (100, "{:.0f}s"),
-        (10, "{:2.1f}s"),
-        (1, "{:1.2f}s"),
-    ]
-
-    for limit, formatter in formatters:
-        if secs >= limit:
-            return formatter.format(secs)
-    # less than 1 sec
-    return "{:.0f}ms".format(secs * 1000)
-
-
 pkg_data = netlib.utils.Data(__name__)
 
 
@@ -117,32 +103,3 @@ def clean_hanging_newline(t):
     if t and t[-1] == "\n":
         return t[:-1]
     return t
-
-
-def parse_size(s):
-    """
-        Parses a size specification. Valid specifications are:
-
-            123: bytes
-            123k: kilobytes
-            123m: megabytes
-            123g: gigabytes
-    """
-    if not s:
-        return None
-    mult = None
-    if s[-1].lower() == "k":
-        mult = 1024**1
-    elif s[-1].lower() == "m":
-        mult = 1024**2
-    elif s[-1].lower() == "g":
-        mult = 1024**3
-
-    if mult:
-        s = s[:-1]
-    else:
-        mult = 1
-    try:
-        return int(s) * mult
-    except ValueError:
-        raise ValueError("Invalid size specification: %s" % s)

netlib/http/response.py

@@ -7,7 +7,7 @@ from . import cookies
 from .headers import Headers
 from .message import Message, _native, _always_bytes, MessageData
 from ..multidict import MultiDictView
-from .. import utils
+from .. import human
 
 
 class ResponseData(MessageData):
@@ -36,7 +36,7 @@ class Response(Message):
         if self.content:
             details = "{}, {}".format(
                 self.headers.get("content-type", "unknown content type"),
-                utils.pretty_size(len(self.content))
+                human.pretty_size(len(self.content))
             )
         else:
             details = "no content"

netlib/human.py

@@ -0,0 +1,51 @@
+SIZE_UNITS = dict(
+    b=1024 ** 0,
+    k=1024 ** 1,
+    m=1024 ** 2,
+    g=1024 ** 3,
+    t=1024 ** 4,
+)
+
+
+def pretty_size(size):
+    suffixes = [
+        ("B", 2 ** 10),
+        ("kB", 2 ** 20),
+        ("MB", 2 ** 30),
+    ]
+    for suf, lim in suffixes:
+        if size >= lim:
+            continue
+        else:
+            x = round(size / float(lim / 2 ** 10), 2)
+            if x == int(x):
+                x = int(x)
+            return str(x) + suf
+
+
+def parse_size(s):
+    try:
+        return int(s)
+    except ValueError:
+        pass
+    for i in SIZE_UNITS.keys():
+        if s.endswith(i):
+            try:
+                return int(s[:-1]) * SIZE_UNITS[i]
+            except ValueError:
+                break
+    raise ValueError("Invalid size specification.")
+
+
+def pretty_duration(secs):
+    formatters = [
+        (100, "{:.0f}s"),
+        (10, "{:2.1f}s"),
+        (1, "{:1.2f}s"),
+    ]
+
+    for limit, formatter in formatters:
+        if secs >= limit:
+            return formatter.format(secs)
+    # less than 1 sec
+    return "{:.0f}ms".format(secs * 1000)

netlib/utils.py

@@ -116,22 +116,6 @@ class BiDi(object):
         return self.values.get(n, default)
 
 
-def pretty_size(size):
-    suffixes = [
-        ("B", 2 ** 10),
-        ("kB", 2 ** 20),
-        ("MB", 2 ** 30),
-    ]
-    for suf, lim in suffixes:
-        if size >= lim:
-            continue
-        else:
-            x = round(size / float(lim / 2 ** 10), 2)
-            if x == int(x):
-                x = int(x)
-            return str(x) + suf
-
-
 class Data(object):
 
     def __init__(self, name):

netlib/websockets/frame.py

@@ -9,6 +9,7 @@ import six
 from .protocol import Masker
 from netlib import tcp
 from netlib import utils
+from netlib import human
 
 
 MAX_16_BIT_INT = (1 << 16)
@@ -98,7 +99,7 @@ class FrameHeader(object):
         if self.masking_key:
             vals.append(":key=%s" % repr(self.masking_key))
         if self.payload_length:
-            vals.append(" %s" % utils.pretty_size(self.payload_length))
+            vals.append(" %s" % human.pretty_size(self.payload_length))
         return "".join(vals)
 
     def human_readable(self):

pathod/language/base.py

@@ -5,8 +5,8 @@ import pyparsing as pp
 
 from six.moves import reduce
 from netlib.utils import escaped_str_to_bytes, bytes_to_escaped_str
+from netlib import human
 
-from .. import utils
 from . import generators, exceptions
 
 
@@ -158,7 +158,7 @@ class TokValueGenerate(Token):
         self.usize, self.unit, self.datatype = usize, unit, datatype
 
     def bytes(self):
-        return self.usize * utils.SIZE_UNITS[self.unit]
+        return self.usize * human.SIZE_UNITS[self.unit]
 
     def get_generator(self, settings_):
         return generators.RandomGenerator(self.datatype, self.bytes())
@@ -173,7 +173,7 @@ class TokValueGenerate(Token):
 
         u = reduce(
             operator.or_,
-            [pp.Literal(i) for i in utils.SIZE_UNITS.keys()]
+            [pp.Literal(i) for i in human.SIZE_UNITS.keys()]
         ).leaveWhitespace()
         e = e + pp.Optional(u, default=None)
 

pathod/pathod_cmdline.py

@@ -4,7 +4,7 @@ import os
 import os.path
 import re
 
-from netlib import tcp
+from netlib import tcp, human
 from . import pathod, version, utils
 
 
@@ -205,7 +205,7 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
     sizelimit = None
     if args.sizelimit:
         try:
-            sizelimit = utils.parse_size(args.sizelimit)
+            sizelimit = human.parse_size(args.sizelimit)
         except ValueError as v:
             return parser.error(v)
     args.sizelimit = sizelimit

pathod/utils.py

@@ -5,15 +5,6 @@ import netlib.utils
 from netlib.utils import bytes_to_escaped_str
 
 
-SIZE_UNITS = dict(
-    b=1024 ** 0,
-    k=1024 ** 1,
-    m=1024 ** 2,
-    g=1024 ** 3,
-    t=1024 ** 4,
-)
-
-
 class MemBool(object):
 
     """
@@ -28,20 +19,6 @@ class MemBool(object):
         return bool(v)
 
 
-def parse_size(s):
-    try:
-        return int(s)
-    except ValueError:
-        pass
-    for i in SIZE_UNITS.keys():
-        if s.endswith(i):
-            try:
-                return int(s[:-1]) * SIZE_UNITS[i]
-            except ValueError:
-                break
-    raise ValueError("Invalid size specification.")
-
-
 def parse_anchor_spec(s):
     """
         Return a tuple, or None on error.

test/mitmproxy/test_utils.py

@@ -43,21 +43,6 @@ def test_pretty_json():
     assert not utils.pretty_json("moo")
 
 
-def test_pretty_duration():
-    assert utils.pretty_duration(0.00001) == "0ms"
-    assert utils.pretty_duration(0.0001) == "0ms"
-    assert utils.pretty_duration(0.001) == "1ms"
-    assert utils.pretty_duration(0.01) == "10ms"
-    assert utils.pretty_duration(0.1) == "100ms"
-    assert utils.pretty_duration(1) == "1.00s"
-    assert utils.pretty_duration(10) == "10.0s"
-    assert utils.pretty_duration(100) == "100s"
-    assert utils.pretty_duration(1000) == "1000s"
-    assert utils.pretty_duration(10000) == "10000s"
-    assert utils.pretty_duration(1.123) == "1.12s"
-    assert utils.pretty_duration(0.123) == "123ms"
-
-
 def test_LRUCache():
     cache = utils.LRUCache(2)
 
@@ -89,13 +74,3 @@ def test_LRUCache():
 
     assert len(cache.cacheList) == 2
     assert len(cache.cache) == 2
-
-
-def test_parse_size():
-    assert not utils.parse_size("")
-    assert utils.parse_size("1") == 1
-    assert utils.parse_size("1k") == 1024
-    assert utils.parse_size("1m") == 1024**2
-    assert utils.parse_size("1g") == 1024**3
-    tutils.raises(ValueError, utils.parse_size, "1f")
-    tutils.raises(ValueError, utils.parse_size, "ak")

test/netlib/test_human.py

@@ -0,0 +1,31 @@
+from netlib import human, tutils
+
+def test_parse_size():
+    assert human.parse_size("1") == 1
+    assert human.parse_size("1k") == 1024
+    assert human.parse_size("1m") == 1024**2
+    assert human.parse_size("1g") == 1024**3
+    tutils.raises(ValueError, human.parse_size, "1f")
+    tutils.raises(ValueError, human.parse_size, "ak")
+
+
+def test_pretty_size():
+    assert human.pretty_size(100) == "100B"
+    assert human.pretty_size(1024) == "1kB"
+    assert human.pretty_size(1024 + (1024 / 2.0)) == "1.5kB"
+    assert human.pretty_size(1024 * 1024) == "1MB"
+
+
+def test_pretty_duration():
+    assert human.pretty_duration(0.00001) == "0ms"
+    assert human.pretty_duration(0.0001) == "0ms"
+    assert human.pretty_duration(0.001) == "1ms"
+    assert human.pretty_duration(0.01) == "10ms"
+    assert human.pretty_duration(0.1) == "100ms"
+    assert human.pretty_duration(1) == "1.00s"
+    assert human.pretty_duration(10) == "10.0s"
+    assert human.pretty_duration(100) == "100s"
+    assert human.pretty_duration(1000) == "1000s"
+    assert human.pretty_duration(10000) == "10000s"
+    assert human.pretty_duration(1.123) == "1.12s"
+    assert human.pretty_duration(0.123) == "123ms"

test/netlib/test_utils.py

@@ -30,13 +30,6 @@ def test_clean_bin():
     assert utils.clean_bin(u"\u2605") == u"\u2605"
 
 
-def test_pretty_size():
-    assert utils.pretty_size(100) == "100B"
-    assert utils.pretty_size(1024) == "1kB"
-    assert utils.pretty_size(1024 + (1024 / 2.0)) == "1.5kB"
-    assert utils.pretty_size(1024 * 1024) == "1MB"
-
-
 def test_safe_subn():
     assert utils.safe_subn("foo", u"bar", "\xc2foo")
 

test/pathod/test_utils.py

@@ -13,13 +13,6 @@ def test_membool():
     assert m.v == 2
 
 
-def test_parse_size():
-    assert utils.parse_size("100") == 100
-    assert utils.parse_size("100k") == 100 * 1024
-    tutils.raises("invalid size spec", utils.parse_size, "foo")
-    tutils.raises("invalid size spec", utils.parse_size, "100kk")
-
-
 def test_parse_anchor_spec():
     assert utils.parse_anchor_spec("foo=200") == ("foo", "200")
     assert utils.parse_anchor_spec("foo") is None