simple benchmarking tool

integration_tests/bin/blockstack-benchmark

@@ -0,0 +1,280 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""
+    Blockstack
+    ~~~~~
+    copyright: (c) 2014-2015 by Halfmoon Labs, Inc.
+    copyright: (c) 2016-2018 by Blockstack.org
+
+    This file is part of Blockstack
+
+    Blockstack is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Blockstack is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+    You should have received a copy of the GNU General Public License
+    along with Blockstack. If not, see <http://www.gnu.org/licenses/>.
+"""
+import time
+import os
+import sys
+import argparse
+import random
+import urlparse
+import json
+
+import virtualchain
+import blockstack
+import blockstack.lib.client as blockstack_client
+
+log = virtualchain.get_logger('blockstack-benchmark')
+
+def benchmark_rpc(client, iterations, method_name, *args, **kw):
+    """
+    Benchmark a given RPC call.
+    Returns [{'time': ..., 'response': ...}]
+    """
+    rpc_call = getattr(client, method_name)
+    assert rpc_call, 'No such RPC call: {}'.format(rpc_call)
+
+    res = rpc_call(*args, **kw)
+    ret = []
+    for i in range(0, iterations):
+        log.debug("Call {}({}) (count {} of {})".format(method_name, ','.join(str(a) for a in args), i, iterations))
+
+        t1 = time.time()
+        res = rpc_call(*args, **kw)
+        t2 = time.time()
+
+        ret.append({'time': t2 - t1, 'response': res})
+
+        log.debug("Call {}({}) (count {} of {}) time={}".format(method_name, ','.join(str(a) for a in args), i, iterations, t2 - t1))
+
+    return ret
+
+
+def get_benchmark_times(benchmark_data, ignore_errors=True):
+    """
+    Get the list of method response times from the benchmark data.
+    By default, ignore error responses.
+    """
+    ret = []
+    for data in benchmark_data:
+        if ignore_errors and blockstack_client.json_is_error(data['response']):
+            continue
+
+        ret.append(data['time'])
+
+    return ret
+
+
+def make_histogram(values, minval, maxval, num_buckets):
+    """
+    Given a sequence of values, create an histogram with a given number of buckets.
+    Returns the CDF's data for plotting it as an histogram
+    """
+    if minval > min(values):
+        raise ValueError("minval == {}; min(values) == {}".format(minval, min(values)))
+
+    if maxval < max(values):
+        raise ValueError("maxval == {}; max(values) == {}".format(maxval, max(values)))
+
+    buckets = [0] * num_buckets
+    for v in values:
+        # interpolate
+        bucket = int(float(v - minval) * num_buckets / float(maxval - minval))
+        if bucket > num_buckets or bucket < 0:
+            raise ValueError('(({} - {}) * {} / ({} - {})) == {}'.format(v, minval, num_buckets, maxval, minval, bucket))
+
+        if bucket == len(buckets):
+            # happens when v == maxval
+            bucket = len(buckets) - 1
+
+        buckets[bucket] += 1
+
+    return {'buckets': buckets, 'minval': minval, 'maxval': maxval}
+
+
+def make_cdf(values, minval, maxval, num_buckets):
+    """
+    Given a sequence of values, create a (bucketed) cumulative distribution function.
+    Returns the CDF's values, between 0 and 1.
+    """
+    histogram_data = make_histogram(values, minval, maxval, num_buckets)
+    histogram = histogram_data['buckets']
+
+    total_values = sum(histogram)
+    cdf = [0] * (len(histogram)+1)
+
+    for i in range(0, len(histogram)):
+        cdf[i+1] = cdf[i] + histogram[i]
+    
+    cdf = cdf[1:]
+
+    # normal_cdf = [float(cdf[i]) / total_values for i in range(0, len(cdf))]
+    return {'buckets': cdf, 'minval': 0, 'maxval': num_buckets}
+    
+
+def make_ccdf(values, minval, maxval, num_buckets):
+    """
+    Given a sequence of values, create a (bucketed) complementary cumulative distribution function.
+    Returns the CCDF's data for plotting it as an histogram
+    """
+    histogram_data = make_histogram(values, minval, maxval, num_buckets)
+    histogram = histogram_data['buckets']
+
+    total_values = sum(histogram)
+    cdf = [0] * (len(histogram)+1)
+
+    for i in range(0, len(histogram)):
+        cdf[i+1] = cdf[i] + histogram[i]
+    
+    cdf = cdf[1:]
+    ccdf = [max(cdf) - cdf[i] for i in range(0, len(cdf))]
+
+    # normal_ccdf = [float(ccdf[i]) / total_values for i in range(0, len(ccdf))]
+    return {'buckets': ccdf, 'minval': 0, 'maxval': num_buckets}
+
+
+def format_histogram(histogram_data):
+    """
+    Render an histogram as a string
+    """
+    minval = histogram_data['minval']
+    maxval = histogram_data['maxval']
+    histogram = histogram_data['buckets']
+
+    ret = ""
+    pad_fmt = '{: >10f}-{: >10f}'
+    for i in range(0, len(histogram)):
+        bucket_min = minval + (float(i) / len(histogram)) * float(maxval - minval)
+        bucket_max = minval + (float(i+1) / len(histogram)) * float(maxval - minval)
+
+        ret += pad_fmt.format(bucket_min, bucket_max) + ': ' + ('#' * histogram[i]) + ' ({})\n'.format(histogram[i])
+
+    return ret
+
+
+def main(argv):
+    """
+    Main method
+    """
+    argparser = argparse.ArgumentParser(description="blockstack-benchmark")
+    subparsers = argparser.add_subparsers(
+        dest='action', help='the action to be taken')
+
+    host_url = 'http://localhost:6264'
+
+    # ---------------------------
+    parser = subparsers.add_parser(
+        'benchmark',
+        help='benchmark an RPC method')
+    
+    parser.add_argument('method', action='store', help='Method to benchmark')
+    parser.add_argument('iterations', action='store', type=int, help='Number of iterations')
+    parser.add_argument('args', nargs='*', action='store', help='Method arguments, if any')
+    parser.add_argument('--url', action='store', help='Blockstackd URL')
+    parser.add_argument('--histogram', action='store_true', help='Print an histogram of the benchmarked method response times')
+    parser.add_argument('--cdf', action='store_true', help='Print a CDF of the benchmarked method response times')
+    parser.add_argument('--ccdf', action='store_true', help='Print a CCDF of the benchmarked method response times')
+    parser.add_argument('--full-responses', action='store_true', help='Print full responses from the node')
+    parser.add_argument('--include-errors', action='store_true', help='Include benchmark data from errors')
+
+    # ---------------------------
+    args, _ = argparser.parse_known_args()
+
+    if args.action == 'benchmark':
+        method = args.method
+        iters = args.iterations
+        method_args = args.args
+        url = args.url
+
+        if url is None:
+            url = host_url
+        
+        log.debug("Blockstack URL: {}".format(url))
+
+        parsed_args = []
+        for method_arg in method_args:
+            try:
+                method_arg = int(method_arg)
+                parsed_args.append(method_arg)
+                continue
+            except:
+                pass
+
+            try:
+                method_arg = json.loads(method_arg)
+                parsed_args.append(method_arg)
+                continue
+            except:
+                pass
+
+        client = blockstack_client.connect_hostport(url)
+        benchmark_data = benchmark_rpc(client, iters, method, *method_args)
+        times = get_benchmark_times(benchmark_data, ignore_errors=(not args.include_errors))
+
+        print_graph = False
+
+        if args.histogram:
+            print_graph = True
+            hist = make_histogram(times, min(times), max(times), 10)
+            print format_histogram(hist)
+
+        if args.cdf:
+            print_graph = True
+            cdf = make_cdf(times, min(times), max(times), 100)
+            print format_histogram(cdf)
+       
+        if args.ccdf:
+            print_graph = True
+            cdf = make_ccdf(times, min(times), max(times), 100)
+            print format_histogram(cdf)
+
+        if not print_graph:
+            if args.full_responses:
+                for data in benchmark_data:
+                    print json.dumps(data, sort_keys=True)
+
+            else:
+                for t in times:
+                    print t
+
+        return True
+
+    return False
+
+
+if __name__ == "__main__":
+    rc = main(sys.argv)
+    sys.exit(0 if rc else 1)
+
+    '''
+    # data = [random.randint(0, 255) for _ in xrange(0, 100)]
+    # data = xrange(0,100)
+    data = [
+        8.78195905685,
+        0.835022926331,
+        0.794891834259,
+        0.885505914688,
+        1.17201900482,
+        1.12744307518,
+        0.794220924377,
+        1.03178310394,
+        0.937813043594,
+        0.786528110504,
+    ]
+    histogram = make_histogram(data, 0, max(data), 8)
+    cdf = make_cdf(data, 0, max(data), 100)
+    ccdf = make_ccdf(data, 0, max(data), 100)
+    
+    print format_histogram(histogram)
+    print format_histogram(cdf)
+    print format_histogram(ccdf)
+    '''
+