diff --git a/integration_tests/bin/blockstack-benchmark b/integration_tests/bin/blockstack-benchmark new file mode 100755 index 000000000..3d13bf385 --- /dev/null +++ b/integration_tests/bin/blockstack-benchmark @@ -0,0 +1,280 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +""" + Blockstack + ~~~~~ + copyright: (c) 2014-2015 by Halfmoon Labs, Inc. + copyright: (c) 2016-2018 by Blockstack.org + + This file is part of Blockstack + + Blockstack is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Blockstack is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with Blockstack. If not, see . +""" +import time +import os +import sys +import argparse +import random +import urlparse +import json + +import virtualchain +import blockstack +import blockstack.lib.client as blockstack_client + +log = virtualchain.get_logger('blockstack-benchmark') + +def benchmark_rpc(client, iterations, method_name, *args, **kw): + """ + Benchmark a given RPC call. + Returns [{'time': ..., 'response': ...}] + """ + rpc_call = getattr(client, method_name) + assert rpc_call, 'No such RPC call: {}'.format(rpc_call) + + res = rpc_call(*args, **kw) + ret = [] + for i in range(0, iterations): + log.debug("Call {}({}) (count {} of {})".format(method_name, ','.join(str(a) for a in args), i, iterations)) + + t1 = time.time() + res = rpc_call(*args, **kw) + t2 = time.time() + + ret.append({'time': t2 - t1, 'response': res}) + + log.debug("Call {}({}) (count {} of {}) time={}".format(method_name, ','.join(str(a) for a in args), i, iterations, t2 - t1)) + + return ret + + +def get_benchmark_times(benchmark_data, ignore_errors=True): + """ + Get the list of method response times from the benchmark data. + By default, ignore error responses. + """ + ret = [] + for data in benchmark_data: + if ignore_errors and blockstack_client.json_is_error(data['response']): + continue + + ret.append(data['time']) + + return ret + + +def make_histogram(values, minval, maxval, num_buckets): + """ + Given a sequence of values, create an histogram with a given number of buckets. + Returns the CDF's data for plotting it as an histogram + """ + if minval > min(values): + raise ValueError("minval == {}; min(values) == {}".format(minval, min(values))) + + if maxval < max(values): + raise ValueError("maxval == {}; max(values) == {}".format(maxval, max(values))) + + buckets = [0] * num_buckets + for v in values: + # interpolate + bucket = int(float(v - minval) * num_buckets / float(maxval - minval)) + if bucket > num_buckets or bucket < 0: + raise ValueError('(({} - {}) * {} / ({} - {})) == {}'.format(v, minval, num_buckets, maxval, minval, bucket)) + + if bucket == len(buckets): + # happens when v == maxval + bucket = len(buckets) - 1 + + buckets[bucket] += 1 + + return {'buckets': buckets, 'minval': minval, 'maxval': maxval} + + +def make_cdf(values, minval, maxval, num_buckets): + """ + Given a sequence of values, create a (bucketed) cumulative distribution function. + Returns the CDF's values, between 0 and 1. + """ + histogram_data = make_histogram(values, minval, maxval, num_buckets) + histogram = histogram_data['buckets'] + + total_values = sum(histogram) + cdf = [0] * (len(histogram)+1) + + for i in range(0, len(histogram)): + cdf[i+1] = cdf[i] + histogram[i] + + cdf = cdf[1:] + + # normal_cdf = [float(cdf[i]) / total_values for i in range(0, len(cdf))] + return {'buckets': cdf, 'minval': 0, 'maxval': num_buckets} + + +def make_ccdf(values, minval, maxval, num_buckets): + """ + Given a sequence of values, create a (bucketed) complementary cumulative distribution function. + Returns the CCDF's data for plotting it as an histogram + """ + histogram_data = make_histogram(values, minval, maxval, num_buckets) + histogram = histogram_data['buckets'] + + total_values = sum(histogram) + cdf = [0] * (len(histogram)+1) + + for i in range(0, len(histogram)): + cdf[i+1] = cdf[i] + histogram[i] + + cdf = cdf[1:] + ccdf = [max(cdf) - cdf[i] for i in range(0, len(cdf))] + + # normal_ccdf = [float(ccdf[i]) / total_values for i in range(0, len(ccdf))] + return {'buckets': ccdf, 'minval': 0, 'maxval': num_buckets} + + +def format_histogram(histogram_data): + """ + Render an histogram as a string + """ + minval = histogram_data['minval'] + maxval = histogram_data['maxval'] + histogram = histogram_data['buckets'] + + ret = "" + pad_fmt = '{: >10f}-{: >10f}' + for i in range(0, len(histogram)): + bucket_min = minval + (float(i) / len(histogram)) * float(maxval - minval) + bucket_max = minval + (float(i+1) / len(histogram)) * float(maxval - minval) + + ret += pad_fmt.format(bucket_min, bucket_max) + ': ' + ('#' * histogram[i]) + ' ({})\n'.format(histogram[i]) + + return ret + + +def main(argv): + """ + Main method + """ + argparser = argparse.ArgumentParser(description="blockstack-benchmark") + subparsers = argparser.add_subparsers( + dest='action', help='the action to be taken') + + host_url = 'http://localhost:6264' + + # --------------------------- + parser = subparsers.add_parser( + 'benchmark', + help='benchmark an RPC method') + + parser.add_argument('method', action='store', help='Method to benchmark') + parser.add_argument('iterations', action='store', type=int, help='Number of iterations') + parser.add_argument('args', nargs='*', action='store', help='Method arguments, if any') + parser.add_argument('--url', action='store', help='Blockstackd URL') + parser.add_argument('--histogram', action='store_true', help='Print an histogram of the benchmarked method response times') + parser.add_argument('--cdf', action='store_true', help='Print a CDF of the benchmarked method response times') + parser.add_argument('--ccdf', action='store_true', help='Print a CCDF of the benchmarked method response times') + parser.add_argument('--full-responses', action='store_true', help='Print full responses from the node') + parser.add_argument('--include-errors', action='store_true', help='Include benchmark data from errors') + + # --------------------------- + args, _ = argparser.parse_known_args() + + if args.action == 'benchmark': + method = args.method + iters = args.iterations + method_args = args.args + url = args.url + + if url is None: + url = host_url + + log.debug("Blockstack URL: {}".format(url)) + + parsed_args = [] + for method_arg in method_args: + try: + method_arg = int(method_arg) + parsed_args.append(method_arg) + continue + except: + pass + + try: + method_arg = json.loads(method_arg) + parsed_args.append(method_arg) + continue + except: + pass + + client = blockstack_client.connect_hostport(url) + benchmark_data = benchmark_rpc(client, iters, method, *method_args) + times = get_benchmark_times(benchmark_data, ignore_errors=(not args.include_errors)) + + print_graph = False + + if args.histogram: + print_graph = True + hist = make_histogram(times, min(times), max(times), 10) + print format_histogram(hist) + + if args.cdf: + print_graph = True + cdf = make_cdf(times, min(times), max(times), 100) + print format_histogram(cdf) + + if args.ccdf: + print_graph = True + cdf = make_ccdf(times, min(times), max(times), 100) + print format_histogram(cdf) + + if not print_graph: + if args.full_responses: + for data in benchmark_data: + print json.dumps(data, sort_keys=True) + + else: + for t in times: + print t + + return True + + return False + + +if __name__ == "__main__": + rc = main(sys.argv) + sys.exit(0 if rc else 1) + + ''' + # data = [random.randint(0, 255) for _ in xrange(0, 100)] + # data = xrange(0,100) + data = [ + 8.78195905685, + 0.835022926331, + 0.794891834259, + 0.885505914688, + 1.17201900482, + 1.12744307518, + 0.794220924377, + 1.03178310394, + 0.937813043594, + 0.786528110504, + ] + histogram = make_histogram(data, 0, max(data), 8) + cdf = make_cdf(data, 0, max(data), 100) + ccdf = make_ccdf(data, 0, max(data), 100) + + print format_histogram(histogram) + print format_histogram(cdf) + print format_histogram(ccdf) + ''' +