From 46803c701eb0e0d53e0ea9ec27a492c25a6051a7 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 20 Jun 2018 11:43:36 -0400 Subject: [PATCH 01/11] add rpc_get_name_history_page() --- blockstack/blockstackd.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/blockstack/blockstackd.py b/blockstack/blockstackd.py index 230dc6b17..be458bd8c 100644 --- a/blockstack/blockstackd.py +++ b/blockstack/blockstackd.py @@ -883,6 +883,29 @@ class BlockstackdRPC(SimpleXMLRPCServer): return self.success_response({'record': res['record']}) + def rpc_get_name_history_page(self, name, page, **con_info): + """ + Get the list of history entries for a name's history, paginated. + Small pages correspond to later history (page = 0 is the page of last updates) + Page size is 20 rows. + Return {'status': True, 'history': [...]} on success + Return {'error': ...} on error + """ + if not self.check_name(name): + return {'error': 'invalid name'} + + if not self.check_count(page): + return {'error': 'invalid page'} + + offset = page * 20 + count = (page + 1) * 20 + + db = get_db_state(self.working_dir) + history_data = db.get_name_history(name, offset, count, reverse=True) + db.close() + return self.success_response({'history': history_data}) + + def rpc_get_name_history_blocks( self, name, **con_info ): """ Get the list of blocks at which the given name was affected. From 6d6259ea59d697a5ada5993f6aea59b735bb054e Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 20 Jun 2018 11:43:51 -0400 Subject: [PATCH 02/11] add code to get a name and its history using pagination, with a fallback to get_name_blockchain_record() --- blockstack/lib/client.py | 163 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 162 insertions(+), 1 deletion(-) diff --git a/blockstack/lib/client.py b/blockstack/lib/client.py index aa7fb2f8e..1082fd9e4 100644 --- a/blockstack/lib/client.py +++ b/blockstack/lib/client.py @@ -872,7 +872,10 @@ def get_name_record(name, include_history=False, include_expired=False, include_ lastblock = None try: if include_history: - resp = proxy.get_name_blockchain_record(name) + resp = get_name_and_history(name, proxy=proxy) + if 'error' in resp: + # fall back to legacy path + resp = proxy.get_name_blockchain_record(name) else: resp = proxy.get_name_record(name) @@ -2100,6 +2103,164 @@ def get_name_history_blocks(name, hostport=None, proxy=None): return resp['history_blocks'] +def get_name_history_page(name, page, hostport=None, proxy=None): + """ + Get a page of the name's history + Returns {'status': True, 'history': ..., 'indexing': ..., 'lastblock': ...} on success + Returns {'error': ...} on error + """ + assert hostport or proxy, 'Need hostport or proxy' + if proxy is None: + proxy = connect_hostport(hostport) + + hist_schema = { + 'type': 'object', + 'patternProperties': { + '^[0-9]+$': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': OP_HISTORY_SCHEMA['properties'], + 'required': [ + 'op', + 'opcode', + 'txid', + 'vtxindex', + ], + }, + }, + }, + } + + hist_resp_schema = { + 'type': 'object', + 'properties': { + 'history': hist_schema, + }, + 'required': [ 'history' ], + } + + resp_schema = json_response_schema(hist_resp_schema) + resp = {} + lastblock = None + indexin = None + + try: + _resp = proxy.get_name_history_page(name, page) + resp = json_validate(resp_schema, _resp) + if json_is_error(resp): + return resp + + lastblock = _resp['lastblock'] + indexing = _resp['indexing'] + + except ValidationError as e: + resp = json_traceback(resp.get('error')) + return resp + + except Exception as ee: + if BLOCKSTACK_DEBUG: + log.exception(ee) + + log.error("Caught exception while connecting to Blockstack node: {}".format(ee)) + resp = {'error': 'Failed to contact Blockstack node. Try again with `--debug`.'} + return resp + + return {'status': True, 'history': resp['history'], 'lastblock': lastblock, 'indexing': indexing} + + +def name_history_merge(h1, h2): + """ + Given two name histories (grouped by block), merge them. + """ + ret = {} + blocks_1 = [int(b) for b in h1.keys()] + blocks_2 = [int(b) for b in h2.keys()] + + # find overlapping blocks + overlap = list(set(blocks_1).intersection(set(blocks_2))) + if len(overlap) > 0: + for b in overlap: + h = h1[str(b)] + h2[str(b)] + h.sort(lambda v1, v2: -1 if v1['vtxindex'] < v2['vtxindex'] else 1) + + uniq = [] + last_vtxindex = None + for i in range(0, len(h)): + if h[i]['vtxindex'] != last_vtxindex: + uniq.append(h[i]) + last_vtxindex = h[i]['vtxindex'] + + ret[str(b)] = uniq + + all_blocks = list(set(blocks_1 + blocks_2)) + for b in all_blocks: + if b in overlap: + continue + + if b in blocks_1: + ret[str(b)] = h1[str(b)] + else: + ret[str(b)] = h2[str(b)] + + return ret + + +def get_name_history(name, hostport=None, proxy=None): + """ + Get the full history of a name + Returns {'status': True, 'history': ...} on success, where history is grouped by block + Returns {'error': ...} on error + """ + assert hostport or proxy, 'Need hostport or proxy' + if proxy is None: + proxy = connect_hostport(hostport) + + hist = {} + indexing = None + lastblock = None + + for i in range(0, 10000): # this is obviously too big + resp = get_name_history_page(name, i, proxy=proxy) + if 'error' in resp: + return resp + + indexing = resp['indexing'] + lastblock = resp['lastblock'] + + if len(resp['history']) == 0: + # caught up + break + + hist = name_history_merge(hist, resp['history']) + + return {'status': True, 'history': hist, 'indexing': indexing, 'lastblock': lastblock} + + +def get_name_and_history(name, include_expired=False, include_grace=True, hostport=None, proxy=None): + """ + Get the current name record and its history + (this is a replacement for proxy.get_name_blockchain_record()) + Return {'status': True, 'record': ...} on success, where .record.history is defined as {block_height: [{history}, {history}, ...], ...} + Return {'error': ...} on error + """ + assert hostport or proxy, 'Need hostport or proxy' + if proxy is None: + proxy = connect_hostport(hostport) + + hist = get_name_history(name, proxy=proxy) + if 'error' in hist: + return hist + + # just the name + rec = get_name_record(name, include_history=False, include_expired=include_expired, include_grace=include_grace, proxy=proxy) + if 'error' in rec: + return rec + + rec['history'] = hist['history'] + return {'status': True, 'record': rec, 'lastblock': hist['lastblock'], 'indexing': hist['indexing']} + + def get_name_at(name, block_id, include_expired=False, hostport=None, proxy=None): """ Get the name as it was at a particular height. From 98c59d4f801d3e17af009c7fd2c38ced8ca1a05d Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 20 Jun 2018 11:44:15 -0400 Subject: [PATCH 03/11] optionally get name history rows in reverse --- blockstack/lib/nameset/namedb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/blockstack/lib/nameset/namedb.py b/blockstack/lib/nameset/namedb.py index 8b54d299f..af40b7974 100644 --- a/blockstack/lib/nameset/namedb.py +++ b/blockstack/lib/nameset/namedb.py @@ -830,12 +830,12 @@ class BlockstackDB( virtualchain.StateEngine ): return namedb_get_namespace_at(cur, namespace_id, block_number, include_expired=True) - def get_name_history( self, name ): + def get_name_history( self, name, offset=None, count=None, reverse=False): """ - Get the historic states for a name + Get the historic states for a name, grouped by block height. """ cur = self.db.cursor() - name_hist = namedb_get_history( cur, name ) + name_hist = namedb_get_history( cur, name, offset=offset, count=count, reverse=reverse ) return name_hist @@ -847,7 +847,7 @@ class BlockstackDB( virtualchain.StateEngine ): cur = self.db.cursor() update_points = namedb_get_blocks_with_ops( cur, name, FIRST_BLOCK_MAINNET, self.lastblock ) return update_points - + def get_all_ops_at( self, block_number, offset=None, count=None, include_history=None, restore_history=None ): """ From cd51534d812b3f7ff9a12d2ba1698fb220f8fc11 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 20 Jun 2018 11:44:29 -0400 Subject: [PATCH 04/11] optionally get name history rows in reverse --- blockstack/lib/nameset/db.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/blockstack/lib/nameset/db.py b/blockstack/lib/nameset/db.py index 221252076..470442206 100644 --- a/blockstack/lib/nameset/db.py +++ b/blockstack/lib/nameset/db.py @@ -1285,13 +1285,17 @@ def namedb_get_blocks_with_ops( cur, history_id, start_block_id, end_block_id ): return ret -def namedb_get_history_rows( cur, history_id, offset=None, count=None ): +def namedb_get_history_rows( cur, history_id, offset=None, count=None, reverse=False ): """ Get the history for a name or namespace from the history table. Use offset/count if given. """ ret = [] - select_query = "SELECT * FROM history WHERE history_id = ? ORDER BY block_id ASC, vtxindex ASC" + if not reverse: + select_query = "SELECT * FROM history WHERE history_id = ? ORDER BY block_id ASC, vtxindex ASC" + else: + select_query = "SELECT * FROM history WHERE history_id = ? ORDER BY block_id DESC, vtxindex DESC" + args = (history_id,) if count is not None: @@ -1325,14 +1329,13 @@ def namedb_get_num_history_rows( cur, history_id ): return count -def namedb_get_history( cur, history_id ): +def namedb_get_history( cur, history_id, offset=None, count=None, reverse=False ): """ Get all of the history for a name or namespace. Returns a dict keyed by block heights, paired to lists of changes (see namedb_history_extract) """ - # get history in increasing order by block_id and then vtxindex - history_rows = namedb_get_history_rows( cur, history_id ) + history_rows = namedb_get_history_rows( cur, history_id, offset=offset, count=count, reverse=reverse ) return namedb_history_extract( history_rows ) From 0025b71d238d1a9ea709edc70b1d384a8ad41e33 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 20 Jun 2018 11:44:41 -0400 Subject: [PATCH 05/11] when servicing a lookup on a subdomain whose domain does not have a _resolver entry, page back through the domain's history until we either find a zone file with a _resolver entry, or we try too many times, or we run out of name history --- blockstack_client/rpc.py | 103 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 7 deletions(-) diff --git a/blockstack_client/rpc.py b/blockstack_client/rpc.py index 58de2404e..5b8b24547 100644 --- a/blockstack_client/rpc.py +++ b/blockstack_client/rpc.py @@ -878,6 +878,90 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): return self._reply_json(resp, status_code=202) + def find_last_resolver_zonefile(self, name, attempts=5): + """ + Find the last known zone file for a name. + Do not carry out more than $attempts zone file requests (prevents a malicious name from opening a DDoS vector). + Return {'status': True, 'zonefile': ...} on success + Return {'error': 'Not found'} if the name doesn't exist, or we have no zone file. + """ + blockstackd_url = get_blockstackd_url(self.server.config_path) + domain_rec = blockstackd_client.get_name_record(name, include_history = False, + hostport = blockstackd_url) + if 'error' in domain_rec: + return {'error': 'Not found', 'http_status': 404} + + if 'zonefile' in domain_rec: + return {'status': True, 'zonefile': domain_rec['zonefile']} + + last_zonefile_hash = None + page = 0 + while attempts > 0: + # paginate back through the name's history to find a zone file + res = blockstackd_client.get_name_history_page(name, page, hostport=blockstackd_url) + if 'error' in res: + log.error("Failed to get name history page {} for {}: {}".format(page, name, res['error'])) + return {'error': 'Failed to get name history: {}'.format(res['error']), 'http_status': 502} + + hist = res['history'] + + if BLOCKSTACK_TEST: + log.debug('Name history page {} for {}:\n{}'.format(page, name, json.dumps(hist, indent=4, sort_keys=True))) + + page += 1 + + if len(hist) == 0: + # out of history + log.debug("Out of history on {}".format(name)) + break + + for block_id in reversed(hist.keys()): + # go in reverse vtx order + vtxs = hist[block_id] + vtxs.sort(lambda v1, v2: 1 if v1['vtxindex'] < v2['vtxindex'] else -1) + for vtx in vtxs: + + if attempts < 0: + log.debug('Tried too many times to find a zone file for {}'.format(name)) + return {'error': 'Not Found', 'http_status': 404} + + if 'value_hash' not in vtx: + log.debug('No value hash in ({}, {}) for {}'.format(block_id, vtx['vtxindex'], name)) + continue + + if vtx['value_hash'] == last_zonefile_hash: + log.debug('Duplicate value hash in ({}, {}) for {}'.format(block_id, vtx['vtxindex'], name)) + continue + + # new zone file + last_zonefile_hash = vtx['value_hash'] + resp = blockstackd_client.get_zonefiles(blockstackd_url, [str(last_zonefile_hash)]) + + attempts -= 1 + + if 'error' in resp: + log.debug('Failed to get {} (from ({}, {}) for {}): {}'.format(last_zonefile_hash, block_id, vtx['vtxindex'], name, resp['error'])) + continue + + if last_zonefile_hash not in resp['zonefiles']: + log.debug('Failed to get {} (from ({}, {}) for {}): missing'.format(last_zonefile_hash, block_id, vtx['vtxindex'], name)) + continue + + # got a zonefile! + # is it well-formed? does it have a _resolver entry? + try: + domain_zf_txt = resp['zonefiles'][last_zonefile_hash] + domain_zf_json = zonefile.decode_name_zonefile(name, domain_zf_txt, allow_legacy=False) + matching_uris = [ x['target'] for x in domain_zf_json['uri'] if x['name'] == '_resolver' ] + except: + log.debug("Malformed zone file {} (from {}, {} for {})".format(last_zonefile_hash, block_id, vtx['vtxindex'], name)) + continue + + return {'status': True, 'zonefile': resp['zonefiles'][last_zonefile_hash]} + + return {'error': 'Not found', 'http_status': 404} + + def GET_name_info( self, ses, path_info, name ): """ Look up a name's zonefile, address, and last TXID @@ -921,12 +1005,18 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): elif 'failed to load subdomain' in name_rec['error'].lower(): # handle redirection to specified resolver _, _, domain_name = blockstackd_scripts.is_address_subdomain(name) - domain_rec = blockstackd_client.get_name_record(domain_name, include_history = False, - hostport = blockstackd_url) - if 'error' in domain_rec or 'zonefile' not in domain_rec: - return self._reply_json( - {'status': 'available', 'more': 'failed to lookup parent domain'}, status_code=404) - domain_zf_txt = base64.b64decode(domain_rec['zonefile']) + + res = self.find_last_resolver_zonefile(domain_name) + if 'error' in res: + if res['http_status'] == 404: + return self._reply_json( + {'status': 'available', 'more': 'failed to look up parent domain'}, status_code=404) + + else: + return self._reply_json( + {'error': res['error']}, status_code=res['http_status']) + + domain_zf_txt = res['zonefile'] domain_zf_json = zonefile.decode_name_zonefile(domain_name, domain_zf_txt, allow_legacy=False) matching_uris = [ x['target'] for x in domain_zf_json['uri'] if x['name'] == '_resolver' ] if len(matching_uris) == 0: @@ -1106,7 +1196,6 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): return - # DEPRECATED def POST_raw_zonefile( self, ses, path_info ): """ Publish a zonefile which has *already* been announced. From 9b9b38a750aec6d680c6372ee8e118d2a4932647 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 20 Jun 2018 11:45:19 -0400 Subject: [PATCH 06/11] test fetching a name's history in pages --- .../scenarios/name_pre_regup_long_history.py | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 integration_tests/blockstack_integration_tests/scenarios/name_pre_regup_long_history.py diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_pre_regup_long_history.py b/integration_tests/blockstack_integration_tests/scenarios/name_pre_regup_long_history.py new file mode 100644 index 000000000..d061f27af --- /dev/null +++ b/integration_tests/blockstack_integration_tests/scenarios/name_pre_regup_long_history.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +""" + Blockstack + ~~~~~ + copyright: (c) 2014-2015 by Halfmoon Labs, Inc. + copyright: (c) 2016 by Blockstack.org + + This file is part of Blockstack + + Blockstack is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Blockstack is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with Blockstack. If not, see . +""" + +# activate F-day 2017 +""" +TEST ENV BLOCKSTACK_EPOCH_1_END_BLOCK 682 +TEST ENV BLOCKSTACK_EPOCH_2_END_BLOCK 683 +TEST ENV BLOCKSTACK_EPOCH_2_NAMESPACE_LIFETIME_MULTIPLIER 1 +""" + +import testlib +import virtualchain +import json +import blockstack + +wallets = [ + testlib.Wallet( "5JesPiN68qt44Hc2nT8qmyZ1JDwHebfoh9KQ52Lazb1m1LaKNj9", 100000000000 ), + testlib.Wallet( "5KHqsiU9qa77frZb6hQy9ocV7Sus9RWJcQGYYBJJBb2Efj1o77e", 100000000000 ), + testlib.Wallet( "5Kg5kJbQHvk1B64rJniEmgbD83FpZpbw2RjdAZEzTefs9ihN3Bz", 100000000000 ), + testlib.Wallet( "5JuVsoS9NauksSkqEjbUZxWwgGDQbMwPsEfoRBSpLpgDX1RtLX7", 100000000000 ), + testlib.Wallet( "5KEpiSRr1BrT8vRD7LKGCEmudokTh1iMHbiThMQpLdwBwhDJB1T", 100000000000 ) +] + +consensus = "17ac43c1d8549c3181b200f1bf97eb7d" + +def scenario( wallets, **kw ): + + testlib.blockstack_namespace_preorder( "test", wallets[1].addr, wallets[0].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_namespace_reveal( "test", wallets[1].addr, 52595, 250, 4, [6,5,4,3,2,1,0,0,0,0,0,0,0,0,0,0], 10, 10, wallets[0].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_namespace_ready( "test", wallets[1].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_name_preorder( "foo.test", wallets[2].privkey, wallets[3].addr ) + testlib.next_block( **kw ) + + preorder_block = str(testlib.get_current_block(**kw)) + + zfdata = 'hello world for the first time' + zfhash = blockstack.lib.storage.get_zonefile_data_hash(zfdata) + + testlib.blockstack_name_register( "foo.test", wallets[2].privkey, wallets[3].addr, zonefile_hash=zfhash) + testlib.next_block( **kw ) + + register_block = str(testlib.get_current_block(**kw)) + + testlib.blockstack_put_zonefile(zfdata) + + name = 'foo.test' + + # get name and history--make sure it works + name_rec = blockstack.lib.client.get_name_record(name, include_history=True, hostport='http://localhost:16264') + if 'error' in name_rec: + print name_rec + return False + + if len(name_rec['history']) != 2: + print 'invalid history' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + if preorder_block not in name_rec['history'] or len(name_rec['history'][preorder_block]) != 1: + print 'missing preorder block' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + if register_block not in name_rec['history'] or len(name_rec['history'][register_block]) != 1: + print 'missing register block' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + # do a bunch of updates in this block + for i in xrange(0, 19): + zfdata = 'hello update {}'.format(i) + zfhash = blockstack.lib.storage.get_zonefile_data_hash(zfdata) + testlib.blockstack_name_update("foo.test", zfhash, wallets[3].privkey) + + testlib.next_block(**kw) + update_block_1 = str(testlib.get_current_block(**kw)) + + for i in xrange(0, 19): + zfdata = 'hello update {}'.format(i) + testlib.blockstack_put_zonefile(zfdata) + + # get name and history--make sure it works + name_rec = blockstack.lib.client.get_name_record(name, include_history=True, hostport='http://localhost:16264') + if 'error' in name_rec: + print json.dumps(name_rec, indent=4, sort_keys=True) + return False + + if len(name_rec['history']) != 3: + print 'invalid history' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + # need to be 21 entries: the preorder, register, and 19 updates + if preorder_block not in name_rec['history'] or len(name_rec['history'][preorder_block]) != 1: + print 'missing preorder block' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + if register_block not in name_rec['history'] or len(name_rec['history'][register_block]) != 1: + print 'missing register block' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + if update_block_1 not in name_rec['history'] or len(name_rec['history'][update_block_1]) != 19: + print 'missing update block' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + # do a bunch more updates in this block + for i in xrange(0, 21): + zfdata = 'hello update round 2 {}'.format(i) + zfhash = blockstack.lib.storage.get_zonefile_data_hash(zfdata) + testlib.blockstack_name_update("foo.test", zfhash, wallets[3].privkey) + + testlib.next_block(**kw) + update_block_2 = str(testlib.get_current_block(**kw)) + + for i in xrange(0, 21): + zfdata = 'hello update round 2 {}'.format(i) + testlib.blockstack_put_zonefile(zfdata) + + # get name and history--make sure it works + name_rec = blockstack.lib.client.get_name_record(name, include_history=True, hostport='http://localhost:16264') + if 'error' in name_rec: + print json.dumps(name_rec, indent=4, sort_keys=True) + return False + + if len(name_rec['history']) != 4: + print 'invalid history' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + # need to be 21 entries: the preorder, register, and 19 updates + if preorder_block not in name_rec['history'] or len(name_rec['history'][preorder_block]) != 1: + print 'missing preorder block' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + if register_block not in name_rec['history'] or len(name_rec['history'][register_block]) != 1: + print 'missing register block' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + if update_block_1 not in name_rec['history'] or len(name_rec['history'][update_block_1]) != 19: + print 'missing update block 1' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + if update_block_2 not in name_rec['history'] or len(name_rec['history'][update_block_2]) != 21: + print 'missing update block 2' + print json.dumps(name_rec['history'], indent=4, sort_keys=True) + return False + + # last page must be just the last updates + hist_page = blockstack.lib.client.get_name_history_page(name, 0, hostport='http://localhost:16264') + if 'error' in hist_page: + print hist_page + return False + + history = hist_page['history'] + if len(history) != 1 or update_block_2 not in history: + print 'missing update block 2 in history page' + print json.dumps(history, indent=4, sort_keys=True) + return False + + for vtx in history[update_block_2]: + # should be vtxindex 2 through 21 + if vtx['vtxindex'] < 2: + print 'got low vtxindex' + print json.dumps(history, indent=4, sort_keys=True) + return False + + +def check( state_engine ): + + # not revealed, but ready + ns = state_engine.get_namespace_reveal( "test" ) + if ns is not None: + return False + + ns = state_engine.get_namespace( "test" ) + if ns is None: + return False + + if ns['namespace_id'] != 'test': + return False + + name = 'foo.test' + + # not preordered + preorder = state_engine.get_name_preorder( name, virtualchain.make_payment_script(wallets[2].addr), wallets[3].addr ) + if preorder is not None: + print 'still have preorder: {}'.format(preorder) + return False + + # registered + name_rec = state_engine.get_name(name) + if name_rec is None: + print 'did not get name {}'.format(name) + return False + + # owned by + if name_rec['address'] != wallets[3].addr or name_rec['sender'] != virtualchain.make_payment_script(wallets[3].addr): + print 'wrong address for {}: {}'.format(name, name_rec) + return False + + return True From e0e302edf1c51dee9c4d8b0cde0b7a20713245cb Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 20 Jun 2018 11:45:36 -0400 Subject: [PATCH 07/11] test looking up a subdomain from a domain name who is missing its latest zone file, and who has too many missing zone files --- .../subdomain_registrar_redirect_prior.py | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 integration_tests/blockstack_integration_tests/scenarios/subdomain_registrar_redirect_prior.py diff --git a/integration_tests/blockstack_integration_tests/scenarios/subdomain_registrar_redirect_prior.py b/integration_tests/blockstack_integration_tests/scenarios/subdomain_registrar_redirect_prior.py new file mode 100644 index 000000000..5ca9464d6 --- /dev/null +++ b/integration_tests/blockstack_integration_tests/scenarios/subdomain_registrar_redirect_prior.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +""" + Blockstack + ~~~~~ + copyright: (c) 2014-2015 by Halfmoon Labs, Inc. + copyright: (c) 2016 by Blockstack.org + + This file is part of Blockstack + + Blockstack is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Blockstack is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with Blockstack. If not, see . +""" +# activate F-day 2017 at the right time +""" +TEST ENV BLOCKSTACK_EPOCH_1_END_BLOCK 682 +TEST ENV BLOCKSTACK_EPOCH_2_END_BLOCK 683 +TEST ENV BLOCKSTACK_EPOCH_2_NAMESPACE_LIFETIME_MULTIPLIER 1 +TEST ENV BLOCKSTACK_EPOCH_3_NAMESPACE_LIFETIME_MULTIPLIER 1 +""" + +import testlib +import virtualchain +import time +import json +import sys +import os +import requests +import blockstack +import blockstack_client +import blockstack_zones +from subprocess import Popen + +wallets = [ + testlib.Wallet( "5JesPiN68qt44Hc2nT8qmyZ1JDwHebfoh9KQ52Lazb1m1LaKNj9", 100000000000 ), + testlib.Wallet( "5KHqsiU9qa77frZb6hQy9ocV7Sus9RWJcQGYYBJJBb2Efj1o77e", 100000000000 ), + testlib.Wallet( "5Kg5kJbQHvk1B64rJniEmgbD83FpZpbw2RjdAZEzTefs9ihN3Bz", 100000000000 ), + testlib.Wallet( "5JuVsoS9NauksSkqEjbUZxWwgGDQbMwPsEfoRBSpLpgDX1RtLX7", 5500 ), + testlib.Wallet( "5KEpiSRr1BrT8vRD7LKGCEmudokTh1iMHbiThMQpLdwBwhDJB1T", 5500 ) +] + +consensus = "17ac43c1d8549c3181b200f1bf97eb7d" + +TRANSACTION_BROADCAST_LOCATION = os.environ.get('BSK_TRANSACTION_BROADCAST_LOCATION', + '/src/transaction-broadcaster') + +SUBDOMAIN_REGISTRAR_LOCATION = os.environ.get('BSK_SUBDOMAIN_REGISTRAR_LOCATION', + '/src/subdomain-registrar') + +def start_transaction_broadcaster(): + try: + os.rename('/tmp/transaction_broadcaster.db', '/tmp/transaction_broadcaster.db.last') + except OSError: + pass + env = {'BSK_TRANSACTION_BROADCAST_DEVELOP' : '1'} + if os.environ.get('BLOCKSTACK_TEST_CLIENT_RPC_PORT', False): + env['BLOCKSTACK_TEST_CLIENT_RPC_PORT'] = os.environ.get('BLOCKSTACK_TEST_CLIENT_RPC_PORT') + + trans_logfile = '/tmp/transaction_broadcaster.log' + fd = open(trans_logfile, 'w') + + Popen(['node', TRANSACTION_BROADCAST_LOCATION + '/lib/index.js'], + env = env, stdout=fd, stderr=fd) + +def start_subdomain_registrar(): + try: + os.rename('/tmp/subdomain_registrar.db', '/tmp/subdomain_registrar.last') + except OSError: + pass + env = {'BSK_SUBDOMAIN_REGTEST' : '1'} + if os.environ.get('BLOCKSTACK_TEST_CLIENT_RPC_PORT', False): + env['BLOCKSTACK_TEST_CLIENT_RPC_PORT'] = os.environ.get('BLOCKSTACK_TEST_CLIENT_RPC_PORT') + + subd_logfile = '/tmp/subdomain_registrar.log' + fd = open(subd_logfile, 'w') + + Popen(['node', SUBDOMAIN_REGISTRAR_LOCATION + '/lib/index.js'], env = env, stdout=fd, stderr=fd) + +def scenario( wallets, **kw ): + + start_transaction_broadcaster() + start_subdomain_registrar() + + testlib.blockstack_namespace_preorder( "id", wallets[1].addr, wallets[0].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_namespace_reveal( "id", wallets[1].addr, 52595, 250, 4, [6,5,4,3,2,1,0,0,0,0,0,0,0,0,0,0], 10, 10, wallets[0].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_namespace_ready( "id", wallets[1].privkey ) + testlib.next_block( **kw ) + + wallet = testlib.blockstack_client_initialize_wallet( "0123456789abcdef", wallets[2].privkey, wallets[3].privkey, wallets[4].privkey ) + + resp = testlib.blockstack_name_preorder('foo.id', wallets[2].privkey, wallets[3].addr) + testlib.next_block(**kw) + + zonefile = blockstack_client.zonefile.make_empty_zonefile('foo.id', None) + zfdata = blockstack_zones.make_zone_file(zonefile) + zfhash = blockstack.lib.storage.get_zonefile_data_hash(zfdata) + + resp = testlib.blockstack_name_register('foo.id', wallets[2].privkey, wallets[3].addr, zonefile_hash=zfhash) + testlib.next_block(**kw) + + testlib.blockstack_put_zonefile(zfdata) + + # now, queue a subdomain registration. + + requests.post('http://localhost:3000/register', + json = { 'zonefile' : 'hello world', + 'name' : 'bar', + 'owner_address': '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa' }) + + # force a batch out of the subdomain registrar + + requests.post('http://localhost:3000/issue_batch', + headers = {'Authorization': 'bearer tester129'}) + + for i in xrange(0, 12): + testlib.next_block( **kw ) + + print >> sys.stderr, "Waiting 10 seconds for the backend to pickup first batch" + time.sleep(10) + + # update the name on-chain + testlib.blockstack_name_update('foo.id', '11' * 20, wallets[3].privkey) + testlib.blockstack_name_update('foo.id', '22' * 20, wallets[3].privkey) + testlib.blockstack_name_update('foo.id', '33' * 20, wallets[3].privkey) + testlib.next_block(**kw) + + # now, queue another registration + + requests.post('http://localhost:3000/register', + json = { 'zonefile' : 'hello world', + 'name' : 'zap', + 'owner_address': '1Ez69SnzzmePmZX3WpEzMKTrcBF2gpNQ55' }) + + res = testlib.blockstack_REST_call('GET', '/v1/names/zap.foo.id', None) + + if 'error' in res: + res['test'] = 'Failed to query zap.foo.id' + print json.dumps(res) + return False + + if res['http_status'] != 200: + res['test'] = 'HTTP status {}, response = {} on name lookup'.format(res['http_status'], res['response']) + print json.dumps(res) + return False + + name_info = res['response'] + try: + if (name_info['zonefile'] != 'hello world' or + name_info['address'] != '1Ez69SnzzmePmZX3WpEzMKTrcBF2gpNQ55'): + res['test'] = 'Unexpected name info lookup for zap.foo.id' + print 'zap.foo.id JSON:' + print json.dumps(name_info) + return False + except: + res['test'] = 'Unexpected name info lookup for zap.foo.id' + print 'zap.foo.id JSON:' + print json.dumps(name_info) + return False + + + # update the name on-chain again, but lots of times + for i in range(0, 20): + zonefile_pattern = '{:x}{:x}'.format(i / 16, i % 16) + testlib.blockstack_name_update('foo.id', zonefile_pattern * 20, wallets[3].privkey) + + testlib.next_block(**kw) + + # should fail--we have too many prior zone files + res = testlib.blockstack_REST_call('GET', '/v1/names/zap.foo.id', None) + + if res['http_status'] != 404: + res['test'] = 'HTTP status {}, response = {} on name lookup'.format(res['http_status'], res['response']) + print json.dumps(res) + return False + +def check( state_engine ): + + # not revealed, but ready + ns = state_engine.get_namespace_reveal( "id" ) + if ns is not None: + print "namespace reveal exists" + return False + + ns = state_engine.get_namespace( "id" ) + if ns is None: + print "no namespace" + return False + + if ns['namespace_id'] != 'id': + print "wrong namespace" + return False + + # registered + name_rec = state_engine.get_name( "foo.id" ) + if name_rec is None: + print "name does not exist" + return False + + return True From 0cdfc9f939ed0f31b150941ec7f161e7a5ea51e4 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Wed, 20 Jun 2018 11:38:14 -0500 Subject: [PATCH 08/11] force paging on the /v1/names//history endpoint --- blockstack/lib/client.py | 24 ++++++++++++++++++------ blockstack_client/rpc.py | 4 +++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/blockstack/lib/client.py b/blockstack/lib/client.py index 1082fd9e4..86b455792 100644 --- a/blockstack/lib/client.py +++ b/blockstack/lib/client.py @@ -810,7 +810,8 @@ def put_zonefiles(hostport, zonefile_data_list, timeout=30, my_hostport=None, pr return push_info -def get_name_record(name, include_history=False, include_expired=False, include_grace=True, proxy=None, hostport=None): +def get_name_record(name, include_history=False, include_expired=False, include_grace=True, proxy=None, hostport=None, + history_page=None): """ Get the record for a name or a subdomain. Optionally include its history, and optionally return an expired name or a name in its grace period. Return the blockchain-extracted information on success. @@ -872,9 +873,10 @@ def get_name_record(name, include_history=False, include_expired=False, include_ lastblock = None try: if include_history: - resp = get_name_and_history(name, proxy=proxy) + resp = get_name_and_history(name, proxy=proxy, history_page=history_page) if 'error' in resp: - # fall back to legacy path + # fall back to legacy path + log.debug(resp) resp = proxy.get_name_blockchain_record(name) else: resp = proxy.get_name_record(name) @@ -2206,7 +2208,7 @@ def name_history_merge(h1, h2): return ret -def get_name_history(name, hostport=None, proxy=None): +def get_name_history(name, hostport=None, proxy=None, history_page=None): """ Get the full history of a name Returns {'status': True, 'history': ...} on success, where history is grouped by block @@ -2220,6 +2222,16 @@ def get_name_history(name, hostport=None, proxy=None): indexing = None lastblock = None + if history_page != None: + resp = get_name_history_page(name, history_page, proxy=proxy) + if 'error' in resp: + return resp + + indexing = resp['indexing'] + lastblock = resp['lastblock'] + + return {'status': True, 'history': resp['history'], 'indexing': indexing, 'lastblock': lastblock} + for i in range(0, 10000): # this is obviously too big resp = get_name_history_page(name, i, proxy=proxy) if 'error' in resp: @@ -2237,7 +2249,7 @@ def get_name_history(name, hostport=None, proxy=None): return {'status': True, 'history': hist, 'indexing': indexing, 'lastblock': lastblock} -def get_name_and_history(name, include_expired=False, include_grace=True, hostport=None, proxy=None): +def get_name_and_history(name, include_expired=False, include_grace=True, hostport=None, proxy=None, history_page=None): """ Get the current name record and its history (this is a replacement for proxy.get_name_blockchain_record()) @@ -2248,7 +2260,7 @@ def get_name_and_history(name, include_expired=False, include_grace=True, hostpo if proxy is None: proxy = connect_hostport(hostport) - hist = get_name_history(name, proxy=proxy) + hist = get_name_history(name, proxy=proxy, history_page=history_page) if 'error' in hist: return hist diff --git a/blockstack_client/rpc.py b/blockstack_client/rpc.py index 5b8b24547..81a9b6a50 100644 --- a/blockstack_client/rpc.py +++ b/blockstack_client/rpc.py @@ -1092,6 +1092,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): qs_values = path_info['qs_values'] start_block = qs_values.get('start_block', None) end_block = qs_values.get('end_block', None) + page = int(qs_values.get('page', 0)) try: if start_block is None: @@ -1111,7 +1112,8 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): blockstackd_url = get_blockstackd_url(self.server.config_path) # res = proxy.get_name_blockchain_history(name, start_block, end_block) - res = blockstackd_client.get_name_record(name, include_history=True, hostport=blockstackd_url) + res = blockstackd_client.get_name_record(name, include_history=True, + hostport=blockstackd_url, history_page=page) if json_is_error(res): self._reply_json({'error': res['error']}, status_code=500) return From 3abbcbc560951307e50cff7a76daeb1883c196f9 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Wed, 20 Jun 2018 11:39:46 -0500 Subject: [PATCH 09/11] update api doc --- docs/api-specs.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api-specs.md b/docs/api-specs.md index 0a9802fd9..4bceb9379 100644 --- a/docs/api-specs.md +++ b/docs/api-specs.md @@ -1872,12 +1872,13 @@ Fetch a list of all names known to the node. } } -## Name history [GET /v1/names/{name}/history] +## Name history [GET /v1/names/{name}/history?page={page}] Get a history of all blockchain records of a registered name. + Public Endpoint + Subdomain aware + Parameters + name: muneeb.id (string) - name to query + + page: 0 (integer) - the page (in 20-entry pages) of the history to fetch + Response 200 (application/json) + Body From 6934613ed1b33f6c1c18e1ac610b7cc954ec6963 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 25 Jun 2018 11:22:19 -0500 Subject: [PATCH 10/11] zonefile record returned as base64, must be decoded before return. --- blockstack_client/rpc.py | 12 +++++++++--- blockstack_client/version.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/blockstack_client/rpc.py b/blockstack_client/rpc.py index 81a9b6a50..307b0da4b 100644 --- a/blockstack_client/rpc.py +++ b/blockstack_client/rpc.py @@ -892,12 +892,18 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): return {'error': 'Not found', 'http_status': 404} if 'zonefile' in domain_rec: - return {'status': True, 'zonefile': domain_rec['zonefile']} + try: + zf_txt = base64.b64decode(domain_rec['zonefile']) + return {'status': True, 'zonefile': zf_txt} + except: + log.error("Failed to parse zonefile returned by blockstackd: contents return: {}" + .format(domain_rec['zonefile'])) + return {'error': 'Failed to parse zonefile', 'http_status': 502} last_zonefile_hash = None page = 0 while attempts > 0: - # paginate back through the name's history to find a zone file + # paginate back through the name's history to find a zone file res = blockstackd_client.get_name_history_page(name, page, hostport=blockstackd_url) if 'error' in res: log.error("Failed to get name history page {} for {}: {}".format(page, name, res['error'])) @@ -1015,7 +1021,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): else: return self._reply_json( {'error': res['error']}, status_code=res['http_status']) - + domain_zf_txt = res['zonefile'] domain_zf_json = zonefile.decode_name_zonefile(domain_name, domain_zf_txt, allow_legacy=False) matching_uris = [ x['target'] for x in domain_zf_json['uri'] if x['name'] == '_resolver' ] diff --git a/blockstack_client/version.py b/blockstack_client/version.py index eca4c50b4..3443e9af1 100644 --- a/blockstack_client/version.py +++ b/blockstack_client/version.py @@ -24,4 +24,4 @@ __version_major__ = '0' __version_minor__ = '18' __version_patch__ = '0' -__version__ = '{}.{}.{}.6'.format(__version_major__, __version_minor__, __version_patch__) +__version__ = '{}.{}.{}.7'.format(__version_major__, __version_minor__, __version_patch__) From dd1aa516584f7b2a130c0ef5198511df315f12f1 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Tue, 26 Jun 2018 16:11:17 -0500 Subject: [PATCH 11/11] build search index then swap, rather than flush+rebuilt --- api/search/basic_index.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/api/search/basic_index.py b/api/search/basic_index.py index 809e84c15..ecccb6472 100644 --- a/api/search/basic_index.py +++ b/api/search/basic_index.py @@ -32,9 +32,18 @@ from .utils import get_json, config_log, pretty_print from api.config import SEARCH_BLOCKCHAIN_DATA_FILE, SEARCH_PROFILE_DATA_FILE -from .db import namespace, profile_data -from .db import search_profiles -from .db import people_cache, twitter_cache, username_cache +client = get_mongo_client() + + +search_db = client['search_db_next'] +search_cache = client['search_cache_next'] + +namespace = search_db.namespace +profile_data = search_db.profile_data +search_profiles = search_db.profiles +people_cache = search_cache.people_cache +twitter_cache = search_cache.twitter_cache +username_cache = search_cache.username_cache """ create the basic index """ @@ -131,11 +140,26 @@ def flush_db(): client = get_mongo_client() # delete any old cache/index - client.drop_database('search_db') - client.drop_database('search_cache') + client.drop_database('search_db_next') + client.drop_database('search_cache_next') log.debug("Flushed DB") +def swap_next_current_db(): + + client = get_mongo_client() + + client.drop_database('search_db_prior') + client.drop_database('search_cache_prior') + + client.admin.command('copydb', fromdb='search_db', todb='search_db_prior') + client.admin.command('copydb', fromdb='search_cache', todb='search_cache_prior') + + client.drop_database('search_db') + client.drop_database('search_cache') + + client.admin.command('copydb', fromdb='search_db_next', todb='search_db') + client.admin.command('copydb', fromdb='search_cache_next', todb='search_cache') def optimize_db(): @@ -281,6 +305,7 @@ if __name__ == "__main__": fetch_profile_data_from_file() fetch_namespace_from_file() create_search_index() + swap_next_current_db() else: print "Usage error"