diff --git a/blockstack/blockstackd.py b/blockstack/blockstackd.py index 03fdf9676..9c4b3030e 100644 --- a/blockstack/blockstackd.py +++ b/blockstack/blockstackd.py @@ -1578,7 +1578,17 @@ class BlockstackdRPC( SimpleXMLRPCServer): - def rpc_get_zonefiles_from_blocks( self, from_block, to_block, offset, count, **con_info ): + def rpc_get_zonefiles_by_block( self, from_block, to_block, offset, count, **con_info ): + """ + Get information about zonefiles announced in blocks [@from_block, @to_block] + @offset - offset into result set + @count - max records to return, must be <= 100 + + Returns {'status': True, 'lastblock' : blockNumber, + 'zonefile_info' : [ { 'block_height' : 470000, + 'txid' : '0000000', + 'zonefile_hash' : '0000000' } ] } + """ conf = get_blockstack_opts() if not conf['atlas']: return {'error': 'Not an atlas node'} diff --git a/blockstack/lib/atlas.py b/blockstack/lib/atlas.py index d679e909f..7caa6a102 100644 --- a/blockstack/lib/atlas.py +++ b/blockstack/lib/atlas.py @@ -729,8 +729,8 @@ def atlasdb_get_zonefiles_by_block( from_block, to_block, offset, count, con=Non with AtlasDBOpen(con=con, path=path) as dbcon: - sql = """SELECT zonefile_hash, txid, block_height FROM zonefiles - WHERE block_height >= ? and block_height <= ? + sql = """SELECT name, zonefile_hash, txid, block_height FROM zonefiles + WHERE block_height >= ? and block_height <= ? ORDER BY inv_index LIMIT ? OFFSET ?;""" args = (from_block, to_block, count, offset) @@ -742,6 +742,7 @@ def atlasdb_get_zonefiles_by_block( from_block, to_block, offset, count, con=Non for zfinfo in res: ret.append({ + 'name' : zfinfo['name'], 'zonefile_hash' : zfinfo['zonefile_hash'], 'block_height' : zfinfo['block_height'], 'txid' : zfinfo['txid'], diff --git a/blockstack/lib/storage/crawl.py b/blockstack/lib/storage/crawl.py index 4b92a3867..74330310f 100644 --- a/blockstack/lib/storage/crawl.py +++ b/blockstack/lib/storage/crawl.py @@ -284,7 +284,7 @@ def store_zonefile_data_to_storage( zonefile_text, txid, required=None, skip=Non log.debug("Failed to cache zonefile %s" % zonefile_hash) # NOTE: this can fail if one of the required drivers needs a non-null txid - res = blockstack_client.storage.put_immutable_data( zonefile_text, txid, data_hash=zonefile_hash, required=required, skip=skip ) + res = blockstack_client.storage.put_immutable_data( zonefile_text, txid, data_hash=zonefile_hash, required=required, skip=skip, required_exclusive=True ) if res is None: log.error("Failed to store zonefile '%s' for '%s'" % (zonefile_hash, txid)) return False diff --git a/blockstack/version.py b/blockstack/version.py index 03e020f83..2ca62f951 100644 --- a/blockstack/version.py +++ b/blockstack/version.py @@ -1,5 +1,5 @@ # this is the only place where version should be updated __version_major__ = '0' __version_minor__ = '14' -__version_patch__ = '2' +__version_patch__ = '4' __version__ = '{}.{}.{}.0'.format(__version_major__, __version_minor__, __version_patch__) diff --git a/blockstack_client/proxy.py b/blockstack_client/proxy.py index b763637d5..60da3964e 100644 --- a/blockstack_client/proxy.py +++ b/blockstack_client/proxy.py @@ -1418,6 +1418,57 @@ def get_op_history_rows(name, proxy=None): return history_rows +def get_zonefiles_by_block(from_block, to_block, proxy=None): + """ + Get zonefile information for zonefiles announced in [@from_block, @to_block] + Returns { 'last_block' : server's last seen block, + 'zonefile_info' : [ { 'zonefile_hash' : '...', + 'txid' : '...', + 'block_height' : '...' } ] } + """ + zonefile_info_schema = { + 'type' : 'array', + 'items' : { + 'type' : 'object', + 'properties' : { + 'name' : {'type' : 'string'}, + 'zonefile_hash' : { 'type' : 'string', + 'pattern' : OP_ZONEFILE_HASH_PATTERN }, + 'txid' : {'type' : 'string', + 'pattern' : OP_TXID_PATTERN}, + 'block_height' : {'type' : 'integer'} + }, + 'required' : [ 'zonefile_hash', 'txid', 'block_height' ] + } + } + response_schema = { + 'type' : 'object', + 'properties' : { + 'lastblock' : {'type' : 'integer'}, + 'zonefile_info' : zonefile_info_schema + }, + 'required' : ['lastblock', 'zonefile_info'] + } + + proxy = get_default_proxy() if proxy is None else proxy + + offset = 0 + output_zonefiles = [] + + last_server_block = 0 + while offset == 0 or len(resp['zonefile_info']) > 0: + resp = proxy.get_zonefiles_by_block(from_block, to_block, offset, 100) + if 'error' in resp: + return resp + resp = json_validate(response_schema, resp) + if json_is_error(resp): + return resp + output_zonefiles += resp['zonefile_info'] + offset += 100 + last_server_block = max(resp['lastblock'], last_server_block) + + return { 'last_block' : last_server_block, + 'zonefile_info' : output_zonefiles } def get_nameops_affected_at(block_id, proxy=None): """ diff --git a/blockstack_client/storage.py b/blockstack_client/storage.py index ab5712b95..97a4232be 100644 --- a/blockstack_client/storage.py +++ b/blockstack_client/storage.py @@ -920,7 +920,7 @@ def get_mutable_data(fq_data_id, data_pubkey, urls=None, data_address=None, data return None -def put_immutable_data(data_text, txid, data_hash=None, required=None, skip=None): +def put_immutable_data(data_text, txid, data_hash=None, required=None, skip=None, required_exclusive=False): """ Given a string of data (which can either be data or a zonefile), store it into our immutable data stores. Do so in a best-effort manner--this method only fails if *all* storage providers fail. @@ -947,6 +947,8 @@ def put_immutable_data(data_text, txid, data_hash=None, required=None, skip=None log.debug(msg.format(data_hash, ','.join(required), ','.join(skip))) for handler in storage_handlers: + if required_exclusive and handler.__name__ not in required: + continue if handler.__name__ in skip: log.debug("Skipping {}".format(handler.__name__)) continue diff --git a/blockstack_client/subdomains.py b/blockstack_client/subdomains.py index a6dd96ea7..499e546b7 100644 --- a/blockstack_client/subdomains.py +++ b/blockstack_client/subdomains.py @@ -43,7 +43,7 @@ from subdomain_registrar.util import (SUBDOMAIN_ZF_PARTS, SUBDOMAIN_ZF_PIECE, log = get_logger() -SUBDOMAINS_FIRST_BLOCK = 478873 +SUBDOMAINS_FIRST_BLOCK = 478872 class DomainNotOwned(Exception): pass @@ -204,40 +204,53 @@ class SubdomainDB(object): last_block = max(last_block, SUBDOMAINS_FIRST_BLOCK) core_last_block = proxy.getinfo()['last_block_processed'] - domains_to_check = set() - log.debug("Fetching nameops affected in range ({}, {})".format( + log.debug("Fetching zonefiles in range ({}, {})".format( last_block + 1, core_last_block)) - for block in range(last_block + 1, core_last_block): - log.debug("Block: {}".format(block)) - nameops_at = proxy.get_nameops_affected_at(block) - for n in nameops_at: - if n['opcode'] == "NAME_UPDATE": - domains_to_check.add(str(n['name'])) + zonefiles_in_blocks = proxy.get_zonefiles_by_block(last_block + 1, + core_last_block) + if 'error' in zonefiles_in_blocks: + log.error(zonefiles_in_blocks) + return + core_last_block = min(zonefiles_in_blocks['last_block'], + core_last_block) + zonefiles_info = zonefiles_in_blocks['zonefile_info'] + if len(zonefiles_info) == 0: + return + zonefiles_info.sort( key = lambda a : a['block_height'] ) + domains, hashes, blockids, txids = map( list, + zip(* [ ( x['name'], x['zonefile_hash'], + x['block_height'], + x['txid'] ) + for x in zonefiles_info ])) + zf_dict = {} + zonefiles_to_fetch_per = 100 + for offset in range(0, len(hashes)/zonefiles_to_fetch_per + 1): + lower = offset * zonefiles_to_fetch_per + upper = min(lower + zonefiles_to_fetch_per, len(hashes)) + zf_resp = proxy.get_zonefiles( + None, hashes[lower:upper], proxy = proxy.get_default_proxy()) + if 'zonefiles' not in zf_resp: + log.error("Couldn't get zonefiles from proxy {}".format(zf_resp)) + return + zf_dict.update( zf_resp['zonefiles'] ) + if len(zf_dict) == 0: + return + could_not_find = [] + zonefiles = [] + for ix, zf_hash in enumerate(hashes): + if zf_hash not in zf_dict: + could_not_find.append(ix) + else: + zonefiles.append(zf_dict[zf_hash]) + could_not_find.sort(reverse=True) + for ix in could_not_find: + del domains[ix] + del hashes[ix] + del blockids[ix] + del txids[ix] - for domain in domains_to_check: - zonefiles, hashes, blockids, txids = data.list_zonefile_history( - domain, return_hashes = True, from_block = last_block + 1, return_blockids = True, - return_txids = True) - - if len(hashes) == 0: - continue - - failed_zonefiles = [] - for ix, zonefile in enumerate(zonefiles): - if 'error' in zonefile: - failed_zonefiles.append(ix) - log.error("Failed to get zonefile for hash ({}), error: {}".format( - hashes[ix], zonefile)) - failed_zonefiles.sort(reverse=True) - for ix in failed_zonefiles: - del zonefiles[ix] - del hashes[ix] - del blockids[ix] - if len(hashes) == 0: - continue - - _build_subdomain_db(domain, zonefiles, self, txids) + _build_subdomain_db(domains, zonefiles, self, txids) last_block = core_last_block @@ -356,19 +369,12 @@ def _transition_valid(from_sub_record, to_sub_record): return False return True -def _build_subdomain_db(domain_fqa, zonefiles, subdomain_db = None, txids = None): +def _build_subdomain_db(domain_fqas, zonefiles, subdomain_db = None, txids = None): if subdomain_db is None: subdomain_db = {} - if txids is not None: - iterator = zip(zonefiles, txids) - else: - iterator = zonefiles - for cur_row in iterator: - if txids is not None: - zf, txid = cur_row - else: - zf, txid = cur_row, None - + if txids is None: + txids = [None for x in zonefiles] + for zf, domain_fqa, txid in zip(zonefiles, domain_fqas, txids): if isinstance(zf, dict): assert "zonefile" not in zf zf_json = zf @@ -441,7 +447,7 @@ def add_subdomains(subdomains, domain_fqa): def get_subdomain_info(subdomain, domain_fqa, use_cache = True): if not use_cache: zonefiles = data.list_zonefile_history(domain_fqa) - subdomain_db = _build_subdomain_db(domain_fqa, zonefiles) + subdomain_db = _build_subdomain_db([domain_fqa for z in zonefiles], zonefiles) else: subdomain_db = SubdomainDB() subdomain_db.update() diff --git a/unittests/subdomains/zonefiles.py b/unittests/subdomains/zonefiles.py index 8b8aa9e67..1b044312e 100644 --- a/unittests/subdomains/zonefiles.py +++ b/unittests/subdomains/zonefiles.py @@ -262,20 +262,20 @@ registrar URI 10 1 "bsreg://foo.com:8234" history.append(blockstack_zones.make_zone_file(zf_json)) - subdomain_db = subdomains._build_subdomain_db("bar.id", history[:1]) + subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(1)], history[:1]) self.assertIn("foo.bar.id", subdomain_db, "Contents actually: {}".format(subdomain_db.keys())) self.assertEqual(subdomain_db["foo.bar.id"].n, 0) self.assertNotIn("bar.bar.id", subdomain_db) - subdomain_db = subdomains._build_subdomain_db("bar.id", history[:2]) + subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(2)], history[:2]) self.assertIn("bar.bar.id", subdomain_db) self.assertEqual(subdomain_db["bar.bar.id"].n, 0) - subdomain_db = subdomains._build_subdomain_db("bar.id", history[:3]) + subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(3)], history[:3]) self.assertEqual(subdomain_db["foo.bar.id"].n, 0) self.assertEqual(subdomain_db["bar.bar.id"].n, 1) - subdomain_db = subdomains._build_subdomain_db("bar.id", history) + subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(len(history))], history) self.assertEqual(subdomain_db["foo.bar.id"].n, 1) self.assertEqual(subdomain_db["bar.bar.id"].n, 1) @@ -327,24 +327,24 @@ registrar URI 10 1 "bsreg://foo.com:8234" subdomain_util._extend_with_subdomain(zf_json, sub2) history.append(blockstack_zones.make_zone_file(zf_json)) - subdomain_db = subdomains._build_subdomain_db("bar.id", history[:1]) + subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(1)], history[:1]) self.assertEqual(subdomain_db["foo.bar.id"].n, 0) self.assertNotIn("bar.bar.id", subdomain_db) - subdomain_db = subdomains._build_subdomain_db("bar.id", history[:2]) + subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(2)], history[:2]) self.assertIn("bar.bar.id", subdomain_db) self.assertEqual(subdomain_db["bar.bar.id"].n, 0) - subdomain_db = subdomains._build_subdomain_db("bar.id", history[:3]) + subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(3)], history[:3]) self.assertEqual(subdomain_db["foo.bar.id"].n, 0) self.assertEqual(subdomain_db["bar.bar.id"].n, 0) # handle repeated zonefile - subdomain_db = subdomains._build_subdomain_db("bar.id", history[:3]) + subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(3)], history[:3]) self.assertEqual(subdomain_db["foo.bar.id"].n, 0) self.assertEqual(subdomain_db["bar.bar.id"].n, 0) - subdomains._build_subdomain_db("bar.id", history[:3], subdomain_db = subdomain_db) + subdomains._build_subdomain_db(["bar.id" for x in range(3)], history[:3], subdomain_db = subdomain_db) self.assertEqual(subdomain_db["foo.bar.id"].n, 0) self.assertEqual(subdomain_db["bar.bar.id"].n, 0)