kylefang/stacks-puppet-node
1
0
Fork 0
mirror of https://github.com/alexgo-io/stacks-puppet-node.git synced 2026-04-22 10:36:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

support for greedily filling the subdomain cache by querying blockstack core for new zonefiles

Browse Source
This commit is contained in:
Aaron Blankstein
2017-08-08 15:00:52 -04:00
parent dfe9aecf2c
commit e0cb5f04f6
8 changed files with 129 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
blockstack/blockstackd.py
View File

@@ -1578,7 +1578,17 @@ class BlockstackdRPC( SimpleXMLRPCServer):
def rpc_get_zonefiles_from_blocks( self, from_block, to_block, offset, count, **con_info ):
def rpc_get_zonefiles_by_block( self, from_block, to_block, offset, count, **con_info ):
"""
Get information about zonefiles announced in blocks [@from_block, @to_block]
@offset - offset into result set
@count - max records to return, must be <= 100
Returns {'status': True, 'lastblock' : blockNumber,
'zonefile_info' : [ { 'block_height' : 470000,
'txid' : '0000000',
'zonefile_hash' : '0000000' } ] }
"""
conf = get_blockstack_opts()
if not conf['atlas']:
return {'error': 'Not an atlas node'}

5
blockstack/lib/atlas.py
View File

@@ -729,8 +729,8 @@ def atlasdb_get_zonefiles_by_block( from_block, to_block, offset, count, con=Non
with AtlasDBOpen(con=con, path=path) as dbcon:
sql = """SELECT zonefile_hash, txid, block_height FROM zonefiles
WHERE block_height >= ? and block_height <= ?
sql = """SELECT name, zonefile_hash, txid, block_height FROM zonefiles
WHERE block_height >= ? and block_height <= ?
ORDER BY inv_index LIMIT ? OFFSET ?;"""
args = (from_block, to_block, count, offset)
@@ -742,6 +742,7 @@ def atlasdb_get_zonefiles_by_block( from_block, to_block, offset, count, con=Non
for zfinfo in res:
ret.append({
'name' : zfinfo['name'],
'zonefile_hash' : zfinfo['zonefile_hash'],
'block_height' : zfinfo['block_height'],
'txid' : zfinfo['txid'],

2
blockstack/lib/storage/crawl.py
View File

@@ -284,7 +284,7 @@ def store_zonefile_data_to_storage( zonefile_text, txid, required=None, skip=Non
log.debug("Failed to cache zonefile %s" % zonefile_hash)
# NOTE: this can fail if one of the required drivers needs a non-null txid
res = blockstack_client.storage.put_immutable_data( zonefile_text, txid, data_hash=zonefile_hash, required=required, skip=skip )
res = blockstack_client.storage.put_immutable_data( zonefile_text, txid, data_hash=zonefile_hash, required=required, skip=skip, required_exclusive=True )
if res is None:
log.error("Failed to store zonefile '%s' for '%s'" % (zonefile_hash, txid))
return False

2
blockstack/version.py
View File

@@ -1,5 +1,5 @@
# this is the only place where version should be updated
__version_major__ = '0'
__version_minor__ = '14'
__version_patch__ = '2'
__version_patch__ = '4'
__version__ = '{}.{}.{}.0'.format(__version_major__, __version_minor__, __version_patch__)

51
blockstack_client/proxy.py
View File

@@ -1418,6 +1418,57 @@ def get_op_history_rows(name, proxy=None):
return history_rows
def get_zonefiles_by_block(from_block, to_block, proxy=None):
"""
Get zonefile information for zonefiles announced in [@from_block, @to_block]
Returns { 'last_block' : server's last seen block,
'zonefile_info' : [ { 'zonefile_hash' : '...',
'txid' : '...',
'block_height' : '...' } ] }
"""
zonefile_info_schema = {
'type' : 'array',
'items' : {
'type' : 'object',
'properties' : {
'name' : {'type' : 'string'},
'zonefile_hash' : { 'type' : 'string',
'pattern' : OP_ZONEFILE_HASH_PATTERN },
'txid' : {'type' : 'string',
'pattern' : OP_TXID_PATTERN},
'block_height' : {'type' : 'integer'}
},
'required' : [ 'zonefile_hash', 'txid', 'block_height' ]
}
}
response_schema = {
'type' : 'object',
'properties' : {
'lastblock' : {'type' : 'integer'},
'zonefile_info' : zonefile_info_schema
},
'required' : ['lastblock', 'zonefile_info']
}
proxy = get_default_proxy() if proxy is None else proxy
offset = 0
output_zonefiles = []
last_server_block = 0
while offset == 0 or len(resp['zonefile_info']) > 0:
resp = proxy.get_zonefiles_by_block(from_block, to_block, offset, 100)
if 'error' in resp:
return resp
resp = json_validate(response_schema, resp)
if json_is_error(resp):
return resp
output_zonefiles += resp['zonefile_info']
offset += 100
last_server_block = max(resp['lastblock'], last_server_block)
return { 'last_block' : last_server_block,
'zonefile_info' : output_zonefiles }
def get_nameops_affected_at(block_id, proxy=None):
"""

4
blockstack_client/storage.py
View File

@@ -920,7 +920,7 @@ def get_mutable_data(fq_data_id, data_pubkey, urls=None, data_address=None, data
return None
def put_immutable_data(data_text, txid, data_hash=None, required=None, skip=None):
def put_immutable_data(data_text, txid, data_hash=None, required=None, skip=None, required_exclusive=False):
"""
Given a string of data (which can either be data or a zonefile), store it into our immutable data stores.
Do so in a best-effort manner--this method only fails if *all* storage providers fail.
@@ -947,6 +947,8 @@ def put_immutable_data(data_text, txid, data_hash=None, required=None, skip=None
log.debug(msg.format(data_hash, ','.join(required), ','.join(skip)))
for handler in storage_handlers:
if required_exclusive and handler.__name__ not in required:
continue
if handler.__name__ in skip:
log.debug("Skipping {}".format(handler.__name__))
continue

94
blockstack_client/subdomains.py
View File

@@ -43,7 +43,7 @@ from subdomain_registrar.util import (SUBDOMAIN_ZF_PARTS, SUBDOMAIN_ZF_PIECE,
log = get_logger()
SUBDOMAINS_FIRST_BLOCK = 478873
SUBDOMAINS_FIRST_BLOCK = 478872
class DomainNotOwned(Exception):
pass
@@ -204,40 +204,53 @@ class SubdomainDB(object):
last_block = max(last_block, SUBDOMAINS_FIRST_BLOCK)
core_last_block = proxy.getinfo()['last_block_processed']
domains_to_check = set()
log.debug("Fetching nameops affected in range ({}, {})".format(
log.debug("Fetching zonefiles in range ({}, {})".format(
last_block + 1, core_last_block))
for block in range(last_block + 1, core_last_block):
log.debug("Block: {}".format(block))
nameops_at = proxy.get_nameops_affected_at(block)
for n in nameops_at:
if n['opcode'] == "NAME_UPDATE":
domains_to_check.add(str(n['name']))
zonefiles_in_blocks = proxy.get_zonefiles_by_block(last_block + 1,
core_last_block)
if 'error' in zonefiles_in_blocks:
log.error(zonefiles_in_blocks)
return
core_last_block = min(zonefiles_in_blocks['last_block'],
core_last_block)
zonefiles_info = zonefiles_in_blocks['zonefile_info']
if len(zonefiles_info) == 0:
return
zonefiles_info.sort( key = lambda a : a['block_height'] )
domains, hashes, blockids, txids = map( list,
zip(* [ ( x['name'], x['zonefile_hash'],
x['block_height'],
x['txid'] )
for x in zonefiles_info ]))
zf_dict = {}
zonefiles_to_fetch_per = 100
for offset in range(0, len(hashes)/zonefiles_to_fetch_per + 1):
lower = offset * zonefiles_to_fetch_per
upper = min(lower + zonefiles_to_fetch_per, len(hashes))
zf_resp = proxy.get_zonefiles(
None, hashes[lower:upper], proxy = proxy.get_default_proxy())
if 'zonefiles' not in zf_resp:
log.error("Couldn't get zonefiles from proxy {}".format(zf_resp))
return
zf_dict.update( zf_resp['zonefiles'] )
if len(zf_dict) == 0:
return
could_not_find = []
zonefiles = []
for ix, zf_hash in enumerate(hashes):
if zf_hash not in zf_dict:
could_not_find.append(ix)
else:
zonefiles.append(zf_dict[zf_hash])
could_not_find.sort(reverse=True)
for ix in could_not_find:
del domains[ix]
del hashes[ix]
del blockids[ix]
del txids[ix]
for domain in domains_to_check:
zonefiles, hashes, blockids, txids = data.list_zonefile_history(
domain, return_hashes = True, from_block = last_block + 1, return_blockids = True,
return_txids = True)
if len(hashes) == 0:
continue
failed_zonefiles = []
for ix, zonefile in enumerate(zonefiles):
if 'error' in zonefile:
failed_zonefiles.append(ix)
log.error("Failed to get zonefile for hash ({}), error: {}".format(
hashes[ix], zonefile))
failed_zonefiles.sort(reverse=True)
for ix in failed_zonefiles:
del zonefiles[ix]
del hashes[ix]
del blockids[ix]
if len(hashes) == 0:
continue
_build_subdomain_db(domain, zonefiles, self, txids)
_build_subdomain_db(domains, zonefiles, self, txids)
last_block = core_last_block
@@ -356,19 +369,12 @@ def _transition_valid(from_sub_record, to_sub_record):
return False
return True
def _build_subdomain_db(domain_fqa, zonefiles, subdomain_db = None, txids = None):
def _build_subdomain_db(domain_fqas, zonefiles, subdomain_db = None, txids = None):
if subdomain_db is None:
subdomain_db = {}
if txids is not None:
iterator = zip(zonefiles, txids)
else:
iterator = zonefiles
for cur_row in iterator:
if txids is not None:
zf, txid = cur_row
else:
zf, txid = cur_row, None
if txids is None:
txids = [None for x in zonefiles]
for zf, domain_fqa, txid in zip(zonefiles, domain_fqas, txids):
if isinstance(zf, dict):
assert "zonefile" not in zf
zf_json = zf
@@ -441,7 +447,7 @@ def add_subdomains(subdomains, domain_fqa):
def get_subdomain_info(subdomain, domain_fqa, use_cache = True):
if not use_cache:
zonefiles = data.list_zonefile_history(domain_fqa)
subdomain_db = _build_subdomain_db(domain_fqa, zonefiles)
subdomain_db = _build_subdomain_db([domain_fqa for z in zonefiles], zonefiles)
else:
subdomain_db = SubdomainDB()
subdomain_db.update()

18
unittests/subdomains/zonefiles.py
View File

@@ -262,20 +262,20 @@ registrar URI 10 1 "bsreg://foo.com:8234"
history.append(blockstack_zones.make_zone_file(zf_json))
subdomain_db = subdomains._build_subdomain_db("bar.id", history[:1])
subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(1)], history[:1])
self.assertIn("foo.bar.id", subdomain_db, "Contents actually: {}".format(subdomain_db.keys()))
self.assertEqual(subdomain_db["foo.bar.id"].n, 0)
self.assertNotIn("bar.bar.id", subdomain_db)
subdomain_db = subdomains._build_subdomain_db("bar.id", history[:2])
subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(2)], history[:2])
self.assertIn("bar.bar.id", subdomain_db)
self.assertEqual(subdomain_db["bar.bar.id"].n, 0)
subdomain_db = subdomains._build_subdomain_db("bar.id", history[:3])
subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(3)], history[:3])
self.assertEqual(subdomain_db["foo.bar.id"].n, 0)
self.assertEqual(subdomain_db["bar.bar.id"].n, 1)
subdomain_db = subdomains._build_subdomain_db("bar.id", history)
subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(len(history))], history)
self.assertEqual(subdomain_db["foo.bar.id"].n, 1)
self.assertEqual(subdomain_db["bar.bar.id"].n, 1)
@@ -327,24 +327,24 @@ registrar URI 10 1 "bsreg://foo.com:8234"
subdomain_util._extend_with_subdomain(zf_json, sub2)
history.append(blockstack_zones.make_zone_file(zf_json))
subdomain_db = subdomains._build_subdomain_db("bar.id", history[:1])
subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(1)], history[:1])
self.assertEqual(subdomain_db["foo.bar.id"].n, 0)
self.assertNotIn("bar.bar.id", subdomain_db)
subdomain_db = subdomains._build_subdomain_db("bar.id", history[:2])
subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(2)], history[:2])
self.assertIn("bar.bar.id", subdomain_db)
self.assertEqual(subdomain_db["bar.bar.id"].n, 0)
subdomain_db = subdomains._build_subdomain_db("bar.id", history[:3])
subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(3)], history[:3])
self.assertEqual(subdomain_db["foo.bar.id"].n, 0)
self.assertEqual(subdomain_db["bar.bar.id"].n, 0)
# handle repeated zonefile
subdomain_db = subdomains._build_subdomain_db("bar.id", history[:3])
subdomain_db = subdomains._build_subdomain_db(["bar.id" for x in range(3)], history[:3])
self.assertEqual(subdomain_db["foo.bar.id"].n, 0)
self.assertEqual(subdomain_db["bar.bar.id"].n, 0)
subdomains._build_subdomain_db("bar.id", history[:3], subdomain_db = subdomain_db)
subdomains._build_subdomain_db(["bar.id" for x in range(3)], history[:3], subdomain_db = subdomain_db)
self.assertEqual(subdomain_db["foo.bar.id"].n, 0)
self.assertEqual(subdomain_db["bar.bar.id"].n, 0)