kylefang/stacks-puppet-node
1
0
Fork 0
mirror of https://github.com/alexgo-io/stacks-puppet-node.git synced 2026-04-16 22:18:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

disable durability pragmas in the subdomain db, since if the node crashes we have to restore the subdomain db anyway (2x speed improvement). also, add a 'zonefile_offset' column to the subdomain db that fixes DID creation in the case where the same owner address created multiple subdomains in the same zone file

Browse Source
This commit is contained in:
Jude Nelson
2018-07-18 18:57:12 -04:00
parent 40e4faa2e7
commit cea93eaaed
1 changed files with 48 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
blockstack/lib/subdomains.py
View File

@@ -98,7 +98,7 @@ class Subdomain(object):
"""
Subdomain entry
"""
def __init__(self, fqn, domain, address, n, zonefile_str, sig, block_height, parent_zonefile_hash, parent_zonefile_index, txid, domain_zonefiles_missing=[], accepted=False, resolver=None):
def __init__(self, fqn, domain, address, n, zonefile_str, sig, block_height, parent_zonefile_hash, parent_zonefile_index, zonefile_offset, txid, domain_zonefiles_missing=[], accepted=False, resolver=None):
"""
@fqn: fully-qualified subdomain name
@domain: the stem name at which this subdomain record was found
@@ -121,6 +121,7 @@ class Subdomain(object):
# pertinent information discovered when querying for subdomains
self.block_height = block_height
self.parent_zonefile_index = parent_zonefile_index
self.zonefile_offset = zonefile_offset
self.parent_zonefile_hash = parent_zonefile_hash
self.txid = txid
self.independent = False # indicates whether or not this record is independent of its domain (i.e. a.b.id is independent of c.id, but not b.id)
@@ -273,7 +274,7 @@ class Subdomain(object):
@staticmethod
def parse_subdomain_record(domain_name, rec, block_height, parent_zonefile_hash, parent_zonefile_index, txid, domain_zonefiles_missing, resolver=None):
def parse_subdomain_record(domain_name, rec, block_height, parent_zonefile_hash, parent_zonefile_index, zonefile_offset, txid, domain_zonefiles_missing, resolver=None):
"""
Parse a subdomain record, and verify its signature.
@domain_name: the stem name
@@ -338,7 +339,7 @@ class Subdomain(object):
# already fully-qualified
subd_name = rec['name']
return Subdomain(str(subd_name), str(domain_name), str(pubkey), int(n), base64.b64decode(b64_zonefile), str(sig), block_height, parent_zonefile_hash, parent_zonefile_index, txid, domain_zonefiles_missing=domain_zonefiles_missing, resolver=resolver)
return Subdomain(str(subd_name), str(domain_name), str(pubkey), int(n), base64.b64decode(b64_zonefile), str(sig), block_height, parent_zonefile_hash, parent_zonefile_index, zonefile_offset, txid, domain_zonefiles_missing=domain_zonefiles_missing, resolver=resolver)
def get_public_key(self):
@@ -585,6 +586,7 @@ class SubdomainIndex(object):
return hist
else:
log.debug("Accept initial {}".format(hist[0]))
pass
hist[0].accepted = True
last_accepted = 0
@@ -752,6 +754,11 @@ class SubdomainIndex(object):
"""
cursor = self.subdomain_db.cursor()
# we can afford to be fast and loose here since if the host crashes while this is going on,
# the node will do a `restore` anyway and wipe this db out.
db_query_execute(cursor, 'PRAGMA synchronous = off;', ())
db_query_execute(cursor, 'PRAGMA journal_mode = off;', ())
db_query_execute(cursor, 'BEGIN', ())
# no matter what we do, store everything.
@@ -765,9 +772,6 @@ class SubdomainIndex(object):
log.debug("Store {}".format(subrec))
self.subdomain_db.update_subdomain_entry(subrec, cur=cursor)
db_query_execute(cursor, 'END', ())
self.subdomain_db.commit()
# at each zone file, find out if its subdomain creates/updates are valid
for subinfo in zonefile_subdomain_info:
@@ -784,8 +788,6 @@ class SubdomainIndex(object):
assert subrec.get_fqn() not in new_subdomain_recs, 'BUG: duplicate subdomain record for "{}" in {}'.format(subrec.get_fqn(), zfhash)
new_subdomain_recs[subrec.get_fqn()] = subrec
db_query_execute(cursor, 'BEGIN', ())
for fqn in new_subdomain_recs:
immediate_history = self.get_subdomain_history_neighbors(cursor, new_subdomain_recs[fqn])
inserted = self.subdomain_try_insert(cursor, new_subdomain_recs[fqn], immediate_history)
@@ -813,9 +815,8 @@ class SubdomainIndex(object):
for subrec in new_fut:
self.subdomain_db.update_subdomain_entry(subrec, cur=cursor)
db_query_execute(cursor, 'END', ())
self.subdomain_db.commit()
db_query_execute(cursor, 'END', ())
def enqueue_zonefile(self, zonefile_hash, block_height):
"""
@@ -928,8 +929,10 @@ class SubdomainIndex(object):
* process any newly-arrived zone files and re-index the affected subdomains
"""
log.debug("BEGIN Processing zonefiles discovered since last re-indexing")
t1 = time.time()
self.index_discovered_zonefiles(block_end)
log.debug("END Processing zonefiles discovered since last re-indexing")
t2 = time.time()
log.debug("END Processing zonefiles discovered since last re-indexing ({} seconds)".format(t2 - t1))
@classmethod
@@ -1025,6 +1028,7 @@ class SubdomainDB(object):
block_height = int(rowdata['block_height'])
parent_zonefile_hash = str(rowdata['parent_zonefile_hash'])
parent_zonefile_index = int(rowdata['parent_zonefile_index'])
zonefile_offset = int(rowdata['zonefile_offset'])
txid = str(rowdata['txid'])
missing = [int(i) for i in rowdata['missing'].split(',')] if rowdata['missing'] is not None and len(rowdata['missing']) > 0 else []
accepted = int(rowdata['accepted'])
@@ -1050,7 +1054,7 @@ class SubdomainDB(object):
log.error("No zone file for {}".format(name))
raise SubdomainNotFound('{}: missing zone file {}'.format(name, zonefile_hash))
return Subdomain(str(name), str(domain), str(encoded_pubkey), int(n), str(zonefile_str), sig, block_height, parent_zonefile_hash, parent_zonefile_index, txid, domain_zonefiles_missing=missing, accepted=accepted, resolver=resolver)
return Subdomain(str(name), str(domain), str(encoded_pubkey), int(n), str(zonefile_str), sig, block_height, parent_zonefile_hash, parent_zonefile_index, zonefile_offset, txid, domain_zonefiles_missing=missing, accepted=accepted, resolver=resolver)
def get_subdomains_count(self, accepted=True, cur=None):
@@ -1240,8 +1244,8 @@ class SubdomainDB(object):
Return {'name_type': ..., 'address': ..., 'index': ...}
"""
subrec = self.get_subdomain_entry_at_sequence(fqn, 0, cur=cur)
cmd = 'SELECT COUNT(*) FROM {} WHERE owner = ? AND sequence = ? AND parent_zonefile_index <= ? AND accepted=1 ORDER BY parent_zonefile_index LIMIT 1;'.format(self.subdomain_table)
args = (subrec.address,0,subrec.parent_zonefile_index)
cmd = 'SELECT zonefile_offset FROM {} WHERE fully_qualified_subdomain = ? AND owner = ? AND sequence=0 AND parent_zonefile_index <= ? AND accepted=1 ORDER BY parent_zonefile_index, zonefile_offset LIMIT 1;'.format(self.subdomain_table)
args = (fqn, subrec.address, subrec.parent_zonefile_index)
cursor = None
if cur is None:
@@ -1251,15 +1255,27 @@ class SubdomainDB(object):
rows = db_query_execute(cursor, cmd, args)
zonefile_offset = None
for r in rows:
zonefile_offset = r['zonefile_offset']
break
if zonefile_offset is None:
raise SubdomainNotFound('No rows for {}'.format(fqn))
cmd = 'SELECT COUNT(*) FROM {} WHERE owner = ? AND sequence=0 AND (parent_zonefile_index < ? OR parent_zonefile_index = ? AND zonefile_offset < ?) AND accepted=1 ORDER BY parent_zonefile_index, zonefile_offset LIMIT 1;'.format(self.subdomain_table)
args = (subrec.address, subrec.parent_zonefile_index, subrec.parent_zonefile_index, zonefile_offset)
rows = db_query_execute(cursor, cmd, args)
count = None
for r in rows:
count = r['COUNT(*)']
break
if not count:
if count is None:
raise SubdomainNotFound('No rows for {}'.format(fqn))
return {'name_type': 'subdomain', 'address': subrec.address, 'index': count-1}
return {'name_type': 'subdomain', 'address': subrec.address, 'index': count}
def get_DID_subdomain(self, did, cur=None):
@@ -1280,8 +1296,8 @@ class SubdomainDB(object):
name_index = did_info['index']
# find the initial subdomain (the nth subdomain created by this address)
cmd = 'SELECT fully_qualified_subdomain FROM {} WHERE owner = ? AND sequence = ? ORDER BY parent_zonefile_index LIMIT 1 OFFSET ?;'.format(self.subdomain_table)
args = (original_address,0,name_index)
cmd = 'SELECT fully_qualified_subdomain FROM {} WHERE owner = ? AND sequence = ? ORDER BY parent_zonefile_index, zonefile_offset LIMIT 1 OFFSET ?;'.format(self.subdomain_table)
args = (original_address, 0, name_index)
cursor = None
if cur is None:
@@ -1370,16 +1386,19 @@ class SubdomainDB(object):
"""
# sanity checks
assert isinstance(subdomain_obj, Subdomain)
# NOTE: there is no need to call fsync() on the zone file fd here---we already have the data from the on-chain name's zone file fsync'ed,
# so this information is already durable (albeit somewhere else) and can ostensibly be restored later.
# We get such high subdomain traffic that we cannot call fsync() here each time; otherwise we could stall the node.
zonefile_hash = get_zonefile_data_hash(subdomain_obj.zonefile_str)
rc = store_atlas_zonefile_data(subdomain_obj.zonefile_str, self.zonefiles_dir)
rc = store_atlas_zonefile_data(subdomain_obj.zonefile_str, self.zonefiles_dir, fsync=False)
if not rc:
raise Exception("Failed to store zone file {} from {}".format(zonefile_hash, subdomain_obj.get_fqn()))
write_cmd = 'INSERT OR REPLACE INTO {} VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)'.format(self.subdomain_table)
write_cmd = 'INSERT OR REPLACE INTO {} VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)'.format(self.subdomain_table)
args = (subdomain_obj.get_fqn(), subdomain_obj.domain, subdomain_obj.n, subdomain_obj.address, zonefile_hash,
subdomain_obj.sig, subdomain_obj.block_height, subdomain_obj.parent_zonefile_hash,
subdomain_obj.parent_zonefile_index, subdomain_obj.txid,
subdomain_obj.parent_zonefile_index, subdomain_obj.zonefile_offset, subdomain_obj.txid,
','.join(str(i) for i in subdomain_obj.domain_zonefiles_missing),
1 if subdomain_obj.accepted else 0,
subdomain_obj.resolver)
@@ -1504,6 +1523,7 @@ class SubdomainDB(object):
block_height INTEGER NOT NULL,
parent_zonefile_hash TEXT NOT NULL,
parent_zonefile_index INTEGER NOT NULL,
zonefile_offset INTEGER NOT NULL,
txid TEXT NOT NULL,
missing TEXT NOT NULL,
accepted INTEGER NOT NULL,
@@ -1570,6 +1590,7 @@ def decode_zonefile_subdomains(domain, zonefile_txt, block_height, zonefile_inde
subdomains = {} # map fully-qualified name to subdomain record with lowest sequence number
subdomain_pos = {} # map fully-qualified name to position in zone file
domain_zonefiles_missing = None # list of zone files declared missing by this domain
zonefile_offset = 0
if "txt" in zonefile_json:
for i, txt in enumerate(zonefile_json['txt']):
@@ -1597,7 +1618,8 @@ def decode_zonefile_subdomains(domain, zonefile_txt, block_height, zonefile_inde
if txt['name'] in SUBDOMAIN_TXT_RR_RESERVED:
continue
subrec = Subdomain.parse_subdomain_record(domain, txt, block_height, zonefile_hash, zonefile_index, txid, domain_zonefiles_missing, resolver=resolver_url)
subrec = Subdomain.parse_subdomain_record(domain, txt, block_height, zonefile_hash, zonefile_index, zonefile_offset, txid, domain_zonefiles_missing, resolver=resolver_url)
zonefile_offset += 1
except ParseError as pe:
if BLOCKSTACK_DEBUG:
log.exception(pe)
@@ -2018,11 +2040,11 @@ def make_subdomain_txt(name_or_fqn, domain, address, n, zonefile_str, privkey_bu
Make a signed subdomain TXT record, to be appended to a (domain's) zone file.
Return the TXT record string
"""
subrec = Subdomain(str(name_or_fqn), str(domain), str(address), int(n), str(zonefile_str), None, None, None, None, None)
subrec = Subdomain(str(name_or_fqn), str(domain), str(address), int(n), str(zonefile_str), None, None, None, None, None, None)
subrec_plaintext = subrec.get_plaintext_to_sign()
sig = sign(privkey_bundle, subrec_plaintext)
subrec = Subdomain(str(name_or_fqn), str(domain), str(address), int(n), str(zonefile_str), str(sig), None, None, None, None)
subrec = Subdomain(str(name_or_fqn), str(domain), str(address), int(n), str(zonefile_str), str(sig), None, None, None, None, None)
return subrec.serialize_to_txt()