kylefang/stacks-puppet-node
1
0
Fork 0
mirror of https://github.com/alexgo-io/stacks-puppet-node.git synced 2026-05-10 12:24:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'hotfix/fast-sync-subdomain-queue'

Browse Source
This commit is contained in:
Jude Nelson
2018-10-17 09:35:47 -04:00
parent eb7ab147a8 1d98b7116e
commit d4f44a41ad
4 changed files with 175 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
blockstack/lib/nameset/namedb.py
View File

@@ -270,7 +270,11 @@ class BlockstackDB( virtualchain.StateEngine ):
"""
Get the paths to the relevant db files to back up
"""
return super(BlockstackDB, cls).get_state_paths(impl, working_dir) + [os.path.join(working_dir, 'atlas.db'), os.path.join(working_dir, 'subdomains.db')]
return super(BlockstackDB, cls).get_state_paths(impl, working_dir) + [
os.path.join(working_dir, 'atlas.db'),
os.path.join(working_dir, 'subdomains.db'),
os.path.join(working_dir, 'subdomains.db.queue')
]
def get_db_path( self ):
@@ -308,9 +312,11 @@ class BlockstackDB( virtualchain.StateEngine ):
snapshots_path = os.path.join(dirpath, os.path.basename(virtualchain.get_snapshots_filename(virtualchain_hooks, self.working_dir)))
atlas_path = os.path.join(dirpath, 'atlas.db')
subdomain_path = os.path.join(dirpath, 'subdomains.db')
subdomain_queue_path = os.path.join(dirpath, 'subdomains.db.queue')
src_atlas_path = os.path.join(self.working_dir, 'atlas.db')
src_subdomain_path = os.path.join(self.working_dir, 'subdomains.db')
src_subdomain_queue_path = os.path.join(self.working_dir, 'subdomains.db.queue')
virtualchain.sqlite3_backup(self.get_db_path(), db_path)
virtualchain.sqlite3_backup(virtualchain.get_snapshots_filename(virtualchain_hooks, self.working_dir), snapshots_path)
@@ -320,6 +326,9 @@ class BlockstackDB( virtualchain.StateEngine ):
if os.path.exists(src_subdomain_path):
virtualchain.sqlite3_backup(src_subdomain_path, subdomain_path)
if os.path.exists(src_subdomain_queue_path):
virtualchain.sqlite3_backup(src_subdomain_queue_path, subdomain_queue_path)
@classmethod

2
blockstack/version.py
View File

@@ -1,5 +1,5 @@
# this is the only place where version should be updated
__version_major__ = '19'
__version_minor__ = '0'
__version_patch__ = '1'
__version_patch__ = '2'
__version__ = '{}.{}.{}.0'.format(__version_major__, __version_minor__, __version_patch__)

261
integration_tests/blockstack_integration_tests/scenarios/name_pre_regup_subdomain_fast_sync.py
View File

@@ -68,7 +68,7 @@ def restore( working_dir, snapshot_path, restore_dir, pubkeys, num_required ):
return False
# database must be identical
db_filenames = ['blockstack-server.db', 'blockstack-server.snapshots', 'atlas.db', 'subdomains.db']
db_filenames = ['blockstack-server.db', 'blockstack-server.snapshots', 'atlas.db', 'subdomains.db', 'subdomains.db.queue']
src_paths = [os.path.join(working_dir, fn) for fn in db_filenames]
backup_paths = [os.path.join(restore_dir, fn) for fn in db_filenames]
@@ -94,12 +94,6 @@ def restore( working_dir, snapshot_path, restore_dir, pubkeys, num_required ):
print 'Missing import keychain {}'.format(import_keychain_path)
return False
# all subdomains are present
subds = ['bar.foo_{}.test'.format(i) for i in range(0,10)]
subdomain_db = blockstack.lib.subdomains.SubdomainDB(os.path.join(restore_dir, 'subdomains.db'), os.path.join(restore_dir, 'zonefiles'))
for subd in subds:
rec = subdomain_db.get_subdomain_entry(subd)
return True
@@ -150,119 +144,190 @@ def scenario( wallets, **kw ):
testlib.next_block( **kw )
# propagate
for name in zonefiles:
# propagate the first five subdomains
for i in range(0,5):
name = 'foo_{}.test'.format(i)
assert testlib.blockstack_put_zonefile(zonefiles[name])
# process subdomains
# process the first five subdomains
testlib.next_block( **kw )
# propagate the last five subdomains, but don't process them
for i in range(5,10):
name = 'foo_{}.test'.format(i)
assert testlib.blockstack_put_zonefile(zonefiles[name])
print 'waiting for all zone files to replicate'
time.sleep(10)
working_dir = os.environ.get('BLOCKSTACK_WORKING_DIR')
restore_dir = os.path.join(working_dir, "snapshot_dir")
# snapshot the latest backup
snapshot_path = os.path.join(working_dir, "snapshot.bsk" )
rc = blockstack.fast_sync_snapshot(kw['working_dir'], snapshot_path, wallets[3].privkey, None )
if not rc:
print "Failed to fast_sync_snapshot"
return False
if not os.path.exists(snapshot_path):
print "Failed to create snapshot {}".format(snapshot_path)
return False
# make a backup
db = testlib.get_state_engine()
# sign with more keys
for i in xrange(4, 6):
rc = blockstack.fast_sync_sign_snapshot( snapshot_path, wallets[i].privkey )
print 'begin make backups of state from {}'.format(testlib.get_current_block(**kw) - 1)
for name in os.listdir(os.path.join(working_dir, 'backups')):
if name.endswith('.{}'.format(testlib.get_current_block(**kw) - 1)):
os.unlink(os.path.join(working_dir, 'backups', name))
db.make_backups(testlib.get_current_block(**kw))
print 'end make backups'
def _backup_and_restore():
# snapshot the latest backup
snapshot_path = os.path.join(working_dir, "snapshot.bsk" )
rc = blockstack.fast_sync_snapshot(working_dir, snapshot_path, wallets[3].privkey, None )
if not rc:
print "Failed to sign with key {}".format(i)
print "Failed to fast_sync_snapshot"
return False
if not os.path.exists(snapshot_path):
print "Failed to create snapshot {}".format(snapshot_path)
return False
# restore!
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex, wallets[5].pubkey_hex], 3 )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
# sign with more keys
for i in xrange(4, 6):
rc = blockstack.fast_sync_sign_snapshot( snapshot_path, wallets[i].privkey )
if not rc:
print "Failed to sign with key {}".format(i)
return False
# restore!
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex, wallets[5].pubkey_hex], 3 )
if not rc:
print "1 failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[5].pubkey_hex, wallets[4].pubkey_hex, wallets[3].pubkey_hex], 3 )
if not rc:
print "2 failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex], 2 )
if not rc:
print "3 failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[5].pubkey_hex], 2 )
if not rc:
print "4 failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[4].pubkey_hex, wallets[5].pubkey_hex], 2 )
if not rc:
print "5 failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex], 1 )
if not rc:
print "6 failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[4].pubkey_hex, wallets[0].pubkey_hex], 1 )
if not rc:
print "7 failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[1].pubkey_hex, wallets[5].pubkey_hex], 1 )
if not rc:
print "8 failed to restore snapshot {}".format(snapshot_path)
return False
shutil.move(restore_dir, restore_dir + '.bak')
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex], 2 )
if rc:
print "restored insufficient signatures snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex], 3 )
if rc:
print "restored insufficient signatures snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex], 1 )
if rc:
print "restored wrongly-signed snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[3].pubkey_hex], 2 )
if rc:
print "restored wrongly-signed snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[3].pubkey_hex, wallets[4].pubkey_hex], 3 )
if rc:
print "restored wrongly-signed snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
shutil.move(restore_dir + '.bak', restore_dir)
return True
# test backup and restore
res = _backup_and_restore()
if not res:
return res
# first five subdomains are all present in the subdomain DB
subds = ['bar.foo_{}.test'.format(i) for i in range(0,5)]
subdomain_db = blockstack.lib.subdomains.SubdomainDB(os.path.join(restore_dir, 'subdomains.db'), os.path.join(restore_dir, 'zonefiles'))
for subd in subds:
rec = subdomain_db.get_subdomain_entry(subd)
if not rec:
print 'not found: {}'.format(subd)
return False
# last 5 subdomains are queued in the subdomain DB queue
queued_zfinfos = blockstack.lib.queue.queuedb_findall(os.path.join(restore_dir, 'subdomains.db.queue'), 'zonefiles')
if len(queued_zfinfos) != 5:
print 'only {} zonefiles queued'.format(queued_zfinfos)
print queued_zfinfos
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[5].pubkey_hex, wallets[4].pubkey_hex, wallets[3].pubkey_hex], 3 )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
# process the last five subdomains
testlib.next_block( **kw )
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex], 2 )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
os.unlink(os.path.join(working_dir, "snapshot.bsk"))
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[5].pubkey_hex], 2 )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
# test backup and restore
res = _backup_and_restore()
if not res:
return res
# all subdomains are all present in the subdomain DB
subds = ['bar.foo_{}.test'.format(i) for i in range(0,10)]
subdomain_db = blockstack.lib.subdomains.SubdomainDB(os.path.join(restore_dir, 'subdomains.db'), os.path.join(restore_dir, 'zonefiles'))
for subd in subds:
rec = subdomain_db.get_subdomain_entry(subd)
if not rec:
print 'not found: {}'.format(subd)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[4].pubkey_hex, wallets[5].pubkey_hex], 2 )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex], 1 )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[4].pubkey_hex, wallets[0].pubkey_hex], 1 )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[1].pubkey_hex, wallets[5].pubkey_hex], 1 )
if not rc:
print "failed to restore snapshot {}".format(snapshot_path)
return False
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex], 2 )
if rc:
print "restored insufficient signatures snapshot {}".format(snapshot_path)
# nothing queued
queued_zfinfos = blockstack.lib.queue.queuedb_findall(os.path.join(restore_dir, 'subdomains.db.queue'), 'zonefiles')
if len(queued_zfinfos) != 0:
print '{} zonefiles queued'.format(queued_zfinfos)
print queued_zfinfos
return False
shutil.rmtree(restore_dir)
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[3].pubkey_hex, wallets[4].pubkey_hex], 3 )
if rc:
print "restored insufficient signatures snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex], 1 )
if rc:
print "restored wrongly-signed snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[3].pubkey_hex], 2 )
if rc:
print "restored wrongly-signed snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
# should fail
rc = restore( kw['working_dir'], snapshot_path, restore_dir, [wallets[0].pubkey_hex, wallets[3].pubkey_hex, wallets[4].pubkey_hex], 3 )
if rc:
print "restored wrongly-signed snapshot {}".format(snapshot_path)
return False
shutil.rmtree(restore_dir)
def check( state_engine ):
# not revealed, but ready

2
integration_tests/blockstack_integration_tests/version.py
View File

@@ -1 +1 @@
__version__='19.0.1.0'
__version__='19.0.2.0'