diff --git a/README.md b/README.md index fc83aad7d..81b33220a 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,10 @@ If you encounter any technical issues in installing or using Blockstack, please ## Development Status -**v0.14.2** is the current stable release of Blockstack Core (available on the master branch).
-**v0.14.3** is the next release candidate for Blockstack Core (available on the [v0.14.3 branch](https://github.com/blockstack/blockstack-core/tree/rc-0.14.3)). +**v0.14.1** is the current stable release of Blockstack Core (available on the master branch).
+**v0.14.2** is the next release candidate for Blockstack Core (available on the [v0.14.2 branch](https://github.com/blockstack/blockstack-core/tree/rc-0.14.2)). -Most of the development is happening in the [v0.14.3 branch](https://github.com/blockstack/blockstack-core/tree/rc-0.14.3). Please submit all +Most of the development is happening in the [v0.14.2 branch](https://github.com/blockstack/blockstack-core/tree/rc-0.14.2). Please submit all pull requests to that branch. In the list of [release notes](https://github.com/blockstack/blockstack-core/tree/master/release_notes) you can find what has changed in these versions. diff --git a/blockstack/blockstackd.py b/blockstack/blockstackd.py index 4405eae92..15d15ed67 100644 --- a/blockstack/blockstackd.py +++ b/blockstack/blockstackd.py @@ -1266,24 +1266,22 @@ class BlockstackdRPC( SimpleXMLRPCServer): res = blockstack_client.get_profile(name, profile_storage_drivers=profile_storage_drivers, zonefile_storage_drivers=zonefile_storage_drivers, user_zonefile=zonefile_dict, name_record=name_rec, - use_zonefile_urls=False, decode=False) + use_zonefile_urls=False, decode_profile=False) if 'error' in res: log.error("Failed to load profile '{}'".format(name)) return res - profile = res['profile'] - if profile is None: - profile = res['legacy_profile'] - - if profile is None: - return {'error': 'No profile could be loaded with the given configuration'} - + zonefile = res['zonefile'] except Exception, e: log.exception(e) log.error("Failed to load profile for '{}'".fomrat(name)) return {'error': 'Failed to load profile'} - return self.success_response( {'profile': profile} ) + if 'error' in zonefile: + return zonefile + + else: + return self.success_response( {'profile': profile} ) def verify_data_timestamp( self, datum ): @@ -1316,6 +1314,49 @@ class BlockstackdRPC( SimpleXMLRPCServer): return {'status': True} + def verify_profile_hash( self, name, name_rec, zonefile_dict, profile_txt, prev_profile_hash, sigb64, user_data_pubkey ): + """ + DEPRECATED + + Verify that the uploader signed the profile's previous hash. + Return {'status': True} on success + Return {'error': ...} on error + """ + + conf = get_blockstack_opts() + if not conf['serve_profiles']: + return {'error': 'No data'} + + profile_storage_drivers = conf['profile_storage_drivers'].split(",") + zonefile_storage_drivers = conf['zonefile_storage_drivers'].split(",") + + # verify that the previous profile actually does have this hash + try: + old_profile_txt, zonefile = blockstack_client.get_profile(name, profile_storage_drivers=profile_storage_drivers, zonefile_storage_drivers=zonefile_storage_drivers, + user_zonefile=zonefile_dict, name_record=name_rec, use_zonefile_urls=False, decode_profile=False) + except Exception, e: + log.exception(e) + log.debug("Failed to load profile for '%s'" % name) + return {'error': 'Failed to load profile'} + + if old_profile_txt is None: + # no profile yet (or error) + old_profile_txt = "" + + old_profile_hash = hex_hash160(old_profile_txt) + if old_profile_hash != prev_profile_hash: + log.debug("Invalid previous profile hash") + return {'error': 'Invalid previous profile hash'} + + # finally, verify the signature over the previous profile hash and this new profile + rc = blockstack_client.storage.verify_raw_data( "%s%s" % (prev_profile_hash, profile_txt), user_data_pubkey, sigb64 ) + if not rc: + log.debug("Invalid signature") + return {'error': 'Invalid signature'} + + return {'status': True} + + def load_mutable_data( self, name, data_txt, max_len=RPC_MAX_PROFILE_LEN, storage_drivers=None ): """ Parse and authenticate user-given data @@ -1449,7 +1490,19 @@ class BlockstackdRPC( SimpleXMLRPCServer): data_info = self.load_mutable_data(name, profile_txt, max_len=RPC_MAX_PROFILE_LEN) if 'error' in data_info: - return data_info + if data_info.has_key('reason') and data_info['reason'] == 'timestamp' and data_info.has_key('data_pubkey') and data_info.has_key('zonefile'): + + user_data_pubkey = data_info['data_pubkey'] + zonefile_dict = data_info['zonefile'] + + # try hash-based verification (deprecated) + res = self.verify_profile_hash( name, name_rec, zonefile_dict, profile_txt, prev_profile_hash_or_ignored, sigb64_or_ignored, user_data_pubkey ) + if 'error' in res: + log.debug("Failed to verify profile by owner hash") + return {'error': 'Failed to validate profile: invalid or missing timestamp and/or previous hash'} + + else: + return data_info res = storage_enqueue_profile( name, str(profile_txt) ) if not res: diff --git a/blockstack/lib/storage/crawl.py b/blockstack/lib/storage/crawl.py index faf15ac63..4b92a3867 100644 --- a/blockstack/lib/storage/crawl.py +++ b/blockstack/lib/storage/crawl.py @@ -326,7 +326,7 @@ def store_mutable_data_to_storage( blockchain_id, data_id, data_txt, profile=Fal nocollide_data_id = '{}-{}'.format(blockchain_id, data_id) log.debug("Store {} to drivers '{}', skipping '{}'".format('profile' if profile else 'mutable datum', ','.join(required if required is not None else []), ','.join(skip if skip is not None else []))) - res = blockstack_client.storage.put_mutable_data(nocollide_data_id, data_txt, raw=True, required=required, skip=skip, fqu=blockchain_id) + res = blockstack_client.storage.put_mutable_data(nocollide_data_id, data_txt, sign=False, required=required, skip=skip, blockchain_id=blockchain_id) return res @@ -334,11 +334,10 @@ def load_mutable_data_from_storage( blockchain_id, data_id, drivers=None ): """ Load mutable data from storage. Used by the storage gateway logic. - Return the data on success - Return None on error + Return """ nocollide_data_id = '{}-{}'.format(blockchain_id, data_id) - res = blockstack_client.storage.get_mutable_data(nocollide_data_id, None, drivers=drivers, decode=False, fqu=blockchain_id) + res = blockstack_client.storage.get_mutable_data(nocollide_data_id, None, blockchain_id=blockchain_id, drivers=drivers, decode=False) return res diff --git a/blockstack/version.py b/blockstack/version.py index 4f3f671cd..03e020f83 100644 --- a/blockstack/version.py +++ b/blockstack/version.py @@ -1,5 +1,5 @@ # this is the only place where version should be updated __version_major__ = '0' __version_minor__ = '14' -__version_patch__ = '3' +__version_patch__ = '2' __version__ = '{}.{}.{}.0'.format(__version_major__, __version_minor__, __version_patch__) diff --git a/blockstack_client/__init__.py b/blockstack_client/__init__.py index 48a1a9c49..49c5ac29c 100644 --- a/blockstack_client/__init__.py +++ b/blockstack_client/__init__.py @@ -31,7 +31,6 @@ import user import snv import rpc import storage -import token_file import backend import zonefile @@ -51,7 +50,7 @@ from data import get_immutable, get_immutable_by_name, get_mutable, put_immutabl from data import set_data_pubkey from storage import get_announcement, put_announcement, verify_zonefile -from profile import get_profile +from profile import get_profile, put_profile, delete_profile from logger import get_logger from config import get_config, get_utxo_provider_client, get_tx_broadcaster, default_bitcoind_opts diff --git a/blockstack_client/actions.py b/blockstack_client/actions.py index 796ddb28b..e6e3efd68 100644 --- a/blockstack_client/actions.py +++ b/blockstack_client/actions.py @@ -82,16 +82,15 @@ from blockstack_client import ( ) from blockstack_client import subdomains - -from blockstack_client.profile import get_profile, \ - profile_list_accounts, profile_get_account, \ +from blockstack_client.profile import put_profile, delete_profile, get_profile, \ + profile_add_device_id, profile_remove_device_id, profile_list_accounts, profile_get_account, \ profile_put_account, profile_delete_account from rpc import local_api_connect, local_api_status import rpc as local_rpc import config -from .config import configure_zonefile, configure, get_utxo_provider_client, get_tx_broadcaster, get_local_device_id +from .config import configure_zonefile, configure, get_utxo_provider_client, get_tx_broadcaster from .constants import ( CONFIG_PATH, CONFIG_DIR, FIRST_BLOCK_MAINNET, NAME_UPDATE, @@ -125,7 +124,7 @@ from .scripts import UTXOException, is_name_valid, is_valid_hash, is_namespace_v from .user import make_empty_user_profile, user_zonefile_data_pubkey from .tx import serialize_tx, sign_tx -from .zonefile import make_empty_zonefile, url_to_uri_record, lookup_name_zonefile_pubkey +from .zonefile import make_empty_zonefile, url_to_uri_record from .utils import exit_with_error, satoshis_to_btc, ScatterGather from .app import app_publish, app_get_config, app_get_resource, \ @@ -139,10 +138,6 @@ from .data import datastore_mkdir, datastore_rmdir, make_datastore_info, put_dat from .schemas import OP_URLENCODED_PATTERN, OP_NAME_PATTERN, OP_USER_ID_PATTERN, OP_BASE58CHECK_PATTERN -from .token_file import token_file_profile_serialize, token_file_update_profile, token_file_get, token_file_put, token_file_delete, \ - lookup_name_privkey, lookup_signing_privkey, lookup_signing_pubkeys, token_file_get_key_order, lookup_delegated_device_pubkeys, \ - token_file_create, lookup_app_pubkeys, token_file_get_application_name, token_file_update_apps - import virtualchain from virtualchain.lib.ecdsalib import * @@ -932,14 +927,11 @@ def cli_lookup(args, config_path=CONFIG_PATH): command: lookup help: Get the zone file and profile for a particular name arg: name (str) 'The name to look up' - opt: full (str) 'If True, then get the whole token file' """ data = {} blockchain_record = None fqu = str(args.name) - full = str(args.full) if hasattr(args, "full") and args.full and len(args.full) > 0 else None - full = True if full else False error = check_valid_name(fqu) if error: @@ -964,44 +956,20 @@ def cli_lookup(args, config_path=CONFIG_PATH): msg = 'Name is revoked. Use get_name_blockchain_record for details.' return {'error': msg} - if not full: - # just getting the profile - try: - res = get_profile(str(args.name), name_record=blockchain_record) - if 'error' in res: - return res + try: + res = get_profile( + str(args.name), name_record=blockchain_record, include_raw_zonefile=True, use_legacy=True, use_legacy_zonefile=True + ) - if res['profile'] is not None: - data['profile'] = res['profile'] - else: - data['profile'] = res['legacy_profile'] + if 'error' in res: + return res - if data['profile'] is None: - return {'error': 'Failed to load a profile for this name. Try again with --debug to diagnose.'} - - data['zonefile'] = res['raw_zonefile'] - except Exception as e: - log.exception(e) - msg = 'Failed to look up name\n{}' - return {'error': msg.format(traceback.format_exc())} - - else: - # getting the token file - try: - res = token_file_get(str(args.name), name_record=blockchain_record) - if 'error' in res: - return res - - if not res['token_file']: - return {'error': 'Name {} does not have a token file'.format(args.name)} - - del res['token_file']['jwts'] - data['zonefile'] = res['raw_zonefile'] - data['token_file'] = res['token_file'] - - except Exception as e: - log.exception(e) - return {'error': 'Failed to look up name\n{}'.format(traceback.format_exc())} + data['profile'] = res['profile'] + data['zonefile'] = res['raw_zonefile'] + except Exception as e: + log.exception(e) + msg = 'Failed to look up name\n{}' + return {'error': msg.format(traceback.format_exc())} result = data analytics_event('Name lookup', {}) @@ -1218,7 +1186,9 @@ def analyze_zonefile_string(fqu, zonefile_data, force_data=False, check_current= if zonefile_data is None: # fetch remotely - zonefile_data_res = get_name_zonefile(fqu, proxy=proxy) + zonefile_data_res = get_name_zonefile( + fqu, proxy=proxy, raw_zonefile=True + ) if 'error' not in zonefile_data_res: zonefile_data = zonefile_data_res['zonefile'] else: @@ -1226,7 +1196,7 @@ def analyze_zonefile_string(fqu, zonefile_data, force_data=False, check_current= # zone file is not given; we had to fetch it ret['downloaded'] = True - ret['raw_zonefile'] = zonefile_data_res['raw_zonefile'] + ret['raw_zonefile'] = zonefile_data ret['is_path'] = False ret['is_string'] = False @@ -1275,7 +1245,7 @@ def analyze_zonefile_string(fqu, zonefile_data, force_data=False, check_current= return ret -def cli_register(args, config_path=CONFIG_PATH, force_data=False, wallet_keys=None, +def cli_register(args, config_path=CONFIG_PATH, force_data=False, cost_satoshis=None, interactive=True, password=None, proxy=None, make_profile = None): """ @@ -1332,8 +1302,6 @@ def cli_register(args, config_path=CONFIG_PATH, force_data=False, wallet_keys=No return {'error': 'Not a valid address'} user_profile = None - new_token_file = None - if user_zonefile: zonefile_info = analyze_zonefile_string(fqu, user_zonefile, force_data=force_data, proxy=proxy) if 'error' in zonefile_info: @@ -1351,7 +1319,11 @@ def cli_register(args, config_path=CONFIG_PATH, force_data=False, wallet_keys=No else: # make a default zonefile - user_zonefile_dict = make_empty_zonefile(fqu, None) + _, _, data_pubkey = get_addresses_from_file(config_dir=config_dir) + if not data_pubkey: + return {'error': 'No data key in wallet. Please add one with `setup_wallet`'} + + user_zonefile_dict = make_empty_zonefile(fqu, data_pubkey) user_zonefile = blockstack_zones.make_zone_file(user_zonefile_dict) if (make_profile and transfer_address): @@ -1362,18 +1334,11 @@ def cli_register(args, config_path=CONFIG_PATH, force_data=False, wallet_keys=No make_profile = not transfer_address if make_profile: - # registering for this wallet. Put an empty token file, signed with this wallet's signing keys - if not wallet_keys: - wallet_keys = get_wallet_keys(config_path, password) - if 'error' in wallet_keys: - return wallet_keys - + # let's put an empty profile + _, _, data_pubkey = get_addresses_from_file(config_dir=config_dir) + if not data_pubkey: + return {'error': 'No data key in wallet. Please add one with `setup_wallet`'} user_profile = make_empty_user_profile() - res = migrate_profile_to_token_file(fqu, user_profile, get_owner_privkey_info(wallet_keys), config_path=config_path) - if 'error' in res: - return {'error': 'Failed to create token file: {}'.format(res['error'])} - - new_token_file = res['token_file'] # operation checks (API server only) if local_rpc.is_api_server(config_dir=config_dir): @@ -1448,12 +1413,14 @@ def cli_register(args, config_path=CONFIG_PATH, force_data=False, wallet_keys=No exit(0) # forward along to RESTful server (or if we're the RESTful server, call the registrar method) - log.debug("Preorder {}, zonefile={}, token_file={}, recipient={} min_confs={}".format(fqu, user_zonefile, new_token_file, transfer_address, min_payment_confs)) + log.debug("Preorder {}, zonefile={}, profile={}, recipient={} min_confs={}".format(fqu, user_zonefile, user_profile, transfer_address, min_payment_confs)) rpc = local_api_connect(config_path=config_path) assert rpc try: - resp = rpc.backend_preorder(fqu, cost_satoshis, user_zonefile, new_token_file, transfer_address, min_payment_confs, unsafe_reg=unsafe_reg) + resp = rpc.backend_preorder(fqu, cost_satoshis, user_zonefile, user_profile, + transfer_address, min_payment_confs, + unsafe_reg = unsafe_reg) except Exception as e: log.exception(e) return {'error': 'Error talking to server, try again.'} @@ -1858,47 +1825,13 @@ def cli_revoke(args, config_path=CONFIG_PATH, interactive=True, password=None, p return resp -def migrate_profile_to_token_file(name, profile, owner_privkey_info, device_id=None, config_path=CONFIG_PATH): - """ - Given a user's existing profile and the wallet keys, - make a serialized token file - - Return {'status': True, 'token_file': token file} on success - Return {'error': ...} on error - """ - config_dir = os.path.dirname(config_path) - - if device_id is None: - device_id = get_local_device_id(config_dir) - - name_owner_privkeys = None - - if virtualchain.is_singlesig(owner_privkey_info): - name_owner_privkeys = { - device_id: str(owner_privkey_info) - } - - else: - # select the first private key as belonging to this device - name_owner_privkeys = { - device_id: str(owner_privkey_info['privkeys'][0]) - } - - res = token_file_create(name, name_owner_privkeys, device_id, profile=profile, config_path=config_path) - if 'error' in res: - return {'error': 'Failed to create token file: {}'.format(res['error'])} - - return {'status': True, 'token_file': res['token_file']} - - -def cli_migrate(args, config_path=CONFIG_PATH, password=None, wallet_keys=None, +def cli_migrate(args, config_path=CONFIG_PATH, password=None, proxy=None, interactive=True, force=False): """ command: migrate - help: Migrate a legacy profile to the latest zonefile and profile format + help: Migrate a legacy blockchain-linked profile to the latest zonefile and profile format arg: name (str) 'The blockchain ID with the profile to migrate' opt: force (str) 'Reset the zone file no matter what.' - opt: device_id (str) 'If given, use this as the device ID.' """ config_dir = os.path.dirname(config_path) @@ -1913,157 +1846,112 @@ def cli_migrate(args, config_path=CONFIG_PATH, password=None, wallet_keys=None, if 'error' in res: return res - if wallet_keys is None: - wallet_keys = get_wallet_keys(config_path, password) - if 'error' in wallet_keys: - return wallet_keys - if proxy is None: proxy = get_default_proxy(config_path=config_path) fqu = str(args.name) - if hasattr(args, 'force') and args.force is not None: - force = args.force.lower() in ['1', 'yes', 'force', 'true'] + force = (force or (getattr(args, 'force', '').lower() in ['1', 'yes', 'force', 'true'])) error = check_valid_name(fqu) if error: return {'error': error} - res = get_name_zonefile(fqu, proxy=proxy) - if 'error' in res: - log.error("Failed to get zone file for {}: {}".format(fqu, res['error'])) - return {'error': "Failed to get zone file for {}".format(fqu)} + # need data public key + _, _, data_pubkey = get_addresses_from_file(config_dir=config_dir) + if data_pubkey is None: + return {'error': 'No data key in wallet'} + + res = get_name_zonefile( + fqu, proxy=proxy, + raw_zonefile=True, include_name_record=True + ) user_zonefile = None user_profile = None - new_token_file = None - name_rec = res['name_record'] - existing_profile = res['profile'] - existing_zonefile = res['zonefile'] - user_zonefile_txt = res['raw_zonefile'] - user_zonefile_hash = get_zonefile_data_hash(user_zonefile_txt) - user_zonefile = None - legacy = False - is_token_file = True - nonstandard = False - need_name_update = False + if 'error' not in res: + name_rec = res['name_record'] + user_zonefile_txt = res['zonefile'] + user_zonefile_hash = get_zonefile_data_hash(user_zonefile_txt) + user_zonefile = None + legacy = False + nonstandard = False - # check the profile. is it a legacy profile, a raw profile, or a token file? - res = get_profile(name, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query current profile or token file'} + # TODO: handle zone files that do not have data keys. + # try to parse + try: + user_zonefile = blockstack_zones.parse_zone_file(user_zonefile_txt) + legacy = blockstack_profiles.is_profile_in_legacy_format(user_zonefile) + except: + log.warning('Non-standard zonefile {}'.format(user_zonefile_hash)) + nonstandard = True - if res['legacy']: - log.warn("Zone file is a legacy profile; migration needed") - legacy = True + if nonstandard: + if force: + # forcibly reset the zone file + user_profile = make_empty_user_profile() + user_zonefile = make_empty_zonefile(fqu, data_pubkey) - if res['token_file'] is None: - log.warn("Got raw profile; token file migration needed") - is_token_file = False - - if res['nonstandard_zonefile']: - log.warn("Zone file is non-standard; migration needed") - nonstandard = True + else: + if os.environ.get("BLOCKSTACK_CLIENT_INTERACTIVE_YES", None) != "1": + # prompt + msg = ( + '' + 'WARNING! Non-standard zone file detected.' + 'If you proceed, your zone file will be reset.' + '' + 'Proceed? (y/N): ' + ) - if nonstandard: - if force: - # forcibly reset the zone file - user_profile = make_empty_user_profile() - user_zonefile = make_empty_zonefile(fqu, None) - need_name_update = True + proceed_str = raw_input(msg) + proceed = proceed_str.lower() in ['y'] + if not proceed: + return {'error': 'Non-standard zonefile'} - else: - if os.environ.get("BLOCKSTACK_CLIENT_INTERACTIVE_YES", None) != "1": - # prompt - msg = ( - '' - 'WARNING! Non-standard zone file detected.' - 'If you proceed, your zone file will be reset.' - '' - 'Proceed? (y/N): ' - ) - - proceed_str = raw_input(msg) - proceed = proceed_str.lower() in ['y'] - if not proceed: + else: + user_profile = make_empty_user_profile() + user_zonefile = make_empty_zonefile(fqu, data_pubkey) + else: return {'error': 'Non-standard zonefile'} - else: - user_profile = make_empty_user_profile() - user_zonefile = make_empty_zonefile(fqu, None) - need_name_update = True - else: - return {'error': 'Non-standard zonefile'} - - # going ahead with zonefile and profile reset - - else: - if not legacy: - # standard zone file - # raw profile? - if not is_token_file: - log.debug("Migrating raw profile to token file") - user_profile = existing_profile - user_zonefile = existing_zonefile - - else: - log.debug("Zone file, profile, and token file are in the latest format.") - return {'status': True} + # going ahead with zonefile and profile reset else: - # legacy zone file that encodes a profile + # standard or legacy zone file + if not legacy: + msg = 'Zone file is in the latest format. No migration needed' + return {'error': msg} + # convert user_profile = blockstack_profiles.get_person_from_legacy_format(user_zonefile) user_zonefile = make_empty_zonefile(fqu, data_pubkey) - need_name_update = True - - res = migrate_profile_to_token_file(name, profile, get_owner_privkey_info(wallet_keys), device_id=device_id, config_path=config_path) - if 'error' in res: - return {'error': res['error']} - - new_token_file = res['token_file'] - resp = {} - - if need_name_update: - # need to update the zone file - zonefile_txt = blockstack_zones.make_zone_file(user_zonefile) - zonefile_hash = get_zonefile_data_hash(zonefile_txt) - - rpc = local_api_connect(config_path=config_path) - assert rpc - - try: - resp = rpc.backend_update(fqu, zonefile_txt, new_token_file, None) - except Exception as e: - log.exception(e) - return {'error': 'Error talking to API endpoint, try again.'} - - if 'error' in resp: - log.debug('RPC error: {}'.format(resp['error'])) - return resp - - if (not 'success' in resp or not resp['success']) and 'message' in resp: - return {'error': resp['message']} - - analytics_event('Migrate name', {}) - resp['zonefile_hash'] = zonefile_hash else: - # find the right signing private key to use - res = find_signing_privkey(name, wallet_keys=wallet_keys, config_path=config_path, password=password, proxy=proxy) - if 'error' in res: - return res + log.error("Failed to get zone file for {}".format(fqu)) + return {'error': res['error']} - signing_privkey = res['signing_privkey'] + zonefile_txt = blockstack_zones.make_zone_file(user_zonefile) + zonefile_hash = get_zonefile_data_hash(zonefile_txt) - # only need to store the token file - res = token_file_put(name, new_token_file, signing_privkey, proxy=proxy, required_drivers=required_storage_drivers, config_path=config_path) - if 'error' in res: - return res - - resp = {'status': True} + rpc = local_api_connect(config_path=config_path) + assert rpc + try: + resp = rpc.backend_update(fqu, zonefile_txt, user_profile, None) + except Exception as e: + log.exception(e) + return {'error': 'Error talking to server, try again.'} + + if 'error' in resp: + log.debug('RPC error: {}'.format(resp['error'])) + return resp + + if (not 'success' in resp or not resp['success']) and 'message' in resp: + return {'error': resp['message']} + + analytics_event('Migrate name', {}) + + resp['zonefile_hash'] = zonefile_hash return resp @@ -2146,7 +2034,7 @@ def cli_setup_wallet(args, config_path=CONFIG_PATH, password=None, interactive=T def cli_get_public_key(args, config_path=CONFIG_PATH, proxy=None): """ command: get_public_key - help: Get the ECDSA signing public key for a blockchain ID + help: Get the ECDSA public key for a blockchain ID arg: name (str) 'The blockchain ID' """ # reply ENODATA if we can't load the zone file @@ -2163,7 +2051,7 @@ def cli_get_public_key(args, config_path=CONFIG_PATH, proxy=None): return {'error': 'zone file for {} has no public key'.format(fqu), 'errno': errno.EINVAL} zfpubkey = keylib.key_formatting.decompress(user_zonefile_data_pubkey(zfinfo['zonefile'])) - return {'status': True, 'public_key': zfpubkey} + return {'public_key': zfpubkey} def cli_list_accounts( args, proxy=None, config_path=CONFIG_PATH ): @@ -2215,7 +2103,6 @@ def cli_put_account( args, proxy=None, config_path=CONFIG_PATH, password=None, w arg: service (str) 'The service this account is for.' arg: identifier (str) 'The name of the account.' arg: content_url (str) 'The URL that points to external contact data.' - opt: signing_privkey (str) 'The device-specific signing private key for this name.' opt: extra_data (str) 'A comma-separated list of "name1=value1,name2=value2,name3=value3..." with any extra account information you need in the account.' """ password = get_default_password(password) @@ -2231,8 +2118,7 @@ def cli_put_account( args, proxy=None, config_path=CONFIG_PATH, password=None, w service = str(args.service) identifier = str(args.identifier) content_url = str(args.content_url) - signing_privkey = None - + if not is_name_valid(args.name): return {'error': 'Invalid name'} @@ -2256,15 +2142,8 @@ def cli_put_account( args, proxy=None, config_path=CONFIG_PATH, password=None, w v = "=".join(parts[1:]) extra_data[k] = v - - # find the right signing private key to use - res = find_signing_privkey(name, getattr(args, 'signing_privkey', None), wallet_keys=wallet_keys, config_path=config_path, password=password, proxy=proxy) - if 'error' in res: - return res - signing_privkey = res['signing_privkey'] - - return profile_put_account(name, service, identifier, content_url, extra_data, signing_privkey, config_path=config_path, proxy=proxy) + return profile_put_account(name, service, identifier, content_url, extra_data, wallet_keys, config_path=config_path, proxy=proxy) def cli_delete_account( args, proxy=None, config_path=CONFIG_PATH, password=None, wallet_keys=None ): @@ -2274,7 +2153,6 @@ def cli_delete_account( args, proxy=None, config_path=CONFIG_PATH, password=None arg: name (str) 'The name to query.' arg: service (str) 'The service the account is for.' arg: identifier (str) 'The identifier of the account to delete.' - opt: signing_privkey (str) 'The device-specific signing private key for this name.' """ password = get_default_password(password) proxy = get_default_proxy(config_path=config_path) if proxy is None else proxy @@ -2288,22 +2166,14 @@ def cli_delete_account( args, proxy=None, config_path=CONFIG_PATH, password=None name = str(args.name) service = str(args.service) identifier = str(args.identifier) - signing_privkey = None - - # find the right signing private key to use - res = find_signing_privkey(name, getattr(args, 'signing_privkey', None), wallet_keys=wallet_keys, config_path=config_path, password=password, proxy=proxy) - if 'error' in res: - return res - signing_privkey = res['signing_privkey'] - - if not is_name_valid(name): + if not is_name_valid(args.name): return {'error': 'Invalid name'} - if len(service) == 0 or len(identifier) == 0: + if len(args.service) == 0 or len(args.identifier) == 0: return {'error': 'Invalid data'} - return profile_delete_account(name, service, identifier, signing_privkey, config_path=config_path, proxy=proxy) + return profile_delete_account(name, service, identifier, wallet_keys, config_path=config_path, proxy=proxy) def cli_import_wallet(args, config_path=CONFIG_PATH, password=None, force=False): @@ -3426,6 +3296,9 @@ def cli_put_mutable(args, config_path=CONFIG_PATH, password=None, proxy=None, st result = put_mutable(mutable_data_info['fq_data_id'], mutable_data_payload, pubkey, sig, mutable_data_info['version'], \ blockchain_id=fqu, config_path=config_path, proxy=proxy, storage_drivers=storage_drivers, storage_drivers_exclusive=storage_drivers_exclusive) + if 'error' in result: + return result + return result @@ -3456,7 +3329,10 @@ def cli_put_immutable(args, config_path=CONFIG_PATH, password=None, proxy=None): proxy = get_default_proxy() if proxy is None else proxy - result = put_immutable(fqu, str(args.data_id), data, wallet_keys=wallet_keys, proxy=proxy) + result = put_immutable( + fqu, str(args.data_id), data, + wallet_keys=wallet_keys, proxy=proxy + ) if 'error' in result: return result @@ -3476,12 +3352,6 @@ def cli_get_mutable(args, config_path=CONFIG_PATH, proxy=None): opt: device_ids (str) 'A CSV of devices to query' """ data_pubkey = str(args.data_pubkey) if hasattr(args, 'data_pubkey') and getattr(args, 'data_pubkey') is not None else None - name = str(args.name) - proxy = get_default_proxy() if proxy is None else proxy - - pubkeys = [] - addresses = [] - device_pubkeys = None # get the list of device IDs to use device_ids = getattr(args, 'device_ids', None) @@ -3489,16 +3359,9 @@ def cli_get_mutable(args, config_path=CONFIG_PATH, proxy=None): device_ids = device_ids.split(',') else: - res = find_signing_pubkeys_and_address(name, data_pubkey, proxy=proxy) - if 'error' in res: - return res + raise NotImplemented("Missing token file parsing logic") - pubkeys = res['pubkeys'] - addresses = [res['address']] - device_pubkeys = res['device_pubkeys'] - device_ids = device_pubkeys.keys() - - result = get_mutable(str(args.data_id), device_ids, proxy=proxy, config_path=config_path, blockchain_id=str(args.name), device_data_pubkeys=device_pubkeys, data_pubkeys=pubkeys, data_pubkey_hashes=addresses) + result = get_mutable(str(args.data_id), device_ids['device_ids'], proxy=proxy, config_path=config_path, blockchain_id=str(args.name), data_pubkey=data_pubkey) if 'error' in result: return result @@ -3526,7 +3389,6 @@ def cli_get_immutable(args, config_path=CONFIG_PATH, proxy=None): return result return { - 'status': True, 'data': result['data'], 'hash': result['hash'] } @@ -3724,36 +3586,28 @@ def cli_get_name_zonefile(args, config_path=CONFIG_PATH, raw=True, proxy=None): arg: name (str) 'The name to query' opt: json (str) 'If true is given, try to parse as JSON' """ - if proxy is None: - proxy = get_default_proxy() - # the 'raw' kwarg is set by the API daemon to False to get back structured data + parse_json = getattr(args, 'json', 'false') parse_json = parse_json is not None and parse_json.lower() in ['true', '1'] - result = get_name_zonefile(str(args.name), proxy=proxy) + result = get_name_zonefile(str(args.name), raw_zonefile=True) if 'error' in result: log.error("get_name_zonefile failed: %s" % result['error']) return result - if 'raw_zonefile' not in result: + if 'zonefile' not in result: return {'error': 'No zonefile data'} if parse_json: # try to parse try: - new_zonefile = decode_name_zonefile(name, result['raw_zonefile']) + new_zonefile = decode_name_zonefile(name, result['zonefile']) assert new_zonefile is not None - result = {'status': True, 'zonefile': new_zonefile} + result['zonefile'] = new_zonefile except: result['warning'] = 'Non-standard zonefile' - else: - result = { - 'status': True, - 'zonefile': result['raw_zonefile'] - } - if raw: return result['zonefile'] @@ -3799,6 +3653,7 @@ def cli_get_all_names(args, config_path=CONFIG_PATH): count = 100 result = get_all_names(offset=offset, count=count) + return result @@ -3823,6 +3678,7 @@ def cli_get_names_in_namespace(args, config_path=CONFIG_PATH): count = 100 result = get_names_in_namespace(str(args.namespace_id), offset, count) + return result @@ -3906,183 +3762,14 @@ def cli_unqueue(args, config_path=CONFIG_PATH): return {'status': True} -def find_signing_privkey(name, args_signing_privkey, wallet_keys=None, config_path=CONFIG_PATH, password=None, proxy=None, parsed_token_file=None): - """ - Find the signing private key to use--either the one given in args_signing_privkey, - or the one from our wallet that corresponds to this device's signing public key. - - Returns {'status': True, 'signing_privkey': privkey} on success - Returns {'error': ...} on error - """ - if args_signing_privkey: - try: - signing_privkey = ECPrivateKey(str(args_privkey)).to_hex() - except: - return {'error': 'Failed to parse private key'} - - else: - if wallet_keys is None: - wallet_keys = get_wallet_keys(config_path, password) - if 'error' in wallet_keys: - return wallet_keys - - res = lookup_signing_privkey(name, get_owner_privkey_info(wallet_keys), proxy=proxy, parsed_token_file=parsed_token_file) - if 'error' in res: - log.error("Failed to load signing key for {}: {}".format(name, res['error'])) - return {'error': 'Failed to look up signing key from wallet. Try passing it explicitly as an argument.'} - - signing_privkey = res['signing_privkey'] - return {'status': True, 'signing_privkey': signing_privkey} - - -def find_signing_pubkeys_and_address(name, args_signing_pubkey, proxy=None): - """ - Find the set of public keys and possibly address to use for a given name. - Use either the one given in the args (args_signing_pubkey), or look it up from the token file, zone file, and name record. - - Return { - 'status': True, - 'pubkeys': [public keys, including zone file pubkey], - 'device_pubkeys': {'$device_id': '$device_signing_pubkey'}, - 'address': owner address} on success - Return {'error': ...} on failure - """ - - pubkeys = [] - - if args_signing_pubkey: - pubkey = str(args_signing_pubkey) - try: - pubkey = ECPublicKey(pubkey).to_hex() - except: - return {'error': 'Invalid public key'} - - pubkeys = [pubkey] - - return {'status': True, 'pubkeys': pubkeys, 'address': None} - - else: - res = lookup_signing_pubkeys(name, proxy=proxy) - if 'error' in res: - log.error("Failed to look up signing keys for {}: {}".format(name, res['error'])) - return {'error': 'Failed to look up public keys for {}'.format(name)} - - device_pubkeys = res['pubkeys'] - pubkeys = res['pubkeys'].values() - - # also grab the zone file public key, if present - res = lookup_name_zonefile_pubkey(name, proxy=proxy) - if 'error' in res: - log.error("Failed to look up zone file public key for {}: {}".format(name, res['error'])) - return {'error': 'Failed to look up zone file public key for {}'.format(name)} - - zonefile_pubkey = res['pubkey'] - if zonefile_pubkey: - pubkeys.append(zonefile_pubkey) - - owner_address = res['name_record']['address'] - - return {'status': True, 'pubkeys': pubkeys, 'device_pubkeys': device_pubkeys, 'address': owner_address} - - -def find_datastore_device_pubkeys(blockchain_id, args_device_ids, args_device_pubkeys, full_application_name=None, datastore_id=None, proxy=None): - """ - Find the list of (device ID, app-specific signing key) pairs for a given name. - Use either the CSV string given in args (args_device_dis), or look it up from the token file. - - Return {'status': True, 'token_file': ..., 'device_ids': [{'device_id': ..., 'public_key': ...}...[} on success - Return {'errro': ...} on error - """ - pubkeys = None - device_ids = None - parsed_token_file = None - - assert full_application_name or datastore_id, "Need either full application name or datastore ID" - - # get the list of device IDs to use - if args_device_ids and args_device_pubkeys: - device_ids = args_device_ids.split(',') - device_pubkeys = args_device_pubkeys.split(',') - assert len(device_ids) == len(device_pubkeys) - - pubkeys = [] - for i in xrange(0, len(device_ids)): - entry = { - 'device_id': device_ids[i], - 'public_key': device_pubkeys[i] - } - pubkeys.append(entry) - - else: - log.debug("Look up datastore public keys for '{}'".format(blockchain_id)) - - if full_application_name is None: - res = get_token_file(blockchain_id, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to load token file for "{}"'.format(blockchain_id)} - - parsed_token_file = res['token_file'] - - res = token_file_get_application_name(datastore_id) - if 'error' in res: - return res - - full_application_name = res['full_application_name'] - - # find from token file - res = lookup_app_pubkeys(blockchain_id, full_application_name, proxy=proxy, parsed_token_file=parsed_token_file) - if 'error' in res: - return {'error': 'Failed to query application device list for {}: {}'.format(blockchain_id, res['error'])} - - parsed_token_file = res['token_file'] - device_ids = res['pubkeys'].keys() - pubkeys = [] - for dev_id in device_ids: - entry = { - 'device_id': dev_id, - 'public_key': res['pubkeys'][dev_id] - } - - pubkeys.append(entry) - - return {'status': True, 'device_ids': device_ids, 'pubkeys': pubkeys, 'token_file': parsed_token_file} - - -def find_datastore_private_key(blockchain_id, full_application_name, arg_app_privkey, wallet_keys=None, config_path=CONFIG_PATH, password=None, proxy=None): - """ - Find the application private key, using the blockchain ID and full application name to generate one - if the given private key (e.g. from args) is None. - - Return {'status': True, 'privkey': ...} on success - Return {'error': ...} on error - """ - app_privkey = None - if arg_app_privkey is None: - # generate one - if not wallet_keys: - wallet_keys = get_wallet_keys(config_path, password) - if 'error' in wallet_keys: - return wallet_keys - - # derive keys - app_root_privkey = get_app_root_privkey(get_owner_privkey_info(wallet_keys)) - app_privkey = get_app_privkey(app_root_privkey, app_domain) - - else: - app_privkey = str(arg_app_privkey) - - return {'status': True, 'privkey': app_privkey} - - def cli_put_profile(args, config_path=CONFIG_PATH, password=None, proxy=None, force_data=False, wallet_keys=None): """ command: put_profile advanced help: Set the profile for a blockchain ID. arg: blockchain_id (str) 'The blockchain ID.' arg: data (str) 'The profile as a JSON string, or a path to the profile.' - opt: signing_privkey (str) 'The signing key for this device.' """ - proxy = get_default_proxy() if proxy is None else proxy + password = get_default_password(password) config_dir = os.path.dirname(config_path) @@ -4090,14 +3777,8 @@ def cli_put_profile(args, config_path=CONFIG_PATH, password=None, proxy=None, fo name = str(args.blockchain_id) profile_json_str = str(args.data) - # find the right signing private key to use - res = find_signing_privkey(name, getattr(args, 'signing_privkey', None), wallet_keys=wallet_keys, config_path=config_path, password=password, proxy=proxy) - if 'error' in res: - return res + proxy = get_default_proxy() if proxy is None else proxy - signing_privkey = res['signing_privkey'] - - # load up the profile profile = None if not force_data and is_valid_path(profile_json_str) and os.path.exists(profile_json_str): # this is a path. try to load it @@ -4113,25 +3794,31 @@ def cli_put_profile(args, config_path=CONFIG_PATH, password=None, proxy=None, fo except: return {'error': 'Invalid profile JSON'} - # get the current profile or token file. If this is a profile, then tell the user to migrate. - res = get_profile(name, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query current profile or token file'} - - if res['legacy']: - return {'error': 'Profile is in legacy format (version 1). Please use the `migrate` command to migrate your profile to the latest format.'} + if wallet_keys is None: + wallet_keys = get_wallet_keys(config_path, password) + if 'error' in wallet_keys: + return wallet_keys - if res['token_file'] is None: - return {'error': 'Name points to a raw profile (version 2). Please use the `migrate` command to migrate your profile to the latest format.'} - - # got a token file - parsed_token_file = res['token_file'] - res = token_file_update_profile(parsed_token_file, profile, signing_privkey) - if 'error' in res: - return {'error': 'Failed to update token file: {}'.format(res['error'])} + required_storage_drivers = conf.get( + 'storage_drivers_required_write', + config.BLOCKSTACK_REQUIRED_STORAGE_DRIVERS_WRITE + ) + required_storage_drivers = required_storage_drivers.split() + + user_zonefile = get_name_zonefile(name, proxy=proxy) + if 'error' in user_zonefile: + return user_zonefile + + user_zonefile = user_zonefile['zonefile'] + if blockstack_profiles.is_profile_in_legacy_format(user_zonefile): + msg = 'Profile in legacy format. Please migrate it with the "migrate" command first.' + return {'error': msg} + + res = put_profile(name, profile, user_zonefile=user_zonefile, + wallet_keys=wallet_keys, proxy=proxy, + required_drivers=required_storage_drivers, blockchain_id=name, + config_path=config_path) - # save it - res = token_file_put(name, res['token_file'], signing_privkey, proxy=proxy, config_path=config_path) if 'error' in res: return res @@ -4143,45 +3830,19 @@ def cli_delete_profile(args, config_path=CONFIG_PATH, password=None, proxy=None, command: delete_profile advanced help: Delete a profile from a blockchain ID. arg: blockchain_id (str) 'The blockchain ID.' - opt: signing_privkey (str) 'The device-specific signing private key for this name, if different from the data key' """ proxy = get_default_proxy() if proxy is None else proxy password = get_default_password(password) - signing_privkey = None name = str(args.blockchain_id) - # find the right signing private key to use - res = find_signing_privkey(name, getattr(args, 'signing_privkey', None), wallet_keys=wallet_keys, config_path=config_path, password=password, proxy=proxy) - if 'error' in res: - return res - - signing_privkey = res['signing_privkey'] - - # get the current profile or token file. If this is a profile, then tell the user to migrate. - res = get_profile(name, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query current profile or token file'} - - if res['legacy']: - return {'error': 'Profile is in legacy format (version 1). Please use the `migrate` command to migrate your profile to the latest format.'} - - if res['token_file'] is None: - return {'error': 'Name points to a raw profile (version 2). Please use the `migrate` command to migrate your profile to the latest format.'} - - # got a token file. put an empty profile - empty_profile = make_empty_user_profile(config_path=config_path) - parsed_token_file = res['token_file'] - res = token_file_update_profile(parsed_token_file, empty_profile, signing_privkey) - if 'error' in res: - return {'error': 'Failed to update token file: {}'.format(res['error'])} - - # save it - res = token_file_put(name, res['token_file'], signing_privkey, proxy=proxy, config_path=config_path) - if 'error' in res: - return res + if wallet_keys is None: + wallet_keys = get_wallet_keys(config_path, password) + if 'error' in wallet_keys: + return wallet_keys + res = delete_profile(name, user_data_privkey=wallet_keys['data_privkey'], proxy=proxy, wallet_keys=wallet_keys) return res @@ -4251,13 +3912,13 @@ def cli_sync_zonefile(args, config_path=CONFIG_PATH, proxy=None, interactive=Tru if user_data is None: # not in queue. Maybe it's available from one of the storage drivers? log.debug('no pending updates for "{}"; try storage'.format(name)) - user_data = get_name_zonefile(name, proxy=proxy) + user_data = get_name_zonefile( name, raw_zonefile=True ) if 'error' in user_data: msg = 'Failed to get zonefile: {}' log.error(msg.format(user_data['error'])) return user_data - user_data = user_data['raw_zonefile'] + user_data = user_data['zonefile'] # have user data zonefile_hash = storage.get_zonefile_data_hash(user_data) @@ -4344,6 +4005,13 @@ def cli_convert_legacy_profile(args, config_path=CONFIG_PATH): return profile +def get_app_name(appname): + """ + Get the application name, or if not given, the default name + """ + return appname if appname is not None else '_default' + + def cli_app_publish( args, config_path=CONFIG_PATH, interactive=False, password=None, proxy=None ): """ command: app_publish advanced @@ -4398,13 +4066,8 @@ def cli_app_publish( args, config_path=CONFIG_PATH, interactive=False, password= wallet_keys = get_wallet_keys(config_path, password) if 'error' in wallet_keys: return wallet_keys - - res = lookup_signing_privkey(name, get_owner_privkey_info(wallet_keys), proxy=proxy) - if 'error' in res: - return res - signing_privkey = res['signing_privkey'] - res = app_publish( blockchain_id, app_domain, methods, uris, index_file_data, signing_privkey, app_driver_hints=drivers, proxy=proxy, config_path=config_path ) + res = app_publish( blockchain_id, app_domain, methods, uris, index_file_data, app_driver_hints=drivers, wallet_keys=wallet_keys, proxy=proxy, config_path=config_path ) if 'error' in res: return res @@ -4485,13 +4148,8 @@ def cli_app_put_resource( args, config_path=CONFIG_PATH, interactive=False, prox wallet_keys = get_wallet_keys(config_path, password) if 'error' in wallet_keys: return wallet_keys - - res = lookup_signing_privkey(name, get_owner_privkey_info(wallet_keys), proxy=proxy) - if 'error' in res: - return res - - signing_privkey = res['signing_privkey'] - res = app_put_resource( blockchain_id, app_domain, res_path, resdata, signing_privkey, proxy=proxy, config_path=config_path ) + + res = app_put_resource( blockchain_id, app_domain, res_path, resdata, proxy=proxy, wallet_keys=wallet_keys, config_path=config_path ) return res @@ -4517,135 +4175,42 @@ def cli_app_delete_resource( args, config_path=CONFIG_PATH, interactive=False, p wallet_keys = get_wallet_keys(config_path, password) if 'error' in wallet_keys: return wallet_keys - - res = lookup_signing_privkey(name, get_owner_privkey_info(wallet_keys), proxy=proxy) - if 'error' in res: - return res - - signing_privkey = res['signing_privkey'] - res = app_delete_resource( blockchain_id, app_domain, res_path, signing_privkey, proxy=proxy, config_path=config_path ) + + res = app_delete_resource( blockchain_id, app_domain, res_path, proxy=proxy, wallet_keys=wallet_keys, config_path=config_path ) return res -def cli_app_signup(args, config_path=CONFIG_PATH, interactive=True, proxy=None, password=None, wallet_keys=None): - """ - command: app_signup advanced - help: Sign up for an application, allowing you to use the app from this device. - arg: blockchain_id (str) 'The blockchain ID to use to sign up' - arg: privkey (str) 'The app-specific private key to use' - arg: app_domain (str) 'The fully-qualified application name, ending in .1 or .x' - arg: api_methods (str) 'A CSV of requested methods to allow' - opt: this_device_id (str) 'The device ID to sign up as' - opt: device_ids (str) 'A CSV of device IDs that can write to the app datastore' - opt: public_keys (str) 'A CSV of public keys that can write to the app datastore' - opt: signing_privkey (str) 'The signing private key to use' - """ - - proxy = get_default_proxy() if proxy is None else proxy - - blockchain_id = str(args.blockchain_id) - app_domain = str(args.app_domain) - api_methods = str(args.api_methods) - api_methods = api_methods.split(',') - password = get_default_password(password) - - this_device_id = None - - if getattr(args, 'this_device_id', None) is not None: - this_device_id = str(args.this_device_id) - - else: - this_device_id = config.get_local_device_id(config_dir=os.path.dirname(config_path)) - - res = find_datastore_device_pubkeys(blockchain_id, getattr(args, 'device_ids', None), getattr(args, 'public_keys', None), full_application_name=app_domain, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query device list for {}: {}'.format(name, res['error'])} - - data_pubkeys = res['pubkeys'] - parsed_token_file = res['token_file'] - - # have we signed up for this app yet? - if this_device_id in [dpk['device_id'] for dpk in data_pubkeys]: - return {'error': "Device '{}' is already signed up to use application '{}'."} - - signing_privkey = None - app_privkey = None - if getattr(args, 'privkey', None) is None: - # generate one - if not wallet_keys: - wallet_keys = get_wallet_keys(config_path, password) - if 'error' in wallet_keys: - return wallet_keys - - # derive keys - app_root_privkey = get_app_root_privkey(get_owner_privkey_info(wallet_keys)) - app_privkey = get_app_privkey(app_root_privkey, app_domain) - - else: - app_privkey = str(args.privkey) - - res = find_signing_privkey(blockchain_id, getattr(args, 'signing_privkey', None), wallet_keys=wallet_keys, config_path=config_path, password=password, proxy=proxy, parsed_token_file=parsed_token_file) - if 'error' in res: - return res - - signing_privkey = res['signing_privkey'] - app_pubkey = get_pubkey_hex(app_privkey) - - # insert new app key - res = token_file_update_apps(parsed_token_file, this_device_id, app_domain, app_pubkey, signing_privkey) - if 'error' in res: - return res - - # save token file - new_token_file = res['token_file'] - res = token_file_put(blockchain_id, new_token_file, signing_privkey, proxy=proxy, config_path=config_path) - if 'error' in res: - return res - - # success! - return {'status': True, 'app_pubkey': app_pubkey} - - -def cli_app_signin(args, config_path=CONFIG_PATH, interactive=True, proxy=None): +def cli_app_signin(args, config_path=CONFIG_PATH, interactive=True): """ command: app_signin advanced help: Create a session token for the RESTful API for a given application - arg: blockchain_id (str) 'The blockchain ID to use to sign in' + arg: blockchain_id (str) 'The blockchain ID of the caller' arg: privkey (str) 'The app-specific private key to use' - arg: app_domain (str) 'The fully-quallifed application name, ending in .1 or .x' + arg: app_domain (str) 'The application domain' arg: api_methods (str) 'A CSV of requested methods to allow' - opt: this_device_id (str) 'The device ID to sign in as' - opt: device_ids (str) 'A CSV of device IDs that can write to the app datastore' - opt: public_keys (str) 'A CSV of public keys that can write to the app datastore' + arg: device_ids (str) 'A CSV of device IDs that can write to the app datastore' + arg: public_keys (str) 'A CSV of public keys that can write to the app datastore' """ - - proxy = get_default_proxy() if proxy is None else proxy blockchain_id = str(args.blockchain_id) app_domain = str(args.app_domain) api_methods = str(args.api_methods) app_privkey = str(args.privkey) + device_ids = str(args.device_ids) + public_keys = str(args.public_keys) + api_methods = api_methods.split(',') - - this_device_id = None + device_ids = device_ids.split(',') + public_keys = public_keys.split(',') - if getattr(args, 'this_device_id', None) is not None: - this_device_id = str(args.this_device_id) + if len(device_ids) != len(public_keys): + return {'error': 'Mismatch between device IDs and public keys'} - else: - this_device_id = config.get_local_device_id(config_dir=os.path.dirname(config_path)) - - res = find_datastore_device_pubkeys(blockchain_id, getattr(args, 'device_ids', None), getattr(args, 'public_keys', None), full_application_name=app_domain, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query device list for {}: {}'.format(name, res['error'])} - - data_pubkeys = res['pubkeys'] - device_ids = [dpk['device_id'] for dpk in data_pubkeys] - public_keys = [dpk['public_key'] for dpk in data_pubkeys] - - # have we signed up for this app yet? - if this_device_id not in device_ids: - return {'error': "Device '{}' is not authorized to use application '{}'. Sign up for it with the `app_signup` command.".format(this_device_id, app_domain)} + for pubk in public_keys: + try: + keylib.ECPublicKey(pubk) + except: + return {'error': 'Invalid public key {}'.format(pubk)} # get API password api_pass = get_secret("BLOCKSTACK_API_PASSWORD") @@ -4666,7 +4231,9 @@ def cli_app_signin(args, config_path=CONFIG_PATH, interactive=True, proxy=None): # TODO: validate API methods # TODO: fetch api methods from app domain, if need be - + + this_device_id = config.get_local_device_id(config_dir=os.path.dirname(config_path)) + rpc = local_api_connect(config_path=config_path, api_pass=api_pass) sesinfo = rpc.backend_signin(blockchain_id, app_privkey, app_domain, api_methods, device_ids, public_keys, this_device_id) if 'error' in sesinfo: @@ -4675,26 +4242,25 @@ def cli_app_signin(args, config_path=CONFIG_PATH, interactive=True, proxy=None): return {'status': True, 'token': sesinfo['token']} -def cli_sign_profile( args, config_path=CONFIG_PATH, proxy=None, password=None, interactive=False, wallet_keys=None ): +def cli_sign_profile( args, config_path=CONFIG_PATH, proxy=None, password=None, interactive=False ): """ command: sign_profile advanced raw help: Sign a JSON file to be used as a profile. - arg: blockchain_id (str) 'The blockchain ID that will own this profile' arg: path (str) 'The path to the profile data on disk.' - opt: privkey (str) 'The device-specific signing key' + opt: privkey (str) 'The optional private key to sign it with (defaults to the data private key in your wallet)' """ + if proxy is None: proxy = get_default_proxy(config_path=config_path) password = get_default_password(password) config_dir = os.path.dirname(config_path) - name = str(args.blockchain_id) path = str(args.path) data_json = None try: with open(path, 'r') as f: - dat = f.read().strip() + dat = f.read() data_json = json.loads(dat) except Exception as e: if os.environ.get("BLOCKSTACK_DEBUG") == "1": @@ -4703,20 +4269,30 @@ def cli_sign_profile( args, config_path=CONFIG_PATH, proxy=None, password=None, log.error("Failed to load {}".format(path)) return {'error': 'Failed to load {}'.format(path)} - # find the right signing private key to use - res = find_signing_privkey(name, getattr(args, 'privkey', None), wallet_keys=wallet_keys, config_path=config_path, password=password, proxy=proxy) - if 'error' in res: - return res + privkey = None + if hasattr(args, "privkey") and args.privkey: + privkey = str(args.privkey) - privkey = res['signing_privkey'] + else: + wallet_keys = get_wallet_keys( config_path, password ) + if 'error' in wallet_keys: + return wallet_keys + + if not wallet_keys.has_key('data_privkey'): + log.error("No data private key in the wallet. You may need to explicitly select a private key.") + return {'error': 'No data private key set.\nTry passing your owner private key.'} + + privkey = wallet_keys['data_privkey'] privkey = ECPrivateKey(privkey).to_hex() pubkey = get_pubkey_hex(privkey) - res = token_file_profile_serialize(data_json, privkey) - if 'error' in res: - log.error("BUG: failed to serialize profile: {}".format(res['error'])) - return {'error': 'BUG: Failed to serialize profile'} + res = storage.serialize_mutable_data(data_json, privkey, pubkey, profile=True) + if res is None: + return {'error': 'Failed to sign and serialize profile'} + + # sanity check + assert storage.parse_mutable_data(res, pubkey) return res @@ -4727,7 +4303,7 @@ def cli_verify_profile( args, config_path=CONFIG_PATH, proxy=None, interactive=F help: Verify a profile JWT and deserialize it into a profile object. arg: name (str) 'The name that points to the public key to use to verify.' arg: path (str) 'The path to the profile data on disk' - opt: pubkey (str) 'The public key to use to verify.' + opt: pubkey (str) 'The public key to use to verify. Overrides `name`.' """ if proxy is None: @@ -4735,18 +4311,48 @@ def cli_verify_profile( args, config_path=CONFIG_PATH, proxy=None, interactive=F name = str(args.name) path = str(args.path) - pubkeys = [] + pubkey = None owner_address = None if not os.path.exists(path): return {'error': 'No such file or directory'} - res = find_signing_pubkeys_and_address(name, getattr(args, 'pubkey', None), proxy=proxy) - if 'error' in res: - return res + if hasattr(args, 'pubkey') and args.pubkey is not None: + pubkey = str(args.pubkey) + try: + pubkey = ECPublicKey(pubkey).to_hex() + except: + return {'error': 'Invalid public key'} - pubkeys = res['pubkeys'] - owner_address = res['address'] + if pubkey is None: + zonefile_data = None + name_rec = None + # get the pubkey + zonefile_data_res = get_name_zonefile( + name, proxy=proxy, raw_zonefile=True, include_name_record=True + ) + if 'error' not in zonefile_data_res: + zonefile_data = zonefile_data_res['zonefile'] + name_rec = zonefile_data_res['name_record'] + else: + return {'error': "Failed to get zonefile data: {}".format(name)} + + # parse + zonefile_dict = None + try: + zonefile_dict = blockstack_zones.parse_zone_file(zonefile_data) + except: + return {'error': 'Nonstandard zone file'} + + pubkey = user_zonefile_data_pubkey(zonefile_dict) + if pubkey is None: + # fall back to owner hash + owner_address = str(name_rec['address']) + if virtualchain.is_multisig_address(owner_address): + return {'error': 'No data public key in zone file, and owner is a p2sh address'} + + else: + log.warn("Falling back to owner address") profile_data = None try: @@ -4754,33 +4360,28 @@ def cli_verify_profile( args, config_path=CONFIG_PATH, proxy=None, interactive=F profile_data = f.read() except: return {'error': 'Failed to read profile file'} - - for pubkey in pubkeys: - log.debug("Try verifying profile with {} or {}".format(pubkey, owner_address)) - res = storage.parse_mutable_data(profile_data, pubkey, public_key_hash=owner_address) - if res is not None: - log.debug("Success with {} or {}".format(pubkey, owner_address)) - return res - return {'error': 'Failed to verify profile with all available public keys'} + res = storage.parse_mutable_data(profile_data, pubkey, public_key_hash=owner_address) + if res is None: + return {'error': 'Failed to verify profile'} + + return res -def cli_sign_data( args, config_path=CONFIG_PATH, proxy=None, password=None, interactive=False, wallet_keys=None ): +def cli_sign_data( args, config_path=CONFIG_PATH, proxy=None, password=None, interactive=False ): """ command: sign_data advanced raw help: Sign data to be used in a data store. - arg: name (str) 'The name who will sign the data.' arg: path (str) 'The path to the profile data on disk.' - opt: privkey (str) 'The optional private key to sign it with' + opt: privkey (str) 'The optional private key to sign it with (defaults to the data private key in your wallet)' """ if proxy is None: proxy = get_default_proxy(config_path=config_path) password = get_default_password(password) + config_dir = os.path.dirname(config_path) - - name = str(args.name) path = str(args.path) data = None try: @@ -4795,12 +4396,20 @@ def cli_sign_data( args, config_path=CONFIG_PATH, proxy=None, password=None, int log.error("Failed to load {}".format(path)) return {'error': 'Failed to load {}'.format(path)} - # find the right signing private key to use - res = find_signing_privkey(name, getattr(args, 'privkey', None), wallet_keys=wallet_keys, config_path=config_path, password=password, proxy=proxy) - if 'error' in res: - return res + privkey = None + if hasattr(args, "privkey") and args.privkey: + privkey = str(args.privkey) - privkey = res['signing_privkey'] + else: + wallet_keys = get_wallet_keys( config_path, password ) + if 'error' in wallet_keys: + return wallet_keys + + if not wallet_keys.has_key('data_privkey'): + log.error("No data private key in the wallet. You may need to explicitly select a private key.") + return {'error': 'No data private key set.\nTry passing your owner private key.'} + + privkey = wallet_keys['data_privkey'] privkey = ECPrivateKey(privkey).to_hex() pubkey = get_pubkey_hex(privkey) @@ -4810,12 +4419,11 @@ def cli_sign_data( args, config_path=CONFIG_PATH, proxy=None, password=None, int return {'error': 'Failed to sign and serialize data'} # sanity check - parser_res = storage.parse_mutable_data(res, pubkey) - if 'error 'in parser_res: - log.error("Failed to parse back into data: {}".format(parser_res['error'])) - raise AssertionError("BUG: failed to generate valid data") + if BLOCKSTACK_DEBUG: + assert storage.parse_mutable_data(res, pubkey) - log.debug("Verified {} with {}".format(res, pubkey)) + if BLOCKSTACK_TEST: + log.debug("Verified {} with {}".format(res, pubkey)) return res @@ -4833,16 +4441,43 @@ def cli_verify_data( args, config_path=CONFIG_PATH, proxy=None, interactive=True name = str(args.name) path = str(args.path) + pubkey = None if not os.path.exists(path): return {'error': 'No such file or directory'} - res = find_signing_pubkeys_and_address(name, getattr(args, 'pubkey', None), proxy=proxy) - if 'error' in res: - return res + if hasattr(args, 'pubkey') and args.pubkey is not None: + pubkey = str(args.pubkey) + try: + pubkey = keylib.ECPublicKey(pubkey).to_hex() + except Exception as e: + if BLOCKSTACK_DEBUG: + log.exception(e) - pubkeys = res['pubkeys'] - owner_address = res['address'] + return {'error': 'Invalid public key'} + + if pubkey is None: + zonefile_data = None + + # get the pubkey + zonefile_data_res = get_name_zonefile( + name, proxy=proxy, raw_zonefile=True + ) + if 'error' not in zonefile_data_res: + zonefile_data = zonefile_data_res['zonefile'] + else: + return {'error': "Failed to get zonefile data: {}".format(name)} + + # parse + zonefile_dict = None + try: + zonefile_dict = blockstack_zones.parse_zone_file(zonefile_data) + except: + return {'error': 'Nonstandard zone file'} + + pubkey = user_zonefile_data_pubkey(zonefile_dict) + if pubkey is None: + return {'error': 'No data public key in zone file'} data = None try: @@ -4854,14 +4489,11 @@ def cli_verify_data( args, config_path=CONFIG_PATH, proxy=None, interactive=True if BLOCKSTACK_TEST: log.debug("Verify {} with {}".format(data, pubkey)) - for pubkey in pubkeys: - log.debug("Try verify with {}".format(pubkey)) - res = storage.parse_mutable_data(data, pubkey) - if res is not None: - log.debug("Success with {}".format(pubkey)) - return data_blob_parse(res) + res = storage.parse_mutable_data(data, pubkey) + if res is None: + return {'error': 'Failed to verify data'} - return {"error": "Failed to verify data with public keys. Try passing one as an argument."} + return data_blob_parse(res) def cli_validate_zone_file(args, config_path=CONFIG_PATH, proxy=None): @@ -4907,31 +4539,22 @@ def cli_list_devices( args, config_path=CONFIG_PATH, proxy=None ): command: list_devices advanced help: Get the list of device IDs and public keys for a particular application arg: blockchain_id (str) 'The blockchain ID whose devices to list' + arg: appname (str) 'The name of the application' """ - proxy = get_default_proxy() if proxy is None else proxy - - res = lookup_delegated_device_pubkeys(str(args.name), proxy=proxy) - if 'error' in res: - return res - - pubkeys = res['pubkeys'] - device_ids = pubkeys.keys() - - return {'status': True, 'device_ids': device_ids} + raise NotImplemented("Missing token file parsing logic") -def cli_add_device( args, config_path=CONFIG_PATH, proxy=None, wallet_keys=None ): +def cli_add_device( args, config_path=CONFIG_PATH, proxy=None ): """ command: add_device advanced help: Add a device that can read and write your data arg: blockchain_id (str) 'The blockchain ID whose profile to update' - arg: device_id (str) 'The ID of the device to add' - opt: signing_privkey (str) 'The signing private key to use' + opt: device_id (str) 'The ID of the device to add, if not this one' """ - + raise NotImplemented("Missing token file parsing logic") - + def cli_remove_device( args, config_path=CONFIG_PATH, proxy=None ): """ @@ -5085,7 +4708,7 @@ def datastore_file_get(datastore_type, blockchain_id, datastore_id, path, data_p def datastore_file_put(datastore_type, blockchain_id, datastore_privkey, path, data, session, create=False, force_data=False, force=False, config_path=CONFIG_PATH ): """ - Put a file into a datastore or collection. + Put a file int oa datastore or collection. Return {'status': True} on success Return {'error': ...} on failure. @@ -5204,6 +4827,20 @@ def datastore_inode_getinode(datastore_type, blockchain_id, datastore_id, inode_ res = datastore_getinode( rpc, blockchain_id, datastore, inode_uuid, data_pubkeys, extended=False, force=force, idata=idata, config_path=config_path ) return res + + +def cli_get_device_keys( args, config_path=CONFIG_PATH ): + """ + command: get_device_keys advanced + help: Get the device IDs and public keys for a blockchain ID + arg: blockchain_id (str) 'The blockchain Id' + """ + blockchain_id = str(args.blockchain_id) + + # TODO: implement token file support + log.warning("Token file support is NOT IMPLEMENTED") + + # find the "data public key" def cli_get_datastore( args, config_path=CONFIG_PATH ): @@ -5216,17 +4853,14 @@ def cli_get_datastore( args, config_path=CONFIG_PATH ): """ blockchain_id = str(args.blockchain_id) datastore_id = str(args.datastore_id) - - device_ids = None - if getattr(args, 'device_ids', None) is not None: - device_ids = args.device_ids.split(',') + + # get the list of device IDs to use + device_ids = getattr(args, 'device_ids', None) + if device_ids: + device_ids = device_ids.split(',') else: - res = lookup_delegated_device_pubkeys(blockchain_id, proxy=proxy) - if 'error' in res: - return res - - device_ids = res['pubkeys'].keys() + raise NotImplemented("Missing token file parsing logic") return get_datastore_by_type('datastore', blockchain_id, datastore_id, device_ids, config_path=config_path ) @@ -5236,7 +4870,7 @@ def cli_create_datastore( args, config_path=CONFIG_PATH, proxy=None ): command: create_datastore advanced help: Make a new datastore arg: blockchain_id (str) 'The blockchain ID that will own this datastore' - arg: privkey (str) 'The app-specific private key of the datastore' + arg: privkey (str) 'The ECDSA private key of the datastore' arg: session (str) 'The API session token' opt: drivers (str) 'A CSV of drivers to use.' """ @@ -5258,7 +4892,7 @@ def cli_delete_datastore( args, config_path=CONFIG_PATH ): command: delete_datastore advanced help: Delete a datastore owned by a given user, and all of the data it contains. arg: blockchain_id (str) 'The owner of this datastore' - arg: privkey (str) 'The app-specific private key of the datastore' + arg: privkey (str) 'The ECDSA private key of the datastore' arg: session (str) 'The API session token' opt: force (str) 'If True, then delete the datastore even if it cannot be emptied' """ @@ -5284,7 +4918,6 @@ def cli_datastore_mkdir( args, config_path=CONFIG_PATH, interactive=False ): blockchain_id = str(args.blockchain_id) path = str(args.path) - datastore_privkey_hex = str(args.privkey) datastore_pubkey_hex = get_pubkey_hex(datastore_privkey_hex) datastore_id = datastore_get_id(datastore_pubkey_hex) @@ -5389,7 +5022,7 @@ def cli_datastore_rmtree( args, config_path=CONFIG_PATH, interactive=False ): return res -def cli_datastore_getfile( args, config_path=CONFIG_PATH, interactive=False, proxy=None): +def cli_datastore_getfile( args, config_path=CONFIG_PATH, interactive=False ): """ command: datastore_getfile advanced raw help: Get a file from a datastore. @@ -5402,8 +5035,6 @@ def cli_datastore_getfile( args, config_path=CONFIG_PATH, interactive=False, pro opt: device_pubkeys (str) 'If given, a CSV of device public keys owned by the blockchain ID' """ - proxy = get_default_proxy() if proxy is None else proxy - blockchain_id = getattr(args, 'blockchain_id', '') if len(blockchain_id) == 0: blockchain_id = None @@ -5422,11 +5053,25 @@ def cli_datastore_getfile( args, config_path=CONFIG_PATH, interactive=False, pro if hasattr(args, 'force') and args.force.lower() in ['1', 'true']: force = True - res = find_datastore_device_pubkeys(blockchain_id, getattr(args, 'device_ids', None), getattr(args, 'device_pubkeys', None), datastore_id=datastore_id, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query device list for {}: {}'.format(name, res['error'])} - - data_pubkeys = res['pubkeys'] + # get the list of device IDs to use + device_ids = getattr(args, 'device_ids', None) + if device_ids: + device_ids = device_ids.split(',') + + else: + raise NotImplemented("Missing token file parsing logic") + + device_pubkeys = None + if hasattr(args, 'device_pubkeys'): + device_pubkeys = str(args.device_pubkeys).split(',') + else: + raise NotImplemented("No support for token files") + + assert len(device_ids) == len(device_pubkeys) + data_pubkeys = [{ + 'device_id': dev_id, + 'public_key': pubkey + } for (dev_id, pubkey) in zip(device_ids, device_pubkeys)] res = datastore_file_get('datastore', blockchain_id, datastore_id, path, data_pubkeys, extended=extended, force=force, config_path=config_path) if json_is_error(res): @@ -5439,7 +5084,7 @@ def cli_datastore_getfile( args, config_path=CONFIG_PATH, interactive=False, pro return res -def cli_datastore_listdir(args, config_path=CONFIG_PATH, interactive=False, proxy=None): +def cli_datastore_listdir(args, config_path=CONFIG_PATH, interactive=False ): """ command: datastore_listdir advanced help: List a directory in the datastore. @@ -5457,13 +5102,12 @@ def cli_datastore_listdir(args, config_path=CONFIG_PATH, interactive=False, prox blockchain_id = None else: blockchain_id = str(blockchain_id) - - proxy = get_default_proxy() if proxy is None else proxy datastore_id = str(args.datastore_id) path = str(args.path) extended = False force = False + device_ids = None if hasattr(args, 'extended') and args.extended.lower() in ['1', 'true']: extended = True @@ -5471,11 +5115,25 @@ def cli_datastore_listdir(args, config_path=CONFIG_PATH, interactive=False, prox if hasattr(args, 'force') and args.force.lower() in ['1', 'true']: force = True - res = find_datastore_device_pubkeys(blockchain_id, getattr(args, 'device_ids', None), getattr(args, 'device_pubkeys', None), datastore_id=datastore_id, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query device list for {}: {}'.format(name, res['error'])} - - data_pubkeys = res['pubkeys'] + # get the list of device IDs to use + device_ids = getattr(args, 'device_ids', None) + if device_ids: + device_ids = device_ids.split(',') + + else: + raise NotImplemented("Missing token file parsing logic") + + device_pubkeys = None + if hasattr(args, 'device_pubkeys'): + device_pubkeys = str(args.device_pubkeys).split(',') + else: + raise NotImplemented("No support for token files") + + assert len(device_ids) == len(device_pubkeys) + data_pubkeys = [{ + 'device_id': dev_id, + 'public_key': pubkey + } for (dev_id, pubkey) in zip(device_ids, device_pubkeys)] res = datastore_dir_list('datastore', blockchain_id, datastore_id, path, data_pubkeys, extended=extended, force=force, config_path=config_path ) if json_is_error(res): @@ -5487,7 +5145,7 @@ def cli_datastore_listdir(args, config_path=CONFIG_PATH, interactive=False, prox return res -def cli_datastore_stat(args, config_path=CONFIG_PATH, interactive=False, proxy=None): +def cli_datastore_stat(args, config_path=CONFIG_PATH, interactive=False ): """ command: datastore_stat advanced help: Stat a file or directory in the datastore, returning only the header for files but returning the entire listing for directories. @@ -5500,8 +5158,6 @@ def cli_datastore_stat(args, config_path=CONFIG_PATH, interactive=False, proxy=N opt: device_pubkeys (str) 'If given, a CSV of device public keys owned by the blockchain ID' """ - proxy = get_default_proxy() if proxy is None else proxy - blockchain_id = getattr(args, 'blockchain_id', '') if len(blockchain_id) == 0: blockchain_id = None @@ -5521,11 +5177,27 @@ def cli_datastore_stat(args, config_path=CONFIG_PATH, interactive=False, proxy=N if hasattr(args, 'force') and args.force.lower() in ['1', 'true']: force = True - res = find_datastore_device_pubkeys(blockchain_id, getattr(args, 'device_ids', None), getattr(args, 'device_pubkeys', None), datastore_id=datastore_id, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query device list for {}: {}'.format(name, res['error'])} - - data_pubkeys = res['pubkeys'] + # get the list of device IDs to use + device_ids = getattr(args, 'device_ids', None) + if device_ids: + device_ids = device_ids.split(',') + + else: + # TODO + raise NotImplemented("Missing token file parsing logic") + + device_pubkeys = None + if hasattr(args, 'device_pubkeys'): + device_pubkeys = str(args.device_pubkeys).split(',') + else: + # TODO + raise NotImplemented("No support for token files") + + assert len(device_ids) == len(device_pubkeys) + data_pubkeys = [{ + 'device_id': dev_id, + 'public_key': pubkey + } for (dev_id, pubkey) in zip(device_ids, device_pubkeys)] res = datastore_path_stat('datastore', blockchain_id, datastore_id, path, data_pubkeys, extended=extended, force=force, config_path=config_path) if json_is_error(res): @@ -5537,7 +5209,7 @@ def cli_datastore_stat(args, config_path=CONFIG_PATH, interactive=False, proxy=N return res -def cli_datastore_getinode(args, config_path=CONFIG_PATH, interactive=False, proxy=None): +def cli_datastore_getinode(args, config_path=CONFIG_PATH, interactive=False): """ command: datastore_getinode advanced help: Get a raw inode from a datastore @@ -5550,8 +5222,6 @@ def cli_datastore_getinode(args, config_path=CONFIG_PATH, interactive=False, pro opt: device_ids (str) 'If given, the CSV of devices owned by the blockchain ID' opt: device_pubkeys (str) 'If given, the CSV of device public keys owned by the blockchain ID' """ - - proxy = get_default_proxy() if proxy is None else proxy blockchain_id = getattr(args, 'blockchain_id', '') if len(blockchain_id) == 0: @@ -5579,11 +5249,27 @@ def cli_datastore_getinode(args, config_path=CONFIG_PATH, interactive=False, pro if hasattr(args, 'idata') and args.idata.lower() in ['1', 'true']: idata = True - res = find_datastore_device_pubkeys(blockchain_id, getattr(args, 'device_ids', None), getattr(args, 'device_pubkeys', None), datastore_id=datastore_id, proxy=proxy) - if 'error' in res: - return {'error': 'Failed to query device list for {}: {}'.format(name, res['error'])} - - data_pubkeys = res['pubkeys'] + # get the list of device IDs to use + device_ids = getattr(args, 'device_ids', None) + if device_ids: + device_ids = device_ids.split(',') + + else: + # TODO + raise NotImplemented("Missing token file parsing logic") + + device_pubkeys = None + if hasattr(args, 'device_pubkeys'): + device_pubkeys = str(args.device_pubkeys).split(',') + else: + # TODO + raise NotImplemented("No support for token files") + + assert len(device_ids) == len(device_pubkeys) + data_pubkeys = [{ + 'device_id': dev_id, + 'public_key': pubkey + } for (dev_id, pubkey) in zip(device_ids, device_pubkeys)] return datastore_inode_getinode('datastore', blockchain_id, datastore_id, inode_uuid, data_pubkeys, extended=extended, force=force, idata=idata, config_path=config_path) @@ -5657,6 +5343,12 @@ def cli_datastore_get_privkey(args, config_path=CONFIG_PATH, interactive=False ) """ raise NotImplemented("Token file support not yet implemented") + app_domain = str(args.app_domain) + master_privkey = str(args.master_privkey) + + datastore_privkey = datastore_get_privkey(master_privkey, app_domain, config_path=config_path) + return {'status': True, 'datastore_privkey': datastore_privkey} + def cli_datastore_get_id(args, config_path=CONFIG_PATH, interactive=False ): """ diff --git a/blockstack_client/app.py b/blockstack_client/app.py index fedd42c02..22c695119 100644 --- a/blockstack_client/app.py +++ b/blockstack_client/app.py @@ -153,7 +153,7 @@ def app_get_datastore_pubkey( session ): return None -def app_publish( dev_blockchain_id, app_domain, app_method_list, app_index_uris, app_index_file, data_privkey, app_driver_hints=[], proxy=None, config_path=CONFIG_PATH ): +def app_publish( dev_blockchain_id, app_domain, app_method_list, app_index_uris, app_index_file, app_driver_hints=[], data_privkey=None, proxy=None, wallet_keys=None, config_path=CONFIG_PATH ): """ Instantiate an application. * replicate the (opaque) app index file to "index.html" to each URL in app_uris @@ -170,6 +170,11 @@ def app_publish( dev_blockchain_id, app_domain, app_method_list, app_index_uris, Return {'error': ...} on error """ + if data_privkey is None: + assert wallet_keys, 'Missing both data private key and wallet keys' + data_privkey = wallet_keys.get('data_privkey') + assert data_privkey, "Wallet does not have a data private key" + proxy = get_default_proxy() if proxy is None else proxy # replicate configuration data (method list and app URIs) @@ -231,7 +236,7 @@ def app_get_config( blockchain_id, app_domain, data_pubkey=None, proxy=None, con proxy = get_default_proxy() if proxy is None else proxy # go get config - res = data.get_mutable( ".blockstack", [app_domain], data_pubkeys=[data_pubkey], proxy=proxy, config_path=config_path, blockchain_id=blockchain_id ) + res = data.get_mutable( ".blockstack", [app_domain], data_pubkey=data_pubkey, proxy=proxy, config_path=config_path, blockchain_id=blockchain_id ) if 'error' in res: log.error("Failed to get application config file {}: {}".format(config_data_id, res['error'])) return res @@ -270,7 +275,7 @@ def app_get_resource( blockchain_id, app_domain, res_name, app_config=None, data driver_hints = app_config['driver_hints'] urls = storage.get_driver_urls( res_data_id, storage.get_storage_handlers() ) - res = data.get_mutable( res_name, [app_domain], data_pubkeys=[data_pubkey], proxy=proxy, config_path=config_path, urls=urls, blockchain_id=blockchain_id ) + res = data.get_mutable( res_name, [app_domain], data_pubkey=data_pubkey, proxy=proxy, config_path=config_path, urls=urls, blockchain_id=blockchain_id ) if 'error' in res: log.error("Failed to get resource {}: {}".format(res_name, res['error'])) return {'error': 'Failed to load resource'} @@ -278,7 +283,7 @@ def app_get_resource( blockchain_id, app_domain, res_name, app_config=None, data return {'status': True, 'res': res['data']} -def app_put_resource( blockchain_id, app_domain, res_name, res_data, data_privkey, app_config=None, proxy=None, config_path=CONFIG_PATH ): +def app_put_resource( blockchain_id, app_domain, res_name, res_data, app_config=None, data_privkey=None, proxy=None, wallet_keys=None, config_path=CONFIG_PATH ): """ Store data to a named application resource in mutable storage. @@ -297,6 +302,11 @@ def app_put_resource( blockchain_id, app_domain, res_name, res_data, data_privke except: raise AssertionError("Resource must be a JSON-serializable string") + if data_privkey is None: + assert wallet_keys, 'Missing both data private key and wallet keys' + data_privkey = wallet_keys.get('data_privkey') + assert data_privkey, "Wallet does not have a data private key" + proxy = get_default_proxy() if proxy is None else proxy res_data_id = storage.make_fq_data_id(app_domain, res_name) @@ -318,7 +328,7 @@ def app_put_resource( blockchain_id, app_domain, res_name, res_data, data_privke return {'status': True, 'version': res_blob['version']} -def app_delete_resource( blockchain_id, app_domain, res_name, data_privkey, app_config=None, proxy=None, config_path=CONFIG_PATH ): +def app_delete_resource( blockchain_id, app_domain, res_name, app_config=None, data_privkey=None, proxy=None, wallet_keys=None, config_path=CONFIG_PATH ): """ Remove data from a named application resource in mutable storage. @@ -331,6 +341,11 @@ def app_delete_resource( blockchain_id, app_domain, res_name, data_privkey, app_ Return {'error': ...} on error """ + if data_privkey is None: + assert wallet_keys, "No data private key or wallet given" + data_privkey = wallet_keys.get('data_privkey', None) + assert data_privkey, "Wallet does not contain a data private key" + data_pubkey = get_pubkey_hex(data_privkey) proxy = get_default_proxy() if proxy is None else proxy diff --git a/blockstack_client/backend/drivers/common.py b/blockstack_client/backend/drivers/common.py index 3877d7fbb..82ab15f63 100644 --- a/blockstack_client/backend/drivers/common.py +++ b/blockstack_client/backend/drivers/common.py @@ -905,7 +905,7 @@ def put_indexed_data( dvconf, name, chunk_buf, raw=False, index=True ): log.debug("Insert ({}, {}) into index".format(name, new_url)) rc = index_insert( dvconf, name, new_url ) if not rc: - log.error("Failed to insert ({}, {}) into index".foramt(name, new_url)) + log.error("Failed to insert ({}, {}) into index".format(name, new_url)) return False return True diff --git a/blockstack_client/backend/drivers/dropbox.py b/blockstack_client/backend/drivers/dropbox.py index 3cc73d680..7e8ebdccf 100644 --- a/blockstack_client/backend/drivers/dropbox.py +++ b/blockstack_client/backend/drivers/dropbox.py @@ -58,10 +58,7 @@ INDEX_DIRNAME = "index" DVCONF = None BLOCKSTACK_DEBUG = (os.environ.get("BLOCKSTACK_DEBUG") == "1") -BLOCKSTACK_TEST = (os.environ.get("BLOCKSTACK_TEST") == "1") -if BLOCKSTACK_TEST: - INDEX_DIRNAME = "index-regtest" def dropbox_url_reformat(url): """ diff --git a/blockstack_client/backend/nameops.py b/blockstack_client/backend/nameops.py index 673e93875..e496b7379 100644 --- a/blockstack_client/backend/nameops.py +++ b/blockstack_client/backend/nameops.py @@ -48,7 +48,7 @@ from ..proxy import get_namespace_blockchain_record as blockstack_get_namespace_ from ..tx import sign_tx, sign_and_broadcast_tx, deserialize_tx, preorder_tx, register_tx, update_tx, transfer_tx, revoke_tx, \ namespace_preorder_tx, namespace_reveal_tx, namespace_ready_tx, announce_tx, name_import_tx -from ..scripts import tx_make_subsidizable, tx_get_unspents +from ..scripts import tx_make_subsidizable, tx_get_unspents, tx_estimate_signature_len_bytes from ..storage import get_blockchain_compat_hash, put_announcement, get_zonefile_data_hash from ..operations import fees_update, fees_transfer, fees_revoke, fees_registration, fees_preorder, \ @@ -61,6 +61,10 @@ from ..utxo import get_unspents import virtualchain from virtualchain.lib.ecdsalib import ecdsa_private_key +from ..constants import get_secret +from .crypto.utils import aes_decrypt +from binascii import hexlify + log = get_logger("blockstack-client") @@ -157,7 +161,7 @@ def make_cheapest_nameop( opcode, utxo_client, payment_address, payment_utxos, * try: log.debug("Try building a {} with inputs 0-{} of {}".format(opcode, i, payment_address)) utxo_client = build_utxo_client(utxo_client, address=payment_address, utxos=payment_utxos[0:i]) - unsigned_tx = tx_builder(*tx_args, **tx_kw) + unsigned_tx = tx_builder(*tx_args, **tx_kw) assert unsigned_tx log.debug("Funded {} with inputs 0-{} of {}".format(opcode, i, payment_address)) @@ -165,71 +169,88 @@ def make_cheapest_nameop( opcode, utxo_client, payment_address, payment_utxos, * except (AssertionError, ValueError): pass - return unsigned_tx + return unsigned_tx, i -def make_cheapest_namespace_preorder( namespace_id, payment_address, reveal_address, cost, consensus_hash, utxo_client, payment_utxos, tx_fee=0 ): +def make_cheapest_namespace_preorder( namespace_id, payment_address, reveal_address, cost, consensus_hash, utxo_client, payment_utxos, tx_fee=0, return_n_funded_inputs = False ): """ Given namespace preorder info, make the cheapest possible namespace preorder transaction. @payment_utxos should be sorted by decreasing value Return the unsigned tx on success. Return None on error """ - return make_cheapest_nameop('NAMESPACE_PREORDER', utxo_client, payment_address, payment_utxos, namespace_id, reveal_address, cost, consensus_hash, payment_address, utxo_client, tx_fee=tx_fee ) + ret = make_cheapest_nameop('NAMESPACE_PREORDER', utxo_client, payment_address, payment_utxos, namespace_id, reveal_address, cost, consensus_hash, payment_address, utxo_client, tx_fee=tx_fee ) + if return_n_funded_inputs: + return ret + return ret[0] - -def make_cheapest_namespace_reveal( namespace_id, reveal_addr, lifetime, coeff, base_cost, bucket_exponents, nonalpha_discount, no_vowel_discount, preorder_addr, utxo_client, payment_utxos, tx_fee=0 ): +def make_cheapest_namespace_reveal( namespace_id, reveal_addr, lifetime, coeff, base_cost, bucket_exponents, nonalpha_discount, no_vowel_discount, preorder_addr, utxo_client, payment_utxos, tx_fee=0, return_n_funded_inputs = False ): """ Given namespace reveal info, make the cheapest possible namespace reveal transaction. @payment_utxos should be sorted by decreasing value Return the unsigned tx on success Return None on error """ - return make_cheapest_nameop('NAMESPACE_REVEAL', utxo_client, preorder_addr, payment_utxos, namespace_id, reveal_addr, lifetime, coeff, base_cost, bucket_exponents, nonalpha_discount, no_vowel_discount, preorder_addr, utxo_client, + ret = make_cheapest_nameop('NAMESPACE_REVEAL', utxo_client, preorder_addr, payment_utxos, namespace_id, reveal_addr, lifetime, coeff, base_cost, bucket_exponents, nonalpha_discount, no_vowel_discount, preorder_addr, utxo_client, tx_fee=tx_fee) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_namespace_ready( namespace_id, reveal_addr, utxo_client, payment_utxos, tx_fee=0 ): +def make_cheapest_namespace_ready( namespace_id, reveal_addr, utxo_client, payment_utxos, tx_fee=0, return_n_funded_inputs = False ): """ Given namespace ready info, make the cheapest possible namespace ready transaction @payment_utxos should be sorted by decreasing value Return the unsigned tx on success Return None on error """ - return make_cheapest_nameop('NAMESPACE_READY', utxo_client, reveal_addr, payment_utxos, namespace_id, reveal_addr, utxo_client, tx_fee=tx_fee) + ret = make_cheapest_nameop('NAMESPACE_READY', utxo_client, reveal_addr, payment_utxos, namespace_id, reveal_addr, utxo_client, tx_fee=tx_fee) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_name_import( name, recipient_address, zonefile_hash, reveal_address, utxo_client, payment_utxos, tx_fee=0 ): +def make_cheapest_name_import( name, recipient_address, zonefile_hash, reveal_address, utxo_client, payment_utxos, tx_fee=0, return_n_funded_inputs = False ): """ Given name import info, make the cheapest possible name import transaction @payment_utxos should be sorted by decreasing value Return the unsigned tx on success Return None on error """ - return make_cheapest_nameop("NAME_IMPORT", utxo_client, reveal_address, payment_utxos, name, recipient_address, zonefile_hash, reveal_address, utxo_client, tx_fee=tx_fee ) + ret = make_cheapest_nameop("NAME_IMPORT", utxo_client, reveal_address, payment_utxos, name, recipient_address, zonefile_hash, reveal_address, utxo_client, tx_fee=tx_fee ) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_name_preorder( name, payment_address, owner_address, cost, consensus_hash, utxo_client, payment_utxos, tx_fee=0 ): +def make_cheapest_name_preorder( name, payment_address, owner_address, cost, consensus_hash, utxo_client, payment_utxos, tx_fee=0, return_n_funded_inputs = False ): """ Given name preorder info, make the cheapest possible name preorder transaction. @payment_utxos should be sorted by decreasing value Return the unsigned tx on success. Return None on error """ - return make_cheapest_nameop('NAME_PREORDER', utxo_client, payment_address, payment_utxos, name, payment_address, owner_address, cost, consensus_hash, utxo_client, tx_fee=tx_fee ) + ret = make_cheapest_nameop('NAME_PREORDER', utxo_client, payment_address, payment_utxos, name, payment_address, owner_address, cost, consensus_hash, utxo_client, tx_fee=tx_fee ) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_name_registration( name, payment_address, owner_address, utxo_client, payment_utxos, tx_fee=0, subsidize=False ): +def make_cheapest_name_registration( name, payment_address, owner_address, utxo_client, payment_utxos, tx_fee=0, subsidize=False, return_n_funded_inputs = False ): """ Given name registration info, make the cheapest possible name register transaction @payment_utxos should be sorted by decreasing value Return the unsigned tx on success. Return None on error """ - return make_cheapest_nameop('NAME_REGISTRATION', utxo_client, payment_address, payment_utxos, name, payment_address, owner_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + ret = make_cheapest_nameop('NAME_REGISTRATION', utxo_client, payment_address, payment_utxos, name, payment_address, owner_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_name_renewal( name, owner_address, renewal_fee, utxo_client, payment_address, payment_utxos, tx_fee=0, subsidize=False ): +def make_cheapest_name_renewal( name, owner_address, renewal_fee, utxo_client, payment_address, payment_utxos, tx_fee=0, subsidize=False, return_n_funded_inputs = False ): """ Given name renewal info, make the cheapest possible name renewal transaction @payment_utxos should be sorted by decreasing value @@ -238,47 +259,62 @@ def make_cheapest_name_renewal( name, owner_address, renewal_fee, utxo_client, p """ # NOTE: for name renewal, the owner address is both the "preorder" and "register" address. # the payment address and UTXOs given here are for the address that will subsidize the operation. - return make_cheapest_nameop('NAME_RENEWAL', utxo_client, payment_address, payment_utxos, name, owner_address, owner_address, utxo_client, renewal_fee=renewal_fee, subsidize=subsidize, tx_fee=tx_fee) + ret = make_cheapest_nameop('NAME_RENEWAL', utxo_client, payment_address, payment_utxos, name, owner_address, owner_address, utxo_client, renewal_fee=renewal_fee, subsidize=subsidize, tx_fee=tx_fee) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_name_update( name, data_hash, consensus_hash, owner_address, utxo_client, payment_address, payment_utxos, tx_fee=0, subsidize=False ): +def make_cheapest_name_update( name, data_hash, consensus_hash, owner_address, utxo_client, payment_address, payment_utxos, tx_fee=0, subsidize=False, return_n_funded_inputs = False ): """ Given name update info, make the cheapest possible name update transaction @payment_utxos should be sorted by decreasing value Return the unsigned tx on success Return None on error """ - return make_cheapest_nameop('NAME_UPDATE', utxo_client, payment_address, payment_utxos, name, data_hash, consensus_hash, owner_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + ret = make_cheapest_nameop('NAME_UPDATE', utxo_client, payment_address, payment_utxos, name, data_hash, consensus_hash, owner_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_name_transfer( name, recipient_address, keepdata, consensus_hash, owner_address, utxo_client, payment_address, payment_utxos, tx_fee=0, subsidize=False ): +def make_cheapest_name_transfer( name, recipient_address, keepdata, consensus_hash, owner_address, utxo_client, payment_address, payment_utxos, tx_fee=0, subsidize=False, return_n_funded_inputs = False ): """ Given name transfer info, make the cheapest possible name transfer transaction @payment_utxos should be sorted by decreasing value Return the unsigned tx on success Return None on error """ - return make_cheapest_nameop('NAME_TRANSFER', utxo_client, payment_address, payment_utxos, name, recipient_address, keepdata, consensus_hash, owner_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + ret = make_cheapest_nameop('NAME_TRANSFER', utxo_client, payment_address, payment_utxos, name, recipient_address, keepdata, consensus_hash, owner_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_name_revoke( name, owner_address, utxo_client, payment_address, payment_utxos, tx_fee=0, subsidize=False ): +def make_cheapest_name_revoke( name, owner_address, utxo_client, payment_address, payment_utxos, tx_fee=0, subsidize=False, return_n_funded_inputs = False ): """ Given name revoke info, make the cheapest possible name revoke transaction @payment_utxos should be sorted by decreasing value Return the unsigned tx on success Return None on error """ - return make_cheapest_nameop('NAME_REVOKE', utxo_client, payment_address, payment_utxos, name, owner_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + ret = make_cheapest_nameop('NAME_REVOKE', utxo_client, payment_address, payment_utxos, name, owner_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + if return_n_funded_inputs: + return ret + return ret[0] -def make_cheapest_announce( announce_hash, sender_address, utxo_client, payment_utxos, tx_fee=0, subsidize=False ): +def make_cheapest_announce( announce_hash, sender_address, utxo_client, payment_utxos, tx_fee=0, subsidize=False, return_n_funded_inputs = False ): """ Given announce info, make the cheapest possible announce transaction @payment_utxos should be sorted by decreasing value Return the unsigned tx on success Return None on error """ - return make_cheapest_nameop('ANNOUNCE', utxo_client, sender_address, payment_utxos, announce_hash, sender_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + ret = make_cheapest_nameop('ANNOUNCE', utxo_client, sender_address, payment_utxos, announce_hash, sender_address, utxo_client, subsidize=subsidize, tx_fee=tx_fee ) + if return_n_funded_inputs: + return ret + return ret[0] def get_estimated_signed_subsidized(unsigned_tx, op_fees, max_fee, payment_privkey_info, @@ -292,16 +328,17 @@ def get_estimated_signed_subsidized(unsigned_tx, op_fees, max_fee, payment_privk MAX_RETRIES = 10 tx_fee_guess = 0 for _ in range(MAX_RETRIES): - subsidized_tx = tx_make_subsidizable( unsigned_tx, op_fees, max_fee, payment_privkey_info, - utxo_client, tx_fee = tx_fee_guess ) + subsidized_tx, sign_lens = tx_make_subsidizable(unsigned_tx, op_fees, max_fee, payment_privkey_info, + utxo_client, tx_fee = tx_fee_guess, + simulated_sign = True) assert subsidized_tx is not None + pad_length = sign_lens + 2*tx_estimate_signature_len_bytes(owner_privkey_info) + padded_tx = subsidized_tx + ("0" * pad_length) - signed_subsidized_tx = sign_tx(subsidized_tx, owner_privkey_info) - - tx_fee = (len(signed_subsidized_tx) * tx_fee_per_byte) / 2 + tx_fee = (len(padded_tx) * tx_fee_per_byte)/2 if tx_fee <= tx_fee_guess: - log.debug("Estimated TX Length and fee per byte: {} + {}".format(len(signed_subsidized_tx) / 2, tx_fee_per_byte)) - return signed_subsidized_tx + log.debug("Estimated TX Length and fee per byte: {} + {}".format(len(padded_tx)/2, tx_fee_per_byte)) + return padded_tx tx_fee_guess = tx_fee raise Exception("Failed to cover the tx_fee in getting estimated subsidized tx") @@ -332,10 +369,14 @@ def estimate_preorder_tx_fee( name, name_cost, payment_privkey_info, owner_privk try: try: - unsigned_tx = make_cheapest_name_preorder(name, payment_addr, owner_address, name_cost, fake_consensus_hash, utxo_client, payment_utxos ) + unsigned_tx, n_inputs = make_cheapest_name_preorder( + name, payment_addr, owner_address, name_cost, fake_consensus_hash, + utxo_client, payment_utxos, return_n_funded_inputs = True) assert unsigned_tx - signed_tx = sign_tx(unsigned_tx, payment_privkey_info) + pad_len = n_inputs * tx_estimate_signature_len_bytes(payment_privkey_info) + signed_tx = unsigned_tx + ("00" * pad_len) + assert signed_tx is not None except AssertionError as e: @@ -396,10 +437,14 @@ def estimate_register_tx_fee( name, payment_privkey_info, owner_privkey_info, tx signed_tx = None try: try: - unsigned_tx = make_cheapest_name_registration(name, payment_addr, owner_addr, utxo_client, payment_utxos) + unsigned_tx, n_inputs = make_cheapest_name_registration( + name, payment_addr, owner_addr, utxo_client, + payment_utxos, return_n_funded_inputs = True) assert unsigned_tx - - signed_tx = sign_tx(unsigned_tx, payment_privkey_info) + + pad_len = n_inputs * tx_estimate_signature_len_bytes(payment_privkey_info) + signed_tx = unsigned_tx + ("00" * pad_len) + assert signed_tx is not None except AssertionError as e: @@ -1224,9 +1269,9 @@ def do_preorder( fqu, payment_privkey_info, owner_privkey_info, cost_satoshis, u payment_address = virtualchain.get_privkey_address( payment_privkey_info ) min_confirmations = utxo_client.min_confirmations - tx_fee = 0 if not dry_run and (safety_checks or (cost_satoshis is None or tx_fee is None)): + tx_fee = 0 # find tx fee, and do sanity checks res = check_preorder(fqu, cost_satoshis, owner_privkey_info, payment_privkey_info, config_path=config_path, proxy=proxy, min_payment_confs=min_confirmations) if 'error' in res and safety_checks: @@ -1312,9 +1357,9 @@ def do_register( fqu, payment_privkey_info, owner_privkey_info, utxo_client, tx_ owner_address = virtualchain.get_privkey_address( owner_privkey_info ) min_confirmations = utxo_client.min_confirmations - tx_fee = 0 if not dry_run and (safety_checks or tx_fee is None): + tx_fee = 0 # find tx fee, and do sanity checks res = check_register(fqu, owner_privkey_info, payment_privkey_info, config_path=config_path, proxy=proxy, min_payment_confs=min_confirmations, @@ -2077,7 +2122,10 @@ def async_preorder(fqu, payment_privkey_info, owner_privkey_info, cost, name_dat @fqu: fully qualified name e.g., muneeb.id @payment_privkey_info: private key that will pay @owner_address: will own the name + @transfer_address: will ultimately receive the name + @zonefile_data: serialized zonefile for the name + @profile: profile for the name Returns True/False and stores tx_hash in queue """ @@ -2115,6 +2163,8 @@ def async_preorder(fqu, payment_privkey_info, owner_privkey_info, cost, name_dat additionals['confirmations_needed'] = 4 if 'min_payment_confs' in name_data: additionals['min_payment_confs'] = name_data['min_payment_confs'] + if 'owner_privkey' in name_data: + additionals['owner_privkey'] = name_data['owner_privkey'] if 'transaction_hash' in resp: if not BLOCKSTACK_DRY_RUN: # watch this preorder, and register it when it gets queued @@ -2123,7 +2173,7 @@ def async_preorder(fqu, payment_privkey_info, owner_privkey_info, cost, name_dat owner_address=owner_address, transfer_address=name_data.get('transfer_address'), zonefile_data=name_data.get('zonefile'), - token_file=name_data.get('token_file'), + profile=name_data.get('profile'), config_path=config_path, path=queue_path, **additionals) else: @@ -2134,6 +2184,18 @@ def async_preorder(fqu, payment_privkey_info, owner_privkey_info, cost, name_dat return resp +def check_owner_privkey_info(owner_privkey_info, name_data): + owner_address = virtualchain.get_privkey_address(owner_privkey_info) + if 'owner_address' in name_data and owner_address != name_data['owner_address']: + log.debug("Registrar owner address changed since beginning registration : from {} to {}".format( + name_data['owner_address'], owner_address)) + owner_address = name_data['owner_address'] + passwd = get_secret('BLOCKSTACK_CLIENT_WALLET_PASSWORD') + owner_privkey_info = aes_decrypt( + str(name_data['owner_privkey']), hexlify( passwd )) + if not virtualchain.get_privkey_address(owner_privkey_info) == owner_address: + raise Exception("Attempting to correct registrar address to {}, but failed!".format(owner_address)) + return owner_address, owner_privkey_info def async_register(fqu, payment_privkey_info, owner_privkey_info, name_data={}, proxy=None, config_path=CONFIG_PATH, queue_path=DEFAULT_QUEUE_PATH, safety_checks=True): @@ -2161,7 +2223,8 @@ def async_register(fqu, payment_privkey_info, owner_privkey_info, name_data={}, tx_broadcaster = get_tx_broadcaster( config_path=config_path ) - owner_address = virtualchain.get_privkey_address( owner_privkey_info ) + owner_address, owner_privkey_info = check_owner_privkey_info( owner_privkey_info, name_data ) + payment_address = virtualchain.get_privkey_address( payment_privkey_info ) # check register_queue first @@ -2198,6 +2261,8 @@ def async_register(fqu, payment_privkey_info, owner_privkey_info, name_data={}, force_register = True if 'min_payment_confs' in name_data: additionals['min_payment_confs'] = name_data['min_payment_confs'] + if 'owner_privkey' in name_data: + additionals['owner_privkey'] = name_data['owner_privkey'] try: resp = do_register( fqu, payment_privkey_info, owner_privkey_info, utxo_client, tx_broadcaster, @@ -2213,7 +2278,7 @@ def async_register(fqu, payment_privkey_info, owner_privkey_info, name_data={}, owner_address=owner_address, transfer_address=name_data.get('transfer_address'), zonefile_data=name_data.get('zonefile'), - token_file=name_data.get('token_file'), + profile=name_data.get('profile'), config_path=config_path, path=queue_path, **additionals) @@ -2226,7 +2291,7 @@ def async_register(fqu, payment_privkey_info, owner_privkey_info, name_data={}, return {'error': 'Failed to send registration: {}'.format(resp['error'])} -def async_update(fqu, zonefile_data, token_file, owner_privkey_info, payment_privkey_info, +def async_update(fqu, zonefile_data, profile, owner_privkey_info, payment_privkey_info, name_data={}, config_path=CONFIG_PATH, zonefile_hash=None, proxy=None, queue_path=DEFAULT_QUEUE_PATH ): """ @@ -2234,7 +2299,7 @@ def async_update(fqu, zonefile_data, token_file, owner_privkey_info, payment_pri @fqu: fully qualified name e.g., muneeb.id @zonefile_data: new zonefile text, hash(zonefile) goes to blockchain. If not given, it will be extracted from name_data - @token_file: the name's token file. If not given, it will be extracted from name_data + @profile: the name's profile. If not given, it will be extracted from name_data @owner_privkey_info: privkey of owner address, to sign update @payment_privkey_info: the privkey which is paying for the cost @@ -2249,10 +2314,10 @@ def async_update(fqu, zonefile_data, token_file, owner_privkey_info, payment_pri elif name_data.get('zonefile') is not None and zonefile_data != name_data.get('zonefile'): assert name_data['zonefile'] == zonefile_data, "Conflicting zone file data given" - if token_file is None: - token_file = name_data.get('token_file') - elif name_data.get('token_file') is not None and token_file != name_data.get('token_file'): - assert name_data['token_file'] == token_file, "Conflicting token_file data given" + if profile is None: + profile = name_data.get('profile') + elif name_data.get('profile') is not None and profile != name_data.get('profile'): + assert name_data['profile'] == profile, "Conflicting profile data given" assert zonefile_hash is not None or zonefile_data is not None, "No zone file or zone file hash given" @@ -2283,7 +2348,7 @@ def async_update(fqu, zonefile_data, token_file, owner_privkey_info, payment_pri tx_broadcaster = get_tx_broadcaster(config_path=config_path) - owner_address = virtualchain.get_privkey_address( owner_privkey_info ) + owner_address, owner_privkey_info = check_owner_privkey_info( owner_privkey_info, name_data ) if in_queue("update", fqu, path=queue_path): log.error("Already in update queue: %s" % fqu) @@ -2299,6 +2364,9 @@ def async_update(fqu, zonefile_data, token_file, owner_privkey_info, payment_pri additionals['confirmations_needed'] = 1 force_update = True + if 'owner_privkey' in name_data: + additionals['owner_privkey'] = name_data['owner_privkey'] + resp = {} try: resp = do_update( fqu, zonefile_hash, owner_privkey_info, payment_privkey_info, utxo_client, tx_broadcaster, @@ -2312,7 +2380,7 @@ def async_update(fqu, zonefile_data, token_file, owner_privkey_info, payment_pri if not BLOCKSTACK_DRY_RUN: queue_append("update", fqu, resp['transaction_hash'], zonefile_data=zonefile_data, - token_file=token_file, + profile=profile, zonefile_hash=zonefile_hash, owner_address=owner_address, transfer_address=name_data.get('transfer_address'), @@ -2330,7 +2398,7 @@ def async_update(fqu, zonefile_data, token_file, owner_privkey_info, payment_pri def async_transfer(fqu, transfer_address, owner_privkey_info, payment_privkey_info, - config_path=CONFIG_PATH, proxy=None, queue_path=DEFAULT_QUEUE_PATH): + config_path=CONFIG_PATH, proxy=None, queue_path=DEFAULT_QUEUE_PATH, name_data = {}): """ Transfer a previously registered fqu, using a different payment address. Preserves the zonefile. @@ -2350,7 +2418,7 @@ def async_transfer(fqu, transfer_address, owner_privkey_info, payment_privkey_in utxo_client = get_utxo_provider_client(config_path=config_path) tx_broadcaster = get_tx_broadcaster(config_path=config_path) - owner_address = virtualchain.get_privkey_address( owner_privkey_info ) + owner_address, owner_privkey_info = check_owner_privkey_info( owner_privkey_info, name_data ) if in_queue("transfer", fqu, path=queue_path): log.error("Already in transfer queue: %s" % fqu) @@ -2363,13 +2431,17 @@ def async_transfer(fqu, transfer_address, owner_privkey_info, payment_privkey_in log.exception(e) return {'error': 'Failed to sign and broadcast transfer transaction'} + additionals = {} + if 'owner_privkey' in name_data: + additionals['owner_privkey'] = name_data['owner_privkey'] + if 'transaction_hash' in resp: if not BLOCKSTACK_DRY_RUN: queue_append("transfer", fqu, resp['transaction_hash'], owner_address=owner_address, transfer_address=transfer_address, config_path=config_path, - path=queue_path) + path=queue_path, **additionals) else: assert 'error' in resp log.error("Error transferring: %s" % fqu) diff --git a/blockstack_client/backend/queue.py b/blockstack_client/backend/queue.py index b541c74e1..786e503b0 100644 --- a/blockstack_client/backend/queue.py +++ b/blockstack_client/backend/queue.py @@ -317,9 +317,10 @@ def in_queue( queue_id, fqu, path=DEFAULT_QUEUE_PATH ): def queue_append(queue_id, fqu, tx_hash, payment_address=None, owner_address=None, transfer_address=None, config_path=CONFIG_PATH, block_height=None, - zonefile_data=None, token_file=None, zonefile_hash=None, - unsafe_reg=None, confirmations_needed=None, - min_payment_confs=None, path=DEFAULT_QUEUE_PATH): + zonefile_data=None, profile=None, zonefile_hash=None, + unsafe_reg = None, confirmations_needed = None, + owner_privkey=None, + min_payment_confs = None, path=DEFAULT_QUEUE_PATH): """ Append a processing name operation to the named queue for the given name. @@ -348,11 +349,13 @@ def queue_append(queue_id, fqu, tx_hash, payment_address=None, new_entry['confirmations_needed'] = confirmations_needed if min_payment_confs is not None: new_entry['min_payment_confs'] = min_payment_confs + if owner_privkey is not None: + new_entry['owner_privkey'] = owner_privkey if zonefile_data is not None: new_entry['zonefile_b64'] = base64.b64encode(zonefile_data) - new_entry['token_file'] = token_file + new_entry['profile'] = profile if zonefile_hash is None and zonefile_data is not None: zonefile_hash = get_zonefile_data_hash(zonefile_data) diff --git a/blockstack_client/backend/registrar.py b/blockstack_client/backend/registrar.py index 3b717e754..53c474af5 100644 --- a/blockstack_client/backend/registrar.py +++ b/blockstack_client/backend/registrar.py @@ -48,19 +48,24 @@ from .queue import queue_add_error_msg from .nameops import async_preorder, async_register, async_update, async_transfer, async_renew, async_revoke -from ..keys import get_data_privkey_info, is_singlesig_hex, get_owner_privkey_info, get_signing_privkey -from ..token_file import token_file_put +from ..keys import get_data_privkey_info, is_singlesig_hex from ..proxy import is_name_registered, is_zonefile_hash_current, get_default_proxy, get_name_blockchain_record, get_atlas_peers, json_is_error from ..zonefile import zonefile_data_replicate from ..user import is_user_zonefile from ..storage import put_mutable_data, get_zonefile_data_hash +from ..data import set_profile_timestamp from ..constants import CONFIG_PATH, DEFAULT_QUEUE_PATH, BLOCKSTACK_DEBUG, BLOCKSTACK_TEST, TX_MIN_CONFIRMATIONS from ..constants import PREORDER_CONFIRMATIONS +from ..constants import get_secret + from ..config import get_config from ..utils import url_to_host_port from ..logger import get_logger +from binascii import hexlify +from .crypto.utils import aes_encrypt, aes_decrypt + DEBUG = True __registrar_state = None @@ -189,12 +194,12 @@ class RegistrarWorker(threading.Thread): if not in_queue("register", name_data['fqu'], path=queue_path): # was preordered but not registered # send the registration - log.debug('Send async register for {}'.format(name_data['fqu'])) - log.debug("async_register({}, zonefile={}, token_file={}, transfer_address={})".format(name_data['fqu'], name_data.get('zonefile'), name_data.get('token_file'), name_data.get('transfer_address'))) + log.debug("async_register({}, zonefile={}, profile={}, transfer_address={})".format( + name_data['fqu'], name_data.get('zonefile'), name_data.get('profile'), + name_data.get('transfer_address'))) res = async_register( name_data['fqu'], payment_privkey_info, owner_privkey_info, name_data=name_data, proxy=proxy, config_path=config_path, queue_path=queue_path ) - return res else: # already queued @@ -238,9 +243,10 @@ class RegistrarWorker(threading.Thread): else: raise Exception("Queue inconsistency: name '%s' is and is not pending update" % up_result['fqu']) - log.debug("update({}, zonefile={}, token_file={}, transfer_address={})".format(name_data['fqu'], name_data.get('zonefile'), name_data.get('token_file'), name_data.get('transfer_address'))) - res = update( name_data['fqu'], name_data.get('zonefile'), name_data.get('token_file'), - name_data.get('zonefile_hash'), name_data.get('transfer_address'), config_path=config_path, proxy=proxy ) + log.debug("update({}, zonefile={}, profile={}, transfer_address={})".format(name_data['fqu'], name_data.get('zonefile'), name_data.get('profile'), name_data.get('transfer_address'))) + res = update( name_data['fqu'], name_data.get('zonefile'), name_data.get('profile'), + name_data.get('zonefile_hash'), name_data.get('transfer_address'), + config_path=config_path, proxy=proxy, prior_name_data = name_data ) assert 'success' in res @@ -262,7 +268,7 @@ class RegistrarWorker(threading.Thread): def set_zonefiles( cls, queue_path, config_path=CONFIG_PATH, proxy=None ): """ Find all confirmed registrations, create empty zonefiles for them and broadcast their hashes to the blockchain. - Queue up the zonefiles and token files for subsequent replication. + Queue up the zonefiles and profiles for subsequent replication. Return {'status': True} on success Return {'error': ...} on failure """ @@ -275,7 +281,7 @@ class RegistrarWorker(threading.Thread): # already migrated? if in_queue("update", register['fqu'], path=queue_path): - log.warn("Already initialized token file for name '%s'" % register['fqu']) + log.warn("Already initialized profile for name '%s'" % register['fqu']) queue_removeall( [register], path=queue_path ) continue @@ -406,12 +412,12 @@ class RegistrarWorker(threading.Thread): @classmethod - def replicate_name_data( cls, name_data, atlas_servers, wallet_data, storage_drivers, config_path, proxy=None, replicated_zonefiles=[], replicated_token_file_hashes=[] ): + def replicate_name_data( cls, name_data, atlas_servers, wallet_data, storage_drivers, config_path, proxy=None, replicated_zonefiles=[], replicated_profile_hashes=[] ): """ Given an update queue entry, replicate the zonefile to as many blockstack atlas servers as we can. - If given, replicate the token file as well. + If given, replicate the profile as well. @atlas_servers should be a list of (host, port) Return {'status': True} on success Return {'error': ...} on error @@ -451,8 +457,9 @@ class RegistrarWorker(threading.Thread): log.info("Replicated zonefile data for %s to %s server(s)" % (name_data['fqu'], len(res['servers']))) replicated_zonefiles.append(zonefile_hash) - # replicate token file to storage, if given - if name_data.has_key('token_file') and name_data['token_file'] is not None: + # replicate profile to storage, if given + # use the data keypair + if name_data.has_key('profile') and name_data['profile'] is not None: # only works this is actually a zonefile, since we need to use # the zonefile to find the appropriate data private key. zonefile = None @@ -463,43 +470,45 @@ class RegistrarWorker(threading.Thread): if BLOCKSTACK_TEST: log.exception(e) - log.warning("Not a standard zone file; not replicating token file for %s" % name_data['fqu']) + log.warning("Not a zone file; not replicating profile for %s" % name_data['fqu']) return {'status': True} - - signing_privkey = get_signing_privkey( get_owner_privkey_info(wallet_data) ) - assert signing_privkey - - log.info("Replicate token file data for %s to %s" % (name_data['fqu'], ",".join(storage_drivers))) - token_file_payload = copy.deepcopy(name_data['token_file']) - token_file_hash = hashlib.sha256(name_data['fqu'] + zonefile_hash + token_file_payload).hexdigest() + data_privkey = get_data_privkey_info( zonefile, wallet_keys=wallet_data, config_path=config_path ) + assert data_privkey is not None and not json_is_error(data_privkey), "No data private key" - # did we replicate this token file for this name and zonefile already? - if token_file_hash in replicated_token_file_hashes: + log.info("Replicate profile data for %s to %s" % (name_data['fqu'], ",".join(storage_drivers))) + + profile_payload = copy.deepcopy(name_data['profile']) + profile_hash = hashlib.sha256(name_data['fqu'] + zonefile_hash + json.dumps(profile_payload, sort_keys=True)).hexdigest() + + # did we replicate this profile for this name and zonefile already? + if profile_hash in replicated_profile_hashes: # already replicated - log.debug("Already replicated token file for {}".format(name_data['fqu'])) + log.debug("Already replicated profile for {}".format(name_data['fqu'])) return {'status': True} + + profile_payload = set_profile_timestamp(profile_payload) - res = token_file_put(name_data['fqu'], token_file_payload, signing_privkey, required_drivers=storage_drivers, config_path=config_path) - if 'error' in res: - log.error("Failed to replicate token file for {}: {}".format(name_data['fqu'], res['error'])) - return {'error': "Failed to replicate token file"} + rc = put_mutable_data( name_data['fqu'], profile_payload, data_privkey=data_privkey, required=storage_drivers, profile=True, blockchain_id=name_data['fqu'] ) + if not rc: + log.info("Failed to replicate profile for %s" % (name_data['fqu'])) + return {'error': 'Failed to store profile'} + else: + log.info("Replicated profile for %s" % (name_data['fqu'])) - log.info("Replicated token file for %s" % (name_data['fqu'])) - - # don't do this again - replicated_token_file_hashes.append(token_file_hash) - return {'status': True} + # don't do this again + replicated_profile_hashes.append(profile_hash) + return {'status': True} else: - log.info("No token file to replicate for '%s'" % (name_data['fqu'])) + log.info("No profile to replicate for '%s'" % (name_data['fqu'])) return {'status': True} @classmethod def replicate_names_data( cls, queue_path, updates, wallet_data, storage_drivers, skip=[], config_path=CONFIG_PATH, proxy=None ): """ - Replicate all zonefiles and token files for each confirmed update or name import. + Replicate all zonefiles and profiles for each confirmed update or name import. @atlas_servers should be a list of (host, port) Do NOT remove items from the queue. @@ -537,7 +546,7 @@ class RegistrarWorker(threading.Thread): @classmethod def replicate_update_data( cls, queue_path, wallet_data, storage_drivers, skip=[], config_path=CONFIG_PATH, proxy=None ): """ - Replicate all zone files and token files for each confirmed NAME_UPDATE + Replicate all zone files and profiles for each confirmed NAME_UPDATE @atlas_servers should be a list of (host, port) Do NOT remove items from the queue. @@ -555,7 +564,7 @@ class RegistrarWorker(threading.Thread): @classmethod def replicate_name_import_data( cls, queue_path, wallet_data, storage_drivers, skip=[], config_path=CONFIG_PATH, proxy=None ): """ - Replicate all zone files and token files for each confirmed NAME_UPDATE + Replicate all zone files and profiles for each confirmed NAME_UPDATE @atlas_servers should be a list of (host, port) Do NOT remove items from the queue. @@ -596,6 +605,9 @@ class RegistrarWorker(threading.Thread): if update.get("transfer_address") is not None: # let's see if the name already got there! name_rec = get_name_blockchain_record( update['fqu'], proxy=proxy ) + if 'address' in name_rec: + log.debug("{} updated, current owner : {}, transfer owner : {}".format( + update['fqu'], name_rec['address'], update['transfer_address'])) if 'address' in name_rec and name_rec['address'] == update['transfer_address']: log.debug("Requested Transfer {} to {} is owned by {} already. Declaring victory.".format( update['fqu'], update['transfer_address'], name_rec['address'])) @@ -810,6 +822,7 @@ class RegistrarWorker(threading.Thread): log.debug("Registrar worker starting up") + is_backing_off = False while self.running: failed = False @@ -838,116 +851,104 @@ class RegistrarWorker(threading.Thread): try: # see if we can complete any registrations # clear out any confirmed preorders - log.debug("register all pending preorders in %s" % (self.queue_path)) + # log.debug("register all pending preorders in %s" % (self.queue_path)) res = RegistrarWorker.register_preorders( self.queue_path, wallet_data, config_path=self.config_path, proxy=proxy ) if 'error' in res: log.warn("Registration failed: %s" % res['error']) # try exponential backoff failed = True - poll_interval = 1.0 except Exception, e: log.exception(e) failed = True - poll_interval = 1.0 try: # see if we can put any zonefiles # clear out any confirmed registers - log.debug("put zonefile hashes for registered names in %s" % (self.queue_path)) + # log.debug("put zonefile hashes for registered names in %s" % (self.queue_path)) res = RegistrarWorker.set_zonefiles( self.queue_path, config_path=self.config_path, proxy=proxy ) if 'error' in res: log.warn('zonefile hash broadcast failed: %s' % res['error']) - # try exponential backoff failed = True - poll_interval = 1.0 except Exception, e: log.exception(e) failed = True - poll_interval = 1.0 try: - # see if we can replicate any zonefiles and token files + # see if we can replicate any zonefiles and profiles # clear out any confirmed updates - log.debug("replicate all pending zone files and token files for updates %s" % (self.queue_path)) + # log.debug("replicate all pending zone files and profiles for updates %s" % (self.queue_path)) res = RegistrarWorker.replicate_update_data( self.queue_path, wallet_data, self.required_storage_drivers, config_path=self.config_path, proxy=proxy ) if 'error' in res: - log.warn("Zone file/token file replication failed for update: %s" % res['error']) + log.warn("Zone file/profile replication failed for update: %s" % res['error']) - # try exponential backoff failed = True - poll_interval = 1.0 failed_names += res['names'] except Exception, e: log.exception(e) failed = True - poll_interval = 1.0 try: # see if we can transfer any names to their new owners - log.debug("transfer all names in {}".format(self.queue_path)) + # log.debug("transfer all names in {}".format(self.queue_path)) res = RegistrarWorker.transfer_names( self.queue_path, skip=failed_names, config_path=self.config_path, proxy=proxy ) if 'error' in res: log.warn("Transfer failed: {}".format(res['error'])) - # try exponential backoff failed = True - poll_interval = 1.0 failed_names += res['names'] except Exception as e: log.exception(e) failed = True - poll_interval = 1.0 try: # see if we can replicate any zonefiles for name imports # clear out any confirmed imports - log.debug("replicate all pending zone files for name imports in {}".format(self.queue_path)) + # log.debug("replicate all pending zone files for name imports in {}".format(self.queue_path)) res = RegistrarWorker.replicate_name_import_data( self.queue_path, wallet_data, self.required_storage_drivers, skip=failed_names, config_path=self.config_path, proxy=proxy ) if 'error' in res: log.warn("Zone file replication failed: {}".format(res['error'])) - # try exponential backoff failed = True - poll_interval = 1.0 failed_names += res['names'] except Exception, e: log.exception(e) failed = True - poll_interval = 1.0 try: # see if we can remove any other confirmed operations, besides preorders, registers, and updates - log.debug("clean out other confirmed operations") + # log.debug("clean out other confirmed operations") res = RegistrarWorker.clear_confirmed( self.config_path, self.queue_path, proxy=proxy ) if 'error' in res: log.warn("Failed to clear out some operations: %s" % res['error']) - # try exponential backoff failed = True - poll_interval = 1.0 except Exception, e: log.exception(e) failed = True - poll_interval = 1.0 # if we failed a step, then try again quickly with exponential backoff if failed: - poll_interval = 2 * poll_interval + random.random() * poll_interval - + if is_backing_off: + poll_interval = 2 * poll_interval + random.random() * poll_interval + poll_interval = min( poll_interval, self.poll_interval ) + else: + poll_interval = 1.0 + is_backing_off = True else: # succeeded. resume normal polling poll_interval = self.poll_interval - + is_backing_off = False + try: - log.debug("Sleep for %s" % poll_interval) + log.debug("Registrar sleeping for %s" % poll_interval) for i in xrange(0, int(poll_interval)): time.sleep(1) @@ -1141,7 +1142,8 @@ def get_wallet(config_path=None, proxy=None): return data -def preorder(fqu, cost_satoshis, zonefile_data, token_file, transfer_address, min_payment_confs, proxy=None, config_path=CONFIG_PATH, unsafe_reg=False): +# RPC method: backend_preorder +def preorder(fqu, cost_satoshis, zonefile_data, profile, transfer_address, min_payment_confs, proxy=None, config_path=CONFIG_PATH, unsafe_reg = False): """ Send preorder transaction and enter it in queue. Queue up additional state so we can update and transfer it as well. @@ -1174,7 +1176,7 @@ def preorder(fqu, cost_satoshis, zonefile_data, token_file, transfer_address, mi name_data = { 'transfer_address': transfer_address, 'zonefile': zonefile_data, - 'token_file': token_file, + 'profile': profile, } if min_payment_confs is None: min_payment_confs = TX_MIN_CONFIRMATIONS @@ -1186,8 +1188,16 @@ def preorder(fqu, cost_satoshis, zonefile_data, token_file, transfer_address, mi name_data['confirmations_needed'] = PREORDER_CONFIRMATIONS name_data['unsafe_reg'] = True - log.debug("async_preorder({}, zonefile_data={}, profile={}, transfer_address={})".format(fqu, zonefile_data, token_file, transfer_address)) + # save the current owner_privkey_info, scrypted with our password + passwd = get_secret('BLOCKSTACK_CLIENT_WALLET_PASSWORD') + if passwd: + name_data['owner_privkey'] = aes_encrypt( + str(owner_privkey_info), hexlify( passwd )) + else: + log.warn("Registrar couldn't access wallet password to encrypt privkey," + + " sheepishly refusing to store the private key unencrypted.") + log.debug("async_preorder({}, zonefile_data={}, profile={}, transfer_address={})".format(fqu, zonefile_data, profile, transfer_address)) resp = async_preorder(fqu, payment_privkey_info, owner_privkey_info, cost_satoshis, name_data=name_data, min_payment_confs=min_payment_confs, proxy=proxy, config_path=config_path, queue_path=state.queue_path) @@ -1210,7 +1220,9 @@ def preorder(fqu, cost_satoshis, zonefile_data, token_file, transfer_address, mi return data -def update( fqu, zonefile_txt, token_file, zonefile_hash, transfer_address, config_path=CONFIG_PATH, proxy=None ): +# RPC method: backend_update +def update( fqu, zonefile_txt, profile, zonefile_hash, transfer_address, config_path=CONFIG_PATH, proxy=None, + prior_name_data = None ): """ Send a new zonefile hash. Queue the zonefile data for subsequent replication. zonefile_txt_b64 must be b64-encoded so we can send it over RPC sanely @@ -1243,12 +1255,15 @@ def update( fqu, zonefile_txt, token_file, zonefile_hash, transfer_address, conf if not is_zonefile_hash_current(fqu, zonefile_hash, proxy=proxy ): # new zonefile data - name_data = { - 'transfer_address': transfer_address - } + if prior_name_data is not None: + name_data = dict(prior_name_data) + else: + name_data = {} - log.debug("async_update({}, zonefile_data={}, token_file={}, transfer_address={})".format(fqu, zonefile_txt, token_file, transfer_address)) - resp = async_update(fqu, zonefile_txt, token_file, + name_data['transfer_address'] = transfer_address + + log.debug("async_update({}, zonefile_data={}, profile={}, transfer_address={})".format(fqu, zonefile_txt, profile, transfer_address)) + resp = async_update(fqu, zonefile_txt, profile, owner_privkey_info, payment_privkey_info, name_data=name_data, @@ -1286,7 +1301,7 @@ def update( fqu, zonefile_txt, token_file, zonefile_hash, transfer_address, conf # RPC method: backend_transfer -def transfer(fqu, transfer_address, config_path=CONFIG_PATH, proxy=None ): +def transfer(fqu, transfer_address, prior_name_data = None, config_path=CONFIG_PATH, proxy=None ): """ Send transfer transaction. Keeps the zonefile data. @@ -1311,12 +1326,15 @@ def transfer(fqu, transfer_address, config_path=CONFIG_PATH, proxy=None ): payment_privkey_info = get_wallet_payment_privkey_info(config_path=config_path, proxy=proxy) owner_privkey_info = get_wallet_owner_privkey_info(config_path=config_path, proxy=proxy) + kwargs = {} + if prior_name_data: + kwargs['name_data'] = prior_name_data resp = async_transfer(fqu, transfer_address, owner_privkey_info, payment_privkey_info, proxy=proxy, config_path=config_path, - queue_path=state.queue_path) + queue_path=state.queue_path, **kwargs) if 'error' not in resp: data['success'] = True diff --git a/blockstack_client/backend/utxo/bitcoind_utxo.py b/blockstack_client/backend/utxo/bitcoind_utxo.py index 57e73bf74..c6cbc65c1 100644 --- a/blockstack_client/backend/utxo/bitcoind_utxo.py +++ b/blockstack_client/backend/utxo/bitcoind_utxo.py @@ -23,8 +23,9 @@ import httplib from virtualchain import AuthServiceProxy -from blockchain_client import BlockchainClient +from .blockchain_client import BlockchainClient +from blockstack_client import constants from decimal import Decimal SATOSHIS_PER_COIN = 10**8 @@ -95,6 +96,11 @@ def get_unspents(address, blockchain_client): unspents = bitcoind.listunspent(min_confirmations, max_confirmation, addresses) + if constants.BLOCKSTACK_TESTNET and len(unspents) == 0: + bitcoind.importaddress(str(address)) + unspents = bitcoind.listunspent(min_confirmations, max_confirmation, + addresses) + return format_unspents(unspents) diff --git a/blockstack_client/backend/utxo/blockchain_info.py b/blockstack_client/backend/utxo/blockchain_info.py index 60dea836d..d4b9e5e61 100644 --- a/blockstack_client/backend/utxo/blockchain_info.py +++ b/blockstack_client/backend/utxo/blockchain_info.py @@ -25,7 +25,7 @@ import requests BLOCKCHAIN_API_BASE_URL = "https://blockchain.info" -from blockchain_client import BlockchainClient +from .blockchain_client import BlockchainClient class BlockchainInfoClient(BlockchainClient): def __init__(self, api_key=None, timeout=30, min_confirmations=None): diff --git a/blockstack_client/backend/utxo/blockcypher.py b/blockstack_client/backend/utxo/blockcypher.py index 455a398ac..80f80ec8b 100644 --- a/blockstack_client/backend/utxo/blockcypher.py +++ b/blockstack_client/backend/utxo/blockcypher.py @@ -26,7 +26,7 @@ import requests BLOCKCYPHER_BASE_URL = 'https://api.blockcypher.com/v1/btc/main' -from blockchain_client import BlockchainClient +from .blockchain_client import BlockchainClient class BlockcypherClient(BlockchainClient): diff --git a/blockstack_client/backend/utxo/blockstack_explorer.py b/blockstack_client/backend/utxo/blockstack_explorer.py index e011c8a2e..738a71902 100644 --- a/blockstack_client/backend/utxo/blockstack_explorer.py +++ b/blockstack_client/backend/utxo/blockstack_explorer.py @@ -21,7 +21,7 @@ along with Blockstack-client. If not, see . """ -from insight_api import InsightClient, _get_unspents, _broadcast_transaction +from .insight_api import InsightClient, _get_unspents, _broadcast_transaction BLOCKSTACK_EXPLORER_URL = "https://explorer.blockstack.org" diff --git a/blockstack_client/backend/utxo/blockstack_utxo.py b/blockstack_client/backend/utxo/blockstack_utxo.py index e7fe0bca9..2ab225c7d 100644 --- a/blockstack_client/backend/utxo/blockstack_utxo.py +++ b/blockstack_client/backend/utxo/blockstack_utxo.py @@ -21,7 +21,7 @@ along with Blockstack-client. If not, see . """ -from insight_api import InsightClient, _get_unspents, _broadcast_transaction +from .insight_api import InsightClient, _get_unspents, _broadcast_transaction BLOCKSTACK_UTXO_URL = "https://utxo.blockstack.org" diff --git a/blockstack_client/backend/utxo/insight_api.py b/blockstack_client/backend/utxo/insight_api.py index 963454034..46803dc65 100644 --- a/blockstack_client/backend/utxo/insight_api.py +++ b/blockstack_client/backend/utxo/insight_api.py @@ -25,7 +25,9 @@ import sys import requests import json import traceback +from ...logger import get_logger +log = get_logger("insight-api") class InsightClient(object): def __init__(self, url, min_confirmations=None): @@ -33,15 +35,10 @@ class InsightClient(object): self.url = url self.min_confirmations = min_confirmations - from ...logger import get_logger - self.log = get_logger("insight-api") - - def get_unspents(self, address): - url = self.url + '/insight-api/addr/{}/utxo'.format(address) resp = None - self.log.debug("GET {}".format(url)) + log.debug("GET {}".format(url)) try: req = requests.get(url) resp = req.json() @@ -52,7 +49,7 @@ class InsightClient(object): # format... try: unspents = format_unspents(resp) - self.log.debug("{} has {} UTXOs".format(address, len(unspents))) + log.debug("{} has {} UTXOs".format(address, len(unspents))) return unspents except Exception as e: traceback.print_exc() @@ -69,12 +66,12 @@ class InsightClient(object): req = None resp = None - self.log.debug("POST {}".format(url)) + log.debug("POST {}".format(url)) try: req = requests.post(url, data=data, headers=headers) except Exception as e: - self.log.error("Failed to send transaction") + log.error("Failed to send transaction") raise try: diff --git a/blockstack_client/client.py b/blockstack_client/client.py index 01354d802..839a898d2 100644 --- a/blockstack_client/client.py +++ b/blockstack_client/client.py @@ -95,17 +95,18 @@ def session(conf=None, config_path=CONFIG_PATH, server_host=None, server_port=No proxy = BlockstackRPCClient(server_host, server_port) # load all storage drivers + loaded = [] for storage_driver in storage_drivers.split(','): storage_impl = load_storage(storage_driver) if storage_impl is None: log.error('Failed to load storage driver "{}"'.format(storage_driver)) sys.exit(1) - + loaded.append(storage_driver) rc = register_storage(storage_impl, conf) if not rc: log.error('Failed to initialize storage driver "{}" ({})'.format(storage_driver, rc)) sys.exit(1) - + log.debug('Loaded storage drivers {}'.format(loaded)) # initialize SPV SPVClient.init(spv_headers_path) proxy.spv_headers_path = spv_headers_path @@ -125,10 +126,10 @@ def load_storage(module_name): prefix = 'blockstack_client.backend.drivers.{}' storage_impl = importlib.import_module(prefix.format(module_name)) storage_impl.__name__ = module_name - log.debug('Loaded storage driver "{}"'.format(module_name)) except ImportError as e: msg = ('Failed to import blockstack_client.backend.drivers.{}. ' 'Please verify that it is installed and is accessible via your PYTHONPATH') + log.exception(e) raise Exception(msg.format(module_name)) return storage_impl @@ -257,7 +258,7 @@ def analytics_user_register(u, email, config_path=CONFIG_PATH, proxy=None): return True -def analytics_user_update(payload, proxy=None): +def analytics_user_update(payload, proxy=None, config_path=CONFIG_PATH): """ Update a user's info on the analytics service """ diff --git a/blockstack_client/config.py b/blockstack_client/config.py index b3006a091..e49055abf 100644 --- a/blockstack_client/config.py +++ b/blockstack_client/config.py @@ -41,9 +41,9 @@ from binascii import hexlify from ConfigParser import SafeConfigParser import virtualchain -from utxo import * -from constants import * -from logger import get_logger +from .utxo import * +from .constants import * +from .logger import get_logger log = get_logger('blockstack-client') diff --git a/blockstack_client/constants.py b/blockstack_client/constants.py index 3e63715c7..e5814f6a2 100644 --- a/blockstack_client/constants.py +++ b/blockstack_client/constants.py @@ -32,7 +32,7 @@ import subprocess import fcntl import virtualchain -from version import __version__, __version_major__, __version_minor__, __version_patch__ +from .version import __version__, __version_major__, __version_minor__, __version_patch__ BLOCKSTACK_TEST = os.environ.get('BLOCKSTACK_TEST', None) BLOCKSTACK_TEST_NODEBUG = os.environ.get('BLOCKSTACK_TEST_NODEBUG', None) @@ -86,10 +86,10 @@ WALLET_PASSWORD_LENGTH = 8 WALLET_DECRYPT_MAX_TRIES = 5 WALLET_DECRYPT_BACKOFF_RESET = 3600 -BLOCKSTACK_DEFAULT_STORAGE_DRIVERS = 'disk,s3,blockstack_resolver,blockstack_server,http,dht' +BLOCKSTACK_DEFAULT_STORAGE_DRIVERS = 'disk,dropbox,s3,blockstack_resolver,blockstack_server,http,dht' # storage drivers that must successfully acknowledge each write -BLOCKSTACK_REQUIRED_STORAGE_DRIVERS_WRITE = 'disk,blockstack_server,dht' +BLOCKSTACK_REQUIRED_STORAGE_DRIVERS_WRITE = 'disk,dropbox' BLOCKSTACK_STORAGE_CLASSES = ['read_public', 'read_private', 'write_public', 'write_private', 'read_local', 'write_local'] diff --git a/blockstack_client/data.py b/blockstack_client/data.py index e3f596660..c44c7e016 100644 --- a/blockstack_client/data.py +++ b/blockstack_client/data.py @@ -45,17 +45,17 @@ from keylib import * import virtualchain from virtualchain.lib.ecdsalib import * -from keys import * -from profile import * -from proxy import * -from storage import hash_zonefile -from zonefile import get_name_zonefile, load_name_zonefile, store_name_zonefile -from utils import ScatterGather +from .keys import * +from .profile import * +from .proxy import * +from .storage import hash_zonefile +from .zonefile import get_name_zonefile, load_name_zonefile, store_name_zonefile +from .utils import ScatterGather -from logger import get_logger -from config import get_config, get_local_device_id -from constants import BLOCKSTACK_TEST, BLOCKSTACK_DEBUG, DATASTORE_SIGNING_KEY_INDEX, BLOCKSTACK_STORAGE_PROTO_VERSION, DEFAULT_DEVICE_ID -from schemas import * +from .logger import get_logger +from .config import get_config, get_local_device_id +from .constants import BLOCKSTACK_TEST, BLOCKSTACK_DEBUG, DATASTORE_SIGNING_KEY_INDEX, BLOCKSTACK_STORAGE_PROTO_VERSION, DEFAULT_DEVICE_ID +from .schemas import * log = get_logger() @@ -225,7 +225,7 @@ class DataCache(object): log.debug("Cache HIT header {}, expires at {} (now={})".format(inode_uuid, deadline, time.time())) else: - log.debug("Cache MISS header {}".format(inode_uuid)) + log.debug("Cache MISS {}".format(inode_uuid)) return res @@ -240,7 +240,7 @@ class DataCache(object): log.debug("Cache HIT directory {}, version {}, expires at {} (now={})".format(inode_uuid, res['version'], deadline, time.time())) else: - log.debug("Cache MISS directory {}".format(inode_uuid)) + log.debug("Cache MISS {}".format(inode_uuid)) return res @@ -255,7 +255,7 @@ class DataCache(object): log.debug("Cache HIT datastore {}, expires at {} (now={})".format(datastore_id, deadline, time.time())) else: - log.debug("Cache MISS datastore {}".format(datastore_id)) + log.debug("Cache MISS {}".format(datastore_id)) return res @@ -326,14 +326,6 @@ class DataCache(object): GLOBAL_CACHE = DataCache() -def cache_evict_all(): - """ - Clear the global inode cache - """ - global GLOBAL_CACHE - GLOBAL_CACHE.evict_all() - - def serialize_mutable_data_id(data_id): """ Turn a data ID into a suitable filesystem name @@ -355,7 +347,7 @@ def get_metadata_dir(conf, config_path=CONFIG_PATH): return metadata_dir -def load_mutable_data_version(device_id, fq_data_id, config_path=CONFIG_PATH): +def load_mutable_data_version(conf, device_id, fq_data_id, config_path=CONFIG_PATH): """ Get the version field of a piece of mutable data from local cache. Return the version on success @@ -364,9 +356,11 @@ def load_mutable_data_version(device_id, fq_data_id, config_path=CONFIG_PATH): """ # try to get the current, locally-cached version - conf = get_config(path=config_path) + conf = get_config(path=config_path) if conf is None else conf + if conf is None: - log.error('No config found; cannot load version for "{}"'.format(fq_data_id)) + msg = 'No config found; cannot load version for "{}"' + log.debug(msg.format(fq_data_id)) return None _, data_id = storage.parse_fq_data_id(fq_data_id) @@ -395,7 +389,7 @@ def load_mutable_data_version(device_id, fq_data_id, config_path=CONFIG_PATH): return None -def store_mutable_data_version(device_id, fq_data_id, ver, config_path=CONFIG_PATH): +def store_mutable_data_version(conf, device_id, fq_data_id, ver, config_path=CONFIG_PATH): """ Locally store the version of a piece of mutable data, so we can ensure that its version is incremented on @@ -406,9 +400,11 @@ def store_mutable_data_version(device_id, fq_data_id, ver, config_path=CONFIG_PA Raise on invalid input """ - conf = get_config(path=config_path) + conf = get_config(path=config_path) if conf is None else conf + if conf is None: - log.error('No config found; cannot store version for "{}"'.format(fq_data_id)) + msg = 'No config found; cannot store version for "{}"' + log.warning(msg.format(fq_data_id)) return False metadata_dir = get_metadata_dir(conf) @@ -468,7 +464,7 @@ def store_mutable_data_version(device_id, fq_data_id, ver, config_path=CONFIG_PA return False -def delete_mutable_data_version(device_id, fq_data_id, config_path=CONFIG_PATH): +def delete_mutable_data_version(conf, device_id, fq_data_id, config_path=CONFIG_PATH): """ Locally delete the version of a piece of mutable data. @@ -477,9 +473,10 @@ def delete_mutable_data_version(device_id, fq_data_id, config_path=CONFIG_PATH): Raise on invalid input """ - conf = get_config(path=config_path) + conf = get_config(path=config_path) if conf is None else conf + if conf is None: - log.error('No config found; cannot delete version for "{}"'.format(fq_data_id)) + msg = 'No config found; cannot delete version for "{}"' return False metadata_dir = get_metadata_dir(conf) @@ -506,181 +503,12 @@ def delete_mutable_data_version(device_id, fq_data_id, config_path=CONFIG_PATH): except Exception as e: # failed for whatever reason msg = 'Failed to remove version file "{}"' - log.warn(msg.format(ver_file_path)) + log.warn(msg.format(ver_path)) return False -def set_partial_create_failure(fq_data_id, config_path=CONFIG_PATH): - """ - Indicate that creating this particular mutable datum - (e.g. a datastore, a directory, a file) failed, and that - we should pretend like it doesn't exist (i.e. mask EEXIST) - - Return True on success - Return False on error - """ - conf = get_config(config_path) - if conf is None: - log.error("Failed to load config {}".format(config_path)) - return False - - metadata_dir = get_metadata_dir(conf) - partial_create_dir = os.path.join(metadata_dir, 'create_failures') - if not os.path.isdir(partial_create_dir): - try: - os.makedirs(partial_create_dir) - except: - log.error("Failed to create {}".format(partial_create_dir)) - return False - - # indicate that we failed to create this item - path = os.path.join(partial_create_dir, fq_data_id) - with open(path, 'w') as f: - pass - - log.debug("Remembering that {} is only partially-created".format(fq_data_id)) - return True - - -def has_partial_create_failure(fq_data_id, config_path=CONFIG_PATH): - """ - Did we try and fail to create this datum in the past? - Return True if so - Return False if not - """ - conf = get_config(config_path) - if conf is None: - log.error("Failed to load config {}".format(config_path)) - return False - - metadata_dir = get_metadata_dir(conf) - partial_create_dir = os.path.join(metadata_dir, 'create_failures') - if not os.path.isdir(partial_create_dir): - return False - - path = os.path.join(partial_create_dir, fq_data_id) - return os.path.exists(path) - - -def clear_partial_create_failure(fq_data_id, config_path=CONFIG_PATH): - """ - Indicate that we succeeded to create this particular mutable datum - - Return True on success - Return False on error - """ - conf = get_config(config_path) - if conf is None: - log.error("Failed to load config {}".format(config_path)) - return False - - metadata_dir = get_metadata_dir(conf) - partial_create_dir = os.path.join(metadata_dir, 'create_failures') - if not os.path.isdir(partial_create_dir): - return True - - path = os.path.join(partial_create_dir, fq_data_id) - if not os.path.exists(path): - return True - - try: - os.unlink(path) - return True - except: - log.error("Failed to remove {}".format(path)) - return False - - -def set_partial_delete_failure(path, fq_data_id, config_path=CONFIG_PATH): - """ - Indicate that deleting this particular mutable adtum - (e.g. a datastore, a directory, a file) failed, and that - we should pretend like it still exists (i.e. mask ENOENT) - """ - conf = get_config(config_path) - if conf is None: - log.error("Failed to load config {}".format(config_path)) - return False - - metadata_dir = get_metadata_dir(conf) - partial_delete_dir = os.path.join(metadata_dir, 'delete_failures') - if not os.path.isdir(partial_delete_dir): - try: - os.makedirs(partial_delete_dir) - except: - log.error("Failed to create {}".format(partial_delete_dir)) - return False - - # indicate that we failed to delete this item - path = path.replace('/', '\\x2f') - storage_path = os.path.join(partial_delete_dir, path) - with open(storage_path, 'w') as f: - f.write(fq_data_id) - - return True - - -def has_partial_delete_failure(path, config_path=CONFIG_PATH): - """ - Did we try and fail to delete this datum in the past? - Return the fully-qualified data ID of the failed path if so - Return None if not - """ - conf = get_config(config_path) - if conf is None: - log.error("Failed to load config {}".format(config_path)) - return None - - metadata_dir = get_metadata_dir(conf) - partial_delete_dir = os.path.join(metadata_dir, 'delete_failures') - if not os.path.isdir(partial_delete_dir): - return None - - path = path.replace('/', '\\x2f') - storage_path = os.path.join(partial_delete_dir, path) - fq_data_id = None - with open(storage_path, 'r') as f: - fq_data_id = f.read() - - return fq_data_id - - -def clear_partial_delete_failure(path, config_path=CONFIG_PATH): - """ - Indicate that we succeeded to delete this particular mutable datum - - Return True on success - Return False on error - """ - conf = get_config(config_path) - if conf is None: - log.error("Failed to load config {}".format(config_path)) - return False - - metadata_dir = get_metadata_dir(conf) - partial_delete_dir = os.path.join(metadata_dir, 'delete_failures') - if not os.path.isdir(partial_delete_dir): - return True - - path = path.replace('/', '\\x2f') - storage_path = os.path.join(partial_delete_dir, path) - if not os.path.exists(storage_path): - return True - - try: - os.unlink(storage_path) - return True - except: - log.error("Failed to remove {}".format(path)) - return False - - def is_obsolete_zonefile(user_zonefile): - """ - Is this zone file in a legacy format? - Return True if so - """ return ( blockstack_profiles.is_profile_in_legacy_format(user_zonefile) or not user_db.is_user_zonefile(user_zonefile) @@ -723,8 +551,8 @@ def get_immutable(name, data_hash, data_id=None, config_path=CONFIG_PATH, proxy= # this tool doesn't allow this to happen (one ID matches # one hash), but that doesn't preclude the user from doing # this with other tools. - if data_hash is not None and data_hash not in h: - return {'error': 'Data ID/hash mismatch: {} not in {} (possibly due to invalid zonefile)'.format(data_hash, h)} + if data_hash is not None and data_hash not in hs: + return {'error': 'Data ID/hash mismatch: {} not in {} (possibly due to invalid zonefile)'.format(data_hash, hs)} else: msg = 'Multiple matches for "{}": {}' return {'error': msg.format(data_id, ','.join(h))} @@ -828,7 +656,7 @@ def list_zonefile_history(name, current_block=None, proxy=None, return_hashes = zonefiles = [] for zh in zonefile_hashes: - zonefile = load_name_zonefile(name, zh) + zonefile = load_name_zonefile(name, zh, raw_zonefile=True) if zonefile is None: zonefile = {'error': 'Failed to load zonefile {}'.format(zh)} else: @@ -897,7 +725,7 @@ def list_immutable_data_history(name, data_id, current_block=None, proxy=None): return hashes -def load_user_data_pubkey_addr( name, proxy=None, config_path=CONFIG_PATH ): +def load_user_data_pubkey_addr( name, storage_drivers=None, proxy=None, config_path=CONFIG_PATH ): """ Get a user's default data public key and/or owner address by getting it's zone file. @@ -905,7 +733,7 @@ def load_user_data_pubkey_addr( name, proxy=None, config_path=CONFIG_PATH ): Return {'error': ...} on error """ # need to find pubkey to use - user_zonefile = get_name_zonefile(name, proxy=proxy) + user_zonefile = get_name_zonefile( name, storage_drivers=storage_drivers, proxy=proxy, include_name_record=True) if 'error' in user_zonefile: log.debug("Unable to load zone file for '{}': {}".format(name, user_zonefile['error'])) return {'error': 'Failed to load zonefile'} @@ -967,9 +795,9 @@ def data_blob_sign( data_blob_str, data_privkey ): return sig -def get_mutable(data_id, device_ids, raw=False, data_pubkeys=None, device_data_pubkeys=None, data_addresses=None, data_hash=None, storage_drivers=None, +def get_mutable(data_id, device_ids, raw=False, blockchain_id=None, data_pubkey=None, data_address=None, data_hash=None, storage_drivers=None, proxy=None, ver_min=None, ver_max=None, force=False, urls=None, is_fq_data_id=False, - config_path=CONFIG_PATH, blockchain_id=None): + config_path=CONFIG_PATH): """ get_mutable @@ -978,12 +806,10 @@ def get_mutable(data_id, device_ids, raw=False, data_pubkeys=None, device_data_p If @ver_min is given, ensure the data's version is greater or equal to it. If @ver_max is given, ensure the data's version is less than it. - Verification order: - The data will be verified against the device-specific public key in device_data_pubkeys, if given. - The data will be verified against *any* public key in data_pubkeys, if given. - The data will be verified against *any* data address in data_addresses, if given. - - Return {'data': the data, 'version': the version, 'timestamp': ..., 'drivers': [driver name]} on success + If data_pubkey or data_address is given, then blockchain_id will be ignored (but it will be passed as a hint to the drivers) + If data_hash is given, then all three will be ignored + + Return {'data': the data, 'version': the version, 'timestamp': ..., 'data_pubkey': ..., 'owner_pubkey_hash': ..., 'drivers': [driver name]} on success If raw=True, then only return {'data': ..., 'drivers': ...} on success. Return {'error': ...} on error @@ -992,8 +818,6 @@ def get_mutable(data_id, device_ids, raw=False, data_pubkeys=None, device_data_p proxy = get_default_proxy(config_path) if proxy is None else proxy conf = get_config(path=config_path) - device_pubkeys = {} - # find all possible fqids for this datum fq_data_ids = [] if is_fq_data_id: @@ -1001,15 +825,27 @@ def get_mutable(data_id, device_ids, raw=False, data_pubkeys=None, device_data_p else: for device_id in device_ids: - fq_data_id = storage.make_fq_data_id(device_id, data_id) - fq_data_ids.append( fq_data_id ) + fq_data_ids.append( storage.make_fq_data_id(device_id, data_id) ) + + lookup = False + if data_address is None and data_pubkey is None and data_hash is None: + # TODO: cut this code path + if blockchain_id is None: + raise ValueError("No data public key, data address, or blockchain ID given") - if device_data_pubkeys and device_data_pubkeys.has_key(device_id): - device_pubkeys[fq_data_id] = device_data_pubkeys[device_id] - - # must have raw=True if we don't have public keys or addresses - if data_pubkeys is None and data_addresses is None: - assert raw, "No data public keys or public key hashes are given" + # need to find pubkey to use + pubkey_info = load_user_data_pubkey_addr( blockchain_id, storage_drivers=storage_drivers, proxy=proxy, config_path=config_path ) + if 'error' in pubkey_info: + return pubkey_info + + data_pubkey = pubkey_info['pubkey'] + data_address = pubkey_info['address'] + + if data_pubkey is None and data_address is None: + log.error("No data public key or address available") + return {'error': 'No data public key or address available'} + + lookup = True if storage_drivers is None: storage_drivers = get_read_storage_drivers(config_path) @@ -1022,8 +858,8 @@ def get_mutable(data_id, device_ids, raw=False, data_pubkeys=None, device_data_p version_info = get_mutable_data_version(data_id, device_ids, config_path=config_path) expected_version = version_info['version'] - log.debug("get_mutable({}, device_ids={}, blockchain_id={}, pubkeys={}, addrs={}, hash={}, expected_version={}, storage_drivers={})".format( - data_id, device_ids, blockchain_id, data_pubkeys, data_addresses, data_hash, expected_version, ','.join(storage_drivers) + log.debug("get_mutable({}, device_ids={}, blockchain_id={}, pubkey={} ({}), addr={}, hash={}, expected_version={}, storage_drivers={})".format( + data_id, device_ids, blockchain_id, data_pubkey, lookup, data_address, data_hash, expected_version, ','.join(storage_drivers) )) mutable_data = None @@ -1039,20 +875,10 @@ def get_mutable(data_id, device_ids, raw=False, data_pubkeys=None, device_data_p for fq_data_id in fq_data_ids: log.debug("get_mutable_data({}) from {}".format(fq_data_id, driver)) - - # prioritize a device-specific public key, if given - device_pubkey = device_pubkeys.get(fq_data_id) - extra_pubkeys = [] - - if device_pubkey: - extra_pubkeys = [device_pubkey] - - if data_pubkeys is None: - data_pubkeys = [] # get the mutable data itsef # NOTE: we only use 'bsk2' data formats; use storage.get_mutable_data() directly for loading things like profiles that have a different format. - data_str = storage.get_mutable_data(fq_data_id, extra_pubkeys + data_pubkeys, urls=urls, drivers=[driver], data_addresses=data_addresses, data_hash=data_hash, bsk_version=2, fqu=blockchain_id) + data_str = storage.get_mutable_data(fq_data_id, data_pubkey, urls=urls, drivers=[driver], data_address=data_address, data_hash=data_hash, blockchain_id=blockchain_id, bsk_version=2) if data_str is None: log.error("Failed to get mutable datum {} from {}".format(fq_data_id, driver)) continue @@ -1136,6 +962,9 @@ def get_mutable(data_id, device_ids, raw=False, data_pubkeys=None, device_data_p 'version': version, 'timestamp': mutable_data['timestamp'], 'fq_data_id': mutable_data['fq_data_id'], + + 'data_pubkey': data_pubkey, + 'owner_pubkey_hash': data_address, 'drivers': mutable_drivers } @@ -1193,7 +1022,10 @@ def put_immutable(blockchain_id, data_id, data_text, data_url=None, txid=None, p if not rc: return {'error': 'Failed to overwrite old immutable data'} - rc = user_db.put_immutable_data_zonefile(user_zonefile, data_id, data_hash, data_url=data_url) + rc = user_db.put_immutable_data_zonefile( + user_zonefile, data_id, data_hash, data_url=data_url + ) + if not rc: return {'error': 'Failed to insert immutable data into user zonefile'} @@ -1201,8 +1033,13 @@ def put_immutable(blockchain_id, data_id, data_text, data_url=None, txid=None, p # update zonefile, if we haven't already if txid is None: - payment_privkey_info = get_payment_privkey_info(wallet_keys=wallet_keys, config_path=proxy.conf['path']) - owner_privkey_info = get_owner_privkey_info(wallet_keys=wallet_keys, config_path=proxy.conf['path']) + payment_privkey_info = get_payment_privkey_info( + wallet_keys=wallet_keys, config_path=proxy.conf['path'] + ) + + owner_privkey_info = get_owner_privkey_info( + wallet_keys=wallet_keys, config_path=proxy.conf['path'] + ) user_zonefile_txt = blockstack_zones.make_zone_file(user_zonefile) @@ -1244,6 +1081,35 @@ def put_immutable(blockchain_id, data_id, data_text, data_url=None, txid=None, p return result +def load_user_data_privkey( blockchain_id, storage_drivers=None, proxy=None, config_path=CONFIG_PATH, wallet_keys=None ): + """ + Get the user's data private key from his/her wallet. + Verify it matches the zone file for this blockchain ID + + Return {'privkey': ...} on success + Return {'error': ...} on error + """ + conf = get_config(path=CONFIG_PATH) + user_zonefile = get_name_zonefile( blockchain_id, storage_drivers=storage_drivers, proxy=proxy) + if 'error' in user_zonefile: + log.debug("Unable to load zone file for '{}': {}".format(blockchain_id, user_zonefile['error'])) + return {'error': 'Failed to load zonefile'} + + # recover name record and zonefile + user_zonefile = user_zonefile['zonefile'] + + # get the data key + data_privkey = get_data_privkey_info(user_zonefile, wallet_keys=wallet_keys, config_path=config_path) + if json_is_error(data_privkey): + # error text + return {'error': data_privkey['error']} + + else: + assert data_privkey is not None + + return {'privkey': data_privkey} + + def make_mutable_data_tombstones( device_ids, data_id ): """ Make tombstones for mutable data across devices @@ -1391,6 +1257,8 @@ def put_mutable(fq_data_id, mutable_data_str, data_pubkey, data_signature, versi """ proxy = get_default_proxy(config_path) if proxy is None else proxy + conf = get_config(path=config_path) + assert conf # NOTE: this will be None if the fq_data_id refers to a profile device_id, _ = storage.parse_fq_data_id(fq_data_id) @@ -1413,8 +1281,8 @@ def put_mutable(fq_data_id, mutable_data_str, data_pubkey, data_signature, versi assert data_pubkey assert data_signature - rc = storage.put_mutable_data(fq_data_id, mutable_data_str, data_pubkey=data_pubkey, data_signature=data_signature, raw=raw, - required=storage_drivers, required_exclusive=storage_drivers_exclusive, fqu=blockchain_id) + rc = storage.put_mutable_data(fq_data_id, mutable_data_str, data_pubkey=data_pubkey, data_signature=data_signature, sign=False, raw=raw, blockchain_id=blockchain_id, + required=storage_drivers, required_exclusive=storage_drivers_exclusive) if not rc: log.error("failed to put mutable data {}".format(fq_data_id)) @@ -1422,7 +1290,7 @@ def put_mutable(fq_data_id, mutable_data_str, data_pubkey, data_signature, versi return result # remember which version this was - rc = store_mutable_data_version(device_id, fq_data_id, version, config_path=config_path) + rc = store_mutable_data_version(conf, device_id, fq_data_id, version, config_path=config_path) if not rc: log.error("failed to put mutable data version {}.{}".format(fq_data_id, version)) result['error'] = 'Failed to store mutable data version' @@ -1448,7 +1316,7 @@ def delete_immutable(blockchain_id, data_key, data_id=None, proxy=None, txid=Non proxy = get_default_proxy(config_path) if proxy is None else proxy - user_zonefile = get_name_zonefile(blockchain_id, proxy=proxy) + user_zonefile = get_name_zonefile(blockchain_id, proxy=proxy, include_name_record=True) if 'error' in user_zonefile: log.debug("Unable to load zone file for '{}': {}".format(blockchain_id, user_zonefile['error'])) return user_zonefile @@ -1489,8 +1357,13 @@ def delete_immutable(blockchain_id, data_key, data_id=None, proxy=None, txid=Non if txid is None: # actually send the transaction - payment_privkey_info = get_payment_privkey_info(wallet_keys=wallet_keys, config_path=proxy.conf['path']) - owner_privkey_info = get_owner_privkey_info(wallet_keys=wallet_keys, config_path=proxy.conf['path']) + payment_privkey_info = get_payment_privkey_info( + wallet_keys=wallet_keys, config_path=proxy.conf['path'] + ) + + owner_privkey_info = get_owner_privkey_info( + wallet_keys=wallet_keys, config_path=proxy.conf['path'] + ) user_zonefile_txt = blockstack_zones.make_zone_file(user_zonefile) @@ -1592,7 +1465,7 @@ def delete_mutable(data_id, signed_data_tombstones, proxy=None, storage_drivers= # only do this if we actually succeeded in deleting from all storage providers for device_id in device_ids: for fq_data_id in fq_data_ids: - delete_mutable_data_version(device_id, fq_data_id, config_path=config_path) + delete_mutable_data_version(conf, device_id, fq_data_id, config_path=config_path) if not worst_rc: return {'error': 'Failed to delete from all storage providers'} @@ -1774,7 +1647,7 @@ def get_datastore( blockchain_id, datastore_id, device_ids, config_path=CONFIG_P cache_ttl = int(conf.get('cache_ttl', 3600)) # 1 hour data_id = '{}.datastore'.format(datastore_id) - datastore_info = get_mutable(data_id, device_ids, blockchain_id=blockchain_id, data_addresses=[datastore_id], proxy=proxy, config_path=config_path) + datastore_info = get_mutable(data_id, device_ids, blockchain_id=blockchain_id, data_address=datastore_id, proxy=proxy, config_path=config_path) if 'error' in datastore_info: log.error("Failed to load public datastore information: {}".format(datastore_info['error'])) return {'error': 'Failed to load public datastore record', 'errno': errno.ENOENT} @@ -2221,7 +2094,7 @@ def get_inode_data(blockchain_id, datastore_id, inode_uuid, inode_type, drivers, # must match owner # data_address = keylib.public_key_to_address(data_pubkey_hex) if full_inode['owner'] != datastore_id: # data_address: - log.error("Inode {} not owned by {} (but by {})".format(full_inode['uuid'], data_address, full_inode['owner'])) + log.error("Inode {} not owned by {} (but by {})".format(full_inode['uuid'], datastore_id, full_inode['owner'])) return {'error': 'Invalid owner'} # preserve reader pubkeys @@ -2248,10 +2121,12 @@ def get_mutable_data_version( data_id, device_ids, config_path=CONFIG_PATH ): Return {'status': True, 'version': version} on success """ new_version = 0 + conf = get_config(config_path) + assert conf for device_id in device_ids: fq_data_id = storage.make_fq_data_id(device_id, data_id) - cur_ver = load_mutable_data_version(device_id, fq_data_id, config_path=config_path) + cur_ver = load_mutable_data_version(conf, device_id, fq_data_id, config_path=config_path) if cur_ver is not None: new_version = max(new_version, cur_ver) @@ -2266,12 +2141,15 @@ def put_mutable_data_version( data_id, new_version, device_ids, config_path=CONF """ # advance header version and inode version + conf = get_config(config_path) + assert conf + res = get_mutable_data_version(data_id, device_ids, config_path=CONFIG_PATH) new_version = max(res['version'], new_version) for device_id in device_ids: fq_data_id = storage.make_fq_data_id(device_id, data_id) - rc = store_mutable_data_version(device_id, fq_data_id, new_version, config_path=config_path) + rc = store_mutable_data_version(conf, device_id, fq_data_id, new_version, config_path=config_path) if not rc: return {'error': 'Failed to advance mutable data version {} to {}'.format(data_id, new_version), 'errno': errno.EIO} @@ -2372,7 +2250,7 @@ def get_inode_header(blockchain_id, datastore_id, inode_uuid, drivers, data_pubk device_id = data_pubkey_info['device_id'] device_pubkey = data_pubkey_info['public_key'] - res = get_mutable(data_id, [device_id], blockchain_id=blockchain_id, ver_min=ver_min, force=force, data_pubkeys=[device_pubkey], storage_drivers=[driver], proxy=proxy, config_path=config_path) + res = get_mutable(data_id, [device_id], blockchain_id=blockchain_id, ver_min=ver_min, force=force, data_pubkey=device_pubkey, storage_drivers=[driver], proxy=proxy, config_path=config_path) if 'error' in res: log.error("Failed to get inode data {} (stale={}): {}".format(inode_uuid, res.get('stale', False), res['error'])) errcode = errno.EREMOTEIO @@ -2655,17 +2533,17 @@ def put_inode_data( datastore, header_blob_str, header_blob_sig, idata_str, conf sg = ScatterGather() driver_save_idata = functools.partial(save_idata, driver) driver_save_header = functools.partial(save_header, driver) - sg.add_task( "save_idata_{}_{}".format(idata_fqid, driver), driver_save_idata) - sg.add_task( "save_header_{}_{}".format(header_fqid, driver), driver_save_header) + sg.add_task( "save_idata", driver_save_idata) + sg.add_task( "save_header", driver_save_header) sg.run_tasks() - res = sg.get_result('save_idata_{}_{}'.format(idata_fqid, driver)) + res = sg.get_result('save_idata') if 'error' in res: log.error('Fastpath: Failed to replicate inode data {}: {}'.format(idata_fqid, res['error'])) return {'error': 'Failed to replicate inode data {}'.format(idata_fqid)} - res = sg.get_result('save_header_{}_{}'.format(header_fqid, driver)) + res = sg.get_result('save_header') if 'error' in res: log.error('Fastpath: Failed to replicate inode header {}: {}'.format(header_fqid, res['error'])) return {'error': 'Failed to replicate inode header {}'.format(header_fqid)} @@ -2713,6 +2591,26 @@ def put_inode_data( datastore, header_blob_str, header_blob_sig, idata_str, conf log.debug("Driver {} succeeded to replicate".format(dname)) driver_succeeded = True + ''' + driver_failed = False + for driver in drivers: + + # TODO: can we do this in parallel along a "fast path", and then try them individually on a "slow path" if one of them fails? + + # store payload (no signature; we'll use the header's hash) + res = put_mutable(idata_fqid, idata_str, None, None, version, raw=True, storage_drivers=[driver], storage_drivers_exclusive=True, config_path=config_path, proxy=proxy ) + if 'error' in res: + log.error("Failed to replicate inode {}: {}".format(idata_fqid, res['error'])) + driver_failed = True + continue + + # store header + res = put_mutable(header_fqid, header_blob_str, datastore['pubkey'], header_blob_sig, version, storage_drivers=[driver], storage_drivers_exclusive=True, config_path=config_path, proxy=proxy ) + if 'error' in res: + log.error("Failed to replicate inode header for {}: {}".format(header_fqid, res['error'])) + driver_failed = True + ''' + if driver_succeeded: # at least one write succeeded; make sure the next read loads fresh data # evict @@ -2798,12 +2696,14 @@ def delete_inode_data( datastore, signed_tombstones, proxy=None, config_path=CON else: inode_tombstones[inode_uuid]['idata_tombstones'].append(ts) + failed_driver = False + # evict for inode_uuid in inode_tombstones.keys(): GLOBAL_CACHE.evict_inode(datastore_id, inode_uuid) - - def _delete_inode_payload(inode_uuid): + # delete inode idata first + for inode_uuid in inode_tombstones.keys(): # delete inode data_id = '{}.{}'.format(datastore_id, inode_uuid) @@ -2812,10 +2712,13 @@ def delete_inode_data( datastore, signed_tombstones, proxy=None, config_path=CON if 'error' in res: log.error("Failed to delete inode {}: {}".format(inode_uuid, res['error'])) + failed_driver = True - return res + if failed_driver: + return {'error': 'Failed to delete inode data', 'errno': errno.EREMOTEIO} - def _delete_inode_header(inode_uuid): + # delete inode headers once all idata is gone + for inode_uuid in inode_tombstones.keys(): hdata_id = '{}.{}.hdr'.format(datastore_id, inode_uuid) res = delete_mutable(hdata_id, inode_tombstones[inode_uuid]['header_tombstones'], proxy=proxy, storage_drivers=drivers, storage_drivers_exclusive=True, device_ids=device_ids, config_path=config_path) @@ -2824,50 +2727,12 @@ def delete_inode_data( datastore, signed_tombstones, proxy=None, config_path=CON log.error("Faled to delete idata for {}: {}".format(inode_uuid, res['error'])) return res - return res - - sg = ScatterGather() - for inode_uuid in inode_tombstones.keys(): - delete_inode_header = functools.partial(_delete_inode_header, inode_uuid) - delete_inode_payload = functools.partial(_delete_inode_payload, inode_uuid) - - sg.add_task('delete_inode_header({})'.format(inode_uuid), delete_inode_header) - sg.add_task('delete_inode_payload({})'.format(inode_uuid), delete_inode_payload) - - sg.run_tasks() - - failure = False - - for inode_uuid in inode_tombstones.keys(): - res = sg.get_result('delete_inode_header({})'.format(inode_uuid)) - if 'error' in res: - log.error("Failed to delete inode header for {}: {}".format(inode_uuid, res['error'])) - failure = True - - res = sg.get_result('delete_inode_payload({})'.format(inode_uuid)) - if 'error' in res: - log.error("Failed to delete inode payload for {}: {}".format(inode_uuid, res['error'])) - failure = True - - # evict (again, to be sure) + # evict (again) for inode_uuid in inode_tombstones.keys(): GLOBAL_CACHE.evict_inode(datastore_id, inode_uuid) - if failure: - return {'error': 'Failed to delete some inode data'} + return {'status': True} - else: - return {'status': True} - - -def get_inode_header_fqids( datastore_id, inode_uuid, data_pubkeys, config_path=CONFIG_PATH): - """ - Get the fully-qualified data IDs for an inode header - """ - # get latest inode and inode header version - device_ids = [dk['device_id'] for dk in data_pubkeys] - data_id = '{}.{}.hdr'.format(datastore_id, inode_uuid) - return [storage.make_fq_data_id(dev_id, data_id) for dev_id in device_ids] def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata=True, force=False, config_path=CONFIG_PATH, proxy=None ): @@ -2885,10 +2750,7 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= '/': {'name': '', 'uuid': ...., 'parent': '', 'inode': directory}, '/a': {'name': 'a', 'uuid': ..., 'parent': '/', 'inode': directory}, '/a/b': {'name': 'b', 'uuid': ..., 'parent': '/a', 'inode': directory}, - '/a/b/c': {'name': 'c', 'uuid': ..., 'parent': '/a/b', 'inode': file or directory} - 'hints': { - 'children_absent': [...] - } + '/a/b/c': {'name': 'c', 'uuid': ..., 'parent': '/a/b', 'inode' file} } Return {'error': ..., 'errno': ...} on error @@ -2899,7 +2761,7 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= log.debug("Resolve {}".format(path)) - def _make_path_entry( name, child_uuid, child_entry, prefix ): + def _make_path_entry( name, child_uuid, child_entry, prefix ): """ Make a path entry to return """ @@ -2914,42 +2776,6 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= return path_ent - - def _is_inode_created( inode_uuid ): - """ - Is the inode actually created? That is, did we - fully create it, instead of partially - """ - fq_data_ids = get_inode_header_fqids(datastore_id, inode_uuid, data_pubkeys, config_path=config_path) - for fq_data_id in fq_data_ids: - if has_partial_create_failure(fq_data_id): - return False - - return True - - - def _hints_find_absent_children( ret ): - """ - Insert hints for absent children - """ - child_entry = ret['/' + path]['inode'] - - # if this is a directory, then also return the list of non-existent children - children_absent = [] - if child_entry['type'] == MUTABLE_DATUM_DIR_TYPE and child_entry.has_key('idata'): - for child_name in child_entry['idata']['children'].keys(): - child_uuid = child_entry['idata']['children'][child_name]['uuid'] - if not _is_inode_created(child_uuid): - log.warning("Child '{}' ({}) of {} is partially created".format(child_name, child_uuid, prefix)) - children_absent.append(child_name) - - ret['hints'] = { - 'children_absent': children_absent - } - - return ret - - path = posixpath.normpath(path).strip("/") path_parts = path.split('/') prefix = '/' @@ -2959,11 +2785,6 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= device_ids = datastore['device_ids'] root_uuid = datastore['root_uuid'] - if not _is_inode_created(root_uuid): - # didn't fully create root - log.error("Root inode {} is only partially created".format(root_uuid)) - return {'error': 'Inode is only partially created', 'errno': errno.ENOENT} - # getting only the root? root_inode = get_inode_data(blockchain_id, datastore_id, root_uuid, MUTABLE_DATUM_DIR_TYPE, drivers, data_pubkeys, force=force, config_path=CONFIG_PATH, proxy=proxy) if 'error' in root_inode: @@ -2976,7 +2797,6 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= if len(path) == 0: # looked up / - ret = _hints_find_absent_children(ret) return ret # walk @@ -3009,11 +2829,6 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= # done searching, and don't want data break - if not _is_inode_created(child_uuid): - # this inode actually doesn't exist - log.error("Failed to get inode {}: not fully created".format(child_uuid)) - return {'error': 'Inode is only partially created', 'errno': errno.ENOENT} - # get child, and only get the idata if it's a directory log.debug("Get {} at '{}'".format(child_uuid, '/' + '/'.join(path_parts[:i+1]))) child_entry = get_inode_data(blockchain_id, datastore_id, child_uuid, child_type, drivers, data_pubkeys, force=force, config_path=CONFIG_PATH, proxy=proxy, file_idata=False) @@ -3023,7 +2838,7 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= last_header_info = child_entry child_entry = child_entry['inode'] - assert child_entry['type'] == child_dirent['type'], "Corrupt inode {}".format(child_uuid) + assert child_entry['type'] == child_dirent['type'], "Corrupt inode {}".format(storage.make_fq_data_id(datastore_id,child_uuid)) path_ent = _make_path_entry(name, child_uuid, child_entry, prefix) ret[prefix + name] = path_ent @@ -3046,15 +2861,9 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= return {'error': 'Not a directory', 'errno': errno.ENOTDIR} if child_type == MUTABLE_DATUM_DIR_TYPE or (get_idata and child_type == MUTABLE_DATUM_FILE_TYPE): - # get file data too (always get directory data) + # get file data too # NOTE: last_header_info will be the return value from the last call to get_inode_data() assert ret.has_key(prefix + name), "BUG: missing {}".format(prefix + name) - - if not _is_inode_created(child_uuid): - # this inode actually doesn't exist - log.error("Failed to get inode {}: not fully created".format(child_uuid)) - return {'error': 'Inode is only partially created', 'errno': errno.ENOENT} - child_entry = get_inode_data(blockchain_id, datastore_id, child_uuid, child_type, drivers, data_pubkeys, force=force, config_path=CONFIG_PATH, proxy=proxy, header_info=last_header_info ) else: @@ -3062,11 +2871,6 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= # didn't request idata, so add a path entry here assert not ret.has_key(prefix + name), "BUG: already defined {}".format(prefix + name) - if not _is_inode_created(child_uuid): - # this inode actually doesn't exist - log.error("Failed to get inode {}: not fully created".format(child_uuid)) - return {'error': 'Inode is only partially created', 'errno': errno.ENOENT} - path_ent = _make_path_entry(name, child_uuid, child_entry, prefix) ret[prefix + name] = path_ent @@ -3080,9 +2884,6 @@ def inode_resolve_path( blockchain_id, datastore, path, data_pubkeys, get_idata= # update ret ret[prefix + name]['inode'] = child_entry - - # add hints - ret = _hints_find_absent_children(ret) log.debug("Resolved /{}".format(path)) return ret @@ -3179,7 +2980,6 @@ def _parse_data_path( data_path ): def inode_path_lookup(blockchain_id, datastore, data_path, data_pubkeys, get_idata=True, force=False, config_path=CONFIG_PATH, proxy=None ): """ Look up all the inodes along the given fully-qualified path, verifying them and ensuring that they're fresh along the way. - If get_idata is True and the path refers to a file, return the file data as well (directory data is always returned) This is a server-side method. @@ -3206,12 +3006,7 @@ def inode_path_lookup(blockchain_id, datastore, data_path, data_pubkeys, get_ida assert data_path in path_info.keys(), "Invalid path data, missing {}:\n{}".format(data_path, json.dumps(path_info, indent=4, sort_keys=True)) inode_info = path_info[data_path] - hints = {} - if 'hints' in path_info: - hints = path_info['hints'] - del path_info['hints'] - - return {'status': True, 'path_info': path_info, 'inode_info': inode_info, 'hints': hints} + return {'status': True, 'path_info': path_info, 'inode_info': inode_info} def datastore_inodes_check_consistent( datastore_id, inode_headers, creates, exists, device_ids, config_path=CONFIG_PATH ): @@ -3358,9 +3153,7 @@ def datastore_do_inode_operation( datastore, inode_headers, inode_payloads, inod Do not call this method directly. Call the op-specific helper methods instead. Return {'status': True} on success - Return {'error': ..., 'inodes': [...], 'tombstones': True/False, 'errno': ...} on error, where - * inodes[i] is set to True if the the ith inode header/payload/signature failed - * tombstones is set to True if we failed to delete data + Return {'error': ...} on error """ if proxy is None: @@ -3369,79 +3162,24 @@ def datastore_do_inode_operation( datastore, inode_headers, inode_payloads, inod assert len(inode_headers) == len(inode_payloads) assert len(inode_payloads) == len(inode_signatures) - # delete data - def _delete_data(): - if len(inode_tombstones) > 0: - res = delete_inode_data( datastore, inode_tombstones, proxy=proxy, config_path=config_path) - if 'error' in res: - log.error("Failed to delete inode with {}".format(','.join(inode_tombstones))) - return res + # process tombstones first + if len(inode_tombstones) > 0: + res = delete_inode_data( datastore, inode_tombstones, proxy=proxy, config_path=config_path) + if 'error' in res: + log.debug("Failed to delete inode with {}".format(','.join(inode_tombstones))) + return res - return {'status': True} - - # store one inode payload - def _store_data(i): + # store data + for i in xrange(0, len(inode_headers)): header_blob = inode_headers[i] payload = inode_payloads[i] signature = inode_signatures[i] res = put_inode_data( datastore, header_blob, signature, payload, config_path=config_path, proxy=proxy) if 'error' in res: - log.error("Failed to put inode {}".format(header_blob)) + log.debug("Failed to put inode {}".format(header_blob)) return res - return res - - # process new data and tombstones in parallel - sg = ScatterGather() - sg.add_task("delete_data", _delete_data) - - for i in xrange(0, len(inode_headers)): - add_data = functools.partial(_store_data, i); - sg.add_task("store_data({})".format(i), add_data) - - # do everything - log.debug("Running operation tasks...") - sg.run_tasks() - log.debug("Finished running operation tasks") - - have_failure = False - failure_errno = None - failures = { - 'tombstones': False, - 'inodes': [False] * len(inode_headers), - } - - # did the delete work? - res = sg.get_result('delete_data') - if 'error' in res: - log.error("Task 'delete_data' failed: Failed to delete inode with {}".format(', '.join(inode_tombstones))) - - have_failure = True - failures['tombstones'] = True - failure_errno = res.get('errno') - - # did the writes work? - for i in xrange(0, len(inode_headers)): - res = sg.get_result("store_data({})".format(i)) - if 'error' in res: - log.error("Task 'store_data({})' failed: Failed to put inode {}".format(i, inode_headers[i])) - - failures['inodes'][i] = True - have_failure = True - - if failure_errno is None: - failure_errno = res.get('errno') - - if have_failure: - failures['error'] = 'Some data failed to replicate' - - if failure_errno is None: - failure_errno = errno.EREMOTEIO - - failures['errno'] = failure_errno - return failures - return {'status': True} @@ -3477,7 +3215,6 @@ def datastore_mkdir_make_inodes(api_client, datastore, data_path, data_pubkeys, drivers = datastore['drivers'] device_ids = datastore['device_ids'] - children_absent = [] if parent_dir is None: parent_info = api_client.backend_datastore_lookup(None, datastore, 'directories', parent_path, data_pubkeys, extended=True, force=force ) @@ -3487,24 +3224,18 @@ def datastore_mkdir_make_inodes(api_client, datastore, data_path, data_pubkeys, parent_dir_info = parent_info['inode_info'] parent_dir = parent_dir_info['inode'] - children_absent = parent_info.get('hints', {}).get('children_absent', []) parent_uuid = parent_dir['uuid'] if parent_dir['type'] != MUTABLE_DATUM_DIR_TYPE: - log.error('Not a directory: {}'.format(dirpath)) + log.error('Not a directory: {}'.format(parent_path)) return {'error': 'Not a directory', 'errno': errno.ENOTDIR} # does a file or directory already exist? if name in parent_dir['idata']['children'].keys(): - if name not in children_absent: - log.error('Already exists in {}: {}'.format(parent_path, name)) - return {'error': 'Entry already exists', 'errno': errno.EEXIST} - - else: - log.warning("Overwriting partially-created directory {} ({})".format(name, parent_dir['idata']['children'][name]['uuid'])) - del parent_dir['idata']['children'][name] - + log.error('Already exists: {}'.format(name)) + return {'error': 'Entry already exists', 'errno': errno.EEXIST} + # make a directory! child_uuid = str(uuid.uuid4()) @@ -3553,8 +3284,6 @@ def datastore_mkdir_put_inodes( datastore, data_path, header_blobs, payloads, si Return {'status': True} on success Return {'error': ..., 'errno': ...} on failure """ - global GLOBAL_CACHE - assert len(header_blobs) == 2 assert len(payloads) == 2 assert len(signatures) == 2 @@ -3562,57 +3291,14 @@ def datastore_mkdir_put_inodes( datastore, data_path, header_blobs, payloads, si creates = [True, False] # create child exists = [False, True] # parent must exist - datastore_id = datastore_get_id(datastore['pubkey']) device_ids = datastore['device_ids'] data_pubkey = datastore['pubkey'] - - # make sure this is valid...we may need it on the error path - child_header_blob = header_blobs[0] - child_header_info = analyze_inode_header_blob(datastore_id, child_header_blob) - if 'error' in child_header_info: - log.error("Unparseable child inode header") - return child_header_info - - # make sure this is valid...we may need it on the success path - parent_header_blob = header_blobs[1] - parent_header_info = analyze_inode_header_blob(datastore_id, parent_header_blob) - if 'error' in parent_header_info: - log.error("Unparseable parent inode header") - return parent_header_info - res = datastore_operation_check( data_pubkey, header_blobs, payloads, signatures, tombstones, creates, exists, device_ids, config_path=config_path ) if 'error' in res: log.debug("Failed to check operation: {}".format(res['error'])) return res - res = datastore_do_inode_operation( datastore, header_blobs, payloads, signatures, tombstones, config_path=config_path, proxy=proxy ) - if 'error' in res: - # create failed - if res['inodes'][0]: - # creating the child failed. - # remember to fail lookups on this child until we fix it. - log.error("mkdir {}: failed to store child ({})".format(data_path, child_header_info['header_fqid'])) - rc = set_partial_create_failure(child_header_info['header_fqid'], config_path=config_path) - if not rc: - log.critical("!!! Failed to mark partial create failure !!! Inconsistency may result") - - GLOBAL_CACHE.evict_inode(datastore_id, child_header_info['uuid']) - - if res['inodes'][1]: - # updating the parent failed - # we may have leaked the child - log.error("mkdir {}: failed to store new parent {} ({})".format(data_path, os.path.dirname(data_path), child_header_info['header_fqid'])) - - GLOBAL_CACHE.evict_inode(datastore_id, parent_header_info['uuid']) - - res = {'error': res['error'], 'errno': res['errno']} - - else: - # this parent and child *definitely* exist - clear_partial_create_failure(parent_header_info['header_fqid'], config_path=config_path) - clear_partial_create_failure(child_header_info['header_fqid'], config_path=config_path) - - return res + return datastore_do_inode_operation( datastore, header_blobs, payloads, signatures, tombstones, config_path=config_path, proxy=proxy ) def datastore_mkdir(api_client, datastore, data_path, data_privkey_hex, data_pubkeys, parent_dir=None, force=False, config_path=CONFIG_PATH): @@ -3766,6 +3452,8 @@ def datastore_rmdir_put_inodes( datastore, data_path, header_blobs, payloads, si if proxy is None: proxy = get_default_proxy(config_path=config_path) + # directory must actually be empty + device_ids = datastore['device_ids'] data_pubkey = datastore['pubkey'] res = datastore_operation_check( data_pubkey, header_blobs, payloads, signatures, tombstones, creates, exists, device_ids, config_path=config_path ) @@ -3773,17 +3461,7 @@ def datastore_rmdir_put_inodes( datastore, data_path, header_blobs, payloads, si log.debug("Failed to check operation: {}".format(res['error'])) return res - res = datastore_do_inode_operation( datastore, header_blobs, payloads, signatures, tombstones, config_path=config_path, proxy=proxy ) - if 'error' in res: - # maybe we need to remember a failed delete? - ts_data = storage.parse_signed_data_tombstone(str(tombstones[0])) - rc = set_partial_delete_failure(data_path, ts_data['id'], config_path=config_path) - if not rc: - log.critical("!! Failed to record delete failure at {} !! inode leakage may result".format(data_path)) - - res = {'error': res['error'], 'errno': res['errno']} - - return res + return datastore_do_inode_operation( datastore, header_blobs, payloads, signatures, tombstones, config_path=config_path, proxy=proxy ) def datastore_rmdir(api_client, datastore, data_path, data_privkey_hex, data_pubkeys, force=False, config_path=CONFIG_PATH): @@ -3845,7 +3523,7 @@ def datastore_getfile(api_client, blockchain_id, datastore, data_path, data_pubk file_info = api_client.backend_datastore_lookup(blockchain_id, datastore, 'files', data_path, data_pubkeys, force=force, extended=True, idata=True ) if 'error' in file_info: - log.error("Failed to resolve {}: {}".format(data_path, file_info)) + log.error("Failed to resolve {}".format(data_path)) return file_info if file_info['inode_info']['inode']['type'] != MUTABLE_DATUM_FILE_TYPE: @@ -3866,7 +3544,6 @@ def datastore_getfile(api_client, blockchain_id, datastore, data_path, data_pubk 'data': file_info['inode_info']['inode']['idata'] } - ret['hints'] = file_info.get('hints', {}) return ret @@ -3897,6 +3574,8 @@ def datastore_listdir(api_client, blockchain_id, datastore, data_path, data_pubk log.error("Not a directory: {}".format(data_path)) return {'error': 'Not a directory', 'errno': errno.ENOTDIR} + # TODO: verify idata's header matches header for this inode + ret = { 'status': True, } @@ -3915,7 +3594,6 @@ def datastore_listdir(api_client, blockchain_id, datastore, data_path, data_pubk 'data': dir_info['inode_info']['inode']['idata'] } - ret['hints'] = dir_info.get('hints', {}) return ret @@ -3948,7 +3626,6 @@ def datastore_putfile_make_inodes(api_client, datastore, data_path, file_data_ha parent_dir_inode = None parent_uuid = None - children_absent = [] if parent_dir is None: parent_path_info = api_client.backend_datastore_lookup(None, datastore, 'directories', parent_dirpath, data_pubkeys, extended=True, force=force ) @@ -3959,22 +3636,15 @@ def datastore_putfile_make_inodes(api_client, datastore, data_path, file_data_ha parent_dir_info = parent_path_info['inode_info'] parent_uuid = parent_dir_info['uuid'] parent_dir_inode = parent_dir_info['inode'] - children_absent = parent_path_info.get('hints', {}).get('children_absent', []) - else: parent_dir_inode = parent_dir parent_uuid = parent_dir['uuid'] # make sure the file doesn't exist if name in parent_dir_inode['idata']['children'].keys() and create: - if name not in children_absent: - # already exists - log.error('Already exists: {}'.format(data_path)) - return {'error': 'Already exists', 'errno': errno.EEXIST} - - else: - log.warning("Overwriting partially-created file {} ({})".format(name, parent_dir_inode['idata']['children'][name]['uuid'])) - del parent_dir_inode['idata']['children'][name] + # already exists + log.error('Already exists: {}'.format(data_path)) + return {'error': 'Already exists', 'errno': errno.EEXIST} child_uuid = None @@ -4034,8 +3704,6 @@ def datastore_putfile_put_inodes( datastore, data_path, header_blobs, payloads, Return {'status': True} on success Return {'error': ..., 'errno': ...} on failure """ - global GLOBAL_CACHE - assert len(header_blobs) == 2 assert len(payloads) == 2 assert len(signatures) == 2 @@ -4046,22 +3714,6 @@ def datastore_putfile_put_inodes( datastore, data_path, header_blobs, payloads, if proxy is None: proxy = get_default_proxy(config_path=config_path) - datastore_id = datastore_get_id(datastore['pubkey']) - - # make sure this is valid...we may need it on the error path - child_header_blob = header_blobs[1] - child_header_info = analyze_inode_header_blob(datastore_id, child_header_blob) - if 'error' in child_header_info: - log.error("Unparseable child inode header") - return child_header_info - - # make sure this is valid...we may need it on the success path - parent_header_blob = header_blobs[1] - parent_header_info = analyze_inode_header_blob(datastore_id, parent_header_blob) - if 'error' in parent_header_info: - log.error("Unparseable parent inode header") - return parent_header_info - device_ids = datastore['device_ids'] data_pubkey = datastore['pubkey'] res = datastore_operation_check( data_pubkey, header_blobs, payloads, signatures, tombstones, creates, exists, device_ids, config_path=config_path ) @@ -4069,38 +3721,7 @@ def datastore_putfile_put_inodes( datastore, data_path, header_blobs, payloads, log.debug("Failed to check operation: {}".format(res['error'])) return res - res = datastore_do_inode_operation( datastore, header_blobs, payloads, signatures, tombstones, config_path=config_path, proxy=proxy ) - if 'error' in res: - # create failed - if res['inodes'][1]: - # creating/updating the child failed. - # did this child exist already? - child_version_info = get_mutable_data_version(child_header_info['header_id'], [child_header_info['device_id']], config_path=CONFIG_PATH) - if create or child_version_info.get('version', 0) == 0: - # we were creating this - # remember to fail lookups on this child until we fix it. - log.error("putfile {}: failed to store child".format(data_path)) - rc = set_partial_create_failure(child_header_info['header_fqid'], config_path=config_path) - if not rc: - log.critical("!!! Failed to mark partial create failure !!! Inconsistency may result") - - GLOBAL_CACHE.evict_inode(datastore_id, child_header_info['uuid']) - - if res['inodes'][0]: - # updating the parent failed - # we may have leaked the child - log.error("putfile {}: failed to store new parent {}. Child leaked!".format(data_path, os.path.dirname(data_path))) - - GLOBAL_CACHE.evict_inode(datastore_id, parent_header_info['uuid']) - - res = {'error': res['error'], 'errno': res['errno']} - - else: - # parent and child *definitely* exist - clear_partial_create_failure(parent_header_info['header_fqid'], config_path=config_path) - clear_partial_create_failure(child_header_info['header_fqid'], config_path=config_path) - - return res + return datastore_do_inode_operation( datastore, header_blobs, payloads, signatures, tombstones, config_path=config_path, proxy=proxy ) def datastore_putfile(api_client, datastore, data_path, file_data_bin, data_privkey_hex, data_pubkeys, create=False, exist=False, force=False, config_path=CONFIG_PATH): @@ -4200,7 +3821,7 @@ def datastore_deletefile_make_inodes(api_client, datastore, data_path, data_pubk reader_pubkeys=parent_dir_inode['reader_pubkeys'], min_version=min_version, config_path=config_path ) if 'error' in parent_dir_info: - log.error("Failed to update directory {}: {}".format(dir_path, parent_dir_info['error'])) + log.error("Failed to update directory {}: {}".format(parent_dir, parent_dir_info['error'])) return {'error': 'Failed to create parent directory', 'errno': errno.EIO} # make a child tombstone @@ -4250,17 +3871,7 @@ def datastore_deletefile_put_inodes( datastore, data_path, header_blobs, payload log.debug("Failed to check operation: {}".format(res['error'])) return res - res = datastore_do_inode_operation( datastore, header_blobs, payloads, signatures, tombstones, config_path=config_path, proxy=proxy ) - if 'error' in res: - # maybe we need to remember a failed delete? - ts_data = storage.parse_signed_data_tombstone(str(tombstones[0])) - rc = set_partial_delete_failure(data_path, ts_data['id'], config_path=config_path) - if not rc: - log.critical("!! Failed to record delete failure at {} !! Inode data may leak".format(data_path)) - - res = {'error': res['error'], 'errno': res['errno']} - - return res + return datastore_do_inode_operation( datastore, header_blobs, payloads, signatures, tombstones, config_path=config_path, proxy=proxy ) def datastore_deletefile(api_client, datastore, data_path, data_privkey_hex, data_pubkeys, force=False, config_path=CONFIG_PATH): @@ -4282,7 +3893,6 @@ def datastore_deletefile(api_client, datastore, data_path, data_privkey_hex, dat inode_info = datastore_deletefile_make_inodes( api_client, datastore, data_path, data_pubkeys, force=force, config_path=config_path ) if 'error' in inode_info: - log.error("Failed to update inodes for deletefile") return inode_info inode_signatures = [] @@ -4296,7 +3906,7 @@ def datastore_deletefile(api_client, datastore, data_path, data_privkey_hex, dat datastore_info = datastore_serialize_and_sign(datastore, data_privkey_hex) res = api_client.backend_datastore_deletefile( datastore_info['str'], datastore_info['sig'], data_path, inode_info['inodes'], inode_info['payloads'], inode_signatures, signed_tombstones ) if 'error' in res: - log.error("Failed to put deletefile inodes: {}".format(res['error'])) + log.debug("Failed to put deletefile inodes") return res return {'status': True} @@ -4335,7 +3945,6 @@ def datastore_stat(api_client, blockchain_id, datastore, data_path, data_pubkeys else: ret['data'] = inode_info['inode_info']['inode'] - ret['hints'] = inode_info.get('hints', {}) return ret @@ -4370,7 +3979,6 @@ def datastore_getinode(api_client, blockchain_id, datastore, inode_uuid, extende else: ret['inode'] = inode_info['inode'] - ret['hints'] = inode_info.get('hints', {}) return ret diff --git a/blockstack_client/keys.py b/blockstack_client/keys.py index 3c87770d9..55f6682c7 100644 --- a/blockstack_client/keys.py +++ b/blockstack_client/keys.py @@ -31,18 +31,12 @@ from keychain import PrivateKeychain import jsonschema from jsonschema.exceptions import ValidationError -from logger import get_logger -from constants import CONFIG_PATH, BLOCKSTACK_DEBUG, BLOCKSTACK_TEST +from .logger import get_logger +from .constants import CONFIG_PATH, BLOCKSTACK_DEBUG, BLOCKSTACK_TEST import virtualchain from virtualchain.lib.ecdsalib import * -NAMES_PRIVKEY_NODE = 888 -NAMES_PRIVKEY_VERSION_NODE = 0 -APP_PRIVKEY_NODE = 0 -SIGNING_PRIVKEY_NODE = 1 -ENCRYPTION_PRIVKEY_NODE = 2 - # for compatibility log = get_logger() @@ -489,6 +483,7 @@ def get_data_privkey_info(user_zonefile, wallet_keys=None, config_path=CONFIG_PA """ Get the user's data private key info """ + privkey = get_data_privkey(user_zonefile, wallet_keys=wallet_keys, config_path=config_path) return privkey @@ -554,97 +549,3 @@ def get_privkey_info_params(privkey_info, config_path=CONFIG_PATH): return None, None - -def find_name_index(name_address, master_privkey_hex, max_tries=25, start=0): - """ - Given a name's device-specific address and device-specific master key, - find index from which it was derived. - - Return the index on success - Return None on failure. - """ - - hdwallet = HDWallet(master_privkey_hex) - for i in xrange(start, max_tries): - child_privkey = hdwallet.get_child_privkey(index=i) - child_pubkey = ecdsalib.get_pubkey_hex(child_privkey) - - child_addresses = [ - keylib.public_key_to_address(keylib.key_formatting.compress(child_pubkey)), - keylib.public_key_to_address(keylib.key_formatting.decompress(child_pubkey)) - ] - - if str(name_address) in child_addresses: - return i - - return None - - -def get_name_privkey(master_privkey_hex, name_index): - """ - Make the device-specific private key that owns the name. - @master_privkey_hex is the wallet master key, e.g. from the Browser. - @name_index is the ith name to be created from this device. - """ - hdwallet = HDWallet(master_privkey_hex) - names_privkey = hdwallet.get_child_privkey(index=NAMES_PRIVKEY_NODE, compressed=False) - - hdwallet = HDWallet(names_privkey) - names_version_privkey = hdwallet.get_child_privkey(index=NAMES_PRIVKEY_VERSION_NODE, compressed=False) - - hdwallet = HDWallet(names_version_privkey) - name_privkey = hdwallet.get_child_privkey(index=name_index, compressesd=False) - - return name_privkey - - -def get_app_root_privkey(name_privkey): - """ - Make the device-specific app private key from the device-specific name owner private key - """ - hdwallet = HDWallet(name_privkey) - app_privkey = hdwallet.get_child_privkey(index=APP_PRIVKEY_NODE, compressed=False) - return app_privkey - - -def get_app_privkey_index(full_application_name): - """ - Get the full application private key index. - Application name must be full. i.e. must end in '.1', or '.x' - """ - full_application_name = str(full_application_name) - hashcode = 0 - for i in xrange(0, len(full_application_name)): - next_byte = ord(full_application_name[i]) - hashcode = ((hashcode << 5) - hashcode) + next_byte - - return hashcode & 0x7fffffff - - -def get_app_privkey(app_root_privkey, full_application_name): - """ - Make the app-specific, device-specific private key from the app root private key - """ - hdwallet = HDWallet(app_root_privkey) - app_index = get_app_privkey_index(full_application_name) - app_privkey = hdwallet.get_child_privkey(index=app_index, compressed=False) - return app_privkey - - -def get_signing_privkey(name_privkey): - """ - Make the device-specific signing private key from the device-specific name owner private key - """ - hdwallet = HDWallet(name_privkey) - signing_privkey = hdwallet.get_child_privkey(index=SIGNING_PRIVKEY_NODE, compressed=False) - return signing_privkey - - -def get_encryption_privkey(name_privkey): - """ - Make the device-specific encryption private key from the device-specific name owner private key - """ - hdwallet = HDWallet(name_privkey) - encryption_privkey = hdwallet.get_child_privkey(index=ENCRYPTION_PRIVKEY_NODE, compressed=False) - return encryption_privkey - diff --git a/blockstack_client/logger.py b/blockstack_client/logger.py index 5eb0a1ce6..aa1c390e0 100644 --- a/blockstack_client/logger.py +++ b/blockstack_client/logger.py @@ -37,7 +37,7 @@ import requests from ConfigParser import SafeConfigParser import virtualchain -from constants import * +from .constants import * class NetworkLogFormatter( logging.Formatter ): diff --git a/blockstack_client/profile.py b/blockstack_client/profile.py index 45624a1a6..f1f14e008 100644 --- a/blockstack_client/profile.py +++ b/blockstack_client/profile.py @@ -32,88 +32,321 @@ import virtualchain from virtualchain.lib.ecdsalib import * import keylib -from proxy import * +from .proxy import * from blockstack_client import storage from blockstack_client import user as user_db -from logger import get_logger -from constants import USER_ZONEFILE_TTL, CONFIG_PATH, BLOCKSTACK_TEST, BLOCKSTACK_DEBUG +from .logger import get_logger +from .constants import USER_ZONEFILE_TTL, CONFIG_PATH, BLOCKSTACK_TEST, BLOCKSTACK_DEBUG -from token_file import token_file_parse, token_file_update_profile, token_file_get, token_file_put -from zonefile import get_name_zonefile -from keys import get_data_privkey_info -from schemas import * -from config import get_config -from constants import BLOCKSTACK_REQUIRED_STORAGE_DRIVERS_WRITE +from .zonefile import get_name_zonefile +from .keys import get_data_privkey_info +from .schemas import * +from .config import get_config +from .constants import BLOCKSTACK_REQUIRED_STORAGE_DRIVERS_WRITE log = get_logger() -def get_profile(name, **kw): + +def set_profile_timestamp(profile, now=None): """ - Legacy compatibility method for get_token_file(). - Wraps get_token_file, and if a token file is successfully resolved, returns - {'status': True, - 'profile': profile, - 'zonefile': zonefile, - 'token_file': token_file, - 'raw_zonefile': unparsed zone file - 'token_file': actual token file (if present) - 'legacy': whether or not the profile was legacy} + Set the profile's timestamp to now + """ + now = time.time() if now is None else now + profile['timestamp'] = now + + return profile + + +def get_profile_timestamp(profile): + """ + Get profile timestamp + """ + return profile['timestamp'] + + +def load_legacy_user_profile(name, expected_hash): + """ + Load a legacy user profile, and convert it into + the new zonefile-esque profile format that can + be serialized into a JWT. + + Verify that the profile hashses to the above expected hash + """ + + # fetch... + storage_host = 'onename.com' + assert name.endswith('.id') + + name_without_namespace = '.'.join(name.split('.')[:-1]) + storage_path = '/{}.json'.format(name_without_namespace) + + try: + req = httplib.HTTPConnection(storage_host) + resp = req.request('GET', storage_path) + data = resp.read() + except Exception as e: + log.error('Failed to fetch http://{}/{}: {}'.format(storage_host, storage_path, e)) + return None + + try: + data_json = json.loads(data) + except Exception as e: + log.error('Unparseable profile data') + return None + + data_hash = storage.get_blockchain_compat_hash(data_json) + if expected_hash != data_hash: + log.error('Hash mismatch: expected {}, got {}'.format(expected_hash, data_hash)) + return None + + assert blockstack_profiles.is_profile_in_legacy_format(data_json) + new_profile = blockstack_profiles.get_person_from_legacy_format(data_json) + return new_profile + + + +def put_profile(name, new_profile, blockchain_id=None, user_data_privkey=None, user_zonefile=None, + proxy=None, wallet_keys=None, required_drivers=None, config_path=CONFIG_PATH): + """ + Set the new profile data. CLIENTS SHOULD NOT CALL THIS METHOD DIRECTLY. + + if user_data_privkey is given, then wallet_keys does not need to be given. + + Return {'status: True} on success + Return {'error': ...} on failure. + """ + + ret = {} + + proxy = get_default_proxy() if proxy is None else proxy + config = proxy.conf + + # deduce storage drivers + required_storage_drivers = None + if required_drivers is not None: + required_storage_drivers = required_drivers + else: + required_storage_drivers = config.get('storage_drivers_required_write', None) + if required_storage_drivers is not None: + required_storage_drivers = required_storage_drivers.split(',') + else: + required_storage_drivers = config.get('storage_drivers', '').split(',') + + # deduce private key + if user_data_privkey is None: + user_data_privkey = get_data_privkey_info(user_zonefile, wallet_keys=wallet_keys, config_path=config_path) + if json_is_error(user_data_privkey): + log.error("Failed to get data private key: {}".format(user_data_privkey['error'])) + return {'error': 'No data key defined'} + + profile_payload = copy.deepcopy(new_profile) + profile_payload = set_profile_timestamp(profile_payload) + + if BLOCKSTACK_DEBUG: + # NOTE: don't calculate this string unless we're actually debugging... + log.debug('Save updated profile for "{}" to {} at {} by {}'.format( + name, ','.join(required_storage_drivers), get_profile_timestamp(profile_payload), get_pubkey_hex(user_data_privkey)) + ) + + rc = storage.put_mutable_data( + name, profile_payload, data_privkey=user_data_privkey, + required=required_storage_drivers, + profile=True, blockchain_id=blockchain_id + ) + + if rc: + ret['status'] = True + else: + ret['error'] = 'Failed to update profile' + + return ret + + +def delete_profile(blockchain_id, user_data_privkey=None, user_zonefile=None, + proxy=None, wallet_keys=None): + """ + Delete profile data. CLIENTS SHOULD NOT CALL THIS DIRECTLY + Return {'status: True} on success + Return {'error': ...} on failure. + """ + + ret = {} + + proxy = get_default_proxy() if proxy is None else proxy + config = proxy.conf + + # deduce private key + if user_data_privkey is None: + user_data_privkey = get_data_privkey_info(user_zonefile, wallet_keys=wallet_keys, config_path=proxy.conf['path']) + if json_is_error(user_data_privkey): + log.error("Failed to get data private key: {}".format(user_data_privkey['error'])) + return {'error': 'No data key defined'} + + rc = storage.delete_mutable_data(blockchain_id, user_data_privkey) + if rc: + ret['status'] = True + else: + ret['error'] = 'Failed to update profile' + + return ret + + +def get_profile(name, zonefile_storage_drivers=None, profile_storage_drivers=None, + proxy=None, user_zonefile=None, name_record=None, + include_name_record=False, include_raw_zonefile=False, use_zonefile_urls=True, + use_legacy=False, use_legacy_zonefile=True, decode_profile=True): + """ + Given a name, look up an associated profile. + Do so by first looking up the zonefile the name points to, + and then loading the profile from that zonefile's public key. + + Notes on backwards compatibility (activated if use_legacy=True and use_legacy_zonefile=True): + + * (use_legacy=True) If the user's zonefile is really a legacy profile from Onename, then + the profile returned will be the converted legacy profile. The returned zonefile will still + be a legacy profile, however. + The caller can check this and perform the conversion automatically. + + * (use_legacy_zonefile=True) If the name points to a current zonefile that does not have a + data public key, then the owner address of the name will be used to verify + the profile's authenticity. + + Returns {'status': True, 'profile': profile, 'zonefile': zonefile} on success. + * If include_name_record is True, then include 'name_record': name_record with the user's blockchain information + * If include_raw_zonefile is True, then include 'raw_zonefile': raw_zonefile with unparsed zone file Returns {'error': ...} on error """ - res = token_file_get(name, **kw) - if 'error' in res: - return res - token_file = res['token_file'] - zonefile = res['zonefile'] + proxy = get_default_proxy() if proxy is None else proxy + + raw_zonefile = None + if user_zonefile is None: + user_zonefile = get_name_zonefile( + name, proxy=proxy, + name_record=name_record, include_name_record=True, + storage_drivers=zonefile_storage_drivers, + include_raw_zonefile=include_raw_zonefile, + allow_legacy=True + ) + + if 'error' in user_zonefile: + return user_zonefile + + raw_zonefile = None + if include_raw_zonefile: + raw_zonefile = user_zonefile.pop('raw_zonefile') + + user_zonefile = user_zonefile['zonefile'] + + # is this really a legacy profile? + if blockstack_profiles.is_profile_in_legacy_format(user_zonefile): + if not use_legacy: + return {'error': 'Profile is in legacy format'} + + # convert it + log.debug('Converting legacy profile to modern profile') + user_profile = blockstack_profiles.get_person_from_legacy_format(user_zonefile) + + elif not user_db.is_user_zonefile(user_zonefile): + if not use_legacy: + return {'error': 'Name zonefile is non-standard'} + + # not a legacy profile, but a custom profile + log.debug('Using custom legacy profile') + user_profile = copy.deepcopy(user_zonefile) - profile = None - legacy = False - if res.get('legacy_profile') is not None: - profile = res['legacy_profile'] - legacy = True - else: - profile = res['profile'] - - raw_zonefile = res.get('raw_zonefile') - name_record = res.get('name_record') - token_file = res.get('token_file') + # get user's data public key + data_address, owner_address = None, None + try: + user_data_pubkey = user_db.user_zonefile_data_pubkey(user_zonefile) + if user_data_pubkey is not None: + user_data_pubkey = str(user_data_pubkey) + data_address = keylib.ECPublicKey(user_data_pubkey).address() + + except ValueError: + # multiple keys defined; we don't know which one to use + user_data_pubkey = None + + if not use_legacy_zonefile and user_data_pubkey is None: + # legacy zonefile without a data public key + return {'error': 'Name zonefile is missing a public key'} + + # find owner address + if name_record is None: + name_record = get_name_blockchain_record(name, proxy=proxy) + if name_record is None or 'error' in name_record: + log.error('Failed to look up name record for "{}"'.format(name)) + return {'error': 'Failed to look up name record'} + + assert 'address' in name_record.keys(), json.dumps(name_record, indent=4, sort_keys=True) + owner_address = name_record['address'] + + # get user's data public key from the zonefile + urls = None + if use_zonefile_urls and user_zonefile is not None: + urls = user_db.user_zonefile_urls(user_zonefile) + + user_profile = storage.get_mutable_data( + name, user_data_pubkey, blockchain_id=name, + data_address=data_address, owner_address=owner_address, + urls=urls, drivers=profile_storage_drivers, decode=decode_profile, + ) + + if user_profile is None or json_is_error(user_profile): + if user_profile is None: + log.error('no user profile for {}'.format(name)) + else: + log.error('failed to load profile for {}: {}'.format(name, user_profile['error'])) + + return {'error': 'Failed to load user profile'} + + # finally, if the caller asked for the name record, and we didn't get a chance to look it up, + # then go get it. ret = { 'status': True, - 'profile': profile, - 'zonefile': zonefile, - 'raw_zonefile': raw_zonefile, - 'name_record': name_record, - 'token_file': token_file, - 'legacy': legacy + 'profile': user_profile, + 'zonefile': user_zonefile } + if include_name_record: + if name_record is None: + name_record = get_name_blockchain_record(name, proxy=proxy) + + if name_record is None or 'error' in name_record: + log.error('Failed to look up name record for "{}"'.format(name)) + return {'error': 'Failed to look up name record'} + + ret['name_record'] = name_record + + if include_raw_zonefile: + if raw_zonefile is not None: + ret['raw_zonefile'] = raw_zonefile + return ret def _get_person_profile(name, proxy=None): """ - Get the person's profile. - Works for raw profiles, and for profiles within token files. - Only works if the profile is a Persona. - - Return {'profile': ..., 'person': ...} on success + Get the person's zonefile and profile. + Handle legacy zonefiles, but not legacy profiles. + Return {'profile': ..., 'zonefile': ..., 'person': ...} on success Return {'error': ...} on error """ - res = get_profile(name, proxy=proxy) + res = get_profile(name, proxy=proxy, use_legacy_zonefile=True) if 'error' in res: - return {'error': 'Failed to load profile: {}'.format(res['error'])} + return {'error': 'Failed to load zonefile: {}'.format(res['error'])} - if res['legacy']: - return {'error': 'Failed to load profile: legacy format'} - - token_file = res.pop('token_file') profile = res.pop('profile') + zonefile = res.pop('zonefile') + + if blockstack_profiles.is_profile_in_legacy_format(profile): + return {'error': 'Legacy profile'} + person = None try: person = blockstack_profiles.Person(profile) @@ -121,28 +354,29 @@ def _get_person_profile(name, proxy=None): log.exception(e) return {'error': 'Failed to parse profile data into a Person record'} - return {'profile': profile, 'person': person, 'token_file': token_file} + return {'profile': profile, 'zonefile': zonefile, 'person': person} -def _save_person_profile(name, cur_token_file, profile, signing_private_key, blockchain_id=None, proxy=None, config_path=CONFIG_PATH): +def _save_person_profile(name, zonefile, profile, wallet_keys, user_data_privkey=None, blockchain_id=None, proxy=None, config_path=CONFIG_PATH): """ Save a person's profile, given information fetched with _get_person_profile. Return {'status': True} on success Return {'error': ...} on error """ - conf = get_config(config_path) assert conf - - required_storage_drivers = conf.get('storage_drivers_required_write', BLOCKSTACK_REQUIRED_STORAGE_DRIVERS_WRITE) + + required_storage_drivers = conf.get( + 'storage_drivers_required_write', + BLOCKSTACK_REQUIRED_STORAGE_DRIVERS_WRITE + ) required_storage_drivers = required_storage_drivers.split() - res = token_file_update_profile(cur_token_file, profile, signing_private_key) - if 'error' in res: - return res + res = put_profile(name, profile, user_zonefile=zonefile, + wallet_keys=wallet_keys, user_data_privkey=user_data_privkey, proxy=proxy, + required_drivers=required_storage_drivers, blockchain_id=name, + config_path=config_path ) - new_token_file = res['token_file'] - res = token_file_put(name, new_token_file, signing_private_key, proxy=proxy, required_drivers=required_storage_drivers, config_path=config_path) return res @@ -159,23 +393,24 @@ def profile_list_accounts(name, proxy=None): name_info = _get_person_profile(name, proxy=proxy) if 'error' in name_info: return name_info - + profile = name_info.pop('profile') + zonefile = name_info.pop('zonefile') person = name_info.pop('person') - person_accounts = [] + accounts = [] if hasattr(person, 'account'): - person_accounts = person.account - - accounts = [] - for acct in person_accounts: + accounts = person.account + + output_accounts = [] + for acct in accounts: try: jsonschema.validate(acct, PROFILE_ACCOUNT_SCHEMA) - accounts.append(acct) + output_accounts.append(acct) except jsonschema.ValidationError: continue - return {'accounts': accounts} + return {'accounts': output_accounts} def profile_get_account(blockchain_id, service, identifier, config_path=CONFIG_PATH, proxy=None): @@ -194,10 +429,10 @@ def profile_get_account(blockchain_id, service, identifier, config_path=CONFIG_P if account['service'] == service and account['identifier'] == identifier: return {'status': True, 'account': account} - return {'error': 'No such account', 'errno': errno.ENOENT} + return {'error': 'No such account'} -def profile_find_accounts(cur_profile, service, identifer): +def profile_find_accounts(cur_profile, service, identifier): """ Given an profile, find accounts that match the service and identifier Returns a list of accounts on success @@ -259,7 +494,7 @@ def profile_patch_account(cur_profile, service, identifier, content_url, extra_d return profile -def profile_put_account(blockchain_id, service, identifier, content_url, extra_data, signing_private_key, config_path=CONFIG_PATH, proxy=None): +def profile_put_account(blockchain_id, service, identifier, content_url, extra_data, wallet_keys, user_data_privkey=None, config_path=CONFIG_PATH, proxy=None): """ Save a new account to a profile. Return {'status': True, 'replaced': True/False} on success @@ -272,23 +507,20 @@ def profile_put_account(blockchain_id, service, identifier, content_url, extra_d person_info = _get_person_profile(blockchain_id, proxy=proxy) if 'error' in person_info: return person_info - - token_file = person_info['token_file'] - if token_file is None: - return {'error': 'Name points to raw, legacy profile. Please use the `migrate` command to migrate to the latest profile data format'} + zonefile = person_info.pop('zonefile') profile = person_info.pop('profile') profile = profile_patch_account(profile, service, identifier, content_url, extra_data) # save - result = _save_person_profile(blockchain_id, token_file, profile, signing_private_key, blockchain_id=blockchain_id, proxy=proxy, config_path=config_path) + result = _save_person_profile(blockchain_id, zonefile, profile, wallet_keys, user_data_privkey=user_data_privkey, blockchain_id=blockchain_id, proxy=proxy, config_path=config_path) if 'error' in result: return result return {'status': True} -def profile_delete_account(blockchain_id, service, identifier, signing_private_key, config_path=CONFIG_PATH, proxy=None): +def profile_delete_account(blockchain_id, service, identifier, wallet_keys, user_data_privkey=None, config_path=CONFIG_PATH, proxy=None): """ Delete an account, given the blockchain ID, service, and identifier Return {'status': True} on success @@ -299,14 +531,11 @@ def profile_delete_account(blockchain_id, service, identifier, signing_private_k if 'error' in person_info: return person_info + zonefile = person_info['zonefile'] profile = person_info['profile'] if not profile.has_key('account'): # nothing to do return {'error': 'No such account'} - - token_file = person_info['token_file'] - if token_file is None: - return {'error': 'Name points to raw, legacy profile. Please use the `migrate` command to migrate to the latest profile data format'} found = False for i in xrange(0, len(profile['account'])): @@ -325,10 +554,38 @@ def profile_delete_account(blockchain_id, service, identifier, signing_private_k if not found: return {'error': 'No such account'} - result = _save_person_profile(blockchain_id, token_file, profile, signing_private_key, blockchain_id=blockchain_id, proxy=proxy, config_path=config_path) + result = _save_person_profile(blockchain_id, zonefile, profile, wallet_keys, user_data_privkey=user_data_privkey, blockchain_id=blockchain_id, proxy=proxy, config_path=config_path) if 'error' in result: return result return {'status': True} +def profile_list_device_ids( blockchain_id, proxy=None ): + """ + Given a blockchain ID, identify the set of device IDs for it. + + Returns {'status': True, 'device_ids': ...} on success + Returns {'error': ...} on error + """ + raise NotImplementedError("Token file logic is not implemented yet") + + +def profile_add_device_id( blockchain_id, device_id, wallet_keys, config_path=CONFIG_PATH, proxy=None): + """ + Add a device ID to a profile + Return {'status': True} on success + Return {'error': ...} on error + """ + raise NotImplementedError("Token file logic is not implemented yet") + + +def profile_remove_device_id( blockchain_id, device_id, wallet_keys, config_path=CONFIG_PATH, proxy=None): + """ + Remove a device ID from a profile + Return {'status': True} on success + Return {'error': ...} on error + """ + raise NotImplementedError("Token file logic is not implemented yet") + + diff --git a/blockstack_client/proxy.py b/blockstack_client/proxy.py index cccfeec86..b763637d5 100644 --- a/blockstack_client/proxy.py +++ b/blockstack_client/proxy.py @@ -43,18 +43,18 @@ xmlrpc.monkey_patch() import storage import scripts -from constants import ( +from .constants import ( MAX_RPC_LEN, CONFIG_PATH, BLOCKSTACK_TEST, DEFAULT_TIMEOUT ) -from logger import get_logger +from .logger import get_logger -from operations import ( +from .operations import ( nameop_history_extract, nameop_restore_from_history, nameop_restore_snv_consensus_fields ) -from schemas import * +from .schemas import * log = get_logger('blockstack-client') diff --git a/blockstack_client/rpc.py b/blockstack_client/rpc.py index 7180d7704..256da2155 100644 --- a/blockstack_client/rpc.py +++ b/blockstack_client/rpc.py @@ -45,6 +45,7 @@ import jsontokens import subprocess import platform import shutil +import urlparse from jsonschema import ValidationError from schemas import * import client as bsk_client @@ -62,7 +63,10 @@ import backend.drivers as backend_drivers import proxy from proxy import json_is_error, json_is_exception -from .constants import BLOCKSTACK_DEBUG, BLOCKSTACK_TEST, RPC_MAX_ZONEFILE_LEN, CONFIG_PATH, WALLET_FILENAME, TX_MIN_CONFIRMATIONS, DEFAULT_API_PORT, SERIES_VERSION, TX_MAX_FEE +DEFAULT_UI_PORT = 8888 +DEVELOPMENT_UI_PORT = 3000 + +from .constants import BLOCKSTACK_DEBUG, BLOCKSTACK_TEST, RPC_MAX_ZONEFILE_LEN, CONFIG_PATH, WALLET_FILENAME, TX_MIN_CONFIRMATIONS, DEFAULT_API_PORT, SERIES_VERSION, TX_MAX_FEE, set_secret, get_secret from .method_parser import parse_methods from .wallet import make_wallet import app @@ -168,14 +172,13 @@ def api_cli_wrapper(method_info, config_path, check_rpc=True, include_kw=False): argwrapper.__name__ = method_info['method'].__name__ return argwrapper +JSONRPC_MAX_SIZE = 1024 * 1024 class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): ''' Blockstack RESTful API endpoint. ''' - JSONRPC_MAX_SIZE = 1024 * 1024 - http_errors = { errno.ENOENT: 404, errno.EINVAL: 401, @@ -233,7 +236,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): return request_str - def _read_json(self, schema=None): + def _read_json(self, schema=None, maxlen=JSONRPC_MAX_SIZE): """ Read a JSON payload from the requester Return the parsed payload on success @@ -247,7 +250,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): log.error("Invalid request of type {} from {}".format(request_type, client_address_str)) return None - request_str = self._read_payload(maxlen=self.JSONRPC_MAX_SIZE) + request_str = self._read_payload(maxlen=maxlen) if request_str is None: log.error("Failed to read request") return None @@ -285,6 +288,33 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): return ret + def verify_origin(self, allowed): + """ + Verify that the Origin: header is present and listed + in our list of allowed origins + Return True if so + Return False if not + """ + allowed_urlparse = [urlparse.urlparse(a) for a in allowed] + allowed_origins = dict() + for parsed, a in zip(allowed_urlparse, allowed): + if not parsed.netloc: + # try to see if we got a domainname (legacy path) + parsed = urlparse.urlparse("http://{}".format(a)) + assert parsed.netloc, "Invalid origin {}".format(a) + allowed_origins[parsed.netloc] = parsed + + origin_header = self.headers.get('origin', None) + if origin_header is not None: + origin_info = urlparse.urlparse(origin_header) + if origin_info.netloc in allowed_origins.keys(): + allowed_origin = allowed_origins[origin_info.netloc] + if origin_info.scheme == allowed_origin.scheme and origin_info.netloc == allowed_origin.netloc: + return True + + return False + + def verify_session(self, qs_values): """ Verify and return the application's session. @@ -453,6 +483,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): if decode_err: log.error('Current decode error:') log.exception(decode_err) + log.error('Legacy decode error:') log.exception(ve) return self._reply_json({'error': 'Invalid authRequest token: does not match any known request schemas'}, status_code=401) @@ -468,7 +499,6 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): if legacy: # legacy; fill in defaults - log.warning("Legacy authToken") app_public_key = str(decoded_token['payload']['app_public_key']) app_private_key = '0000000000000000000000000000000000000000000000000000000000000001' app_public_keys = [] @@ -755,7 +785,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): self._reply_json({'error': 'Failed to lookup name'}, status_code=500) return - zonefile_res = zonefile.get_name_zonefile(name, name_record=name_rec) + zonefile_res = zonefile.get_name_zonefile(name, raw_zonefile=True, name_record=name_rec) zonefile_txt = None if 'error' in zonefile_res: error = "No zonefile for name" @@ -765,25 +795,25 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): log.error("Failed to get name zonefile for {}: {}".format(name, error)) else: - zonefile_txt = zonefile_res.pop("raw_zonefile") + zonefile_txt = zonefile_res.pop("zonefile") status = 'revoked' if name_rec['revoked'] else 'registered' + address = name_rec['address'] + if address: + address = virtualchain.address_reencode(str(address), network='mainnet') + log.debug("{} is {}".format(name, status)) ret = { 'status': status, 'zonefile': zonefile_txt, 'zonefile_hash': name_rec['value_hash'], - 'address': name_rec['address'], + 'address': address, 'last_txid': name_rec['txid'], 'blockchain': 'bitcoin', 'expire_block': name_rec['expire_block'], } - # make sure the address is in the right format - blockchain_network = os.environ.get("BLOCKSTACK_RPC_MOCK_BLOCKCHAIN_NETWORK", None) - ret['address'] = virtualchain.address_reencode(str(ret['address']), network=blockchain_network) - self._reply_json(ret) return @@ -820,14 +850,6 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): self._reply_json({'error': res['error']}, status_code=500) return - # re-encode addresses, if need be - blockchain_network = os.environ.get("BLOCKSTACK_RPC_MOCK_BLOCKCHAIN_NETWORK", None) - for block_id in res.keys(): - for state in res[block_id]: - for addr_key in ['address', 'recipient_address', 'importer_address']: - if state.has_key(addr_key) and state[addr_key]: - state[addr_key] = virtualchain.address_reencode(str(state[addr_key]), network=blockchain_network) - self._reply_json(res) return @@ -1047,7 +1069,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): Reply 500 on failure to fetch data """ internal = self.server.get_internal_proxy() - resp = internal.cli_get_name_zonefile(name, "false", raw=False) + resp = internal.cli_get_name_zonefile(name, "true", raw=False) if json_is_error(resp): self._reply_json({"error": resp['error']}, status_code=500) return @@ -1105,8 +1127,8 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): 'message': 'Name queued for update. The process takes ~1 hour. You can check the status with `blockstack info`.' } - if 'tx' in res: - ret['tx'] = res['tx'] + if 'tx' in resp: + ret['tx'] = resp['tx'] self._reply_json(ret) return @@ -1576,6 +1598,8 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): Return 200 on successful save Return 401 on invalid request return 503 on storage failure + + Allows arbitrary sized PUTs/POSTs """ if datastore_id != ses['app_user_id']: @@ -1595,7 +1619,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): create = qs.get('create', '0') == '1' exist = qs.get('exist', '0') == '1' - request = self._read_json() + request = self._read_json(maxlen=None) if request: # sent externally-signed data operation = None @@ -1623,7 +1647,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): Reply 503 on failure to contact remote storage providers """ if datastore_id != ses['app_user_id']: - return self._reply_json({'error': 'Invalid user', 'errno': errno.EACCES}, status=403) + return self._reply_json({'error': 'Invalid user', 'errno': errno.EACCES}, status_code=403) if inode_type not in ['files', 'directories', 'inodes']: self._reply_json({'error': 'Invalid request', 'errno': errno.EINVAL}, status_code=401) @@ -2065,8 +2089,8 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): Return 500 on error """ wallet = self._read_json(schema=WALLET_SCHEMA_CURRENT) - if request is None: - return self._reply_json({'error': 'Failed to validate keys'}, status_code=401) + if wallet is None: + return self._reply_json({'error': 'Failed to validate wallet keys'}, status_code=401) res = backend.registrar.set_wallet( (wallet['payment_addresses'][0], wallet['payment_privkey']), (wallet['owner_addresses'][0], wallet['owner_privkey']), @@ -2332,6 +2356,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): if index_url: driver_config['index_url'] = index_url + ret = {} ret[driver_name] = driver_config return self._reply_json(ret) @@ -2538,7 +2563,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): else: status_code = 404 - self._reply_json({'error': consensus_hash['error']}, status_code=status_code) + self._reply_json({'error': info['error']}, status_code=status_code) return self._reply_json({'consensus_hash': info['consensus']}) @@ -2672,7 +2697,7 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): elif command == 'clearcache': # clear the cache - data.cache_evct_all() + data.cache_evict_all() return self._send_headers(status_code=200, content_type='text/plain') else: @@ -3503,6 +3528,15 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): }, }, } + + LOCALHOST = [] + for port in [DEFAULT_UI_PORT, DEVELOPMENT_UI_PORT]: + LOCALHOST += [ + 'http://localhost:{}'.format(port), + 'http://{}:{}'.format(socket.gethostname(), port), + 'http://127.0.0.1:{}'.format(port), + 'http://::1:{}'.format(port) + ] path_info = self.get_path_and_qs() if 'error' in path_info: @@ -3527,12 +3561,34 @@ class BlockstackAPIEndpointHandler(SimpleHTTPRequestHandler): use_session = whitelist_info['auth_session'] use_password = whitelist_info['auth_pass'] - log.debug("\nfull path: {}\nmethod: {}\npath: {}\nqs: {}\nheaders:\n {}\n".format(self.path, method_name, path_info['path'], qs_values, '\n'.join( '{}: {}'.format(k, v) for (k, v) in self.headers.items() ))) + log.debug("\nfull path: {}\nmethod: {}\npath: {}\nqs: {}\nheaders:\n{}\n".format(self.path, method_name, path_info['path'], qs_values, '\n'.join( '{}: {}'.format(k, v) for (k, v) in self.headers.items() ))) have_password = False session = self.verify_session(qs_values) if not session: - have_password = self.verify_password() + # password authentication + # don't even try to authenticate with the password unless the Origin is set appropriately + if self.verify_origin(LOCALHOST): + # can authenticate + have_password = self.verify_password() + else: + log.warning("Origin is absent or not local") + + else: + # got a session. + # check origin. + app_domain = session['app_domain'] + try: + session_verified = self.verify_origin([app_domain]) + except AssertionError as e: + session = None + err = {'error' : e.message} + return self._reply_json(err, status_code = 403) + + if not session_verified: + # invalid session + log.warning("Invalid session: app domain '{}' does not match Origin '{}'".format(app_domain, self.headers.get('origin', ''))) + session = None authorized = False @@ -3651,7 +3707,7 @@ class BlockstackAPIEndpoint(SocketServer.ThreadingMixIn, SocketServer.TCPServer) (i.e. a mock module with all of the wrapped CLI methods that follow the Python calling convention) """ - name = func.__name__ if name is None else name + name = func_internal.__name__ if name is None else name assert name setattr(self.internal_proxy, name, func_internal) @@ -3786,7 +3842,7 @@ class BlockstackAPIEndpointClient(object): # random ID to match in logs r = random.randint(0, 2 ** 16) if r == -1 else r if self.debug_timeline: - log.debug('RPC({}) {} {} {}'.format(r, event, self.url, key)) + log.debug('RPC({}) {} {}'.format(r, event, key)) return r @@ -3813,6 +3869,7 @@ class BlockstackAPIEndpointClient(object): elif self.session: headers['Authorization'] = 'bearer {}'.format(self.session) + headers['Origin'] = 'http://localhost:{}'.format(DEFAULT_UI_PORT) return headers @@ -4264,7 +4321,9 @@ class BlockstackAPIEndpointClient(object): """ if is_api_server(self.config_dir): # directly get the inode - return data.get_inode_data(blockchain_id, data.datastore_get_id(datastore['pubkey']), inode_uuid, 0, data_pubkeys, datastore['drivers'], datastore['device_ids'], idata=idata, config_path=self.config_path ) + return data.get_inode_data(blockchain_id, data.datastore_get_id(datastore['pubkey']), inode_uuid, 0, + datastore['drivers'], data_pubkeys, idata=idata, + config_path=self.config_path ) else: res = self.check_version() @@ -4432,7 +4491,7 @@ class BlockstackAPIEndpointClient(object): """ if is_api_server(self.config_dir): # delete - return data.datastore_rmtree_put_inodes( datastore, inoes, payloads, signatures, tombstones, config_path=self.config_path ) + return data.datastore_rmtree_put_inodes( datastore, inodes, payloads, signatures, tombstones, config_path=self.config_path ) else: res = self.check_version() @@ -4689,6 +4748,9 @@ def local_api_unlink_pidfile(pidfile_path): except: pass +# used when running in a separate process +rpc_pidpath = None +rpc_srv = None def local_api_atexit(): """ @@ -4762,11 +4824,6 @@ def local_api_start_wait( api_host='localhost', api_port=DEFAULT_API_PORT, confi return running -# used when running in a separate process -rpc_pidpath = None -rpc_srv = None - - def local_api_check_alive(config_path, api_host=None, api_port=None): """ Is the local API daemon alive? @@ -4826,6 +4883,9 @@ def local_api_start( port=None, host=None, config_dir=blockstack_constants.CONFI global rpc_pidpath, rpc_srv, running + if password: + set_secret("BLOCKSTACK_CLIENT_WALLET_PASSWORD", password) + p = subprocess.Popen("pip freeze", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if p.returncode != 0: diff --git a/blockstack_client/schemas.py b/blockstack_client/schemas.py index c71321e13..b9c53df5d 100644 --- a/blockstack_client/schemas.py +++ b/blockstack_client/schemas.py @@ -24,7 +24,7 @@ from __future__ import print_function along with Blockstack-client. If not, see . """ -from constants import * +from .constants import * import blockstack_profiles OP_CONSENSUS_HASH_PATTERN = r'^([0-9a-fA-F]{{{}}})$'.format(LENGTH_CONSENSUS_HASH * 2) @@ -50,9 +50,7 @@ OP_NAME_PATTERN = r'^([a-z0-9\-_.+]{{{},{}}})$'.format(3, LENGTH_MAX_NAME) OP_NAMESPACE_PATTERN = r'^([a-z0-9\-_+]{{{},{}}})$'.format(1, LENGTH_MAX_NAMESPACE_ID) OP_NAMESPACE_HASH_PATTERN = r'^([0-9a-fA-F]{16})$' OP_BASE64_PATTERN_SECTION = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})' -OP_BASE64_URLSAFE_PATTERN_SECTION = r'(?:[A-Za-z0-9\-_]{4})*(?:[A-Za-z0-9\-_]{2}==|[A-Za-z0-9\-_]{3}=|[A-Za-z0-9\-_]{4})' OP_BASE64_PATTERN = r'^({})$'.format(OP_BASE64_PATTERN_SECTION) -OP_BASE64_URLSAFE_PATTERN = r'^({})$'.format(OP_BASE64_URLSAFE_PATTERN_SECTION) OP_URLENCODED_NOSLASH_PATTERN = r'^([a-zA-Z0-9\-_.~%]+)$' # intentionally left out / OP_URLENCODED_NOSLASH_OR_EMPTY_PATTERN = r'^([a-zA-Z0-9\-_.~%]*)$' # intentionally left out /, allow empty OP_URLENCODED_OR_EMPTY_PATTERN = r'^([a-zA-Z0-9\-_.~%/]*)$' @@ -63,9 +61,6 @@ OP_USER_ID_PATTERN = r'^({}+)$'.format(OP_USER_ID_CLASS) OP_DATASTORE_ID_PATTERN = r'^({}+)$'.format(OP_DATASTORE_ID_CLASS) OP_URI_TARGET_PATTERN = r'^([a-z0-9+]+)://([a-zA-Z0-9\-_.~%#?&\\:/=]+)$' OP_URI_TARGET_PATTERN_NOSCHEME = r'^([a-zA-Z0-9\-_.~%#?&\\:/=]+)$' -OP_DNS_NAME_PATTERN_SECTION = '([a-z0-9_\-]+\.)*([a-z0-9\-]+)\.([a-z0-9]+)' -OP_BLOCKSTACK_APP_NAME_PATTERN_SECTION = '[a-z0-9\-_.+]{{{},{}}}'.format(3, LENGTH_MAX_NAME) -OP_APP_NAME_PATTERN = r'^({})\.x|({})\.1$'.format(OP_BLOCKSTACK_APP_NAME_PATTERN_SECTION, OP_DNS_NAME_PATTERN_SECTION) OP_ANY_TYPE_SCHEMA = [ { @@ -1091,6 +1086,7 @@ DATASTORE_LOOKUP_RESPONSE_SCHEMA = { 'type': 'boolean', }, }, + 'additionalProperties': False, 'required': [ 'data', 'status', @@ -1112,6 +1108,7 @@ DATASTORE_LOOKUP_EXTENDED_RESPONSE_SCHEMA = { 'type': 'boolean', }, }, + 'additionalProperties': False, 'required': [ 'path_info', 'inode_info', @@ -1458,45 +1455,6 @@ PROFILE_ACCOUNT_SCHEMA = { } -# single delegated key bundle -KEY_DELEGATION_DEVICE_RECORD_SCHEMA = { - 'type': 'object', - 'properties': { - 'app': { - 'type': 'string', - 'pattern': OP_PUBKEY_PATTERN, - }, - 'enc': { - 'type': 'string', - 'pattern': OP_PUBKEY_PATTERN, - }, - 'sign': { - 'type': 'string', - 'pattern': OP_PUBKEY_PATTERN, - }, - 'index': { - 'type': 'integer', - 'minimum': 0, - 'maximum': 2**31 - 1, - }, - }, - 'required': [ - 'app', - 'enc', - 'sign', - 'index', - ], -} - - -# set of device-specific key delegations -KEY_DELEGATION_DEVICES_SCHEMA = { - 'type': 'object', - 'patternProperties': { - '.+': KEY_DELEGATION_DEVICE_RECORD_SCHEMA, - }, -} - # key delegation schema KEY_DELEGATION_SCHEMA = { 'type': 'object', @@ -1509,7 +1467,40 @@ KEY_DELEGATION_SCHEMA = { 'type': 'string', 'pattern': OP_NAME_PATTERN, }, - 'devices': KEY_DELEGATION_DEVICES_SCHEMA, + 'devices': { + 'type': 'object', + 'patternProperties': { + '^.+$': { + 'type': 'object', + 'properties': { + 'app': { + 'type': 'string', + 'pattern': OP_PUBKEY_PATTERN, + }, + 'enc': { + 'type': 'string', + 'pattern': OP_PUBKEY_PATTERN, + }, + 'sign': { + 'type': 'string', + 'pattern': OP_PUBKEY_PATTERN, + }, + 'index': { + 'type': 'integer', + 'minimum': 0, + 'maximum': 2**31 - 1, + }, + }, + 'required': [ + 'app', + 'enc', + 'sign', + 'index', + ], + 'additionalProperties': False, + }, + }, + }, }, 'required': [ 'version', @@ -1531,7 +1522,7 @@ APP_KEY_BUNDLE_SCHEMA = { 'apps': { 'type': 'object', 'patternProperties': { - OP_APP_NAME_PATTERN: { + OP_NAME_PATTERN: { 'type': 'string', 'pattern': OP_PUBKEY_PATTERN, }, @@ -1554,56 +1545,26 @@ BLOCKSTACK_TOKEN_FILE_SCHEMA = { 'type': 'string', 'pattern': '^3\.0$', }, - 'profile': { - 'type': 'string', - 'pattern': '.+', - }, + 'profile': blockstack_profiles.person.PERSON_SCHEMA, 'keys': { - 'type': 'object', - 'properties': { - 'name': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': OP_PUBKEY_PATTERN, - }, - }, - 'delegation': { - 'type': 'string', - 'pattern': '.+', # KEY_DELEGATION_SCHEMA JWT - }, - 'apps': { - 'type': 'object', - 'patternProperties': { - '.+': { # device ID - 'type': 'string', - 'pattern': '.+', # APP_KEY_BUNDLE_SCHEMA JWT - }, - }, + 'delegation': KEY_DELEGATION_SCHEMA, + 'apps': { + 'type': 'object', + 'patternProperties': { + '^.+$': APP_KEY_BUNDLE_SCHEMA }, }, 'required': [ - 'name', 'delegation', 'apps', ], 'additionalProperties': False, }, - 'writes': { - 'type': 'integer', - 'minimum': 0, - }, - 'timestamp': { - 'type': 'integer', - 'minimum': 0, - }, }, 'required': [ 'version', 'profile', 'keys', - 'writes', - 'timestamp', ], 'additionalProperties': False, } diff --git a/blockstack_client/scripts.py b/blockstack_client/scripts.py index fd6e5a60c..1d7ce0234 100644 --- a/blockstack_client/scripts.py +++ b/blockstack_client/scripts.py @@ -33,11 +33,11 @@ from virtualchain.lib.hashing import * from virtualchain import tx_extend, tx_sign_input -from b40 import * -from constants import MAGIC_BYTES, NAME_OPCODES, LENGTH_MAX_NAME, LENGTH_MAX_NAMESPACE_ID, TX_MIN_CONFIRMATIONS -from keys import * -from utxo import get_unspents -from logger import get_logger +from .b40 import * +from .constants import MAGIC_BYTES, NAME_OPCODES, LENGTH_MAX_NAME, LENGTH_MAX_NAMESPACE_ID, TX_MIN_CONFIRMATIONS +from .keys import * +from .utxo import get_unspents +from .logger import get_logger log = get_logger('blockstack-client') @@ -262,7 +262,7 @@ def tx_make_subsidization_output(payer_utxo_inputs, payer_address, op_fee, dust_ def tx_make_subsidizable(blockstack_tx, fee_cb, max_fee, subsidy_key_info, utxo_client, tx_fee=0, - subsidy_address=None, add_dust_fee=True): + subsidy_address=None, add_dust_fee=True, simulated_sign = False): """ Given an unsigned serialized transaction from Blockstack, make it into a subsidized transaction for the client to go sign off on. @@ -271,8 +271,10 @@ def tx_make_subsidizable(blockstack_tx, fee_cb, max_fee, subsidy_key_info, utxo_ * Sign our inputs with SIGHASH_ANYONECANPAY (if subsidy_key_info is not None) @tx_fee should be in fundamental units (i.e. satoshis) + @simulated_sign tells us not to actually sign, but just compute expected sig lengths - Returns the transaction; signed if subsidy_key_info is given; unsigned otherwise + Returns the transaction; signed if subsidy_key_info is given; unsigned otherwise; + if simulated_sign, returns a tuple (unsigned tx, expected length of hex encoded signatures) Returns None if we can't get subsidy info Raise ValueError if there are not enough inputs to subsidize """ @@ -330,18 +332,31 @@ def tx_make_subsidizable(blockstack_tx, fee_cb, max_fee, subsidy_key_info, utxo_ # sign each of our inputs with our key, but use # SIGHASH_ANYONECANPAY so the client can sign its inputs - if subsidy_key_info is not None: + log.debug("Length of unsigned subsidized = {}".format(len(subsidized_tx))) + + unsigned = subsidized_tx + if subsidy_key_info is not None and not simulated_sign: for i in range(len(consumed_inputs)): idx = i + len(tx_inputs) subsidized_tx = tx_sign_input( subsidized_tx, idx, subsidy_key_info, hashcode=virtualchain.SIGHASH_ANYONECANPAY ) - + elif simulated_sign: + return subsidized_tx, 2*(len(consumed_inputs) * tx_estimate_signature_len_bytes(subsidy_key_info)) else: log.debug("Warning: no subsidy key given; transaction will be subsidized but not signed") return subsidized_tx +def tx_estimate_signature_len_bytes(privkey_info): + if virtualchain.is_singlesig(privkey_info): + return 73 + else: + m, _ = virtualchain.parse_multisig_redeemscript( privkey_info['redeem_script'] ) + siglengths = 74 * m + scriptlen = len(privkey_info['redeem_script']) / 2 + return 6 + scriptlen + siglengths + def tx_get_unspents(address, utxo_client, min_confirmations=None): """ diff --git a/blockstack_client/storage.py b/blockstack_client/storage.py index 513e74538..ab5712b95 100644 --- a/blockstack_client/storage.py +++ b/blockstack_client/storage.py @@ -35,7 +35,7 @@ import time import blockstack_zones import blockstack_profiles -from logger import get_logger +from .logger import get_logger from constants import BLOCKSTACK_TEST, BLOCKSTACK_DEBUG, BLOCKSTACK_STORAGE_CLASSES from config import get_config from scripts import hex_hash160 @@ -73,7 +73,7 @@ def get_zonefile_data_hash(data_txt): Generate a hash over a user's zonefile. Return the hex string. """ - return hex_hash160(str(data_txt)) + return hex_hash160(data_txt) def get_blockchain_compat_hash(data_txt): @@ -319,7 +319,7 @@ def parse_signed_data_tombstone( tombstone_data ): return {'id': parts2[0], 'signature': parts2[1], 'timestamp': ts} -def serialize_mutable_data(data_text_or_json, data_privkey=None, data_pubkey=None, data_signature=None): +def serialize_mutable_data(data_text_or_json, data_privkey=None, data_pubkey=None, data_signature=None, profile=False): """ Generate a serialized mutable data record from the given information. Sign it with privatekey. @@ -329,24 +329,39 @@ def serialize_mutable_data(data_text_or_json, data_privkey=None, data_pubkey=Non Return the serialized data (as a string) on success """ + + if profile: + # private key required to generate signature + assert data_privkey is not None - # version 2 format for mutable data - assert data_privkey or (data_pubkey and data_signature) + # profiles must conform to a particular standard format + tokenized_data = blockstack_profiles.sign_token_records( + [data_text_or_json], data_privkey + ) - if data_signature is None: - assert isinstance(data_text_or_json, (str, unicode)), "data must be a string" - data_str = str(data_text_or_json) - data_signature = sign_data_payload( data_str, data_privkey ) + del tokenized_data[0]['decodedToken'] - # make sure it's compressed - if data_pubkey is None: - data_pubkey = get_pubkey_hex(data_privkey) + serialized_data = json.dumps(tokenized_data, sort_keys=True) + return serialized_data + + else: + # version 2 format for mutable data + assert data_privkey or (data_pubkey and data_signature) - pubkey_hex_compressed = keylib.key_formatting.compress(data_pubkey) - data_payload = serialize_data_payload( data_text_or_json ) - res = "bsk2.{}.{}.{}".format(pubkey_hex_compressed, data_signature, data_payload) + if data_signature is None: + assert isinstance(data_text_or_json, (str, unicode)), "data must be a string" + data_str = str(data_text_or_json) + data_signature = sign_data_payload( data_str, data_privkey ) - return res + # make sure it's compressed + if data_pubkey is None: + data_pubkey = get_pubkey_hex(data_privkey) + + pubkey_hex_compressed = keylib.key_formatting.compress(data_pubkey) + data_payload = serialize_data_payload( data_text_or_json ) + res = "bsk2.{}.{}.{}".format(pubkey_hex_compressed, data_signature, data_payload) + + return res def parse_mutable_data_v2(mutable_data_json_txt, public_key_hex, public_key_hash=None, data_hash=None, raw=False): @@ -442,7 +457,9 @@ def parse_mutable_data_v2(mutable_data_json_txt, public_key_hex, public_key_hash if public_key_hash is not None: pubkey_hash = keylib.address_formatting.bin_hash160_to_address( - keylib.address_formatting.address_to_bin_hash160(str(public_key_hash)), + keylib.address_formatting.address_to_bin_hash160( + str(public_key_hash), + ), version_byte=0 ) @@ -471,8 +488,6 @@ def parse_mutable_data(mutable_data_json_txt, public_key, public_key_hash=None, parse it into a JSON document. Verify that it was signed by public_key's or public_key_hash's private key. - Works on bsk2 data, as well as profiles (but not token files) - Try to verify with both keys, if given. Return the parsed JSON dict on success @@ -491,9 +506,7 @@ def parse_mutable_data(mutable_data_json_txt, public_key, public_key_hash=None, return parse_mutable_data_v2(mutable_data_json_txt, public_key, public_key_hash=public_key_hash, data_hash=data_hash, raw=raw) - # Legacy parser for profiles. - # Does not work with token files or Gaia datastore info. - + # legacy parser assert public_key is not None or public_key_hash is not None, 'Need a public key or public key hash' mutable_data_jwt = None @@ -509,7 +522,9 @@ def parse_mutable_data(mutable_data_json_txt, public_key, public_key_hash=None, # try pubkey, if given if public_key is not None: - mutable_data_json = blockstack_profiles.get_profile_from_tokens(mutable_data_jwt, str(public_key)) + mutable_data_json = blockstack_profiles.get_profile_from_tokens( + mutable_data_jwt, str(public_key) + ) if len(mutable_data_json) > 0: return mutable_data_json @@ -528,7 +543,9 @@ def parse_mutable_data(mutable_data_json_txt, public_key, public_key_hash=None, version_byte=0 ) - mutable_data_json = blockstack_profiles.get_profile_from_tokens(mutable_data_jwt, public_key_hash_0) + mutable_data_json = blockstack_profiles.get_profile_from_tokens( + mutable_data_jwt, public_key_hash_0 + ) if len(mutable_data_json) > 0: log.debug('Verified with {}'.format(public_key_hash)) @@ -692,7 +709,7 @@ def get_immutable_data(data_hash, data_url=None, hash_func=get_data_hash, fqu=No urlh.close() except Exception as e: log.exception(e) - msg = 'Failed to load data from "{}"' + msg = 'Failed to load profile from "{}"' log.error(msg.format(data_url)) continue else: @@ -769,12 +786,12 @@ def get_driver_urls( fq_data_id, storage_drivers ): return ret -def get_mutable_data(fq_data_id, data_pubkeys, urls=None, data_addresses=None, data_hash=None, - drivers=None, decode=True, bsk_version=None, **driver_kw): +def get_mutable_data(fq_data_id, data_pubkey, urls=None, data_address=None, data_hash=None, + owner_address=None, blockchain_id=None, drivers=None, decode=True, bsk_version=None): """ Low-level call to get mutable data, given a fully-qualified data name. - if decode is False, then data_pubkeys and data_addresses are not needed and raw bytes will be returned. + if decode is False, then data_pubkey, data_address, and owner_address are not needed and raw bytes will be returned. Return a mutable data dict on success (or raw bytes if decode=False) Return None on error @@ -782,7 +799,10 @@ def get_mutable_data(fq_data_id, data_pubkeys, urls=None, data_addresses=None, d global storage_handlers - assert data_pubkeys or data_addresses or data_hash or not decode, "BUG: no means to decode data" + # fully-qualified username hint + fqu = None + if blockchain_id is not None: + fqu = blockchain_id handlers_to_use = [] if drivers is None: @@ -795,19 +815,16 @@ def get_mutable_data(fq_data_id, data_pubkeys, urls=None, data_addresses=None, d ) # ripemd160(sha256(pubkey)) - encoded_data_pubkey_hashes = [] data_pubkey_hashes = [] - if data_addresses: - for a in filter(lambda x: x is not None, data_addresses): - try: - h = keylib.b58check.b58check_decode(str(a)).encode('hex') - data_pubkey_hashes.append(h) - encoded_data_pubkey_hashes.append(a) - except: - log.debug("Invalid address '{}'".format(a)) - continue + for a in filter(lambda x: x is not None, [data_address, owner_address]): + try: + h = keylib.b58check.b58check_decode(str(a)).encode('hex') + data_pubkey_hashes.append(h) + except: + log.debug("Invalid address '{}'".format(a)) + continue - log.debug('get_mutable_data {} bsk_version={}'.format(fq_data_id, bsk_version)) + log.debug('get_mutable_data {} fqu={} bsk_version={}'.format(fq_data_id, fqu, bsk_version)) for storage_handler in handlers_to_use: if not getattr(storage_handler, 'get_mutable_handler', None): continue @@ -853,7 +870,7 @@ def get_mutable_data(fq_data_id, data_pubkeys, urls=None, data_addresses=None, d log.debug('Try {} ({})'.format(storage_handler.__name__, url)) try: - data_txt = storage_handler.get_mutable_handler(url, data_pubkeys=data_pubkeys, data_pubkey_hashes=data_pubkey_hashes, **driver_kw) + data_txt = storage_handler.get_mutable_handler(url, fqu=fqu, data_pubkey=data_pubkey, data_pubkey_hashes=data_pubkey_hashes) except UnhandledURLException as uue: # handler doesn't handle this URL msg = 'Storage handler {} does not handle URLs like {}' @@ -872,30 +889,19 @@ def get_mutable_data(fq_data_id, data_pubkeys, urls=None, data_addresses=None, d # parse it, if desired if decode: data = None - if data_pubkeys: - # try public keys - for data_pubkey in data_pubkeys: - log.debug("Try to verify {} bytes with public key {}".format(len(data_txt), data_pubkey)) - data = parse_mutable_data(data_txt, data_pubkey, data_hash=data_hash, bsk_version=bsk_version) - if data is not None: - break + if data_pubkey is not None or data_address is not None or data_hash is not None: + data = parse_mutable_data( + data_txt, data_pubkey, public_key_hash=data_address, data_hash=data_hash, bsk_version=bsk_version + ) + + if data is None and owner_address is not None: + data = parse_mutable_data( + data_txt, None, public_key_hash=owner_address, bsk_version=bsk_version + ) - if data is None and len(encoded_data_pubkey_hashes) > 0: - # try public key hashes - for pubkey_hash in encoded_data_pubkey_hashes: - log.debug("Try to verify {} bytes with public key hash {}".format(len(data_txt), pubkey_hash)) - data = parse_mutable_data(data_txt, None, public_key_hash=pubkey_hash, data_hash=data_hash, bsk_version=bsk_version) - if data is not None: - break - - if data_hash is not None and (data_pubkeys is None or len(data_pubkeys) == 0) and len(encoded_data_pubkey_hashes) == 0: - # try data hash - log.debug("Try to verify {} bytes with data hash {}".format(len(data_txt), data_hash)) - data = parse_mutable_data(data_txt, None, data_hash=data_hash, bsk_version=bsk_version) - if data is None: - # out of options - log.error("Unparseable data from '{}'".format(url)) + msg = 'Unparseable data from "{}"' + log.error(msg.format(url)) continue msg = 'Loaded "{}" with {}' @@ -986,7 +992,7 @@ def put_immutable_data(data_text, txid, data_hash=None, required=None, skip=None return None if successes == 0 and required_successes == len(set(required) - set(skip)) else data_hash -def put_mutable_data(fq_data_id, data_text, raw=False, data_privkey=None, data_pubkey=None, data_signature=None, required=None, skip=None, required_exclusive=False, **driver_kw): +def put_mutable_data(fq_data_id, data_text_or_json, sign=True, raw=False, data_privkey=None, data_pubkey=None, data_signature=None, profile=False, blockchain_id=None, required=None, skip=None, required_exclusive=False): """ Given the unserialized data, store it into our mutable data stores. Do so in a best-effort way. This method fails if all storage providers fail, @@ -995,7 +1001,8 @@ def put_mutable_data(fq_data_id, data_text, raw=False, data_privkey=None, data_p @required: list of required drivers to use. All of them must succeed for this method to succeed. @skip: list of drivers we can skip. None of them will be tried. @required_exclusive: if True, then only the required drivers will be tried (none of the loaded but not-required drivers will be invoked) - @raw: If True, then the data will be put as-is without any ancilliary metadata. + @sign: if True, then a private key is required. if False, then simply store the data without serializing it or including a public key and signature. + @raw: If True, then the data will be put as-is without any ancilliary metadata. Requires sign=False Return True on success Return False on error @@ -1004,9 +1011,9 @@ def put_mutable_data(fq_data_id, data_text, raw=False, data_privkey=None, data_p global storage_handlers assert len(storage_handlers) > 0, "No storage handlers initialized" - # sanity check: only take structured data if this is a token file - if not isinstance(data_text, (str, unicode)): - raise ValueError("Only takes strings") + # sanity check: only take structured data if this is a profile + if not isinstance(data_text_or_json, (str, unicode)): + assert profile, "Structured data is only supported when profile=True" required = [] if required is None else required skip = [] if skip is None else skip @@ -1015,6 +1022,11 @@ def put_mutable_data(fq_data_id, data_text, raw=False, data_privkey=None, data_p log.debug('put_mutable_data({}), required={}, skip={} required_exclusive={}'.format(fq_data_id, ','.join(required), ','.join(skip), required_exclusive)) + # fully-qualified username hint + fqu = None + if blockchain_id is not None: + fqu = blockchain_id + # sanity check: only support single-sig private keys if data_privkey is not None: if not is_singlesig_hex(data_privkey): @@ -1023,12 +1035,15 @@ def put_mutable_data(fq_data_id, data_text, raw=False, data_privkey=None, data_p data_pubkey = get_pubkey_hex( data_privkey ) + elif sign: + assert data_pubkey is not None + assert data_signature is not None + serialized_data = None - if not raw: - assert data_privkey or data_signature - serialized_data = serialize_mutable_data(data_text, data_privkey=data_privkey, data_pubkey=data_pubkey, data_signature=data_signature) + if sign or not raw: + serialized_data = serialize_mutable_data(data_text_or_json, data_privkey=data_privkey, data_pubkey=data_pubkey, data_signature=data_signature, profile=profile) else: - serialized_data = data_text + serialized_data = data_text_or_json if BLOCKSTACK_TEST: log.debug("data ({}): {}".format(type(serialized_data), serialized_data)) @@ -1057,7 +1072,7 @@ def put_mutable_data(fq_data_id, data_text, raw=False, data_privkey=None, data_p log.debug('Try "{}"'.format(handler.__name__)) try: - rc = handler.put_mutable_handler(fq_data_id, serialized_data, **driver_kw) + rc = handler.put_mutable_handler(fq_data_id, serialized_data, fqu=fqu, profile=profile) except Exception as e: log.exception(e) if handler.__name__ not in required: @@ -1122,7 +1137,7 @@ def delete_immutable_data(data_hash, txid, privkey=None, signed_data_tombstone=N return True -def delete_mutable_data(fq_data_id, privatekey=None, signed_data_tombstone=None, required=None, required_exclusive=False, skip=None, blockchain_id=None, token_file=False): +def delete_mutable_data(fq_data_id, privatekey=None, signed_data_tombstone=None, required=None, required_exclusive=False, skip=None, blockchain_id=None, profile=False): """ Given the data ID and private key of a user, go and delete the associated mutable data. @@ -1151,7 +1166,7 @@ def delete_mutable_data(fq_data_id, privatekey=None, signed_data_tombstone=None, if signed_data_tombstone is None: assert privatekey ts = make_data_tombstone(fq_data_id) - signed_data_tombstone = sign_data_tombstone(ts, privkey) + signed_data_tombstone = sign_data_tombstone(ts, privatekey) required_successes = 0 @@ -1170,7 +1185,7 @@ def delete_mutable_data(fq_data_id, privatekey=None, signed_data_tombstone=None, rc = False try: - rc = handler.delete_mutable_handler(fq_data_id, signed_data_tombstone, fqu=fqu, token_file=token_file) + rc = handler.delete_mutable_handler(fq_data_id, signed_data_tombstone, fqu=fqu, profile=profile) except Exception as e: log.exception(e) rc = False diff --git a/blockstack_client/token_file.py b/blockstack_client/token_file.py deleted file mode 100644 index 03cf55a1f..000000000 --- a/blockstack_client/token_file.py +++ /dev/null @@ -1,1223 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - Blockstack-client - ~~~~~ - copyright: (c) 2014-2015 by Halfmoon Labs, Inc. - copyright: (c) 2016 by Blockstack.org - - This file is part of Blockstack-client. - - Blockstack-client is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Blockstack-client is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with Blockstack-client. If not, see . -""" - -# This module contains the code needed to generate and authenticate key-delegation JWTs. -# This is NOT the Blockstack Token code. - -import schemas -import storage -import user as user_db - -from constants import BLOCKSTACK_TEST, BLOCKSTACK_DEBUG, CONFIG_PATH -from proxy import get_default_proxy, get_name_blockchain_record -from zonefile import get_name_zonefile - -import keychain -import virtualchain -from virtualchain.lib import ecdsalib -import blockstack_profiles - -from keys import HDWallet, get_app_root_privkey, get_signing_privkey, get_encryption_privkey - -import copy -import time -import json -import jsontokens -import jsonschema -from jsonschema import ValidationError - -import keylib - -from logger import get_logger -log = get_logger() - - -def token_file_get_name_public_keys(token_file, name_addr): - """ - Given the parsed (but not yet verified) token file and an address, get the public keys - from the token file if they match the address. - - Return {'status': True, 'public_keys': [...]} on success - Return {'error': ...} if the public keys in the token file do not match the address - """ - - name_addr = virtualchain.address_reencode(str(name_addr)) - name_owner_pubkeys = [] - - if virtualchain.is_multisig_address(name_addr): - - public_keys = token_file['keys']['name'] - if name_addr != virtualchain.make_multisig_address(public_keys, len(public_keys)): - return {'error': 'Multisig address {} does not match public keys {}'.format(name_addr, ','.join(public_keys))} - - # match! - name_owner_pubkeys = [str(pubk) for pubk in public_keys] - - elif virtualchain.is_singlesig_address(name_addr): - - public_keys = token_file['keys']['name'] - for public_key in public_keys: - if virtualchain.address_reencode(keylib.public_key_to_address(str(public_key))) == name_addr: - name_owner_pubkeys = [str(public_key)] - break - - if len(name_owner_pubkeys) == 0: - # no match - return {'error': 'Address {} does not match any public key {}'.format(name_addr, ','.join(public_keys))} - - else: - # invalid - return {'error': 'Invalid address {}'.format(name_owner_pubkeys_or_addr)} - - return {'status': True, 'public_keys': name_owner_pubkeys} - - -def token_file_make_datastore_index(apps): - """ - Given the .keys.apps section of the token file, generate an index - that maps datastore IDs onto application names. - Return {'status': True, 'index': {'$datastore_id': '$app_name'}} on success - """ - from data import datastore_get_id - index = {} - for dev_id in apps.keys(): - dev_apps = apps[dev_id]['apps'] - for app_name in dev_apps.keys(): - datastore_id = datastore_get_id(dev_apps[app_name]) - index[datastore_id] = app_name - - return {'status': True, 'index': index} - - -def token_file_get_application_name(token_file, datastore_id): - """ - Given a parsed token file and a datastore ID, find the application domain. - Return {'status': True, 'full_application_name': ...} on success - Return {'error': ...} on failure - """ - if 'datastore_index' not in token_file: - raise ValueError("Token file does not have a datastore index") - - full_application_name = token_file['datastore_index'].get(datastore_id) - if full_application_name is None: - return {'error': 'No application name for "{}"'.format(datastore_id)} - - return {'status': True, 'full_application_name': full_application_name} - - -def token_file_parse(token_txt, name_owner_pubkeys_or_addr, min_writes=None): - """ - Given a compact-format JWT encoding a token file, this device's name-owner private key, and the list of name-owner public keys, - go verify that the token file is well-formed and authentic. - Return {'status': True, 'token_file': the parsed, decoded token file} on success - Return {'error': ...} on error - """ - unverified_token_file = None - unverified_profile = None - unverified_apps = None - - signing_public_keys = {} - app_public_keys = {} - - token_file = None - profile_jwt_txt = None - delegation_jwt_txt = None - delegation_jwt = None - delegation_file = None - profile = None - apps = {} - apps_jwts_txt = {} - - name_owner_pubkeys = [] - - # get the delegation file out of the token file - try: - unverified_token_file = jsontokens.decode_token(token_txt)['payload'] - except jsontokens.utils.DecodeError: - return {'error': 'Invalid token file: not a JWT'} - - try: - jsonschema.validate(unverified_token_file, schemas.BLOCKSTACK_TOKEN_FILE_SCHEMA) - except ValidationError as ve: - if BLOCKSTACK_TEST: - log.exception(ve) - - return {'error': 'Invalid token file: does not match token file schema'} - - except Exception as e: - log.exception(e) - return {'error': 'Invalid token file: failed to parse'} - - try: - delegation_jwt_txt = unverified_token_file['keys']['delegation'] - try: - delegation_jwt = json.loads(delegation_jwt_txt) - except ValueError: - delegation_jwt = delegation_jwt_txt - - except ValueError as ve: - if BLOCKSTACK_TEST: - log.exception(ve) - - return {'error': 'Invalid delegation file'} - - # if we're given an address (b58check-encoded hash of a public key or list of pbulic keys), - # see if we can authenticate based on the keys given - if isinstance(name_owner_pubkeys_or_addr, (str, unicode)): - res = token_file_get_name_public_keys(unverified_token_file, str(name_owner_pubkeys_or_addr)) - if 'error' in res: - return res - - name_owner_pubkeys = res['public_keys'] - - else: - if not isinstance(name_owner_pubkeys_or_addr, list): - return {'error': 'Not a valid address or list: {}'.format(name_owner_pubkeys_or_addr)} - - name_owner_pubkeys = [str(pubk) for pubk in name_owner_pubkeys_or_addr] - - # authenticate the delegation file with the name owner public keys - try: - delegation_verifier = jsontokens.TokenVerifier() - - if len(name_owner_pubkeys) > 1: - assert delegation_verifier.verify(delegation_jwt, name_owner_pubkeys) - else: - assert delegation_verifier.verify(delegation_jwt, name_owner_pubkeys[0]) - - except AssertionError as ae: - if BLOCKSTACK_TEST: - log.exception(ae) - - return {'error': 'Delegation file verification failed'} - - # decode the delegation file - try: - delegation_file = jsontokens.decode_token(delegation_jwt)['payload'] - except Exception as e: - if BLOCKSTACK_TEST: - log.exception(e) - - return {'error': 'Invalid delegation file: failed to parse'} - - # have verified, well-formed delegation file - # extract signing public keys and app public keys - for device_id in delegation_file['devices'].keys(): - try: - signing_public_keys[device_id] = keylib.ECPublicKey(str(delegation_file['devices'][device_id]['sign'])).to_hex() - except Exception as e: - log.exception(e) - return {'error': 'Invalid signing public key for device "{}"'.format(device_id)} - - # validate the rest of the public keys - for key_type in ['app', 'enc']: - try: - keylib.ECPublicKey(str(delegation_file['devices'][device_id][key_type])) - except Exception as e: - log.exception(e) - return {'error': 'Invalid public key "{}" for device "{}"'.format(key_type, device_id)} - - # verify the token file, using any of the signing keys - for (device_id, signing_public_key) in signing_public_keys.items(): - try: - token_file_verifier = jsontokens.TokenVerifier() - token_file_valid = token_file_verifier.verify(token_txt, signing_public_key) - assert token_file_valid - - # success! - token_file = unverified_token_file - break - - except AssertionError as ae: - continue - - if not token_file: - # unverifiable - return {'error': 'Failed to verify token file with name owner public keys'} - - # the device IDs in the delegation file must include all of the device IDs in the app key bundles - for device_id in token_file['keys']['apps'].keys(): - if device_id not in delegation_file['devices'].keys(): - return {'error': 'Application key bundle contains a non-delegated device ID "{}"'.format(device_id)} - - # now go verify the profile, using any of the signing public keys - for (device_id, signing_public_key) in signing_public_keys.items(): - try: - profile_jwt_txt = token_file['profile'] - profile = storage.parse_mutable_data(profile_jwt_txt, signing_public_key) - assert profile - - # success - break - except AssertionError as ae: - continue - - if profile is None: - return {'error': 'Failed to verify profile using signing keys in delegation file'} - - # verify app key bundles, using each device's respective public key - for (device_id, signing_public_key) in signing_public_keys.items(): - if not token_file['keys']['apps'].has_key(device_id): - continue - - apps_jwt_txt = token_file['keys']['apps'][device_id] - try: - apps_verifier = jsontokens.TokenVerifier() - apps_is_valid = apps_verifier.verify(apps_jwt_txt, signing_public_key) - assert apps_is_valid - - # valid! but well-formed? - app_token = jsontokens.decode_token(apps_jwt_txt)['payload'] - jsonschema.validate(app_token, schemas.APP_KEY_BUNDLE_SCHEMA) - - # valid and well-formed! - apps[device_id] = app_token - apps_jwts_txt[device_id] = apps_jwt_txt - - except AssertionError as ae: - return {'error': 'Application key bundle for "{}" has an invalid signature'.format(device_id)} - - except ValidationError as ve: - if BLOCKSTACK_TEST: - log.exception(ve) - - return {'error': 'Application key bundle for "{}" is not well-formed'.format(device_id)} - - # verify fresh - if min_writes is not None: - if token_file['writes'] < min_writes: - return {'error': 'Stale token file with only {} writes'.format(token_file['writes'])} - - # map datastore_id to names - res = token_file_make_datastore_index(apps) - if 'error' in res: - return {'error': 'Failed to build datastore index: {}'.format(res['error'])} - - datastore_index = res['index'] - - # success! - token_file_data = { - 'profile': profile, - 'keys': { - 'name': token_file['keys']['name'], - 'delegation': delegation_file, - 'apps': apps, - }, - 'writes': token_file['writes'], - 'timestamp': token_file['timestamp'], - 'jwts': { - 'profile': profile_jwt_txt, - 'keys': { - 'name': token_file['keys']['name'], - 'delegation': delegation_jwt_txt, - 'apps': apps_jwts_txt, - }, - }, - 'datastore_index': datastore_index - } - - return {'status': True, 'token_file': token_file_data} - - -def token_file_make_delegation_entry(name_owner_privkey, device_id, key_index): - """ - Make a delegation file entry for a specific device. - Returns {'status': True, 'delegation': delegation entry, 'private_keys': delegation private keys} - """ - signing_privkey = get_signing_privkey(name_owner_privkey) - encryption_privkey = get_encryption_privkey(name_owner_privkey) - app_privkey = get_app_root_privkey(name_owner_privkey) - - delg = { - 'app': ecdsalib.get_pubkey_hex(app_privkey), - 'enc': ecdsalib.get_pubkey_hex(encryption_privkey), - 'sign': ecdsalib.get_pubkey_hex(signing_privkey), - 'index': key_index - } - - privkeys = { - 'app': app_privkey, - 'enc': encryption_privkey, - 'sign': signing_privkey - } - - return {'status': True, 'delegation': delg, 'private_keys': privkeys} - - -def token_file_get_key_order(name_owner_privkeys, pubkeys): - """ - Given the device -> privkey owner mapping, and a list of public keys - (e.g. from an on-chain multisig redeem script), calculate the key-signing order - (e.g. to be fed into token_file_create()) - - Return {'status': True, 'key_order': [...]} on success - Return {'error': ...} on failure - """ - key_order = [None] * len(name_owner_pubkeys) - for (dev_id, privkey) in name_owner_privkeys.items(): - compressed_form = keylib.key_formatting.compress(keylib.ECPrivateKey(privkey).public_key().to_hex()) - uncompressed_form = keylib.key_formatting.decompress(keylib.ECPrivateKey(privkey).public_key().to_hex()) - - index = None - if compressed_form in pubkeys: - index = pubkeys.index(compressed_form) - - elif uncompressed_form in pubkeys: - index = pubkeys.index(uncompressed_form) - - else: - return {'error': 'Public key {} is not present in name owner keys'.format(compressed_form)} - - key_order[index] = dev_id - - return {'status': True, 'key_order': key_order} - - -def token_file_create(name, name_owner_privkeys, device_id, key_order=None, write_version=1, apps=None, profile=None, delegations=None, config_path=CONFIG_PATH): - """ - Make a new token file from a profile. Sign and serialize the delegations file, - and sign and serialize each of the app bundles. - - @name_owner_privkeys is a dict of {'$device_id': '$private_key'} - @apps is a dict of {'$device_id': {'$app_name': '$app_public_key'}} - - Return {'status': True, 'token_file': compact-serialized JWT} on success, signed with this device's signing key. - Return {'error': ...} on error - """ - if apps is None: - # default - apps = {} - for dev_id in name_owner_privkeys.keys(): - apps[dev_id] = {'version': '1.0', 'apps': {}} - - if profile is None: - # default - profile = user_db.make_empty_user_profile(config_path=config_path) - - if delegations is None: - # default - delegations = { - 'version': '1.0', - 'name': name, - 'devices': {}, - } - - for dev_id in name_owner_privkeys.keys(): - delg = token_file_make_delegation_entry(name_owner_privkeys[dev_id], dev_id, 0)['delegation'] - delegations['devices'][dev_id] = delg - - # sanity check: apps must be per-device app key bundles - for dev_id in apps.keys(): - try: - jsonschema.validate(apps[dev_id], schemas.APP_KEY_BUNDLE_SCHEMA) - except ValidationError as e: - if BLOCKSTACK_TEST: - log.exception(e) - - return {'error': 'Invalid app bundle'} - - # sanity check: delegations must be well-formed - try: - jsonschema.validate(delegations, schemas.KEY_DELEGATION_SCHEMA) - except ValidationError as e: - if BLOCKSTACK_TEST: - log.exception(e) - - return {'error': 'Invalid key delegations object'} - - try: - jsonschema.validate(profile, blockstack_profiles.person.PERSON_SCHEMA) - except ValidationError as e: - if BLOCKSTACK_TEST: - log.exception(e) - - return {'error': 'Invalid profile'} - - device_specific_name_owner_privkey = name_owner_privkeys[device_id] - - # derive the appropriate signing keys - signing_keys = dict([(dev_id, get_signing_privkey(name_owner_privkeys[dev_id])) for dev_id in name_owner_privkeys.keys()]) - signing_public_keys = dict([(dev_id, ecdsalib.get_pubkey_hex(signing_keys[dev_id])) for dev_id in signing_keys.keys()]) - - # make profile jwt - profile_jwt_txt = token_file_profile_serialize(profile, signing_keys[device_id]) - - # make delegation jwt (to be signed by each name owner key) - signer = jsontokens.TokenSigner() - - # store compact-form delegation JWT if there is one signature - delegation_jwt_txt = None - if len(name_owner_privkeys) == 1: - delegation_jwt = signer.sign(delegations, name_owner_privkeys.values()[0]) - delegation_jwt_txt = delegation_jwt - else: - delegation_jwt = signer.sign(delegations, name_owner_privkeys.values()) - delegation_jwt_txt = json.dumps(delegation_jwt) - - # make the app jwt - apps_jwt_txt = {} - for dev_id in apps.keys(): - signing_privkey = signing_keys.get(dev_id) - if signing_privkey is None: - raise ValueError("No key for {}".format(dev_id)) - - signer = jsontokens.TokenSigner() - app_jwt_txt = signer.sign(apps[dev_id], signing_privkey) - - # only want the token - apps_jwt_txt[dev_id] = app_jwt_txt - - # name public keys are alphabetically sorted on device ID upon creation by default. - # otherwise, follow a key order - name_owner_pubkeys = [] - if key_order is None: - for dev_id in sorted(name_owner_privkeys.keys()): - name_owner_pubkeys.append( keylib.key_formatting.compress(ecdsalib.get_pubkey_hex(name_owner_privkeys[dev_id])) ) - - else: - if len(key_order) != len(name_owner_privkeys.keys()): - return {'error': 'Invalid key order: length mismatch'} - - for dev_id in key_order: - if dev_id not in name_owner_privkeys.keys(): - return {'error': 'Invalid key order: device "{}" not present in private key set'.format(dev_id)} - - name_owner_pubkeys.append( keylib.key_formatting.compress(ecdsalib.get_pubkey_hex(name_owner_privkeys[dev_id])) ) - - # make the token file - token_file = { - 'version': '3.0', - 'profile': profile_jwt_txt, - 'keys': { - 'name': name_owner_pubkeys, - 'delegation': delegation_jwt_txt, - 'apps': apps_jwt_txt, - }, - 'writes': write_version, - 'timestamp': int(time.time()), - } - - return {'status': True, 'token_file': token_file_sign(token_file, signing_keys[device_id])} - - -def token_file_sign(parsed_token_file, signing_private_key): - """ - Given a parsed token file, sign it with the private key - and return the serialized JWT (in compact serialization) - - Return {'status': True, 'token_file': token file text} - """ - signer = jsontokens.TokenSigner() - jwt = signer.sign(parsed_token_file, signing_private_key) - return jwt - - -def token_file_profile_serialize(data_text_or_json, data_privkey): - """ - Serialize a profile to a string - """ - # profiles must conform to a particular standard format - tokenized_data = blockstack_profiles.sign_token_records([data_text_or_json], data_privkey) - - del tokenized_data[0]['decodedToken'] - - serialized_data = json.dumps(tokenized_data, sort_keys=True) - return serialized_data - - -def token_file_update_profile(parsed_token_file, new_profile, signing_private_key): - """ - Given a parsed token file, a new profile, and the signing key for this device, - generate a new (serialized) token file with the new profile. - - Return {'status': True, 'token_file': serialized token file} - Return {'error': ...} on failure - """ - - keys_jwts = parsed_token_file.get('jwts', {}).get('keys', None) - if keys_jwts is None: - return {'error': 'Invalid parsed token file: missing jwts'} - - profile_jwt_txt = token_file_profile_serialize(new_profile, signing_private_key) - tok = { - 'version': '3.0', - 'profile': profile_jwt_txt, - 'keys': keys_jwts, - 'writes': parsed_token_file['writes'] + 1, - 'timestamp': int(time.time()), - } - - return {'status': True, 'token_file': token_file_sign(tok, signing_private_key)} - - -def token_file_update_apps(parsed_token_file, device_id, app_name, app_pubkey, signing_private_key): - """ - Given a parsed token file, a device ID, an application name, its public key, and the device's signing private key, - insert a new entry for the application for this device - - Return {'status': True, 'token_file': serialized token file} on success - Return {'error': ...} on failure - """ - - key_jwts = parsed_token_file.get('jwts', {}).get('keys', None) - if key_jwts is None: - return {'error': 'Invalid parsed token file: missing jwts'} - - profile_jwt = parsed_token_file.get('jwts', {}).get('profile', None) - if profile_jwt is None: - return {'error': 'Invalid parsed token file: missing profile JWT'} - - delegation_jwt = key_jwts.get('delegation', None) - if delegation_jwt is None: - return {'error': 'Invalid parsed token file: missing delegations JWT'} - - if device_id not in parsed_token_file['keys']['delegation']['devices'].keys(): - return {'error': 'Device "{}" not present in delegation file'.format(device_id)} - - cur_apps = parsed_token_file['keys']['apps'] - if not cur_apps.has_key(device_id): - cur_apps[device_id] = {'version': '1.0', 'apps': {}} - - cur_apps[device_id]['apps'][app_name] = app_pubkey - - apps_signer = jsontokens.TokenSigner() - apps_jwt = apps_signer.sign(cur_apps[device_id], signing_private_key) - - apps_jwts = key_jwts['apps'] - apps_jwts[device_id] = apps_jwt - - tok = { - 'version': '3.0', - 'profile': profile_jwt, - 'keys': { - 'name': parsed_token_file['keys']['name'], - 'delegation': delegation_jwt, - 'apps': apps_jwts, - }, - 'writes': parsed_token_file['writes'] + 1, - 'timestamp': int(time.time()), - } - - return {'status': True, 'token_file': token_file_sign(tok, signing_private_key)} - - -def token_file_update_delegation(parsed_token_file, device_delegation, name_owner_privkeys, signing_private_key): - """ - Given a parsed token file, a device delegation object, and a list of name owner private keys, - insert a new entry for the token file's delegation records. - - Return {'status': True, 'token_file': serialized token file} on success - Return {'error': ...} on failure - """ - - keys_jwts = parsed_token_file.get('jwts', {}).get('keys', None) - if keys_jwts is None: - return {'error': 'Invalid parsed token file: missing jwts'} - - profile_jwt = parsed_token_file.get('jwts', {}).get('profile', None) - if profile_jwt is None: - return {'error': 'Invalid parsed token file: missing profile JWT'} - - apps_jwt = keys_jwts.get('apps', None) - if apps_jwt is None: - return {'error': 'Invalid parsed token file: missing apps JWT'} - - try: - jsonschema.validate(device_delegation, schemas.KEY_DELEGATION_DEVICES_SCHEMA) - except ValidationError as ve: - if BLOCKSTACK_TEST: - log.exception(ve) - - return {'error': 'Invalid device delegation'} - - new_delegation = copy.deepcopy(parsed_token_file['keys']['delegation']) - new_delegation['devices'].update(device_delegation) - - signer = jsontokens.TokenSigner() - new_delegation_jwt = signer.sign(new_delegation, name_owner_privkeys) - new_delegation_jwt_txt = json.dumps(new_delegation_jwt) - - tok = { - 'version': '3.0', - 'profile': profile_jwt, - 'keys': { - 'name': parsed_token_file['keys']['name'], - 'delegation': new_delegation_jwt_txt, - 'apps': apps_jwt, - }, - 'writes': parsed_token_file['writes'] + 1, - 'timestamp': int(time.time()), - } - - return {'status': True, 'token_file': token_file_sign(tok, signing_private_key)} - - -def token_file_get_delegated_device_pubkeys(parsed_token_file, device_id): - """ - Get the public keys for a delegated device. - Returns {'status': true, 'version': ..., 'pubkeys': {'app': ..., 'sign': ..., 'enc': ...}} on success - Returns {'error': ...} on error - """ - delegation = parsed_token_file.get('keys', {}).get('delegation', None) - if not delegation: - raise ValueError('Token file does not have a "delegation" entry') - - device_info = delegation['devices'].get(device_id, None) - if device_info is None: - return {'error': 'No device entry in delegation file for "{}"'.format(device_id)} - - res = { - 'status': True, - 'version': delegation['version'], - 'app': device_info['app'], - 'enc': device_info['enc'], - 'sign': device_info['sign'], - } - - return res - - -def token_file_get_app_device_ids(parsed_token_file): - """ - Get the list of app-specific device IDs - - Returns {'status': True, 'device_ids': [...]} on success - Return {'error': ...} on error - """ - apps = parsed_token_file.get('keys', {}).get('apps', None) - if not apps: - raise ValueError('Token file does not have a "apps" entry') - - return {'status': True, 'device_ids': apps.keys()} - - -def token_file_get_app_device_pubkeys(parsed_token_file, device_id): - """ - Get the public keys for apps available from a particular device - Returns {'status': True, 'version': ..., 'pubkeys': {...}} on success - Returns {'error': ...} on error - """ - apps = parsed_token_file.get('keys', {}).get('apps', None) - if not apps: - raise ValueError('Token file does not have an "apps" entry') - - apps_info = apps.get(device_id, None) - if apps_info is None: - return {'error': 'No device entry in apps file for {}'.format(device_id)} - - res = { - 'status': True, - 'version': apps_info['version'], - 'app_pubkeys': apps_info['apps'], - } - return res - - -def token_file_get_delegated_device_ids(parsed_token_file): - """ - Get the list of delegated device IDs - - Returns {'status': True, 'device_ids': [...]} on success - Return {'error': ...} on error - """ - delegation = parsed_token_file.get('keys', {}).get('delegation', None) - if not delegation: - raise ValueError('Token file does not have a "delegation" entry') - - return {'status': True, 'device_ids': delegation['devices'].keys()} - - -def deduce_name_privkey(parsed_token_file, owner_privkey_info): - """ - Given owner private key info, and the token file and device ID, - determine the name-owning private key to use for this device. - - Return {'status': True, 'name_privkey': privkey} on success - Return {'error': ...} on failure - """ - privkey_candidates = [] - if virtualchain.is_singlesig(owner_privkey_info): - # one owner key, and this is it. - privkey_candidates = [owner_privkey_info] - - else: - # multisig bundle - privkey_candidates = owner_privkey_info['privkeys'] - - # map signing public keys back to the name private key that generated it - signing_pubkey_candidates = dict([(ecdsalib.get_pubkey_hex(get_signing_privkey(pk)), pk) for pk in privkey_candidates]) - - all_device_ids = token_file_get_delegated_device_ids(parsed_token_file) - for device_id in all_device_ids['device_ids']: - pubkeys = token_file_get_delegated_device_pubkeys(parsed_token_file, device_id) - assert 'error' not in pubkeys, pubkeys['error'] - - signing_pubkey = pubkeys['sign'] - compressed_form = keylib.key_formatting.compress(signing_pubkey) - uncompressed_form = keylib.key_formatting.decompress(signing_pubkey) - - if compressed_form in signing_pubkey_candidates.keys(): - # found! - return {'status': True, 'name_privkey': signing_pubkey_candidates[compressed_form]} - - if uncompressed_form in signing_pubkey_candidates.keys(): - # found! - return {'status': True, 'name_privkey': signing_pubkey_candidates[uncompressed_form]} - - # absent - return {'error': 'Token file is missing name public keys'} - - -def lookup_name_privkey(name, owner_privkey_info, proxy=None, parsed_token_file=None): - """ - Given a name and wallet keys, get the name private key - Return {'status': True, 'name_privkey': ...} on success - Return {'error': ...} on error - """ - proxy = get_default_proxy() if proxy is None else proxy - - if parsed_token_file is None: - res = token_file_get(name, proxy=proxy) - if 'error' in res: - log.error("Failed to get token file for {}: {}".format(name, res['error'])) - return {'error': 'Failed to get token file for {}: {}'.format(name, res['error'])} - - parsed_token_file = res['token_file'] - if parsed_token_file is None: - log.error("No token file for {}".format(name)) - return {'error': 'No token file available for {}'.format(name)} - - return deduce_name_privkey(parsed_token_file, owner_privkey_info) - - -def lookup_signing_privkey(name, owner_privkey_info, proxy=None, parsed_token_file=None): - """ - Given a name and wallet keys, get the signing private key - Return {"status': True, 'signing_privkey': ...} on success - Return {'error': ...} on error - """ - res = lookup_name_privkey(name, owner_privkey_info, proxy=proxy, parsed_token_file=parsed_token_file) - if 'error' in res: - return res - - name_privkey = res['name_privkey'] - signing_privkey = get_signing_privkey(name_privkey) - return {'status': True, 'signing_privkey': signing_privkey} - - -def lookup_delegated_device_pubkeys(name, proxy=None): - """ - Given a blockchain ID (name), get all of its delegated devices' public keys - Return {'status': True, 'pubkeys': {'$device-id': {...}}} on success - Return {'error': ...} on error - """ - res = token_file_get(name, proxy=proxy) - if 'error' in res: - log.error("Failed to get token file for {}".format(name)) - return {'error': 'Failed to get token file for {}: {}'.format(name, res['error'])} - - parsed_token_file = res['token_file'] - if parsed_token_file is None: - log.error("No token file for {}".format(name)) - return {'error': 'No token file available for {}'.format(name)} - - all_device_ids = token_file_get_delegated_device_ids(parsed_token_file) - all_pubkeys = {} - for dev_id in all_device_ids['device_ids']: - pubkey_info = token_file_get_delegated_device_pubkeys(parsed_token_file, dev_id) - assert 'error' not in pubkey_info, pubkey_info['error'] - - all_pubkeys[dev_id] = pubkey_info - - return {'status': True, 'pubkeys': all_pubkeys, 'token_file': parsed_token_file} - - -def lookup_signing_pubkeys(name, proxy=None): - """ - Given a blockchain ID (name), get its signing public keys. - Return {'status': True, 'token_file': ..., 'pubkeys': {'$device_id': ...}} on success - Return {'error': ...} on error - """ - res = lookup_delegated_device_pubkeys(name, proxy=proxy) - if 'error' in res: - return res - - token_file = res['token_file'] - all_pubkeys = res['pubkeys'] - signing_keys = {} - for dev_id in all_pubkeys.keys(): - signing_keys[dev_id] = all_pubkeys[dev_id].get('sign') - - return {'status': True, 'pubkeys': signing_keys, 'token_file': token_file} - - -def lookup_app_pubkeys(name, full_application_name, proxy=None, parsed_token_file=None): - """ - Given a blockchain ID (name), and the full application name (i.e. ending in .1 or .x), - go and get all of the public keys for it in the app keys file - Return {'status': True, 'token_file': ..., 'pubkeys': {'$device_id': ...}} on success - Return {'error': ...} on error - """ - if parsed_token_file is None: - res = token_file_get(name, proxy=proxy) - if 'error' in res: - log.error("Failed to get token file for {}".format(name)) - return {'error': 'Failed to get token file for {}: {}'.format(name, res['error'])} - - parsed_token_file = res['token_file'] - if parsed_token_file is None: - log.error("No token file for {}".format(name)) - return {'error': 'No token file available for {}'.format(name)} - - all_device_ids = token_file_get_app_device_ids(parsed_token_file) - app_pubkeys = {} - for dev_id in all_device_ids['device_ids']: - dev_app_pubkey_info = token_file_get_app_device_pubkeys(parsed_token_file, dev_id) - assert 'error' not in dev_app_pubkey_info, dev_app_pubkey_info['error'] - - dev_app_pubkeys = dev_app_pubkey_info['app_pubkeys'] - if full_application_name not in dev_app_pubkeys.keys(): - # this device may not access this app - continue - - app_pubkeys[dev_id] = dev_app_pubkeys[full_application_name] - - return {'status': True, 'pubkeys': app_pubkeys, 'token_file': parsed_token_file} - - -def token_file_get(name, zonefile_storage_drivers=None, profile_storage_drivers=None, - proxy=None, user_zonefile=None, name_record=None, - use_zonefile_urls=True, decode=True): - """ - Given a name, look up an associated key token file. - Do so by first looking up the zonefile the name points to, - and then loading the token file from that zonefile's public key. - - Returns { - 'status': True, - 'token_file': token_file (if present), - 'profile': profile, - 'zonefile': zonefile - 'raw_zonefile': unparesed zone file, - 'nonstandard_zonefile': bool whether or not this is a non-standard zonefile - 'legacy_profile': legacy parsed profile - 'name_record': name record (if needed) - } on success. - - 'token_file' may be None, if this name still points to an off-zonefile profile - 'legacy_profile' will be set if this name does not even have an off-zonefile profile (but instead a zone file that parses to a profile) - - Returns {'error': ...} on error - """ - - proxy = get_default_proxy() if proxy is None else proxy - - ret = { - 'status': True, - 'token_file': None, - 'profile': None, - 'legacy_profile': None, - 'raw_zonefile': None, - 'nonstandard_zonefile': False, - 'zonefile': user_zonefile, - 'name_record': name_record, - } - - token_file = None - - if user_zonefile is None: - user_zonefile = get_name_zonefile(name, proxy=proxy, name_record=name_record, storage_drivers=zonefile_storage_drivers, allow_legacy=True) - if 'error' in user_zonefile: - return user_zonefile - - ret['raw_zonefile'] = user_zonefile['raw_zonefile'] - ret['user_zonefile'] = user_zonefile['zonefile'] - - user_zonefile = user_zonefile['zonefile'] - - # is this really a legacy profile? - if blockstack_profiles.is_profile_in_legacy_format(user_zonefile): - # convert it - log.warning('Converting legacy profile to modern profile') - legacy_profile = blockstack_profiles.get_person_from_legacy_format(user_zonefile) - - # nothing more to do - ret['legacy_profile'] = legacy_profile - return ret - - elif not user_db.is_user_zonefile(user_zonefile): - # not a legacy profile, but a custom profile - log.warning('Non-standard zone file; treating as legacy profile') - ret['legacy_profile'] = copy.deepcopy(user_zonefile) - ret['nonstandard_zonefile'] = True - return ret - - # get user's data public key from their zone file, if it is set. - # this is only needed for legacy lookups in off-zonefile profiles - # (i.e. pre-token file) - data_address, owner_address = None, None - - try: - user_data_pubkey = user_db.user_zonefile_data_pubkey(user_zonefile) - if user_data_pubkey is not None: - user_data_pubkey = str(user_data_pubkey) - data_address = keylib.ECPublicKey(user_data_pubkey).address() - - except ValueError: - # multiple keys defined; we don't know which one to use - user_data_pubkey = None - - # find owner address - if name_record is None: - name_record = get_name_blockchain_record(name, proxy=proxy) - if name_record is None or 'error' in name_record: - log.error('Failed to look up name record for "{}"'.format(name)) - return {'error': 'Failed to look up name record'} - - ret['name_record'] = name_record - - assert 'address' in name_record.keys(), json.dumps(name_record, indent=4, sort_keys=True) - owner_address = name_record['address'] - - # find the set of URLs, if none are given - urls = None - if use_zonefile_urls and user_zonefile is not None: - urls = user_db.user_zonefile_urls(user_zonefile) - - # actually go and load the profile or token file (but do not decode it yet) - profile_or_token_file_txt = storage.get_mutable_data(name, None, urls=urls, drivers=profile_storage_drivers, decode=False, fqu=name) - if profile_or_token_file_txt is None: - log.error('no token file or profile for {}'.format(name)) - return {'error': 'Failed to load profile or token file from zone file for {}'.format(name)} - - # try to parse as a token file... - token_file = None - profile = None - token_file_data = token_file_parse(profile_or_token_file_txt, owner_address) - if 'error' in token_file_data: - log.warning("Failed to parse token file: {}".format(token_file_data['error'])) - - # try to parse as a legacy profile - profile = storage.parse_mutable_data(profile_or_token_file_txt, user_data_pubkey, public_key_hash=owner_address) - if profile is None: - log.error("Failed to parse data as a token file or a profile") - return {'error': 'Failed to load profile or token file'} - - else: - # got a token file! - token_file = token_file_data['token_file'] - profile = token_file['profile'] - - ret['token_file'] = token_file - ret['profile'] = profile - return ret - - -def token_file_put(name, new_token_file, signing_privkey, proxy=None, required_drivers=None, config_path=CONFIG_PATH): - """ - Set the new token file data. CLIENTS SHOULD NOT CALL THIS METHOD DIRECTLY. - Takes a serialized token file (as a string) - - Return {'status: True} on success - Return {'error': ...} on failure. - """ - if not isinstance(new_token_file, (str, unicode)): - raise ValueError("Invalid token file: string or unicode compact-form JWT required") - - ret = {} - - proxy = get_default_proxy() if proxy is None else proxy - config = proxy.conf - - # deduce storage drivers - required_storage_drivers = None - if required_drivers is not None: - required_storage_drivers = required_drivers - else: - required_storage_drivers = config.get('storage_drivers_required_write', None) - if required_storage_drivers is not None: - required_storage_drivers = required_storage_drivers.split(',') - else: - required_storage_drivers = config.get('storage_drivers', '').split(',') - - log.debug('Save updated token file for "{}" to {}'.format(name, ','.join(required_storage_drivers))) - - rc = storage.put_mutable_data(name, new_token_file, raw=True, required=required_storage_drivers, token_file=True, fqu=name) - if not rc: - return {'error': 'Failed to store token file for {}'.format(name)} - - return {'status': True} - - -def token_file_delete(blockchain_id, signing_private_key, proxy=None): - """ - Delete token file data. CLIENTS SHOULD NOT CALL THIS DIRECTLY - Return {'status: True} on success - Return {'error': ...} on failure. - """ - - proxy = get_default_proxy() if proxy is None else proxy - rc = storage.delete_mutable_data(blockchain_id, signing_private_key) - if not rc: - return {'error': 'Failed to delete token file'} - - return {'status': True} - - - -if __name__ == "__main__": - - name_owner_privkeys = { - 'test_device_1': '2acbababb77e2d52845fd5c9f710ff83595c01b0f4a431927c74afc88dd4c2d501', - 'test_device_2': 'b261db1ae6e0dfeb947b3e1eb67e8426157c6b0abea9de863ce01e76499b231501', - 'test_device_3': '7150f2b6275c1e29f3cf27fb2442ccb17a15ef0de50bc633e14a80175207066b01', - } - - name_owner_pubkeys = dict([(dev_id, ecdsalib.get_pubkey_hex(nopk)) for (dev_id, nopk) in name_owner_privkeys.items()]) - name_owner_address = virtualchain.make_multisig_address( [keylib.key_formatting.compress(name_owner_pubkeys[dev_id]) for dev_id in sorted(name_owner_pubkeys.keys())], len(name_owner_pubkeys) ) - - name = 'test.id' - device_id = 'test_device_1' - profile = { - '@type': 'Person', - 'accounts': [] - } - apps = { - 'test_device_1': { - 'version': '1.0', - 'apps': {} - }, - } - delegations = { - 'version': '1.0', - 'name': name, - 'devices': { - 'test_device_1': token_file_make_delegation_entry(name_owner_privkeys['test_device_1'], 'test_device_1', 0)['delegation'], - }, - } - - # make token file - token_info = token_file_create("test.id", name_owner_privkeys, device_id, profile=profile, delegations=delegations, apps=apps) - assert 'error' not in token_info, token_info - - token_file_txt = token_info['token_file'] - token_file = token_file_parse(token_file_txt, name_owner_pubkeys.values()) - assert 'error' not in token_file, token_file - - token_file = token_file_parse(token_file_txt, name_owner_address) - assert 'error' not in token_file, token_file - - token_file = token_file['token_file'] - - print 'initial token file is \n{}'.format(json.dumps(token_file, indent=4, sort_keys=True)) - - assert token_file['profile'] == profile - assert token_file['keys']['delegation'] == delegations - assert token_file['keys']['apps'] == apps, 'token_file[keys][apps] = {}, apps = {}'.format(token_file['keys']['apps'], apps) - - # update the token file's profile - new_profile = { - '@type': 'Person', - 'accounts': [], - 'name': { - 'formatted': 'Hello World', - }, - } - - print 'update profile' - res = token_file_update_profile(token_file, new_profile, get_signing_privkey(name_owner_privkeys['test_device_1'])) - assert 'error' not in res - - # re-extract - token_file_txt = res['token_file'] - token_file = token_file_parse(token_file_txt, name_owner_pubkeys.values()) - assert 'error' not in token_file - - token_file = token_file_parse(token_file_txt, name_owner_address) - assert 'error' not in token_file, token_file - - token_file = token_file['token_file'] - - print 'token file with new profile is \n{}'.format(json.dumps(token_file, indent=4, sort_keys=True)) - - assert token_file['profile'] == new_profile - assert token_file['keys']['delegation'] == delegations - assert token_file['keys']['apps'] == apps - - # update the token file's delegations - new_delegations = { - 'test_device_1': delegations['devices']['test_device_1'], - 'test_device_2': token_file_make_delegation_entry(name_owner_privkeys['test_device_2'], 'test_device_2', 0)['delegation'], - } - - print 'update delegation' - res = token_file_update_delegation(token_file, new_delegations, name_owner_privkeys.values(), get_signing_privkey(name_owner_privkeys['test_device_1'])) - assert 'error' not in res, res['error'] - - # re-extract - token_file_txt = res['token_file'] - token_file = token_file_parse(token_file_txt, name_owner_pubkeys.values()) - assert 'error' not in token_file, token_file['error'] - - token_file = token_file_parse(token_file_txt, name_owner_address) - assert 'error' not in token_file, token_file - - token_file = token_file['token_file'] - - print 'token file with new profile and new delegation is \n{}'.format(json.dumps(token_file, indent=4, sort_keys=True)) - - assert token_file['profile'] == new_profile - assert token_file['keys']['delegation'] == {'version': '1.0', 'name': name, 'devices': new_delegations} - assert token_file['keys']['apps'] == apps - - # update the token file's apps - helloblockstack_com_pubkey = keylib.ECPrivateKey().public_key().to_hex() - res = token_file_update_apps(token_file, 'test_device_1', "helloblockstack.com.1", helloblockstack_com_pubkey, get_signing_privkey(name_owner_privkeys['test_device_1'])) - assert 'error' not in res, res['error'] - - # re-extract - token_file_txt = res['token_file'] - token_file = token_file_parse(token_file_txt, name_owner_pubkeys.values()) - assert 'error' not in token_file - - token_file = token_file_parse(token_file_txt, name_owner_address) - assert 'error' not in token_file, token_file - - token_file = token_file['token_file'] - - print 'token file with new profile and new delegation and new app is \n{}'.format(json.dumps(token_file, indent=4, sort_keys=True)) - - assert token_file['profile'] == new_profile - assert token_file['keys']['delegation'] == {'version': '1.0', 'name': name, 'devices': new_delegations} - assert token_file['keys']['apps'].has_key('test_device_1') - assert token_file['keys']['apps']['test_device_1']['apps'].has_key('helloblockstack.com.1') - assert token_file['keys']['apps']['test_device_1']['apps']['helloblockstack.com.1'] == helloblockstack_com_pubkey - - - - - # def token_file_parse(token_txt, name_owner_pubkeys_or_addrs, min_writes=None): - # def token_file_create(profile, delegations, apps, name_owner_privkeys, device_id, write_version=1): - # def token_file_update_profile(parsed_token_file, new_profile, signing_private_key): - # def token_file_update_delegation(parsed_token_file, device_delegation, name_owner_privkeys): - # def token_file_update_apps(parsed_token_file, device_id, app_name, app_pubkey, signing_private_key): diff --git a/blockstack_client/user.py b/blockstack_client/user.py index f2d164a18..476200a40 100644 --- a/blockstack_client/user.py +++ b/blockstack_client/user.py @@ -32,12 +32,12 @@ import urlparse import virtualchain from virtualchain.lib.ecdsalib import * -from schemas import * -from constants import BLOCKSTACK_TEST, CONFIG_PATH, BLOCKSTACK_DEBUG +from .schemas import * +from .constants import BLOCKSTACK_TEST, CONFIG_PATH, BLOCKSTACK_DEBUG import scripts -from logger import get_logger +from .logger import get_logger log = get_logger() @@ -80,7 +80,8 @@ def user_zonefile_data_pubkey(user_zonefile, key_prefix='pubkey:data:'): continue if data_pubkey is not None: - log.error('Invalid zone file: multiple data keys') + msg = 'Invalid zone file: multiple data keys' + log.error(msg) raise ValueError('{} starting with "{}"'.format(msg, key_prefix)) data_pubkey = txtrec['txt'][len(key_prefix):] diff --git a/blockstack_client/utils.py b/blockstack_client/utils.py index 716f78c4a..c4d2f201f 100644 --- a/blockstack_client/utils.py +++ b/blockstack_client/utils.py @@ -29,8 +29,8 @@ import hashlib import threading import traceback -from constants import DEFAULT_BLOCKSTACKD_PORT -from logger import get_logger +from .constants import DEFAULT_BLOCKSTACKD_PORT +from .logger import get_logger log = get_logger('blockstack-client') diff --git a/blockstack_client/utxo.py b/blockstack_client/utxo.py index 035beda29..57ecf05df 100644 --- a/blockstack_client/utxo.py +++ b/blockstack_client/utxo.py @@ -401,7 +401,8 @@ def connect_utxo_provider( utxo_opts, min_confirmations=TX_MIN_CONFIRMATIONS ): min_confirmations=min_confirmations ) elif utxo_provider == "blockstack_core": - return BlockstackCoreUTXOClient( utxo_opts['server'], utxo_opts['port'], min_confirmations=min_confirmations ) + # setting min confirmations not supported by this backend + return BlockstackCoreUTXOClient( utxo_opts['server'], utxo_opts['port'] ) elif utxo_provider == "blockstack_explorer": return BlockstackExplorerClient( url=utxo_opts['url'], min_confirmations=min_confirmations ) diff --git a/blockstack_client/wallet.py b/blockstack_client/wallet.py index 543742815..6b97dd763 100644 --- a/blockstack_client/wallet.py +++ b/blockstack_client/wallet.py @@ -232,8 +232,9 @@ def make_legacy_wallet_keys(data, password): if BLOCKSTACK_DEBUG is not None: log.exception(e) - log.debug('Failed to decrypt encrypted_master_private_key: {}'.format(ret['error'])) - return ret + err = 'Failed to decrypt encrypted_master_private_key' + log.debug(err) + return {'error' : err} # legacy compat: use the master private key to generate child keys. # If the specific key they are purposed for is not defined in the wallet, @@ -897,7 +898,7 @@ def unlock_wallet(password=None, config_dir=CONFIG_DIR, wallet_path=None): try: res = save_keys_to_memory( wallet, config_path=config_path ) except KeyError as ke: - if BLOCKSACK_DEBUG is not None: + if BLOCKSTACK_DEBUG is not None: data = json.dumps(wallet, indent=4, sort_keys=True) log.error('data:\n{}\n'.format(data)) raise diff --git a/blockstack_client/zonefile.py b/blockstack_client/zonefile.py index 68903d334..8963734f3 100644 --- a/blockstack_client/zonefile.py +++ b/blockstack_client/zonefile.py @@ -28,13 +28,13 @@ import base64 import socket from keylib import ECPrivateKey -from proxy import * +from .proxy import * import storage import user as user_db -from config import get_config -from logger import get_logger -from constants import USER_ZONEFILE_TTL, CONFIG_PATH, BLOCKSTACK_TEST, BLOCKSTACK_DEBUG +from .config import get_config +from .logger import get_logger +from .constants import USER_ZONEFILE_TTL, CONFIG_PATH, BLOCKSTACK_TEST, BLOCKSTACK_DEBUG log = get_logger() @@ -178,13 +178,16 @@ def decode_name_zonefile(name, zonefile_txt, allow_legacy=False): return user_zonefile -def load_name_zonefile(name, expected_zonefile_hash, storage_drivers=None, proxy=None ): +def load_name_zonefile(name, expected_zonefile_hash, storage_drivers=None, raw_zonefile=False, allow_legacy=False, proxy=None ): """ - Fetch and load a raw user zonefile from the storage implementation with the given hex string hash, + Fetch and load a user zonefile from the storage implementation with the given hex string hash, The user zonefile hash should have been loaded from the blockchain, and thereby be the authentic hash. - Return the raw user zonefile (as a string) + If raw_zonefile is True, then return the raw zonefile data. Don't parse it. + If however, raw_zonefile is False, the zonefile will be parsed. If name is given, the $ORIGIN will be checked. + + Return the user zonefile (as a dict) on success Return None on error """ @@ -219,17 +222,21 @@ def load_name_zonefile(name, expected_zonefile_hash, storage_drivers=None, proxy log.debug('Fetched {} from Atlas peer {}'.format(expected_zonefile_hash, hostport)) zonefile_txt = res['zonefiles'][expected_zonefile_hash] - try: - assert isinstance(zonefile_txt, (str, unicode)), msg - except AssertionError as ae: - if BLOCKSTACK_TEST is not None: - log.exception(ae) - + if raw_zonefile: msg = 'Driver did not return a serialized zonefile' - log.error(msg) - return None + try: + assert isinstance(zonefile_txt, (str, unicode)), msg + except AssertionError as ae: + if BLOCKSTACK_TEST is not None: + log.exception(ae) - return zonefile_txt + log.error(msg) + return None + + return zonefile_txt + + parsed_zonefile = decode_name_zonefile(name, zonefile_txt, allow_legacy=allow_legacy) + return parsed_zonefile def load_data_pubkey_for_new_zonefile(wallet_keys={}, config_path=CONFIG_PATH): @@ -251,14 +258,22 @@ def load_data_pubkey_for_new_zonefile(wallet_keys={}, config_path=CONFIG_PATH): def get_name_zonefile(name, storage_drivers=None, proxy=None, - name_record=None, allow_legacy=False): + name_record=None, include_name_record=False, + raw_zonefile=False, include_raw_zonefile=False, allow_legacy=False): """ Given a name, go fetch its zonefile. Verifies that the hash on the blockchain matches the zonefile. - Returns {'status': True, 'zonefile': zonefile dict (if well-formed, otherwise None), 'raw_zonefile': raw bytes, 'name_record': bns name record} on success. - Return {'error': ...} if we failed to load the zone file + Returns {'status': True, 'zonefile': zonefile dict} on success. + Returns a dict with "error" defined and a message on failure to load. + Return None if there is no zonefile (i.e. the hash is null) + if 'include_name_record' is true, then zonefile will contain + an extra key called 'name_record' that includes the blockchain name record. + + If 'raw_zonefile' is true, no attempt to parse the zonefile will be made. + The raw zonefile will be returned in 'zonefile'. allow_legacy is ignored. + If 'allow_legacy' is true, then support returning older supported versions of the zone file (including old Onename profiles). Otherwise, this method fails. """ @@ -288,19 +303,41 @@ def get_name_zonefile(name, storage_drivers=None, proxy=None, raw_zonefile_data = None user_zonefile_data = None - raw_zonefile_data = load_name_zonefile(name, user_zonefile_hash, storage_drivers=storage_drivers, proxy=proxy) - if raw_zonefile_data is None: - return {'error': 'Failed to load raw name zonefile'} + if raw_zonefile or include_raw_zonefile: + raw_zonefile_data = load_name_zonefile( + name, user_zonefile_hash, storage_drivers=storage_drivers, + raw_zonefile=True, proxy=proxy, allow_legacy=allow_legacy + ) - # further decode (it's okay if this is None) - user_zonefile_data = decode_name_zonefile(name, raw_zonefile_data, allow_legacy=allow_legacy) + if raw_zonefile_data is None: + return {'error': 'Failed to load raw name zonefile'} + + if raw_zonefile: + user_zonefile_data = raw_zonefile_data + + else: + # further decode + user_zonefile_data = decode_name_zonefile(name, raw_zonefile_data, allow_legacy=allow_legacy) + if user_zonefile_data is None: + return {'error': 'Failed to decode name zonefile'} + + else: + user_zonefile_data = load_name_zonefile( + name, user_zonefile_hash, storage_drivers=storage_drivers, proxy=proxy, allow_legacy=allow_legacy + ) + if user_zonefile_data is None: + return {'error': 'Failed to load or decode name zonefile'} ret = { - 'zonefile': user_zonefile_data, - 'raw_zonefile': raw_zonefile_data, - 'name_record': name_record, + 'zonefile': user_zonefile_data } + if include_name_record: + ret['name_record'] = name_record + + if include_raw_zonefile: + ret['raw_zonefile'] = raw_zonefile_data + return ret @@ -442,32 +479,3 @@ def zonefile_data_replicate(fqu, zonefile_data, tx_hash, server_list, config_pat return res return {'status': True, 'servers': res['servers']} - - -def lookup_name_zonefile_pubkey(name, proxy=None): - """ - Given a name, get the public key in its zone file - Returns {'status': True, 'pubkey': ..., 'name_record': ...} on success - Returns {'status': True, 'pubkey': None, 'name_record': ...} if there is no public key - Returns {'error': ...} on failure - """ - zonefile_data = None - name_rec = None - # get the pubkey - zonefile_data_res = get_name_zonefile(name, proxy=proxy) - if 'error' not in zonefile_data_res: - zonefile_data = zonefile_data_res['raw_zonefile'] - name_rec = zonefile_data_res['name_record'] - else: - return {'error': "Failed to get zonefile data: {}".format(name)} - - # parse - zonefile_dict = None - try: - zonefile_dict = blockstack_zones.parse_zone_file(zonefile_data) - except: - return {'error': 'Nonstandard zone file'} - - pubkey = user_zonefile_data_pubkey(zonefile_dict) - return {'status': True, 'pubkey': pubkey, 'name_record': name_rec} - diff --git a/docs/api-specs.md b/docs/api-specs.md index 90cd7f008..ec80f501b 100644 --- a/docs/api-specs.md +++ b/docs/api-specs.md @@ -177,14 +177,21 @@ This call instructs the blockstack core node to use a particular key instead of the core node's configured wallet key. The setting of this key is *temporary*. It is not written to `~/.blockstack/wallet.json`, and on a subsequent restart, the key will return to the original key. -Therefore, particular care should be taken when registering with such -a key, as the registrar may be unable to issue a `REGISTER` or -`UPDATE` if the node restarts in the middle of the registration process. +However, the core registrar *tracks* the owner key used for each `PREORDER`, +and stores that private key encrypted (with `scrypt` and the core wallet +password) in the queue. When the registrar detects that the key being used +for a particular name has changed, it will recover by submitting further +transactions with the stored key. + Requires root authorization + Parameters + keyname: owner (string) - which key to set (one of 'owner', 'data', 'payment') ++ Request (application/json) + + Body + + "cPo24qGYz76xSbUCug6e8LzmzLGJPZoowQC7fCVPLN2tzCUJgfcW" + + Request (application/json) + Schema @@ -238,6 +245,11 @@ a key, as the registrar may be unable to issue a `REGISTER` or ] } ++ Response 200 (application/json) + + Body + + {"status": true} + ## Get payment wallet balance [GET /v1/wallet/balance/{minconfs}] Fetches wallet balance, including UTXOs from transactions with at @@ -426,8 +438,9 @@ Sets the user's zonefile hash, and, if supplied, propagates the zonefile. If you supply the zonefile, the hash will be calculated from that. Ultimately, your requests should only supply one of `zonefile`, `zonefile_b64`, or `zonefile_hash`. - + Authorization: `update` ++ Parameters + + name: bar.test (string) - fully-qualified name + Request (application/json) + Schema @@ -436,20 +449,18 @@ that. Ultimately, your requests should only supply one of `zonefile`, 'properties': { "zonefile": { 'type': 'string', - 'maxLength': RPC_MAX_ZONEFILE_LEN, }, 'zonefile_b64': { 'type': 'string', - 'maxLength': (RPC_MAX_ZONEFILE_LEN * 4) / 3 + 1, }, 'zonefile_hash': { 'type': 'string', - 'pattern': OP_ZONEFILE_HASH_PATTERN, + 'pattern': '^([0-9a-fA-F]{20})$' }, 'tx_fee': { 'type': 'integer', 'minimum': 0, - 'maximum': TX_MAX_FEE + 'maximum': 500000 }, }, 'additionalProperties': False, @@ -460,6 +471,18 @@ that. Ultimately, your requests should only supply one of `zonefile`, {'transaction_hash' : '...'} +## Fetch zone file [GET /v1/names/{name}/zonefile] +Fetch a user's raw zonefile. ++ Parameters + + name: bar.test (string) - fully-qualified name ++ Response 200 (application/json) + + Body + + { + "zonefile": "$ORIGIN bar.test\n$TTL 3600\n_https._tcp URI 10 1 \"https://blockstack.s3.amazonaws.com/bar.test\"\n" + } + + # Group Name Querying This family of API endpoints deals with querying name information. diff --git a/docs/setup_core_portal.md b/docs/setup_core_portal.md index 489ceca1e..f44949737 100644 --- a/docs/setup_core_portal.md +++ b/docs/setup_core_portal.md @@ -2,8 +2,8 @@ We provide a [script](../images/scripts/ubuntu-17.04.sh) which will perform all the steps outlined in this doc (except for creating a protocol handler -- see the bottom of the doc). The script creates a virtualenv of -Blockstack Core and installs Portal in a subdirectory. It additionally creates some -scripts for starting Core and Portal together. +Blockstack Core and installs Browser in a subdirectory. It additionally creates some +scripts for starting Core and Browser together. However, if you'd like to customize your install, step through it yourself, or you are on a different distro, continue on with this doc! @@ -24,12 +24,12 @@ pip install virtualenv virtualenv --python=python2.7 ~/.blockstack.venv/ && source ~/.blockstack.venv/bin/activate ``` -Let's install virtualchain 0.14.2 and blockstack 0.14.2 +Let's install virtualchain 0.14.3 and blockstack 0.14.3 ``` sudo apt install git -pip install git+https://github.com/blockstack/virtualchain.git@rc-0.14.2 -pip install git+https://github.com/blockstack/blockstack-core.git@rc-0.14.2 +pip install git+https://github.com/blockstack/virtualchain.git@rc-0.14.3 +pip install git+https://github.com/blockstack/blockstack-core.git@rc-0.14.3 ``` Get Blockstack core configured with default settings and choose your Bitcoin wallet password @@ -37,7 +37,7 @@ Get Blockstack core configured with default settings and choose your Bitcoin wal blockstack setup -y --password BITCOIN_WALLET_PASSWORD --debug ``` -# Setting up Blockstack Portal Node Application +# Setting up Blockstack Browser Node Application Install NodeJS through NodeSource PPA @@ -46,20 +46,20 @@ curl -sL https://deb.nodesource.com/setup_7.x | sudo -E bash - sudo apt install -y nodejs ``` -Download Blockstack Portal and install its dependencies +Download Blockstack Browser and install its dependencies ``` -git clone https://github.com/blockstack/blockstack-portal.git -bv0.8 -cd blockstack-portal +git clone https://github.com/blockstack/blockstack-browser.git -bv0.11.1 +cd blockstack-browser npm install node-sass npm install ``` -Note that `blockstack-portal` depends on `node-sass` which can sometimes install strangely on Linux, running `npm install node-sass` before trying to install the other dependencies solves that problem. +Note that `blockstack-browser` depends on `node-sass` which can sometimes install strangely on Linux, running `npm install node-sass` before trying to install the other dependencies solves that problem. -# Running Blockstack Portal +# Running Blockstack Browser -Now we're ready to run our Core API service and start the Portal node app. +Now we're ready to run our Core API service and start the Browser node app. First, start the Core API service. @@ -79,7 +79,7 @@ Start the Node Application npm run dev ``` -Then you can open `http://localhost:3000/` in your browser to get to the portal. +Then you can open `http://localhost:3000/` in your browser to get to the Blockstack Browser. You can copy your api password to your clipboard with this command: @@ -100,7 +100,7 @@ With the following contents: Type=Application Terminal=false Exec=bash -c 'xdg-open http://localhost:3000/auth?authRequest=$(echo "%u" | sed s,blockstack:/*,,)' -Name=Blockstack-Portal +Name=Blockstack-Browser MimeType=x-scheme-handler/blockstack; ``` @@ -111,4 +111,4 @@ $ chmod +x ~/.local/share/applications/blockstack.desktop $ xdg-mime default blockstack.desktop x-scheme-handler/blockstack ``` -Now, `blockstack:` protocol URLs should get handled by your Blockstack Portal. If you're running Portal in your browser's private mode, you may have to copy and paste the link, as this protocol handler will try to open in a regular browser window. +Now, `blockstack:` protocol URLs should get handled by your Blockstack Browser. If you're running Browser in your browser's private mode, you may have to copy and paste the link, as this protocol handler will try to open in a regular browser window. diff --git a/docs/wire-format.md b/docs/wire-format.md new file mode 100644 index 000000000..550163a7f --- /dev/null +++ b/docs/wire-format.md @@ -0,0 +1,387 @@ +This page is for organizations who want to be able to create and send name operation transactions to the blockchain(s) Blockstack supports. + +# Bitcoin + +This section describes the transaction formats for the Bitcoin blockchain. + +## Transaction format + +Each Bitcoin transaction for Blockstack contains signatures from two sets of keys: the name owner, and the payer. The owner `scriptSig` and `scriptPubKey` fields are generated from the key(s) that own the given name. The payer `scriptSig` and `scriptPubKey` fields are used to *subsidize* the operation. The owner keys do not pay for any operations; the owner keys only control the minimum amount of BTC required to make the transaction standard. The payer keys only pay for the transaction's fees, and (when required) they pay the name fee. + +This construction is meant to allow the payer to be wholly separate from the owner. The principal that owns the name can fund their own transactions, or they can create a signed transaction that carries out the desired operation and request some other principal (e.g. a parent organization) to actually pay for and broadcast the transaction. + +The general transaction layout is as follows: + +| **Inputs** | **Outputs** | +| ------------------------ | ----------------------- | +| Owner scriptSig (1) | `OP_RETURN ` (2) | +| Payment scriptSig | Owner scriptPubKey (3) | +| Payment scriptSig... (4) | +| ... (4) | ... (5) | + +(1) The owner `scriptSig` is *always* the first input. +(2) The `OP_RETURN` script that describes the name operation is *always* the first output. +(3) The owner `scriptPubKey` is *always* the second output. +(4) The payer can use as many payment inputs as (s)he likes. +(5) At most one output will be the "change" `scriptPubKey` for the payer. + +## Payload Format + +Each Blockstack transaction in Bitcoin describes the name operation within an `OP_RETURN` output. It encodes name ownership, name fees, and payments as `scriptPubKey` outputs. The specific operations are described below. + +Each `OP_RETURN` payload *always* starts with `id` (called the "magic" bytes in this document), followed by a one-byte `op` that describes the operation. + +### NAME_PREORDER + +Op: `?` + +`OP_RETURN` wire format: +``` + 0 2 3 23 39 + |-----|--|--------------------------------------------------|--------------| + magic op hash_name(name.ns_id,script_pubkey,register_addr) consensus hash +``` + +Inputs: +* Payment `scriptSig`'s + +Outputs: +* `OP_RETURN` payload +* Payment `scriptPubkey` script for change +* `p2pkh` `scriptPubkey` to the burn address (0x00000000000000000000000000000000000000) + +Notes: +* `register_addr` is a base58check-encoded `ripemd160(sha256(pubkey))` (i.e. an address). This address **must not** have been used before in the underlying blockchain. +* `script_pubkey` is either a `p2pkh` or `p2sh` compiled Bitcoin script for the payer's address. + +### NAME_REGISTRATION + +Op: `:` + +`OP_RETURN` wire format: +``` + 0 2 3 39 + |----|--|-----------------------------| + magic op name.ns_id (37 bytes) +``` + +Inputs: +* Payer `scriptSig`'s + +Outputs: +* `OP_RETURN` payload +* `scriptPubkey` for the owner's address +* `scriptPubkey` for the payer's change + +### NAME_RENEWAL + +Op: `:` + +`OP_RETURN` wire format: +``` + 0 2 3 39 + |----|--|-----------------------------| + magic op name.ns_id (37 bytes) +``` + +Inputs: + +* Payer `scriptSig`'s + +Outputs: + +* `OP_RETURN` payload +* `scriptPubkey` for the owner's address +* `scriptPubkey` for the payer's change +* `scriptPubkey` for the burn address (to pay the name cost) + +Notes: + +* This transaction is identical to a `NAME_REGISTRATION`, except for the presence of the fourth output that pays for the name cost (to the burn address). + +### NAME_UPDATE + +Op: `+` + +`OP_RETURN` wire format: +``` + 0 2 3 19 39 + |-----|--|-----------------------------------|-----------------------| + magic op hash128(name.ns_id,consensus hash) hash160(data) +``` + +Note that `hash128(name.ns_id, consensus hash)` is a hash over the name concatenated to the hexadecimal string of the consensus hash (not the bytes corresponding to that hex string). + +Inputs: +* owner `scriptSig` +* payment `scriptSig`'s + +Outputs: +* `OP_RETURN` payload +* owner's `scriptPubkey` +* payment `scriptPubkey` change + +### NAME_TRANSFER + +Op: `>` + +`OP_RETURN` wire format: +``` + 0 2 3 4 20 36 + |-----|--|----|-------------------|---------------| + magic op keep hash128(name.ns_id) consensus hash + data? +``` + +Inputs: + +* Owner `scriptSig` +* Payment `scriptSig`'s + +Outputs: + +* `OP_RETURN` payload +* new name owner's `scriptPubkey` +* old name owner's `scriptPubkey` +* payment `scriptPubkey` change + +Notes: + +* The `keep data?` byte controls whether or not the zone file hash for the name is preserved. This value is either `>` to preserve it, or `~` to delete it. + +### NAME_REVOKE + +Op: `~` + +`OP_RETURN` wire format: +``` + 0 2 3 39 + |----|--|-----------------------------| + magic op name.ns_id (37 bytes) +``` + +Inputs: + +* owner `scriptSig` +* payment `scriptSig`'s + +Outputs: + +* `OP_RETURN` payload +* owner `scriptPubkey` +* payment `scriptPubkey` change + +### ANNOUNCE + +Op: `#` + +`OP_RETURN` wire format: + +``` + 0 2 3 23 + |----|--|-----------------------------| + magic op hash160(message) +``` + +Inputs: + +* The payer `scriptSig`'s + +Outputs: + +* `OP_RETURN` payload +* change `scriptPubKey` + +Notes: + +* The payer key should be an owner key for an existing name, since Blockstack users can subscribe to announcements from specific name-owners. + +### NAMESPACE_PREORDER + +Op: `*` + +`OP_RETURN` wire format: +``` + 0 2 3 23 39 + |-----|---|-----------------------------------------|----------------| + magic op hash_name(ns_id,script_pubkey,reveal_addr) consensus hash +``` + +Inputs: + +* Namespace payer `scriptSig` + +Outputs: + +* `OP_RETURN` payload +* Namespace payer `scriptPubkey` change address +* `p2pkh` script to the burn address (0x00000000000000000000000000000000) + +Notes: + +* The `reveal_addr` field is the address of the namespace revealer public key. The revealer private key will be used to generate `NAME_IMPORT` transactions. + +### NAMESPACE_REVEAL + +Op: `&` + +`OP_RETURN` wire format: +``` + 0 2 3 7 8 9 10 11 12 13 14 15 16 17 18 20 39 + |-----|---|--------|-----|-----|----|----|----|----|----|-----|-----|-----|--------|----------|-------------------------| + magic op life coeff. base 1-2 3-4 5-6 7-8 9-10 11-12 13-14 15-16 nonalpha version namespace ID + bucket exponents no-vowel + discounts +``` + +Inputs: + +* Namespace payer `scriptSig`s + +Outputs: + +* `OP_RETURN` payload +* namespace revealer `scriptPubkey` +* namespace payer change `scriptPubkey` + +Notes: + +* This transaction must be sent within 1 day of the `NAMESPACE_PREORDER` +* The second output (with the namespace revealer) **must** be a `p2pkh` script +* The address of the second output **must** be the `reveal_addr` in the `NAMESPACE_PREORDER` + +Pricing: + +The rules for a namespace are as follows: + + * a name can fall into one of 16 buckets, measured by length. Bucket 16 incorporates all names at least 16 characters long. + * the pricing structure applies a multiplicative penalty for having numeric characters, or punctuation characters. + * the price of a name in a bucket is ((coeff) * (base) ^ (bucket exponent)) / ((numeric discount multiplier) * (punctuation discount multiplier)) + +Example: +* base = 10 +* coeff = 2 +* nonalpha discount: 10 +* no-vowel discount: 10 +* buckets 1, 2: 9 +* buckets 3, 4, 5, 6: 8 +* buckets 7, 8, 9, 10, 11, 12, 13, 14: 7 +* buckets 15, 16+: + +With the above example configuration, the following are true: + +* The price of "john" would be 2 * 10^8, since "john" falls into bucket 4 and has no punctuation or numerics. +* The price of "john1" would be 2 * 10^6, since "john1" falls into bucket 5 but has a number (and thus receives a 10x discount) +* The price of "john_1" would be 2 * 10^6, since "john_1" falls into bucket 6 but has a number and punctuation (and thus receives a 10x discount) +* The price of "j0hn_1" would be 2 * 10^5, since "j0hn_1" falls into bucket 6 but has a number and punctuation and lacks vowels (and thus receives a 100x discount) + + +### NAME_IMPORT + +Op: `;` + +`OP_RETURN` wire format: +``` + 0 2 3 39 + |----|--|-----------------------------| + magic op name.ns_id (37 bytes) +``` + +Inputs: + +* The namespace reveal `scriptSig` (with the namespace revealer's public key), or one of its first 300 extended public keys +* Any payment inputs + +Outputs: + +* `OP_RETURN` payload +* recipient `scriptPubKey` +* zone file hash (using the 20-byte hash in a standard `p2pkh` script) +* payment change `scriptPubKey` + +Notes: + +* These transactions can only be sent between the `NAMESPACE_REVEAL` and `NAMESPACE_READY`. +* The first `NAME_IMPORT` transaction **must** have a `scriptSig` input that matches the `NAMESPACE_REVEAL`'s second output (i.e. the reveal output). +* Any subsequent `NAME_IMPORT` transactions **may** have a `scriptSig` input whose public key is one of the first 300 extended public keys from the `NAMESPACE_REVEAL`'s `scriptSig` public key. + +### NAMESPACE_READY + +Op: `!` + +`OP_RETURN` wire format: +``` + + 0 2 3 4 23 + |-----|--|--|------------| + magic op . ns_id +``` + +Inputs: +* Namespace revealer's `scriptSig`s + +Outputs: +* `OP_RETURN` payload +* Change output to the namespace revealer's `p2pkh` script + +Notes: +* This transaction must be sent within 1 year of the corresponding `NAMESPACE_REVEAL` to be accepted. + +## Method Glossary + +Some hashing primitives are used to construct the wire-format representation of each name operation. They are enumerated here: + +``` +B40_REGEX = '^[a-z0-9\-_.+]*$' + +def is_b40(s): + return isinstance(s, str) and re.match(B40_REGEX, s) is not None + +def b40_to_bin(s): + if not is_b40(s): + raise ValueError('{} must only contain characters in the b40 char set'.format(s)) + return unhexlify(charset_to_hex(s, B40_CHARS)) + +def hexpad(x): + return ('0' * (len(x) % 2)) + x + +def charset_to_hex(s, original_charset): + return hexpad(change_charset(s, original_charset, B16_CHARS)) + +def bin_hash160(s, hex_format=False): + """ s is in hex or binary format + """ + if hex_format and is_hex(s): + s = unhexlify(s) + return hashlib.new('ripemd160', bin_sha256(s)).digest() + +def hex_hash160(s, hex_format=False): + """ s is in hex or binary format + """ + if hex_format and is_hex(s): + s = unhexlify(s) + return hexlify(bin_hash160(s)) + +def hash_name(name, script_pubkey, register_addr=None): + """ + Generate the hash over a name and hex-string script pubkey. + Returns the hex-encoded string RIPEMD160(SHA256(x)), where + x is the byte string composed of the concatenation of the + binary + """ + bin_name = b40_to_bin(name) + name_and_pubkey = bin_name + unhexlify(script_pubkey) + + if register_addr is not None: + name_and_pubkey += str(register_addr) + + # make hex-encoded hash + return hex_hash160(name_and_pubkey) + +def hash128(data): + """ + Hash a string of data by taking its 256-bit sha256 and truncating it to 128 bits. + """ + return hexlify(bin_sha256(data)[0:16]) +``` + diff --git a/images/scripts/ubuntu-17.04.sh b/images/scripts/ubuntu-17.04.sh index a44147bc3..a5a13642e 100644 --- a/images/scripts/ubuntu-17.04.sh +++ b/images/scripts/ubuntu-17.04.sh @@ -4,11 +4,11 @@ # This script will create a folder `blockstack` in your current directory # this folder will contain: # a Python virtualenv with Blockstack-Core -# a git clone of the Blockstack-Portal node app +# a git clone of the Blockstack-Browser node app # (with dependencies installed) # a `bin` directory with scripts: -# blockstack_portal_start.sh -> for starting core and portal -# blockstack_portal_stop.sh -> for stopping portal +# blockstack_browser_start.sh -> for starting core and browser +# blockstack_browser_stop.sh -> for stopping browser # blockstack_core_stop.sh -> for stopping core # blockstack_copy_api_pass.sh-> copies the API key to the clipboard # @@ -74,33 +74,33 @@ mkdir -p "$DIR" virtualenv --python=python2.7 "$CORE_VENV" -"$CORE_VENV/bin/python" -m pip install git+https://github.com/blockstack/virtualchain.git@rc-0.14.2 -"$CORE_VENV/bin/python" -m pip install git+https://github.com/blockstack/blockstack-core.git@rc-0.14.2 +"$CORE_VENV/bin/python" -m pip install git+https://github.com/blockstack/virtualchain.git@rc-0.14.3 +"$CORE_VENV/bin/python" -m pip install git+https://github.com/blockstack/blockstack-core.git@rc-0.14.3 "$CORE_VENV/bin/python" "$CORE_VENV/bin/blockstack" setup -y --password "$BITCOIN_WALLET_PASSWORD" cd "$DIR" -git clone https://github.com/blockstack/blockstack-portal.git -bv0.9 -cd blockstack-portal +git clone https://github.com/blockstack/blockstack-browser.git -bv0.11.1 +cd blockstack-browser npm install node-sass npm install -echo "Installed Blockstack Core + Portal!" +echo "Installed Blockstack Core + Browser!" # make some bin scripts. mkdir "$DIR/bin" cd "$DIR/bin" -START_PORTAL_NAME=blockstack_portal_start.sh -STOP_PORTAL_NAME=blockstack_portal_stop.sh +START_PORTAL_NAME=blockstack_browser_start.sh +STOP_PORTAL_NAME=blockstack_browser_stop.sh STOP_CORE_NAME=blockstack_core_stop.sh COPY_API_NAME=blockstack_copy_api_pass.sh echo "#!/bin/bash" > $START_PORTAL_NAME -echo "cd \"$DIR/blockstack-portal\"" >> $START_PORTAL_NAME +echo "cd \"$DIR/blockstack-browser\"" >> $START_PORTAL_NAME echo "\"$CORE_VENV/bin/python\" \"$CORE_VENV/bin/blockstack\" api status -y | grep 'true' > /dev/null" >> $START_PORTAL_NAME echo "if [ \$? -ne 0 ]; then" >> $START_PORTAL_NAME echo "\"$CORE_VENV/bin/python\" \"$CORE_VENV/bin/blockstack\" api start -y --debug" >> $START_PORTAL_NAME @@ -118,7 +118,7 @@ echo "#!/bin/bash" > $STOP_PORTAL_NAME echo "tokill=\$(cat /tmp/dev.pid)" >> $STOP_PORTAL_NAME echo "echo 'Terminating process group of \$tokill'" >> $STOP_PORTAL_NAME echo "kill -s SIGTERM -\$(ps -o pgid= \$tokill | cut -d\\ -f2)" >> $STOP_PORTAL_NAME -echo "echo 'Killed Blockstack Portal'" >> $STOP_PORTAL_NAME +echo "echo 'Killed Blockstack Browser'" >> $STOP_PORTAL_NAME echo "#!/bin/bash" > $STOP_CORE_NAME echo "\"$CORE_VENV/bin/python\" \"$CORE_VENV/bin/blockstack\" api stop -y" >> $STOP_CORE_NAME @@ -134,3 +134,13 @@ chmod +x $STOP_CORE_NAME echo "Made app scripts!" echo "You can add bins to your path with \$ export PATH=\$PWD/blockstack/bin:\$PATH" +echo "You may need to add a protocol handler for your system, add a .desktop like the following and it should work: " +echo +echo "[Desktop Entry]" +echo "Type=Application" +echo "Terminal=false" +echo "Exec=bash -c 'xdg-open http://localhost:3000/auth?authRequest=\$(echo \"%u\" | sed s,blockstack:/*,,)'" +echo "Name=Blockstack-Browser" +echo "MimeType=x-scheme-handler/blockstack;" +echo +echo diff --git a/integration_tests/bin/blockstack-browser-test-mode.sh b/integration_tests/bin/blockstack-browser-test-mode.sh index 00e5df50d..c8865d2a0 100755 --- a/integration_tests/bin/blockstack-browser-test-mode.sh +++ b/integration_tests/bin/blockstack-browser-test-mode.sh @@ -1,11 +1,11 @@ #!/bin/bash -# launcher for browser +# launcher for browser/portal -BROWSER_DIR='/tmp/blockstack-browser' -if ! [ -d "$BROWSER_DIR" ]; then +PORTAL_DIR='/tmp/blockstack-portal' +if ! [ -d "$PORTAL_DIR" ]; then echo "" - echo "Missing $BROWSER_DIR" + echo "Missing $PORTAL_DIR" echo "Please make sure Blockstack Browser is available in that directory" echo "(https://github.com/blockstack/blockstack-browser)" echo "" @@ -16,13 +16,9 @@ TEST_PID= DEV_PROXY_PID= # start environment -# override RPC port export BLOCKSTACK_TEST_CLIENT_RPC_PORT=6270 -# tell the RPC system to encode all addresses to mainnet, even though we're in testing -export BLOCKSTACK_RPC_MOCK_BLOCKCHAIN_NETWORK="mainnet" - -blockstack-test-scenario --interactive-web 3001 blockstack_integration_tests.scenarios.portal_empty_namespace 2>&1 | tee browser_test.log & +blockstack-test-scenario --interactive-web 3001 blockstack_integration_tests.scenarios.portal_empty_namespace 2>&1 | tee portal_test.log & TEST_PID=$! # wait for it to begin serving @@ -35,12 +31,12 @@ while true; do fi done -pushd "$BROWSER_DIR" >/dev/null +pushd "$PORTAL_DIR" >/dev/null echo "" echo "Starting Browser CORS proxy" echo "" -npm run dev-proxy 2>&1 | tee "$BROWSER_DIR/cors-proxy.log" & +npm run dev-proxy 2>&1 | tee "$PORTAL_DIR/cors-proxy.log" & DEV_PROXY_PID=$! sleep 5 diff --git a/integration_tests/bin/blockstack-test-all-junit b/integration_tests/bin/blockstack-test-all-junit new file mode 100755 index 000000000..c3c32e30a --- /dev/null +++ b/integration_tests/bin/blockstack-test-all-junit @@ -0,0 +1,109 @@ +#!/bin/sh + +usage() { + echo "Usage: $0 [path/to/test/output/dir] [OPTIONAL: path/to/tests/to/skip.txt] [OPTIONAL: path/to/existing/test/logdir/]" + exit 1 +} + +if [ $# -lt 1 ]; then + usage $0 +fi + +RUN_SCENARIO="blockstack-test-scenario" +CHECK_SERIALIZATION="blockstack-test-check-serialization" +SCENARIOS_PYTHON="blockstack_integration_tests.scenarios" +SCENARIOS="$(BLOCKSTACK_TESTNET=1 python -c "import blockstack_integration_tests; import blockstack_integration_tests.scenarios; print blockstack_integration_tests.scenarios.__path__[0]")" +if [ $? -ne 0 ]; then + echo >&2 "Failed to load blockstack integration test scenarios" + exit 1 +fi + +OUTPUTS="$1" +TESTS_SKIP="$2" +EXISTING_LOGS="$3" + +test -d "$OUTPUTS" || mkdir -p "$OUTPUTS" + +while IFS= read SCENARIO_FILE; do + + if ! [ "$(echo "$SCENARIO_FILE" | egrep '.py$')" ]; then + continue + fi + + if [ "$SCENARIO_FILE" = "__init__.py" ] || [ "$SCENARIO_FILE" = "testlib.py" ]; then + continue + fi + + SCENARIO_MODULE_BASE="$(echo "$SCENARIO_FILE" | sed 's/\.py//g')" + SCENARIO_MODULE="$SCENARIOS_PYTHON.$SCENARIO_MODULE_BASE" + + if [ -n "$TESTS_SKIP" ] && [ -n "$(fgrep "$SCENARIO_MODULE_BASE" "$TESTS_SKIP")" ]; then + continue + fi + + TESTDIR="/tmp/blockstack-test" + + if ! [ -f "$OUTPUTS/$SCENARIO_MODULE_BASE.log" ]; then + + echo -n "$SCENARIO_MODULE ... " + + mkdir -p "$TESTDIR" + "$RUN_SCENARIO" "$SCENARIO_MODULE" "$TESTDIR" > "$OUTPUTS/$SCENARIO_MODULE_BASE.log" 2>&1 + + RC=$? + + # make the xunit file + OUTPUT_FILE="$OUTPUTS/$SCENARIO_MODULE_BASE.xml" + + echo '' > "$OUTPUT_FILE" + echo "" >> "$OUTPUT_FILE" + if [ $RC -ne 0 ]; then + echo '' >> "$OUTPUT_FILE" + fi + echo "" >> "$OUTPUT_FILE" + + if [ $RC -ne 0 ]; then + # failed + echo " FAILURE" + mv "$TESTDIR" "$OUTPUTS/$SCENARIO_MODULE_BASE.d" + + continue + fi + + # compare serialized consensus fields with older release + EXISTING_LOG="$EXISTING_LOGS/$SCENARIO_MODULE_BASE.log" + if [ -f "$EXISTING_LOG" ]; then + + "$CHECK_SERIALIZATION" "$EXISTING_LOGS/$SCENARIO_MODULE_BASE.log" "$OUTPUTS/$SCENARIO_MODULE_BASE.log" + RC=$? + + if [ $RC -ne 0 ]; then + + if [ $RC -eq 1 ]; then + # generated incorrect serialization output + echo " (ERROR: mismatched serialization) FAILURE" + mv "$TESTDIR" "$OUTPUTS/$SCENARIO_MODULE_BASE.d" + + # TODO: only exit with option + exit 1 + + else + # exit 2 means no serialization check happened + echo " (WARN: SKIPPED SERIALIZATION CHECK) SUCCESS" + fi + fi + else + + echo -n " (WARN: no existing log) " + fi + + echo " SUCCESS" + rm -rf "$TESTDIR" + killall bitcoind + fi + +done <. +""" + +import testlib +import pybitcoin +import json +import time +import blockstack_client +import sys + +wallets = [ + testlib.Wallet( "5JesPiN68qt44Hc2nT8qmyZ1JDwHebfoh9KQ52Lazb1m1LaKNj9", 100000000000 ), + testlib.Wallet( "5KHqsiU9qa77frZb6hQy9ocV7Sus9RWJcQGYYBJJBb2Efj1o77e", 100000000000 ), + testlib.Wallet( "5Kg5kJbQHvk1B64rJniEmgbD83FpZpbw2RjdAZEzTefs9ihN3Bz", 100000000000 ), + testlib.Wallet( "5JuVsoS9NauksSkqEjbUZxWwgGDQbMwPsEfoRBSpLpgDX1RtLX7", 100000000000 ), + testlib.Wallet( "5KEpiSRr1BrT8vRD7LKGCEmudokTh1iMHbiThMQpLdwBwhDJB1T", 100000000000 ) +] + +consensus = "17ac43c1d8549c3181b200f1bf97eb7d" + +def scenario( wallets, **kw ): + + testlib.blockstack_namespace_preorder( "test", wallets[1].addr, wallets[0].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_namespace_reveal( "test", wallets[1].addr, 52595, 250, 4, [6,5,4,3,2,1,0,0,0,0,0,0,0,0,0,0], 10, 10, wallets[0].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_namespace_ready( "test", wallets[1].privkey ) + testlib.next_block( **kw ) + + wallet = testlib.blockstack_client_initialize_wallet( "0123456789abcdef", wallets[2].privkey, wallets[3].privkey, wallets[4].privkey ) + resp = testlib.blockstack_cli_register( "foo.test", "0123456789abcdef" ) + if 'error' in resp: + print >> sys.stderr, json.dumps(resp, indent=4, sort_keys=True) + return False + + # wait for the preorder to get confirmed + for i in xrange(0, 12): + testlib.next_block( **kw ) + + # wait for the poller to pick it up + print >> sys.stderr, "Waiting 10 seconds for the backend to submit the register" + time.sleep(10) + + # wait for the register to get confirmed + for i in xrange(0, 12): + testlib.next_block( **kw ) + + print >> sys.stderr, "Waiting 10 seconds for the backend to acknowledge registration" + time.sleep(10) + + # wait for update to get confirmed + for i in xrange(0, 12): + testlib.next_block( **kw ) + + print >> sys.stderr, "Waiting 10 seconds for the backend to acknowledge update" + time.sleep(10) + + # let's make sure the zonefile propagates... + + res = testlib.blockstack_REST_call("GET", "/v1/names/foo.test", None) + if 'error' in res or res['http_status'] != 200: + res['test'] = 'Failed to get name bar.test' + print json.dumps(res) + return False + + response_json = json.loads(res['raw']) + + # assert that we're getting a mainnet address + if response_json['address'] != "1K4SCfqffVHTnHvqsW2whH1Jn57dDjDdQA": + print "Address returned to REST call isn't converted to mainnet!" + print "Returned: {}, Expected: {}".format(response_json['address'], "1K4SCfqffVHTnHvqsW2whH1Jn57dDjDdQA") + return False + +def check( state_engine ): + + return True diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder.py index d8c1baa66..f13806b9d 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder.py @@ -69,7 +69,10 @@ def check( state_engine ): return False # paid fee - if preorder['op_fee'] < blockstackd.get_name_cost( 'foo.test' ): + proxy = testlib.make_proxy() + + if preorder['op_fee'] < proxy.get_name_cost( 'foo.test' )['satoshis']: + print "{} < {}".format(preorder['op_fee'], proxy.get_name_cost( 'foo.test' )) print "Insufficient fee" return False diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_portal.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_portal.py index 08ab3119b..e5baae068 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_portal.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_portal.py @@ -59,15 +59,8 @@ def scenario( wallets, **kw ): testlib.blockstack_name_register( "demo.id", wallets[2].privkey, wallets[3].addr ) testlib.next_block( **kw ) - - # add a token file - res = testlib.migrate_profile( "demo.id", wallet_keys=wallet ) - if 'error' in res: - res['test'] = 'Failed to initialize demo.id profile' - print json.dumps(res, indent=4, sort_keys=True) - return False - - testlib.next_block( **kw ) + + def check( state_engine ): diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore.py index c63e245ad..2ba1822cf 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore.py @@ -93,7 +93,7 @@ def scenario( wallets, **kw ): # sign in and make a token datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() - res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_blockchain_id.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_blockchain_id.py index b2042a294..46feca386 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_blockchain_id.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_blockchain_id.py @@ -93,7 +93,7 @@ def scenario( wallets, **kw ): # sign in and make a token datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() - res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiservice.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiservice.py index b61127b40..aa0e4cfd7 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiservice.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiservice.py @@ -96,7 +96,7 @@ def scenario( wallets, **kw ): # sign in and make a token datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() - res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiservice_failedservice.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiservice_failedservice.py index 6853b249b..f168d4ed8 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiservice_failedservice.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiservice_failedservice.py @@ -96,7 +96,7 @@ def scenario( wallets, **kw ): # sign in and make a token datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() - res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True @@ -133,8 +133,6 @@ def scenario( wallets, **kw ): if 'error' not in res: print 'accidentally succeeded to mkdir {}: {}'.format(dpath, res) return False - - # clear inode cache # stat directories (should all fail locally due to ENOENT, since the parent directory will not have been updated) for dpath in ['/dir1/dir3/dir4', '/dir1/dir3', '/dir2', '/dir1']: @@ -345,22 +343,16 @@ def scenario( wallets, **kw ): print 'accidentally got {}: {}'.format(dpath, res) return False - # if res['errno'] != errno.EREMOTEIO: - if res['errno'] != errno.ENOENT: # ENOENT since the latest data is in disk, or cache + if res['errno'] != errno.EREMOTEIO: print 'wrong errno: {}'.format(res) return False - # stat files (should fail) + # stat files (should still work) for dpath in ['/file1', '/file2', '/dir1/file3', '/dir1/dir3/file4', '/dir1/dir3/dir4/file5']: - print 'stat {} (should fail)'.format(dpath) + print 'stat {} (should still work)'.format(dpath) res = testlib.blockstack_cli_datastore_stat( 'foo.test', datastore_id, dpath, ses ) - if 'error' not in res: - print 'accidentally got {}: {}'.format(dpath, res) - return False - - # if res['errno'] != errno.EREMOTEIO: - if res['errno'] != errno.ENOENT: # ENOENT since the latest data is in disk, or cache - print 'wrong errno: {}'.format(res) + if 'error' in res: + print 'failed to stat {}: {}'.format(path, res) return False # restore test driver @@ -369,19 +361,6 @@ def scenario( wallets, **kw ): print 'failed to setenv: {}'.format(res) return False - # stat files (should fail) - for dpath in ['/file1', '/file2', '/dir1/file3', '/dir1/dir3/file4', '/dir1/dir3/dir4/file5']: - print 'stat {} (should fail)'.format(dpath) - res = testlib.blockstack_cli_datastore_stat( 'foo.test', datastore_id, dpath, ses ) - if 'error' not in res: - print 'accidentally got {}: {}'.format(dpath, res) - return False - - # if res['errno'] != errno.EREMOTEIO: - if res['errno'] != errno.ENOENT: # ENOENT since the latest data is in disk, or cache - print 'wrong errno: {}'.format(res) - return False - ''' # stat files (should still work) for dpath in ['/file1', '/file2', '/dir1/file3', '/dir1/dir3/file4', '/dir1/dir3/dir4/file5']: print 'stat {}'.format(dpath) @@ -389,26 +368,14 @@ def scenario( wallets, **kw ): if 'error' in res: print 'failed to stat {}: {}'.format(path, res) return False - ''' - # remove files (should fail) + # remove files (should work now) for dpath in ['/file1', '/file2', '/dir1/file3', '/dir1/dir3/file4', '/dir1/dir3/dir4/file5']: print 'deletefile {}'.format(dpath) res = testlib.blockstack_cli_datastore_deletefile( 'foo.test', datastore_pk, dpath, ses ) - if 'error' not in res: - print 'accidentally succeeded to delete {}: {}'.format(dpath, res) - return False - - # if res['errno'] != errno.EREMOTEIO: - if res['errno'] != errno.ENOENT: # ENOENT since the latest data is in disk, or cache - print 'wrong errno: {}'.format(res) - return False - - ''' if 'error' in res: print 'failed to deletefile {}: {}'.format(dpath, res) return False - ''' # put file data (should succeed on both 'disk' and 'test') for dpath in ['/file1', '/file2', '/dir1/file3', '/dir1/dir3/file4', '/dir1/dir3/dir4/file5']: @@ -585,9 +552,8 @@ def scenario( wallets, **kw ): return False if dpath not in ['/dir1/dir3/dir4', '/dir2']: - #if res.get('errno') != errno.ENOTEMPTY: - if res.get('errno') != errno.EREMOTEIO: - print 'wrong errno for deleting {}: expected EREMOTEIO'.format(res) + if res.get('errno') != errno.ENOTEMPTY: + print 'wrong errno for deleting {}'.format(res) return False # list directories (should fail for /dir1/dir3/dir4 and /dir2 since its idata got deleted, but it should still work for everyone else) @@ -598,11 +564,10 @@ def scenario( wallets, **kw ): print 'accidentally succeeded to list {}: {}'.format(dpath, res) return False - if res['errno'] != errno.ENOENT: + if res['errno'] != errno.EREMOTEIO: print 'wrong errno: {}'.format(res) return False - ''' # these should still work for dpath, expected in [('/', ['dir1', 'dir2']), ('/dir1', ['dir3']), ('/dir1/dir3', ['dir4'])]: print 'listdir {} (should still work)'.format(dpath) @@ -619,7 +584,6 @@ def scenario( wallets, **kw ): if not res['children'].has_key(child): print 'invalid directory: missing {} in {}'.format(child, res) return False - ''' # restore service res = testlib.blockstack_test_setenv('BLOCKSTACK_INTEGRATION_TEST_STORAGE_FAILURE', '0') @@ -627,7 +591,6 @@ def scenario( wallets, **kw ): print 'failed to setenv: {}'.format(res) return False - ''' # remove directories (should succeed) for dpath in ['/dir1/dir3/dir4', '/dir1/dir3', '/dir2', '/dir1']: print 'rmdir {}'.format(dpath) @@ -635,7 +598,6 @@ def scenario( wallets, **kw ): if 'error' in res: print 'failed to rmdir {}: {}'.format(dpath, res['error']) return False - ''' # stat directories (should all fail) for dpath in ['/dir1/dir3/dir4', '/dir1/dir3', '/dir2', '/dir1']: @@ -700,13 +662,11 @@ def scenario( wallets, **kw ): print json.dumps(res) return False - ''' # no more data in test-disk driver names = os.listdir("/tmp/blockstack-integration-test-storage/mutable") if names != ['foo.test']: print 'improper cleanup on test' return False - ''' # due to our failed mkdir of /dir1 and /dir2, these # will have leaked. Expect 5 entries (including foo.test): diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiuser.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiuser.py index e43a9b0e7..293e733ad 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiuser.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_multiuser.py @@ -116,7 +116,7 @@ def activate_account(blockchain_id, datastore_pk): return False # sign in and make a token with the given blockchain ID (whose wallet must be currently set) - res = testlib.blockstack_cli_app_signin(blockchain_id, datastore_pk, 'app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin(blockchain_id, datastore_pk, 'app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) return False @@ -137,7 +137,7 @@ def core_signin(datastore_pk, blockchain_id): global sessions # sign in and make a token with the given blockchain ID (whose wallet must be currently set) - res = testlib.blockstack_cli_app_signin(blockchain_id, datastore_pk, 'app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin(blockchain_id, datastore_pk, 'app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) return False @@ -167,26 +167,8 @@ def target_datastore(blockchain_id): if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) return False - - old_root = os.environ.get('TEST_BLOCKSTACK_TEST_DISK_ROOT', None) - old_blockchain_id = None - - if old_root: - old_root_parts = old_root.split('-') - if old_root_parts[-1].endswith('.test'): - old_blockchain_id = old_root_parts[-1] - - new_root = '/tmp/blockstack-integration-test-storage{}'.format(blockchain_id) - os.environ['TEST_BLOCKSTACK_TEST_DISK_ROOT'] = new_root - - if old_blockchain_id and os.path.exists(new_root + '/index'): - # sanity check: look for weird race conditions - rc = os.system("fgrep '{}' -r '{}/index' >/dev/null".format(old_blockchain_id, new_root)) - if rc != 0: - print '\n\nBUG: blockchain ID {} found in {}/index'.format(old_blockchain_id, new_root) - return False - - return True + + os.environ['TEST_BLOCKSTACK_TEST_DISK_ROOT'] = '/tmp/blockstack-integration-test-storage{}'.format(blockchain_id) def setup_datastore(datastore_pk, blockchain_id, write_iteration): @@ -560,9 +542,7 @@ def scenario( wallets, **kw ): print 'failed to start API for foo.test: {}'.format(res) return False - rc = target_datastore('foo.test') - if not rc: - return False + target_datastore('foo.test') # instantiate foo.test's test driver res = testlib.blockstack_REST_call('POST', '/v1/node/drivers/storage/test?index=1&force=1', None, api_pass='blockstack_integration_test_api_password') @@ -585,9 +565,7 @@ def scenario( wallets, **kw ): # link test account for bar.test # BUT! make sure we store the profile for bar.test into foo.test's and bar.test's storage directories! - rc = target_datastore('bar.test') - if not rc: - return False + target_datastore('bar.test') # instantiate bar.test's test driver res = testlib.blockstack_REST_call('POST', '/v1/node/drivers/storage/test?index=1&force=1', None, api_pass='blockstack_integration_test_api_password') @@ -641,9 +619,7 @@ def scenario( wallets, **kw ): # make *absolutely certain* that the test driver does not load data from # foo.test's or bar.test's storage directories. We want to verify that we can look up # the index manifest URLs from the profile - rc = target_datastore(None) - if not rc: - return False + target_datastore(None) print "\n\nbar.test tries to read foo.test's datastore {}\n\n".format(foo_datastore_id) res = read_datastore(foo_datastore_id, "foo.test", 1) @@ -666,9 +642,7 @@ def scenario( wallets, **kw ): # make *absolutely certain* that the test driver does not load data from # foo.test's or bar.test's storage directories. We want to verify that we can look up # the index manifest URLs from the profile - rc = target_datastore(None) - if not rc: - return False + target_datastore(None) # try to read all of bar.test's files print "\n\nfoo.test tries to read bar.test's datastore {}\n\n".format(bar_datastore_id) @@ -678,9 +652,7 @@ def scenario( wallets, **kw ): return False # re-target foo.test's datastore - rc = target_datastore('foo.test') - if not rc: - return False + target_datastore('foo.test') # have foo.test write new files print '\n\nupdate foo.test datastore\n\n' @@ -698,9 +670,7 @@ def scenario( wallets, **kw ): # make *absolutely certain* that the test driver does not load data from # foo.test's or bar.test's storage directories. We want to verify that we can look up # the index manifest URLs from the profile - rc = target_datastore(None) - if not rc: - return False + target_datastore(None) # get foo.test's new files res = read_datastore(foo_datastore_id, 'foo.test', 3) @@ -709,9 +679,7 @@ def scenario( wallets, **kw ): return False # re-target bar.test's datastore - rc = target_datastore('bar.test') - if not rc: - return False + target_datastore('bar.test') # have bar write some new files print '\n\nupdate bar.test datastore\n\n' @@ -742,9 +710,7 @@ def scenario( wallets, **kw ): # make *absolutely certain* that the test driver does not load data from # foo.test's or bar.test's storage directories. We want to verify that we can look up # the index manifest URLs from the profile - rc = target_datastore(None) - if not rc: - return False + target_datastore(None) # verify that foo's files are gone res = check_datastore_files_absent(foo_datastore_id, 'foo.test') @@ -753,9 +719,7 @@ def scenario( wallets, **kw ): return False # re-target bar.test's datastore - rc = target_datastore('bar.test') - if not rc: - return False + target_datastore('bar.test') # clear bar.test's files print '\n\ndelete bar.test files\n\n' @@ -773,9 +737,7 @@ def scenario( wallets, **kw ): # make *absolutely certain* that the test driver does not load data from # foo.test's or bar.test's storage directories. We want to verify that we can look up # the index manifest URLs from the profile - rc = target_datastore(None) - if not rc: - return False + target_datastore(None) # verify that bar's files are gone res = check_datastore_files_absent(bar_datastore_id, 'bar.test') @@ -784,9 +746,7 @@ def scenario( wallets, **kw ): return False # re-target foo.test's datastore - rc = target_datastore("foo.test") - if not rc: - return False + target_datastore("foo.test") # clear foo's directories res = clear_datastore_directories('foo.test', foo_datastore_pk) @@ -803,9 +763,7 @@ def scenario( wallets, **kw ): # make *absolutely certain* that the test driver does not load data from # foo.test's or bar.test's storage directories. We want to verify that we can look up # the index manifest URLs from the profile - rc = target_datastore(None) - if not rc: - return False + target_datastore(None) # verify that foo's directories are gone res = check_datastore_directories_absent(foo_datastore_id, "foo.test") @@ -814,9 +772,7 @@ def scenario( wallets, **kw ): return False # re-target bar.test's datastore - rc = target_datastore('bar.test') - if not rc: - return False + target_datastore('bar.test') # clear bar's directories res = clear_datastore_directories('bar.test', bar_datastore_pk) @@ -833,9 +789,7 @@ def scenario( wallets, **kw ): # make *absolutely certain* that the test driver does not load data from # foo.test's or bar.test's storage directories. We want to verify that we can look up # the index manifest URLs from the profile - rc = target_datastore(None) - if not rc: - return False + target_datastore(None) # verify that bar's directories are gone res = check_datastore_directories_absent(bar_datastore_id, "bar.test") diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_rmtree.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_rmtree.py index 4649e3f34..e0cab24c9 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_rmtree.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_rmtree.py @@ -93,7 +93,7 @@ def scenario( wallets, **kw ): # sign in and make a token datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() - res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_stale_directory.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_stale_directory.py index 043ece466..43ade9333 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_stale_directory.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_stale_directory.py @@ -95,7 +95,7 @@ def scenario( wallets, **kw ): # sign in and make a token datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() - res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True diff --git a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_stale_file.py b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_stale_file.py index 21242c25b..f1d3685ce 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_stale_file.py +++ b/integration_tests/blockstack_integration_tests/scenarios/name_preorder_register_update_app_datastore_stale_file.py @@ -95,7 +95,7 @@ def scenario( wallets, **kw ): # sign in and make a token datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() - res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com.1', ['store_read', 'store_write', 'store_admin']) + res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'foo-app.com', ['store_read', 'store_write', 'store_admin']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True diff --git a/integration_tests/blockstack_integration_tests/scenarios/rest_register_recipient_set_key.py b/integration_tests/blockstack_integration_tests/scenarios/rest_register_recipient_set_key.py index 6a9d70f90..421e86fb2 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/rest_register_recipient_set_key.py +++ b/integration_tests/blockstack_integration_tests/scenarios/rest_register_recipient_set_key.py @@ -54,7 +54,11 @@ error = False index_file_data = "foo.test hello world" resource_data = "hello world" - + +new_key = "cPo24qGYz76xSbUCug6e8LzmzLGJPZoowQC7fCVPLN2tzCUJgfcW" +new_addr = "mqnupoveYRrSHmrxFT9nQQEZt3RLsetbBQ" + +insanity_key = "cSCyE5Q1AFVyDAL8LkHo1sFMVqmwdvFcCbGJ71xEvto2Nrtzjm67" def scenario( wallets, **kw ): @@ -134,8 +138,8 @@ def scenario( wallets, **kw ): api_pass = conf['api_password'] - res = testlib.blockstack_REST_call('PUT', '/v1/wallet/keys/owner', None, api_pass=api_pass, - data=wallets[4].privkey) + res = testlib.blockstack_REST_call('PUT', '/v1/wallet/keys/owner', None, api_pass=api_pass, + data=new_key) if res['http_status'] != 200 or 'error' in res: print 'failed to set owner key' print res @@ -150,7 +154,7 @@ def scenario( wallets, **kw ): # leaving the call format of this one the same to make sure that our registrar correctly # detects that the requested TRANSFER is superfluous # register the name bar.test - res = testlib.blockstack_REST_call('POST', '/v1/names', ses, data={'name': 'bar.test', 'zonefile': zonefile_txt, 'owner_address': wallets[4].addr}) + res = testlib.blockstack_REST_call('POST', '/v1/names', ses, data={'name': 'bar.test', 'zonefile': zonefile_txt, 'owner_address': new_addr }) if 'error' in res: res['test'] = 'Failed to register user' print json.dumps(res) @@ -261,6 +265,104 @@ def scenario( wallets, **kw ): print json.dumps(res) return False + + ### Now, we'll do it again, but this time, we're going to CHANGE THE KEY in the middle of registrations. + ### to test the different paths, I'll start 3 registrations: + # 1 has submitted preorder + # 1 has submitted register + # 1 has submitted update + ### And then I'll issue a change-key + + + # make zonefile for recipients + zonefiles = [] + for i in [1,2,3]: + name = "tricky{}.test".format(i) + driver_urls = blockstack_client.storage.make_mutable_data_urls(name, use_only=['dht', 'disk']) + zonefile = blockstack_client.zonefile.make_empty_zonefile(name, wallets[4].pubkey_hex, urls=driver_urls) + zonefiles.append(blockstack_zones.make_zone_file( zonefile, origin=name, ttl=3600 )) + + # leaving the call format of this one the same to make sure that our registrar correctly + # detects that the requested TRANSFER is superfluous + res = testlib.blockstack_REST_call( + 'POST', '/v1/names', ses, data={'name':'tricky1.test', 'zonefile':zonefiles[0], 'owner_address':new_addr}) + if 'error' in res: + res['test'] = 'Failed to register tricky1.test' + print json.dumps(res) + error = True + return False + + tx_hash = res['response']['transaction_hash'] + # wait for preorder to get confirmed... + for i in xrange(0, 6): + testlib.next_block( **kw ) + + res = testlib.blockstack_REST_call( + 'POST', '/v1/names', ses, data={'name':'tricky2.test', 'zonefile':zonefiles[1], 'owner_address':new_addr}) + if 'error' in res: + res['test'] = 'Failed to register tricky2.test' + print json.dumps(res) + error = True + return False + + # wait for the preorder to get confirmed + for i in xrange(0, 6): + testlib.next_block( **kw ) + + # wait for register to go through + print 'Wait for register to be submitted' + time.sleep(10) + + # wait for the register to get confirmed + for i in xrange(0, 6): + testlib.next_block( **kw ) + + res = testlib.blockstack_REST_call( + 'POST', '/v1/names', ses, data={'name':'tricky3.test', 'zonefile':zonefiles[2], 'owner_address':new_addr}) + if 'error' in res: + res['test'] = 'Failed to register tricky3.test' + print json.dumps(res) + error = True + return False + + for i in xrange(0, 6): + testlib.next_block( **kw ) + + print 'Wait for update to be submitted' + time.sleep(10) + + for i in xrange(0, 1): + testlib.next_block( **kw ) + + + + res = testlib.verify_in_queue(ses, 'tricky1.test', 'update', None) + res = testlib.verify_in_queue(ses, 'tricky2.test', 'register', None) + res = testlib.verify_in_queue(ses, 'tricky3.test', 'preorder', None) + + # let's go crazy. + res = testlib.blockstack_REST_call('PUT', '/v1/wallet/keys/owner', None, api_pass=api_pass, + data=insanity_key) + if res['http_status'] != 200 or 'error' in res: + print 'failed to set owner key' + print res + return False + + + # wait for preorder to get confirmed + for i in xrange(0, 6): + testlib.next_block( **kw ) + # wake up registrar, submit register + time.sleep(10) + for i in xrange(0, 12): + testlib.next_block( **kw ) + # wake up registrar, submit update + time.sleep(10) + for i in xrange(0, 12): + testlib.next_block( **kw ) + # wake up registrar, propogate zonefile + time.sleep(10) + def check( state_engine ): global wallet_keys, error, index_file_data, resource_data @@ -287,9 +389,8 @@ def check( state_engine ): print "wrong namespace" return False - names = ['foo.test', 'bar.test'] - owners = [ wallets[3].addr , wallets[4].addr] - wallet_keys_list = [wallet_keys, wallet_keys] + names = ['foo.test', 'bar.test', 'tricky1.test', 'tricky2.test', 'tricky3.test'] + owners = [ wallets[3].addr , new_addr, new_addr, new_addr, new_addr ] test_proxy = testlib.TestAPIProxy() for i in xrange(0, len(names)): diff --git a/integration_tests/blockstack_integration_tests/scenarios/rest_register_transfer.py b/integration_tests/blockstack_integration_tests/scenarios/rest_register_transfer.py index f91a977f0..f6e07c706 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/rest_register_transfer.py +++ b/integration_tests/blockstack_integration_tests/scenarios/rest_register_transfer.py @@ -30,6 +30,7 @@ import blockstack_profiles import sys import keylib import time +import virtualchain from keylib import ECPrivateKey, ECPublicKey @@ -204,8 +205,10 @@ def scenario( wallets, **kw ): print json.dumps(res) return False - if res['response']['address'] != wallets[7].addr: + address_testnet = virtualchain.address_reencode(str(res['response']['address']), network='testnet') + if address_testnet != wallets[7].addr: res['test'] = 'Failed to transfer name to new address {}'.format(wallets[7].addr) + res['owner_address_testnet'] = address_testnet print json.dumps(res) return False diff --git a/integration_tests/blockstack_integration_tests/scenarios/rest_register_update_zf.py b/integration_tests/blockstack_integration_tests/scenarios/rest_register_update_zf.py new file mode 100644 index 000000000..a7358b0f0 --- /dev/null +++ b/integration_tests/blockstack_integration_tests/scenarios/rest_register_update_zf.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + Blockstack + ~~~~~ + copyright: (c) 2014-2015 by Halfmoon Labs, Inc. + copyright: (c) 2016 by Blockstack.org + + This file is part of Blockstack + + Blockstack is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Blockstack is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with Blockstack. If not, see . +""" +import os +import testlib +import pybitcoin +import urllib2 +import json +import blockstack_client +import blockstack_profiles +import sys +import keylib +import time + +from keylib import ECPrivateKey, ECPublicKey + +wallets = [ + testlib.Wallet( "5JesPiN68qt44Hc2nT8qmyZ1JDwHebfoh9KQ52Lazb1m1LaKNj9", 100000000000 ), + testlib.Wallet( "5KHqsiU9qa77frZb6hQy9ocV7Sus9RWJcQGYYBJJBb2Efj1o77e", 100000000000 ), + testlib.Wallet( "5Kg5kJbQHvk1B64rJniEmgbD83FpZpbw2RjdAZEzTefs9ihN3Bz", 100000000000 ), + testlib.Wallet( "5JuVsoS9NauksSkqEjbUZxWwgGDQbMwPsEfoRBSpLpgDX1RtLX7", 100000000000 ), + testlib.Wallet( "5KEpiSRr1BrT8vRD7LKGCEmudokTh1iMHbiThMQpLdwBwhDJB1T", 100000000000 ), + testlib.Wallet( "5K5hDuynZ6EQrZ4efrchCwy6DLhdsEzuJtTDAf3hqdsCKbxfoeD", 100000000000 ), + testlib.Wallet( "5J39aXEeHh9LwfQ4Gy5Vieo7sbqiUMBXkPH7SaMHixJhSSBpAqz", 100000000000 ), + testlib.Wallet( "5K9LmMQskQ9jP1p7dyieLDAeB6vsAj4GK8dmGNJAXS1qHDqnWhP", 100000000000 ), + testlib.Wallet( "5KcNen67ERBuvz2f649t9F2o1ddTjC5pVUEqcMtbxNgHqgxG2gZ", 100000000000 ), + testlib.Wallet( "5KMbNjgZt29V6VNbcAmebaUT2CZMxqSridtM46jv4NkKTP8DHdV", 100000000000 ), +] + +consensus = "17ac43c1d8549c3181b200f1bf97eb7d" +wallet_keys = None +wallet_keys_2 = None +error = False + +index_file_data = "foo.test hello world" +resource_data = "hello world" + + +def scenario( wallets, **kw ): + + global wallet_keys, wallet_keys_2, error, index_file_data, resource_data + + wallet_keys = testlib.blockstack_client_initialize_wallet( "0123456789abcdef", wallets[5].privkey, wallets[3].privkey, wallets[4].privkey ) + test_proxy = testlib.TestAPIProxy() + blockstack_client.set_default_proxy( test_proxy ) + + testlib.blockstack_namespace_preorder( "test", wallets[1].addr, wallets[0].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_namespace_reveal( "test", wallets[1].addr, 52595, 250, 4, [6,5,4,3,2,1,0,0,0,0,0,0,0,0,0,0], 10, 10, wallets[0].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_namespace_ready( "test", wallets[1].privkey ) + testlib.next_block( **kw ) + + testlib.blockstack_name_preorder( "foo.test", wallets[2].privkey, wallets[3].addr ) + testlib.next_block( **kw ) + + testlib.blockstack_name_register( "foo.test", wallets[2].privkey, wallets[3].addr ) + testlib.next_block( **kw ) + + # migrate profiles + res = testlib.migrate_profile( "foo.test", proxy=test_proxy, wallet_keys=wallet_keys ) + if 'error' in res: + res['test'] = 'Failed to initialize foo.test profile' + print json.dumps(res, indent=4, sort_keys=True) + error = True + return + + # tell serialization-checker that value_hash can be ignored here + print "BLOCKSTACK_SERIALIZATION_CHECK_IGNORE value_hash" + sys.stdout.flush() + + testlib.next_block( **kw ) + + config_path = os.environ.get("BLOCKSTACK_CLIENT_CONFIG", None) + + # make a session + datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() + res = testlib.blockstack_cli_app_signin("foo.test", datastore_pk, 'register.app', ['names', 'register', 'prices', 'zonefiles', 'blockchain', 'node_read', 'update']) + if 'error' in res: + print json.dumps(res, indent=4, sort_keys=True) + error = True + return + + ses = res['token'] + + # for funsies, get the price of .test + res = testlib.blockstack_REST_call('GET', '/v1/prices/namespaces/test', ses ) + if 'error' in res or res['http_status'] != 200: + res['test'] = 'Failed to get price of .test' + print json.dumps(res) + return False + + test_price = res['response']['satoshis'] + print '\n\n.test costed {} satoshis\n\n'.format(test_price) + + # get the price for bar.test + res = testlib.blockstack_REST_call('GET', '/v1/prices/names/bar.test', ses ) + if 'error' in res or res['http_status'] != 200: + res['test'] = 'Failed to get price of bar.test' + print json.dumps(res) + return False + + bar_price = res['response']['total_estimated_cost']['satoshis'] + print "\n\nbar.test will cost {} satoshis\n\n".format(bar_price) + + # register the name bar.test. autogenerate the rest + res = testlib.blockstack_REST_call('POST', '/v1/names', ses, data={'name': 'bar.test'} ) + if 'error' in res: + res['test'] = 'Failed to register user' + print json.dumps(res) + error = True + return False + + print res + tx_hash = res['response']['transaction_hash'] + + # wait for preorder to get confirmed... + for i in xrange(0, 6): + testlib.next_block( **kw ) + + res = testlib.verify_in_queue(ses, 'bar.test', 'preorder', tx_hash ) + if not res: + return False + + # wait for the preorder to get confirmed + for i in xrange(0, 6): + testlib.next_block( **kw ) + + # wait for register to go through + print 'Wait for register to be submitted' + time.sleep(10) + + # wait for the register to get confirmed + for i in xrange(0, 6): + testlib.next_block( **kw ) + + res = testlib.verify_in_queue(ses, 'bar.test', 'register', None ) + if not res: + return False + + for i in xrange(0, 6): + testlib.next_block( **kw ) + + print 'Wait for update to be submitted' + time.sleep(10) + + # wait for update to get confirmed + for i in xrange(0, 6): + testlib.next_block( **kw ) + + res = testlib.verify_in_queue(ses, 'bar.test', 'update', None ) + if not res: + return False + + for i in xrange(0, 12): + testlib.next_block( **kw ) + + print 'Wait for update to be confirmed' + time.sleep(10) + + res = testlib.blockstack_REST_call("GET", "/v1/names/bar.test", ses) + if 'error' in res or res['http_status'] != 200: + res['test'] = 'Failed to get name bar.test' + print json.dumps(res) + return False + + zonefile_hash = res['response']['zonefile_hash'] + + # do we have the history for the name? + res = testlib.blockstack_REST_call("GET", "/v1/names/bar.test/history", ses ) + if 'error' in res or res['http_status'] != 200: + res['test'] = "Failed to get name history for bar.test" + print json.dumps(res) + return False + + # valid history? + hist = res['response'] + if len(hist.keys()) != 3: + res['test'] = 'Failed to get update history' + res['history'] = hist + print json.dumps(res, indent=4, sort_keys=True) + return False + + # get the zonefile + res = testlib.blockstack_REST_call("GET", "/v1/names/bar.test/zonefile/{}".format(zonefile_hash), ses ) + if 'error' in res or res['http_status'] != 200: + res['test'] = 'Failed to get name zonefile' + print json.dumps(res) + return False + + # now, let's set a new zonefile + zf_str = "$ORIGIN bar.test\n$TTL 3600\nmy_vote TXT \"045a501e341fbf1b403ce3a6e66836a3a40a\"" + res = testlib.blockstack_REST_call("PUT", "/v1/names/bar.test/zonefile/", ses, + ses, data={'zonefile' : zf_str}) + + # wait for update to get confirmed + for i in xrange(0, 6): + testlib.next_block( **kw ) + + res = testlib.verify_in_queue(ses, 'bar.test', 'update', None ) + if not res: + return False + + for i in xrange(0, 6): + testlib.next_block( **kw ) + + res = testlib.blockstack_REST_call("GET", "/v1/names/bar.test/zonefile/", ses ) + if 'error' in res or res['http_status'] != 200: + res['test'] = 'Failed to get name zonefile' + print json.dumps(res) + return False + + if res['response']['zonefile'] != zf_str: + print "Zonefile wasn't updated." + print "Expected: {}".format(zf_str) + print "Received: {}".format(res['response']['zonefile']) + +def check( state_engine ): + + global wallet_keys, error, index_file_data, resource_data + + config_path = os.environ.get("BLOCKSTACK_CLIENT_CONFIG") + assert config_path + + if error: + print "Key operation failed." + return False + + # not revealed, but ready + ns = state_engine.get_namespace_reveal( "test" ) + if ns is not None: + print "namespace not ready" + return False + + ns = state_engine.get_namespace( "test" ) + if ns is None: + print "no namespace" + return False + + if ns['namespace_id'] != 'test': + print "wrong namespace" + return False + + names = ['foo.test', 'bar.test'] + wallet_keys_list = [wallet_keys, wallet_keys] + test_proxy = testlib.TestAPIProxy() + + for i in xrange(0, len(names)): + name = names[i] + wallet_payer = 5 + wallet_owner = 3 + wallet_data_pubkey = 4 + + # not preordered + preorder = state_engine.get_name_preorder( name, pybitcoin.make_pay_to_address_script(wallets[wallet_payer].addr), wallets[wallet_owner].addr ) + if preorder is not None: + print "still have preorder" + return False + + # registered + name_rec = state_engine.get_name( name ) + if name_rec is None: + print "name does not exist" + return False + + # owned + if name_rec['address'] != wallets[wallet_owner].addr or name_rec['sender'] != pybitcoin.make_pay_to_address_script(wallets[wallet_owner].addr): + print "name {} has wrong owner".format(name) + return False + + return True diff --git a/integration_tests/blockstack_integration_tests/scenarios/rpc_register_cli_queries.py b/integration_tests/blockstack_integration_tests/scenarios/rpc_register_cli_queries.py index b09bd4b19..fd7326017 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/rpc_register_cli_queries.py +++ b/integration_tests/blockstack_integration_tests/scenarios/rpc_register_cli_queries.py @@ -294,14 +294,10 @@ def check( state_engine ): return False # zonefile info - if zonefile_info is None or type(zonefile_info) != dict: + if zonefile_info is None or type(zonefile_info) != str: print "invalid zonefile\n%s\n" % zonefile_info return False - if not zonefile_info.has_key('zonefile'): - print "missing zonefile\n%s\n" % zonefile_info - return False - # name query if type(all_names_info) == dict and 'error' in all_names_info: print "error in all_names: %s" % all_names_info @@ -334,8 +330,8 @@ def check( state_engine ): print "missing '%s'\n%s" % (k, json.dumps(lookup_info, indent=4, sort_keys=True)) return False - if lookup_info['zonefile'] != zonefile_info['zonefile']: - print "unequal zonefiles:\n%s\n%s" % (json.dumps(lookup_info['zonefile'], indent=4, sort_keys=True), json.dumps(zonefile_info['zonefile'], indent=4, sort_keys=True)) + if lookup_info['zonefile'] != zonefile_info: + print "unequal zonefiles:\n%s\n%s" % (json.dumps(lookup_info['zonefile'], indent=4, sort_keys=True), json.dumps(zonefile_info, indent=4, sort_keys=True)) return False # update history (2 items) @@ -344,8 +340,9 @@ def check( state_engine ): return False # zonefile history (expect 2 items) - if len(zonefile_history) != 2 or zonefile_history[1] != zonefile_info['zonefile']: + if len(zonefile_history) != 2 or zonefile_history[1] != zonefile_info: print "invalid zonefile history\n%s" % json.dumps(zonefile_history, indent=4, sort_keys=True) + print "zonefile current:\n%s" % json.dumps(zonefile_info, indent=4, sort_keys=True) return False # names info diff --git a/integration_tests/blockstack_integration_tests/scenarios/testlib.py b/integration_tests/blockstack_integration_tests/scenarios/testlib.py index 8f92fc374..49ee44090 100644 --- a/integration_tests/blockstack_integration_tests/scenarios/testlib.py +++ b/integration_tests/blockstack_integration_tests/scenarios/testlib.py @@ -301,7 +301,10 @@ def blockstack_name_register( name, privatekey, register_addr, wallet=None, subs owner_privkey_info = find_wallet(register_addr).privkey register_addr = virtualchain.address_reencode(register_addr) - resp = blockstack_client.do_register( name, privatekey, owner_privkey_info, test_proxy, test_proxy, config_path=config_path, proxy=test_proxy, safety_checks=safety_checks ) + kwargs = {} + if not safety_checks: + kwargs = {'tx_fee' : 1} # regtest shouldn't care about the tx_fee + resp = blockstack_client.do_register( name, privatekey, owner_privkey_info, test_proxy, test_proxy, config_path=config_path, proxy=test_proxy, safety_checks=safety_checks, **kwargs ) api_call_history.append( APICallRecord( "register", name, resp ) ) return resp @@ -331,7 +334,7 @@ def blockstack_name_transfer( name, address, keepdata, privatekey, user_public_k if subsidy_key is not None: payment_key = subsidy_key - resp = blockstack_client.do_transfer( name, address, keepdata, privatekey, payment_key, test_proxy, test_proxy, consensus_hash=consensus_hash, config_path=config_path, proxy=test_proxy, safety_checks=safety_checks) + resp = blockstack_client.do_transfer( name, address, keepdata, privatekey, payment_key, test_proxy, test_proxy, consensus_hash=consensus_hash, config_path=config_path, proxy=test_proxy, safety_checks=safety_checks ) api_call_history.append( APICallRecord( "transfer", name, resp ) ) return resp @@ -1729,8 +1732,7 @@ def blockstack_cli_app_put_resource( blockchain_id, app_domain, res_path, res_fi def blockstack_cli_app_signin( blockchain_id, app_privkey, app_domain, api_methods, device_ids=None, public_keys=None, config_path=None ): """ - sign in and get a token. - sign up if we need to. + sign in and get a token """ if config_path is None: test_proxy = make_proxy(config_path=config_path) @@ -1744,21 +1746,16 @@ def blockstack_cli_app_signin( blockchain_id, app_privkey, app_domain, api_metho args.api_methods = ','.join(api_methods) args.privkey = app_privkey - res = cli_app_signin( args, config_path=config_path ) - if 'error' in res: - if '`app_signup`' in res['error']: - # need to sign up first - res = cli_app_signup(args, config_path=config_path) - if 'error' in res: - return res - - else: - return res + if device_ids is None and public_keys is None: + device_ids = [blockstack_client.config.get_local_device_id()] + public_keys = [keylib.ECPrivateKey(app_privkey).public_key().to_hex()] - # now sign in - res = cli_app_signin(args, config_path=config_path) + device_ids = ','.join(device_ids) + public_keys = ','.join(public_keys) + args.device_ids = device_ids + args.public_keys = public_keys - return res + return cli_app_signin( args, config_path=config_path ) def blockstack_cli_create_datastore(blockchain_id, datastore_privkey, drivers, ses, device_ids=None, config_path=None): @@ -2229,8 +2226,11 @@ def blockstack_REST_call( method, route, session, api_pass=None, app_fqu=None, a headers = {} if session: headers['authorization'] = 'bearer {}'.format(session) + app_domain = jsontokens.decode_token(session)["payload"]["app_domain"] + headers['origin'] = "http://{}".format(app_domain) elif api_pass: headers['authorization'] = 'bearer {}'.format(api_pass) + headers['origin'] = 'http://localhost:3000' assert not (data and raw_data), "Multiple data given" @@ -3057,11 +3057,11 @@ def put_test_data( relpath, data, **kw ): def migrate_profile( name, proxy=None, wallet_keys=None, zonefile_has_data_key=True, config_path=None ): """ - Migrate a user's profile from the legacy format to the token-file/zonefile format. + Migrate a user's profile from the legacy format to the profile/zonefile format. Broadcast an update transaction with the zonefile hash. - Replicate the zonefile and token file. + Replicate the zonefile and profile. - Return {'status': True, 'zonefile': ..., 'profile': ..., 'token_file': ..., 'transaction_hash': ...} on success, if the profile was migrated + Return {'status': True, 'zonefile': ..., 'profile': ..., 'transaction_hash': ...} on success, if the profile was migrated Return {'status': True} on success, if the profile is already migrated Return {'error': ...} on error """ @@ -3078,7 +3078,6 @@ def migrate_profile( name, proxy=None, wallet_keys=None, zonefile_has_data_key=T user_profile = None user_zonefile = None user_zonefile_txt = None - new_token_file = None res = blockstack_cli_lookup(name, config_path=config_path) if 'error' in res: @@ -3118,29 +3117,31 @@ def migrate_profile( name, proxy=None, wallet_keys=None, zonefile_has_data_key=T # not JSON user_zonefile = blockstack_zones.parse_zone_file(user_zonefile_txt) + # add public key, if absent + if blockstack_client.user.user_zonefile_data_pubkey(user_zonefile) is None and zonefile_has_data_key: + log.debug("Adding zone file public key") + user_zonefile = blockstack_client.user.user_zonefile_set_data_pubkey(user_zonefile, keylib.ECPrivateKey(wallet_keys['data_privkey']).public_key().to_hex()) + payment_privkey_info = blockstack_client.get_payment_privkey_info( wallet_keys=wallet_keys, config_path=proxy.conf['path'] ) owner_privkey_info = blockstack_client.get_owner_privkey_info( wallet_keys=wallet_keys, config_path=proxy.conf['path'] ) data_privkey_info = blockstack_client.get_data_privkey_info( user_zonefile, wallet_keys=wallet_keys, config_path=proxy.conf['path'] ) - - ''' + assert data_privkey_info is not None assert 'error' not in data_privkey_info, str(data_privkey_info) assert virtualchain.is_singlesig(data_privkey_info) - ''' - res = blockstack_client.actions.migrate_profile_to_token_file(name, user_profile, owner_privkey_info, config_path=config_path) - if 'error' in res: - return {'error': 'Failed to create new token file'} - - new_token_file = res['token_file'] user_zonefile_hash = blockstack_client.hash_zonefile( user_zonefile ) - # replicate the token file - signing_key = blockstack_client.keys.get_signing_privkey(owner_privkey_info) - rc = blockstack_client.token_file.token_file_put(name, new_token_file, signing_key, config_path=config_path) + # replicate the profile + # TODO: this is onename-specific + + rc = blockstack_client.profile.put_profile(name, user_profile, blockchain_id=name, + user_data_privkey=data_privkey_info, user_zonefile=user_zonefile, + proxy=proxy, wallet_keys=wallet_keys ) + if 'error' in rc: - log.error("Failed to put token file: {}".format(rc['error'])) - return {'error': 'Failed to move legacy profile to token file and zonefile'} + log.error("Failed to put profile: {}".format(rc['error'])) + return {'error': 'Failed to move legacy profile to profile zonefile'} # do the update res = blockstack_client.do_update( name, user_zonefile_hash, owner_privkey_info, payment_privkey_info, proxy, proxy, config_path=proxy.config_path, proxy=proxy ) @@ -3160,8 +3161,7 @@ def migrate_profile( name, proxy=None, wallet_keys=None, zonefile_has_data_key=T 'transaction_hash': res['transaction_hash'], 'zonefile': user_zonefile, 'zonefile_txt': user_zonefile_txt, - 'profile': user_profile, - 'token_file': new_token_file, + 'profile': user_profile } return result diff --git a/integration_tests/setup.py b/integration_tests/setup.py index 29cfffa9b..ab363c568 100755 --- a/integration_tests/setup.py +++ b/integration_tests/setup.py @@ -21,6 +21,7 @@ setup( 'bin/blockstack-test-scenario', 'bin/blockstack-test-check-serialization', 'bin/blockstack-test-all', + 'bin/blockstack-test-all-junit', 'bin/blockstack-netlog-server', ], download_url='https://github.com/blockstack/blockstack-integration-tests/archive/master.zip', diff --git a/release_notes/changelog-0.14.3.md b/release_notes/changelog-0.14.3.md index e07ce64fe..b78bb6e66 100644 --- a/release_notes/changelog-0.14.3.md +++ b/release_notes/changelog-0.14.3.md @@ -45,12 +45,3 @@ Selected Bugfixes and Fixes * Issue #469 : Blockstack Core used to die in error cases when it should be able to fail more gracefully. This release fixes several such cases. - -* Removed hash-based fresh-profile validation from the storage gateway interface -in the indexer. This interface had been deprecated for some time, and new -clients have been relying on timestamp-based fresh-profile validation for some -time now. - -* Signing profiles and data is now done with a name's token file signing key, -not the catch-all data private key. You can still verify data signed with the -old data key, however. diff --git a/setup.py b/setup.py index f6619e1ac..e2bec7744 100755 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( zip_safe=False, include_package_data=True, install_requires=[ - 'virtualchain>=0.14.2', + 'virtualchain>=0.14.3', 'keychain>=0.14.2.0', 'protocoin>=0.2', 'blockstack-profiles>=0.14.1', @@ -52,11 +52,10 @@ setup( 'simplejson>=3.8.2', 'jsonschema>=2.5.1', 'scrypt>=0.8.0', - 'jsontokens>=0.0.4', 'pyparsing>=2.2.0', # not required, but causes problems if not installed properly 'basicrpc>=0.0.2', # DHT storage driver 'boto>=2.38.0', # S3 storage driver - 'dropbox>=8.0.0', # Dropbox driver + 'dropbox>=7.1.1', # Dropbox driver 'pydrive>=1.3.1', # Google Drive driver 'onedrivesdk>=1.1.8', # Microsoft OneDrive driver ],