#!/usr/bin/python2 # -*- coding: utf-8 -*- """ Blockstack-integration-tests ~~~~~ copyright: (c) 2014-2015 by Halfmoon Labs, Inc. copyright: (c) 2016-2017 by Blockstack.org This file is part of Blockstack-integration-tests. Blockstack-integration-tests is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. Blockstack-integration-tests is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Blockstack-integration-tests. If not, see . """ import os import sys # debugging? DEBUG = False if '--debug' in sys.argv: os.environ['BLOCKSTACK_DEBUG'] = '1' DEBUG = True import blockstack_client import virtualchain import argparse import json import jsonschema import importlib def hash_data( d ): return virtualchain.lib.hashing.hex_hash160( d ) def print_debug(s): if DEBUG: print s def test_put_immutable_handler( driver_mod, dataset ): """ Run the put_immutable_handler() method on the given dataset Return True on success Raise on error """ key = dataset['data_hash'] data = dataset['data'] txid = dataset['txid'] kw = dataset['kwargs'] res = driver_mod.put_immutable_handler(key, data, txid, **kw) assert res return True def test_put_mutable_handler( driver_mod, dataset ): """ Run the put_mutable_handler() method on the given dataset Return True on success Raise on error """ data_id = dataset['data_id'] data = dataset['data'] kw = dataset['kwargs'] res = driver_mod.put_mutable_handler(data_id, data, **kw) assert res return True def test_get_immutable_handler( driver_mod, dataset ): """ Run the get_immutable_handler() method on the given dataset Return True on success Raise on error """ key = dataset['data_hash'] kw = dataset['kwargs'] res = driver_mod.get_immutable_handler(key, **kw) assert res == dataset['data'] return True def test_get_mutable_handler( driver_mod, dataset ): """ Run the get_mutable_handler() method on the given dataset Return True on success Raise on error """ data_id = dataset['data_id'] kw = dataset['kwargs'] url = driver_mod.make_mutable_url(data_id, **kw) assert url assert driver_mod.handles_url(url) res = driver_mod.get_mutable_handler(url, **kw) assert res == dataset['data'] return True def test_delete_immutable_handler( driver_mod, dataset ): """ Run the delete_immutable_handler() method on the given dataset Return True on success Raise on error """ key = dataset['data_hash'] txid = dataset['txid'] tombstone = dataset['tombstone'] kw = dataset['kwargs'] res = driver_mod.delete_immutable_handler(key, txid, tombstone, **kw) assert res return True def test_delete_mutable_handler( driver_mod, dataset ): """ Run the delete_mutable_handler() method on the given dataset Return True on success Raise on error """ data_id = dataset['data_id'] tombstone = dataset['tombstone'] kw = dataset['kwargs'] res = driver_mod.delete_mutable_handler(data_id, tombstone, **kw) assert res return True def make_dataset( data_id, data, **kw ): """ Create a dataset structure with enough information to test all of the storage operations. """ data_hash = hash_data(data) # fake txid, derived from the data txid = virtualchain.lib.hashing.bin_double_sha256(data).encode('hex') # fake tombstone, using a known private key privk = '91b6c0d341fa8dcaa2340dd781ced822aeeb196fc9b37ddc8afd471324e137ff01' tombstone = blockstack_client.storage.make_data_tombstone('test-{}'.format(data_id)) tombstone = blockstack_client.storage.sign_data_tombstone(tombstone, privk) dataset = { 'data_id': data_id, 'data': data, 'data_hash': data_hash, 'txid': txid, 'tombstone': tombstone, 'kwargs': kw } return dataset def load_driver(driver_mod_name): """ Load a driver module. The string will be interpreted as a python module string. Returns the module on success Raises on error """ try: mod = importlib.import_module(driver_mod_name) return mod except: print >> sys.stderr, "\nModule not found: '{}'\n".format(driver_mod_name) raise def load_datasets(dataset_path): """ Load a set of datasets to test with Returns the parsed datasets on success Raises on error """ with open(dataset_path, 'r') as f: dataset_txt = f.read() try: datasets = json.loads(dataset_txt) except: print >> sys.stderr, "\nFailed to parse JSON from '{}'\n".format(dataset_path) raise datasets_schema = { 'type': 'array', 'items': { 'type': 'object', 'properties': { 'data_id': { 'type': 'string', }, 'data': { 'anyOf': [ { 'type': 'string', }, { 'type': 'boolean', }, ], }, 'kwargs': { 'type': 'object', 'patternProperties': { '.+': { 'anyOf': [ { 'type': 'string' }, { 'type': 'boolean', }, { 'type': 'integer', }, { 'type': 'number', }, ], }, }, }, }, 'required': [ 'data_id', 'data', 'kwargs' ], }, } try: jsonschema.validate(datasets, datasets_schema) except jsonschema.ValidationError: print >> sys.stderr, "\nFailed to validate JSON datasets from '{}'\n".format(dataset_path) raise return datasets def test_gets(driver_mod, datasets): """ Run all 'get' tests """ print_debug("Testing get_immutable_handler...") for dataset in datasets: print_debug(" Get immutable '{}' ({})".format(dataset['data_id'], dataset['data_hash'])) test_get_immutable_handler(driver_mod, dataset) print_debug("Testing get_mutable_handler...") for dataset in datasets: print_debug(" Get mutable '{}' ({})".format(dataset['data_id'], dataset['data_hash'])) test_get_mutable_handler(driver_mod, dataset) return True def test_puts(driver_mod, datasets): """ Run all 'put' tests """ print_debug("Testing put_immutable_handler...") for dataset in datasets: print_debug(" Put immutable '{}' ({})".format(dataset['data_id'], dataset['data_hash'])) test_put_immutable_handler(driver_mod, dataset) print_debug("Testing put_mutable_handler...") for dataset in datasets: print_debug(" Put mutable '{}' ({})".format(dataset['data_id'], dataset['data_hash'])) test_put_mutable_handler(driver_mod, dataset) return True def test_deletes(driver_mod, datasets): """ Run all 'delete' tests """ print_debug("Testing delete_immutale_handler...") for dataset in datasets: print_debug(" Delete immutable '{}' ({})".format(dataset['data_id'], dataset['data_hash'])) test_delete_immutable_handler(driver_mod, dataset) print_debug("Testing delete_mutable_handler...") for dataset in datasets: print_debug(" Delete mutable '{}' ({})".format(dataset['data_id'], dataset['data_hash'])) test_delete_mutable_handler(driver_mod, dataset) return True DEFAULT_DATASETS = [ { 'data_id': 'my_first_datum', 'data': 'hello world', 'kwargs': {}, }, { 'data_id': '/my/second/datum', 'data': 'hello world 2', 'kwargs': { 'fqu': 'foo.id' }, }, { 'data_id': 'user\"_profile', 'data': '{"name": {"formatted": "judecn"}, "v": "2"}', 'kwargs': { 'profile': True, 'fqu': 'judecn.id' }, }, { 'data_id': 'binary data', 'data': '\x01\x02\x03\x04\x05\x11\x12\x13\x14\x15\xfa\xfb\xfc\xfd\xfe\xff', 'kwargs': { 'fqu': 'foo.app', }, }, { 'data_id': 'zonefile', 'data': '$ORIGIN judecn.id\n$TTL 3600\npubkey TXT \"pubkey:data:04cabba0b5b9a871dbaa11c044066e281c5feb57243c7d2a452f06a0d708613a46ced59f9f806e601b3353931d1e4a98d7040127f31016311050bedc0d4f1f62ff\"\n_file URI 10 1 \"file:///home/jude/.blockstack/storage-disk/mutable/judecn.id\"\n_https._tcp URI 10 1 \"https://blockstack.s3.amazonaws.com/judecn.id\"\n_http._tcp URI 10 1 \"http://node.blockstack.org:6264/RPC2#judecn.id\"\n_dht._udp URI 10 1 \"dht+udp://fc4d9c1481a6349fe99f0e3dd7261d67b23dadc5\"\n', 'kwargs': { 'fqu': 'judecn.id', 'zonefile': True }, }, { 'data_id': 'empty string', 'data': '', 'kwargs': {} }, ] def main(argv): """ Entry point """ parser = argparse.ArgumentParser(description='Test a storage driver') parser.add_argument("--config", dest='config_file', type=str, help="Path to config file to use", required=False) parser.add_argument("--index", dest="index", action='store_true', help='Instantiate an index, if we have not done so already', required=False) parser.add_argument("--force-index", dest="force_index", action='store_true', help='Force (re-)creating the data index', required=False) parser.add_argument("--datasets", type=str, help="Ppath to a JSON file with [{'data_id': ..., 'data': ...}] objects to test", required=False) parser.add_argument("--debug", dest='debug', action='store_true', help='Activate verbose output from Blockstack libraries', required=False) parser.add_argument("driver_module", type=str, help="Blockstack storage driver module to run (e.g. 'disk', 'blockstack_client.backend.drivers.s3', etc.)") parser.add_argument("operation", type=str, help="Family of operations to test ('get', 'put', 'delete', 'all')") parser.add_argument("--fqu", type=str, help="blockchain ID to access user-specific index manifest URI", required=False) args, _ = parser.parse_known_args(argv[1:]) datasets = [make_dataset(ds['data_id'], ds['data'], **ds['kwargs']) for ds in DEFAULT_DATASETS] operations = 'all' config_path = blockstack_client.constants.CONFIG_PATH force_index = False index = False driver_name = args.driver_module.rsplit('.', 1)[-1] driver_mod = load_driver(args.driver_module) if args.operation is not None: operations = args.operation if args.datasets is not None: datasets = load_datasets(args.datasets) datasets = [make_dataset(ds['data_id'], ds['data'], **ds['kwargs']) for ds in datasets] if args.config_file is not None: config_path = args.config_file if args.index: index = True if args.force_index: force_index = True conf = blockstack_client.get_config(config_path) assert conf res = driver_mod.storage_init(conf, index=index, force_index=force_index, fqu=args.fqu) assert res if operations == 'get': test_gets(driver_mod, datasets) elif operations == 'put': test_puts(driver_mod, datasets) elif operations == 'delete': test_deletes(driver_mod, datasets) elif operations == 'all': test_puts(driver_mod, datasets) test_gets(driver_mod, datasets) test_deletes(driver_mod, datasets) else: print >> sys.stderr, 'Unrecognized operation suite "{}"'.format(operations) parser.print_help() sys.exit(1) if index: # print out the driver's index manifest url print "" print " index manifest URI: {}".format(blockstack_client.backend.drivers.common.index_settings_get_index_manifest_url(driver_name, config_path)) print "" if __name__ == "__main__": main(sys.argv)