Add operations for namespacing and data read/write, as well as

preliminary schema verification.
This commit is contained in:
Jude Nelson
2015-07-26 06:35:43 -04:00
parent 31512f0bc6
commit 5830d31670
5 changed files with 888 additions and 0 deletions

View File

@@ -0,0 +1,57 @@
from coinkit import embed_data_in_blockchain, BlockchainInfoClient, hex_hash160
from utilitybelt import is_hex
from binascii import hexlify, unhexlify
from ..b40 import b40_to_hex, bin_to_b40, is_b40
from ..config import *
from ..scripts import blockstore_script_to_hex, add_magic_bytes
def build( namespace_id, testset=False ):
"""
Record to mark the end of a namespace import in the blockchain.
Takes an base40-encoded namespace ID to mark the end.
Format:
0 2 3 4 23
|-----|--|--|-------------|
magic op len ns_id
"""
# sanity check
if not is_b40( namespace_id ):
raise Exception("Namespace ID '%s' is not base-40" % namespace_id)
if len(namespace_id) == 0 or len(namespace_id) > LENGTHS['blockchain_id_namespace_id']:
raise Exception("Invalid namespace ID '%s (expected length between 1 and %s)" % (namespace_id, LENGTHS['blockchain_id_namespace_id']))
readable_script = "NAMESPACE_BEGIN %i %s" % (len(namespace_id), namespace_id)
hex_script = blockstore_script_to_hex(readable_script)
packaged_script = add_magic_bytes(hex_script, testset=testset)
return packaged_script
def broadcast( namespace_id, private_key, blockchain_client=BlockchainInfoClient(), testset=False ):
nulldata = build( namespace_id, testset=testset )
# response = {'success': True }
response = embed_data_in_blockchain( nulldata, private_key, blockchain_client, format='hex')
response.update({'data': nulldata})
return response
def parse( bin_payload ):
"""
NOTE: the first three bytes will be missing
"""
namespace_id_len = ord( bin_payload[0:LENGTHS['blockchain_id_namespace_id_len']] )
namespace_id = bin_payload[ LENGTHS['blockchain_id_namespace_id_len']:LENGTHS['blockchain_id_namespace_id_len'] + namespace_id_len ]
return {
'opcode': 'NAMESPACE_BEGIN',
'namespace_id': namespace_id
}

View File

@@ -0,0 +1,156 @@
from coinkit import embed_data_in_blockchain, BlockchainInfoClient, hex_hash160
from utilitybelt import is_hex
from binascii import hexlify, unhexlify
from ..b40 import b40_to_hex, bin_to_b40, is_b40
from ..config import *
from ..scripts import blockstore_script_to_hex, add_magic_bytes, get_script_pubkey
def namespace_decay_to_float( namespace_decay_fixedpoint ):
"""
Convert the raw namespace decay rate (a fixedpoint decimal)
to a floating-point number.
Upper 8 bits: integer
Lower 24 bits: decimal
"""
ipart = namespace_decay_fixedpoint >> 24
fpart = namespace_decay_fixedpoint & 0xff000000
return ipart + (float(fpart) / 2**24)
def namespace_decay_to_fixpoint( namespace_decay_float ):
"""
Convert a floating-point number to a namespace decay rate.
Return None if invalid
"""
if namespace_decay_float < 0:
return None
ipart = int(namespace_decay_float)
if( ipart > 255 ):
return None
fpart = (namespace_decay_float - ipart)
fixpoint = (ipart << 24) | int(fpart * (1 << 24))
return fixpoint
# name lifetime (blocks): 4 bytes (0xffffffff for infinite)
# baseline price for one-letter names (satoshis): 8 bytes
# price decay rate per letter (fixed-point decimal: 2**8 integer part, 2**24 decimal part): 4 bytes
# namespace ID: up to 19 bytes
def build( namespace_id, script_pubkey, lifetime, satoshi_cost, price_decay_rate, testset=False ):
"""
Record to mark the beginning of a namespace import in the blockchain.
Takes an ASCII-encoded namespace ID and parameters and registers the beginning of a namespace definition.
NOTE: "namespace_id" must not start with ., but can contain anything else we want
We put the hash of the namespace ID instead of the namespace ID itself to avoid races with squatters (akin to pre-ordering)
Format:
0 2 3 7 15 19 39
|-----|---|-----|----------|------|----------------------------|
magic op life cost decay hash(ns_id,script_pubkey)
"""
# sanity check
if not is_b40( namespace_id ):
raise Exception("Namespace identifier '%s' is not base-40" % namespace_id)
if lifetime < 0 or lifetime > (2**32 - 1):
raise Exception("Lifetime '%s' out of range (expected unsigned 32-bit integer)" % lifetime)
if satoshi_cost < 0 or satoshi_cost > (2**64 - 1):
raise Exception("Cost '%s' out of range (expected unsigned 64-bit integer)" % satoshi_cost)
if price_decay_rate < 0 or price_decay_rate > (2**32 - 1):
raise Exception("Decay rate '%s' out of range (expected unsigned 32-bit integer)" % price_decay_rate)
if len(namespace_id) == 0 or len(namespace_id) > LENGTHS['blockchain_id_namespace_id']:
raise Exception("Invalid namespace ID length '%s (expected length between 1 and %s)" % (namespace_id, LENGTHS['blockchain_id_namespace_id']))
namespace_id_hash = hash_name(namespace_id, script_pubkey)
readable_script = "NAMESPACE_DEFINE %i %i %i %s" % (lifetime, satoshi_cost, price_decay_rate, namespace_id_hash)
hex_script = blockstore_script_to_hex(readable_script)
packaged_script = add_magic_bytes(hex_script, testset=testset)
return packaged_script
def broadcast( namespace_id, lifetime, satoshi_cost, price_decay_rate, private_key, blockchain_client=BlockchainInfoClient(), testset=False ):
"""
Propagate a namespace.
Arguments:
namespace_id human-readable (i.e. base-40) name of the namespace
lifetime: the number of blocks for which names will be valid (pass a negative value for "infinite")
satoshi_cost: the base cost (i.e. cost of a 1-character name), in satoshis
price_decay_rate a positive float representing the rate at which names get cheaper. The formula is satoshi_cost / (price_decay_rate)^(name_length - 1).
private_key the Bitcoin address that created this namespace, and can populate it.
"""
script_pubkey = get_script_pubkey( private_key )
price_decay_rate_fixedpoint = namespace_decay_to_fixpoint( price_decay_rate )
if price_decay_rate_fixedpoint is None:
raise Exception("Invalid price decay rate '%s'" % price_decay_rate)
if lifetime < 0:
lifetime = NAMESPACE_LIFE_INFINITE
nulldata = build( namespace_id, script_pubkey, lifetime, satoshi_cost, price_decay_rate_fixedpoint, testset=testset )
# response = {'success': True }
response = embed_data_in_blockchain( nulldata, private_key, blockchain_client, format='hex')
response.update({'data': nulldata})
return response
def parse( bin_payload ):
"""
NOTE: the first three bytes will be missing
"""
off = 0
life = None
cost = None
decay = None
namespace_id_len = None
namespace_id = None
life = ord( bin_payload[off:off+LENGTHS['blockchain_id_namespace_life']] )
off += LENGTHS['blockchain_id_namespace_life']
cost = ord( bin_payload[off:off+LENGTHS['blockchain_id_namespace_cost']] )
off += LENGTHS['blockchain_id_namespace_cost']
decay_fixedpoint = ord( bin_payload[off:off+LENGTHS['blockchain_id_namespace_price_decay']] )
off += LENGTHS['blockchain_id_namespace_price_decay']
namespace_id_len = ord( bin_payload[off:off+LENGTHS['blockchain_id_namespace_id_len']] )
off += LENGTHS['blockchain_id_namespace_id_len']
namespace_id_hash = bin_payload[off:off+namespace_id_len]
return {
'opcode': 'NAMESPACE_DEFINE',
'lifetime': life,
'cost': cost,
'price_decay': namespace_decay_to_float( decay_fixedpoint ),
'namespace_id_hash': namespace_id_hash
}

View File

@@ -0,0 +1,71 @@
from coinkit import embed_data_in_blockchain, BlockchainInfoClient, hex_hash160, bin_hash160, BitcoinPrivateKey
from utilitybelt import is_hex
from binascii import hexlify, unhexlify
from ..b40 import b40_to_hex, bin_to_b40, is_b40
from ..config import *
from ..scripts import blockstore_script_to_hex, add_magic_bytes, get_script_pubkey
from ..hashing import hash256_trunc128
def build(name, data_hash=None, data=None, testset=False):
"""
Write a signed storage record: takes the name of the owner and the data or the data's hash (i.e. the key to the data).
Name must include the namespace ID, but not the protocol scheme.
In either case, we only want the name.ns_id hash and the data hash.
Format:
0 2 3 19 39
|-----|--|--------------------|--------------|
magic op hash128(name) hash160(data)
WARNING: the caller must verify that the data is well-formed.
"""
hex_name = hash256_trunc128(name)
name_len = len(hex_name)/2
if not data_hash:
if not data:
raise ValueError('A data hash or data string is required.')
data_hash = hex_hash160(data)
elif not (is_hex(data_hash) and len(data_hash) == 40):
raise ValueError('Data hash must be a 20 byte hex string.')
readable_script = 'DATA_PUT %s %s' % (hex_name, data_hash)
hex_script = blockstore_script_to_hex(readable_script)
packaged_script = add_magic_bytes(hex_script, testset=testset)
return packaged_script
def broadcast(name, data_hash, private_key, blockchain_client=BlockchainInfoClient(), testset=False):
"""
Put the DATA_PUT message into the blockchain.
This effectively signs the data's hash with the user's private key.
"""
nulldata = build(name, data_hash=data_hash, testset=testset)
response = embed_data_in_blockchain(nulldata, private_key, blockchain_client, format='hex')
response.update({'data': nulldata})
return response
def parse(bin_payload):
"""
Recover the hashed name and data from a blockchain record.
"""
name_hash_bin = bin_payload[:LENGTHS['name_hash']]
data_hash_bin = bin_payload[LENGTHS['name_hash']:]
name_hash = hexlify( name_hash_bin )
data_hash = hexlify( data_hash_bin )
return {
'opcode': 'DATA_PUT',
'name_hash': name_hash,
'data_hash': data_hash
}

View File

@@ -0,0 +1,72 @@
from coinkit import embed_data_in_blockchain, BlockchainInfoClient, hex_hash160
from utilitybelt import is_hex
from binascii import hexlify, unhexlify
from ..b40 import b40_to_hex, bin_to_b40
from ..config import *
from ..scripts import blockstore_script_to_hex, add_magic_bytes
def build(key, name, data=None, testset=False):
"""
Delete a chunk of signed data, owned by a particular name.
Record format:
0 2 3 19 39
|-----|--|-------------------|-----------------------|
magic op hash128(name) hash160(data)
"""
if name.startswith(NAME_SCHEME):
raise Exception("Invalid name %s: must not start with %s" % (name, NAME_SCHEME))
hex_name = hash256_trunc128(name)
if key is None:
if data is None:
raise ValueError('A data hash or data string is required.')
key = hex_hash160(data)
elif not (is_hex(key) and len(key) == 2*LENGTHS['data_hash']):
raise ValueError('Data hash must be a %s byte hex string.' % LENGTHS['data_hash'])
readable_script = 'DATA_RM %s %s' % (hex_name, key)
hex_script = blockstore_script_to_hex(readable_script)
packaged_script = add_magic_bytes(hex_script, testset=testset)
return packaged_script
def broadcast(key, name, private_key, data=None, blockchain_client=BlockchainInfoClient(), testset=False):
"""
Broadcast a 'delete data' message to the blockchain.
"""
if key is None and data is None:
raise ValueError("A key or the raw data string is required.")
nulldata = build( key, name, data=data, testset=testset)
response = embed_data_in_blockchain(nulldata, private_key, blockchain_client, format='hex')
response.update({'data': nulldata})
return response
def parse(bin_payload):
"""
Parse a binary 'delete data' message, starting from the 3rd byte.
"""
name_hash_bin = bin_payload[0:LENGTHS['name_hash']]
data_hash_bin = bin_payload[LENGHTS['name_hash']:]
name_hash = hexlify( name_hash )
data_hash = hexlify( data_hash )
return {
'opcode': 'DATA_RM',
'name_hash': name_hash,
'data_hash': data_hash
}

532
blockstore/lib/schemas.py Normal file
View File

@@ -0,0 +1,532 @@
#!/usr/bin/python
"""
Listing of data schemas, as well as methods for validating them.
"""
import types
import re
class SchemaField( object ):
def __init__(self, name):
self.name = name
def __repr__(self):
return self.name
def __eq__(self, value):
return self.name == value
class SchemaType( object ):
def __init__(self, *args):
self.types = args
def get_types(self):
return self.types
def valid( self, value ):
# children override this
return type(value) in self.get_types()
class BitcoinAddressType( SchemaType ):
def __init__(self):
super( BitcoinAddressType, self ).__init__( types.StringType, types.UnicodeType )
def valid( self, value ):
if type(value) != types.StringType and type(value) != types.UnicodeType:
print "mismatch on type (%s)" % type(value)
return False
strvalue = str(value)
if re.match(r"[a-zA-Z1-9]{27,35}$", strvalue) is None:
print "mismatch on regex (%s)" % strvalue
return False
return True
class PGPFingerprintType( SchemaType ):
def __init__(self):
super( PGPFingerprintType, self ).__init__( types.StringType, types.UnicodeType )
def valid( self, value ):
if type(value) != types.StringType and type(value) != types.UnicodeType:
return False
strvalue = str(value)
if re.match(r"[a-fA-F0-9 :]$", strvalue) is None:
return False
return True
class EmailType( SchemaType ):
# RFC-822 compliant, as long as there aren't any comments in the address.
# taken from http://chrisbailey.blogs.ilrt.org/2013/08/19/validating-email-addresses-in-python/
email_regex_str = r"^(?=^.{1,256}$)(?=.{1,64}@)(?:[^\x00-\x20\x22\x28\x29\x2c\x2e\x3a-\x3c\x3e\x40\x5b-\x5d\x7f-\xff]+|\x22(?:[^\x0d\x22\x5c\x80-\xff]|\x5c[\x00-\x7f])*\x22)(?:\x2e(?:[^\x00-\x20\x22\x28\x29\x2c\x2e\x3a-\x3c\x3e\x40\x5b-\x5d\x7f-\xff]+|\x22(?:[^\x0d\x22\x5c\x80-\xff]|\x5c[\x00-\x7f])*\x22))*\x40(?:[^\x00-\x20\x22\x28\x29\x2c\x2e\x3a-\x3c\x3e\x40\x5b-\x5d\x7f-\xff]+|[\x5b](?:[^\x0d\x5b-\x5d\x80-\xff]|\x5c[\x00-\x7f])*[\x5d])(?:\x2e(?:[^\x00-\x20\x22\x28\x29\x2c\x2e\x3a-\x3c\x3e\x40\x5b-\x5d\x7f-\xff]+|[\x5b](?:[^\x0d\x5b-\x5d\x80-\xff]|\x5c[\x00-\x7f])*[\x5d]))*$"
def __init__(self):
super( EmailType, self ).__init__( types.StringType, types.UnicodeType )
def valid( self, value ):
if type(value) != types.StringType and type(value) != types.UnicodeType:
return False
strvalue = str(value)
if re.match( self.email_regex_str, strvalue ) is None:
return False
return True
class OptionalField( SchemaField ):
pass
STRING = SchemaType( types.StringType, types.UnicodeType )
URL = STRING
PGP_FINGERPRINT = PGPFingerprintType()
BITCOIN_ADDRESS = BitcoinAddressType()
EMAIL = EmailType()
OPTIONAL = OptionalField
def match( schema, obj, verbose=False ):
"""
Recursively verify that the given object has the given schema.
Return True if so
Return False if not
"""
def debug( msg ):
if verbose:
print msg
# object is literal?
if type(obj) != types.DictType:
if obj != schema:
debug( "Literal '%s' does not match '%s'" % (literal, schema) )
return False
else:
return True
# all object keys must be acceptable to the schema
for literal in obj.keys():
if literal not in schema.keys():
debug("Unmatched object literal '%s'" % literal)
return False
# all non-optional schema keys must be present in the object
for field in schema.keys():
optional = False
literal = field
if isinstance( field, OptionalField ):
literal = str(field)
optional = True
if literal not in obj.keys():
if not optional:
debug( "Literal '%s' not found in object" % literal )
return False
else:
continue
sub_object = obj[literal]
sub_schema = schema[field]
is_match = False
if type(sub_schema) != types.DictType:
if isinstance( sub_schema, SchemaType ):
# check custom validation
is_match = sub_schema.valid( sub_object )
if is_match is False:
print "schema not valid: %s" % (sub_object)
else:
# check type
is_match = (type(sub_schema) == type(sub_object))
if is_match is False:
print "%s != %s" % (type(sub_schema), type(sub_object))
else:
# recursively verify match
is_match = match( sub_schema, sub_object )
if not is_match:
debug( "Mismatch on key '%s'" % literal)
return False
# all checks pass
return True
# tests
"""
if __name__ == "__main__":
PASSCARD_SCHEMA_V2 = {
"name": {
"formatted": STRING
},
"bio": STRING,
"location": {
"formatted": STRING
},
"website": URL,
"bitcoin": {
"address": BITCOIN_ADDRESS
},
"avatar": {
"url": URL,
},
"cover": {
"url": URL,
},
OPTIONAL("pgp"): {
"url": URL,
"fingerprint": PGP_FINGERPRINT,
},
OPTIONAL("email"): EMAIL,
"twitter": {
"username": STRING,
"proof": {
"url": URL
}
},
"facebook": {
"username": STRING,
"proof": {
"url": URL
}
},
"github": {
"username": STRING,
"proof": {
"url": URL
}
},
"v": STRING
}
testcases = [
# valid
{
"website": "http://www.cs.princeton.edu/~jcnelson",
"location": {
"formatted": "Princeton University"
},
"github": {
"proof": {
"url": "https://gist.github.com/jcnelson/70c02f80f8d4b0b8fc15"
},
"username": "jcnelson"
},
"bio": "PhD student",
"bitcoin": {
"address": "17zf596xPvV8Z8ThbWHZHYQZEURSwebsKE"
},
"twitter": {
"proof": {
"url": "https://twitter.com/judecnelson/status/507374756291555328"
},
"username": "judecnelson"
},
"email": "judecn@gmail.com",
"avatar": {
"url": "https://s3.amazonaws.com/kd4/judecn"
},
"name": {
"formatted": "Jude Nelson"
},
"facebook": {
"proof": {
"url": "https://facebook.com/sunspider/posts/674912239245011"
},
"username": "sunspider"
},
"cover": {
"url": "https://s3.amazonaws.com/97p/gQZ.jpg"
},
"v": "0.2"
},
# valid (missing email)
{
"website": "http://www.cs.princeton.edu/~jcnelson",
"location": {
"formatted": "Princeton University"
},
"github": {
"proof": {
"url": "https://gist.github.com/jcnelson/70c02f80f8d4b0b8fc15"
},
"username": "jcnelson"
},
"bio": "PhD student",
"bitcoin": {
"address": "17zf596xPvV8Z8ThbWHZHYQZEURSwebsKE"
},
"twitter": {
"proof": {
"url": "https://twitter.com/judecnelson/status/507374756291555328"
},
"username": "judecnelson"
},
"avatar": {
"url": "https://s3.amazonaws.com/kd4/judecn"
},
"name": {
"formatted": "Jude Nelson"
},
"facebook": {
"proof": {
"url": "https://facebook.com/sunspider/posts/674912239245011"
},
"username": "sunspider"
},
"cover": {
"url": "https://s3.amazonaws.com/97p/gQZ.jpg"
},
"v": "0.2"
},
# invalid: missing name
{
"website": "http://www.cs.princeton.edu/~jcnelson",
"location": {
"formatted": "Princeton University"
},
"github": {
"proof": {
"url": "https://gist.github.com/jcnelson/70c02f80f8d4b0b8fc15"
},
"username": "jcnelson"
},
"bio": "PhD student",
"bitcoin": {
"address": "17zf596xPvV8Z8ThbWHZHYQZEURSwebsKE"
},
"twitter": {
"proof": {
"url": "https://twitter.com/judecnelson/status/507374756291555328"
},
"username": "judecnelson"
},
"avatar": {
"url": "https://s3.amazonaws.com/kd4/judecn"
},
"facebook": {
"proof": {
"url": "https://facebook.com/sunspider/posts/674912239245011"
},
"username": "sunspider"
},
"cover": {
"url": "https://s3.amazonaws.com/97p/gQZ.jpg"
},
"v": "0.2"
},
# invalid: invalid website type
{
"website": 1,
"location": {
"formatted": "Princeton University"
},
"github": {
"proof": {
"url": "https://gist.github.com/jcnelson/70c02f80f8d4b0b8fc15"
},
"username": "jcnelson"
},
"bio": "PhD student",
"bitcoin": {
"address": "17zf596xPvV8Z8ThbWHZHYQZEURSwebsKE"
},
"twitter": {
"proof": {
"url": "https://twitter.com/judecnelson/status/507374756291555328"
},
"username": "judecnelson"
},
"avatar": {
"url": "https://s3.amazonaws.com/kd4/judecn"
},
"name": {
"formatted": "Jude Nelson"
},
"facebook": {
"proof": {
"url": "https://facebook.com/sunspider/posts/674912239245011"
},
"username": "sunspider"
},
"cover": {
"url": "https://s3.amazonaws.com/97p/gQZ.jpg"
},
"v": "0.2"
},
# invalid: extra field
{
"extra_field": "foo",
"website": "http://www.cs.princeton.edu/~jcnelson",
"location": {
"formatted": "Princeton University"
},
"github": {
"proof": {
"url": "https://gist.github.com/jcnelson/70c02f80f8d4b0b8fc15"
},
"username": "jcnelson"
},
"bio": "PhD student",
"bitcoin": {
"address": "17zf596xPvV8Z8ThbWHZHYQZEURSwebsKE"
},
"twitter": {
"proof": {
"url": "https://twitter.com/judecnelson/status/507374756291555328"
},
"username": "judecnelson"
},
"avatar": {
"url": "https://s3.amazonaws.com/kd4/judecn"
},
"name": {
"formatted": "Jude Nelson"
},
"facebook": {
"proof": {
"url": "https://facebook.com/sunspider/posts/674912239245011"
},
"username": "sunspider"
},
"cover": {
"url": "https://s3.amazonaws.com/97p/gQZ.jpg"
},
"v": "0.2"
},
# invalid: extra field in cover
{
"website": "http://www.cs.princeton.edu/~jcnelson",
"location": {
"formatted": "Princeton University"
},
"github": {
"proof": {
"url": "https://gist.github.com/jcnelson/70c02f80f8d4b0b8fc15"
},
"username": "jcnelson"
},
"bio": "PhD student",
"bitcoin": {
"address": "17zf596xPvV8Z8ThbWHZHYQZEURSwebsKE"
},
"twitter": {
"proof": {
"url": "https://twitter.com/judecnelson/status/507374756291555328"
},
"username": "judecnelson"
},
"avatar": {
"url": "https://s3.amazonaws.com/kd4/judecn"
},
"name": {
"formatted": "Jude Nelson"
},
"facebook": {
"proof": {
"url": "https://facebook.com/sunspider/posts/674912239245011"
},
"username": "sunspider"
},
"cover": {
"url": "https://s3.amazonaws.com/97p/gQZ.jpg",
"extra_field": "foo",
},
"v": "0.2"
},
# invalid: missing subobject "url" in "github"
{
"website": "http://www.cs.princeton.edu/~jcnelson",
"location": {
"formatted": "Princeton University"
},
"github": {
"username": "jcnelson"
},
"bio": "PhD student",
"bitcoin": {
"address": "17zf596xPvV8Z8ThbWHZHYQZEURSwebsKE"
},
"twitter": {
"proof": {
"url": "https://twitter.com/judecnelson/status/507374756291555328"
},
"username": "judecnelson"
},
"avatar": {
"url": "https://s3.amazonaws.com/kd4/judecn"
},
"name": {
"formatted": "Jude Nelson"
},
"facebook": {
"proof": {
"url": "https://facebook.com/sunspider/posts/674912239245011"
},
"username": "sunspider"
},
"cover": {
"url": "https://s3.amazonaws.com/97p/gQZ.jpg"
},
"v": "0.2"
},
]
for i in xrange(0, len(testcases)):
testcase = testcases[i]
rc = match( PASSCARD_SCHEMA_V2, testcase, verbose=True )
print "test case %s: %s" % (i, rc)
"""