Started a new repository for blockchain operations (separate from coinrpc):

-- Support for registering, transferring, and updating names
-- Tools for registering names in bulk (given a csv file)
-- Support for running as a service/daemon

blockstack_cli/blockstack_registrar/blockdata/activate.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import json
+import requests
+
+from time import sleep
+from coinrpc.coinrpc import namecoind_blocks, namecoind_firstupdate
+
+blocks = namecoind_blocks()
+
+from pymongo import Connection
+con = Connection()
+db = con['namecoin']
+queue = db.queue
+
+import os 
+
+LOAD_BALANCER = os.environ['LOAD_BALANCER']
+
+#-----------------------------------
+def check_name_firstupdate_errors(key):
+
+    reply = queue.find_one({'key':key})
+
+    #all activated entries should have final tx_id
+    try:
+        if(reply['activated'] is True):
+        
+            try: 
+                temp = json.loads(reply['tx_id'])
+                print temp['code']
+                print key + " had error"
+                reply['activated'] = False
+                queue.save(reply)
+            except:
+                pass
+    except Exception as e:
+        print key + " not in DB"
+
+#-----------------------------------
+def do_name_firstupdate():
+
+    print "Checking for new activations"
+    #print '---'
+
+    #first check for errors of last run and set activated = False
+    #for entry in queue.find():
+    #    check_name_firstupdate_errors(entry['key'])
+    
+    for entry in queue.find():
+
+        #entry is registered; but not activated
+        if entry.get('activated') is not None and entry.get('activated') == False:
+            
+            #print "Processing: " + entry['key'] 
+
+            #compare the current block with 'wait_till_block'
+            current_blocks = blocks['blocks']
+
+            if current_blocks > entry['wait_till_block'] and entry['backend_server'] == int(LOAD_BALANCER):
+                #lets activate the entry
+                print "Activating: " + entry['key']
+                print '----'
+
+                #check if 'value' is a json or not
+                try:
+                    update_value = json.loads(entry['value'])
+                    update_value = json.dumps(update_value)     #no error while parsing; dump into json again
+                except:
+                    update_value = entry['value']    #error: treat it as a string
+
+                print "Activating entry: '%s' to point to '%s'" % (entry['key'], update_value)
+            
+                output = namecoind_firstupdate(entry['key'],entry['rand'],update_value,entry['longhex'])
+
+                #output = namecoind_firstupdate(entry['key'], entry['rand'], update_value, entry['longhex'])
+                print "Transaction ID ", output
+
+                if 'message' in output and output['message'] == "this name is already active":
+                    entry['activated'] = True
+                elif 'code' in output:
+                    entry['activated'] = False
+                    print "Not activated. Try again."
+                else:
+                    entry['activated'] = True
+
+                entry['tx_id'] = output
+                queue.save(entry)
+
+                #sleep(1)
+            else:
+                pass
+
+#-----------------------------------
+if __name__ == '__main__':
+
+    do_name_firstupdate()

blockstack_cli/blockstack_registrar/blockdata/register.py

@@ -0,0 +1,355 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+SLEEP_INTERVAL = 1
+
+from time import sleep
+import csv
+import requests
+import json
+
+VALUE_MAX_LIMIT = 512
+
+from coinrpc.coinrpc import namecoind_blocks, namecoind_name_new, check_registration
+from coinrpc.coinrpc import namecoind_name_update, namecoind_name_show
+
+from pymongo import Connection
+conn = Connection()
+db = conn['namecoin']
+queue = db.queue
+codes = db.codes
+remoteusers = db.remoteusers
+
+from pymongo import MongoClient
+import os 
+
+LOAD_BALANCER = os.environ['LOAD_BALANCER']
+
+MONGODB_URI = os.environ['MONGODB_URI']
+HEROKU_APP = os.environ['HEROKU_APP'] 
+remote_client = MongoClient(MONGODB_URI)
+users = remote_client[HEROKU_APP].user
+
+#-----------------------------------
+def utf8len(s):
+
+	if type(s) == unicode:
+		return len(s)
+	else:
+		return len(s.encode('utf-8'))
+
+#-----------------------------------
+def save_name_new_info(info,key,value):
+	reply = {}
+  
+	try:
+		
+		reply['longhex'] = info[0]
+		reply['rand'] = info[1]
+		reply['key'] = key
+		reply['value'] = value
+		reply['backend_server'] = int(LOAD_BALANCER)
+
+		#get current block...
+		blocks = namecoind_blocks()
+
+		reply['current_block'] = blocks['blocks']
+		reply['wait_till_block'] = blocks['blocks'] + 12
+		reply['activated'] = False
+		
+		#save this data to Mongodb...
+		queue.insert(reply)
+
+		reply['message'] = 'Your registration will be completed in roughly two hours'
+		del reply['_id']        #reply[_id] is causing a json encode error
+	
+	except Exception as e:
+		reply['message'] = "ERROR:" + str(e)
+	
+	return reply 
+
+#-----------------------------------
+def slice_profile(username, profile, old_keys=None):
+
+	keys = []
+	values = [] 
+
+	key = 'u/' + username.lower()
+	keys.append(key)
+
+	def max_size(username):
+		return VALUE_MAX_LIMIT - len('next: i-' + username + '000000')
+
+	#-----------------------------------
+	def splitter(remaining,username):
+
+		split = {} 
+
+		if utf8len(json.dumps(remaining)) < max_size(username):
+			return remaining, None 
+		else:
+			for key in remaining.keys(): 
+				split[key] = remaining[key]
+
+				if utf8len(json.dumps(split)) < max_size(username):
+					del remaining[key]
+				else:
+					del split[key]
+					break 
+			return split, remaining
+
+	#-----------------------------------
+	def get_key(key_counter):
+		return 'i/' + username.lower() + '-' + str(key_counter)
+
+	split, remaining = splitter(profile, username) 
+	values.append(split)
+
+	key_counter = 0
+	counter = 0 
+
+	while(remaining is not None):
+		
+		key_counter += 1
+		key = get_key(key_counter)
+
+		if old_keys is not None and key in old_keys:
+			pass
+		else:
+			while check_registration(key):
+				key_counter += 1 
+				key = get_key(key_counter)
+
+		split, remaining = splitter(remaining, username)
+		keys.append(key) 
+		values.append(split)
+
+		values[counter]['next'] = key
+		counter += 1
+
+	return keys, values 
+
+#-----------------------------------
+def register_name(key,value):
+
+	info = namecoind_name_new(key,json.dumps(value))
+
+	reply = save_name_new_info(info,key,json.dumps(value))
+	
+	print reply
+	print '---'
+	sleep(SLEEP_INTERVAL)
+
+#-----------------------------------
+def update_name(key,value):
+
+	reply = {}
+
+	info = namecoind_name_update(key,json.dumps(value))
+
+	reply['key'] = key
+	reply['value'] = value
+	reply['activated'] = True
+	reply['backend_server'] = int(LOAD_BALANCER) 
+
+	#save this data to Mongodb...
+	check = queue.find_one({'key':key})
+
+	if check is None:
+		queue.insert(reply)
+	else:
+		queue.save(reply)
+
+	print reply
+	print info
+	print '---'
+	sleep(SLEEP_INTERVAL)
+
+#----------------------------------
+def get_old_keys(username):
+
+	#----------------------------------
+	def get_next_key(key): 
+	
+		check_profile = namecoind_name_show(key)
+
+		try:
+			check_profile = check_profile['value']
+
+			if 'next' in check_profile:
+				return check_profile['next']
+		except:
+			pass 
+
+		return None 
+
+
+	old_keys = []
+	key1 = "u/" + username
+
+	old_keys.append(str(key1))
+	next_key = get_next_key(key1)
+
+	while(next_key is not None):
+		old_keys.append(str(next_key))
+		next_key = get_next_key(next_key)
+		
+	return old_keys
+
+#-----------------------------------
+def process_user(username,profile,accesscode=None):
+
+	old_keys = get_old_keys(username) 
+
+	keys, values = slice_profile(username,profile,old_keys)
+
+	index = 0
+	key1 = keys[index]
+	value1 = values[index]
+
+	print utf8len(json.dumps(value1))
+
+	if check_registration(key1):
+		
+		#if name is registered
+		check_profile = namecoind_name_show(key1)
+
+		try:
+			check_profile = check_profile['value']
+			
+			#if name is reserved, check code
+			if 'status' in check_profile and check_profile['status'] == 'reserved':
+			
+				print "name reserved: " + key1
+				code = codes.find_one({'username':key1})
+				if code['accesscode'] == accesscode:
+					print "code match"
+					update_name(key1,value1)
+			else:
+				#if registered but not reserved
+				print "name update: " + key1
+				update_name(key1,value1)
+
+		except Exception as e:
+			#if registered but not reserved
+			print "name update: " + key1
+			update_name(key1,value1)
+
+	else: 
+		#if not registered 
+		print "name new: " + key1 
+		register_name(key1,value1)
+
+	process_additional_keys(keys, values)
+
+#-----------------------------------
+def process_additional_keys(keys,values):
+
+	#register/update remaining keys
+	size = len(keys)
+	index = 1
+	while index < size: 
+		next_key = keys[index]
+		next_value = values[index]
+
+		if check_registration(next_key):
+			print "name update: " + next_key
+			print utf8len(json.dumps(next_value))
+			update_name(next_key,next_value)
+		else: 
+			print "name new: " + next_key
+			print utf8len(json.dumps(next_value))
+			register_name(next_key,next_value)
+			
+		index += 1
+
+#-----------------------------------
+def set_backend_server(DISTRIBUTE=True):
+
+	DEFAULT_SERVER = 2	
+	BACKEND_SERVER_FOR_RESERVER = 1
+	loadbalancer_counter = 0
+
+	for i in users.find():
+
+		if 'dispatched' in i and i['dispatched'] is False:
+
+			loadbalancer_counter += 1
+
+			if(loadbalancer_counter == 8):
+				loadbalancer_counter = 0
+
+			#hardcoded backend_server for reserved names
+			if 'backend_server' not in i:
+				selected_server = DEFAULT_SERVER
+
+				if 'accesscode' in i:
+					print "found reserved user, " + i['username'] + " using backend_server ", BACKEND_SERVER_FOR_RESERVER
+					selected_server = BACKEND_SERVER_FOR_RESERVER
+				else:
+					if DISTRIBUTE:
+						selected_server = loadbalancer_counter
+
+				i['backend_server'] = selected_server
+				users.save(i)
+
+				print "sending " + i['username'] + " to backend_server " + str(selected_server)
+				
+#-----------------------------------
+def check_new_registrations(IS_LIVE=False):
+
+	registered_counter = 0
+	unregistered_counter = 0
+
+	print '-' * 5
+	print "Checking for new users"
+	for user in users.find():
+
+		if 'dispatched' in user and user['dispatched'] is False:
+
+			if not IS_LIVE:
+				print user['username']
+				print user['email']
+				print '-' * 5
+			unregistered_counter += 1
+
+			accesscode = None
+			if 'accesscode' in user:
+				accesscode = user['accesscode']
+		
+			if ('backend_server' in user) and (user['backend_server'] == int(LOAD_BALANCER)):
+				if IS_LIVE:
+					try:
+						process_user(user['username'],json.loads(user['profile']),accesscode)
+						print user['backend_server']
+					except Exception as e:
+						print e
+						continue 
+				
+				username = 'u/' + user['username'].lower()
+				extended = 'i/' + user['username'].lower() + '-1'
+
+				local = queue.find_one({'key':username})
+				if local is not None:
+					print "in local DB"
+					if IS_LIVE:
+						user['dispatched'] = True
+						user['accepted'] = True
+						remoteusers.insert(user)
+						users.save(user)
+			
+				print '-' * 5
+		else:
+			registered_counter += 1
+
+
+	print "Registered users: ", registered_counter
+	print "Not registered users: ", unregistered_counter
+
+#-----------------------------------
+if __name__ == '__main__':
+
+	IS_LIVE = True
+	DISTRIBUTE = False
+	set_backend_server(DISTRIBUTE)
+	check_new_registrations(IS_LIVE)

blockstack_cli/blockstack_registrar/blockdata/transfer.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import json
+from pymongo import Connection
+from coinrpc.coinrpc import namecoind_transfer, namecoind_name_show, check_registration
+from onename_register import process_user
+from pymongo import MongoClient
+import os 
+
+LOAD_BALANCER = os.environ['LOAD_BALANCER']
+import ssl
+
+MONGODB_URI = os.environ['MONGODB_URI']
+HEROKU_APP = os.environ['HEROKU_APP'] 
+remote_client = MongoClient(MONGODB_URI)
+users = remote_client[HEROKU_APP].user
+
+#-----------------------------------
+def test_private_key(passphrase,nmc_address):
+
+	from coinkit.keypair import NamecoinKeypair
+
+	keypair = NamecoinKeypair.from_passphrase(passphrase)
+	
+	print keypair.wif_pk()
+	
+	generated_nmc_address = keypair.address()
+
+	if(generated_nmc_address == nmc_address):
+		print "found a match"
+		return True
+	else:
+		print "don't match"
+		return False
+
+#-----------------------------------
+def do_name_transfer(username,live=False):
+
+	try:
+		entry = users.find_one({'username':username})
+		nmc_address = entry['namecoin_address']
+		backend_server = entry['backend_server']
+	except:
+		print "no such user in DB"
+		return 
+
+	key = 'u/' + username
+
+	if check_registration(key):
+
+		value = namecoind_name_show(key)['value']
+
+		next_blob = None 
+
+		try:
+			next_blob = value['next']
+		except:
+			pass
+
+		if(live):
+			reply = namecoind_transfer(key,nmc_address)
+			if 'message' in reply:
+				print reply['message']
+			else:
+				print reply
+				entry['name_transferred'] = True
+				users.save(entry)
+		
+		print key, nmc_address
+		print backend_server
+
+		passphrase = ''
+		#test_private_key(passphrase,nmc_address)
+
+		if next_blob is not None: 
+			print next_blob, nmc_address
+			if(live):
+				print namecoind_transfer(next_blob,nmc_address)	
+				
+	else:	
+		print "activate the name first"
+
+#-----------------------------------
+if __name__ == '__main__':
+
+	live = True
+
+	username = "grapeape"
+
+	user = users.find_one({"username":username})
+
+	do_name_transfer(user['username'],live)
+
+	'''
+
+	MyWbJHsddjgqvzY22Z7Qiupwin3TiEoXkf
+
+	user_list = ['ryanshea','samsmith','leena','saqib','ali','darthvader','ibrahim','mohammed','asjad']
+
+	for user in users.find():
+
+		if 'accepted' in user and user['accepted'] is True:
+
+			if user['username'] in user_list:
+				pass
+			elif user['backend_server'] != int(LOAD_BALANCER):
+				pass
+			elif 'name_transferred' in user and user['name_transferred'] is True:
+				print "already transferred: " + user['username']
+			else:
+				try:
+					#print user
+					do_name_transfer(user['username'],live)
+				except ssl.SSLError:
+					pass 
+				except KeyError:
+					pass
+				print '-' * 5
+
+	'''

blockstack_cli/blockstack_registrar/blockdata/update.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import json
+from pymongo import MongoClient
+from onename_register import process_user, utf8len
+from coinrpc.coinrpc import check_registration
+
+import os 
+
+LOAD_BALANCER = os.environ['LOAD_BALANCER']
+
+MONGODB_URI = os.environ['MONGODB_URI']
+HEROKU_APP = os.environ['HEROKU_APP'] 
+remote_client = MongoClient(MONGODB_URI)
+users = remote_client[HEROKU_APP].user
+private_key = remote_client[HEROKU_APP].private_key
+
+from datetime import datetime
+
+#-----------------------------------
+def update_profile(username,profile):
+ 
+	#update_name does json.dumps internally
+	process_user(username,profile)
+
+#-----------------------------------
+def update_profile_from_DB(username):
+
+	entry = users.find_one({'username':username})
+
+	profile = json.loads(entry['profile'])
+
+	#print profile
+	update_profile(username,profile)
+
+#-----------------------------------
+def update_profile_from_file(username,file_name='tools/json_profile.json'):
+
+	json_data=open(file_name)
+
+	try:
+		profile = json.load(json_data)
+	except Exception as e:
+		print e
+		return 
+
+	update_profile(username,profile)
+
+#-----------------------------------
+def private_key_expired(time):
+	
+	now = datetime.now()
+	diff = now - time
+	day_diff = diff.days
+
+	if diff.days < -1:
+		return True
+	else:
+		return False
+
+#-----------------------------------
+def delete_expired_private_keys():
+	print "Private keys: "
+	for user in private_key.find():
+
+		print user['username']
+		if private_key_expired(user['created_at']):
+			private_key.remove(user)
+			
+	print '-' * 5
+	
+#-----------------------------------
+def update_profile_from_private_key(username):
+
+	user_private_key = private_key.find_one({"username":username})
+	user = users.find_one({"username":username})
+
+	if ('backend_server' in user) and (user['backend_server'] == int(LOAD_BALANCER)):
+		process_user(user['username'],json.loads(user['profile']))
+		user['profile_update_pending'] = False
+		users.save(user)
+		private_key.remove(user_private_key)
+
+#-----------------------------------
+def process_profile_updates():
+	
+	print "Profile update pending: "
+	for user in users.find(): 
+
+		if 'name_transferred' in user and user['name_transferred'] is True:
+			#print "name already transferred: " + user['username']
+			continue 
+
+		if 'profile_update_pending' in user and user['profile_update_pending']:
+
+			if not check_registration('u/' + user['username']):
+				print "Not registered yet: " + user['username']
+				continue
+			else:
+				print "Updating user: " + user['username']
+				print user['backend_server']
+				update_profile_from_private_key(user['username'])
+
+	print '-' * 5
+
+	#delete_expired_private_keys()
+
+#-----------------------------------
+if __name__ == '__main__':
+
+	#username = 'muneebali'
+	#user = users.find_one({"username":username})
+	#print user['backend_server'] 
+	#update_profile_from_DB(username)
+	#update_profile_from_file(username)
+	process_profile_updates()

blockstack_cli/blockstack_registrar/registration_loop.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from tools.onename_register import check_new_registrations
+from tools.onename_activate import do_name_firstupdate
+
+from time import sleep
+
+POLLING_INTERVAL = 10
+
+#-----------------------------------
+if __name__ == '__main__':
+
+	while(1):
+		check_new_registrations()
+		do_name_firstupdate()
+		print "sleeping ... "
+		sleep(POLLING_INTERVAL)

blockstack_cli/blockstack_registrar/tools/csv_reserve.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from time import sleep
+import csv
+import requests
+import json
+import os
+import binascii
+from onename_register import register_name
+from coinrpc.coinrpc import check_registration
+
+from pymongo import Connection
+con = Connection()
+db = con['namecoin']
+queue = db.queue
+codes = db.codes
+
+#-----------------------------------
+def get_random_hex(size=10):
+    #every byte of data is converted into the corresponding 2-digit hex representation
+    return binascii.b2a_hex(os.urandom(size))
+
+#-----------------------------------
+def format_key_value(key, name=None):
+
+    #need u/ for OneName usernames
+    key = 'u/' + key.lower()
+
+    value = {}
+
+    value['status'] = "reserved"
+
+    if name is not None and name != '' and name != ' ': 
+
+        value["message"] = "This OneName username is reserved for " + name.lstrip(' ') 
+        value["message"] += ". If this is you, please email reservations@onename.io to claim it for free."
+
+    else:
+
+        value["message"] = "This OneName username was parked to evade name squatting, but can be made available upon reasonable request"
+        value["message"] += " at no charge. If you are interested in this name, please email reservations@onename.io with your twitter"
+        value["message"] += " handle and why you would like this particular name."
+
+    return key, value 
+
+
+#-----------------------------------
+#codes are only assigned to 'reserved' names
+def assign_code(key):
+    
+    reply = queue.find_one({'key':key})
+
+    if reply is not None and reply['activated'] is False:     
+        print "Not activated: " + reply['key']
+    else:
+
+        print key
+        check_code = codes.find_one({'username':key})
+        print check_code
+        
+        if check_code is not None: 
+            print check_code['username'] + ',' + check_code['code']
+        else:
+            print "Creating code for: " + key    
+
+            code = get_random_hex()
+        
+        
+        
+        #new_code = {}
+        #new_code["username"] = reply["username"]
+        #new_code['code'] = code
+        #codes.save(new_code)
+        
+
+#-----------------------------------
+def main_loop(key, name=None):
+
+    key, value = format_key_value(key,name)
+
+    reply = queue.find_one({'key':key})
+
+    if check_registration(key) or reply is not None:
+        print "already registered: " + key
+        try:
+            assign_code(key)
+        except Exception as e:
+            print "couldn't assign code"
+            #print e
+    else:
+        #not in DB 
+        print "not registered: " + key
+        register_name(key,value)
+    
+#-----------------------------------
+if __name__ == '__main__':
+
+    with open('tools/data.csv') as csvfile:
+        spamreader = csv.reader(csvfile)
+        for row in spamreader:
+            main_loop(row[0], row[1])
+

blockstack_cli/blockstack_registrar/tools/tests.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import json
+import csv
+
+from coinrpc.coinrpc import check_registration, namecoind_name_show
+from tools.onename_register import process_user, utf8len
+
+from pymongo import MongoClient
+import os 
+
+MONGODB_URI = os.environ['MONGODB_URI']
+HEROKU_APP = os.environ['HEROKU_APP'] 
+remote_client = MongoClient(MONGODB_URI)
+users = remote_client[HEROKU_APP].user
+
+local_client = MongoClient()
+registered = local_client['onename'].registered
+queue = local_client['namecoin'].queue
+
+#-----------------------------------
+def check_linked_list(key):
+
+	if check_registration(key):
+
+		check_profile = namecoind_name_show(key)
+		
+		check_profile = check_profile['value']
+		
+		if 'next' in check_profile:
+			return check_linked_list(check_profile['next'])
+		else:
+			return True 
+	else:
+		return False 
+
+#-----------------------------------
+if __name__ == '__main__':
+
+	for user in users.find():
+
+		username = user['username']
+		key = 'u/' + username 
+
+		check = registered.find_one({"username":username})
+
+		if check is not None:
+			continue 
+
+		in_queue = queue.find_one({"username":username})
+
+		if in_queue is not None:
+			continue 
+
+		process_user(user['username'],json.loads(user['profile']))
+
+		'''
+		if check_linked_list(key):
+			print "Registered: " + username 
+			registered.insert(user)
+		else:
+			print "Not registered: " + username
+		'''
+#-----------------------------------