small updates

blockstack_cli_0.14.1/blockstack_registrar/blockdata/register.py

@@ -157,6 +157,68 @@ def slice_profile(username, profile, old_keys=None):
 	key_counter = 0
 	counter = 0 
 
+	while(remaining is not None):
+	
+		key_counter += 1
+		key = get_key(key_counter)
+		
+		while(1):
+
+			if namecoind.check_registration(key):
+				key_counter += 1
+				key = get_key(key_counter)
+			else:
+				break
+
+		split, remaining = splitter(remaining, username)
+		keys.append(key) 
+		values.append(split)
+
+		values[counter]['next'] = key
+		counter += 1
+
+	return keys, values 
+
+#-----------------------------------
+def slice_profile_update(username, profile, old_keys=None):
+
+	keys = []
+	values = [] 
+
+	key = 'u/' + username.lower()
+	keys.append(key)
+
+	def max_size(username):
+		return VALUE_MAX_LIMIT - len('next: i-' + username + '000000')
+
+	#-----------------------------------
+	def splitter(remaining,username):
+
+		split = {} 
+
+		if utf8len(json.dumps(remaining)) < max_size(username):
+			return remaining, None 
+		else:
+			for key in remaining.keys(): 
+				split[key] = remaining[key]
+
+				if utf8len(json.dumps(split)) < max_size(username):
+					del remaining[key]
+				else:
+					del split[key]
+					break 
+			return split, remaining
+
+	#-----------------------------------
+	def get_key(key_counter):
+		return 'i/' + username.lower() + '-' + str(key_counter)
+
+	split, remaining = splitter(profile, username) 
+	values.append(split)
+
+	key_counter = 0
+	counter = 0 
+
 	while(remaining is not None):
 		
 		key_counter += 1
@@ -169,7 +231,7 @@ def slice_profile(username, profile, old_keys=None):
 		values[counter]['next'] = key
 		counter += 1
 
-	return keys, values 
+	return keys, values
 
 #----------------------------------
 def get_old_keys(username):
@@ -208,7 +270,12 @@ def process_user(username,profile,server=NAMECOIND_SERVER):
 
 	#old_keys = get_old_keys(username) 
 
-	keys, values = slice_profile(username,profile)
+	master_key = 'u/' + username 
+
+	if namecoind.check_registration(master_key):
+		keys, values = slice_profile_update(username,profile)
+	else:
+		keys, values = slice_profile(username,profile)
 
 	index = 0
 	key1 = keys[index]

blockstack_cli_0.14.1/blockstack_registrar/register_daemon.py

@@ -59,7 +59,7 @@ def process_profile(username,profile):
 #-----------------------------------
 def profile_on_blockchain(username,DB_profile):
 
-	sleep(1)
+	sleep(5)
 	try:
 		block_profile = namecoind.get_full_profile('u/' + username)
 	except:
@@ -89,14 +89,23 @@ def check_banned(username):
 #-----------------------------------
 def register_users(): 
 
+	counter = 0 
+
 	for new_user in registrations.find():
 
 		user_id = new_user['user_id']
 		user = users.find_one({"_id":user_id})
 
+		if user is None:
+			continue 
+
 		if check_banned(user['username']):
 			continue
 			
+		print "checking: " + user['username']
+
+		counter += 1
+
 		if 'dispatched' in new_user and new_user['dispatched'] is False: 
 	
 			if datetime.datetime.utcnow() - new_user['created_at'] > datetime.timedelta(minutes=15):
@@ -105,9 +114,10 @@ def register_users():
 				process_profile(user['username'],user['profile'])
 				new_user['dispatched'] = True 
 				registrations.save(new_user)
+				sleep(20)
 			else:
 				print "New user (within 15 mins): " + user['username']
-		
+
 		elif 'dispatched' in new_user and new_user['dispatched'] is True:
 		
 			try:
@@ -128,6 +138,8 @@ def register_users():
 			print "Random: " + user['username']
 			#registrations.remove(new_user)
 
+	print counter
+
 #-----------------------------------
 def check_users(): 
 
@@ -180,6 +192,8 @@ def check_transfer():
 def update_users(): 
 
 	for new_user in updates.find():
+
+		sleep(1)
 		
 		user_id = new_user['user_id']
 		user = users.find_one({"_id":user_id})
@@ -246,9 +260,12 @@ def cleanup_db():
 		user_id = new_user['user_id']
 		user = users.find_one({"_id":user_id})
 
+		if user is None:
+			continue
+
 		if check_banned(user['username']):
 			continue		
-
+		
 		try:
 			if profile_on_blockchain(user["username"],user["profile"]):
 				print "cleaning: " + user["username"]
@@ -262,7 +279,7 @@ def cleanup_db():
 #-----------------------------------
 if __name__ == '__main__':
 
-	check_transfer()
+	#check_transfer()
 	update_users()
 	register_users()
 	

blockstack_cli_0.14.1/blockstack_registrar/tools/csv_reserve.py

@@ -15,6 +15,15 @@ db = con['namecoin']
 queue = db.queue
 
 from ast import literal_eval
+import json 
+
+from config import MONGODB_URI
+
+#-----------------------------------
+from pymongo import MongoClient
+remote_client = MongoClient(MONGODB_URI)
+remote_db = remote_client.get_default_database()
+codes = remote_db.codes 
 
 #-----------------------------------
 def format_key_value(key, name=None):
@@ -53,7 +62,7 @@ def main_loop(key, name=None):
         profile = profile['value']
         if 'status' in profile and profile['status'] == 'reserved':
             print "already reserved: " + key
-            update_name(key,value)
+            #update_name(key,value)
         else:
             print "registered but not reserved: " + key
             #update_name(key,value)
@@ -64,10 +73,23 @@ def main_loop(key, name=None):
         #not in DB and not registered
         print "not registered: " + key
         register_name(key,value)
-        
+
+    print '-' * 5
+
+#-----------------------------------
+from base64 import b64encode
+def get_url(username, access_code):
+    return 'http://onename.io?a=' + b64encode(username + '-' + access_code)
+
+#-----------------------------------
+def get_random_hex(size=10):
+    #every byte of data is converted into the corresponding 2-digit hex representation
+    return binascii.b2a_hex(os.urandom(size))
+
 #-----------------------------------
 if __name__ == '__main__':
 
+    '''
     with open('tools/data.csv') as csvfile:
         spamreader = csv.reader(csvfile)
         for row in spamreader:
@@ -77,15 +99,18 @@ if __name__ == '__main__':
                 main_loop(row[0])
    
     '''
-    with open('tools/angel_list.txt') as f:
-        users = [list(literal_eval(line)) for line in f]
+    with open('tools/email_invites_dataset.txt') as f:
+        users = json.loads(f.read())
 
-        for user in users:
-            for i in user:
-                local = queue.find_one({'key':"u/" + i})
+        counter = 0
+        skip = 881
+        for i in users:
+            counter += 1
 
-                if local is not None:
-                    print "already in DB"
-                else:
-                    main_loop(i) 
-    '''
+            if counter < skip:
+                continue  
+            
+            #print i['twitter_handle'], i['full_name'], i['email']
+            print counter
+            main_loop(i['twitter_handle'],i['full_name']) 
+