kylefang/stacks-puppet-node
1
0
Fork 0
mirror of https://github.com/alexgo-io/stacks-puppet-node.git synced 2026-04-24 11:55:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

integrating search as a blueprint flask, plus getting its imports to play nicely with the api module

Browse Source
This commit is contained in:
Aaron Blankstein
2017-04-11 12:13:51 -04:00
parent 928bbe1bc6
commit bccf08e2d2
13 changed files with 86 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
api/search/attributes_index.py
View File

@@ -27,9 +27,8 @@ import sys
import json
import requests
from proofchecker import profile_to_proofs
from proofchecker import contains_valid_proof_statement
from proofchecker.domain import get_proof_from_txt_record
from blockstack_proofs import profile_to_proofs, contains_valid_proof_statement
from blockstack_proofs.domain import get_proof_from_txt_record
from pybitcoin import is_b58check_address
@@ -39,7 +38,7 @@ from .db import twitter_index, facebook_index
from .db import github_index, domain_index
from .db import proofs_cache
from .config import SUPPORTED_PROOFS
from api.config import SEARCH_SUPPORTED_PROOFS
def flush_collection():
@@ -301,7 +300,7 @@ def validProofQuery(query):
except:
return False
if query_type in SUPPORTED_PROOFS:
if query_type in SEARCH_SUPPORTED_PROOFS:
return True
return False

6
api/search/basic_index.py
View File

@@ -32,7 +32,7 @@ from pymongo import MongoClient
from .utils import validUsername
from .utils import get_json, config_log, pretty_print
from .config import BLOCKCHAIN_DATA_FILE, PROFILE_DATA_FILE
from api.config import SEARCH_BLOCKCHAIN_DATA_FILE, SEARCH_PROFILE_DATA_FILE
from .db import namespace, profile_data
from .db import search_profiles
@@ -48,7 +48,7 @@ def fetch_profile_data_from_file():
""" takes profile data from file and saves in the profile_data DB
"""
profile_data_file = open(PROFILE_DATA_FILE, 'r')
profile_data_file = open(SEARCH_PROFILE_DATA_FILE, 'r')
profiles = profile_data_file.read()
profiles = json.loads(profiles)
@@ -80,7 +80,7 @@ def fetch_profile_data_from_file():
def fetch_namespace_from_file():
blockchain_file = open(BLOCKCHAIN_DATA_FILE, 'r')
blockchain_file = open(SEARCH_BLOCKCHAIN_DATA_FILE, 'r')
blockchain_state = blockchain_file.read()
blockchain_state = json.loads(blockchain_state)

30
api/search/config.py
View File

@@ -23,33 +23,3 @@ This file is part of Blockstack.
along with Blockstack. If not, see <http://www.gnu.org/licenses/>.
"""
import os
DEBUG = True
MEMCACHED_ENABLED = True
LUCENE_ENABLED = False
DEFAULT_PORT = 5000
DEFAULT_HOST = '127.0.0.1'
BULK_INSERT_LIMIT = 1000
DEFAULT_LIMIT = 50
MEMCACHED_TIMEOUT = 6 * 60 * 60
BLOCKCHAIN_DATA_FILENAME = "data/blockchain_data.json"
PROFILE_DATA_FILENAME = "data/profile_data.json"
current_dir = os.path.abspath(os.path.dirname(__file__))
parent_dir = os.path.abspath(current_dir + "/../")
BLOCKCHAIN_DATA_FILE = os.path.join(parent_dir, BLOCKCHAIN_DATA_FILENAME)
PROFILE_DATA_FILE = os.path.join(parent_dir, PROFILE_DATA_FILENAME)
SUPPORTED_PROOFS = ['twitter', 'facebook', 'github', 'domain']
try:
# to overrite things like MEMCACHED_ENABLED
from config_local import *
except:
pass

38
api/search/fetch_data.py
View File

@@ -26,17 +26,18 @@ This file is part of Blockstack.
import sys
import json
from api.config import SEARCH_BLOCKCHAIN_DATA_FILE as BLOCKCHAIN_DATA_FILE, \
SEARCH_PROFILE_DATA_FILE as PROFILE_DATA_FILE
from .utils import validUsername
from .utils import get_json, config_log
from .config import BLOCKCHAIN_DATA_FILE, PROFILE_DATA_FILE
from blockstack_client.proxy import get_all_names
from blockstack_client.profile import get_profile
import logging
log = config_log(__name__)
def fetch_namespace():
"""
Fetch all names in a namespace that should be indexed.
@@ -51,8 +52,14 @@ def fetch_namespace():
return
def print_status_bar(filled, total):
pct = float(filled) / total
bar = max((int(pct * 60) - 1), 0)
out = "\r[%s>%s] %.1f%%" % ( ("=" * bar), " " * (59 - bar), pct * 100)
sys.stderr.write(out)
sys.stderr.flush()
def fetch_profiles():
def fetch_profiles(max_to_fetch = None):
"""
Fetch profile data using Blockstack Core and save the data.
Data is saved in: data/profile_data.json
@@ -68,10 +75,15 @@ def fetch_profiles():
all_names = json.loads(file)
all_profiles = []
if max_to_fetch == None:
max_to_fetch = len(all_names)
counter = 0
for fqu in all_names:
for ix, fqu in enumerate(all_names):
if ix % 100 == 0:
print_status_bar(ix, max_to_fetch)
if ix >= max_to_fetch:
break
resp = {}
resp['fqu'] = fqu
@@ -79,13 +91,11 @@ def fetch_profiles():
try:
resp['profile'] = get_profile(fqu)[0]
all_profiles.append(resp)
except KeyboardInterrupt as e:
raise e
except:
pass
counter += 1
if counter % 100 == 0:
print counter
fout = open(PROFILE_DATA_FILE, 'w')
fout.write(json.dumps(all_profiles))
fout.close()
@@ -107,7 +117,11 @@ if __name__ == "__main__":
elif(option == '--fetch_profiles'):
# Step 2
fetch_profiles()
if len(sys.argv) > 2:
max_to_fetch = int(sys.argv[2])
fetch_profiles(max_to_fetch)
else:
fetch_profiles()
else:
print "Usage error"

45
api/search/server.py
View File

@@ -29,11 +29,11 @@ import threading
import pylibmc
from time import time
from flask import request, jsonify, Flask, make_response, render_template
from flask import request, jsonify, make_response, render_template, Blueprint
from flask_crossdomain import crossdomain
from .config import DEFAULT_HOST, DEFAULT_PORT, DEBUG, MEMCACHED_TIMEOUT
from .config import DEFAULT_LIMIT
from .config import MEMCACHED_ENABLED, LUCENE_ENABLED
from api.config import DEFAULT_HOST, DEFAULT_PORT, DEBUG, MEMCACHED_TIMEOUT, MEMCACHED_ENABLED
from api.config import SEARCH_DEFAULT_LIMIT as DEFAULT_LIMIT, SEARCH_LUCENE_ENABLED as LUCENE_ENABLED
from .substring_search import search_people_by_name, search_people_by_twitter
from .substring_search import search_people_by_username, search_people_by_bio
@@ -41,14 +41,11 @@ from .substring_search import fetch_profiles
from .attributes_index import search_proofs, validProofQuery
searcher = Blueprint('searcher', __name__, url_prefix='')
app = Flask(__name__)
mc = pylibmc.Client(["127.0.0.1:11211"], binary=True,
behaviors={'tcp_nodelay': True,
'connect_timeout': 100,
'no_block': True})
from api.resolver import get_mc_client
mc = get_mc_client()
class QueryThread(threading.Thread):
""" for performing multi-threaded search on three search sub-systems
@@ -114,7 +111,8 @@ def test_alphanumeric(query):
return True
@app.route('/search')
@searcher.route('/search', methods = ["GET", "POST"], strict_slashes = False)
@crossdomain(origin='*')
def search_by_name():
query = request.args.get('query')
@@ -224,28 +222,3 @@ def search_proofs_index(query):
mc.set(cache_key, results, int(time() + MEMCACHED_TIMEOUT))
return jsonify(results)
@app.route('/')
def index():
return render_template('index.html')
@app.errorhandler(500)
def internal_error(error):
reply = []
return json.dumps(reply)
@app.errorhandler(404)
def not_found(error):
resp = {}
resp['error'] = "Not found"
return make_response(jsonify(resp), 404)
if __name__ == '__main__':
app.run(host=DEFAULT_HOST, port=DEFAULT_PORT, debug=DEBUG)

10
api/search/substring_search.py
View File

@@ -32,14 +32,10 @@ import sys
import json
current_dir = os.path.abspath(os.path.dirname(__file__))
parent_dir = os.path.abspath(current_dir + "/../")
sys.path.insert(0, parent_dir)
from api.search.db import search_db, search_profiles
from api.search.db import search_cache
from search.db import search_db, search_profiles
from search.db import search_cache
from search.config import DEFAULT_LIMIT
from api.config import SEARCH_DEFAULT_LIMIT as DEFAULT_LIMIT
from .utils import get_json,pretty_print
def anyword_substring_search_inner(query_word, target_words):

2
api/search/utils.py
View File

@@ -28,7 +28,7 @@ import re
from json import JSONEncoder
import logging
from config import DEBUG
from api.config import DEBUG
def config_log(name):