From a9c9de15c395b1e49244c6063c9c1cb204e450da Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 10 Jan 2018 15:05:03 +0300 Subject: Add configuration variables for external services * Add configuration variables for GitHub and ORCID which will be used by the system to allow users to login. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 57f97a81..d2e689a4 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,6 +251,10 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) +GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') +ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) -- cgit v1.2.3 From b4e4ca152256e1f1f06359d584034554db06402e Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 10 Jan 2018 15:07:11 +0300 Subject: Add template elements for OAuth login * Add html elements that will be used to prompt users to login with either GitHub or ORCID. --- wqflask/wqflask/templates/new_security/login_user.html | 18 ++++++++++++++++-- wqflask/wqflask/user_manager.py | 9 ++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index b9f49a61..15f0a27e 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -18,8 +18,22 @@ Create a new account - -
+
+

Login with external services

+ + {% if external_login: %} +
+ {% if external_login["github"]: %} + Login with Github + {% endif %} + + {% if external_login["orcid"]: %} + Login with ORCID + {% endif %} +
+ +
+ {% endif %}

Already have an account? Sign in here.

diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index f7fcd2d0..25833464 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -506,7 +506,14 @@ class LoginUser(object): params = request.form if request.form else request.args logger.debug("in login params are:", params) if not params: - return render_template("new_security/login_user.html") + from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL + external_login = None + if GITHUB_AUTH_URL or ORCID_AUTH_URL: + external_login={ + "github": GITHUB_AUTH_URL, + "orcid": ORCID_AUTH_URL + } + return render_template("new_security/login_user.html", external_login=external_login) else: try: user = model.User.query.filter_by(email_address=params['email_address']).one() -- cgit v1.2.3 From 76299c30c265919fd4025e11017b687c2b63fd82 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:01:11 +0300 Subject: Add client_id and client_secret configurations * Provide the OAuth2 client_id and client_secret values in configuration variables. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index d2e689a4..310b4cea 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,7 +251,11 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) +GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -- cgit v1.2.3 From 3e70cab812e29e504f714c782c71bc4b79793686 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:05:45 +0300 Subject: Add elasticsearch_tools module * Collect variables and functions for using the elasticsearch system in a separate module. --- wqflask/utility/elasticsearch_tools.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 wqflask/utility/elasticsearch_tools.py diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py new file mode 100644 index 00000000..bc7bb240 --- /dev/null +++ b/wqflask/utility/elasticsearch_tools.py @@ -0,0 +1,22 @@ +from elasticsearch import Elasticsearch, TransportError +from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + +es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT +}]) + +def get_user_by_unique_column(column_name, column_value): + user_details = None + try: + response = es.search( + index = "users" + , doc_type = "local" + , body = { + "query": { "match": { column_name: column_value } } + }) + if len(response["hits"]["hits"]) > 0: + user_details = response["hits"]["hits"][0]["_source"] + except TransportError as te: + pass + return user_details -- cgit v1.2.3 From 7004c0ee5e86bfb7ebe491356ca3210d2dc2b67b Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:07:13 +0300 Subject: Add functions to help handle github login * Add functions to help with the github OAuth2 login process --- wqflask/wqflask/user_manager.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 25833464..9012c842 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -494,6 +494,37 @@ def login(): lu = LoginUser() return lu.standard_login() +@app.route("/n/login/github_oauth2", methods=('GET', 'POST')) +def github_oauth2(): + from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET + from utility.elasticsearch_tools import get_user_by_unique_column + import requests + code = request.args.get("code") + data = { + "client_id": GITHUB_CLIENT_ID, + "client_secret": GITHUB_CLIENT_SECRET, + "code": code + } + result = requests.post("https://github.com/login/oauth/access_token", json=data) + result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} + + github_user = get_github_user_details(result_dict["access_token"]) + user_details = get_user_by_unique_column("github_id", github_user["id"]) + if user_details == None: + user_details = { + "user_id": str(uuid4()) + , "name": github_user["name"] + , "github_id": github_user["id"] + , "user_url": github_user["html_url"] + , "login_type": "github" + } + url = "/n/login?type=github" + return redirect(url) + +def get_github_user_details(access_token): + from utility.tools import GITHUB_API_URL + result = requests.get(GITHUB_API_URL, params={"access_token":access_token}) + return result.json() class LoginUser(object): remember_time = 60 * 60 * 24 * 30 # One month in seconds -- cgit v1.2.3 From 63a5c8a42ad02e9126bb207465ff5eca98f6515d Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:36:19 +0300 Subject: Add elasticsearch module to the path * Add some code to set up the path for the python-elasticsearch module. --- bin/genenetwork2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index a7edb1c2..ccd9b1d9 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -61,6 +61,11 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides +if [ -z $ELASTICSEARCH_PROFILE ]; then + echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m"; +else + PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +fi if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) @@ -72,7 +77,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages + export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH" export R_LIBS_SITE=$GN2_PROFILE/site-library export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0 export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript -- cgit v1.2.3 From 7282e0b47c1c6d12fc4c06a080af07c8e67ca75c Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:38:04 +0300 Subject: Add save_user() function * On successful login via OAuth2, save the details of the user in elasticsearch store, to avoid hitting the external provider for the basic details. --- wqflask/utility/elasticsearch_tools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index bc7bb240..74db489b 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -20,3 +20,10 @@ def get_user_by_unique_column(column_name, column_value): except TransportError as te: pass return user_details + +def save_user(user, user_id, index="users", doc_type="local"): + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT + }]) + es.create(index, doc_type, body=user, id=user_id) -- cgit v1.2.3 From 4ae020160a9ae9c399fe338c9415897b26425201 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:41:41 +0300 Subject: Add more configuration variables. * Add configurations for elasticsearch and github. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 310b4cea..df066e67 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -254,11 +254,15 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +GITHUB_API_URL = get_setting('GITHUB_API_URL') ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') +ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) -- cgit v1.2.3 From e185fd3895473e86f2c9fdf174a36b1d325a8c36 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 12:00:04 +0300 Subject: Delay after save for indexing * Elasticsearch need a short delay after adding document for it to index the document for subsequent access. --- wqflask/utility/elasticsearch_tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 74db489b..c2c999ea 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -22,8 +22,10 @@ def get_user_by_unique_column(column_name, column_value): return user_details def save_user(user, user_id, index="users", doc_type="local"): + from time import sleep es = Elasticsearch([{ "host": ELASTICSEARCH_HOST , "port": ELASTICSEARCH_PORT }]) es.create(index, doc_type, body=user, id=user_id) + sleep(1) # Delay 1 second to allow indexing -- cgit v1.2.3 From 2e7335182e55b22e9c61eef111e00f119760d365 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 12:03:10 +0300 Subject: Add code to enable OAuth2 login * Add some code to handle the login if the user chooses to login via GitHub or ORCID. --- wqflask/wqflask/user_manager.py | 58 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 9012c842..daeb7bc5 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -54,6 +54,9 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list +import requests +from utility.elasticsearch_tools import get_user_by_unique_column, save_user + THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -492,13 +495,16 @@ class DecodeUser(object): @app.route("/n/login", methods=('GET', 'POST')) def login(): lu = LoginUser() - return lu.standard_login() + login_type = request.args.get("type") + if login_type: + uid = request.args.get("uid") + return lu.oauth2_login(login_type, uid) + else: + return lu.standard_login() @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) def github_oauth2(): from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET - from utility.elasticsearch_tools import get_user_by_unique_column - import requests code = request.args.get("code") data = { "client_id": GITHUB_CLIENT_ID, @@ -512,13 +518,15 @@ def github_oauth2(): user_details = get_user_by_unique_column("github_id", github_user["id"]) if user_details == None: user_details = { - "user_id": str(uuid4()) + "user_id": str(uuid.uuid4()) , "name": github_user["name"] , "github_id": github_user["id"] , "user_url": github_user["html_url"] , "login_type": "github" + , "organization": "" } - url = "/n/login?type=github" + save_user(user_details, user_details.get("user_id")) + url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) def get_github_user_details(access_token): @@ -532,6 +540,46 @@ class LoginUser(object): def __init__(self): self.remember_me = False + def oauth2_login(self, login_type, user_id): + """Login via an OAuth2 provider""" + user_details = get_user_by_unique_column("user_id", user_id) + if user_details: + user = model.User() + user.id = user_details["user_id"] + user.full_name = user_details["name"] + user.login_type = user_details["login_type"] + return self.actual_login_oauth2(user) + else: + flash("Error logging in via OAuth2") + return make_response(redirect(url_for('login'))) + + def actual_login_oauth2(self, user, assumed_by=None, import_collections=None): + """The meat of the logging in process""" + session_id_signed = self.successful_login_oauth2(user) + flash("Thank you for logging in {}.".format(user.full_name), "alert-success") + print("IMPORT1:", import_collections) + response = make_response(redirect(url_for('index_page', import_collections=import_collections))) + if self.remember_me: + max_age = self.remember_time + else: + max_age = None + response.set_cookie(UserSession.cookie_name, session_id_signed, max_age=max_age) + return response + + def successful_login_oauth2(self, user, assumed_by=None): + login_rec = model.Login(user) + login_rec.successful = True + login_rec.session_id = str(uuid.uuid4()) + login_rec.assumed_by = assumed_by + session_id_signature = actual_hmac_creation(login_rec.session_id) + session_id_signed = login_rec.session_id + ":" + session_id_signature + logger.debug("session_id_signed:", session_id_signed) + + session = dict(login_time = time.time(), + user_id = user.id, + user_login_type = user.login_type) + return session_id_signed + def standard_login(self): """Login through the normal form""" params = request.form if request.form else request.args -- cgit v1.2.3 From d30cbe103946d70b9c61bc48575f23ead1d4048c Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 13:00:51 +0300 Subject: Update configurations * Have the authorisation URLs build up from the client id and client secret values. --- wqflask/utility/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index df066e67..8a32c9b4 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -253,11 +253,11 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') -GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET GITHUB_API_URL = get_setting('GITHUB_API_URL') ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') -ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') +ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -- cgit v1.2.3 From 03127502a982af308e6cd134e218214f6a156cd8 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 13:02:11 +0300 Subject: Add OAuth2 login code for ORCID * Add code to handle the login via ORCID --- wqflask/wqflask/user_manager.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index daeb7bc5..c3f190c3 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -529,6 +529,40 @@ def github_oauth2(): url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) +@app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) +def orcid_oauth2(): + from uuid import uuid4 + from utility.tools import ORCID_CLIENT_ID, ORCID_CLIENT_SECRET, ORCID_TOKEN_URL, ORCID_AUTH_URL + code = request.args.get("code") + error = request.args.get("error") + url = "/n/login" + if code: + data = { + "client_id": ORCID_CLIENT_ID + , "client_secret": ORCID_CLIENT_SECRET + , "grant_type": "authorization_code" + , "code": code + } + result = requests.post(ORCID_TOKEN_URL, data=data) + result_dict = json.loads(result.text.encode("utf-8")) + print("The dict: ", result_dict); + user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) + if user_details == None: + user_details = { + "user_id": str(uuid4()) + , "name": result_dict["name"] + , "orcid": result_dict["orcid"] + , "user_url": "%s/%s" % ( + "/".join(ORCID_AUTH_URL.split("/")[:-2]), + result_dict["orcid"]) + , "login_type": "orcid" + } + save_user(user_details, user_details["user_id"]) + url = "/n/login?type=orcid&uid="+user_details["user_id"] + else: + flash("There was an error getting code from ORCID") + return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, params={"access_token":access_token}) -- cgit v1.2.3 From 309eca58943bf94ce80deb91cf83135957d20980 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 17:48:43 +0300 Subject: Use elasticsearch rather than mysql for local accounts * Register local accounts onto elasticsearch rather than mysql. * Login from the accounts on elasticsearch * Harmonise local and oauth2 logins to use the same code. --- wqflask/wqflask/user_manager.py | 106 ++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 70 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index c3f190c3..4322945b 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -272,15 +272,19 @@ class RegisterUser(object): self.errors = [] self.user = Bunch() - self.user.email_address = kw.get('email_address', '').strip() + self.user.email_address = kw.get('email_address', '').encode("utf-8").strip() if not (5 <= len(self.user.email_address) <= 50): self.errors.append('Email Address needs to be between 5 and 50 characters.') - self.user.full_name = kw.get('full_name', '').strip() + email_exists = get_user_by_unique_column("email_address", self.user.email_address) + if email_exists: + self.errors.append('User already exists with that email') + + self.user.full_name = kw.get('full_name', '').encode("utf-8").strip() if not (5 <= len(self.user.full_name) <= 50): self.errors.append('Full Name needs to be between 5 and 50 characters.') - self.user.organization = kw.get('organization', '').strip() + self.user.organization = kw.get('organization', '').encode("utf-8").strip() if self.user.organization and not (5 <= len(self.user.organization) <= 50): self.errors.append('Organization needs to be empty or between 5 and 50 characters.') @@ -297,28 +301,11 @@ class RegisterUser(object): logger.debug("No errors!") set_password(password, self.user) + self.user.user_id = str(uuid.uuid4()) + self.user.confirmed = 1 self.user.registration_info = json.dumps(basic_info(), sort_keys=True) - - self.new_user = model.User(**self.user.__dict__) - db_session.add(self.new_user) - - try: - db_session.commit() - except sqlalchemy.exc.IntegrityError: - # This exception is thrown if the email address is already in the database - # To do: Perhaps put a link to sign in using an existing account here - self.errors.append("An account with this email address already exists. " - "Click the button above to sign in using an existing account.") - return - - logger.debug("Adding verification email to queue") - #self.send_email_verification() - VerificationEmail(self.new_user) - logger.debug("Added verification email to queue") - - self.thank_you_mode = True - + save_user(self.user.__dict__, self.user.user_id) def set_password(password, user): pwfields = Bunch() @@ -364,7 +351,7 @@ class VerificationEmail(object): verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - data = json.dumps(dict(id=user.id, + data = json.dumps(dict(id=user.user_id, timestamp=timestamp()) ) @@ -519,13 +506,15 @@ def github_oauth2(): if user_details == None: user_details = { "user_id": str(uuid.uuid4()) - , "name": github_user["name"] + , "name": github_user["name"].encode("utf-8") , "github_id": github_user["id"] - , "user_url": github_user["html_url"] + , "user_url": github_user["html_url"].encode("utf-8") , "login_type": "github" , "organization": "" + , "active": 1 + , "confirmed": 1 } - save_user(user_details, user_details.get("user_id")) + save_user(user_details, user_details["user_id"]) url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) @@ -545,7 +534,7 @@ def orcid_oauth2(): } result = requests.post(ORCID_TOKEN_URL, data=data) result_dict = json.loads(result.text.encode("utf-8")) - print("The dict: ", result_dict); + user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) if user_details == None: user_details = { @@ -556,6 +545,9 @@ def orcid_oauth2(): "/".join(ORCID_AUTH_URL.split("/")[:-2]), result_dict["orcid"]) , "login_type": "orcid" + , "organization": "" + , "active": 1 + , "confirmed": 1 } save_user(user_details, user_details["user_id"]) url = "/n/login?type=orcid&uid="+user_details["user_id"] @@ -582,38 +574,11 @@ class LoginUser(object): user.id = user_details["user_id"] user.full_name = user_details["name"] user.login_type = user_details["login_type"] - return self.actual_login_oauth2(user) + return self.actual_login(user) else: flash("Error logging in via OAuth2") return make_response(redirect(url_for('login'))) - def actual_login_oauth2(self, user, assumed_by=None, import_collections=None): - """The meat of the logging in process""" - session_id_signed = self.successful_login_oauth2(user) - flash("Thank you for logging in {}.".format(user.full_name), "alert-success") - print("IMPORT1:", import_collections) - response = make_response(redirect(url_for('index_page', import_collections=import_collections))) - if self.remember_me: - max_age = self.remember_time - else: - max_age = None - response.set_cookie(UserSession.cookie_name, session_id_signed, max_age=max_age) - return response - - def successful_login_oauth2(self, user, assumed_by=None): - login_rec = model.Login(user) - login_rec.successful = True - login_rec.session_id = str(uuid.uuid4()) - login_rec.assumed_by = assumed_by - session_id_signature = actual_hmac_creation(login_rec.session_id) - session_id_signed = login_rec.session_id + ":" + session_id_signature - logger.debug("session_id_signed:", session_id_signed) - - session = dict(login_time = time.time(), - user_id = user.id, - user_login_type = user.login_type) - return session_id_signed - def standard_login(self): """Login through the normal form""" params = request.form if request.form else request.args @@ -628,20 +593,23 @@ class LoginUser(object): } return render_template("new_security/login_user.html", external_login=external_login) else: - try: - user = model.User.query.filter_by(email_address=params['email_address']).one() - except sqlalchemy.orm.exc.NoResultFound: - logger.debug("No account exists for that email address") - valid = False - user = None - else: + user_details = get_user_by_unique_column("email_address", params["email_address"]) + user = None + if user_details: + user = model.User(); + for key in user_details: + user.__dict__[key] = user_details[key] + print("RETRIEVED USER: ", user) + valid = False; + submitted_password = params['password'] pwfields = Struct(json.loads(user.password)) - encrypted = Password(submitted_password, - pwfields.salt, - pwfields.iterations, - pwfields.keylength, - pwfields.hashfunc) + encrypted = Password( + submitted_password, + pwfields.salt, + pwfields.iterations, + pwfields.keylength, + pwfields.hashfunc) logger.debug("\n\nComparing:\n{}\n{}\n".format(encrypted.password, pwfields.password)) valid = pbkdf2.safe_str_cmp(encrypted.password, pwfields.password) logger.debug("valid is:", valid) @@ -709,8 +677,6 @@ class LoginUser(object): else: expire_time = THREE_DAYS Redis.expire(key, expire_time) - db_session.add(login_rec) - db_session.commit() return session_id_signed def unsuccessful_login(self, user): -- cgit v1.2.3 From fb0ad54313366a3f0efda3ef20e800d5e52d1d53 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 30 Jan 2018 12:23:31 +0300 Subject: Set to None if no value provided * Add a method to set the configuration variables to None if the configuration values are not provided at startup or in the configuration files. The system already checks for these values, and if they are absent, it simply fails to display the OAuth service as available for use to login. --- wqflask/utility/tools.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 8a32c9b4..feeeccfc 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,17 +251,28 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) -GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') -GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') -GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET -GITHUB_API_URL = get_setting('GITHUB_API_URL') -ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') -ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') -ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET -ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') - -ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') +def get_setting_safe(setting): + try: + return get_setting(setting) + except: + print("Could not find the setting '", setting, "'. Continuing with value unset") + return None + +GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET') +GITHUB_AUTH_URL = None +if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET: + GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET +GITHUB_API_URL = get_setting_safe('GITHUB_API_URL') +ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET') +ORCID_AUTH_URL = None +if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET: + ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET +ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') + +ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) -- cgit v1.2.3 From afaea1b1297d0cf08565746799d2900a6981823a Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 30 Jan 2018 13:07:58 +0300 Subject: Fail safely if elasticsearch is down or unconfigured * If elasticsearch server is down, or the configuration variables are not provided at startup or in a configuration file, then do not allow the system to simply crash, but instead, inform the user that they cannot use the services that depend on elasticsearch to be running. --- wqflask/utility/elasticsearch_tools.py | 16 ++++++++------ .../wqflask/templates/new_security/login_user.html | 25 ++++++++++++++++++---- wqflask/wqflask/user_manager.py | 6 +++++- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index c2c999ea..8b8ad9cc 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,10 +1,14 @@ -from elasticsearch import Elasticsearch, TransportError -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT +es = None +try: + from elasticsearch import Elasticsearch, TransportError + from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT -es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST - , "port": ELASTICSEARCH_PORT -}]) + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT + }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None +except: + es = None def get_user_by_unique_column(column_name, column_value): user_details = None diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 15f0a27e..0dae3503 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -16,7 +16,14 @@

Don't have an account?

- Create a new account + {% if es_server: %} + Create a new account + {% else: %} +
+

You cannot create an account at this moment.
+ Please try again later.

+
+ {% endif %}

Login with external services

@@ -31,13 +38,17 @@ Login with ORCID {% endif %} - -
+ {% else: %} +
+

You cannot login with external services at this time.
+ Please try again later.

+
{% endif %} +

Already have an account? Sign in here.

- + {% if es_server: %}
@@ -75,6 +86,12 @@
+ {% else: %} +
+

You cannot login at this moment using your GeneNetwork account.
+ Please try again later.

+
+ {% endif %} diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 4322945b..772d6c83 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -585,13 +585,17 @@ class LoginUser(object): logger.debug("in login params are:", params) if not params: from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL + from utility.elasticsearch_tools import es external_login = None if GITHUB_AUTH_URL or ORCID_AUTH_URL: external_login={ "github": GITHUB_AUTH_URL, "orcid": ORCID_AUTH_URL } - return render_template("new_security/login_user.html", external_login=external_login) + return render_template( + "new_security/login_user.html" + , external_login=external_login + , es_server=es) else: user_details = get_user_by_unique_column("email_address", params["email_address"]) user = None -- cgit v1.2.3 From 6ec9dcc365b899f7f4aaab739103f4559c2c1620 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 16:19:13 +0300 Subject: Check each part of the PYTHONPATH * PYTHONPATH may be composed of multiple locations in the filesystem, thus the need to test that each location exists and is a directory. --- bin/genenetwork2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 34fbc72e..145ce395 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -126,7 +126,11 @@ else export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" - if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi + while IFS=":" read -ra PPATH; do + for PPart in "${PPATH[@]}"; do + if [ ! -d $PPart ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi + done + done <<< "$PYTHONPATH" if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi fi -- cgit v1.2.3 From 832a82d4732290fcae033976f6dcb6bab16f61b7 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 17:47:29 +0300 Subject: Add SMTP configuration variables * Add configuration variables to enable the system connect to the configured SMTP server to send out emails. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index feeeccfc..ec673cf5 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -274,6 +274,10 @@ ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') +SMTP_CONNECT = get_setting_safe('SMTP_CONNECT') +SMTP_USERNAME = get_setting_safe('SMTP_USERNAME') +SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) -- cgit v1.2.3 From 2b42e6ec444936b7b3594c5fc07c540a74fe4c05 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 17:55:36 +0300 Subject: Send emails for forgotten passwords * Update the code so that it sends out emails for the "forgot password" feature. --- wqflask/wqflask/user_manager.py | 71 ++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 772d6c83..ec29062d 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -368,23 +368,33 @@ class ForgotPasswordEmail(VerificationEmail): template_name = "email/forgot_password.txt" key_prefix = "forgot_password_code" subject = "GeneNetwork password reset" + fromaddr = "no-reply@genenetwork.org" - def __init__(self, user): + def __init__(self, toaddr): + from email.MIMEMultipart import MIMEMultipart + from email.MIMEText import MIMEText verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - data = json.dumps(dict(id=user.id, - timestamp=timestamp()) - ) + # data = json.dumps(dict(id=user.id, + # timestamp=timestamp()) + # ) + + # Redis.set(key, data) + # Redis.expire(key, THREE_DAYS) - Redis.set(key, data) - #two_days = 60 * 60 * 24 * 2 - Redis.expire(key, THREE_DAYS) - to = user.email_address subject = self.subject - body = render_template(self.template_name, - verification_code = verification_code) - send_email(to, subject, body) + body = render_template( + self.template_name, + verification_code = verification_code) + + msg = MIMEMultipart() + msg["To"] = toaddr + msg["Subject"] = self.subject + msg["From"] = self.fromaddr + msg.attach(MIMEText(body, "plain")) + + send_email(toaddr, msg.as_string()) class Password(object): @@ -708,13 +718,16 @@ def forgot_password(): def forgot_password_submit(): params = request.form email_address = params['email_address'] - try: - user = model.User.query.filter_by(email_address=email_address).one() - except orm.exc.NoResultFound: - flash("Couldn't find a user associated with the email address {}. Sorry.".format( - email_address)) - return redirect(url_for("login")) - ForgotPasswordEmail(user) + user_details = get_user_by_unique_column("email_address", email_address) + if user_details: + ForgotPasswordEmail(user_details["email_address"]) + # try: + # user = model.User.query.filter_by(email_address=email_address).one() + # except orm.exc.NoResultFound: + # flash("Couldn't find a user associated with the email address {}. Sorry.".format( + # email_address)) + # return redirect(url_for("login")) + # ForgotPasswordEmail(user) return render_template("new_security/forgot_password_step2.html", subject=ForgotPasswordEmail.subject) @@ -861,13 +874,21 @@ app.jinja_env.globals.update(url_for_hmac=url_for_hmac, ####################################################################################### -def send_email(to, subject, body): - msg = json.dumps(dict(From="no-reply@genenetwork.org", - To=to, - Subject=subject, - Body=body)) - Redis.rpush("mail_queue", msg) - +# def send_email(to, subject, body): +# msg = json.dumps(dict(From="no-reply@genenetwork.org", +# To=to, +# Subject=subject, +# Body=body)) +# Redis.rpush("mail_queue", msg) + +def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): + from smtplib import SMTP + from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD + server = SMTP(SMTP_CONNECT) + server.starttls() + server.login(SMTP_USERNAME, SMTP_PASSWORD) + server.sendmail(fromaddr, toaddr, msg) + server.quit() -- cgit v1.2.3 From 1c9540879d8761d9252c3fb3f749ae0b6d5be2b9 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 20:01:45 +0300 Subject: Refactor common items to more generic methods. * Refactor code that can be used in more than one place to a more generic method/function that's called by other methods --- wqflask/utility/elasticsearch_tools.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 8b8ad9cc..ea636b2e 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -11,25 +11,27 @@ except: es = None def get_user_by_unique_column(column_name, column_value): - user_details = None + return get_item_by_unique_column(column_name, column_value, index="users", doc_type="local") + +def save_user(user, user_id): + es_save_data("users", "local", user, user_id) + +def get_item_by_unique_column(column_name, column_value, index, doc_type): + item_details = None try: response = es.search( - index = "users" - , doc_type = "local" + index = index + , doc_type = doc_type , body = { "query": { "match": { column_name: column_value } } }) if len(response["hits"]["hits"]) > 0: - user_details = response["hits"]["hits"][0]["_source"] + item_details = response["hits"]["hits"][0]["_source"] except TransportError as te: pass - return user_details + return item_details -def save_user(user, user_id, index="users", doc_type="local"): +def es_save_data(index, doc_type, data_item, data_id,): from time import sleep - es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST - , "port": ELASTICSEARCH_PORT - }]) - es.create(index, doc_type, body=user, id=user_id) + es.create(index, doc_type, body=data_item, id=data_id) sleep(1) # Delay 1 second to allow indexing -- cgit v1.2.3 From 03edecad79c4e8e1e10734e9e8f21f5da7912851 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 20:04:09 +0300 Subject: Add code to allow user to change password * After the email is sent to the user, there is need to provide a way for the user to actually change their password, and have the results saved. --- wqflask/wqflask/user_manager.py | 65 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index ec29062d..8f09c206 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -55,7 +55,7 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list import requests -from utility.elasticsearch_tools import get_user_by_unique_column, save_user +from utility.elasticsearch_tools import get_user_by_unique_column, save_user, es_save_data THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -376,12 +376,12 @@ class ForgotPasswordEmail(VerificationEmail): verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - # data = json.dumps(dict(id=user.id, - # timestamp=timestamp()) - # ) - - # Redis.set(key, data) - # Redis.expire(key, THREE_DAYS) + data = { + "verification_code": verification_code, + "email_address": toaddr, + "timestamp": timestamp() + } + es_save_data(self.key_prefix, "local", data, verification_code) subject = self.subject body = render_template( @@ -429,38 +429,59 @@ def verify_email(): response.set_cookie(UserSession.cookie_name, session_id_signed) return response -@app.route("/n/password_reset") +@app.route("/n/password_reset", methods=['GET']) def password_reset(): + from utility.elasticsearch_tools import get_item_by_unique_column logger.debug("in password_reset request.url is:", request.url) # We do this mainly just to assert that it's in proper form for displaying next page # Really not necessary but doesn't hurt - user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() - - return render_template("new_security/password_reset.html", user_encode=user_encode) + # user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() + verification_code = request.args.get('code') + hmac = request.args.get('hm') + if verification_code: + code_details = get_item_by_unique_column( + "verification_code", + verification_code, + ForgotPasswordEmail.key_prefix, + "local") + if code_details: + user_details = get_user_by_unique_column( + "email_address", + code_details["email_address"]) + if user_details: + return render_template( + "new_security/password_reset.html", user_encode=user_details["user_id"]) + else: + flash("Invalid code: User no longer exists!", "error") + else: + flash("Invalid code: Password reset code does not exist or might have expired!", "error") + return redirect(url_for("login"))#render_template("new_security/login_user.html", error=error) @app.route("/n/password_reset_step2", methods=('POST',)) def password_reset_step2(): + from utility.elasticsearch_tools import es logger.debug("in password_reset request.url is:", request.url) errors = [] + user_id = request.form['user_encode'] - user_encode = request.form['user_encode'] - verification_code, separator, hmac = user_encode.partition(':') - - hmac_verified = actual_hmac_creation(verification_code) logger.debug("locals are:", locals()) - assert hmac == hmac_verified, "Someone has been naughty" - - user = DecodeUser.actual_get_user(ForgotPasswordEmail.key_prefix, verification_code) - logger.debug("user is:", user) - + user = Bunch() password = request.form['password'] - set_password(password, user) - db_session.commit() + + es.update( + index = "users" + , doc_type = "local" + , id = user_id + , body = { + "doc": { + "password": user.__dict__.get("password") + } + }) flash("Password changed successfully. You can now sign in.", "alert-info") response = make_response(redirect(url_for('login'))) -- cgit v1.2.3 From 25ff5e9e47c568547a855e42a4fd43477db2a61e Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Feb 2018 10:11:21 +0300 Subject: Add check for elasticsearch * Add some extra checks to ensure that elasticsearch is running before presenting the UI to the user. --- wqflask/utility/elasticsearch_tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index ea636b2e..a0383033 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -7,6 +7,10 @@ try: "host": ELASTICSEARCH_HOST , "port": ELASTICSEARCH_PORT }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + + # Check if elasticsearch is running + if not es.ping(): + es = None except: es = None -- cgit v1.2.3 From 4fe60697964d81ce1425000e764fd0a27d73eb29 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Feb 2018 11:02:07 +0300 Subject: Check elasticsearch at point of use * Instead of checking for the state of elasticsearch at startup, check the state at the moment the user requests a feature that depends on elasticsearch. This reduces the chances that the user is dropped onto an exception page when elasticsearch server goes down. --- wqflask/utility/elasticsearch_tools.py | 3 --- wqflask/wqflask/user_manager.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index a0383033..4fc0035c 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -8,9 +8,6 @@ try: , "port": ELASTICSEARCH_PORT }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - # Check if elasticsearch is running - if not es.ping(): - es = None except: es = None diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 8f09c206..630be9aa 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -626,7 +626,7 @@ class LoginUser(object): return render_template( "new_security/login_user.html" , external_login=external_login - , es_server=es) + , es_server=es.ping()) else: user_details = get_user_by_unique_column("email_address", params["email_address"]) user = None -- cgit v1.2.3 From a91c0c240e3534567f65b16578bcfaf72a9055c9 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 12 Feb 2018 17:08:58 +0000 Subject: Quick parallel correlation should work, though I'm not sure why the code suddenly decided to start running it when it wasn't in the past. Switched correlation table to tabulator --- wqflask/wqflask/correlation/show_corr_results.py | 22 +++++++++- wqflask/wqflask/templates/correlation_page.html | 51 ++++++++++++++++++------ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 24432ad0..bb109f60 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -120,8 +120,15 @@ class CorrelationResults(object): self.min_expr = float(start_vars['min_expr']) else: self.min_expr = None - self.p_range_lower = float(start_vars['p_range_lower']) - self.p_range_upper = float(start_vars['p_range_upper']) + logger.debug("P RANGE:", start_vars['p_range_lower']) + try: + self.p_range_lower = float(start_vars['p_range_lower']) + except: + self.p_range_lower = -1.00 + try: + self.p_range_upper = float(start_vars['p_range_upper']) + except: + self.p_range_upper = 1.00 if ('loc_chr' in start_vars and 'min_loc_mb' in start_vars and @@ -1153,6 +1160,17 @@ class CorrelationResults(object): import math import reaper + def cmpOrder2(A,B): + try: + if A[-1] < B[-1]: + return -1 + elif A[-1] == B[-1]: + return 0 + else: + return 1 + except: + return 0 + def calCorrelation(dbdata,userdata,N): X = [] Y = [] diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index fa9e3585..c7fb7cc6 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -1,9 +1,10 @@ {% extends "base.html" %} {% block css %} + + {% endblock %} {% block content %} @@ -63,15 +64,14 @@
- +
- {% for header in target_dataset.header_fields %} {% if header == 'Year' %} {% elif header == 'Max LRS' %} - + {% elif header == 'Max LRS Location' %} {% elif header == 'Location' %} @@ -79,7 +79,7 @@ {% elif header == 'Mean' %} {% elif header == 'Additive Effect' %} - + {% elif header == 'Index' %} {% elif header == 'N' %} @@ -91,7 +91,7 @@ {% if target_dataset.type == "ProbeSet" %} {% if corr_method == 'pearson' %} - + @@ -129,8 +129,7 @@ {% for trait in correlation_results %} - - + {% for uc in collections %} + {% if g.user_session.user_ob %} + {% else %} + + {% endif %} diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 1254ea6a..0d305559 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -275,29 +275,32 @@ {% if target_dataset.type == "ProbeSet" %} - var json_array = new Array(); + var json_array = []; for (i=0; i < table_json.length; i++){ - json_array.push({ - checkbox: table_json[i]["checkbox"], - index: table_json[i]["index"], - trait_id: table_json[i]["trait_id"], - symbol: table_json[i]["symbol"], - description: table_json[i]["description"], - location: table_json[i]["location"], - mean: table_json[i]["mean"], - lrs_score: table_json[i]["lrs_score"], - lrs_location: table_json[i]["lrs_location"], - additive: table_json[i]["additive"], - sample_r: table_json[i]["sample_r"], - num_overlap: table_json[i]["num_overlap"], - sample_p: table_json[i]["sample_p"], - lit_corr: table_json[i]["lit_corr"], - tissue_corr: table_json[i]["tissue_corr"] - }); - console.log("JSON_DATA:", json_array); + this_array = json_array + + var row_dict = {}; + row_dict.checkbox = table_json[i]["checkbox"] + row_dict.index = table_json[i]["index"] + row_dict.trait_id = table_json[i]["trait_id"] + row_dict.symbol = table_json[i]["symbol"] + row_dict.description = table_json[i]["description"] + row_dict.location = table_json[i]["location"] + row_dict.mean = table_json[i]["mean"] + row_dict.lrs_score = table_json[i]["lrs_score"] + row_dict.lrs_location = table_json[i]["lrs_location"] + row_dict.additive = table_json[i]["additive"] + row_dict.sample_r = table_json[i]["sample_r"] + row_dict.num_overlap = table_json[i]["num_overlap"] + row_dict.sample_p = table_json[i]["sample_p"] + row_dict.lit_corr = table_json[i]["lit_corr"] + row_dict.tissue_corr = table_json[i]["tissue_corr"] + this_array.push(row_dict); + json_array.concat(this_array); + //console.log("JSON_ARRAY:", json_array); } - //console.log("JSON_DATA:", json_array) + console.log("JSON_DATA:", json_array) $("#trait_table").tabulator({ data: json_array, diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index dcec2b9e..ab2e2aae 100644 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -1,6 +1,6 @@
{% if (use_pylmm_rqtl and dataset.group.species != "human") or use_plink_gemma %} -
+
{{header}}Max LRS ?Max LRS{{header}}{{header}}Additive Effect ?Additive Effect{{header}}Sample r  NN Sample p(r) Lit r Tissue r
{{ loop.index }}{{ loop.index }} + + + + + - {% endblock %} -- cgit v1.2.3 From a88f9c25e1b6eaaeede23d9217b1998f28fa7bbf Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:24:55 +0300 Subject: Remove failing code * With the parameterized module, the testcase cannot be run independently at this time. --- test/requests/test_login_local.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index 8e2dec4e..808649ca 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -58,19 +58,3 @@ class TestLoginLocal(ParametrizedTest): result = requests.post(self.login_url, data=data) index = result.content.find(expected) self.assertTrue(index >= 0, message) - - -def main(gn2, es): - import unittest - suite = unittest.TestSuite() - suite.addTest(TestLoginLocal(methodName="testLoginNonRegisteredUser", gn2_url=gn2, es_url=es)) - suite.addTest(TestLoginLocal(methodName="testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse", gn2_url=gn2, es_url=es)) - runner = unittest.TextTestRunner() - runner.run(suite) - -if __name__ == "__main__": - import sys - if len(sys.argv) < 3: - raise Exception("Required arguments missing") - else: - main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From 4d7c565aac196ba1913b65ed27481c6974f1a0b5 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:26:42 +0300 Subject: Add new test for github logins --- test/requests/run-integration-tests.py | 4 +++- test/requests/test_login_github.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 test/requests/test_login_github.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py index 0fd7bb20..fc795779 100644 --- a/test/requests/run-integration-tests.py +++ b/test/requests/run-integration-tests.py @@ -1,11 +1,13 @@ import sys from test_login_local import TestLoginLocal +from test_login_github import TestLoginGithub from test_registration import TestRegistration from unittest import TestSuite, TextTestRunner, TestLoader test_cases = [ - TestLoginLocal, TestRegistration + , TestLoginLocal + , TestLoginGithub ] def suite(gn2_url, es_url): diff --git a/test/requests/test_login_github.py b/test/requests/test_login_github.py new file mode 100644 index 00000000..15bf18ae --- /dev/null +++ b/test/requests/test_login_github.py @@ -0,0 +1,40 @@ +import uuid +import requests +from time import sleep +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +login_link_text = 'Sign in' +logout_link_text = 'Sign out' +uid = str(uuid.uuid4()) + +class TestLoginGithub(ParametrizedTest): + + def setUp(self): + super(TestLoginGithub, self).setUp() + data = { + "user_id": uid + , "name": "A. T. Est User" + , "github_id": 693024 + , "user_url": "https://fake-github.com/atestuser" + , "login_type": "github" + , "organization": "" + , "active": 1 + , "confirmed": 1 + } + self.es.create(index="users", doc_type="local", body=data, id=uid) + sleep(1) + + def tearDown(self): + super(TestLoginGithub, self).tearDown() + self.es.delete(index="users", doc_type="local", id=uid) + + @parameterized.expand([ + ("1234", login_link_text, "Login should have failed with non-existing user") + , (uid, logout_link_text, "Login should have been successful with existing user") + ]) + def testLogin(self, test_uid, expected, message): + url = self.gn2_url+"/n/login?type=github&uid="+test_uid + result = requests.get(url) + index = result.content.find(expected) + self.assertTrue(index >= 0, message) -- cgit v1.2.3 From 4d656976eae920066261e26bf22a999d90a0beab Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:36:29 +0300 Subject: Add new tests for orcid logins --- test/requests/run-integration-tests.py | 2 ++ test/requests/test_login_orcid.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/requests/test_login_orcid.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py index fc795779..5e816549 100644 --- a/test/requests/run-integration-tests.py +++ b/test/requests/run-integration-tests.py @@ -1,5 +1,6 @@ import sys from test_login_local import TestLoginLocal +from test_login_orcid import TestLoginOrcid from test_login_github import TestLoginGithub from test_registration import TestRegistration from unittest import TestSuite, TextTestRunner, TestLoader @@ -8,6 +9,7 @@ test_cases = [ TestRegistration , TestLoginLocal , TestLoginGithub + , TestLoginOrcid ] def suite(gn2_url, es_url): diff --git a/test/requests/test_login_orcid.py b/test/requests/test_login_orcid.py new file mode 100644 index 00000000..6e40ece5 --- /dev/null +++ b/test/requests/test_login_orcid.py @@ -0,0 +1,40 @@ +import uuid +import requests +from time import sleep +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +login_link_text = 'Sign in' +logout_link_text = 'Sign out' +uid = str(uuid.uuid4()) + +class TestLoginOrcid(ParametrizedTest): + + def setUp(self): + super(TestLoginOrcid, self).setUp() + data = { + "user_id": uid + , "name": "A. T. Est User" + , "orcid": 345872 + , "user_url": "https://fake-orcid.org/atestuser" + , "login_type": "orcid" + , "organization": "" + , "active": 1 + , "confirmed": 1 + } + self.es.create(index="users", doc_type="local", body=data, id=uid) + sleep(1) + + def tearDown(self): + super(TestLoginOrcid, self).tearDown() + self.es.delete(index="users", doc_type="local", id=uid) + + @parameterized.expand([ + ("1234", login_link_text, "Login should have failed with non-existing user") + , (uid, logout_link_text, "Login should have been successful with existing user") + ]) + def testLogin(self, test_uid, expected, message): + url = self.gn2_url+"/n/login?type=orcid&uid="+test_uid + result = requests.get(url) + index = result.content.find(expected) + self.assertTrue(index >= 0, message) -- cgit v1.2.3 From 1ee48887d93cace2d1b00b2f1adc9a6407fdb882 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:37:06 +0300 Subject: Provide connections to elasticsearch * Provide connections to elasticsearch at various points in the code. These oversight was caught while running tests. --- wqflask/wqflask/user_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index e1920f4e..c344ea27 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -540,6 +540,7 @@ def github_oauth2(): result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} github_user = get_github_user_details(result_dict["access_token"]) + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "github_id", github_user["id"]) if user_details == None: user_details = { @@ -573,6 +574,7 @@ def orcid_oauth2(): result = requests.post(ORCID_TOKEN_URL, data=data) result_dict = json.loads(result.text.encode("utf-8")) + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "orcid", result_dict["orcid"]) if user_details == None: user_details = { @@ -606,6 +608,7 @@ class LoginUser(object): def oauth2_login(self, login_type, user_id): """Login via an OAuth2 provider""" + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "user_id", user_id) if user_details: user = model.User() -- cgit v1.2.3 From 1f591baa129a7a42d6c0cadf73441274c8e373d3 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 18:11:33 +0300 Subject: Add tests to check that the UI is setup correctly * Check that the links for OAuth2 logins via ORCID and GitHub are setup correctly. --- test/requests/test_login_github.py | 7 +++++++ test/requests/test_login_orcid.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/test/requests/test_login_github.py b/test/requests/test_login_github.py index 15bf18ae..1bf4f695 100644 --- a/test/requests/test_login_github.py +++ b/test/requests/test_login_github.py @@ -1,6 +1,7 @@ import uuid import requests from time import sleep +from wqflask import app from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -29,6 +30,12 @@ class TestLoginGithub(ParametrizedTest): super(TestLoginGithub, self).tearDown() self.es.delete(index="users", doc_type="local", id=uid) + def testLoginUrl(self): + login_button_text = 'Login with Github' + result = requests.get(self.gn2_url+"/n/login") + index = result.content.find(login_button_text) + self.assertTrue(index >= 0, "Should have found `Login with Github` button") + @parameterized.expand([ ("1234", login_link_text, "Login should have failed with non-existing user") , (uid, logout_link_text, "Login should have been successful with existing user") diff --git a/test/requests/test_login_orcid.py b/test/requests/test_login_orcid.py index 6e40ece5..ea15642e 100644 --- a/test/requests/test_login_orcid.py +++ b/test/requests/test_login_orcid.py @@ -1,6 +1,7 @@ import uuid import requests from time import sleep +from wqflask import app from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -29,6 +30,12 @@ class TestLoginOrcid(ParametrizedTest): super(TestLoginOrcid, self).tearDown() self.es.delete(index="users", doc_type="local", id=uid) + def testLoginUrl(self): + login_button_text = 'a href="https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=' + app.config.get("ORCID_CLIENT_ID") + '&client_secret=' + app.config.get("ORCID_CLIENT_SECRET") + '" title="Login with ORCID" class="btn btn-info btn-group">Login with ORCID' + result = requests.get(self.gn2_url+"/n/login") + index = result.content.find(login_button_text) + self.assertTrue(index >= 0, "Should have found `Login with ORCID` button") + @parameterized.expand([ ("1234", login_link_text, "Login should have failed with non-existing user") , (uid, logout_link_text, "Login should have been successful with existing user") -- cgit v1.2.3 From 04240ba98c040887a318b9f19dff0be3f68f5e4d Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 26 Feb 2018 23:01:55 +0000 Subject: Committing some in-progress changes related to loading correlation page from JSON --- wqflask/wqflask/correlation/show_corr_results.py | 63 ++++++++++++++++++++++- wqflask/wqflask/static/dbdoc/TODO.md | 1 + wqflask/wqflask/templates/correlation_page.html | 65 ++++++++++++++++++------ wqflask/wqflask/templates/empty_collection.html | 2 +- wqflask/wqflask/views.py | 2 +- 5 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 wqflask/wqflask/static/dbdoc/TODO.md diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 4d7fea61..9b048346 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -32,6 +32,7 @@ import pp import math import collections import resource +import json import scipy @@ -51,6 +52,7 @@ import utility.webqtlUtil #this is for parallel computing only. from wqflask.correlation import correlation_functions from utility.benchmark import Bench import utility.webqtlUtil +from wqflask import user_manager from MySQLdb import escape_string as escape @@ -248,7 +250,7 @@ class CorrelationResults(object): else: self.this_trait_vals.append("None") num_overlap = len(self.this_trait_vals) - + logger.debug("DOING PARALLEL") self.do_parallel_correlation(db_filename, num_overlap) else: for trait, values in self.target_dataset.trait_data.iteritems(): @@ -335,6 +337,7 @@ class CorrelationResults(object): #print("self.correlation_results: ", pf(self.correlation_results)) + self.json_results = generate_corr_json(self.correlation_results, self.this_trait, self.dataset, self.target_dataset) #XZ, 09/18/2008: get all information about the user selected database. #target_db_name = fd.corr_dataset @@ -1431,3 +1434,61 @@ class CorrelationResults(object): # for one_result in results: # for one_traitinfo in one_result: # allcorrelations.append( one_traitinfo ) + +def generate_corr_json(corr_results, this_trait, dataset, target_dataset): + results_list = [] + for i, trait in enumerate(corr_results): + results_dict = {} + results_dict['checkbox'] = "" + results_dict['index'] = i + 1 + results_dict['trait_id'] = ""+str(trait.name)+"" + if target_dataset.type == "ProbeSet": + results_dict['symbol'] = trait.symbol + results_dict['description'] = trait.description_display + results_dict['location'] = trait.location_repr + results_dict['mean'] = float(trait.mean) + if trait.LRS_score_repr != "N/A": + results_dict['lrs_score'] = "%.1f" % float(trait.LRS_score_repr) + else: + results_dict['lrs_score'] = "N/A" + results_dict['lrs_location'] = trait.LRS_location_repr + if trait.additive != "": + results_dict['additive'] = "%0.3f" % float(trait.additive) + else: + results_dict['additive'] = "N/A" + results_dict['sample_r'] = "" + "%0.3f" % float(trait.sample_r) + "" + results_dict['num_overlap'] = trait.num_overlap + results_dict['sample_p'] = "%0.3e" % float(trait.sample_p) + if trait.lit_corr == "" or trait.lit_corr == 0: + results_dict['lit_corr'] = "--" + else: + results_dict['lit_corr'] = "%0.3f" % float(trait.lit_corr) + if trait.tissue_corr == "" or trait.tissue_corr == 0: + results_dict['tissue_corr'] = "--" + else: + results_dict['tissue_corr'] = "%0.3f" % float(trait.tissue_corr) + elif target_dataset.type == "Publish": + results_dict['description'] = trait.description_display + results_dict['authors'] = trait.authors + if trait.pubmed_id: + results_dict['pubmed'] = " " + trait.pubmed_text + "" + else: + results_dict['pubmed'] = "N/A" + results_dict['lrs_score'] = trait.LRS_score_repr + results_dict['lrs_location'] = trait.LRS_location_repr + if trait.additive != "": + results_dict['additive'] = "%0.3f" % float(trait.additive) + else: + results_dict['additive'] = "N/A" + results_dict['sample_r'] = "" + "%0.3f" % trait.sample_r + "" + results_dict['num_overlap'] = trait.num_overlap + results_dict['sample_p'] = "%0.3e" % float(trait.sample_p) + else: + results_dict['lrs_location'] = trait.LRS_location_repr + results_dict['sample_r'] = "" + "%0.3f" % float(trait.sample_r) + "" + results_dict['num_overlap'] = trait.num_overlap + results_dict['sample_p'] = "%0.3e" % float(trait.sample_p) + + results_list.append(results_dict) + + return json.dumps(results_list) diff --git a/wqflask/wqflask/static/dbdoc/TODO.md b/wqflask/wqflask/static/dbdoc/TODO.md new file mode 100644 index 00000000..c0a8bab7 --- /dev/null +++ b/wqflask/wqflask/static/dbdoc/TODO.md @@ -0,0 +1 @@ +TODO: Add all database documentation into this folder diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 629724c0..1254ea6a 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -65,8 +65,10 @@
+
@@ -267,24 +271,55 @@ $(document).ready( function () { + var table_json = {{ json_results | safe }} + + + {% if target_dataset.type == "ProbeSet" %} + var json_array = new Array(); + + for (i=0; i < table_json.length; i++){ + json_array.push({ + checkbox: table_json[i]["checkbox"], + index: table_json[i]["index"], + trait_id: table_json[i]["trait_id"], + symbol: table_json[i]["symbol"], + description: table_json[i]["description"], + location: table_json[i]["location"], + mean: table_json[i]["mean"], + lrs_score: table_json[i]["lrs_score"], + lrs_location: table_json[i]["lrs_location"], + additive: table_json[i]["additive"], + sample_r: table_json[i]["sample_r"], + num_overlap: table_json[i]["num_overlap"], + sample_p: table_json[i]["sample_p"], + lit_corr: table_json[i]["lit_corr"], + tissue_corr: table_json[i]["tissue_corr"] + }); + console.log("JSON_DATA:", json_array); + } + //console.log("JSON_DATA:", json_array) + $("#trait_table").tabulator({ + data: json_array, columns:[ - {title:"Index", formatter:"html"}, - {title:"Record", formatter:"html"}, - {title:"Symbol", formatter:"plaintext"}, - {title:"Description", formatter:"textarea", width:"25%"}, - {title:"Location", formatter:"plaintext"}, - {title:"Mean", formatter:"plaintext"}, - {title:"Max LRS", formatter:"plaintext"}, - {title:"Max LRS Location", formatter:"plaintext"}, - {title:"Additive Effect", formatter:"plaintext"}, - {title:"Sample r", formatter:"html"}, - {title:"N", formatter:"plaintext"}, - {title:"Sample p(r)", formatter:"plaintext"}, - {title:"Lit r", formatter:"plaintext"}, - {title:"Tissue r", formatter:"plaintext"} + {title:"", field:"checkbox", formatter:"html"}, + {title:"Index", field:"index", formatter:"plaintext"}, + {title:"Record", field:"trait_id", formatter:"html"}, + {title:"Symbol", field:"symbol", formatter:"plaintext"}, + {title:"Description", field:"description", formatter:"textarea", width:"25%"}, + {title:"Location", field:"location", formatter:"plaintext"}, + {title:"Mean", field:"mean", formatter:"plaintext"}, + {title:"Max LRS", field:"lrs_score", formatter:"plaintext"}, + {title:"Max LRS Location", field:"lrs_location", formatter:"plaintext"}, + {title:"Additive Effect", field:"additive", formatter:"plaintext"}, + {title:"Sample r", field:"sample_r", formatter:"html"}, + {title:"N", field:"num_overlap", formatter:"plaintext"}, + {title:"Sample p(r)", field:"sample_p", formatter:"plaintext"}, + {title:"Lit r", field:"lit_corr", formatter:"plaintext"}, + {title:"Tissue r", field:"tissue_corr", formatter:"plaintext"} ] }); + {% endif %} $('#trait_table tr').click(function(event) { if (event.target.type !== 'checkbox') { diff --git a/wqflask/wqflask/templates/empty_collection.html b/wqflask/wqflask/templates/empty_collection.html index 3f2b3786..d1b779ef 100644 --- a/wqflask/wqflask/templates/empty_collection.html +++ b/wqflask/wqflask/templates/empty_collection.html @@ -6,7 +6,7 @@
-

You must select at least one trait to use the {{ tool }}.

+

You must select at least two traits to use the {{ tool }}.

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 4e81c29c..187b60dc 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -743,7 +743,7 @@ def corr_matrix_page(): start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] - if traits[0] != "": + if len(traits) > 1: template_vars = show_corr_matrix.CorrelationMatrix(start_vars) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, -- cgit v1.2.3 From 51383c96dc275f4462716398f9ec665fbd84802a Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 2 Mar 2018 17:59:28 +0000 Subject: Added option to display selected covariates for GEMMA mapping Changed layout/appearance of mapping options Partially fixed issue with deleting collections Changes to correlation table still in progress; having trouble getting tabulator to load JSON --- wqflask/wqflask/collect.py | 7 +- wqflask/wqflask/marker_regression/gemma_mapping.py | 4 +- .../javascript/get_covariates_from_collection.js | 5 + .../wqflask/static/new/javascript/show_trait.js | 3 +- wqflask/wqflask/templates/collections/list.html | 12 +- wqflask/wqflask/templates/correlation_page.html | 43 +++-- .../templates/show_trait_mapping_tools.html | 208 ++++++++++----------- 7 files changed, 146 insertions(+), 136 deletions(-) diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 2f6c3a96..0f72b29d 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -349,8 +349,11 @@ def delete_collection(): db_session.delete(uc) db_session.commit() else: - collection_name = params['collection_name'] - user_manager.AnonUser().delete_collection(collection_name) + if "collection_name" in params: + collection_name = params['collection_name'] + else: + for this_collection in params['uc_id'].split(":"): + user_manager.AnonUser().delete_collection(this_collection) flash("We've deleted the collection: {}.".format(collection_name), "alert-info") diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index 233a5c76..7563eb2c 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -235,7 +235,7 @@ def parse_loco_output(this_dataset, gwa_output_filename): else: marker['chr'] = line.split("\t")[0] marker['Mb'] = float(line.split("\t")[2]) / 1000000 - marker['p_value'] = float(line.split("\t")[10]) + marker['p_value'] = float(line.split("\t")[11]) if math.isnan(marker['p_value']) or (marker['p_value'] <= 0): marker['lod_score'] = 0 #marker['lrs_value'] = 0 @@ -245,6 +245,6 @@ def parse_loco_output(this_dataset, gwa_output_filename): marker_obs.append(marker) included_markers.append(line.split("\t")[1]) - p_values.append(float(line.split("\t")[10])) + p_values.append(float(line.split("\t")[11])) return marker_obs \ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js index 92e2b13b..f2e694d8 100644 --- a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js +++ b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js @@ -24,6 +24,7 @@ collection_click = function() { submit_click = function() { var covariates_string = ""; + var covariates_display_string = ""; $('#collections_holder').find('input[type=checkbox]:checked').each(function() { var this_dataset, this_trait; this_trait = $(this).parents('tr').find('.trait').text(); @@ -31,12 +32,15 @@ submit_click = function() { this_dataset = $(this).parents('tr').find('.dataset').text(); console.log("this_dataset is:", this_dataset); covariates_string += this_trait + ":" + this_dataset + "," + covariates_display_string += this_trait + "\n" }); // Trim the last comma covariates_string = covariates_string.substring(0, covariates_string.length - 1) + //covariates_display_string = covariates_display_string.substring(0, covariates_display_string.length - 2) console.log("COVARIATES:", covariates_string) $("input[name=covariates]").val(covariates_string) + $(".selected_covariates").val(covariates_display_string) return $.colorbox.close(); }; @@ -46,6 +50,7 @@ trait_click = function() { trait = $(this).parent().find('.trait').text(); dataset = $(this).parent().find('.dataset').text(); $("input[name=covariates]").val(trait + ":" + dataset) + $(".selected_covariates").text(trait) return $.colorbox.close(); }; diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 099d8010..df10c060 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -88,7 +88,8 @@ }; })(this)); $("#remove_covariates").click(function () { - $("input[name=covariates]").val("") + $("input[name=covariates]").val("") + $(".selected_covariates").val("") }); d3.select("#clear_compare_trait").on("click", (function(_this) { return function() { diff --git a/wqflask/wqflask/templates/collections/list.html b/wqflask/wqflask/templates/collections/list.html index ad72052e..e7f3229b 100644 --- a/wqflask/wqflask/templates/collections/list.html +++ b/wqflask/wqflask/templates/collections/list.html @@ -26,13 +26,13 @@ {% endif %}
-
- -
+
+ +
- +

@@ -51,7 +51,11 @@
{{ loop.index }} {% if g.user_session.user_ob %} {{ uc.name }}
- +{% endif %} {% endblock %} {% block js %} diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html index c5f815ce..ef233333 100644 --- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html +++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html @@ -1,9 +1,10 @@
+
- -
+ +
{% for tissue in corr_tools.dataset_menu %} {% if tissue.tissue %} @@ -37,8 +38,8 @@
- -
+ +
{% for group, pretty_group in sample_group_types.items() %} @@ -64,8 +65,8 @@
- -
+ +
- -
+ +
Chr:     Mb:  to  @@ -89,8 +90,8 @@
- -
+ +
@@ -103,48 +104,45 @@
- -
+ +
- -
- -
- - The Sample Correlation - is computed - between trait data and any - other traits in the sample database selected above. Use - Spearman - Rank - when the sample size is small (<20) or when there are influential outliers. - - - -
-
+
+
+ + The Sample Correlation + is computed + between trait data and any + other traits in the sample database selected above. Use + Spearman + Rank + when the sample size is small (<20) or when there are influential outliers. + + + +
\ No newline at end of file diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index ab2e2aae..03590c2c 100644 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -1,5 +1,5 @@
- {% if (use_pylmm_rqtl and dataset.group.species != "human") or use_plink_gemma %} + {% if dataset.group.mapping_names|length > 0 %}
@@ -41,7 +41,7 @@
{% if genofiles and genofiles|length>0 %}
- +
- -
+ +
@@ -75,7 +75,7 @@
-->
- +
{% if g.user_session.user_ob and (g.user_session.user_ob.display_num_collections() == "") %} No collections available. Please add traits to a collection to use them as covariates. @@ -86,13 +86,13 @@
- + {% endif %}
- -
+ +
@@ -116,7 +116,7 @@
{% if genofiles and genofiles|length>0 %}
- +
- +
- -
+ +
{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != "" %} {% else %} @@ -184,8 +184,8 @@
- -
+ +
- -
+ +
@@ -212,7 +212,7 @@
{% if genofiles and genofiles|length>0 %}
- +
- -
+ +
{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != "" %} {% else %} @@ -248,7 +248,7 @@
- +
@@ -289,8 +289,8 @@
-->
- -
+ +
- -
+ +
@@ -316,7 +316,7 @@
{% if genofiles and genofiles|length>0 %}
- +
{% for item in genofiles %} @@ -354,13 +354,13 @@
{% endif %}
- +
- +
{% if g.user_session.user_ob and (g.user_session.user_ob.display_num_collections() == "") %} No collections available. Please add traits to a collection to use them as covariates. @@ -376,8 +376,8 @@
- -
+ +
@@ -389,16 +389,16 @@