From a9c9de15c395b1e49244c6063c9c1cb204e450da Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 10 Jan 2018 15:05:03 +0300 Subject: Add configuration variables for external services * Add configuration variables for GitHub and ORCID which will be used by the system to allow users to login. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 57f97a81..d2e689a4 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,6 +251,10 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) +GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') +ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) -- cgit v1.2.3 From b4e4ca152256e1f1f06359d584034554db06402e Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 10 Jan 2018 15:07:11 +0300 Subject: Add template elements for OAuth login * Add html elements that will be used to prompt users to login with either GitHub or ORCID. --- wqflask/wqflask/templates/new_security/login_user.html | 18 ++++++++++++++++-- wqflask/wqflask/user_manager.py | 9 ++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index b9f49a61..15f0a27e 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -18,8 +18,22 @@ Create a new account - -
+
+

Login with external services

+ + {% if external_login: %} +
+ {% if external_login["github"]: %} + Login with Github + {% endif %} + + {% if external_login["orcid"]: %} + Login with ORCID + {% endif %} +
+ +
+ {% endif %}

Already have an account? Sign in here.

diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index f7fcd2d0..25833464 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -506,7 +506,14 @@ class LoginUser(object): params = request.form if request.form else request.args logger.debug("in login params are:", params) if not params: - return render_template("new_security/login_user.html") + from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL + external_login = None + if GITHUB_AUTH_URL or ORCID_AUTH_URL: + external_login={ + "github": GITHUB_AUTH_URL, + "orcid": ORCID_AUTH_URL + } + return render_template("new_security/login_user.html", external_login=external_login) else: try: user = model.User.query.filter_by(email_address=params['email_address']).one() -- cgit v1.2.3 From 76299c30c265919fd4025e11017b687c2b63fd82 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:01:11 +0300 Subject: Add client_id and client_secret configurations * Provide the OAuth2 client_id and client_secret values in configuration variables. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index d2e689a4..310b4cea 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,7 +251,11 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) +GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -- cgit v1.2.3 From 3e70cab812e29e504f714c782c71bc4b79793686 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:05:45 +0300 Subject: Add elasticsearch_tools module * Collect variables and functions for using the elasticsearch system in a separate module. --- wqflask/utility/elasticsearch_tools.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 wqflask/utility/elasticsearch_tools.py diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py new file mode 100644 index 00000000..bc7bb240 --- /dev/null +++ b/wqflask/utility/elasticsearch_tools.py @@ -0,0 +1,22 @@ +from elasticsearch import Elasticsearch, TransportError +from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + +es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT +}]) + +def get_user_by_unique_column(column_name, column_value): + user_details = None + try: + response = es.search( + index = "users" + , doc_type = "local" + , body = { + "query": { "match": { column_name: column_value } } + }) + if len(response["hits"]["hits"]) > 0: + user_details = response["hits"]["hits"][0]["_source"] + except TransportError as te: + pass + return user_details -- cgit v1.2.3 From 7004c0ee5e86bfb7ebe491356ca3210d2dc2b67b Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:07:13 +0300 Subject: Add functions to help handle github login * Add functions to help with the github OAuth2 login process --- wqflask/wqflask/user_manager.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 25833464..9012c842 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -494,6 +494,37 @@ def login(): lu = LoginUser() return lu.standard_login() +@app.route("/n/login/github_oauth2", methods=('GET', 'POST')) +def github_oauth2(): + from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET + from utility.elasticsearch_tools import get_user_by_unique_column + import requests + code = request.args.get("code") + data = { + "client_id": GITHUB_CLIENT_ID, + "client_secret": GITHUB_CLIENT_SECRET, + "code": code + } + result = requests.post("https://github.com/login/oauth/access_token", json=data) + result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} + + github_user = get_github_user_details(result_dict["access_token"]) + user_details = get_user_by_unique_column("github_id", github_user["id"]) + if user_details == None: + user_details = { + "user_id": str(uuid4()) + , "name": github_user["name"] + , "github_id": github_user["id"] + , "user_url": github_user["html_url"] + , "login_type": "github" + } + url = "/n/login?type=github" + return redirect(url) + +def get_github_user_details(access_token): + from utility.tools import GITHUB_API_URL + result = requests.get(GITHUB_API_URL, params={"access_token":access_token}) + return result.json() class LoginUser(object): remember_time = 60 * 60 * 24 * 30 # One month in seconds -- cgit v1.2.3 From 63a5c8a42ad02e9126bb207465ff5eca98f6515d Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:36:19 +0300 Subject: Add elasticsearch module to the path * Add some code to set up the path for the python-elasticsearch module. --- bin/genenetwork2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index a7edb1c2..ccd9b1d9 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -61,6 +61,11 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides +if [ -z $ELASTICSEARCH_PROFILE ]; then + echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m"; +else + PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +fi if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) @@ -72,7 +77,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages + export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH" export R_LIBS_SITE=$GN2_PROFILE/site-library export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0 export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript -- cgit v1.2.3 From 7282e0b47c1c6d12fc4c06a080af07c8e67ca75c Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:38:04 +0300 Subject: Add save_user() function * On successful login via OAuth2, save the details of the user in elasticsearch store, to avoid hitting the external provider for the basic details. --- wqflask/utility/elasticsearch_tools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index bc7bb240..74db489b 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -20,3 +20,10 @@ def get_user_by_unique_column(column_name, column_value): except TransportError as te: pass return user_details + +def save_user(user, user_id, index="users", doc_type="local"): + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT + }]) + es.create(index, doc_type, body=user, id=user_id) -- cgit v1.2.3 From 4ae020160a9ae9c399fe338c9415897b26425201 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:41:41 +0300 Subject: Add more configuration variables. * Add configurations for elasticsearch and github. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 310b4cea..df066e67 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -254,11 +254,15 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +GITHUB_API_URL = get_setting('GITHUB_API_URL') ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') +ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) -- cgit v1.2.3 From e185fd3895473e86f2c9fdf174a36b1d325a8c36 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 12:00:04 +0300 Subject: Delay after save for indexing * Elasticsearch need a short delay after adding document for it to index the document for subsequent access. --- wqflask/utility/elasticsearch_tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 74db489b..c2c999ea 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -22,8 +22,10 @@ def get_user_by_unique_column(column_name, column_value): return user_details def save_user(user, user_id, index="users", doc_type="local"): + from time import sleep es = Elasticsearch([{ "host": ELASTICSEARCH_HOST , "port": ELASTICSEARCH_PORT }]) es.create(index, doc_type, body=user, id=user_id) + sleep(1) # Delay 1 second to allow indexing -- cgit v1.2.3 From 2e7335182e55b22e9c61eef111e00f119760d365 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 12:03:10 +0300 Subject: Add code to enable OAuth2 login * Add some code to handle the login if the user chooses to login via GitHub or ORCID. --- wqflask/wqflask/user_manager.py | 58 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 9012c842..daeb7bc5 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -54,6 +54,9 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list +import requests +from utility.elasticsearch_tools import get_user_by_unique_column, save_user + THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -492,13 +495,16 @@ class DecodeUser(object): @app.route("/n/login", methods=('GET', 'POST')) def login(): lu = LoginUser() - return lu.standard_login() + login_type = request.args.get("type") + if login_type: + uid = request.args.get("uid") + return lu.oauth2_login(login_type, uid) + else: + return lu.standard_login() @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) def github_oauth2(): from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET - from utility.elasticsearch_tools import get_user_by_unique_column - import requests code = request.args.get("code") data = { "client_id": GITHUB_CLIENT_ID, @@ -512,13 +518,15 @@ def github_oauth2(): user_details = get_user_by_unique_column("github_id", github_user["id"]) if user_details == None: user_details = { - "user_id": str(uuid4()) + "user_id": str(uuid.uuid4()) , "name": github_user["name"] , "github_id": github_user["id"] , "user_url": github_user["html_url"] , "login_type": "github" + , "organization": "" } - url = "/n/login?type=github" + save_user(user_details, user_details.get("user_id")) + url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) def get_github_user_details(access_token): @@ -532,6 +540,46 @@ class LoginUser(object): def __init__(self): self.remember_me = False + def oauth2_login(self, login_type, user_id): + """Login via an OAuth2 provider""" + user_details = get_user_by_unique_column("user_id", user_id) + if user_details: + user = model.User() + user.id = user_details["user_id"] + user.full_name = user_details["name"] + user.login_type = user_details["login_type"] + return self.actual_login_oauth2(user) + else: + flash("Error logging in via OAuth2") + return make_response(redirect(url_for('login'))) + + def actual_login_oauth2(self, user, assumed_by=None, import_collections=None): + """The meat of the logging in process""" + session_id_signed = self.successful_login_oauth2(user) + flash("Thank you for logging in {}.".format(user.full_name), "alert-success") + print("IMPORT1:", import_collections) + response = make_response(redirect(url_for('index_page', import_collections=import_collections))) + if self.remember_me: + max_age = self.remember_time + else: + max_age = None + response.set_cookie(UserSession.cookie_name, session_id_signed, max_age=max_age) + return response + + def successful_login_oauth2(self, user, assumed_by=None): + login_rec = model.Login(user) + login_rec.successful = True + login_rec.session_id = str(uuid.uuid4()) + login_rec.assumed_by = assumed_by + session_id_signature = actual_hmac_creation(login_rec.session_id) + session_id_signed = login_rec.session_id + ":" + session_id_signature + logger.debug("session_id_signed:", session_id_signed) + + session = dict(login_time = time.time(), + user_id = user.id, + user_login_type = user.login_type) + return session_id_signed + def standard_login(self): """Login through the normal form""" params = request.form if request.form else request.args -- cgit v1.2.3 From d30cbe103946d70b9c61bc48575f23ead1d4048c Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 13:00:51 +0300 Subject: Update configurations * Have the authorisation URLs build up from the client id and client secret values. --- wqflask/utility/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index df066e67..8a32c9b4 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -253,11 +253,11 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') -GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET GITHUB_API_URL = get_setting('GITHUB_API_URL') ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') -ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') +ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -- cgit v1.2.3 From 03127502a982af308e6cd134e218214f6a156cd8 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 13:02:11 +0300 Subject: Add OAuth2 login code for ORCID * Add code to handle the login via ORCID --- wqflask/wqflask/user_manager.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index daeb7bc5..c3f190c3 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -529,6 +529,40 @@ def github_oauth2(): url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) +@app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) +def orcid_oauth2(): + from uuid import uuid4 + from utility.tools import ORCID_CLIENT_ID, ORCID_CLIENT_SECRET, ORCID_TOKEN_URL, ORCID_AUTH_URL + code = request.args.get("code") + error = request.args.get("error") + url = "/n/login" + if code: + data = { + "client_id": ORCID_CLIENT_ID + , "client_secret": ORCID_CLIENT_SECRET + , "grant_type": "authorization_code" + , "code": code + } + result = requests.post(ORCID_TOKEN_URL, data=data) + result_dict = json.loads(result.text.encode("utf-8")) + print("The dict: ", result_dict); + user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) + if user_details == None: + user_details = { + "user_id": str(uuid4()) + , "name": result_dict["name"] + , "orcid": result_dict["orcid"] + , "user_url": "%s/%s" % ( + "/".join(ORCID_AUTH_URL.split("/")[:-2]), + result_dict["orcid"]) + , "login_type": "orcid" + } + save_user(user_details, user_details["user_id"]) + url = "/n/login?type=orcid&uid="+user_details["user_id"] + else: + flash("There was an error getting code from ORCID") + return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, params={"access_token":access_token}) -- cgit v1.2.3 From 309eca58943bf94ce80deb91cf83135957d20980 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 17:48:43 +0300 Subject: Use elasticsearch rather than mysql for local accounts * Register local accounts onto elasticsearch rather than mysql. * Login from the accounts on elasticsearch * Harmonise local and oauth2 logins to use the same code. --- wqflask/wqflask/user_manager.py | 106 ++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 70 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index c3f190c3..4322945b 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -272,15 +272,19 @@ class RegisterUser(object): self.errors = [] self.user = Bunch() - self.user.email_address = kw.get('email_address', '').strip() + self.user.email_address = kw.get('email_address', '').encode("utf-8").strip() if not (5 <= len(self.user.email_address) <= 50): self.errors.append('Email Address needs to be between 5 and 50 characters.') - self.user.full_name = kw.get('full_name', '').strip() + email_exists = get_user_by_unique_column("email_address", self.user.email_address) + if email_exists: + self.errors.append('User already exists with that email') + + self.user.full_name = kw.get('full_name', '').encode("utf-8").strip() if not (5 <= len(self.user.full_name) <= 50): self.errors.append('Full Name needs to be between 5 and 50 characters.') - self.user.organization = kw.get('organization', '').strip() + self.user.organization = kw.get('organization', '').encode("utf-8").strip() if self.user.organization and not (5 <= len(self.user.organization) <= 50): self.errors.append('Organization needs to be empty or between 5 and 50 characters.') @@ -297,28 +301,11 @@ class RegisterUser(object): logger.debug("No errors!") set_password(password, self.user) + self.user.user_id = str(uuid.uuid4()) + self.user.confirmed = 1 self.user.registration_info = json.dumps(basic_info(), sort_keys=True) - - self.new_user = model.User(**self.user.__dict__) - db_session.add(self.new_user) - - try: - db_session.commit() - except sqlalchemy.exc.IntegrityError: - # This exception is thrown if the email address is already in the database - # To do: Perhaps put a link to sign in using an existing account here - self.errors.append("An account with this email address already exists. " - "Click the button above to sign in using an existing account.") - return - - logger.debug("Adding verification email to queue") - #self.send_email_verification() - VerificationEmail(self.new_user) - logger.debug("Added verification email to queue") - - self.thank_you_mode = True - + save_user(self.user.__dict__, self.user.user_id) def set_password(password, user): pwfields = Bunch() @@ -364,7 +351,7 @@ class VerificationEmail(object): verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - data = json.dumps(dict(id=user.id, + data = json.dumps(dict(id=user.user_id, timestamp=timestamp()) ) @@ -519,13 +506,15 @@ def github_oauth2(): if user_details == None: user_details = { "user_id": str(uuid.uuid4()) - , "name": github_user["name"] + , "name": github_user["name"].encode("utf-8") , "github_id": github_user["id"] - , "user_url": github_user["html_url"] + , "user_url": github_user["html_url"].encode("utf-8") , "login_type": "github" , "organization": "" + , "active": 1 + , "confirmed": 1 } - save_user(user_details, user_details.get("user_id")) + save_user(user_details, user_details["user_id"]) url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) @@ -545,7 +534,7 @@ def orcid_oauth2(): } result = requests.post(ORCID_TOKEN_URL, data=data) result_dict = json.loads(result.text.encode("utf-8")) - print("The dict: ", result_dict); + user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) if user_details == None: user_details = { @@ -556,6 +545,9 @@ def orcid_oauth2(): "/".join(ORCID_AUTH_URL.split("/")[:-2]), result_dict["orcid"]) , "login_type": "orcid" + , "organization": "" + , "active": 1 + , "confirmed": 1 } save_user(user_details, user_details["user_id"]) url = "/n/login?type=orcid&uid="+user_details["user_id"] @@ -582,38 +574,11 @@ class LoginUser(object): user.id = user_details["user_id"] user.full_name = user_details["name"] user.login_type = user_details["login_type"] - return self.actual_login_oauth2(user) + return self.actual_login(user) else: flash("Error logging in via OAuth2") return make_response(redirect(url_for('login'))) - def actual_login_oauth2(self, user, assumed_by=None, import_collections=None): - """The meat of the logging in process""" - session_id_signed = self.successful_login_oauth2(user) - flash("Thank you for logging in {}.".format(user.full_name), "alert-success") - print("IMPORT1:", import_collections) - response = make_response(redirect(url_for('index_page', import_collections=import_collections))) - if self.remember_me: - max_age = self.remember_time - else: - max_age = None - response.set_cookie(UserSession.cookie_name, session_id_signed, max_age=max_age) - return response - - def successful_login_oauth2(self, user, assumed_by=None): - login_rec = model.Login(user) - login_rec.successful = True - login_rec.session_id = str(uuid.uuid4()) - login_rec.assumed_by = assumed_by - session_id_signature = actual_hmac_creation(login_rec.session_id) - session_id_signed = login_rec.session_id + ":" + session_id_signature - logger.debug("session_id_signed:", session_id_signed) - - session = dict(login_time = time.time(), - user_id = user.id, - user_login_type = user.login_type) - return session_id_signed - def standard_login(self): """Login through the normal form""" params = request.form if request.form else request.args @@ -628,20 +593,23 @@ class LoginUser(object): } return render_template("new_security/login_user.html", external_login=external_login) else: - try: - user = model.User.query.filter_by(email_address=params['email_address']).one() - except sqlalchemy.orm.exc.NoResultFound: - logger.debug("No account exists for that email address") - valid = False - user = None - else: + user_details = get_user_by_unique_column("email_address", params["email_address"]) + user = None + if user_details: + user = model.User(); + for key in user_details: + user.__dict__[key] = user_details[key] + print("RETRIEVED USER: ", user) + valid = False; + submitted_password = params['password'] pwfields = Struct(json.loads(user.password)) - encrypted = Password(submitted_password, - pwfields.salt, - pwfields.iterations, - pwfields.keylength, - pwfields.hashfunc) + encrypted = Password( + submitted_password, + pwfields.salt, + pwfields.iterations, + pwfields.keylength, + pwfields.hashfunc) logger.debug("\n\nComparing:\n{}\n{}\n".format(encrypted.password, pwfields.password)) valid = pbkdf2.safe_str_cmp(encrypted.password, pwfields.password) logger.debug("valid is:", valid) @@ -709,8 +677,6 @@ class LoginUser(object): else: expire_time = THREE_DAYS Redis.expire(key, expire_time) - db_session.add(login_rec) - db_session.commit() return session_id_signed def unsuccessful_login(self, user): -- cgit v1.2.3 From fb0ad54313366a3f0efda3ef20e800d5e52d1d53 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 30 Jan 2018 12:23:31 +0300 Subject: Set to None if no value provided * Add a method to set the configuration variables to None if the configuration values are not provided at startup or in the configuration files. The system already checks for these values, and if they are absent, it simply fails to display the OAuth service as available for use to login. --- wqflask/utility/tools.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 8a32c9b4..feeeccfc 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,17 +251,28 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) -GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') -GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') -GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET -GITHUB_API_URL = get_setting('GITHUB_API_URL') -ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') -ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') -ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET -ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') - -ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') +def get_setting_safe(setting): + try: + return get_setting(setting) + except: + print("Could not find the setting '", setting, "'. Continuing with value unset") + return None + +GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET') +GITHUB_AUTH_URL = None +if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET: + GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET +GITHUB_API_URL = get_setting_safe('GITHUB_API_URL') +ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET') +ORCID_AUTH_URL = None +if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET: + ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET +ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') + +ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) -- cgit v1.2.3 From afaea1b1297d0cf08565746799d2900a6981823a Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 30 Jan 2018 13:07:58 +0300 Subject: Fail safely if elasticsearch is down or unconfigured * If elasticsearch server is down, or the configuration variables are not provided at startup or in a configuration file, then do not allow the system to simply crash, but instead, inform the user that they cannot use the services that depend on elasticsearch to be running. --- wqflask/utility/elasticsearch_tools.py | 16 ++++++++------ .../wqflask/templates/new_security/login_user.html | 25 ++++++++++++++++++---- wqflask/wqflask/user_manager.py | 6 +++++- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index c2c999ea..8b8ad9cc 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,10 +1,14 @@ -from elasticsearch import Elasticsearch, TransportError -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT +es = None +try: + from elasticsearch import Elasticsearch, TransportError + from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT -es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST - , "port": ELASTICSEARCH_PORT -}]) + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT + }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None +except: + es = None def get_user_by_unique_column(column_name, column_value): user_details = None diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 15f0a27e..0dae3503 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -16,7 +16,14 @@

Don't have an account?

- Create a new account + {% if es_server: %} + Create a new account + {% else: %} +
+

You cannot create an account at this moment.
+ Please try again later.

+
+ {% endif %}

Login with external services

@@ -31,13 +38,17 @@ Login with ORCID {% endif %} - -
+ {% else: %} +
+

You cannot login with external services at this time.
+ Please try again later.

+
{% endif %} +

Already have an account? Sign in here.

- + {% if es_server: %}
@@ -75,6 +86,12 @@
+ {% else: %} +
+

You cannot login at this moment using your GeneNetwork account.
+ Please try again later.

+
+ {% endif %} diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 4322945b..772d6c83 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -585,13 +585,17 @@ class LoginUser(object): logger.debug("in login params are:", params) if not params: from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL + from utility.elasticsearch_tools import es external_login = None if GITHUB_AUTH_URL or ORCID_AUTH_URL: external_login={ "github": GITHUB_AUTH_URL, "orcid": ORCID_AUTH_URL } - return render_template("new_security/login_user.html", external_login=external_login) + return render_template( + "new_security/login_user.html" + , external_login=external_login + , es_server=es) else: user_details = get_user_by_unique_column("email_address", params["email_address"]) user = None -- cgit v1.2.3 From 6ec9dcc365b899f7f4aaab739103f4559c2c1620 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 16:19:13 +0300 Subject: Check each part of the PYTHONPATH * PYTHONPATH may be composed of multiple locations in the filesystem, thus the need to test that each location exists and is a directory. --- bin/genenetwork2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 34fbc72e..145ce395 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -126,7 +126,11 @@ else export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" - if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi + while IFS=":" read -ra PPATH; do + for PPart in "${PPATH[@]}"; do + if [ ! -d $PPart ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi + done + done <<< "$PYTHONPATH" if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi fi -- cgit v1.2.3 From 832a82d4732290fcae033976f6dcb6bab16f61b7 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 17:47:29 +0300 Subject: Add SMTP configuration variables * Add configuration variables to enable the system connect to the configured SMTP server to send out emails. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index feeeccfc..ec673cf5 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -274,6 +274,10 @@ ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') +SMTP_CONNECT = get_setting_safe('SMTP_CONNECT') +SMTP_USERNAME = get_setting_safe('SMTP_USERNAME') +SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) -- cgit v1.2.3 From 2b42e6ec444936b7b3594c5fc07c540a74fe4c05 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 17:55:36 +0300 Subject: Send emails for forgotten passwords * Update the code so that it sends out emails for the "forgot password" feature. --- wqflask/wqflask/user_manager.py | 71 ++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 772d6c83..ec29062d 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -368,23 +368,33 @@ class ForgotPasswordEmail(VerificationEmail): template_name = "email/forgot_password.txt" key_prefix = "forgot_password_code" subject = "GeneNetwork password reset" + fromaddr = "no-reply@genenetwork.org" - def __init__(self, user): + def __init__(self, toaddr): + from email.MIMEMultipart import MIMEMultipart + from email.MIMEText import MIMEText verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - data = json.dumps(dict(id=user.id, - timestamp=timestamp()) - ) + # data = json.dumps(dict(id=user.id, + # timestamp=timestamp()) + # ) + + # Redis.set(key, data) + # Redis.expire(key, THREE_DAYS) - Redis.set(key, data) - #two_days = 60 * 60 * 24 * 2 - Redis.expire(key, THREE_DAYS) - to = user.email_address subject = self.subject - body = render_template(self.template_name, - verification_code = verification_code) - send_email(to, subject, body) + body = render_template( + self.template_name, + verification_code = verification_code) + + msg = MIMEMultipart() + msg["To"] = toaddr + msg["Subject"] = self.subject + msg["From"] = self.fromaddr + msg.attach(MIMEText(body, "plain")) + + send_email(toaddr, msg.as_string()) class Password(object): @@ -708,13 +718,16 @@ def forgot_password(): def forgot_password_submit(): params = request.form email_address = params['email_address'] - try: - user = model.User.query.filter_by(email_address=email_address).one() - except orm.exc.NoResultFound: - flash("Couldn't find a user associated with the email address {}. Sorry.".format( - email_address)) - return redirect(url_for("login")) - ForgotPasswordEmail(user) + user_details = get_user_by_unique_column("email_address", email_address) + if user_details: + ForgotPasswordEmail(user_details["email_address"]) + # try: + # user = model.User.query.filter_by(email_address=email_address).one() + # except orm.exc.NoResultFound: + # flash("Couldn't find a user associated with the email address {}. Sorry.".format( + # email_address)) + # return redirect(url_for("login")) + # ForgotPasswordEmail(user) return render_template("new_security/forgot_password_step2.html", subject=ForgotPasswordEmail.subject) @@ -861,13 +874,21 @@ app.jinja_env.globals.update(url_for_hmac=url_for_hmac, ####################################################################################### -def send_email(to, subject, body): - msg = json.dumps(dict(From="no-reply@genenetwork.org", - To=to, - Subject=subject, - Body=body)) - Redis.rpush("mail_queue", msg) - +# def send_email(to, subject, body): +# msg = json.dumps(dict(From="no-reply@genenetwork.org", +# To=to, +# Subject=subject, +# Body=body)) +# Redis.rpush("mail_queue", msg) + +def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): + from smtplib import SMTP + from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD + server = SMTP(SMTP_CONNECT) + server.starttls() + server.login(SMTP_USERNAME, SMTP_PASSWORD) + server.sendmail(fromaddr, toaddr, msg) + server.quit() -- cgit v1.2.3 From 1c9540879d8761d9252c3fb3f749ae0b6d5be2b9 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 20:01:45 +0300 Subject: Refactor common items to more generic methods. * Refactor code that can be used in more than one place to a more generic method/function that's called by other methods --- wqflask/utility/elasticsearch_tools.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 8b8ad9cc..ea636b2e 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -11,25 +11,27 @@ except: es = None def get_user_by_unique_column(column_name, column_value): - user_details = None + return get_item_by_unique_column(column_name, column_value, index="users", doc_type="local") + +def save_user(user, user_id): + es_save_data("users", "local", user, user_id) + +def get_item_by_unique_column(column_name, column_value, index, doc_type): + item_details = None try: response = es.search( - index = "users" - , doc_type = "local" + index = index + , doc_type = doc_type , body = { "query": { "match": { column_name: column_value } } }) if len(response["hits"]["hits"]) > 0: - user_details = response["hits"]["hits"][0]["_source"] + item_details = response["hits"]["hits"][0]["_source"] except TransportError as te: pass - return user_details + return item_details -def save_user(user, user_id, index="users", doc_type="local"): +def es_save_data(index, doc_type, data_item, data_id,): from time import sleep - es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST - , "port": ELASTICSEARCH_PORT - }]) - es.create(index, doc_type, body=user, id=user_id) + es.create(index, doc_type, body=data_item, id=data_id) sleep(1) # Delay 1 second to allow indexing -- cgit v1.2.3 From 03edecad79c4e8e1e10734e9e8f21f5da7912851 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 20:04:09 +0300 Subject: Add code to allow user to change password * After the email is sent to the user, there is need to provide a way for the user to actually change their password, and have the results saved. --- wqflask/wqflask/user_manager.py | 65 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index ec29062d..8f09c206 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -55,7 +55,7 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list import requests -from utility.elasticsearch_tools import get_user_by_unique_column, save_user +from utility.elasticsearch_tools import get_user_by_unique_column, save_user, es_save_data THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -376,12 +376,12 @@ class ForgotPasswordEmail(VerificationEmail): verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - # data = json.dumps(dict(id=user.id, - # timestamp=timestamp()) - # ) - - # Redis.set(key, data) - # Redis.expire(key, THREE_DAYS) + data = { + "verification_code": verification_code, + "email_address": toaddr, + "timestamp": timestamp() + } + es_save_data(self.key_prefix, "local", data, verification_code) subject = self.subject body = render_template( @@ -429,38 +429,59 @@ def verify_email(): response.set_cookie(UserSession.cookie_name, session_id_signed) return response -@app.route("/n/password_reset") +@app.route("/n/password_reset", methods=['GET']) def password_reset(): + from utility.elasticsearch_tools import get_item_by_unique_column logger.debug("in password_reset request.url is:", request.url) # We do this mainly just to assert that it's in proper form for displaying next page # Really not necessary but doesn't hurt - user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() - - return render_template("new_security/password_reset.html", user_encode=user_encode) + # user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() + verification_code = request.args.get('code') + hmac = request.args.get('hm') + if verification_code: + code_details = get_item_by_unique_column( + "verification_code", + verification_code, + ForgotPasswordEmail.key_prefix, + "local") + if code_details: + user_details = get_user_by_unique_column( + "email_address", + code_details["email_address"]) + if user_details: + return render_template( + "new_security/password_reset.html", user_encode=user_details["user_id"]) + else: + flash("Invalid code: User no longer exists!", "error") + else: + flash("Invalid code: Password reset code does not exist or might have expired!", "error") + return redirect(url_for("login"))#render_template("new_security/login_user.html", error=error) @app.route("/n/password_reset_step2", methods=('POST',)) def password_reset_step2(): + from utility.elasticsearch_tools import es logger.debug("in password_reset request.url is:", request.url) errors = [] + user_id = request.form['user_encode'] - user_encode = request.form['user_encode'] - verification_code, separator, hmac = user_encode.partition(':') - - hmac_verified = actual_hmac_creation(verification_code) logger.debug("locals are:", locals()) - assert hmac == hmac_verified, "Someone has been naughty" - - user = DecodeUser.actual_get_user(ForgotPasswordEmail.key_prefix, verification_code) - logger.debug("user is:", user) - + user = Bunch() password = request.form['password'] - set_password(password, user) - db_session.commit() + + es.update( + index = "users" + , doc_type = "local" + , id = user_id + , body = { + "doc": { + "password": user.__dict__.get("password") + } + }) flash("Password changed successfully. You can now sign in.", "alert-info") response = make_response(redirect(url_for('login'))) -- cgit v1.2.3 From 25ff5e9e47c568547a855e42a4fd43477db2a61e Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Feb 2018 10:11:21 +0300 Subject: Add check for elasticsearch * Add some extra checks to ensure that elasticsearch is running before presenting the UI to the user. --- wqflask/utility/elasticsearch_tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index ea636b2e..a0383033 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -7,6 +7,10 @@ try: "host": ELASTICSEARCH_HOST , "port": ELASTICSEARCH_PORT }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + + # Check if elasticsearch is running + if not es.ping(): + es = None except: es = None -- cgit v1.2.3 From 4fe60697964d81ce1425000e764fd0a27d73eb29 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Feb 2018 11:02:07 +0300 Subject: Check elasticsearch at point of use * Instead of checking for the state of elasticsearch at startup, check the state at the moment the user requests a feature that depends on elasticsearch. This reduces the chances that the user is dropped onto an exception page when elasticsearch server goes down. --- wqflask/utility/elasticsearch_tools.py | 3 --- wqflask/wqflask/user_manager.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index a0383033..4fc0035c 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -8,9 +8,6 @@ try: , "port": ELASTICSEARCH_PORT }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - # Check if elasticsearch is running - if not es.ping(): - es = None except: es = None diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 8f09c206..630be9aa 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -626,7 +626,7 @@ class LoginUser(object): return render_template( "new_security/login_user.html" , external_login=external_login - , es_server=es) + , es_server=es.ping()) else: user_details = get_user_by_unique_column("email_address", params["email_address"]) user = None -- cgit v1.2.3 From 1d45fac66b19340ed9378fcc08a928555ad95f74 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 13 Feb 2018 13:48:28 +0300 Subject: Update module to make it more testable * Update functions to make them more testable. * Update code using updated functions. --- wqflask/utility/elasticsearch_tools.py | 40 +++++++++++++--------- wqflask/wqflask/user_manager.py | 61 +++++++++++++++++++--------------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 4fc0035c..a964b025 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,23 +1,31 @@ -es = None -try: - from elasticsearch import Elasticsearch, TransportError - from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT +from elasticsearch import Elasticsearch, TransportError +import logging - es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST - , "port": ELASTICSEARCH_PORT - }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - -except: +def get_elasticsearch_connection(): es = None + try: + from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT + }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + + es_logger = logging.getLogger("elasticsearch") + es_logger.setLevel(logging.INFO) + es_logger.addHandler(logging.NullHandler()) + except: + es = None + + return es -def get_user_by_unique_column(column_name, column_value): - return get_item_by_unique_column(column_name, column_value, index="users", doc_type="local") +def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): + return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) -def save_user(user, user_id): - es_save_data("users", "local", user, user_id) +def save_user(es, user, user_id): + es_save_data(es, "users", "local", user, user_id) -def get_item_by_unique_column(column_name, column_value, index, doc_type): +def get_item_by_unique_column(es, column_name, column_value, index, doc_type): item_details = None try: response = es.search( @@ -32,7 +40,7 @@ def get_item_by_unique_column(column_name, column_value, index, doc_type): pass return item_details -def es_save_data(index, doc_type, data_item, data_id,): +def es_save_data(es, index, doc_type, data_item, data_id,): from time import sleep es.create(index, doc_type, body=data_item, id=data_id) sleep(1) # Delay 1 second to allow indexing diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 630be9aa..6b667615 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -55,8 +55,9 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list import requests -from utility.elasticsearch_tools import get_user_by_unique_column, save_user, es_save_data +from utility.elasticsearch_tools import * +es = get_elasticsearch_connection() THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -271,14 +272,18 @@ class RegisterUser(object): self.thank_you_mode = False self.errors = [] self.user = Bunch() + es = kw.get('es_connection', None) + + if not es: + self.errors.append("Missing connection object") self.user.email_address = kw.get('email_address', '').encode("utf-8").strip() if not (5 <= len(self.user.email_address) <= 50): self.errors.append('Email Address needs to be between 5 and 50 characters.') - - email_exists = get_user_by_unique_column("email_address", self.user.email_address) - if email_exists: - self.errors.append('User already exists with that email') + else: + email_exists = get_user_by_unique_column(es, "email_address", self.user.email_address) + if email_exists: + self.errors.append('User already exists with that email') self.user.full_name = kw.get('full_name', '').encode("utf-8").strip() if not (5 <= len(self.user.full_name) <= 50): @@ -305,7 +310,7 @@ class RegisterUser(object): self.user.confirmed = 1 self.user.registration_info = json.dumps(basic_info(), sort_keys=True) - save_user(self.user.__dict__, self.user.user_id) + save_user(es, self.user.__dict__, self.user.user_id) def set_password(password, user): pwfields = Bunch() @@ -381,7 +386,7 @@ class ForgotPasswordEmail(VerificationEmail): "email_address": toaddr, "timestamp": timestamp() } - es_save_data(self.key_prefix, "local", data, verification_code) + es_save_data(es, self.key_prefix, "local", data, verification_code) subject = self.subject body = render_template( @@ -431,7 +436,6 @@ def verify_email(): @app.route("/n/password_reset", methods=['GET']) def password_reset(): - from utility.elasticsearch_tools import get_item_by_unique_column logger.debug("in password_reset request.url is:", request.url) # We do this mainly just to assert that it's in proper form for displaying next page @@ -441,14 +445,16 @@ def password_reset(): hmac = request.args.get('hm') if verification_code: code_details = get_item_by_unique_column( - "verification_code", - verification_code, - ForgotPasswordEmail.key_prefix, - "local") + es + , "verification_code" + , verification_code + , ForgotPasswordEmail.key_prefix + , "local") if code_details: user_details = get_user_by_unique_column( - "email_address", - code_details["email_address"]) + es + , "email_address" + , code_details["email_address"]) if user_details: return render_template( "new_security/password_reset.html", user_encode=user_details["user_id"]) @@ -533,7 +539,7 @@ def github_oauth2(): result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} github_user = get_github_user_details(result_dict["access_token"]) - user_details = get_user_by_unique_column("github_id", github_user["id"]) + user_details = get_user_by_unique_column(es, "github_id", github_user["id"]) if user_details == None: user_details = { "user_id": str(uuid.uuid4()) @@ -545,7 +551,7 @@ def github_oauth2(): , "active": 1 , "confirmed": 1 } - save_user(user_details, user_details["user_id"]) + save_user(es, user_details, user_details["user_id"]) url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) @@ -566,7 +572,7 @@ def orcid_oauth2(): result = requests.post(ORCID_TOKEN_URL, data=data) result_dict = json.loads(result.text.encode("utf-8")) - user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) + user_details = get_user_by_unique_column(es, "orcid", result_dict["orcid"]) if user_details == None: user_details = { "user_id": str(uuid4()) @@ -580,7 +586,7 @@ def orcid_oauth2(): , "active": 1 , "confirmed": 1 } - save_user(user_details, user_details["user_id"]) + save_user(es, user_details, user_details["user_id"]) url = "/n/login?type=orcid&uid="+user_details["user_id"] else: flash("There was an error getting code from ORCID") @@ -599,7 +605,7 @@ class LoginUser(object): def oauth2_login(self, login_type, user_id): """Login via an OAuth2 provider""" - user_details = get_user_by_unique_column("user_id", user_id) + user_details = get_user_by_unique_column(es, "user_id", user_id) if user_details: user = model.User() user.id = user_details["user_id"] @@ -616,7 +622,6 @@ class LoginUser(object): logger.debug("in login params are:", params) if not params: from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL - from utility.elasticsearch_tools import es external_login = None if GITHUB_AUTH_URL or ORCID_AUTH_URL: external_login={ @@ -628,8 +633,9 @@ class LoginUser(object): , external_login=external_login , es_server=es.ping()) else: - user_details = get_user_by_unique_column("email_address", params["email_address"]) + user_details = get_user_by_unique_column(es, "email_address", params["email_address"]) user = None + valid = None if user_details: user = model.User(); for key in user_details: @@ -672,7 +678,7 @@ class LoginUser(object): else: if user: self.unsuccessful_login(user) - flash("Invalid email-address or password. Please try again.", "alert-error") + flash("Invalid email-address or password. Please try again.", "alert-danger") response = make_response(redirect(url_for('login'))) return response @@ -739,7 +745,7 @@ def forgot_password(): def forgot_password_submit(): params = request.form email_address = params['email_address'] - user_details = get_user_by_unique_column("email_address", email_address) + user_details = get_user_by_unique_column(es, "email_address", email_address) if user_details: ForgotPasswordEmail(user_details["email_address"]) # try: @@ -815,16 +821,17 @@ def register(): params = request.form if request.form else request.args + params = params.to_dict(flat=True) + params["es_connection"] = es if params: logger.debug("Attempting to register the user...") result = RegisterUser(params) errors = result.errors - if result.thank_you_mode: - assert not errors, "Errors while in thank you mode? That seems wrong..." - return render_template("new_security/registered.html", - subject=VerificationEmail.subject) + if len(errors) == 0: + flash("Registration successful. You may login with your new account", "alert-info") + return redirect(url_for("login")) return render_template("new_security/register_user.html", values=params, errors=errors) -- cgit v1.2.3 From 1defefd05d0eb658fb5922fc755547261a5e914a Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 13 Feb 2018 18:32:18 +0300 Subject: Add tests for Registration process. --- wqflask/tests/__init__.py | 0 wqflask/tests/es_double.py | 30 ++++++++++ wqflask/tests/test_registration.py | 113 +++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 wqflask/tests/__init__.py create mode 100644 wqflask/tests/es_double.py create mode 100644 wqflask/tests/test_registration.py diff --git a/wqflask/tests/__init__.py b/wqflask/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wqflask/tests/es_double.py b/wqflask/tests/es_double.py new file mode 100644 index 00000000..00739016 --- /dev/null +++ b/wqflask/tests/es_double.py @@ -0,0 +1,30 @@ +class ESDouble(object): + def __init__(self): + self.items = { + "users": { + "local": [] + }} + + def ping(self): + return true + + def create(self, index, doc_type, body, id): + item = {"id": id, "_source": body} + if not self.items.get("index", None): + self.items[index] = {doc_type: [item]} + else: + self.items[index][doc_type].append(item) + + def search(self, index, doc_type, body): + d = body["query"]["match"] + column = [(key, d[key]) for key in d] + + items = [] + for thing in self.items[index][doc_type]: + if thing["_source"][column[0][0]] == column[0][1]: + items.append(thing) + break + return { + "hits": { + "hits": items + }} diff --git a/wqflask/tests/test_registration.py b/wqflask/tests/test_registration.py new file mode 100644 index 00000000..50a2a84c --- /dev/null +++ b/wqflask/tests/test_registration.py @@ -0,0 +1,113 @@ +import unittest +import es_double +import wqflask.user_manager +from wqflask.user_manager import RegisterUser + +class TestRegisterUser(unittest.TestCase): + def setUp(self): + # Mock elasticsearch + self.es = es_double.ESDouble() + + # Patch method + wqflask.user_manager.basic_info = lambda : {"basic_info": "some info"} + + def tearDown(self): + self.es = None + + def testRegisterUserWithNoData(self): + data = {} + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Data was not provided. Error was expected") + + def testRegisterUserWithNoEmail(self): + data = { + "email_address": "" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Email not provided. Error was expected") + + def testRegisterUserWithNoName(self): + data = { + "email_address": "user@example.com" + , "full_name": "" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Name not provided. Error was expected") + + def testRegisterUserWithNoOrganisation(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertEqual(len(result.errors), 0, "Organisation not provided. Error not expected") + + def testRegisterUserWithShortOrganisation(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "SO" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Organisation name too short. Error expected") + + def testRegisterUserWithNoPassword(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": None + , "password_confirm": None + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Password not provided. Error was expected") + + def testRegisterUserWithNonMatchingPasswords(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "stilltesting" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Password mismatch. Error was expected") + + def testRegisterUserWithCorrectData(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + result = RegisterUser(data) + self.assertEqual(len(result.errors), 0, "All data items provided. Errors were not expected") + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3 From 9d47ad572f46397bdd778d8f6e1c37d40e12aa2a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 15 Feb 2018 10:30:52 +0000 Subject: Fixing authentication stuff so it uses parameters properly. Also no PYTHONPATH needed as it is now in the Guix build. --- bin/genenetwork2 | 12 +++++++----- etc/default_settings.py | 18 ++++++++++++++++++ wqflask/run_gunicorn.py | 3 +++ wqflask/utility/elasticsearch_tools.py | 16 ++++++++++++---- wqflask/utility/tools.py | 29 +++++++++++------------------ wqflask/wqflask/user_manager.py | 7 +++++-- 6 files changed, 56 insertions(+), 29 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5438c1c0..42f79650 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -95,11 +95,13 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides -if [ -z $ELASTICSEARCH_PROFILE ]; then - echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m"; -else - PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" -fi +# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead +# if [ -z $ELASTICSEARCH_PROFILE ]; then +# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; +# else +# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +# fi + if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) diff --git a/etc/default_settings.py b/etc/default_settings.py index 699d21f1..a70d8aec 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -41,6 +41,24 @@ SECURITY_POST_LOGIN_VIEW = "/thank_you" SERVER_PORT = 5003 # running on localhost SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' +GITHUB_CLIENT_ID = "UNKNOWN" +GITHUB_CLIENT_SECRET = "UNKNOWN" +GITHUB_AUTH_URL = "UNKNOWN" +GITHUB_API_URL = "UNKNOWN" + +ORCID_CLIENT_ID = "UNKNOWN" +ORCID_CLIENT_SECRET = "UNKNOWN" +ORCID_AUTH_URL = "UNKNOWN" +ORCID_TOKEN_URL = "UNKNOWN" + +ELASTICSEARCH_HOST = "localhost" +ELASTICSEARCH_PORT = '9200' + +SMTP_CONNECT = "UNKNOWN" +SMTP_USERNAME = "UNKNOWN" +SMTP_PASSWORD = "UNKNOWN" + + # ---- Behavioural settings (defaults) note that logger and log levels can # be overridden at the module level and with enviroment settings WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index 14a2d689..ebe3add5 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -11,6 +11,9 @@ print "Starting up Gunicorn process" from wqflask import app +app.config['SESSION_TYPE'] = 'filesystem' +app.config['SECRET_KEY'] = 'super secret key' + @app.route("/gunicorn") def hello(): return "

Hello There!

" diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index a964b025..2d3d5add 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,10 +1,18 @@ from elasticsearch import Elasticsearch, TransportError import logging +from utility.logger import getLogger +logger = getLogger(__name__) + +from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + def get_elasticsearch_connection(): + logger.info("get_elasticsearch_connection") es = None try: - from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + assert(ELASTICSEARCH_HOST) + assert(ELASTICSEARCH_PORT) + logger.info("ES HOST",ELASTICSEARCH_HOST) es = Elasticsearch([{ "host": ELASTICSEARCH_HOST @@ -31,12 +39,12 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type): response = es.search( index = index , doc_type = doc_type - , body = { - "query": { "match": { column_name: column_value } } + , body = { + "query": { "match": { column_name: column_value } } }) if len(response["hits"]["hits"]) > 0: item_details = response["hits"]["hits"][0]["_source"] - except TransportError as te: + except TransportError as te: pass return item_details diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 005f9b0f..8c9fed96 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,32 +251,25 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) -def get_setting_safe(setting): - try: - return get_setting(setting) - except: - print("Could not find the setting '", setting, "'. Continuing with value unset") - return None - -GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID') -GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET') +GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = None if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET: GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET -GITHUB_API_URL = get_setting_safe('GITHUB_API_URL') -ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID') -ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET') +GITHUB_API_URL = get_setting('GITHUB_API_URL') +ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = None if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET: ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET -ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') +ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') +ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') -SMTP_CONNECT = get_setting_safe('SMTP_CONNECT') -SMTP_USERNAME = get_setting_safe('SMTP_USERNAME') -SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD') +SMTP_CONNECT = get_setting('SMTP_CONNECT') +SMTP_USERNAME = get_setting('SMTP_USERNAME') +SMTP_PASSWORD = get_setting('SMTP_PASSWORD') PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 6b667615..c8471cb1 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -55,9 +55,8 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list import requests -from utility.elasticsearch_tools import * +from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column, save_user -es = get_elasticsearch_connection() THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -479,6 +478,7 @@ def password_reset_step2(): password = request.form['password'] set_password(password, user) + es = get_elasticsearch_connection() es.update( index = "users" , doc_type = "local" @@ -620,6 +620,7 @@ class LoginUser(object): """Login through the normal form""" params = request.form if request.form else request.args logger.debug("in login params are:", params) + es = get_elasticsearch_connection() if not params: from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL external_login = None @@ -628,6 +629,7 @@ class LoginUser(object): "github": GITHUB_AUTH_URL, "orcid": ORCID_AUTH_URL } + assert(es is not None) return render_template( "new_security/login_user.html" , external_login=external_login @@ -822,6 +824,7 @@ def register(): params = request.form if request.form else request.args params = params.to_dict(flat=True) + es = get_elasticsearch_connection() params["es_connection"] = es if params: -- cgit v1.2.3 From 4097723cfd6331f41d8b8e7abec322cb12f39f10 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 15 Feb 2018 14:03:38 +0000 Subject: Added unittest example --- test/unittest/test_registration.py | 27 +++++++++ wqflask/mock/__init__.py | 0 wqflask/mock/es_double.py | 15 +++++ wqflask/tests/__init__.py | 0 wqflask/tests/es_double.py | 30 ---------- wqflask/tests/test_registration.py | 113 ------------------------------------- 6 files changed, 42 insertions(+), 143 deletions(-) create mode 100644 test/unittest/test_registration.py create mode 100644 wqflask/mock/__init__.py create mode 100644 wqflask/mock/es_double.py delete mode 100644 wqflask/tests/__init__.py delete mode 100644 wqflask/tests/es_double.py delete mode 100644 wqflask/tests/test_registration.py diff --git a/test/unittest/test_registration.py b/test/unittest/test_registration.py new file mode 100644 index 00000000..98d0cdff --- /dev/null +++ b/test/unittest/test_registration.py @@ -0,0 +1,27 @@ +# Run test with something like +# +# env GN2_PROFILE=~/opt/gn-latest GENENETWORK_FILES=$HOME/gn2_data ./bin/genenetwork2 ./etc/default_settings.py -c ../test/unittest/test_registration.py +# + +import unittest +import mock.es_double as es +from wqflask.user_manager import RegisterUser + +class TestRegisterUser(unittest.TestCase): + def setUp(self): + self.es = es.ESDouble() + + def testRegisterUserWithCorrectData(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + result = RegisterUser(data) + self.assertEqual(len(result.errors), 0, "Errors were not expected") + +if __name__ == "__main__": + unittest.main() diff --git a/wqflask/mock/__init__.py b/wqflask/mock/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wqflask/mock/es_double.py b/wqflask/mock/es_double.py new file mode 100644 index 00000000..6ef8a1b9 --- /dev/null +++ b/wqflask/mock/es_double.py @@ -0,0 +1,15 @@ +class ESDouble(object): + def __init__(self): + self.items = {} + + def ping(self): + return true + + def create(self, index, doc_type, body, id): + self.items["index"] = {doc_type: {"id": id, "_source": data}} + + def search(self, index, doc_type, body): + return { + "hits": { + "hits": self.items[index][doc_type][body] + }} diff --git a/wqflask/tests/__init__.py b/wqflask/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/wqflask/tests/es_double.py b/wqflask/tests/es_double.py deleted file mode 100644 index 00739016..00000000 --- a/wqflask/tests/es_double.py +++ /dev/null @@ -1,30 +0,0 @@ -class ESDouble(object): - def __init__(self): - self.items = { - "users": { - "local": [] - }} - - def ping(self): - return true - - def create(self, index, doc_type, body, id): - item = {"id": id, "_source": body} - if not self.items.get("index", None): - self.items[index] = {doc_type: [item]} - else: - self.items[index][doc_type].append(item) - - def search(self, index, doc_type, body): - d = body["query"]["match"] - column = [(key, d[key]) for key in d] - - items = [] - for thing in self.items[index][doc_type]: - if thing["_source"][column[0][0]] == column[0][1]: - items.append(thing) - break - return { - "hits": { - "hits": items - }} diff --git a/wqflask/tests/test_registration.py b/wqflask/tests/test_registration.py deleted file mode 100644 index 50a2a84c..00000000 --- a/wqflask/tests/test_registration.py +++ /dev/null @@ -1,113 +0,0 @@ -import unittest -import es_double -import wqflask.user_manager -from wqflask.user_manager import RegisterUser - -class TestRegisterUser(unittest.TestCase): - def setUp(self): - # Mock elasticsearch - self.es = es_double.ESDouble() - - # Patch method - wqflask.user_manager.basic_info = lambda : {"basic_info": "some info"} - - def tearDown(self): - self.es = None - - def testRegisterUserWithNoData(self): - data = {} - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Data was not provided. Error was expected") - - def testRegisterUserWithNoEmail(self): - data = { - "email_address": "" - , "full_name": "A.N. Other" - , "organization": "Some Organisation" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Email not provided. Error was expected") - - def testRegisterUserWithNoName(self): - data = { - "email_address": "user@example.com" - , "full_name": "" - , "organization": "Some Organisation" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Name not provided. Error was expected") - - def testRegisterUserWithNoOrganisation(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertEqual(len(result.errors), 0, "Organisation not provided. Error not expected") - - def testRegisterUserWithShortOrganisation(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "SO" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Organisation name too short. Error expected") - - def testRegisterUserWithNoPassword(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "Some Organisation" - , "password": None - , "password_confirm": None - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Password not provided. Error was expected") - - def testRegisterUserWithNonMatchingPasswords(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "Some Organisation" - , "password": "testing" - , "password_confirm": "stilltesting" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Password mismatch. Error was expected") - - def testRegisterUserWithCorrectData(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "Some Organisation" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - result = RegisterUser(data) - self.assertEqual(len(result.errors), 0, "All data items provided. Errors were not expected") - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3 From 3a26e0fb7b88cd9c105a1f7e9ca9d5a8b2130d82 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 15 Feb 2018 15:00:47 +0000 Subject: Requests added --- bin/genenetwork2 | 10 ++-- bin/mechnical-rob | 111 ++++++++++++++++++++++++++++++++++++++++ bin/test-website | 114 ++---------------------------------------- test/requests/test-website.py | 20 ++++++++ 4 files changed, 142 insertions(+), 113 deletions(-) create mode 100755 bin/mechnical-rob create mode 100755 test/requests/test-website.py diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 42f79650..2a83a1cd 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -168,8 +168,9 @@ if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} echo PYTHONPATH=$PYTHONPATH - echo RUNNING COMMAND $cmd - python $cmd + shift ; shift + echo RUNNING COMMAND $cmd $* + python $cmd $* exit $? fi # Now handle command parameter -cli which runs in bash @@ -178,8 +179,9 @@ if [ "$1" = "-cli" ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 echo PYTHONPATH=$PYTHONPATH - echo RUNNING COMMAND $cmd - $cmd + shift ; shift + echo RUNNING COMMAND $cmd $* + $cmd $* exit $? fi if [ "$1" = '-gunicorn' ] ; then diff --git a/bin/mechnical-rob b/bin/mechnical-rob new file mode 100755 index 00000000..be223d94 --- /dev/null +++ b/bin/mechnical-rob @@ -0,0 +1,111 @@ +#!/usr/bin/env ruby + + +USAGE = <0 + ARGV.shift + else + "http://localhost:5003" + end + +$stderr.print "Testing <",$host,">\n" + +require 'mechanize' +require 'minitest/spec' +require 'minitest/autorun' + +# These are the actual testing modules + +libpath = File.dirname(File.dirname(__FILE__)) +$: << File.join(libpath,'test/lib') + +require 'main_web_functionality' + +if options[:all] or options[:mapping] + require 'mapping' +end + +if options[:all] or options[:link_checker] + require 'link_checker' +end + +if options[:all] or options[:navigation] + require 'navigation' +end diff --git a/bin/test-website b/bin/test-website index be223d94..5935f016 100755 --- a/bin/test-website +++ b/bin/test-website @@ -1,111 +1,7 @@ -#!/usr/bin/env ruby +#! /bin/bash - -USAGE = <0 - ARGV.shift - else - "http://localhost:5003" - end - -$stderr.print "Testing <",$host,">\n" - -require 'mechanize' -require 'minitest/spec' -require 'minitest/autorun' - -# These are the actual testing modules - -libpath = File.dirname(File.dirname(__FILE__)) -$: << File.join(libpath,'test/lib') - -require 'main_web_functionality' - -if options[:all] or options[:mapping] - require 'mapping' -end - -if options[:all] or options[:link_checker] - require 'link_checker' -end - -if options[:all] or options[:navigation] - require 'navigation' -end +fi diff --git a/test/requests/test-website.py b/test/requests/test-website.py new file mode 100755 index 00000000..d02b71aa --- /dev/null +++ b/test/requests/test-website.py @@ -0,0 +1,20 @@ +# Run with something like +# +# env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 +# +# Mostly to pick up the Guix GN2_PROFILE and python modules + +import requests as req +import sys + +print "Mechanical Rob firing up..." + +if len(sys.argv)<1: + raise "Problem with arguments" + +url = sys.argv[1] +print url + +r = req.get(url) + +print r -- cgit v1.2.3 From 2ac52ea39a7a8db6cab756e3af2f65b228bb1c09 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 16 Feb 2018 14:22:17 +0300 Subject: Add registration test * Add integration test to check the registration process. --- test/requests/test-registration.py | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/requests/test-registration.py diff --git a/test/requests/test-registration.py b/test/requests/test-registration.py new file mode 100644 index 00000000..bcb1642f --- /dev/null +++ b/test/requests/test-registration.py @@ -0,0 +1,59 @@ +import sys +import unittest +import requests +import logging +from elasticsearch import Elasticsearch, TransportError +#from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + +GN2_SERVER = None +ES_SERVER = None + +class TestRegistration(unittest.TestCase): + + + def setUp(self): + self.url = GN2_SERVER+"/n/register" + self.es = Elasticsearch([ES_SERVER]) + self.es_cleanup = [] + + es_logger = logging.getLogger("elasticsearch") + es_logger.addHandler( + logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) + es_trace_logger = logging.getLogger("elasticsearch.trace") + es_trace_logger.addHandler( + logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) + + def tearDown(self): + for item in self.es_cleanup: + self.es.delete(index="users", doc_type="local", id=item["_id"]) + + def testRegistrationPage(self): + if self.es.ping(): + data = { + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + requests.post(self.url, data) + response = self.es.search( + index="users" + , doc_type="local" + , body={ + "query": {"match": {"email_address": "test@user.com"}}}) + self.assertEqual(len(response["hits"]["hits"]), 1) + self.es_cleanup.append(response["hits"]["hits"][0]) + else: + self.skipTest("The elasticsearch server is down") + +def main(): + suite = unittest.TestSuite() + suite.addTest(TestRegistration("testRegistrationPage")) + runner = unittest.TextTestRunner() + runner.run(suite) + +if __name__ == "__main__": + GN2_SERVER = sys.argv[1] + ES_SERVER = sys.argv[2] + main() -- cgit v1.2.3 From 5dd7e61c43bac52ae52735c38c337a5a198c6c39 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 12:46:47 +0300 Subject: Rename file and move common code * Rename the file to make it an importable module * Refactor the test to move common code out to a super class. --- test/requests/test-registration.py | 59 -------------------------------------- test/requests/test_registration.py | 41 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 59 deletions(-) delete mode 100644 test/requests/test-registration.py create mode 100644 test/requests/test_registration.py diff --git a/test/requests/test-registration.py b/test/requests/test-registration.py deleted file mode 100644 index bcb1642f..00000000 --- a/test/requests/test-registration.py +++ /dev/null @@ -1,59 +0,0 @@ -import sys -import unittest -import requests -import logging -from elasticsearch import Elasticsearch, TransportError -#from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT - -GN2_SERVER = None -ES_SERVER = None - -class TestRegistration(unittest.TestCase): - - - def setUp(self): - self.url = GN2_SERVER+"/n/register" - self.es = Elasticsearch([ES_SERVER]) - self.es_cleanup = [] - - es_logger = logging.getLogger("elasticsearch") - es_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) - es_trace_logger = logging.getLogger("elasticsearch.trace") - es_trace_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) - - def tearDown(self): - for item in self.es_cleanup: - self.es.delete(index="users", doc_type="local", id=item["_id"]) - - def testRegistrationPage(self): - if self.es.ping(): - data = { - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - requests.post(self.url, data) - response = self.es.search( - index="users" - , doc_type="local" - , body={ - "query": {"match": {"email_address": "test@user.com"}}}) - self.assertEqual(len(response["hits"]["hits"]), 1) - self.es_cleanup.append(response["hits"]["hits"][0]) - else: - self.skipTest("The elasticsearch server is down") - -def main(): - suite = unittest.TestSuite() - suite.addTest(TestRegistration("testRegistrationPage")) - runner = unittest.TextTestRunner() - runner.run(suite) - -if __name__ == "__main__": - GN2_SERVER = sys.argv[1] - ES_SERVER = sys.argv[2] - main() diff --git a/test/requests/test_registration.py b/test/requests/test_registration.py new file mode 100644 index 00000000..0047e8a6 --- /dev/null +++ b/test/requests/test_registration.py @@ -0,0 +1,41 @@ +import sys +import requests +from parametrized_test import ParametrizedTest + +class TestRegistration(ParametrizedTest): + + def tearDown(self): + for item in self.es_cleanup: + self.es.delete(index="users", doc_type="local", id=item["_id"]) + + def testRegistrationPage(self): + if self.es.ping(): + data = { + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + requests.post(self.gn2_url+"/n/register", data) + response = self.es.search( + index="users" + , doc_type="local" + , body={ + "query": {"match": {"email_address": "test@user.com"}}}) + self.assertEqual(len(response["hits"]["hits"]), 1) + else: + self.skipTest("The elasticsearch server is down") + +def main(gn2, es): + import unittest + suite = unittest.TestSuite() + suite.addTest(TestRegistration(methodName="testRegistrationPage", gn2_url=gn2, es_url=es)) + runner = unittest.TextTestRunner() + runner.run(suite) + +if __name__ == "__main__": + if len(sys.argv) < 3: + raise Exception("Required arguments missing") + else: + main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From 230bf08f9aac697fc4c8dc092348be3b7026046f Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 12:48:36 +0300 Subject: Create parametrized superclass for tests * Since the tests require that some parameters be provided while running the tests, create a class that helps abstract away the details of retrieving and setting the expected parameters. --- test/requests/parametrized_test.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 test/requests/parametrized_test.py diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py new file mode 100644 index 00000000..abf98fce --- /dev/null +++ b/test/requests/parametrized_test.py @@ -0,0 +1,27 @@ +import logging +import unittest +from elasticsearch import Elasticsearch, TransportError + +class ParametrizedTest(unittest.TestCase): + + def __init__(self, methodName='runTest', gn2_url="http://localhost:5003", es_url="localhost:9200"): + super(ParametrizedTest, self).__init__(methodName=methodName) + self.gn2_url = gn2_url + self.es_url = es_url + + def setUp(self): + self.es = Elasticsearch([self.es_url]) + self.es_cleanup = [] + + es_logger = logging.getLogger("elasticsearch") + es_logger.addHandler( + logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) + es_trace_logger = logging.getLogger("elasticsearch.trace") + es_trace_logger.addHandler( + logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) + + def tearDown(self): + self.es.delete_by_query( + index="users" + , doc_type="local" + , body={"query":{"match":{"email_address":"test@user.com"}}}) -- cgit v1.2.3 From 8c4833dbf9f4ae32afbfbe6a3cb8e4630abc3d25 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 12:50:30 +0300 Subject: Add test for local login * Add an integration test to test that the login process for users registered locally to genenetwork2 works as expected. --- test/requests/test_login_local.py | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/requests/test_login_local.py diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py new file mode 100644 index 00000000..bced1ee9 --- /dev/null +++ b/test/requests/test_login_local.py @@ -0,0 +1,56 @@ +import requests +from wqflask import user_manager +from parametrized_test import ParametrizedTest + +class TestLoginLocal(ParametrizedTest): + + def setUp(self): + super(TestLoginLocal, self).setUp() + self.login_url = self.gn2_url +"/n/login" + data = { + "es_connection": self.es, + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + user_manager.basic_info = lambda : { "basic_info": "basic" } + user_manager.RegisterUser(data) + + def testLoginNonRegisteredUser(self): + data = { + "email_address": "non@existent.email", + "password": "doesitmatter?" + } + result = requests.post(self.login_url, data=data) + self.assertEqual(result.url, self.login_url, "") + + def testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse(self): + data = { + "email_address": "test@user.com", + "password": "test_password" + } + result = requests.post(self.login_url, data=data) + print("THE COOKIES? ", result.cookies) + self.assertEqual( + result.url + , self.gn2_url+"/?import_collections=false" + , "Login should have been successful") + + + +def main(gn2, es): + import unittest + suite = unittest.TestSuite() + suite.addTest(TestLoginLocal(methodName="testLoginNonRegisteredUser", gn2_url=gn2, es_url=es)) + suite.addTest(TestLoginLocal(methodName="testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse", gn2_url=gn2, es_url=es)) + runner = unittest.TextTestRunner() + runner.run(suite) + +if __name__ == "__main__": + import sys + if len(sys.argv) < 3: + raise Exception("Required arguments missing") + else: + main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From 85015343fd99885ec00db86b4e3705bcc25e62f1 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 12:52:32 +0300 Subject: Add a runner for all integration tests. --- test/requests/run-integration-tests.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/requests/run-integration-tests.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py new file mode 100644 index 00000000..0fd7bb20 --- /dev/null +++ b/test/requests/run-integration-tests.py @@ -0,0 +1,30 @@ +import sys +from test_login_local import TestLoginLocal +from test_registration import TestRegistration +from unittest import TestSuite, TextTestRunner, TestLoader + +test_cases = [ + TestLoginLocal, + TestRegistration +] + +def suite(gn2_url, es_url): + the_suite = TestSuite() + for case in test_cases: + the_suite.addTests(initTest(case, gn2_url, es_url)) + return the_suite + +def initTest(klass, gn2_url, es_url): + loader = TestLoader() + methodNames = loader.getTestCaseNames(klass) + return [klass(mname, gn2_url, es_url) for mname in methodNames] + +def main(gn2_url, es_url): + runner = TextTestRunner() + runner.run(suite(gn2_url, es_url)) + +if __name__ == "__main__": + if len(sys.argv) < 3: + raise Exception("Required arguments missing:\n\tTry running `run-integration-test.py `") + else: + main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From 2035387823ffc87bc2b6a817a06cb3a47aa006a0 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 14:12:59 +0300 Subject: Add more login tests. --- test/requests/test_login_local.py | 39 +++++++++++++++++++++++++++++++++++++-- wqflask/wqflask/user_manager.py | 2 -- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index bced1ee9..acad45c9 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -32,12 +32,47 @@ class TestLoginLocal(ParametrizedTest): "password": "test_password" } result = requests.post(self.login_url, data=data) - print("THE COOKIES? ", result.cookies) self.assertEqual( result.url , self.gn2_url+"/?import_collections=false" , "Login should have been successful") - + + def testLoginWithRegisteredUserImportCollectionsTrueAndRememberMeFalse(self): + data = { + "email_address": "test@user.com", + "password": "test_password", + "import_collections": "y" + } + result = requests.post(self.login_url, data=data) + self.assertEqual( + result.url + , self.gn2_url+"/?import_collections=true" + , "Login should have been successful") + + def testLoginWithRegisteredUserImportCollectionsFalseAndRememberMeTrue(self): + data = { + "email_address": "test@user.com", + "password": "test_password", + "remember_me": "y" + } + result = requests.post(self.login_url, data=data) + self.assertEqual( + result.url + , self.gn2_url+"/?import_collections=false" + , "Login should have been successful") + + def testLoginWithRegisteredUserBothImportCollectionsAndRememberMeTrue(self): + data = { + "email_address": "test@user.com", + "password": "test_password", + "remember_me": "y", + "import_collections": "y" + } + result = requests.post(self.login_url, data=data) + self.assertEqual( + result.url + , self.gn2_url+"/?import_collections=true" + , "Login should have been successful") def main(gn2, es): diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index c8471cb1..fd1d56ff 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -642,7 +642,6 @@ class LoginUser(object): user = model.User(); for key in user_details: user.__dict__[key] = user_details[key] - print("RETRIEVED USER: ", user) valid = False; submitted_password = params['password'] @@ -689,7 +688,6 @@ class LoginUser(object): """The meat of the logging in process""" session_id_signed = self.successful_login(user, assumed_by) flash("Thank you for logging in {}.".format(user.full_name), "alert-success") - print("IMPORT1:", import_collections) response = make_response(redirect(url_for('index_page', import_collections=import_collections))) if self.remember_me: max_age = self.remember_time -- cgit v1.2.3 From f8970931932ed9b32c078cf3f2a1203f50f73ab0 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 21 Feb 2018 11:46:52 +0300 Subject: Fix assumption of existing collection * When logging in, if a user selects "Import existing collections", the system would throw an exception, since Redis would return a NoneType, which would then be accessed by json.loads(), which doesn't seem to know how to process that. This fixes that, by providing a string representing an empty json array ("[]"), in place of the NoneType. --- wqflask/wqflask/user_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index fd1d56ff..e1920f4e 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -122,7 +122,8 @@ class AnonUser(object): return collections def import_traits_to_user(self): - collections_list = json.loads(Redis.get(self.key)) + result = Redis.get(self.key) + collections_list = json.loads(result if result else "[]") for collection in collections_list: uc = model.UserCollection() uc.name = collection['name'] -- cgit v1.2.3 From a8e328a105dac1d137c3d08653c9164d489332b3 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 21 Feb 2018 12:51:34 +0300 Subject: Simplify test. Check for content, rather than url * The test functions were very similar, so this commit refactors out the common test code to a single method, and passes in the data to the test using the parameterized package. * Check that the page content after a login attempt is the expected content, rather than checking the url. --- test/requests/test_login_local.py | 93 ++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 54 deletions(-) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index acad45c9..8e2dec4e 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -1,7 +1,11 @@ import requests from wqflask import user_manager +from parameterized import parameterized from parametrized_test import ParametrizedTest +login_link_text = 'Sign in' +logout_link_text = 'Sign out' + class TestLoginLocal(ParametrizedTest): def setUp(self): @@ -18,61 +22,42 @@ class TestLoginLocal(ParametrizedTest): user_manager.basic_info = lambda : { "basic_info": "basic" } user_manager.RegisterUser(data) - def testLoginNonRegisteredUser(self): - data = { - "email_address": "non@existent.email", - "password": "doesitmatter?" - } - result = requests.post(self.login_url, data=data) - self.assertEqual(result.url, self.login_url, "") - - def testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse(self): - data = { - "email_address": "test@user.com", - "password": "test_password" - } - result = requests.post(self.login_url, data=data) - self.assertEqual( - result.url - , self.gn2_url+"/?import_collections=false" - , "Login should have been successful") - - def testLoginWithRegisteredUserImportCollectionsTrueAndRememberMeFalse(self): - data = { - "email_address": "test@user.com", - "password": "test_password", - "import_collections": "y" - } - result = requests.post(self.login_url, data=data) - self.assertEqual( - result.url - , self.gn2_url+"/?import_collections=true" - , "Login should have been successful") - - def testLoginWithRegisteredUserImportCollectionsFalseAndRememberMeTrue(self): - data = { - "email_address": "test@user.com", - "password": "test_password", - "remember_me": "y" - } - result = requests.post(self.login_url, data=data) - self.assertEqual( - result.url - , self.gn2_url+"/?import_collections=false" - , "Login should have been successful") - - def testLoginWithRegisteredUserBothImportCollectionsAndRememberMeTrue(self): - data = { - "email_address": "test@user.com", - "password": "test_password", - "remember_me": "y", - "import_collections": "y" - } + + @parameterized.expand([ + ( + { + "email_address": "non@existent.email", + "password": "doesitmatter?" + }, login_link_text, "Login should have failed with the wrong user details."), + ( + { + "email_address": "test@user.com", + "password": "test_password" + }, logout_link_text, "Login should have been successful with correct user details and neither import_collections nor remember_me set"), + ( + { + "email_address": "test@user.com", + "password": "test_password", + "import_collections": "y" + }, logout_link_text, "Login should have been successful with correct user details and only import_collections set"), + ( + { + "email_address": "test@user.com", + "password": "test_password", + "remember_me": "y" + }, logout_link_text, "Login should have been successful with correct user details and only remember_me set"), + ( + { + "email_address": "test@user.com", + "password": "test_password", + "remember_me": "y", + "import_collections": "y" + }, logout_link_text, "Login should have been successful with correct user details, and both remember_me, and import_collections set") + ]) + def testLogin(self, data, expected, message): result = requests.post(self.login_url, data=data) - self.assertEqual( - result.url - , self.gn2_url+"/?import_collections=true" - , "Login should have been successful") + index = result.content.find(expected) + self.assertTrue(index >= 0, message) def main(gn2, es): -- cgit v1.2.3 From a88f9c25e1b6eaaeede23d9217b1998f28fa7bbf Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:24:55 +0300 Subject: Remove failing code * With the parameterized module, the testcase cannot be run independently at this time. --- test/requests/test_login_local.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index 8e2dec4e..808649ca 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -58,19 +58,3 @@ class TestLoginLocal(ParametrizedTest): result = requests.post(self.login_url, data=data) index = result.content.find(expected) self.assertTrue(index >= 0, message) - - -def main(gn2, es): - import unittest - suite = unittest.TestSuite() - suite.addTest(TestLoginLocal(methodName="testLoginNonRegisteredUser", gn2_url=gn2, es_url=es)) - suite.addTest(TestLoginLocal(methodName="testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse", gn2_url=gn2, es_url=es)) - runner = unittest.TextTestRunner() - runner.run(suite) - -if __name__ == "__main__": - import sys - if len(sys.argv) < 3: - raise Exception("Required arguments missing") - else: - main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From 4d7c565aac196ba1913b65ed27481c6974f1a0b5 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:26:42 +0300 Subject: Add new test for github logins --- test/requests/run-integration-tests.py | 4 +++- test/requests/test_login_github.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 test/requests/test_login_github.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py index 0fd7bb20..fc795779 100644 --- a/test/requests/run-integration-tests.py +++ b/test/requests/run-integration-tests.py @@ -1,11 +1,13 @@ import sys from test_login_local import TestLoginLocal +from test_login_github import TestLoginGithub from test_registration import TestRegistration from unittest import TestSuite, TextTestRunner, TestLoader test_cases = [ - TestLoginLocal, TestRegistration + , TestLoginLocal + , TestLoginGithub ] def suite(gn2_url, es_url): diff --git a/test/requests/test_login_github.py b/test/requests/test_login_github.py new file mode 100644 index 00000000..15bf18ae --- /dev/null +++ b/test/requests/test_login_github.py @@ -0,0 +1,40 @@ +import uuid +import requests +from time import sleep +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +login_link_text = 'Sign in' +logout_link_text = 'Sign out' +uid = str(uuid.uuid4()) + +class TestLoginGithub(ParametrizedTest): + + def setUp(self): + super(TestLoginGithub, self).setUp() + data = { + "user_id": uid + , "name": "A. T. Est User" + , "github_id": 693024 + , "user_url": "https://fake-github.com/atestuser" + , "login_type": "github" + , "organization": "" + , "active": 1 + , "confirmed": 1 + } + self.es.create(index="users", doc_type="local", body=data, id=uid) + sleep(1) + + def tearDown(self): + super(TestLoginGithub, self).tearDown() + self.es.delete(index="users", doc_type="local", id=uid) + + @parameterized.expand([ + ("1234", login_link_text, "Login should have failed with non-existing user") + , (uid, logout_link_text, "Login should have been successful with existing user") + ]) + def testLogin(self, test_uid, expected, message): + url = self.gn2_url+"/n/login?type=github&uid="+test_uid + result = requests.get(url) + index = result.content.find(expected) + self.assertTrue(index >= 0, message) -- cgit v1.2.3 From 4d656976eae920066261e26bf22a999d90a0beab Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:36:29 +0300 Subject: Add new tests for orcid logins --- test/requests/run-integration-tests.py | 2 ++ test/requests/test_login_orcid.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/requests/test_login_orcid.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py index fc795779..5e816549 100644 --- a/test/requests/run-integration-tests.py +++ b/test/requests/run-integration-tests.py @@ -1,5 +1,6 @@ import sys from test_login_local import TestLoginLocal +from test_login_orcid import TestLoginOrcid from test_login_github import TestLoginGithub from test_registration import TestRegistration from unittest import TestSuite, TextTestRunner, TestLoader @@ -8,6 +9,7 @@ test_cases = [ TestRegistration , TestLoginLocal , TestLoginGithub + , TestLoginOrcid ] def suite(gn2_url, es_url): diff --git a/test/requests/test_login_orcid.py b/test/requests/test_login_orcid.py new file mode 100644 index 00000000..6e40ece5 --- /dev/null +++ b/test/requests/test_login_orcid.py @@ -0,0 +1,40 @@ +import uuid +import requests +from time import sleep +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +login_link_text = 'Sign in' +logout_link_text = 'Sign out' +uid = str(uuid.uuid4()) + +class TestLoginOrcid(ParametrizedTest): + + def setUp(self): + super(TestLoginOrcid, self).setUp() + data = { + "user_id": uid + , "name": "A. T. Est User" + , "orcid": 345872 + , "user_url": "https://fake-orcid.org/atestuser" + , "login_type": "orcid" + , "organization": "" + , "active": 1 + , "confirmed": 1 + } + self.es.create(index="users", doc_type="local", body=data, id=uid) + sleep(1) + + def tearDown(self): + super(TestLoginOrcid, self).tearDown() + self.es.delete(index="users", doc_type="local", id=uid) + + @parameterized.expand([ + ("1234", login_link_text, "Login should have failed with non-existing user") + , (uid, logout_link_text, "Login should have been successful with existing user") + ]) + def testLogin(self, test_uid, expected, message): + url = self.gn2_url+"/n/login?type=orcid&uid="+test_uid + result = requests.get(url) + index = result.content.find(expected) + self.assertTrue(index >= 0, message) -- cgit v1.2.3 From 1ee48887d93cace2d1b00b2f1adc9a6407fdb882 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:37:06 +0300 Subject: Provide connections to elasticsearch * Provide connections to elasticsearch at various points in the code. These oversight was caught while running tests. --- wqflask/wqflask/user_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index e1920f4e..c344ea27 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -540,6 +540,7 @@ def github_oauth2(): result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} github_user = get_github_user_details(result_dict["access_token"]) + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "github_id", github_user["id"]) if user_details == None: user_details = { @@ -573,6 +574,7 @@ def orcid_oauth2(): result = requests.post(ORCID_TOKEN_URL, data=data) result_dict = json.loads(result.text.encode("utf-8")) + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "orcid", result_dict["orcid"]) if user_details == None: user_details = { @@ -606,6 +608,7 @@ class LoginUser(object): def oauth2_login(self, login_type, user_id): """Login via an OAuth2 provider""" + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "user_id", user_id) if user_details: user = model.User() -- cgit v1.2.3 From 1f591baa129a7a42d6c0cadf73441274c8e373d3 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 18:11:33 +0300 Subject: Add tests to check that the UI is setup correctly * Check that the links for OAuth2 logins via ORCID and GitHub are setup correctly. --- test/requests/test_login_github.py | 7 +++++++ test/requests/test_login_orcid.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/test/requests/test_login_github.py b/test/requests/test_login_github.py index 15bf18ae..1bf4f695 100644 --- a/test/requests/test_login_github.py +++ b/test/requests/test_login_github.py @@ -1,6 +1,7 @@ import uuid import requests from time import sleep +from wqflask import app from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -29,6 +30,12 @@ class TestLoginGithub(ParametrizedTest): super(TestLoginGithub, self).tearDown() self.es.delete(index="users", doc_type="local", id=uid) + def testLoginUrl(self): + login_button_text = 'Login with Github' + result = requests.get(self.gn2_url+"/n/login") + index = result.content.find(login_button_text) + self.assertTrue(index >= 0, "Should have found `Login with Github` button") + @parameterized.expand([ ("1234", login_link_text, "Login should have failed with non-existing user") , (uid, logout_link_text, "Login should have been successful with existing user") diff --git a/test/requests/test_login_orcid.py b/test/requests/test_login_orcid.py index 6e40ece5..ea15642e 100644 --- a/test/requests/test_login_orcid.py +++ b/test/requests/test_login_orcid.py @@ -1,6 +1,7 @@ import uuid import requests from time import sleep +from wqflask import app from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -29,6 +30,12 @@ class TestLoginOrcid(ParametrizedTest): super(TestLoginOrcid, self).tearDown() self.es.delete(index="users", doc_type="local", id=uid) + def testLoginUrl(self): + login_button_text = 'a href="https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=' + app.config.get("ORCID_CLIENT_ID") + '&client_secret=' + app.config.get("ORCID_CLIENT_SECRET") + '" title="Login with ORCID" class="btn btn-info btn-group">Login with ORCID' + result = requests.get(self.gn2_url+"/n/login") + index = result.content.find(login_button_text) + self.assertTrue(index >= 0, "Should have found `Login with ORCID` button") + @parameterized.expand([ ("1234", login_link_text, "Login should have failed with non-existing user") , (uid, logout_link_text, "Login should have been successful with existing user") -- cgit v1.2.3 From 34fa7690fe2255d44b129c7d2397113d713342b6 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Mar 2018 18:12:58 +0300 Subject: Use argparse to handle arguments * Use argparse to handle commandline arguments. * Create initial layout of how the code might end up - lots of the code is currently commented out. --- test/requests/test-website.py | 60 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index d02b71aa..9637b87f 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -3,18 +3,60 @@ # env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 # # Mostly to pick up the Guix GN2_PROFILE and python modules +from __future__ import print_function +from link_checker import check_links +import argparse -import requests as req -import sys +print("Mechanical Rob firing up...") -print "Mechanical Rob firing up..." +def run_all(args_obj, parser): + print("") + print("Running all tests.") + check_links(args_obj, parser) + # TODO: Add other functions as they are created. -if len(sys.argv)<1: - raise "Problem with arguments" +def print_help(args_obj, parser): + print(parser.format_help()) -url = sys.argv[1] -print url +def dummy(args_obj, parser): + print("Not implemented yet.") -r = req.get(url) -print r +desc = """ +This is Mechanical-Rob - an automated web server tester for + Genenetwork.org +""" +parser = argparse.ArgumentParser(description=desc) + +parser.add_argument("-d", "--database", metavar="DB", type=str + , default="db_webqtl_s" + , help="Use database (default db_webqtl_s)") + +parser.add_argument("host", metavar="HOST", type=str + , default="http://localhost:5003" + , help="The url to the web server") + +parser.add_argument("-a", "--all", dest="accumulate", action="store_const" + , const=run_all, default=print_help + , help="Runs all tests.") + +parser.add_argument("-l", "--link-checker", dest="accumulate" + , action='store_const', const=check_links, default=print_help + , help="Checks for dead links.") + +# parser.add_argument("-n", "--navigation", dest="accumulate" +# , action="store_const", const=check_navigation, default=print_help +# , help="Checks for navigation.") + +# parser.add_argument("-m", "--mapping", dest="accumulate" +# , action="store_const", const=check_mapping, default=print_help +# , help="Checks for mapping.") + +# parser.add_argument("-s", "--skip-broken", dest="accumulate" +# , action="store_const", const=dummy, default=print_help +# , help="Skip tests that are known to be broken.") + +args = parser.parse_args() +# print("The arguments object: ", args) + +args.accumulate(args, parser) -- cgit v1.2.3 From e147bc1ec354f9e07b7887109967afe8aae87f5b Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Mar 2018 18:15:19 +0300 Subject: Add link_checker module * Add the module that will hold the code to test the links on the system. --- test/requests/link_checker.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/requests/link_checker.py diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py new file mode 100644 index 00000000..f171c12a --- /dev/null +++ b/test/requests/link_checker.py @@ -0,0 +1,9 @@ +import requests + +def check_links(args_obj, parser): + print("") + print("Checking links") + print("########################") + print("Not implemented yet.") + print("This is supposed to check all links in the system.") + print("########################") -- cgit v1.2.3 From 655e741375b3fad7e3b7657662d33ca8017c0220 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 12 Mar 2018 13:00:03 +0300 Subject: Add tests to check links. --- test/requests/link_checker.py | 62 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index f171c12a..256bf6ef 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -1,9 +1,63 @@ +from __future__ import print_function +import re import requests +from lxml.html import parse +from requests.exceptions import ConnectionError + +def is_root_link(link): + pattern = re.compile("^/$") + return pattern.match(link) + +def is_mailto_link(link): + pattern = re.compile("^mailto:.*") + return pattern.match(link) + +def is_internal_link(link): + pattern = re.compile("^/.*") + return pattern.match(link) + +def get_links(doc): + return filter( + lambda x: not ( + is_root_link(x) + or is_mailto_link(x)) + , map(lambda y: y.get("href") + , doc.cssselect("a"))) + +def verify_link(link): + try: + result = requests.get(link, timeout=20) + if result.status_code == 200: + print(link+" ==> OK") + else: + print("ERROR: link `"+link+"` failed with status " + , result.status_code) + except ConnectionError as ex: + print("ERROR: ", link, ex) + +def check_page(host, start_url): + print("") + print("Checking links in page `"+start_url+"`") + doc = parse(start_url).getroot() + links = get_links(doc) + internal_links = filter(is_internal_link, links) + external_links = filter(lambda x: not is_internal_link(x), links) + external_links.append("http://somenon-existentsite.brr") + for link in internal_links: + verify_link(host+link) + + for link in external_links: + verify_link(link) def check_links(args_obj, parser): print("") print("Checking links") - print("########################") - print("Not implemented yet.") - print("This is supposed to check all links in the system.") - print("########################") + host = args_obj.host + + # Check the home page + check_page(host, host) + + # Check traits page + check_page( + host, + host+"/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") -- cgit v1.2.3 From ed0c0018166166f1dc05905f0aed1fa1930d963b Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 14 Mar 2018 02:16:59 +0300 Subject: Add tests for main web functionality --- test/requests/main_web_functionality.py | 40 +++++++++++++++++++++++++++++++++ test/requests/test-website.py | 7 ++++++ 2 files changed, 47 insertions(+) create mode 100644 test/requests/main_web_functionality.py diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py new file mode 100644 index 00000000..f6b32340 --- /dev/null +++ b/test/requests/main_web_functionality.py @@ -0,0 +1,40 @@ +from __future__ import print_function +import re +import requests +from lxml.html import parse +from requests.exceptions import ConnectionError + +def check_home(url): + doc = parse(url).getroot() + search_button = doc.cssselect("#btsearch") + assert(search_button[0].value == "Search") + print("OK") + +def check_search_page(host): + data = dict( + species="mouse" + , group="BXD" + , type="Hippocampus mRNA" + , dataset="HC_M2_0606_P" + , search_terms_or="" + , search_terms_and="MEAN=(15 16) LRS=(23 46)") + result = requests.get(host+"/search", params=data) + found = result.text.find("/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") + assert(found >= 0) + print("OK") + check_traits_page(host, "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") + +def check_traits_page(host, traits_url): + from link_checker import check_page + doc = parse(host+traits_url).getroot() + traits_form = doc.forms[1] + assert(traits_form.fields["corr_dataset"] == "HC_M2_0606_P") + print("OK") + check_page(host, host+traits_url) + +def check_main_web_functionality(args_obj, parser): + print("") + print("Checking main web functionality...") + host = args_obj.host + check_home(host) + check_search_page(host) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 9637b87f..f65c3fc8 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -4,6 +4,7 @@ # # Mostly to pick up the Guix GN2_PROFILE and python modules from __future__ import print_function +from main_web_functionality import check_main_web_functionality from link_checker import check_links import argparse @@ -12,6 +13,7 @@ print("Mechanical Rob firing up...") def run_all(args_obj, parser): print("") print("Running all tests.") + check_main_web_functionality(args_obj, parser) check_links(args_obj, parser) # TODO: Add other functions as they are created. @@ -44,6 +46,11 @@ parser.add_argument("-l", "--link-checker", dest="accumulate" , action='store_const', const=check_links, default=print_help , help="Checks for dead links.") +parser.add_argument("-f", "--main-functionality", dest="accumulate" + , action='store_const', const=check_main_web_functionality + , default=print_help + , help="Checks for main web functionality.") + # parser.add_argument("-n", "--navigation", dest="accumulate" # , action="store_const", const=check_navigation, default=print_help # , help="Checks for navigation.") -- cgit v1.2.3 From 61a9e8663a3fb36ce4afbe26431e74e66621e175 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 18 Mar 2018 08:51:55 +0300 Subject: Check that key exists before using it * Ensure the key exists in the json object/dictionary, before trying to use it to retrieve a value. --- wqflask/wqflask/marker_regression/marker_regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 087b95b4..c189fb96 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -114,7 +114,7 @@ class MarkerRegression(object): self.num_perm = 0 self.perm_output = [] self.bootstrap_results = [] - self.covariates = start_vars['covariates'] + self.covariates = start_vars['covariates'] if "covariates" in start_vars else None #ZS: This is passed to GN1 code for single chr mapping self.selected_chr = -1 -- cgit v1.2.3 From 55f42f8f972ab3940711fc7b5e39335bc35445b5 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 18 Mar 2018 08:54:03 +0300 Subject: Move import to the top of the page. * Mainly to tell the dependencies easily. --- test/requests/main_web_functionality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py index f6b32340..7b89b833 100644 --- a/test/requests/main_web_functionality.py +++ b/test/requests/main_web_functionality.py @@ -2,6 +2,7 @@ from __future__ import print_function import re import requests from lxml.html import parse +from link_checker import check_page from requests.exceptions import ConnectionError def check_home(url): @@ -25,7 +26,6 @@ def check_search_page(host): check_traits_page(host, "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") def check_traits_page(host, traits_url): - from link_checker import check_page doc = parse(host+traits_url).getroot() traits_form = doc.forms[1] assert(traits_form.fields["corr_dataset"] == "HC_M2_0606_P") -- cgit v1.2.3 From 5ccf077f53c6546bb9258c7116b4b1cf8903375f Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 18 Mar 2018 08:56:03 +0300 Subject: Initialise mapping tests * Add mapping tests, and build the first of the tests. --- test/requests/mapping_tests.py | 43 ++++++++++++++++++++++++++++++++++++++++++ test/requests/test-website.py | 14 ++++++++------ 2 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 test/requests/mapping_tests.py diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py new file mode 100644 index 00000000..fd20df11 --- /dev/null +++ b/test/requests/mapping_tests.py @@ -0,0 +1,43 @@ +from __future__ import print_function +import re +import json +import requests +from lxml.html import fromstring + +def get_data(list_item): + try: + value = list_item[1] + except: + value = None + #print("list_item:", list_item, "==>", value) + return value + +def load_data_from_file(): + filename = "../test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json" + file_handle = open(filename, "r") + file_data = json.loads(file_handle.read().encode("utf-8")) + return file_data + +def check_pylmm_tool_selection(host, data): + data["method"] = "pylmm" + page = requests.post(host+"/marker_regression", data=data) + doc = fromstring(page.text) + form = doc.forms[1] + assert form.fields["dataset"] == "HC_M2_0606_P" + assert form.fields["value:BXD1"] == "15.034" # Check value in the file + +def check_R_qtl_tool_selection(host, data): + pass + +def check_CIM_tool_selection(host, data): + pass + +def check_mapping(args_obj, parser): + print("") + print("Checking mapping") + + host = args_obj.host + data = load_data_from_file() + check_pylmm_tool_selection(host, data) + check_R_qtl_tool_selection(host, data) + check_CIM_tool_selection(host, data) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index f65c3fc8..2bef6eb1 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -4,9 +4,10 @@ # # Mostly to pick up the Guix GN2_PROFILE and python modules from __future__ import print_function -from main_web_functionality import check_main_web_functionality -from link_checker import check_links import argparse +from link_checker import check_links +from mapping_tests import check_mapping +from main_web_functionality import check_main_web_functionality print("Mechanical Rob firing up...") @@ -15,6 +16,7 @@ def run_all(args_obj, parser): print("Running all tests.") check_main_web_functionality(args_obj, parser) check_links(args_obj, parser) + check_mapping(args_obj, parser) # TODO: Add other functions as they are created. def print_help(args_obj, parser): @@ -51,14 +53,14 @@ parser.add_argument("-f", "--main-functionality", dest="accumulate" , default=print_help , help="Checks for main web functionality.") +parser.add_argument("-m", "--mapping", dest="accumulate" + , action="store_const", const=check_mapping, default=print_help + , help="Checks for mapping.") + # parser.add_argument("-n", "--navigation", dest="accumulate" # , action="store_const", const=check_navigation, default=print_help # , help="Checks for navigation.") -# parser.add_argument("-m", "--mapping", dest="accumulate" -# , action="store_const", const=check_mapping, default=print_help -# , help="Checks for mapping.") - # parser.add_argument("-s", "--skip-broken", dest="accumulate" # , action="store_const", const=dummy, default=print_help # , help="Skip tests that are known to be broken.") -- cgit v1.2.3 From cfb3ff04e921cda320d46bf4eaabf1610019e1c7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 19 Mar 2018 13:06:13 +0000 Subject: - Disabled PYTHONPATH injection - fix Guix instead, see https://github.com/pjotrp/genenetwork2/commit/63a5c8a42ad02e9126bb207465ff5eca98f6515d - Renamed WQFLASK_SETTINGS to GN2_SETTINGS --- bin/genenetwork2 | 33 +++++++++++++++------------------ wqflask/utility/tools.py | 15 +-------------- wqflask/wqflask/__init__.py | 4 ++-- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 2a83a1cd..b9e0a02b 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -68,32 +68,29 @@ if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then echo "Can not use $1 switch without default settings file" exit 1 fi -# Handle settings parameter (can be .py or .json) -if [ ! -z $1 ]; then - settings=$(realpath "$1") - if [ ! -e $settings ]; then - settings=$GN2_BASE_DIR/etc/default_settings.py - else - shift - fi -fi -ext="${settings##*.}" -if [ "$ext" = "json" -o "$ext" = "JSON" ]; then - overrides=$settings +settings=$1 +if [ -z $settings ]; then + settings=$GN2_BASE_DIR/etc/default_settings.py else - echo $settings + shift fi +settings=$(realpath $settings) + +# ext="${settings##*.}" +# if [ "$ext" = "json" -o "$ext" = "JSON" ]; then +# overrides=$settings +# else +# echo $settings +# fi if [ ! -e $settings ]; then echo "ERROR: can not locate settings file - pass it in the command line" exit 1 fi -export WQFLASK_SETTINGS=$settings # Python -export WQFLASK_OVERRIDES=$overrides # JSON -echo WQFLASK_SETTINGS=$settings -echo WQFLASK_OVERRIDES=$overrides +export GN2_SETTINGS=$settings # Python +echo GN2_SETTINGS=$settings # This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead # if [ -z $ELASTICSEARCH_PROFILE ]; then @@ -115,7 +112,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH" + export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!! export R_LIBS_SITE=$GN2_PROFILE/site-library export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0 export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 8c9fed96..bae3a7f4 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -54,7 +54,7 @@ def get_setting(command_id,guess=None): # print("Looking for "+command_id+"\n") command = value(os.environ.get(command_id)) if command is None or command == "": - command = OVERRIDES.get(command_id) + command = OVERRIDES.get(command_id) # currently not in use if command is None: # ---- Check whether setting exists in app command = value(app.config.get(command_id)) @@ -285,18 +285,5 @@ assert_dir(JS_TWITTER_POST_FETCHER_PATH) from six import string_types -if os.environ.get('WQFLASK_OVERRIDES'): - jsonfn = get_setting('WQFLASK_OVERRIDES') - logger.info("WQFLASK_OVERRIDES: %s" % jsonfn) - with open(jsonfn) as data_file: - overrides = json.load(data_file) - for k in overrides: - cmd = overrides[k] - if isinstance(cmd, string_types): - OVERRIDES[k] = eval(cmd) - else: - OVERRIDES[k] = cmd - logger.debug(OVERRIDES) - # assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData") assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 2188ce17..bc8e9900 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -13,8 +13,8 @@ logging.basicConfig(level=logging.INFO) app = Flask(__name__) app.config.from_object('cfg.default_settings') # Get the defaults from cfg.default_settings -app.config.from_envvar('WQFLASK_SETTINGS') # See http://flask.pocoo.org/docs/config/#configuring-from-files -# Note we also use WQFLASK_OVERRIDES +app.config.from_envvar('GN2_SETTINGS') # See http://flask.pocoo.org/docs/config/#configuring-from-files +# Note no longer use the badly named WQFLASK_OVERRIDES (nyi) app.jinja_env.globals.update( undefined = jinja2.StrictUndefined, -- cgit v1.2.3 From 7ab7a55f68178aa55d9e242d74f2a2d28f6e2aba Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 19 Mar 2018 13:47:34 +0000 Subject: RELEASENOTES --- RELEASE-NOTES.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 RELEASE-NOTES.md diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md new file mode 100644 index 00000000..d15cad1a --- /dev/null +++ b/RELEASE-NOTES.md @@ -0,0 +1,17 @@ +## ChangeLog v2.11 (date unknown) + +This is a massive bug fix release with many improvements. For contributions +see +[contributors](https://github.com/genenetwork/genenetwork2/contributors) +and +[commits](https://github.com/genenetwork/genenetwork2/commits/master). + +### Added GEMMA support + +* Front-end support + +### Added test framework and unit tests + +* Added python integration and unit tests + + -- cgit v1.2.3 From fd236176d9011887662a0203f9af4a2f206b5af2 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 20 Mar 2018 09:41:36 +0000 Subject: Authentication: some mods around defaults --- wqflask/utility/tools.py | 15 +++++++++------ wqflask/wqflask/templates/new_security/login_user.html | 2 +- wqflask/wqflask/user_manager.py | 13 ++++++------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index bae3a7f4..ed54f77c 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -254,15 +254,18 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = None -if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET: - GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET -GITHUB_API_URL = get_setting('GITHUB_API_URL') +if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET: + GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \ + GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET + GITHUB_API_URL = get_setting('GITHUB_API_URL') + ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = None -if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET: - ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET -ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') +if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: + ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + + ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET + ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 0dae3503..6d597f6b 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -88,7 +88,7 @@ {% else: %}
-

You cannot login at this moment using your GeneNetwork account.
+

You cannot login at this moment using your GeneNetwork account (the authentication service is down).
Please try again later.

{% endif %} diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index c344ea27..c55649f3 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -626,13 +626,12 @@ class LoginUser(object): logger.debug("in login params are:", params) es = get_elasticsearch_connection() if not params: - from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL - external_login = None - if GITHUB_AUTH_URL or ORCID_AUTH_URL: - external_login={ - "github": GITHUB_AUTH_URL, - "orcid": ORCID_AUTH_URL - } + from utility.tools import GITHUB_AUTH_URL, GITHUB_CLIENT_ID, ORCID_AUTH_URL, ORCID_CLIENT_ID + external_login = {} + if GITHUB_AUTH_URL and GITHUB_CLIENT_ID != 'UNKNOWN': + external_login["github"] = GITHUB_AUTH_URL + if ORCID_AUTH_URL and ORCID_CLIENT_ID != 'UNKNOWN': + external_login["orcid"] = ORCID_AUTH_URL assert(es is not None) return render_template( "new_security/login_user.html" -- cgit v1.2.3 From ab369995d4a16de5d20123192279e32658f870a0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 20 Mar 2018 09:50:34 +0000 Subject: Fix --- wqflask/utility/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index ed54f77c..4b4cd633 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -263,7 +263,7 @@ ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = None if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: - ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + + ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \ ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -- cgit v1.2.3 From afd8d38c19be78b7e962f4edea0972ac3f48c25d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 20 Mar 2018 11:02:25 +0000 Subject: Refactor startup config for gunicorn and werkzeug --- wqflask/run_gunicorn.py | 6 ++-- wqflask/runserver.py | 17 ++-------- wqflask/utility/elasticsearch_tools.py | 5 +++ wqflask/utility/startup_config.py | 39 ++++++++++++++++++++++ wqflask/utility/tools.py | 4 ++- .../wqflask/templates/new_security/login_user.html | 8 ++++- 6 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 wqflask/utility/startup_config.py diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index ebe3add5..adffdca3 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -7,12 +7,12 @@ # from flask import Flask # application = Flask(__name__) -print "Starting up Gunicorn process" +print "===> Starting up Gunicorn process" from wqflask import app +from utility.startup_config import app_config -app.config['SESSION_TYPE'] = 'filesystem' -app.config['SECRET_KEY'] = 'super secret key' +app_config() @app.route("/gunicorn") def hello(): diff --git a/wqflask/runserver.py b/wqflask/runserver.py index a0c76e51..5f41d04d 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -21,22 +21,9 @@ GREEN = '\033[92m' BOLD = '\033[1m' ENDC = '\033[0m' -import os -app.config['SECRET_KEY'] = os.urandom(24) +from utility.startup_config import app_config -from utility.tools import WEBSERVER_MODE,get_setting_int,get_setting,get_setting_bool - -port = get_setting_int("SERVER_PORT") - -print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]") - -if get_setting_bool("USE_GN_SERVER"): - import requests - page = requests.get(get_setting("GN_SERVER_URL")) - if page.status_code != 200: - raise Exception("API server not found!") - -print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) +app_config() werkzeug_logger = logging.getLogger('werkzeug') diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 2d3d5add..734379f7 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -6,6 +6,11 @@ logger = getLogger(__name__) from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT +def test_elasticsearch_connection(): + es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+ELASTICSEARCH_PORT+'/'], verify_certs=True) + if not es.ping(): + logger.warning("Elasticsearch is DOWN") + def get_elasticsearch_connection(): logger.info("get_elasticsearch_connection") es = None diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py new file mode 100644 index 00000000..02e8e56c --- /dev/null +++ b/wqflask/utility/startup_config.py @@ -0,0 +1,39 @@ + +from wqflask import app +from utility.tools import WEBSERVER_MODE, show_settings, get_setting_int, get_setting, get_setting_bool + +import utility.logger +logger = utility.logger.getLogger(__name__ ) + +BLUE = '\033[94m' +GREEN = '\033[92m' +BOLD = '\033[1m' +ENDC = '\033[0m' + +def app_config(): + app.config['SESSION_TYPE'] = 'filesystem' + if not app.config.get('SECRET_KEY'): + import os + app.config['SECRET_KEY'] = str(os.urandom(24)) + + mode = WEBSERVER_MODE + if mode == "DEV" or mode == "DEBUG": + app.config['TEMPLATES_AUTO_RELOAD'] = True + if mode == "DEBUG": + app.config['EXPLAIN_TEMPLATE_LOADING'] = True + print("==========================================") + show_settings() + + port = get_setting_int("SERVER_PORT") + + if get_setting_bool("USE_GN_SERVER"): + print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]") + import requests + page = requests.get(get_setting("GN_SERVER_URL")) + if page.status_code != 200: + raise Exception("API server not found!") + + import utility.elasticsearch_tools as es + es.test_elasticsearch_connection() + + print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 4b4cd633..59bb49d8 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -220,7 +220,7 @@ def show_settings(): logger.info(OVERRIDES) logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC) - print "runserver.py: ****** Webserver configuration ******" + print "runserver.py: ****** Webserver configuration - k,v pairs from app.config ******" keylist = app.config.keys() keylist.sort() for k in keylist: @@ -269,6 +269,8 @@ if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') +import utility.elasticsearch_tools as es +es.test_elasticsearch_connection() SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 6d597f6b..949760b6 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -15,7 +15,6 @@

Don't have an account?

- {% if es_server: %} Create a new account {% else: %} @@ -92,6 +91,13 @@ Please try again later.

{% endif %} + {% if not es_server and not external_login: %} +
+
+ Note: it is safe to use GeneNetwork without a login. Login is only required for keeping track of + collections and getting access to some types of restricted data. +
+ {% endif %} -- cgit v1.2.3 From fe08b7773b2903baf80adf97bad9db4317774c2a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 21 Mar 2018 10:05:54 +0000 Subject: Skip automatic use of EXPLAIN_TEMPLATE_LOADING. --- wqflask/utility/startup_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 02e8e56c..5a62cc50 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -19,8 +19,8 @@ def app_config(): mode = WEBSERVER_MODE if mode == "DEV" or mode == "DEBUG": app.config['TEMPLATES_AUTO_RELOAD'] = True - if mode == "DEBUG": - app.config['EXPLAIN_TEMPLATE_LOADING'] = True + # if mode == "DEBUG": + # app.config['EXPLAIN_TEMPLATE_LOADING'] = True <--- use overriding app param instead print("==========================================") show_settings() -- cgit v1.2.3 From c0df4a12d17af0233e525d07eb9e6e341aded298 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 21 Mar 2018 10:13:04 +0000 Subject: Use logger.info instead of logger.debug, fixes https://github.com/genenetwork/genenetwork2/issues/296 --- wqflask/wqflask/views.py | 66 ++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 4e81c29c..8476b6c5 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -123,7 +123,7 @@ def handle_bad_request(e): @app.route("/") def index_page(): logger.info("Sending index_page") - logger.error(request.url) + logger.info(request.url) params = request.args if 'import_collections' in params: import_collections = params['import_collections'] @@ -141,7 +141,7 @@ def index_page(): def tmp_page(img_path): logger.info("In tmp_page") logger.info("img_path:", img_path) - logger.error(request.url) + logger.info(request.url) initial_start_vars = request.form logger.info("initial_start_vars:", initial_start_vars) imgfile = open(GENERATED_IMAGE_DIR + img_path, 'rb') @@ -174,7 +174,7 @@ def twitter(filename): @app.route("/search", methods=('GET',)) def search_page(): logger.info("in search_page") - logger.error(request.url) + logger.info(request.url) if 'info_database' in request.args: logger.info("Going to sharing_info_page") template_vars = sharing_info_page() @@ -213,7 +213,7 @@ def search_page(): @app.route("/gsearch", methods=('GET',)) def gsearchact(): - logger.error(request.url) + logger.info(request.url) result = gsearch.GSearch(request.args).__dict__ type = request.args['type'] if type == "gene": @@ -224,7 +224,7 @@ def gsearchact(): @app.route("/gsearch_updating", methods=('POST',)) def gsearch_updating(): logger.info("REQUEST ARGS:", request.values) - logger.error(request.url) + logger.info(request.url) result = update_search_results.GSearch(request.args).__dict__ return result['results'] # type = request.args['type'] @@ -235,31 +235,31 @@ def gsearch_updating(): @app.route("/docedit") def docedit(): - logger.error(request.url) + logger.info(request.url) doc = docs.Docs(request.args['entry']) return render_template("docedit.html", **doc.__dict__) @app.route('/generated/') def generated_file(filename): - logger.error(request.url) + logger.info(request.url) return send_from_directory(GENERATED_IMAGE_DIR,filename) @app.route("/help") def help(): - logger.error(request.url) + logger.info(request.url) doc = docs.Docs("help") return render_template("docs.html", **doc.__dict__) @app.route("/wgcna_setup", methods=('POST',)) def wcgna_setup(): logger.info("In wgcna, request.form is:", request.form) # We are going to get additional user input for the analysis - logger.error(request.url) + logger.info(request.url) return render_template("wgcna_setup.html", **request.form) # Display them using the template @app.route("/wgcna_results", methods=('POST',)) def wcgna_results(): logger.info("In wgcna, request.form is:", request.form) - logger.error(request.url) + logger.info(request.url) wgcna = wgcna_analysis.WGCNA() # Start R, load the package and pointers and create the analysis wgcnaA = wgcna.run_analysis(request.form) # Start the analysis, a wgcnaA object should be a separate long running thread result = wgcna.process_results(wgcnaA) # After the analysis is finished store the result @@ -268,13 +268,13 @@ def wcgna_results(): @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): logger.info("In ctl, request.form is:", request.form) # We are going to get additional user input for the analysis - logger.error(request.url) + logger.info(request.url) return render_template("ctl_setup.html", **request.form) # Display them using the template @app.route("/ctl_results", methods=('POST',)) def ctl_results(): logger.info("In ctl, request.form is:", request.form) - logger.error(request.url) + logger.info(request.url) ctl = ctl_analysis.CTL() # Start R, load the package and pointers and create the analysis ctlA = ctl.run_analysis(request.form) # Start the analysis, a ctlA object should be a separate long running thread result = ctl.process_results(ctlA) # After the analysis is finished store the result @@ -313,13 +313,13 @@ def environments(): @app.route("/submit_trait") def submit_trait_form(): - logger.error(request.url) + logger.info(request.url) species_and_groups = get_species_groups() return render_template("submit_trait.html", **{'species_and_groups' : species_and_groups, 'gn_server_url' : GN_SERVER_URL, 'version' : GN_VERSION}) @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): - logger.error(request.url) + logger.info(request.url) print("REQUEST.FORM:", request.form) #template_vars = submit_trait.SubmitTrait(request.form) @@ -332,7 +332,7 @@ def export_trait_excel(): """Excel file consisting of the sample data from the trait data and analysis page""" logger.info("In export_trait_excel") logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) sample_data = export_trait_data.export_sample_table(request.form) logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) @@ -358,7 +358,7 @@ def export_trait_csv(): """CSV file consisting of the sample data from the trait data and analysis page""" logger.info("In export_trait_csv") logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) sample_data = export_trait_data.export_sample_table(request.form) logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) @@ -379,7 +379,7 @@ def export_traits_csv(): """CSV file consisting of the traits from the search result page""" logger.info("In export_traits_csv") logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) csv_data = export_traits.export_search_results_csv(request.form) return Response(csv_data, @@ -389,7 +389,7 @@ def export_traits_csv(): @app.route('/export_perm_data', methods=('POST',)) def export_perm_data(): """CSV file consisting of the permutation data for the mapping results""" - logger.error(request.url) + logger.info(request.url) num_perm = float(request.form['num_perm']) perm_data = json.loads(request.form['perm_results']) @@ -412,7 +412,7 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): - logger.error(request.url) + logger.info(request.url) template_vars = show_trait.ShowTrait(request.form) #logger.info("js_data before dump:", template_vars.js_data) template_vars.js_data = json.dumps(template_vars.js_data, @@ -427,7 +427,7 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): - logger.error(request.url) + logger.info(request.url) template_vars = show_trait.ShowTrait(request.args) #logger.info("js_data before dump:", template_vars.js_data) template_vars.js_data = json.dumps(template_vars.js_data, @@ -443,7 +443,7 @@ def show_trait_page(): @app.route("/heatmap", methods=('POST',)) def heatmap_page(): logger.info("In heatmap, request.form is:", pf(request.form)) - logger.error(request.url) + logger.info(request.url) start_vars = request.form temp_uuid = uuid.uuid4() @@ -493,7 +493,7 @@ def mapping_results_container_page(): @app.route("/loading", methods=('POST',)) def loading_page(): - logger.error(request.url) + logger.info(request.url) initial_start_vars = request.form logger.debug("Marker regression called with initial_start_vars:", initial_start_vars.items()) #temp_uuid = initial_start_vars['temp_uuid'] @@ -552,7 +552,7 @@ def loading_page(): def marker_regression_page(): initial_start_vars = request.form logger.debug("Marker regression called with initial_start_vars:", initial_start_vars.items()) - logger.error(request.url) + logger.info(request.url) temp_uuid = initial_start_vars['temp_uuid'] wanted = ( 'trait_id', @@ -678,7 +678,7 @@ def marker_regression_page(): @app.route("/export_mapping_results", methods = ('POST',)) def export_mapping_results(): logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) file_path = request.form.get("results_path") results_csv = open(file_path, "r").read() response = Response(results_csv, @@ -691,7 +691,7 @@ def export_mapping_results(): @app.route("/export", methods = ('POST',)) def export(): logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) svg_xml = request.form.get("data", "Invalid data") filename = request.form.get("filename", "manhattan_plot_snp") response = Response(svg_xml, mimetype="image/svg+xml") @@ -702,7 +702,7 @@ def export(): def export_pdf(): import cairosvg logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) svg_xml = request.form.get("data", "Invalid data") logger.info("svg_xml:", svg_xml) filename = request.form.get("filename", "interval_map_pdf") @@ -715,7 +715,7 @@ def export_pdf(): @app.route("/network_graph", methods=('POST',)) def network_graph_page(): logger.info("In network_graph, request.form is:", pf(request.form)) - logger.error(request.url) + logger.info(request.url) start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] if traits[0] != "": @@ -731,7 +731,7 @@ def network_graph_page(): @app.route("/corr_compute", methods=('POST',)) def corr_compute_page(): logger.info("In corr_compute, request.form is:", pf(request.form)) - logger.error(request.url) + logger.info(request.url) #fd = webqtlFormData.webqtlFormData(request.form) template_vars = show_corr_results.CorrelationResults(request.form) return render_template("correlation_page.html", **template_vars.__dict__) @@ -739,7 +739,7 @@ def corr_compute_page(): @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): logger.info("In corr_matrix, request.form is:", pf(request.form)) - logger.error(request.url) + logger.info(request.url) start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -755,7 +755,7 @@ def corr_matrix_page(): @app.route("/corr_scatter_plot") def corr_scatter_plot_page(): - logger.error(request.url) + logger.info(request.url) template_vars = corr_scatter_plot.CorrScatterPlot(request.args) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, @@ -764,7 +764,7 @@ def corr_scatter_plot_page(): @app.route("/submit_bnw", methods=('POST',)) def submit_bnw(): - logger.error(request.url) + logger.info(request.url) template_vars = get_bnw_input(request.form) return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) @@ -772,7 +772,7 @@ def submit_bnw(): def sharing_info_page(): """Info page displayed when the user clicks the "Info" button next to the dataset selection""" logger.info("In sharing_info_page") - logger.error(request.url) + logger.info(request.url) fd = webqtlFormData.webqtlFormData(request.args) template_vars = SharingInfoPage.SharingInfoPage(fd) return template_vars @@ -780,7 +780,7 @@ def sharing_info_page(): # Take this out or secure it before putting into production @app.route("/get_temp_data") def get_temp_data(): - logger.error(request.url) + logger.info(request.url) temp_uuid = request.args['key'] return flask.jsonify(temp_data.TempData(temp_uuid).get_all()) -- cgit v1.2.3 From 121ecdeb9d923e9964969007ac56210fb17e2c4d Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 10 Jan 2018 15:05:03 +0300 Subject: Add configuration variables for external services * Add configuration variables for GitHub and ORCID which will be used by the system to allow users to login. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index d3113302..e7a673ad 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,6 +251,10 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) +GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') +ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) assert(GEMMA_COMMAND is not None) -- cgit v1.2.3 From 06a34983c913f697f10f0440240cffb28f5ddb79 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 10 Jan 2018 15:07:11 +0300 Subject: Add template elements for OAuth login * Add html elements that will be used to prompt users to login with either GitHub or ORCID. --- wqflask/wqflask/templates/new_security/login_user.html | 18 ++++++++++++++++-- wqflask/wqflask/user_manager.py | 9 ++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index b9f49a61..15f0a27e 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -18,8 +18,22 @@ Create a new account - -
+
+

Login with external services

+ + {% if external_login: %} +
+ {% if external_login["github"]: %} + Login with Github + {% endif %} + + {% if external_login["orcid"]: %} + Login with ORCID + {% endif %} +
+ +
+ {% endif %}

Already have an account? Sign in here.

diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index f7fcd2d0..25833464 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -506,7 +506,14 @@ class LoginUser(object): params = request.form if request.form else request.args logger.debug("in login params are:", params) if not params: - return render_template("new_security/login_user.html") + from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL + external_login = None + if GITHUB_AUTH_URL or ORCID_AUTH_URL: + external_login={ + "github": GITHUB_AUTH_URL, + "orcid": ORCID_AUTH_URL + } + return render_template("new_security/login_user.html", external_login=external_login) else: try: user = model.User.query.filter_by(email_address=params['email_address']).one() -- cgit v1.2.3 From ccdf9b81147fba35e4df704f39026af6a64e3a20 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:01:11 +0300 Subject: Add client_id and client_secret configurations * Provide the OAuth2 client_id and client_secret values in configuration variables. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index e7a673ad..330344d1 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,7 +251,11 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) +GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -- cgit v1.2.3 From d0f071a3871a2bcbb2c5170996a4afb145c21f9c Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:05:45 +0300 Subject: Add elasticsearch_tools module * Collect variables and functions for using the elasticsearch system in a separate module. --- wqflask/utility/elasticsearch_tools.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 wqflask/utility/elasticsearch_tools.py diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py new file mode 100644 index 00000000..bc7bb240 --- /dev/null +++ b/wqflask/utility/elasticsearch_tools.py @@ -0,0 +1,22 @@ +from elasticsearch import Elasticsearch, TransportError +from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + +es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT +}]) + +def get_user_by_unique_column(column_name, column_value): + user_details = None + try: + response = es.search( + index = "users" + , doc_type = "local" + , body = { + "query": { "match": { column_name: column_value } } + }) + if len(response["hits"]["hits"]) > 0: + user_details = response["hits"]["hits"][0]["_source"] + except TransportError as te: + pass + return user_details -- cgit v1.2.3 From 2524691b03cb629eb79e4d11c8ce8e451a5cd870 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 12 Jan 2018 18:07:13 +0300 Subject: Add functions to help handle github login * Add functions to help with the github OAuth2 login process --- wqflask/wqflask/user_manager.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 25833464..9012c842 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -494,6 +494,37 @@ def login(): lu = LoginUser() return lu.standard_login() +@app.route("/n/login/github_oauth2", methods=('GET', 'POST')) +def github_oauth2(): + from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET + from utility.elasticsearch_tools import get_user_by_unique_column + import requests + code = request.args.get("code") + data = { + "client_id": GITHUB_CLIENT_ID, + "client_secret": GITHUB_CLIENT_SECRET, + "code": code + } + result = requests.post("https://github.com/login/oauth/access_token", json=data) + result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} + + github_user = get_github_user_details(result_dict["access_token"]) + user_details = get_user_by_unique_column("github_id", github_user["id"]) + if user_details == None: + user_details = { + "user_id": str(uuid4()) + , "name": github_user["name"] + , "github_id": github_user["id"] + , "user_url": github_user["html_url"] + , "login_type": "github" + } + url = "/n/login?type=github" + return redirect(url) + +def get_github_user_details(access_token): + from utility.tools import GITHUB_API_URL + result = requests.get(GITHUB_API_URL, params={"access_token":access_token}) + return result.json() class LoginUser(object): remember_time = 60 * 60 * 24 * 30 # One month in seconds -- cgit v1.2.3 From 941e42ef72bd6aa6761c35cf78e64df41977c5d3 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:36:19 +0300 Subject: Add elasticsearch module to the path * Add some code to set up the path for the python-elasticsearch module. --- bin/genenetwork2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 8886e4bc..3fe8baa0 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -95,6 +95,11 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides +if [ -z $ELASTICSEARCH_PROFILE ]; then + echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m"; +else + PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +fi if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) @@ -108,7 +113,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages + export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH" export R_LIBS_SITE=$GN2_PROFILE/site-library export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0 export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript -- cgit v1.2.3 From 98de7bf9649115b67a13018454a2c3766be1fc12 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:38:04 +0300 Subject: Add save_user() function * On successful login via OAuth2, save the details of the user in elasticsearch store, to avoid hitting the external provider for the basic details. --- wqflask/utility/elasticsearch_tools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index bc7bb240..74db489b 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -20,3 +20,10 @@ def get_user_by_unique_column(column_name, column_value): except TransportError as te: pass return user_details + +def save_user(user, user_id, index="users", doc_type="local"): + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT + }]) + es.create(index, doc_type, body=user, id=user_id) -- cgit v1.2.3 From 38c85f80090322b2dc9278812b7b1f011bb35390 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:41:41 +0300 Subject: Add more configuration variables. * Add configurations for elasticsearch and github. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 330344d1..ea7b8120 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -254,11 +254,15 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +GITHUB_API_URL = get_setting('GITHUB_API_URL') ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') +ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) assert(GEMMA_COMMAND is not None) -- cgit v1.2.3 From 7959930d3276b5317d933a428a3c2f9ea8f7ddf4 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 12:00:04 +0300 Subject: Delay after save for indexing * Elasticsearch need a short delay after adding document for it to index the document for subsequent access. --- wqflask/utility/elasticsearch_tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 74db489b..c2c999ea 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -22,8 +22,10 @@ def get_user_by_unique_column(column_name, column_value): return user_details def save_user(user, user_id, index="users", doc_type="local"): + from time import sleep es = Elasticsearch([{ "host": ELASTICSEARCH_HOST , "port": ELASTICSEARCH_PORT }]) es.create(index, doc_type, body=user, id=user_id) + sleep(1) # Delay 1 second to allow indexing -- cgit v1.2.3 From 2cf5b40e163ac17210a7ed64cbeffd98de9789a6 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 12:03:10 +0300 Subject: Add code to enable OAuth2 login * Add some code to handle the login if the user chooses to login via GitHub or ORCID. --- wqflask/wqflask/user_manager.py | 58 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 9012c842..daeb7bc5 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -54,6 +54,9 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list +import requests +from utility.elasticsearch_tools import get_user_by_unique_column, save_user + THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -492,13 +495,16 @@ class DecodeUser(object): @app.route("/n/login", methods=('GET', 'POST')) def login(): lu = LoginUser() - return lu.standard_login() + login_type = request.args.get("type") + if login_type: + uid = request.args.get("uid") + return lu.oauth2_login(login_type, uid) + else: + return lu.standard_login() @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) def github_oauth2(): from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET - from utility.elasticsearch_tools import get_user_by_unique_column - import requests code = request.args.get("code") data = { "client_id": GITHUB_CLIENT_ID, @@ -512,13 +518,15 @@ def github_oauth2(): user_details = get_user_by_unique_column("github_id", github_user["id"]) if user_details == None: user_details = { - "user_id": str(uuid4()) + "user_id": str(uuid.uuid4()) , "name": github_user["name"] , "github_id": github_user["id"] , "user_url": github_user["html_url"] , "login_type": "github" + , "organization": "" } - url = "/n/login?type=github" + save_user(user_details, user_details.get("user_id")) + url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) def get_github_user_details(access_token): @@ -532,6 +540,46 @@ class LoginUser(object): def __init__(self): self.remember_me = False + def oauth2_login(self, login_type, user_id): + """Login via an OAuth2 provider""" + user_details = get_user_by_unique_column("user_id", user_id) + if user_details: + user = model.User() + user.id = user_details["user_id"] + user.full_name = user_details["name"] + user.login_type = user_details["login_type"] + return self.actual_login_oauth2(user) + else: + flash("Error logging in via OAuth2") + return make_response(redirect(url_for('login'))) + + def actual_login_oauth2(self, user, assumed_by=None, import_collections=None): + """The meat of the logging in process""" + session_id_signed = self.successful_login_oauth2(user) + flash("Thank you for logging in {}.".format(user.full_name), "alert-success") + print("IMPORT1:", import_collections) + response = make_response(redirect(url_for('index_page', import_collections=import_collections))) + if self.remember_me: + max_age = self.remember_time + else: + max_age = None + response.set_cookie(UserSession.cookie_name, session_id_signed, max_age=max_age) + return response + + def successful_login_oauth2(self, user, assumed_by=None): + login_rec = model.Login(user) + login_rec.successful = True + login_rec.session_id = str(uuid.uuid4()) + login_rec.assumed_by = assumed_by + session_id_signature = actual_hmac_creation(login_rec.session_id) + session_id_signed = login_rec.session_id + ":" + session_id_signature + logger.debug("session_id_signed:", session_id_signed) + + session = dict(login_time = time.time(), + user_id = user.id, + user_login_type = user.login_type) + return session_id_signed + def standard_login(self): """Login through the normal form""" params = request.form if request.form else request.args -- cgit v1.2.3 From 49067569b4236da81013f30e9382d77277a1cd75 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 13:00:51 +0300 Subject: Update configurations * Have the authorisation URLs build up from the client id and client secret values. --- wqflask/utility/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index ea7b8120..c96b3699 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -253,11 +253,11 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') -GITHUB_AUTH_URL = get_setting('GITHUB_AUTH_URL') +GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET GITHUB_API_URL = get_setting('GITHUB_API_URL') ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') -ORCID_AUTH_URL = get_setting('ORCID_AUTH_URL') +ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -- cgit v1.2.3 From e8b91c219e94b9a7252e44914352c425556f3c71 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 13:02:11 +0300 Subject: Add OAuth2 login code for ORCID * Add code to handle the login via ORCID --- wqflask/wqflask/user_manager.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index daeb7bc5..c3f190c3 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -529,6 +529,40 @@ def github_oauth2(): url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) +@app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) +def orcid_oauth2(): + from uuid import uuid4 + from utility.tools import ORCID_CLIENT_ID, ORCID_CLIENT_SECRET, ORCID_TOKEN_URL, ORCID_AUTH_URL + code = request.args.get("code") + error = request.args.get("error") + url = "/n/login" + if code: + data = { + "client_id": ORCID_CLIENT_ID + , "client_secret": ORCID_CLIENT_SECRET + , "grant_type": "authorization_code" + , "code": code + } + result = requests.post(ORCID_TOKEN_URL, data=data) + result_dict = json.loads(result.text.encode("utf-8")) + print("The dict: ", result_dict); + user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) + if user_details == None: + user_details = { + "user_id": str(uuid4()) + , "name": result_dict["name"] + , "orcid": result_dict["orcid"] + , "user_url": "%s/%s" % ( + "/".join(ORCID_AUTH_URL.split("/")[:-2]), + result_dict["orcid"]) + , "login_type": "orcid" + } + save_user(user_details, user_details["user_id"]) + url = "/n/login?type=orcid&uid="+user_details["user_id"] + else: + flash("There was an error getting code from ORCID") + return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, params={"access_token":access_token}) -- cgit v1.2.3 From de628e01635f62d1edefabb30712763f04c2e552 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 17:48:43 +0300 Subject: Use elasticsearch rather than mysql for local accounts * Register local accounts onto elasticsearch rather than mysql. * Login from the accounts on elasticsearch * Harmonise local and oauth2 logins to use the same code. --- wqflask/wqflask/user_manager.py | 106 ++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 70 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index c3f190c3..4322945b 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -272,15 +272,19 @@ class RegisterUser(object): self.errors = [] self.user = Bunch() - self.user.email_address = kw.get('email_address', '').strip() + self.user.email_address = kw.get('email_address', '').encode("utf-8").strip() if not (5 <= len(self.user.email_address) <= 50): self.errors.append('Email Address needs to be between 5 and 50 characters.') - self.user.full_name = kw.get('full_name', '').strip() + email_exists = get_user_by_unique_column("email_address", self.user.email_address) + if email_exists: + self.errors.append('User already exists with that email') + + self.user.full_name = kw.get('full_name', '').encode("utf-8").strip() if not (5 <= len(self.user.full_name) <= 50): self.errors.append('Full Name needs to be between 5 and 50 characters.') - self.user.organization = kw.get('organization', '').strip() + self.user.organization = kw.get('organization', '').encode("utf-8").strip() if self.user.organization and not (5 <= len(self.user.organization) <= 50): self.errors.append('Organization needs to be empty or between 5 and 50 characters.') @@ -297,28 +301,11 @@ class RegisterUser(object): logger.debug("No errors!") set_password(password, self.user) + self.user.user_id = str(uuid.uuid4()) + self.user.confirmed = 1 self.user.registration_info = json.dumps(basic_info(), sort_keys=True) - - self.new_user = model.User(**self.user.__dict__) - db_session.add(self.new_user) - - try: - db_session.commit() - except sqlalchemy.exc.IntegrityError: - # This exception is thrown if the email address is already in the database - # To do: Perhaps put a link to sign in using an existing account here - self.errors.append("An account with this email address already exists. " - "Click the button above to sign in using an existing account.") - return - - logger.debug("Adding verification email to queue") - #self.send_email_verification() - VerificationEmail(self.new_user) - logger.debug("Added verification email to queue") - - self.thank_you_mode = True - + save_user(self.user.__dict__, self.user.user_id) def set_password(password, user): pwfields = Bunch() @@ -364,7 +351,7 @@ class VerificationEmail(object): verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - data = json.dumps(dict(id=user.id, + data = json.dumps(dict(id=user.user_id, timestamp=timestamp()) ) @@ -519,13 +506,15 @@ def github_oauth2(): if user_details == None: user_details = { "user_id": str(uuid.uuid4()) - , "name": github_user["name"] + , "name": github_user["name"].encode("utf-8") , "github_id": github_user["id"] - , "user_url": github_user["html_url"] + , "user_url": github_user["html_url"].encode("utf-8") , "login_type": "github" , "organization": "" + , "active": 1 + , "confirmed": 1 } - save_user(user_details, user_details.get("user_id")) + save_user(user_details, user_details["user_id"]) url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) @@ -545,7 +534,7 @@ def orcid_oauth2(): } result = requests.post(ORCID_TOKEN_URL, data=data) result_dict = json.loads(result.text.encode("utf-8")) - print("The dict: ", result_dict); + user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) if user_details == None: user_details = { @@ -556,6 +545,9 @@ def orcid_oauth2(): "/".join(ORCID_AUTH_URL.split("/")[:-2]), result_dict["orcid"]) , "login_type": "orcid" + , "organization": "" + , "active": 1 + , "confirmed": 1 } save_user(user_details, user_details["user_id"]) url = "/n/login?type=orcid&uid="+user_details["user_id"] @@ -582,38 +574,11 @@ class LoginUser(object): user.id = user_details["user_id"] user.full_name = user_details["name"] user.login_type = user_details["login_type"] - return self.actual_login_oauth2(user) + return self.actual_login(user) else: flash("Error logging in via OAuth2") return make_response(redirect(url_for('login'))) - def actual_login_oauth2(self, user, assumed_by=None, import_collections=None): - """The meat of the logging in process""" - session_id_signed = self.successful_login_oauth2(user) - flash("Thank you for logging in {}.".format(user.full_name), "alert-success") - print("IMPORT1:", import_collections) - response = make_response(redirect(url_for('index_page', import_collections=import_collections))) - if self.remember_me: - max_age = self.remember_time - else: - max_age = None - response.set_cookie(UserSession.cookie_name, session_id_signed, max_age=max_age) - return response - - def successful_login_oauth2(self, user, assumed_by=None): - login_rec = model.Login(user) - login_rec.successful = True - login_rec.session_id = str(uuid.uuid4()) - login_rec.assumed_by = assumed_by - session_id_signature = actual_hmac_creation(login_rec.session_id) - session_id_signed = login_rec.session_id + ":" + session_id_signature - logger.debug("session_id_signed:", session_id_signed) - - session = dict(login_time = time.time(), - user_id = user.id, - user_login_type = user.login_type) - return session_id_signed - def standard_login(self): """Login through the normal form""" params = request.form if request.form else request.args @@ -628,20 +593,23 @@ class LoginUser(object): } return render_template("new_security/login_user.html", external_login=external_login) else: - try: - user = model.User.query.filter_by(email_address=params['email_address']).one() - except sqlalchemy.orm.exc.NoResultFound: - logger.debug("No account exists for that email address") - valid = False - user = None - else: + user_details = get_user_by_unique_column("email_address", params["email_address"]) + user = None + if user_details: + user = model.User(); + for key in user_details: + user.__dict__[key] = user_details[key] + print("RETRIEVED USER: ", user) + valid = False; + submitted_password = params['password'] pwfields = Struct(json.loads(user.password)) - encrypted = Password(submitted_password, - pwfields.salt, - pwfields.iterations, - pwfields.keylength, - pwfields.hashfunc) + encrypted = Password( + submitted_password, + pwfields.salt, + pwfields.iterations, + pwfields.keylength, + pwfields.hashfunc) logger.debug("\n\nComparing:\n{}\n{}\n".format(encrypted.password, pwfields.password)) valid = pbkdf2.safe_str_cmp(encrypted.password, pwfields.password) logger.debug("valid is:", valid) @@ -709,8 +677,6 @@ class LoginUser(object): else: expire_time = THREE_DAYS Redis.expire(key, expire_time) - db_session.add(login_rec) - db_session.commit() return session_id_signed def unsuccessful_login(self, user): -- cgit v1.2.3 From 3029ae6ffe66f083f02d35d39e09500f76977197 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 30 Jan 2018 12:23:31 +0300 Subject: Set to None if no value provided * Add a method to set the configuration variables to None if the configuration values are not provided at startup or in the configuration files. The system already checks for these values, and if they are absent, it simply fails to display the OAuth service as available for use to login. --- wqflask/utility/tools.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index c96b3699..a67a930b 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,17 +251,28 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) -GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') -GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') -GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET -GITHUB_API_URL = get_setting('GITHUB_API_URL') -ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') -ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') -ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET -ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') - -ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') +def get_setting_safe(setting): + try: + return get_setting(setting) + except: + print("Could not find the setting '", setting, "'. Continuing with value unset") + return None + +GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET') +GITHUB_AUTH_URL = None +if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET: + GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET +GITHUB_API_URL = get_setting_safe('GITHUB_API_URL') +ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET') +ORCID_AUTH_URL = None +if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET: + ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET +ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') + +ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) -- cgit v1.2.3 From e0295504fb0097db394e99568339e24a71406123 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 30 Jan 2018 13:07:58 +0300 Subject: Fail safely if elasticsearch is down or unconfigured * If elasticsearch server is down, or the configuration variables are not provided at startup or in a configuration file, then do not allow the system to simply crash, but instead, inform the user that they cannot use the services that depend on elasticsearch to be running. --- wqflask/utility/elasticsearch_tools.py | 16 ++++++++------ .../wqflask/templates/new_security/login_user.html | 25 ++++++++++++++++++---- wqflask/wqflask/user_manager.py | 6 +++++- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index c2c999ea..8b8ad9cc 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,10 +1,14 @@ -from elasticsearch import Elasticsearch, TransportError -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT +es = None +try: + from elasticsearch import Elasticsearch, TransportError + from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT -es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST - , "port": ELASTICSEARCH_PORT -}]) + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT + }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None +except: + es = None def get_user_by_unique_column(column_name, column_value): user_details = None diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 15f0a27e..0dae3503 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -16,7 +16,14 @@

Don't have an account?

- Create a new account + {% if es_server: %} + Create a new account + {% else: %} +
+

You cannot create an account at this moment.
+ Please try again later.

+
+ {% endif %}

Login with external services

@@ -31,13 +38,17 @@ Login with ORCID {% endif %} - -
+ {% else: %} +
+

You cannot login with external services at this time.
+ Please try again later.

+
{% endif %} +

Already have an account? Sign in here.

- + {% if es_server: %}
@@ -75,6 +86,12 @@
+ {% else: %} +
+

You cannot login at this moment using your GeneNetwork account.
+ Please try again later.

+
+ {% endif %} diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 4322945b..772d6c83 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -585,13 +585,17 @@ class LoginUser(object): logger.debug("in login params are:", params) if not params: from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL + from utility.elasticsearch_tools import es external_login = None if GITHUB_AUTH_URL or ORCID_AUTH_URL: external_login={ "github": GITHUB_AUTH_URL, "orcid": ORCID_AUTH_URL } - return render_template("new_security/login_user.html", external_login=external_login) + return render_template( + "new_security/login_user.html" + , external_login=external_login + , es_server=es) else: user_details = get_user_by_unique_column("email_address", params["email_address"]) user = None -- cgit v1.2.3 From 193a0c19647b37eeb9d18df02e1436d422b4f843 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 26 Mar 2018 09:28:26 +0000 Subject: Fix conflict --- bin/genenetwork2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 3fe8baa0..74ed7f9b 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -129,7 +129,11 @@ else if [ -z $GEMMA_WRAPPER_COMMAND ]; then export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi - if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi + while IFS=":" read -ra PPATH; do + for PPART in "${PPATH[@]}"; do + if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi + done + done <<< "$PYTHONPATH" if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi fi -- cgit v1.2.3 From d0530d880e11dba4cd6dd35e3d04ac4541c11722 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 17:47:29 +0300 Subject: Add SMTP configuration variables * Add configuration variables to enable the system connect to the configured SMTP server to send out emails. --- wqflask/utility/tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index a67a930b..005f9b0f 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -274,6 +274,10 @@ ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') +SMTP_CONNECT = get_setting_safe('SMTP_CONNECT') +SMTP_USERNAME = get_setting_safe('SMTP_USERNAME') +SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD') + PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) assert(GEMMA_COMMAND is not None) -- cgit v1.2.3 From dfc415edf77279078503f7b09ac8c14a442d5db3 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 17:55:36 +0300 Subject: Send emails for forgotten passwords * Update the code so that it sends out emails for the "forgot password" feature. --- wqflask/wqflask/user_manager.py | 71 ++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 772d6c83..ec29062d 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -368,23 +368,33 @@ class ForgotPasswordEmail(VerificationEmail): template_name = "email/forgot_password.txt" key_prefix = "forgot_password_code" subject = "GeneNetwork password reset" + fromaddr = "no-reply@genenetwork.org" - def __init__(self, user): + def __init__(self, toaddr): + from email.MIMEMultipart import MIMEMultipart + from email.MIMEText import MIMEText verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - data = json.dumps(dict(id=user.id, - timestamp=timestamp()) - ) + # data = json.dumps(dict(id=user.id, + # timestamp=timestamp()) + # ) + + # Redis.set(key, data) + # Redis.expire(key, THREE_DAYS) - Redis.set(key, data) - #two_days = 60 * 60 * 24 * 2 - Redis.expire(key, THREE_DAYS) - to = user.email_address subject = self.subject - body = render_template(self.template_name, - verification_code = verification_code) - send_email(to, subject, body) + body = render_template( + self.template_name, + verification_code = verification_code) + + msg = MIMEMultipart() + msg["To"] = toaddr + msg["Subject"] = self.subject + msg["From"] = self.fromaddr + msg.attach(MIMEText(body, "plain")) + + send_email(toaddr, msg.as_string()) class Password(object): @@ -708,13 +718,16 @@ def forgot_password(): def forgot_password_submit(): params = request.form email_address = params['email_address'] - try: - user = model.User.query.filter_by(email_address=email_address).one() - except orm.exc.NoResultFound: - flash("Couldn't find a user associated with the email address {}. Sorry.".format( - email_address)) - return redirect(url_for("login")) - ForgotPasswordEmail(user) + user_details = get_user_by_unique_column("email_address", email_address) + if user_details: + ForgotPasswordEmail(user_details["email_address"]) + # try: + # user = model.User.query.filter_by(email_address=email_address).one() + # except orm.exc.NoResultFound: + # flash("Couldn't find a user associated with the email address {}. Sorry.".format( + # email_address)) + # return redirect(url_for("login")) + # ForgotPasswordEmail(user) return render_template("new_security/forgot_password_step2.html", subject=ForgotPasswordEmail.subject) @@ -861,13 +874,21 @@ app.jinja_env.globals.update(url_for_hmac=url_for_hmac, ####################################################################################### -def send_email(to, subject, body): - msg = json.dumps(dict(From="no-reply@genenetwork.org", - To=to, - Subject=subject, - Body=body)) - Redis.rpush("mail_queue", msg) - +# def send_email(to, subject, body): +# msg = json.dumps(dict(From="no-reply@genenetwork.org", +# To=to, +# Subject=subject, +# Body=body)) +# Redis.rpush("mail_queue", msg) + +def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): + from smtplib import SMTP + from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD + server = SMTP(SMTP_CONNECT) + server.starttls() + server.login(SMTP_USERNAME, SMTP_PASSWORD) + server.sendmail(fromaddr, toaddr, msg) + server.quit() -- cgit v1.2.3 From a494260a8f9cf0e3ecf0c428bb70d4066623f1dd Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 20:01:45 +0300 Subject: Refactor common items to more generic methods. * Refactor code that can be used in more than one place to a more generic method/function that's called by other methods --- wqflask/utility/elasticsearch_tools.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 8b8ad9cc..ea636b2e 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -11,25 +11,27 @@ except: es = None def get_user_by_unique_column(column_name, column_value): - user_details = None + return get_item_by_unique_column(column_name, column_value, index="users", doc_type="local") + +def save_user(user, user_id): + es_save_data("users", "local", user, user_id) + +def get_item_by_unique_column(column_name, column_value, index, doc_type): + item_details = None try: response = es.search( - index = "users" - , doc_type = "local" + index = index + , doc_type = doc_type , body = { "query": { "match": { column_name: column_value } } }) if len(response["hits"]["hits"]) > 0: - user_details = response["hits"]["hits"][0]["_source"] + item_details = response["hits"]["hits"][0]["_source"] except TransportError as te: pass - return user_details + return item_details -def save_user(user, user_id, index="users", doc_type="local"): +def es_save_data(index, doc_type, data_item, data_id,): from time import sleep - es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST - , "port": ELASTICSEARCH_PORT - }]) - es.create(index, doc_type, body=user, id=user_id) + es.create(index, doc_type, body=data_item, id=data_id) sleep(1) # Delay 1 second to allow indexing -- cgit v1.2.3 From 20950a4dc73a310aea12c601b837ccad369ba49b Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 20:04:09 +0300 Subject: Add code to allow user to change password * After the email is sent to the user, there is need to provide a way for the user to actually change their password, and have the results saved. --- wqflask/wqflask/user_manager.py | 65 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index ec29062d..8f09c206 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -55,7 +55,7 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list import requests -from utility.elasticsearch_tools import get_user_by_unique_column, save_user +from utility.elasticsearch_tools import get_user_by_unique_column, save_user, es_save_data THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -376,12 +376,12 @@ class ForgotPasswordEmail(VerificationEmail): verification_code = str(uuid.uuid4()) key = self.key_prefix + ":" + verification_code - # data = json.dumps(dict(id=user.id, - # timestamp=timestamp()) - # ) - - # Redis.set(key, data) - # Redis.expire(key, THREE_DAYS) + data = { + "verification_code": verification_code, + "email_address": toaddr, + "timestamp": timestamp() + } + es_save_data(self.key_prefix, "local", data, verification_code) subject = self.subject body = render_template( @@ -429,38 +429,59 @@ def verify_email(): response.set_cookie(UserSession.cookie_name, session_id_signed) return response -@app.route("/n/password_reset") +@app.route("/n/password_reset", methods=['GET']) def password_reset(): + from utility.elasticsearch_tools import get_item_by_unique_column logger.debug("in password_reset request.url is:", request.url) # We do this mainly just to assert that it's in proper form for displaying next page # Really not necessary but doesn't hurt - user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() - - return render_template("new_security/password_reset.html", user_encode=user_encode) + # user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() + verification_code = request.args.get('code') + hmac = request.args.get('hm') + if verification_code: + code_details = get_item_by_unique_column( + "verification_code", + verification_code, + ForgotPasswordEmail.key_prefix, + "local") + if code_details: + user_details = get_user_by_unique_column( + "email_address", + code_details["email_address"]) + if user_details: + return render_template( + "new_security/password_reset.html", user_encode=user_details["user_id"]) + else: + flash("Invalid code: User no longer exists!", "error") + else: + flash("Invalid code: Password reset code does not exist or might have expired!", "error") + return redirect(url_for("login"))#render_template("new_security/login_user.html", error=error) @app.route("/n/password_reset_step2", methods=('POST',)) def password_reset_step2(): + from utility.elasticsearch_tools import es logger.debug("in password_reset request.url is:", request.url) errors = [] + user_id = request.form['user_encode'] - user_encode = request.form['user_encode'] - verification_code, separator, hmac = user_encode.partition(':') - - hmac_verified = actual_hmac_creation(verification_code) logger.debug("locals are:", locals()) - assert hmac == hmac_verified, "Someone has been naughty" - - user = DecodeUser.actual_get_user(ForgotPasswordEmail.key_prefix, verification_code) - logger.debug("user is:", user) - + user = Bunch() password = request.form['password'] - set_password(password, user) - db_session.commit() + + es.update( + index = "users" + , doc_type = "local" + , id = user_id + , body = { + "doc": { + "password": user.__dict__.get("password") + } + }) flash("Password changed successfully. You can now sign in.", "alert-info") response = make_response(redirect(url_for('login'))) -- cgit v1.2.3 From 49f3111bcac8d69dea52ff75bdd39e01e99c2f02 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Feb 2018 10:11:21 +0300 Subject: Add check for elasticsearch * Add some extra checks to ensure that elasticsearch is running before presenting the UI to the user. --- wqflask/utility/elasticsearch_tools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index ea636b2e..a0383033 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -7,6 +7,10 @@ try: "host": ELASTICSEARCH_HOST , "port": ELASTICSEARCH_PORT }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + + # Check if elasticsearch is running + if not es.ping(): + es = None except: es = None -- cgit v1.2.3 From 690e525a8a063d1be107c15521474052bd6ae2b4 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Feb 2018 11:02:07 +0300 Subject: Check elasticsearch at point of use * Instead of checking for the state of elasticsearch at startup, check the state at the moment the user requests a feature that depends on elasticsearch. This reduces the chances that the user is dropped onto an exception page when elasticsearch server goes down. --- wqflask/utility/elasticsearch_tools.py | 3 --- wqflask/wqflask/user_manager.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index a0383033..4fc0035c 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -8,9 +8,6 @@ try: , "port": ELASTICSEARCH_PORT }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - # Check if elasticsearch is running - if not es.ping(): - es = None except: es = None diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 8f09c206..630be9aa 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -626,7 +626,7 @@ class LoginUser(object): return render_template( "new_security/login_user.html" , external_login=external_login - , es_server=es) + , es_server=es.ping()) else: user_details = get_user_by_unique_column("email_address", params["email_address"]) user = None -- cgit v1.2.3 From c68cffc414ac6d7536db36e79914f7d57af741c6 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 13 Feb 2018 13:48:28 +0300 Subject: Update module to make it more testable * Update functions to make them more testable. * Update code using updated functions. --- wqflask/utility/elasticsearch_tools.py | 40 +++++++++++++--------- wqflask/wqflask/user_manager.py | 61 +++++++++++++++++++--------------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 4fc0035c..a964b025 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,23 +1,31 @@ -es = None -try: - from elasticsearch import Elasticsearch, TransportError - from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT +from elasticsearch import Elasticsearch, TransportError +import logging - es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST - , "port": ELASTICSEARCH_PORT - }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - -except: +def get_elasticsearch_connection(): es = None + try: + from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST + , "port": ELASTICSEARCH_PORT + }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + + es_logger = logging.getLogger("elasticsearch") + es_logger.setLevel(logging.INFO) + es_logger.addHandler(logging.NullHandler()) + except: + es = None + + return es -def get_user_by_unique_column(column_name, column_value): - return get_item_by_unique_column(column_name, column_value, index="users", doc_type="local") +def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): + return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) -def save_user(user, user_id): - es_save_data("users", "local", user, user_id) +def save_user(es, user, user_id): + es_save_data(es, "users", "local", user, user_id) -def get_item_by_unique_column(column_name, column_value, index, doc_type): +def get_item_by_unique_column(es, column_name, column_value, index, doc_type): item_details = None try: response = es.search( @@ -32,7 +40,7 @@ def get_item_by_unique_column(column_name, column_value, index, doc_type): pass return item_details -def es_save_data(index, doc_type, data_item, data_id,): +def es_save_data(es, index, doc_type, data_item, data_id,): from time import sleep es.create(index, doc_type, body=data_item, id=data_id) sleep(1) # Delay 1 second to allow indexing diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 630be9aa..6b667615 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -55,8 +55,9 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list import requests -from utility.elasticsearch_tools import get_user_by_unique_column, save_user, es_save_data +from utility.elasticsearch_tools import * +es = get_elasticsearch_connection() THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -271,14 +272,18 @@ class RegisterUser(object): self.thank_you_mode = False self.errors = [] self.user = Bunch() + es = kw.get('es_connection', None) + + if not es: + self.errors.append("Missing connection object") self.user.email_address = kw.get('email_address', '').encode("utf-8").strip() if not (5 <= len(self.user.email_address) <= 50): self.errors.append('Email Address needs to be between 5 and 50 characters.') - - email_exists = get_user_by_unique_column("email_address", self.user.email_address) - if email_exists: - self.errors.append('User already exists with that email') + else: + email_exists = get_user_by_unique_column(es, "email_address", self.user.email_address) + if email_exists: + self.errors.append('User already exists with that email') self.user.full_name = kw.get('full_name', '').encode("utf-8").strip() if not (5 <= len(self.user.full_name) <= 50): @@ -305,7 +310,7 @@ class RegisterUser(object): self.user.confirmed = 1 self.user.registration_info = json.dumps(basic_info(), sort_keys=True) - save_user(self.user.__dict__, self.user.user_id) + save_user(es, self.user.__dict__, self.user.user_id) def set_password(password, user): pwfields = Bunch() @@ -381,7 +386,7 @@ class ForgotPasswordEmail(VerificationEmail): "email_address": toaddr, "timestamp": timestamp() } - es_save_data(self.key_prefix, "local", data, verification_code) + es_save_data(es, self.key_prefix, "local", data, verification_code) subject = self.subject body = render_template( @@ -431,7 +436,6 @@ def verify_email(): @app.route("/n/password_reset", methods=['GET']) def password_reset(): - from utility.elasticsearch_tools import get_item_by_unique_column logger.debug("in password_reset request.url is:", request.url) # We do this mainly just to assert that it's in proper form for displaying next page @@ -441,14 +445,16 @@ def password_reset(): hmac = request.args.get('hm') if verification_code: code_details = get_item_by_unique_column( - "verification_code", - verification_code, - ForgotPasswordEmail.key_prefix, - "local") + es + , "verification_code" + , verification_code + , ForgotPasswordEmail.key_prefix + , "local") if code_details: user_details = get_user_by_unique_column( - "email_address", - code_details["email_address"]) + es + , "email_address" + , code_details["email_address"]) if user_details: return render_template( "new_security/password_reset.html", user_encode=user_details["user_id"]) @@ -533,7 +539,7 @@ def github_oauth2(): result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} github_user = get_github_user_details(result_dict["access_token"]) - user_details = get_user_by_unique_column("github_id", github_user["id"]) + user_details = get_user_by_unique_column(es, "github_id", github_user["id"]) if user_details == None: user_details = { "user_id": str(uuid.uuid4()) @@ -545,7 +551,7 @@ def github_oauth2(): , "active": 1 , "confirmed": 1 } - save_user(user_details, user_details["user_id"]) + save_user(es, user_details, user_details["user_id"]) url = "/n/login?type=github&uid="+user_details["user_id"] return redirect(url) @@ -566,7 +572,7 @@ def orcid_oauth2(): result = requests.post(ORCID_TOKEN_URL, data=data) result_dict = json.loads(result.text.encode("utf-8")) - user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) + user_details = get_user_by_unique_column(es, "orcid", result_dict["orcid"]) if user_details == None: user_details = { "user_id": str(uuid4()) @@ -580,7 +586,7 @@ def orcid_oauth2(): , "active": 1 , "confirmed": 1 } - save_user(user_details, user_details["user_id"]) + save_user(es, user_details, user_details["user_id"]) url = "/n/login?type=orcid&uid="+user_details["user_id"] else: flash("There was an error getting code from ORCID") @@ -599,7 +605,7 @@ class LoginUser(object): def oauth2_login(self, login_type, user_id): """Login via an OAuth2 provider""" - user_details = get_user_by_unique_column("user_id", user_id) + user_details = get_user_by_unique_column(es, "user_id", user_id) if user_details: user = model.User() user.id = user_details["user_id"] @@ -616,7 +622,6 @@ class LoginUser(object): logger.debug("in login params are:", params) if not params: from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL - from utility.elasticsearch_tools import es external_login = None if GITHUB_AUTH_URL or ORCID_AUTH_URL: external_login={ @@ -628,8 +633,9 @@ class LoginUser(object): , external_login=external_login , es_server=es.ping()) else: - user_details = get_user_by_unique_column("email_address", params["email_address"]) + user_details = get_user_by_unique_column(es, "email_address", params["email_address"]) user = None + valid = None if user_details: user = model.User(); for key in user_details: @@ -672,7 +678,7 @@ class LoginUser(object): else: if user: self.unsuccessful_login(user) - flash("Invalid email-address or password. Please try again.", "alert-error") + flash("Invalid email-address or password. Please try again.", "alert-danger") response = make_response(redirect(url_for('login'))) return response @@ -739,7 +745,7 @@ def forgot_password(): def forgot_password_submit(): params = request.form email_address = params['email_address'] - user_details = get_user_by_unique_column("email_address", email_address) + user_details = get_user_by_unique_column(es, "email_address", email_address) if user_details: ForgotPasswordEmail(user_details["email_address"]) # try: @@ -815,16 +821,17 @@ def register(): params = request.form if request.form else request.args + params = params.to_dict(flat=True) + params["es_connection"] = es if params: logger.debug("Attempting to register the user...") result = RegisterUser(params) errors = result.errors - if result.thank_you_mode: - assert not errors, "Errors while in thank you mode? That seems wrong..." - return render_template("new_security/registered.html", - subject=VerificationEmail.subject) + if len(errors) == 0: + flash("Registration successful. You may login with your new account", "alert-info") + return redirect(url_for("login")) return render_template("new_security/register_user.html", values=params, errors=errors) -- cgit v1.2.3 From 11577cd84db5d7cc9cf10c2178d1f782a9809260 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Tue, 13 Feb 2018 18:32:18 +0300 Subject: Add tests for Registration process. --- wqflask/tests/__init__.py | 0 wqflask/tests/es_double.py | 30 ++++++++++ wqflask/tests/test_registration.py | 113 +++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 wqflask/tests/__init__.py create mode 100644 wqflask/tests/es_double.py create mode 100644 wqflask/tests/test_registration.py diff --git a/wqflask/tests/__init__.py b/wqflask/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wqflask/tests/es_double.py b/wqflask/tests/es_double.py new file mode 100644 index 00000000..00739016 --- /dev/null +++ b/wqflask/tests/es_double.py @@ -0,0 +1,30 @@ +class ESDouble(object): + def __init__(self): + self.items = { + "users": { + "local": [] + }} + + def ping(self): + return true + + def create(self, index, doc_type, body, id): + item = {"id": id, "_source": body} + if not self.items.get("index", None): + self.items[index] = {doc_type: [item]} + else: + self.items[index][doc_type].append(item) + + def search(self, index, doc_type, body): + d = body["query"]["match"] + column = [(key, d[key]) for key in d] + + items = [] + for thing in self.items[index][doc_type]: + if thing["_source"][column[0][0]] == column[0][1]: + items.append(thing) + break + return { + "hits": { + "hits": items + }} diff --git a/wqflask/tests/test_registration.py b/wqflask/tests/test_registration.py new file mode 100644 index 00000000..50a2a84c --- /dev/null +++ b/wqflask/tests/test_registration.py @@ -0,0 +1,113 @@ +import unittest +import es_double +import wqflask.user_manager +from wqflask.user_manager import RegisterUser + +class TestRegisterUser(unittest.TestCase): + def setUp(self): + # Mock elasticsearch + self.es = es_double.ESDouble() + + # Patch method + wqflask.user_manager.basic_info = lambda : {"basic_info": "some info"} + + def tearDown(self): + self.es = None + + def testRegisterUserWithNoData(self): + data = {} + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Data was not provided. Error was expected") + + def testRegisterUserWithNoEmail(self): + data = { + "email_address": "" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Email not provided. Error was expected") + + def testRegisterUserWithNoName(self): + data = { + "email_address": "user@example.com" + , "full_name": "" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Name not provided. Error was expected") + + def testRegisterUserWithNoOrganisation(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertEqual(len(result.errors), 0, "Organisation not provided. Error not expected") + + def testRegisterUserWithShortOrganisation(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "SO" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Organisation name too short. Error expected") + + def testRegisterUserWithNoPassword(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": None + , "password_confirm": None + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Password not provided. Error was expected") + + def testRegisterUserWithNonMatchingPasswords(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "stilltesting" + , "es_connection": self.es + } + + result = RegisterUser(data) + self.assertNotEqual(len(result.errors), 0, "Password mismatch. Error was expected") + + def testRegisterUserWithCorrectData(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + result = RegisterUser(data) + self.assertEqual(len(result.errors), 0, "All data items provided. Errors were not expected") + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3 From 3de1ecfa37b73b4cb011b634c8b4afc2362f858c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 15 Feb 2018 10:30:52 +0000 Subject: Fixing authentication stuff so it uses parameters properly. Also no PYTHONPATH needed as it is now in the Guix build. --- bin/genenetwork2 | 12 +++++++----- etc/default_settings.py | 18 ++++++++++++++++++ wqflask/run_gunicorn.py | 3 +++ wqflask/utility/elasticsearch_tools.py | 16 ++++++++++++---- wqflask/utility/tools.py | 29 +++++++++++------------------ wqflask/wqflask/user_manager.py | 7 +++++-- 6 files changed, 56 insertions(+), 29 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 74ed7f9b..31fefbd3 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -95,11 +95,13 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides -if [ -z $ELASTICSEARCH_PROFILE ]; then - echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m"; -else - PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" -fi +# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead +# if [ -z $ELASTICSEARCH_PROFILE ]; then +# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; +# else +# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +# fi + if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) diff --git a/etc/default_settings.py b/etc/default_settings.py index 699d21f1..a70d8aec 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -41,6 +41,24 @@ SECURITY_POST_LOGIN_VIEW = "/thank_you" SERVER_PORT = 5003 # running on localhost SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' +GITHUB_CLIENT_ID = "UNKNOWN" +GITHUB_CLIENT_SECRET = "UNKNOWN" +GITHUB_AUTH_URL = "UNKNOWN" +GITHUB_API_URL = "UNKNOWN" + +ORCID_CLIENT_ID = "UNKNOWN" +ORCID_CLIENT_SECRET = "UNKNOWN" +ORCID_AUTH_URL = "UNKNOWN" +ORCID_TOKEN_URL = "UNKNOWN" + +ELASTICSEARCH_HOST = "localhost" +ELASTICSEARCH_PORT = '9200' + +SMTP_CONNECT = "UNKNOWN" +SMTP_USERNAME = "UNKNOWN" +SMTP_PASSWORD = "UNKNOWN" + + # ---- Behavioural settings (defaults) note that logger and log levels can # be overridden at the module level and with enviroment settings WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index 14a2d689..ebe3add5 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -11,6 +11,9 @@ print "Starting up Gunicorn process" from wqflask import app +app.config['SESSION_TYPE'] = 'filesystem' +app.config['SECRET_KEY'] = 'super secret key' + @app.route("/gunicorn") def hello(): return "

Hello There!

" diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index a964b025..2d3d5add 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,10 +1,18 @@ from elasticsearch import Elasticsearch, TransportError import logging +from utility.logger import getLogger +logger = getLogger(__name__) + +from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + def get_elasticsearch_connection(): + logger.info("get_elasticsearch_connection") es = None try: - from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + assert(ELASTICSEARCH_HOST) + assert(ELASTICSEARCH_PORT) + logger.info("ES HOST",ELASTICSEARCH_HOST) es = Elasticsearch([{ "host": ELASTICSEARCH_HOST @@ -31,12 +39,12 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type): response = es.search( index = index , doc_type = doc_type - , body = { - "query": { "match": { column_name: column_value } } + , body = { + "query": { "match": { column_name: column_value } } }) if len(response["hits"]["hits"]) > 0: item_details = response["hits"]["hits"][0]["_source"] - except TransportError as te: + except TransportError as te: pass return item_details diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 005f9b0f..8c9fed96 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,32 +251,25 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) -def get_setting_safe(setting): - try: - return get_setting(setting) - except: - print("Could not find the setting '", setting, "'. Continuing with value unset") - return None - -GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID') -GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET') +GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = None if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET: GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET -GITHUB_API_URL = get_setting_safe('GITHUB_API_URL') -ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID') -ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET') +GITHUB_API_URL = get_setting('GITHUB_API_URL') +ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = None if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET: ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET -ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') +ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') +ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') -SMTP_CONNECT = get_setting_safe('SMTP_CONNECT') -SMTP_USERNAME = get_setting_safe('SMTP_USERNAME') -SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD') +SMTP_CONNECT = get_setting('SMTP_CONNECT') +SMTP_USERNAME = get_setting('SMTP_USERNAME') +SMTP_PASSWORD = get_setting('SMTP_PASSWORD') PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 6b667615..c8471cb1 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -55,9 +55,8 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list import requests -from utility.elasticsearch_tools import * +from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column, save_user -es = get_elasticsearch_connection() THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -479,6 +478,7 @@ def password_reset_step2(): password = request.form['password'] set_password(password, user) + es = get_elasticsearch_connection() es.update( index = "users" , doc_type = "local" @@ -620,6 +620,7 @@ class LoginUser(object): """Login through the normal form""" params = request.form if request.form else request.args logger.debug("in login params are:", params) + es = get_elasticsearch_connection() if not params: from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL external_login = None @@ -628,6 +629,7 @@ class LoginUser(object): "github": GITHUB_AUTH_URL, "orcid": ORCID_AUTH_URL } + assert(es is not None) return render_template( "new_security/login_user.html" , external_login=external_login @@ -822,6 +824,7 @@ def register(): params = request.form if request.form else request.args params = params.to_dict(flat=True) + es = get_elasticsearch_connection() params["es_connection"] = es if params: -- cgit v1.2.3 From e7f80eb6cf60bb117f943132049ec0b220ddcab4 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 15 Feb 2018 14:03:38 +0000 Subject: Added unittest example --- test/unittest/test_registration.py | 27 +++++++++ wqflask/mock/__init__.py | 0 wqflask/mock/es_double.py | 15 +++++ wqflask/tests/__init__.py | 0 wqflask/tests/es_double.py | 30 ---------- wqflask/tests/test_registration.py | 113 ------------------------------------- 6 files changed, 42 insertions(+), 143 deletions(-) create mode 100644 test/unittest/test_registration.py create mode 100644 wqflask/mock/__init__.py create mode 100644 wqflask/mock/es_double.py delete mode 100644 wqflask/tests/__init__.py delete mode 100644 wqflask/tests/es_double.py delete mode 100644 wqflask/tests/test_registration.py diff --git a/test/unittest/test_registration.py b/test/unittest/test_registration.py new file mode 100644 index 00000000..98d0cdff --- /dev/null +++ b/test/unittest/test_registration.py @@ -0,0 +1,27 @@ +# Run test with something like +# +# env GN2_PROFILE=~/opt/gn-latest GENENETWORK_FILES=$HOME/gn2_data ./bin/genenetwork2 ./etc/default_settings.py -c ../test/unittest/test_registration.py +# + +import unittest +import mock.es_double as es +from wqflask.user_manager import RegisterUser + +class TestRegisterUser(unittest.TestCase): + def setUp(self): + self.es = es.ESDouble() + + def testRegisterUserWithCorrectData(self): + data = { + "email_address": "user@example.com" + , "full_name": "A.N. Other" + , "organization": "Some Organisation" + , "password": "testing" + , "password_confirm": "testing" + , "es_connection": self.es + } + result = RegisterUser(data) + self.assertEqual(len(result.errors), 0, "Errors were not expected") + +if __name__ == "__main__": + unittest.main() diff --git a/wqflask/mock/__init__.py b/wqflask/mock/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wqflask/mock/es_double.py b/wqflask/mock/es_double.py new file mode 100644 index 00000000..6ef8a1b9 --- /dev/null +++ b/wqflask/mock/es_double.py @@ -0,0 +1,15 @@ +class ESDouble(object): + def __init__(self): + self.items = {} + + def ping(self): + return true + + def create(self, index, doc_type, body, id): + self.items["index"] = {doc_type: {"id": id, "_source": data}} + + def search(self, index, doc_type, body): + return { + "hits": { + "hits": self.items[index][doc_type][body] + }} diff --git a/wqflask/tests/__init__.py b/wqflask/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/wqflask/tests/es_double.py b/wqflask/tests/es_double.py deleted file mode 100644 index 00739016..00000000 --- a/wqflask/tests/es_double.py +++ /dev/null @@ -1,30 +0,0 @@ -class ESDouble(object): - def __init__(self): - self.items = { - "users": { - "local": [] - }} - - def ping(self): - return true - - def create(self, index, doc_type, body, id): - item = {"id": id, "_source": body} - if not self.items.get("index", None): - self.items[index] = {doc_type: [item]} - else: - self.items[index][doc_type].append(item) - - def search(self, index, doc_type, body): - d = body["query"]["match"] - column = [(key, d[key]) for key in d] - - items = [] - for thing in self.items[index][doc_type]: - if thing["_source"][column[0][0]] == column[0][1]: - items.append(thing) - break - return { - "hits": { - "hits": items - }} diff --git a/wqflask/tests/test_registration.py b/wqflask/tests/test_registration.py deleted file mode 100644 index 50a2a84c..00000000 --- a/wqflask/tests/test_registration.py +++ /dev/null @@ -1,113 +0,0 @@ -import unittest -import es_double -import wqflask.user_manager -from wqflask.user_manager import RegisterUser - -class TestRegisterUser(unittest.TestCase): - def setUp(self): - # Mock elasticsearch - self.es = es_double.ESDouble() - - # Patch method - wqflask.user_manager.basic_info = lambda : {"basic_info": "some info"} - - def tearDown(self): - self.es = None - - def testRegisterUserWithNoData(self): - data = {} - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Data was not provided. Error was expected") - - def testRegisterUserWithNoEmail(self): - data = { - "email_address": "" - , "full_name": "A.N. Other" - , "organization": "Some Organisation" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Email not provided. Error was expected") - - def testRegisterUserWithNoName(self): - data = { - "email_address": "user@example.com" - , "full_name": "" - , "organization": "Some Organisation" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Name not provided. Error was expected") - - def testRegisterUserWithNoOrganisation(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertEqual(len(result.errors), 0, "Organisation not provided. Error not expected") - - def testRegisterUserWithShortOrganisation(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "SO" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Organisation name too short. Error expected") - - def testRegisterUserWithNoPassword(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "Some Organisation" - , "password": None - , "password_confirm": None - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Password not provided. Error was expected") - - def testRegisterUserWithNonMatchingPasswords(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "Some Organisation" - , "password": "testing" - , "password_confirm": "stilltesting" - , "es_connection": self.es - } - - result = RegisterUser(data) - self.assertNotEqual(len(result.errors), 0, "Password mismatch. Error was expected") - - def testRegisterUserWithCorrectData(self): - data = { - "email_address": "user@example.com" - , "full_name": "A.N. Other" - , "organization": "Some Organisation" - , "password": "testing" - , "password_confirm": "testing" - , "es_connection": self.es - } - result = RegisterUser(data) - self.assertEqual(len(result.errors), 0, "All data items provided. Errors were not expected") - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3 From 375935b23778148eba40ebce4a4f4def823aa3f0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 15 Feb 2018 15:00:47 +0000 Subject: Requests added --- bin/genenetwork2 | 10 ++-- bin/mechnical-rob | 111 ++++++++++++++++++++++++++++++++++++++++ bin/test-website | 114 ++---------------------------------------- test/requests/test-website.py | 20 ++++++++ 4 files changed, 142 insertions(+), 113 deletions(-) create mode 100755 bin/mechnical-rob create mode 100755 test/requests/test-website.py diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 31fefbd3..ceafeedd 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -168,8 +168,9 @@ if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} echo PYTHONPATH=$PYTHONPATH - echo RUNNING COMMAND $cmd - python $cmd + shift ; shift + echo RUNNING COMMAND $cmd $* + python $cmd $* exit $? fi # Now handle command parameter -cli which runs in bash @@ -178,8 +179,9 @@ if [ "$1" = "-cli" ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 echo PYTHONPATH=$PYTHONPATH - echo RUNNING COMMAND $cmd - $cmd + shift ; shift + echo RUNNING COMMAND $cmd $* + $cmd $* exit $? fi if [ "$1" = '-gunicorn' ] ; then diff --git a/bin/mechnical-rob b/bin/mechnical-rob new file mode 100755 index 00000000..be223d94 --- /dev/null +++ b/bin/mechnical-rob @@ -0,0 +1,111 @@ +#!/usr/bin/env ruby + + +USAGE = <0 + ARGV.shift + else + "http://localhost:5003" + end + +$stderr.print "Testing <",$host,">\n" + +require 'mechanize' +require 'minitest/spec' +require 'minitest/autorun' + +# These are the actual testing modules + +libpath = File.dirname(File.dirname(__FILE__)) +$: << File.join(libpath,'test/lib') + +require 'main_web_functionality' + +if options[:all] or options[:mapping] + require 'mapping' +end + +if options[:all] or options[:link_checker] + require 'link_checker' +end + +if options[:all] or options[:navigation] + require 'navigation' +end diff --git a/bin/test-website b/bin/test-website index be223d94..5935f016 100755 --- a/bin/test-website +++ b/bin/test-website @@ -1,111 +1,7 @@ -#!/usr/bin/env ruby +#! /bin/bash - -USAGE = <0 - ARGV.shift - else - "http://localhost:5003" - end - -$stderr.print "Testing <",$host,">\n" - -require 'mechanize' -require 'minitest/spec' -require 'minitest/autorun' - -# These are the actual testing modules - -libpath = File.dirname(File.dirname(__FILE__)) -$: << File.join(libpath,'test/lib') - -require 'main_web_functionality' - -if options[:all] or options[:mapping] - require 'mapping' -end - -if options[:all] or options[:link_checker] - require 'link_checker' -end - -if options[:all] or options[:navigation] - require 'navigation' -end +fi diff --git a/test/requests/test-website.py b/test/requests/test-website.py new file mode 100755 index 00000000..d02b71aa --- /dev/null +++ b/test/requests/test-website.py @@ -0,0 +1,20 @@ +# Run with something like +# +# env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 +# +# Mostly to pick up the Guix GN2_PROFILE and python modules + +import requests as req +import sys + +print "Mechanical Rob firing up..." + +if len(sys.argv)<1: + raise "Problem with arguments" + +url = sys.argv[1] +print url + +r = req.get(url) + +print r -- cgit v1.2.3 From c5b1028a15c771508b2b88994869912330c1148e Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 16 Feb 2018 14:22:17 +0300 Subject: Add registration test * Add integration test to check the registration process. --- test/requests/test-registration.py | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/requests/test-registration.py diff --git a/test/requests/test-registration.py b/test/requests/test-registration.py new file mode 100644 index 00000000..bcb1642f --- /dev/null +++ b/test/requests/test-registration.py @@ -0,0 +1,59 @@ +import sys +import unittest +import requests +import logging +from elasticsearch import Elasticsearch, TransportError +#from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + +GN2_SERVER = None +ES_SERVER = None + +class TestRegistration(unittest.TestCase): + + + def setUp(self): + self.url = GN2_SERVER+"/n/register" + self.es = Elasticsearch([ES_SERVER]) + self.es_cleanup = [] + + es_logger = logging.getLogger("elasticsearch") + es_logger.addHandler( + logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) + es_trace_logger = logging.getLogger("elasticsearch.trace") + es_trace_logger.addHandler( + logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) + + def tearDown(self): + for item in self.es_cleanup: + self.es.delete(index="users", doc_type="local", id=item["_id"]) + + def testRegistrationPage(self): + if self.es.ping(): + data = { + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + requests.post(self.url, data) + response = self.es.search( + index="users" + , doc_type="local" + , body={ + "query": {"match": {"email_address": "test@user.com"}}}) + self.assertEqual(len(response["hits"]["hits"]), 1) + self.es_cleanup.append(response["hits"]["hits"][0]) + else: + self.skipTest("The elasticsearch server is down") + +def main(): + suite = unittest.TestSuite() + suite.addTest(TestRegistration("testRegistrationPage")) + runner = unittest.TextTestRunner() + runner.run(suite) + +if __name__ == "__main__": + GN2_SERVER = sys.argv[1] + ES_SERVER = sys.argv[2] + main() -- cgit v1.2.3 From 0f1197ce8afdc2a869e4cc6f78122641df009e42 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 12:46:47 +0300 Subject: Rename file and move common code * Rename the file to make it an importable module * Refactor the test to move common code out to a super class. --- test/requests/test-registration.py | 59 -------------------------------------- test/requests/test_registration.py | 41 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 59 deletions(-) delete mode 100644 test/requests/test-registration.py create mode 100644 test/requests/test_registration.py diff --git a/test/requests/test-registration.py b/test/requests/test-registration.py deleted file mode 100644 index bcb1642f..00000000 --- a/test/requests/test-registration.py +++ /dev/null @@ -1,59 +0,0 @@ -import sys -import unittest -import requests -import logging -from elasticsearch import Elasticsearch, TransportError -#from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT - -GN2_SERVER = None -ES_SERVER = None - -class TestRegistration(unittest.TestCase): - - - def setUp(self): - self.url = GN2_SERVER+"/n/register" - self.es = Elasticsearch([ES_SERVER]) - self.es_cleanup = [] - - es_logger = logging.getLogger("elasticsearch") - es_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) - es_trace_logger = logging.getLogger("elasticsearch.trace") - es_trace_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) - - def tearDown(self): - for item in self.es_cleanup: - self.es.delete(index="users", doc_type="local", id=item["_id"]) - - def testRegistrationPage(self): - if self.es.ping(): - data = { - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - requests.post(self.url, data) - response = self.es.search( - index="users" - , doc_type="local" - , body={ - "query": {"match": {"email_address": "test@user.com"}}}) - self.assertEqual(len(response["hits"]["hits"]), 1) - self.es_cleanup.append(response["hits"]["hits"][0]) - else: - self.skipTest("The elasticsearch server is down") - -def main(): - suite = unittest.TestSuite() - suite.addTest(TestRegistration("testRegistrationPage")) - runner = unittest.TextTestRunner() - runner.run(suite) - -if __name__ == "__main__": - GN2_SERVER = sys.argv[1] - ES_SERVER = sys.argv[2] - main() diff --git a/test/requests/test_registration.py b/test/requests/test_registration.py new file mode 100644 index 00000000..0047e8a6 --- /dev/null +++ b/test/requests/test_registration.py @@ -0,0 +1,41 @@ +import sys +import requests +from parametrized_test import ParametrizedTest + +class TestRegistration(ParametrizedTest): + + def tearDown(self): + for item in self.es_cleanup: + self.es.delete(index="users", doc_type="local", id=item["_id"]) + + def testRegistrationPage(self): + if self.es.ping(): + data = { + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + requests.post(self.gn2_url+"/n/register", data) + response = self.es.search( + index="users" + , doc_type="local" + , body={ + "query": {"match": {"email_address": "test@user.com"}}}) + self.assertEqual(len(response["hits"]["hits"]), 1) + else: + self.skipTest("The elasticsearch server is down") + +def main(gn2, es): + import unittest + suite = unittest.TestSuite() + suite.addTest(TestRegistration(methodName="testRegistrationPage", gn2_url=gn2, es_url=es)) + runner = unittest.TextTestRunner() + runner.run(suite) + +if __name__ == "__main__": + if len(sys.argv) < 3: + raise Exception("Required arguments missing") + else: + main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From f74b1615567d5cbf2cb00572cc14450ffd4b0c1c Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 12:48:36 +0300 Subject: Create parametrized superclass for tests * Since the tests require that some parameters be provided while running the tests, create a class that helps abstract away the details of retrieving and setting the expected parameters. --- test/requests/parametrized_test.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 test/requests/parametrized_test.py diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py new file mode 100644 index 00000000..abf98fce --- /dev/null +++ b/test/requests/parametrized_test.py @@ -0,0 +1,27 @@ +import logging +import unittest +from elasticsearch import Elasticsearch, TransportError + +class ParametrizedTest(unittest.TestCase): + + def __init__(self, methodName='runTest', gn2_url="http://localhost:5003", es_url="localhost:9200"): + super(ParametrizedTest, self).__init__(methodName=methodName) + self.gn2_url = gn2_url + self.es_url = es_url + + def setUp(self): + self.es = Elasticsearch([self.es_url]) + self.es_cleanup = [] + + es_logger = logging.getLogger("elasticsearch") + es_logger.addHandler( + logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) + es_trace_logger = logging.getLogger("elasticsearch.trace") + es_trace_logger.addHandler( + logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) + + def tearDown(self): + self.es.delete_by_query( + index="users" + , doc_type="local" + , body={"query":{"match":{"email_address":"test@user.com"}}}) -- cgit v1.2.3 From c84bd98971aa5839cb6aa999889a92071890f579 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 12:50:30 +0300 Subject: Add test for local login * Add an integration test to test that the login process for users registered locally to genenetwork2 works as expected. --- test/requests/test_login_local.py | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/requests/test_login_local.py diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py new file mode 100644 index 00000000..bced1ee9 --- /dev/null +++ b/test/requests/test_login_local.py @@ -0,0 +1,56 @@ +import requests +from wqflask import user_manager +from parametrized_test import ParametrizedTest + +class TestLoginLocal(ParametrizedTest): + + def setUp(self): + super(TestLoginLocal, self).setUp() + self.login_url = self.gn2_url +"/n/login" + data = { + "es_connection": self.es, + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + user_manager.basic_info = lambda : { "basic_info": "basic" } + user_manager.RegisterUser(data) + + def testLoginNonRegisteredUser(self): + data = { + "email_address": "non@existent.email", + "password": "doesitmatter?" + } + result = requests.post(self.login_url, data=data) + self.assertEqual(result.url, self.login_url, "") + + def testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse(self): + data = { + "email_address": "test@user.com", + "password": "test_password" + } + result = requests.post(self.login_url, data=data) + print("THE COOKIES? ", result.cookies) + self.assertEqual( + result.url + , self.gn2_url+"/?import_collections=false" + , "Login should have been successful") + + + +def main(gn2, es): + import unittest + suite = unittest.TestSuite() + suite.addTest(TestLoginLocal(methodName="testLoginNonRegisteredUser", gn2_url=gn2, es_url=es)) + suite.addTest(TestLoginLocal(methodName="testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse", gn2_url=gn2, es_url=es)) + runner = unittest.TextTestRunner() + runner.run(suite) + +if __name__ == "__main__": + import sys + if len(sys.argv) < 3: + raise Exception("Required arguments missing") + else: + main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From db86c21577e50b30ee222c47aae43e49a574c007 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 12:52:32 +0300 Subject: Add a runner for all integration tests. --- test/requests/run-integration-tests.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/requests/run-integration-tests.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py new file mode 100644 index 00000000..0fd7bb20 --- /dev/null +++ b/test/requests/run-integration-tests.py @@ -0,0 +1,30 @@ +import sys +from test_login_local import TestLoginLocal +from test_registration import TestRegistration +from unittest import TestSuite, TextTestRunner, TestLoader + +test_cases = [ + TestLoginLocal, + TestRegistration +] + +def suite(gn2_url, es_url): + the_suite = TestSuite() + for case in test_cases: + the_suite.addTests(initTest(case, gn2_url, es_url)) + return the_suite + +def initTest(klass, gn2_url, es_url): + loader = TestLoader() + methodNames = loader.getTestCaseNames(klass) + return [klass(mname, gn2_url, es_url) for mname in methodNames] + +def main(gn2_url, es_url): + runner = TextTestRunner() + runner.run(suite(gn2_url, es_url)) + +if __name__ == "__main__": + if len(sys.argv) < 3: + raise Exception("Required arguments missing:\n\tTry running `run-integration-test.py `") + else: + main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From e4b7e72d83a6dba40a38c58e62e338bd257862ef Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Feb 2018 14:12:59 +0300 Subject: Add more login tests. --- test/requests/test_login_local.py | 39 +++++++++++++++++++++++++++++++++++++-- wqflask/wqflask/user_manager.py | 2 -- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index bced1ee9..acad45c9 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -32,12 +32,47 @@ class TestLoginLocal(ParametrizedTest): "password": "test_password" } result = requests.post(self.login_url, data=data) - print("THE COOKIES? ", result.cookies) self.assertEqual( result.url , self.gn2_url+"/?import_collections=false" , "Login should have been successful") - + + def testLoginWithRegisteredUserImportCollectionsTrueAndRememberMeFalse(self): + data = { + "email_address": "test@user.com", + "password": "test_password", + "import_collections": "y" + } + result = requests.post(self.login_url, data=data) + self.assertEqual( + result.url + , self.gn2_url+"/?import_collections=true" + , "Login should have been successful") + + def testLoginWithRegisteredUserImportCollectionsFalseAndRememberMeTrue(self): + data = { + "email_address": "test@user.com", + "password": "test_password", + "remember_me": "y" + } + result = requests.post(self.login_url, data=data) + self.assertEqual( + result.url + , self.gn2_url+"/?import_collections=false" + , "Login should have been successful") + + def testLoginWithRegisteredUserBothImportCollectionsAndRememberMeTrue(self): + data = { + "email_address": "test@user.com", + "password": "test_password", + "remember_me": "y", + "import_collections": "y" + } + result = requests.post(self.login_url, data=data) + self.assertEqual( + result.url + , self.gn2_url+"/?import_collections=true" + , "Login should have been successful") def main(gn2, es): diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index c8471cb1..fd1d56ff 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -642,7 +642,6 @@ class LoginUser(object): user = model.User(); for key in user_details: user.__dict__[key] = user_details[key] - print("RETRIEVED USER: ", user) valid = False; submitted_password = params['password'] @@ -689,7 +688,6 @@ class LoginUser(object): """The meat of the logging in process""" session_id_signed = self.successful_login(user, assumed_by) flash("Thank you for logging in {}.".format(user.full_name), "alert-success") - print("IMPORT1:", import_collections) response = make_response(redirect(url_for('index_page', import_collections=import_collections))) if self.remember_me: max_age = self.remember_time -- cgit v1.2.3 From baa60e0ae0513e70aa52737bf6059f2043c539bc Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 21 Feb 2018 11:46:52 +0300 Subject: Fix assumption of existing collection * When logging in, if a user selects "Import existing collections", the system would throw an exception, since Redis would return a NoneType, which would then be accessed by json.loads(), which doesn't seem to know how to process that. This fixes that, by providing a string representing an empty json array ("[]"), in place of the NoneType. --- wqflask/wqflask/user_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index fd1d56ff..e1920f4e 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -122,7 +122,8 @@ class AnonUser(object): return collections def import_traits_to_user(self): - collections_list = json.loads(Redis.get(self.key)) + result = Redis.get(self.key) + collections_list = json.loads(result if result else "[]") for collection in collections_list: uc = model.UserCollection() uc.name = collection['name'] -- cgit v1.2.3 From e1e3ea41578947e47b2f394e6e3ba4f45eceeb82 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 21 Feb 2018 12:51:34 +0300 Subject: Simplify test. Check for content, rather than url * The test functions were very similar, so this commit refactors out the common test code to a single method, and passes in the data to the test using the parameterized package. * Check that the page content after a login attempt is the expected content, rather than checking the url. --- test/requests/test_login_local.py | 93 ++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 54 deletions(-) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index acad45c9..8e2dec4e 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -1,7 +1,11 @@ import requests from wqflask import user_manager +from parameterized import parameterized from parametrized_test import ParametrizedTest +login_link_text = 'Sign in' +logout_link_text = 'Sign out' + class TestLoginLocal(ParametrizedTest): def setUp(self): @@ -18,61 +22,42 @@ class TestLoginLocal(ParametrizedTest): user_manager.basic_info = lambda : { "basic_info": "basic" } user_manager.RegisterUser(data) - def testLoginNonRegisteredUser(self): - data = { - "email_address": "non@existent.email", - "password": "doesitmatter?" - } - result = requests.post(self.login_url, data=data) - self.assertEqual(result.url, self.login_url, "") - - def testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse(self): - data = { - "email_address": "test@user.com", - "password": "test_password" - } - result = requests.post(self.login_url, data=data) - self.assertEqual( - result.url - , self.gn2_url+"/?import_collections=false" - , "Login should have been successful") - - def testLoginWithRegisteredUserImportCollectionsTrueAndRememberMeFalse(self): - data = { - "email_address": "test@user.com", - "password": "test_password", - "import_collections": "y" - } - result = requests.post(self.login_url, data=data) - self.assertEqual( - result.url - , self.gn2_url+"/?import_collections=true" - , "Login should have been successful") - - def testLoginWithRegisteredUserImportCollectionsFalseAndRememberMeTrue(self): - data = { - "email_address": "test@user.com", - "password": "test_password", - "remember_me": "y" - } - result = requests.post(self.login_url, data=data) - self.assertEqual( - result.url - , self.gn2_url+"/?import_collections=false" - , "Login should have been successful") - - def testLoginWithRegisteredUserBothImportCollectionsAndRememberMeTrue(self): - data = { - "email_address": "test@user.com", - "password": "test_password", - "remember_me": "y", - "import_collections": "y" - } + + @parameterized.expand([ + ( + { + "email_address": "non@existent.email", + "password": "doesitmatter?" + }, login_link_text, "Login should have failed with the wrong user details."), + ( + { + "email_address": "test@user.com", + "password": "test_password" + }, logout_link_text, "Login should have been successful with correct user details and neither import_collections nor remember_me set"), + ( + { + "email_address": "test@user.com", + "password": "test_password", + "import_collections": "y" + }, logout_link_text, "Login should have been successful with correct user details and only import_collections set"), + ( + { + "email_address": "test@user.com", + "password": "test_password", + "remember_me": "y" + }, logout_link_text, "Login should have been successful with correct user details and only remember_me set"), + ( + { + "email_address": "test@user.com", + "password": "test_password", + "remember_me": "y", + "import_collections": "y" + }, logout_link_text, "Login should have been successful with correct user details, and both remember_me, and import_collections set") + ]) + def testLogin(self, data, expected, message): result = requests.post(self.login_url, data=data) - self.assertEqual( - result.url - , self.gn2_url+"/?import_collections=true" - , "Login should have been successful") + index = result.content.find(expected) + self.assertTrue(index >= 0, message) def main(gn2, es): -- cgit v1.2.3 From e982d7223141d247954ab5b5c1ff9bbed907c603 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:24:55 +0300 Subject: Remove failing code * With the parameterized module, the testcase cannot be run independently at this time. --- test/requests/test_login_local.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index 8e2dec4e..808649ca 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -58,19 +58,3 @@ class TestLoginLocal(ParametrizedTest): result = requests.post(self.login_url, data=data) index = result.content.find(expected) self.assertTrue(index >= 0, message) - - -def main(gn2, es): - import unittest - suite = unittest.TestSuite() - suite.addTest(TestLoginLocal(methodName="testLoginNonRegisteredUser", gn2_url=gn2, es_url=es)) - suite.addTest(TestLoginLocal(methodName="testLoginWithRegisteredUserBothRememberMeAndImportCollectionsFalse", gn2_url=gn2, es_url=es)) - runner = unittest.TextTestRunner() - runner.run(suite) - -if __name__ == "__main__": - import sys - if len(sys.argv) < 3: - raise Exception("Required arguments missing") - else: - main(sys.argv[1], sys.argv[2]) -- cgit v1.2.3 From 7ea87a27735242889ac369dcbd5aec83a1e1caff Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:26:42 +0300 Subject: Add new test for github logins --- test/requests/run-integration-tests.py | 4 +++- test/requests/test_login_github.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 test/requests/test_login_github.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py index 0fd7bb20..fc795779 100644 --- a/test/requests/run-integration-tests.py +++ b/test/requests/run-integration-tests.py @@ -1,11 +1,13 @@ import sys from test_login_local import TestLoginLocal +from test_login_github import TestLoginGithub from test_registration import TestRegistration from unittest import TestSuite, TextTestRunner, TestLoader test_cases = [ - TestLoginLocal, TestRegistration + , TestLoginLocal + , TestLoginGithub ] def suite(gn2_url, es_url): diff --git a/test/requests/test_login_github.py b/test/requests/test_login_github.py new file mode 100644 index 00000000..15bf18ae --- /dev/null +++ b/test/requests/test_login_github.py @@ -0,0 +1,40 @@ +import uuid +import requests +from time import sleep +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +login_link_text = 'Sign in' +logout_link_text = 'Sign out' +uid = str(uuid.uuid4()) + +class TestLoginGithub(ParametrizedTest): + + def setUp(self): + super(TestLoginGithub, self).setUp() + data = { + "user_id": uid + , "name": "A. T. Est User" + , "github_id": 693024 + , "user_url": "https://fake-github.com/atestuser" + , "login_type": "github" + , "organization": "" + , "active": 1 + , "confirmed": 1 + } + self.es.create(index="users", doc_type="local", body=data, id=uid) + sleep(1) + + def tearDown(self): + super(TestLoginGithub, self).tearDown() + self.es.delete(index="users", doc_type="local", id=uid) + + @parameterized.expand([ + ("1234", login_link_text, "Login should have failed with non-existing user") + , (uid, logout_link_text, "Login should have been successful with existing user") + ]) + def testLogin(self, test_uid, expected, message): + url = self.gn2_url+"/n/login?type=github&uid="+test_uid + result = requests.get(url) + index = result.content.find(expected) + self.assertTrue(index >= 0, message) -- cgit v1.2.3 From 20d9af4812426e4457e7417cfcea601c1a870168 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:36:29 +0300 Subject: Add new tests for orcid logins --- test/requests/run-integration-tests.py | 2 ++ test/requests/test_login_orcid.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/requests/test_login_orcid.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py index fc795779..5e816549 100644 --- a/test/requests/run-integration-tests.py +++ b/test/requests/run-integration-tests.py @@ -1,5 +1,6 @@ import sys from test_login_local import TestLoginLocal +from test_login_orcid import TestLoginOrcid from test_login_github import TestLoginGithub from test_registration import TestRegistration from unittest import TestSuite, TextTestRunner, TestLoader @@ -8,6 +9,7 @@ test_cases = [ TestRegistration , TestLoginLocal , TestLoginGithub + , TestLoginOrcid ] def suite(gn2_url, es_url): diff --git a/test/requests/test_login_orcid.py b/test/requests/test_login_orcid.py new file mode 100644 index 00000000..6e40ece5 --- /dev/null +++ b/test/requests/test_login_orcid.py @@ -0,0 +1,40 @@ +import uuid +import requests +from time import sleep +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +login_link_text = 'Sign in' +logout_link_text = 'Sign out' +uid = str(uuid.uuid4()) + +class TestLoginOrcid(ParametrizedTest): + + def setUp(self): + super(TestLoginOrcid, self).setUp() + data = { + "user_id": uid + , "name": "A. T. Est User" + , "orcid": 345872 + , "user_url": "https://fake-orcid.org/atestuser" + , "login_type": "orcid" + , "organization": "" + , "active": 1 + , "confirmed": 1 + } + self.es.create(index="users", doc_type="local", body=data, id=uid) + sleep(1) + + def tearDown(self): + super(TestLoginOrcid, self).tearDown() + self.es.delete(index="users", doc_type="local", id=uid) + + @parameterized.expand([ + ("1234", login_link_text, "Login should have failed with non-existing user") + , (uid, logout_link_text, "Login should have been successful with existing user") + ]) + def testLogin(self, test_uid, expected, message): + url = self.gn2_url+"/n/login?type=orcid&uid="+test_uid + result = requests.get(url) + index = result.content.find(expected) + self.assertTrue(index >= 0, message) -- cgit v1.2.3 From 9a9fc91dad7fe66883d48826b4cf1a1892a62d54 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 17:37:06 +0300 Subject: Provide connections to elasticsearch * Provide connections to elasticsearch at various points in the code. These oversight was caught while running tests. --- wqflask/wqflask/user_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index e1920f4e..c344ea27 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -540,6 +540,7 @@ def github_oauth2(): result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} github_user = get_github_user_details(result_dict["access_token"]) + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "github_id", github_user["id"]) if user_details == None: user_details = { @@ -573,6 +574,7 @@ def orcid_oauth2(): result = requests.post(ORCID_TOKEN_URL, data=data) result_dict = json.loads(result.text.encode("utf-8")) + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "orcid", result_dict["orcid"]) if user_details == None: user_details = { @@ -606,6 +608,7 @@ class LoginUser(object): def oauth2_login(self, login_type, user_id): """Login via an OAuth2 provider""" + es = get_elasticsearch_connection() user_details = get_user_by_unique_column(es, "user_id", user_id) if user_details: user = model.User() -- cgit v1.2.3 From 07b792f0898d7ecec94b938aa3081e6aa64bc435 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 23 Feb 2018 18:11:33 +0300 Subject: Add tests to check that the UI is setup correctly * Check that the links for OAuth2 logins via ORCID and GitHub are setup correctly. --- test/requests/test_login_github.py | 7 +++++++ test/requests/test_login_orcid.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/test/requests/test_login_github.py b/test/requests/test_login_github.py index 15bf18ae..1bf4f695 100644 --- a/test/requests/test_login_github.py +++ b/test/requests/test_login_github.py @@ -1,6 +1,7 @@ import uuid import requests from time import sleep +from wqflask import app from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -29,6 +30,12 @@ class TestLoginGithub(ParametrizedTest): super(TestLoginGithub, self).tearDown() self.es.delete(index="users", doc_type="local", id=uid) + def testLoginUrl(self): + login_button_text = 'Login with Github' + result = requests.get(self.gn2_url+"/n/login") + index = result.content.find(login_button_text) + self.assertTrue(index >= 0, "Should have found `Login with Github` button") + @parameterized.expand([ ("1234", login_link_text, "Login should have failed with non-existing user") , (uid, logout_link_text, "Login should have been successful with existing user") diff --git a/test/requests/test_login_orcid.py b/test/requests/test_login_orcid.py index 6e40ece5..ea15642e 100644 --- a/test/requests/test_login_orcid.py +++ b/test/requests/test_login_orcid.py @@ -1,6 +1,7 @@ import uuid import requests from time import sleep +from wqflask import app from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -29,6 +30,12 @@ class TestLoginOrcid(ParametrizedTest): super(TestLoginOrcid, self).tearDown() self.es.delete(index="users", doc_type="local", id=uid) + def testLoginUrl(self): + login_button_text = 'a href="https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=' + app.config.get("ORCID_CLIENT_ID") + '&client_secret=' + app.config.get("ORCID_CLIENT_SECRET") + '" title="Login with ORCID" class="btn btn-info btn-group">Login with ORCID' + result = requests.get(self.gn2_url+"/n/login") + index = result.content.find(login_button_text) + self.assertTrue(index >= 0, "Should have found `Login with ORCID` button") + @parameterized.expand([ ("1234", login_link_text, "Login should have failed with non-existing user") , (uid, logout_link_text, "Login should have been successful with existing user") -- cgit v1.2.3 From 428371a67a7c742e239d96914a96558171f9f59e Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Mar 2018 18:12:58 +0300 Subject: Use argparse to handle arguments * Use argparse to handle commandline arguments. * Create initial layout of how the code might end up - lots of the code is currently commented out. --- test/requests/test-website.py | 60 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index d02b71aa..9637b87f 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -3,18 +3,60 @@ # env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 # # Mostly to pick up the Guix GN2_PROFILE and python modules +from __future__ import print_function +from link_checker import check_links +import argparse -import requests as req -import sys +print("Mechanical Rob firing up...") -print "Mechanical Rob firing up..." +def run_all(args_obj, parser): + print("") + print("Running all tests.") + check_links(args_obj, parser) + # TODO: Add other functions as they are created. -if len(sys.argv)<1: - raise "Problem with arguments" +def print_help(args_obj, parser): + print(parser.format_help()) -url = sys.argv[1] -print url +def dummy(args_obj, parser): + print("Not implemented yet.") -r = req.get(url) -print r +desc = """ +This is Mechanical-Rob - an automated web server tester for + Genenetwork.org +""" +parser = argparse.ArgumentParser(description=desc) + +parser.add_argument("-d", "--database", metavar="DB", type=str + , default="db_webqtl_s" + , help="Use database (default db_webqtl_s)") + +parser.add_argument("host", metavar="HOST", type=str + , default="http://localhost:5003" + , help="The url to the web server") + +parser.add_argument("-a", "--all", dest="accumulate", action="store_const" + , const=run_all, default=print_help + , help="Runs all tests.") + +parser.add_argument("-l", "--link-checker", dest="accumulate" + , action='store_const', const=check_links, default=print_help + , help="Checks for dead links.") + +# parser.add_argument("-n", "--navigation", dest="accumulate" +# , action="store_const", const=check_navigation, default=print_help +# , help="Checks for navigation.") + +# parser.add_argument("-m", "--mapping", dest="accumulate" +# , action="store_const", const=check_mapping, default=print_help +# , help="Checks for mapping.") + +# parser.add_argument("-s", "--skip-broken", dest="accumulate" +# , action="store_const", const=dummy, default=print_help +# , help="Skip tests that are known to be broken.") + +args = parser.parse_args() +# print("The arguments object: ", args) + +args.accumulate(args, parser) -- cgit v1.2.3 From a29c16b770aa0bb53c632eecc1764caf02789fe7 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 9 Mar 2018 18:15:19 +0300 Subject: Add link_checker module * Add the module that will hold the code to test the links on the system. --- test/requests/link_checker.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/requests/link_checker.py diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py new file mode 100644 index 00000000..f171c12a --- /dev/null +++ b/test/requests/link_checker.py @@ -0,0 +1,9 @@ +import requests + +def check_links(args_obj, parser): + print("") + print("Checking links") + print("########################") + print("Not implemented yet.") + print("This is supposed to check all links in the system.") + print("########################") -- cgit v1.2.3 From dd5ec1d3080b74fa065620f6915eeacf2cca8a2d Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 12 Mar 2018 13:00:03 +0300 Subject: Add tests to check links. --- test/requests/link_checker.py | 62 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index f171c12a..256bf6ef 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -1,9 +1,63 @@ +from __future__ import print_function +import re import requests +from lxml.html import parse +from requests.exceptions import ConnectionError + +def is_root_link(link): + pattern = re.compile("^/$") + return pattern.match(link) + +def is_mailto_link(link): + pattern = re.compile("^mailto:.*") + return pattern.match(link) + +def is_internal_link(link): + pattern = re.compile("^/.*") + return pattern.match(link) + +def get_links(doc): + return filter( + lambda x: not ( + is_root_link(x) + or is_mailto_link(x)) + , map(lambda y: y.get("href") + , doc.cssselect("a"))) + +def verify_link(link): + try: + result = requests.get(link, timeout=20) + if result.status_code == 200: + print(link+" ==> OK") + else: + print("ERROR: link `"+link+"` failed with status " + , result.status_code) + except ConnectionError as ex: + print("ERROR: ", link, ex) + +def check_page(host, start_url): + print("") + print("Checking links in page `"+start_url+"`") + doc = parse(start_url).getroot() + links = get_links(doc) + internal_links = filter(is_internal_link, links) + external_links = filter(lambda x: not is_internal_link(x), links) + external_links.append("http://somenon-existentsite.brr") + for link in internal_links: + verify_link(host+link) + + for link in external_links: + verify_link(link) def check_links(args_obj, parser): print("") print("Checking links") - print("########################") - print("Not implemented yet.") - print("This is supposed to check all links in the system.") - print("########################") + host = args_obj.host + + # Check the home page + check_page(host, host) + + # Check traits page + check_page( + host, + host+"/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") -- cgit v1.2.3 From 8ff3dbd9bd5737b6cd0df8d279a70073c40786ec Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 14 Mar 2018 02:16:59 +0300 Subject: Add tests for main web functionality --- test/requests/main_web_functionality.py | 40 +++++++++++++++++++++++++++++++++ test/requests/test-website.py | 7 ++++++ 2 files changed, 47 insertions(+) create mode 100644 test/requests/main_web_functionality.py diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py new file mode 100644 index 00000000..f6b32340 --- /dev/null +++ b/test/requests/main_web_functionality.py @@ -0,0 +1,40 @@ +from __future__ import print_function +import re +import requests +from lxml.html import parse +from requests.exceptions import ConnectionError + +def check_home(url): + doc = parse(url).getroot() + search_button = doc.cssselect("#btsearch") + assert(search_button[0].value == "Search") + print("OK") + +def check_search_page(host): + data = dict( + species="mouse" + , group="BXD" + , type="Hippocampus mRNA" + , dataset="HC_M2_0606_P" + , search_terms_or="" + , search_terms_and="MEAN=(15 16) LRS=(23 46)") + result = requests.get(host+"/search", params=data) + found = result.text.find("/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") + assert(found >= 0) + print("OK") + check_traits_page(host, "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") + +def check_traits_page(host, traits_url): + from link_checker import check_page + doc = parse(host+traits_url).getroot() + traits_form = doc.forms[1] + assert(traits_form.fields["corr_dataset"] == "HC_M2_0606_P") + print("OK") + check_page(host, host+traits_url) + +def check_main_web_functionality(args_obj, parser): + print("") + print("Checking main web functionality...") + host = args_obj.host + check_home(host) + check_search_page(host) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 9637b87f..f65c3fc8 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -4,6 +4,7 @@ # # Mostly to pick up the Guix GN2_PROFILE and python modules from __future__ import print_function +from main_web_functionality import check_main_web_functionality from link_checker import check_links import argparse @@ -12,6 +13,7 @@ print("Mechanical Rob firing up...") def run_all(args_obj, parser): print("") print("Running all tests.") + check_main_web_functionality(args_obj, parser) check_links(args_obj, parser) # TODO: Add other functions as they are created. @@ -44,6 +46,11 @@ parser.add_argument("-l", "--link-checker", dest="accumulate" , action='store_const', const=check_links, default=print_help , help="Checks for dead links.") +parser.add_argument("-f", "--main-functionality", dest="accumulate" + , action='store_const', const=check_main_web_functionality + , default=print_help + , help="Checks for main web functionality.") + # parser.add_argument("-n", "--navigation", dest="accumulate" # , action="store_const", const=check_navigation, default=print_help # , help="Checks for navigation.") -- cgit v1.2.3 From b50e82383bfbf44e35f7376bbd13dd68e4e1f201 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 18 Mar 2018 08:51:55 +0300 Subject: Check that key exists before using it * Ensure the key exists in the json object/dictionary, before trying to use it to retrieve a value. --- wqflask/wqflask/marker_regression/marker_regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index db4f8f46..3ec61e55 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -114,7 +114,7 @@ class MarkerRegression(object): self.num_perm = 0 self.perm_output = [] self.bootstrap_results = [] - self.covariates = start_vars['covariates'] + self.covariates = start_vars['covariates'] if "covariates" in start_vars else None #ZS: This is passed to GN1 code for single chr mapping self.selected_chr = -1 -- cgit v1.2.3 From 6a111c64c50ffe6daa387034ef8fc3ad3e90fc75 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 18 Mar 2018 08:54:03 +0300 Subject: Move import to the top of the page. * Mainly to tell the dependencies easily. --- test/requests/main_web_functionality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py index f6b32340..7b89b833 100644 --- a/test/requests/main_web_functionality.py +++ b/test/requests/main_web_functionality.py @@ -2,6 +2,7 @@ from __future__ import print_function import re import requests from lxml.html import parse +from link_checker import check_page from requests.exceptions import ConnectionError def check_home(url): @@ -25,7 +26,6 @@ def check_search_page(host): check_traits_page(host, "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") def check_traits_page(host, traits_url): - from link_checker import check_page doc = parse(host+traits_url).getroot() traits_form = doc.forms[1] assert(traits_form.fields["corr_dataset"] == "HC_M2_0606_P") -- cgit v1.2.3 From c3fab61242796ef4b7544d784fceb39f4545828e Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 18 Mar 2018 08:56:03 +0300 Subject: Initialise mapping tests * Add mapping tests, and build the first of the tests. --- test/requests/mapping_tests.py | 43 ++++++++++++++++++++++++++++++++++++++++++ test/requests/test-website.py | 14 ++++++++------ 2 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 test/requests/mapping_tests.py diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py new file mode 100644 index 00000000..fd20df11 --- /dev/null +++ b/test/requests/mapping_tests.py @@ -0,0 +1,43 @@ +from __future__ import print_function +import re +import json +import requests +from lxml.html import fromstring + +def get_data(list_item): + try: + value = list_item[1] + except: + value = None + #print("list_item:", list_item, "==>", value) + return value + +def load_data_from_file(): + filename = "../test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json" + file_handle = open(filename, "r") + file_data = json.loads(file_handle.read().encode("utf-8")) + return file_data + +def check_pylmm_tool_selection(host, data): + data["method"] = "pylmm" + page = requests.post(host+"/marker_regression", data=data) + doc = fromstring(page.text) + form = doc.forms[1] + assert form.fields["dataset"] == "HC_M2_0606_P" + assert form.fields["value:BXD1"] == "15.034" # Check value in the file + +def check_R_qtl_tool_selection(host, data): + pass + +def check_CIM_tool_selection(host, data): + pass + +def check_mapping(args_obj, parser): + print("") + print("Checking mapping") + + host = args_obj.host + data = load_data_from_file() + check_pylmm_tool_selection(host, data) + check_R_qtl_tool_selection(host, data) + check_CIM_tool_selection(host, data) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index f65c3fc8..2bef6eb1 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -4,9 +4,10 @@ # # Mostly to pick up the Guix GN2_PROFILE and python modules from __future__ import print_function -from main_web_functionality import check_main_web_functionality -from link_checker import check_links import argparse +from link_checker import check_links +from mapping_tests import check_mapping +from main_web_functionality import check_main_web_functionality print("Mechanical Rob firing up...") @@ -15,6 +16,7 @@ def run_all(args_obj, parser): print("Running all tests.") check_main_web_functionality(args_obj, parser) check_links(args_obj, parser) + check_mapping(args_obj, parser) # TODO: Add other functions as they are created. def print_help(args_obj, parser): @@ -51,14 +53,14 @@ parser.add_argument("-f", "--main-functionality", dest="accumulate" , default=print_help , help="Checks for main web functionality.") +parser.add_argument("-m", "--mapping", dest="accumulate" + , action="store_const", const=check_mapping, default=print_help + , help="Checks for mapping.") + # parser.add_argument("-n", "--navigation", dest="accumulate" # , action="store_const", const=check_navigation, default=print_help # , help="Checks for navigation.") -# parser.add_argument("-m", "--mapping", dest="accumulate" -# , action="store_const", const=check_mapping, default=print_help -# , help="Checks for mapping.") - # parser.add_argument("-s", "--skip-broken", dest="accumulate" # , action="store_const", const=dummy, default=print_help # , help="Skip tests that are known to be broken.") -- cgit v1.2.3 From 3c8089e2c3957ca2d4a5906249b4818d367221dc Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 19 Mar 2018 13:06:13 +0000 Subject: - Disabled PYTHONPATH injection - fix Guix instead, see https://github.com/pjotrp/genenetwork2/commit/63a5c8a42ad02e9126bb207465ff5eca98f6515d - Renamed WQFLASK_SETTINGS to GN2_SETTINGS --- bin/genenetwork2 | 33 +++++++++++++++------------------ wqflask/utility/tools.py | 15 +-------------- wqflask/wqflask/__init__.py | 4 ++-- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index ceafeedd..3f06e7f9 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -68,32 +68,29 @@ if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then echo "Can not use $1 switch without default settings file" exit 1 fi -# Handle settings parameter (can be .py or .json) -if [ ! -z $1 ]; then - settings=$(realpath "$1") - if [ ! -e $settings ]; then - settings=$GN2_BASE_DIR/etc/default_settings.py - else - shift - fi -fi -ext="${settings##*.}" -if [ "$ext" = "json" -o "$ext" = "JSON" ]; then - overrides=$settings +settings=$1 +if [ -z $settings ]; then + settings=$GN2_BASE_DIR/etc/default_settings.py else - echo $settings + shift fi +settings=$(realpath $settings) + +# ext="${settings##*.}" +# if [ "$ext" = "json" -o "$ext" = "JSON" ]; then +# overrides=$settings +# else +# echo $settings +# fi if [ ! -e $settings ]; then echo "ERROR: can not locate settings file - pass it in the command line" exit 1 fi -export WQFLASK_SETTINGS=$settings # Python -export WQFLASK_OVERRIDES=$overrides # JSON -echo WQFLASK_SETTINGS=$settings -echo WQFLASK_OVERRIDES=$overrides +export GN2_SETTINGS=$settings # Python +echo GN2_SETTINGS=$settings # This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead # if [ -z $ELASTICSEARCH_PROFILE ]; then @@ -115,7 +112,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH" + export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!! export R_LIBS_SITE=$GN2_PROFILE/site-library export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0 export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 8c9fed96..bae3a7f4 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -54,7 +54,7 @@ def get_setting(command_id,guess=None): # print("Looking for "+command_id+"\n") command = value(os.environ.get(command_id)) if command is None or command == "": - command = OVERRIDES.get(command_id) + command = OVERRIDES.get(command_id) # currently not in use if command is None: # ---- Check whether setting exists in app command = value(app.config.get(command_id)) @@ -285,18 +285,5 @@ assert_dir(JS_TWITTER_POST_FETCHER_PATH) from six import string_types -if os.environ.get('WQFLASK_OVERRIDES'): - jsonfn = get_setting('WQFLASK_OVERRIDES') - logger.info("WQFLASK_OVERRIDES: %s" % jsonfn) - with open(jsonfn) as data_file: - overrides = json.load(data_file) - for k in overrides: - cmd = overrides[k] - if isinstance(cmd, string_types): - OVERRIDES[k] = eval(cmd) - else: - OVERRIDES[k] = cmd - logger.debug(OVERRIDES) - # assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData") assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 2188ce17..bc8e9900 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -13,8 +13,8 @@ logging.basicConfig(level=logging.INFO) app = Flask(__name__) app.config.from_object('cfg.default_settings') # Get the defaults from cfg.default_settings -app.config.from_envvar('WQFLASK_SETTINGS') # See http://flask.pocoo.org/docs/config/#configuring-from-files -# Note we also use WQFLASK_OVERRIDES +app.config.from_envvar('GN2_SETTINGS') # See http://flask.pocoo.org/docs/config/#configuring-from-files +# Note no longer use the badly named WQFLASK_OVERRIDES (nyi) app.jinja_env.globals.update( undefined = jinja2.StrictUndefined, -- cgit v1.2.3 From 690a9f716219c868932d2e6fe4071633dd0257ee Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 19 Mar 2018 13:47:34 +0000 Subject: RELEASENOTES --- RELEASE-NOTES.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 RELEASE-NOTES.md diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md new file mode 100644 index 00000000..d15cad1a --- /dev/null +++ b/RELEASE-NOTES.md @@ -0,0 +1,17 @@ +## ChangeLog v2.11 (date unknown) + +This is a massive bug fix release with many improvements. For contributions +see +[contributors](https://github.com/genenetwork/genenetwork2/contributors) +and +[commits](https://github.com/genenetwork/genenetwork2/commits/master). + +### Added GEMMA support + +* Front-end support + +### Added test framework and unit tests + +* Added python integration and unit tests + + -- cgit v1.2.3 From 2b75b5d56c99d58e205aed59e721bf578489e802 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 20 Mar 2018 09:41:36 +0000 Subject: Authentication: some mods around defaults --- wqflask/utility/tools.py | 15 +++++++++------ wqflask/wqflask/templates/new_security/login_user.html | 2 +- wqflask/wqflask/user_manager.py | 13 ++++++------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index bae3a7f4..ed54f77c 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -254,15 +254,18 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = None -if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET: - GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET -GITHUB_API_URL = get_setting('GITHUB_API_URL') +if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET: + GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \ + GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET + GITHUB_API_URL = get_setting('GITHUB_API_URL') + ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = None -if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET: - ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET -ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') +if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: + ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + + ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET + ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 0dae3503..6d597f6b 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -88,7 +88,7 @@ {% else: %}
-

You cannot login at this moment using your GeneNetwork account.
+

You cannot login at this moment using your GeneNetwork account (the authentication service is down).
Please try again later.

{% endif %} diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index c344ea27..c55649f3 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -626,13 +626,12 @@ class LoginUser(object): logger.debug("in login params are:", params) es = get_elasticsearch_connection() if not params: - from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL - external_login = None - if GITHUB_AUTH_URL or ORCID_AUTH_URL: - external_login={ - "github": GITHUB_AUTH_URL, - "orcid": ORCID_AUTH_URL - } + from utility.tools import GITHUB_AUTH_URL, GITHUB_CLIENT_ID, ORCID_AUTH_URL, ORCID_CLIENT_ID + external_login = {} + if GITHUB_AUTH_URL and GITHUB_CLIENT_ID != 'UNKNOWN': + external_login["github"] = GITHUB_AUTH_URL + if ORCID_AUTH_URL and ORCID_CLIENT_ID != 'UNKNOWN': + external_login["orcid"] = ORCID_AUTH_URL assert(es is not None) return render_template( "new_security/login_user.html" -- cgit v1.2.3 From e2339a0a491f1853b8411d499be1e08bf62a2da8 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 20 Mar 2018 09:50:34 +0000 Subject: Fix --- wqflask/utility/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index ed54f77c..4b4cd633 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -263,7 +263,7 @@ ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = None if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: - ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + + ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \ ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -- cgit v1.2.3 From 7aae77762e47c9269591e138ab25320d3ed3a85f Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 20 Mar 2018 11:02:25 +0000 Subject: Refactor startup config for gunicorn and werkzeug --- wqflask/run_gunicorn.py | 6 ++-- wqflask/runserver.py | 17 ++-------- wqflask/utility/elasticsearch_tools.py | 5 +++ wqflask/utility/startup_config.py | 39 ++++++++++++++++++++++ wqflask/utility/tools.py | 4 ++- .../wqflask/templates/new_security/login_user.html | 8 ++++- 6 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 wqflask/utility/startup_config.py diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index ebe3add5..adffdca3 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -7,12 +7,12 @@ # from flask import Flask # application = Flask(__name__) -print "Starting up Gunicorn process" +print "===> Starting up Gunicorn process" from wqflask import app +from utility.startup_config import app_config -app.config['SESSION_TYPE'] = 'filesystem' -app.config['SECRET_KEY'] = 'super secret key' +app_config() @app.route("/gunicorn") def hello(): diff --git a/wqflask/runserver.py b/wqflask/runserver.py index a0c76e51..5f41d04d 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -21,22 +21,9 @@ GREEN = '\033[92m' BOLD = '\033[1m' ENDC = '\033[0m' -import os -app.config['SECRET_KEY'] = os.urandom(24) +from utility.startup_config import app_config -from utility.tools import WEBSERVER_MODE,get_setting_int,get_setting,get_setting_bool - -port = get_setting_int("SERVER_PORT") - -print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]") - -if get_setting_bool("USE_GN_SERVER"): - import requests - page = requests.get(get_setting("GN_SERVER_URL")) - if page.status_code != 200: - raise Exception("API server not found!") - -print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) +app_config() werkzeug_logger = logging.getLogger('werkzeug') diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 2d3d5add..734379f7 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -6,6 +6,11 @@ logger = getLogger(__name__) from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT +def test_elasticsearch_connection(): + es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+ELASTICSEARCH_PORT+'/'], verify_certs=True) + if not es.ping(): + logger.warning("Elasticsearch is DOWN") + def get_elasticsearch_connection(): logger.info("get_elasticsearch_connection") es = None diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py new file mode 100644 index 00000000..02e8e56c --- /dev/null +++ b/wqflask/utility/startup_config.py @@ -0,0 +1,39 @@ + +from wqflask import app +from utility.tools import WEBSERVER_MODE, show_settings, get_setting_int, get_setting, get_setting_bool + +import utility.logger +logger = utility.logger.getLogger(__name__ ) + +BLUE = '\033[94m' +GREEN = '\033[92m' +BOLD = '\033[1m' +ENDC = '\033[0m' + +def app_config(): + app.config['SESSION_TYPE'] = 'filesystem' + if not app.config.get('SECRET_KEY'): + import os + app.config['SECRET_KEY'] = str(os.urandom(24)) + + mode = WEBSERVER_MODE + if mode == "DEV" or mode == "DEBUG": + app.config['TEMPLATES_AUTO_RELOAD'] = True + if mode == "DEBUG": + app.config['EXPLAIN_TEMPLATE_LOADING'] = True + print("==========================================") + show_settings() + + port = get_setting_int("SERVER_PORT") + + if get_setting_bool("USE_GN_SERVER"): + print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]") + import requests + page = requests.get(get_setting("GN_SERVER_URL")) + if page.status_code != 200: + raise Exception("API server not found!") + + import utility.elasticsearch_tools as es + es.test_elasticsearch_connection() + + print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 4b4cd633..59bb49d8 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -220,7 +220,7 @@ def show_settings(): logger.info(OVERRIDES) logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC) - print "runserver.py: ****** Webserver configuration ******" + print "runserver.py: ****** Webserver configuration - k,v pairs from app.config ******" keylist = app.config.keys() keylist.sort() for k in keylist: @@ -269,6 +269,8 @@ if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') +import utility.elasticsearch_tools as es +es.test_elasticsearch_connection() SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 6d597f6b..949760b6 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -15,7 +15,6 @@

Don't have an account?

- {% if es_server: %} Create a new account {% else: %} @@ -92,6 +91,13 @@ Please try again later.

{% endif %} + {% if not es_server and not external_login: %} +
+
+ Note: it is safe to use GeneNetwork without a login. Login is only required for keeping track of + collections and getting access to some types of restricted data. +
+ {% endif %} -- cgit v1.2.3 From a0514e4d0119791267094581e2dc5c7eefb86b8f Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 21 Mar 2018 10:05:54 +0000 Subject: Skip automatic use of EXPLAIN_TEMPLATE_LOADING. --- wqflask/utility/startup_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 02e8e56c..5a62cc50 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -19,8 +19,8 @@ def app_config(): mode = WEBSERVER_MODE if mode == "DEV" or mode == "DEBUG": app.config['TEMPLATES_AUTO_RELOAD'] = True - if mode == "DEBUG": - app.config['EXPLAIN_TEMPLATE_LOADING'] = True + # if mode == "DEBUG": + # app.config['EXPLAIN_TEMPLATE_LOADING'] = True <--- use overriding app param instead print("==========================================") show_settings() -- cgit v1.2.3 From b386e390020581946a17b56dd3b9c841adfd4646 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 21 Mar 2018 10:13:04 +0000 Subject: Use logger.info instead of logger.debug, fixes https://github.com/genenetwork/genenetwork2/issues/296 --- wqflask/wqflask/views.py | 66 ++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 17a2d762..3ebef046 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -123,7 +123,7 @@ def handle_bad_request(e): @app.route("/") def index_page(): logger.info("Sending index_page") - logger.error(request.url) + logger.info(request.url) params = request.args if 'import_collections' in params: import_collections = params['import_collections'] @@ -141,7 +141,7 @@ def index_page(): def tmp_page(img_path): logger.info("In tmp_page") logger.info("img_path:", img_path) - logger.error(request.url) + logger.info(request.url) initial_start_vars = request.form logger.info("initial_start_vars:", initial_start_vars) imgfile = open(GENERATED_IMAGE_DIR + img_path, 'rb') @@ -174,7 +174,7 @@ def twitter(filename): @app.route("/search", methods=('GET',)) def search_page(): logger.info("in search_page") - logger.error(request.url) + logger.info(request.url) if 'info_database' in request.args: logger.info("Going to sharing_info_page") template_vars = sharing_info_page() @@ -213,7 +213,7 @@ def search_page(): @app.route("/gsearch", methods=('GET',)) def gsearchact(): - logger.error(request.url) + logger.info(request.url) result = gsearch.GSearch(request.args).__dict__ type = request.args['type'] if type == "gene": @@ -224,7 +224,7 @@ def gsearchact(): @app.route("/gsearch_updating", methods=('POST',)) def gsearch_updating(): logger.info("REQUEST ARGS:", request.values) - logger.error(request.url) + logger.info(request.url) result = update_search_results.GSearch(request.args).__dict__ return result['results'] # type = request.args['type'] @@ -235,31 +235,31 @@ def gsearch_updating(): @app.route("/docedit") def docedit(): - logger.error(request.url) + logger.info(request.url) doc = docs.Docs(request.args['entry']) return render_template("docedit.html", **doc.__dict__) @app.route('/generated/') def generated_file(filename): - logger.error(request.url) + logger.info(request.url) return send_from_directory(GENERATED_IMAGE_DIR,filename) @app.route("/help") def help(): - logger.error(request.url) + logger.info(request.url) doc = docs.Docs("help") return render_template("docs.html", **doc.__dict__) @app.route("/wgcna_setup", methods=('POST',)) def wcgna_setup(): logger.info("In wgcna, request.form is:", request.form) # We are going to get additional user input for the analysis - logger.error(request.url) + logger.info(request.url) return render_template("wgcna_setup.html", **request.form) # Display them using the template @app.route("/wgcna_results", methods=('POST',)) def wcgna_results(): logger.info("In wgcna, request.form is:", request.form) - logger.error(request.url) + logger.info(request.url) wgcna = wgcna_analysis.WGCNA() # Start R, load the package and pointers and create the analysis wgcnaA = wgcna.run_analysis(request.form) # Start the analysis, a wgcnaA object should be a separate long running thread result = wgcna.process_results(wgcnaA) # After the analysis is finished store the result @@ -268,13 +268,13 @@ def wcgna_results(): @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): logger.info("In ctl, request.form is:", request.form) # We are going to get additional user input for the analysis - logger.error(request.url) + logger.info(request.url) return render_template("ctl_setup.html", **request.form) # Display them using the template @app.route("/ctl_results", methods=('POST',)) def ctl_results(): logger.info("In ctl, request.form is:", request.form) - logger.error(request.url) + logger.info(request.url) ctl = ctl_analysis.CTL() # Start R, load the package and pointers and create the analysis ctlA = ctl.run_analysis(request.form) # Start the analysis, a ctlA object should be a separate long running thread result = ctl.process_results(ctlA) # After the analysis is finished store the result @@ -313,13 +313,13 @@ def environments(): @app.route("/submit_trait") def submit_trait_form(): - logger.error(request.url) + logger.info(request.url) species_and_groups = get_species_groups() return render_template("submit_trait.html", **{'species_and_groups' : species_and_groups, 'gn_server_url' : GN_SERVER_URL, 'version' : GN_VERSION}) @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): - logger.error(request.url) + logger.info(request.url) print("REQUEST.FORM:", request.form) #template_vars = submit_trait.SubmitTrait(request.form) @@ -332,7 +332,7 @@ def export_trait_excel(): """Excel file consisting of the sample data from the trait data and analysis page""" logger.info("In export_trait_excel") logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) sample_data = export_trait_data.export_sample_table(request.form) logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) @@ -358,7 +358,7 @@ def export_trait_csv(): """CSV file consisting of the sample data from the trait data and analysis page""" logger.info("In export_trait_csv") logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) sample_data = export_trait_data.export_sample_table(request.form) logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) @@ -379,7 +379,7 @@ def export_traits_csv(): """CSV file consisting of the traits from the search result page""" logger.info("In export_traits_csv") logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) csv_data = export_traits.export_search_results_csv(request.form) return Response(csv_data, @@ -389,7 +389,7 @@ def export_traits_csv(): @app.route('/export_perm_data', methods=('POST',)) def export_perm_data(): """CSV file consisting of the permutation data for the mapping results""" - logger.error(request.url) + logger.info(request.url) num_perm = float(request.form['num_perm']) perm_data = json.loads(request.form['perm_results']) @@ -412,7 +412,7 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): - logger.error(request.url) + logger.info(request.url) template_vars = show_trait.ShowTrait(request.form) #logger.info("js_data before dump:", template_vars.js_data) template_vars.js_data = json.dumps(template_vars.js_data, @@ -427,7 +427,7 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): - logger.error(request.url) + logger.info(request.url) template_vars = show_trait.ShowTrait(request.args) #logger.info("js_data before dump:", template_vars.js_data) template_vars.js_data = json.dumps(template_vars.js_data, @@ -443,7 +443,7 @@ def show_trait_page(): @app.route("/heatmap", methods=('POST',)) def heatmap_page(): logger.info("In heatmap, request.form is:", pf(request.form)) - logger.error(request.url) + logger.info(request.url) start_vars = request.form temp_uuid = uuid.uuid4() @@ -493,7 +493,7 @@ def mapping_results_container_page(): @app.route("/loading", methods=('POST',)) def loading_page(): - logger.error(request.url) + logger.info(request.url) initial_start_vars = request.form logger.debug("Marker regression called with initial_start_vars:", initial_start_vars.items()) #temp_uuid = initial_start_vars['temp_uuid'] @@ -552,7 +552,7 @@ def loading_page(): def marker_regression_page(): initial_start_vars = request.form logger.debug("Marker regression called with initial_start_vars:", initial_start_vars.items()) - logger.error(request.url) + logger.info(request.url) temp_uuid = initial_start_vars['temp_uuid'] wanted = ( 'trait_id', @@ -678,7 +678,7 @@ def marker_regression_page(): @app.route("/export_mapping_results", methods = ('POST',)) def export_mapping_results(): logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) file_path = request.form.get("results_path") results_csv = open(file_path, "r").read() response = Response(results_csv, @@ -691,7 +691,7 @@ def export_mapping_results(): @app.route("/export", methods = ('POST',)) def export(): logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) svg_xml = request.form.get("data", "Invalid data") filename = request.form.get("filename", "manhattan_plot_snp") response = Response(svg_xml, mimetype="image/svg+xml") @@ -702,7 +702,7 @@ def export(): def export_pdf(): import cairosvg logger.info("request.form:", request.form) - logger.error(request.url) + logger.info(request.url) svg_xml = request.form.get("data", "Invalid data") logger.info("svg_xml:", svg_xml) filename = request.form.get("filename", "interval_map_pdf") @@ -715,7 +715,7 @@ def export_pdf(): @app.route("/network_graph", methods=('POST',)) def network_graph_page(): logger.info("In network_graph, request.form is:", pf(request.form)) - logger.error(request.url) + logger.info(request.url) start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] if traits[0] != "": @@ -731,7 +731,7 @@ def network_graph_page(): @app.route("/corr_compute", methods=('POST',)) def corr_compute_page(): logger.info("In corr_compute, request.form is:", pf(request.form)) - logger.error(request.url) + logger.info(request.url) #fd = webqtlFormData.webqtlFormData(request.form) template_vars = show_corr_results.CorrelationResults(request.form) return render_template("correlation_page.html", **template_vars.__dict__) @@ -739,7 +739,7 @@ def corr_compute_page(): @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): logger.info("In corr_matrix, request.form is:", pf(request.form)) - logger.error(request.url) + logger.info(request.url) start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -755,7 +755,7 @@ def corr_matrix_page(): @app.route("/corr_scatter_plot") def corr_scatter_plot_page(): - logger.error(request.url) + logger.info(request.url) template_vars = corr_scatter_plot.CorrScatterPlot(request.args) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, @@ -764,7 +764,7 @@ def corr_scatter_plot_page(): @app.route("/submit_bnw", methods=('POST',)) def submit_bnw(): - logger.error(request.url) + logger.info(request.url) template_vars = get_bnw_input(request.form) return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) @@ -772,7 +772,7 @@ def submit_bnw(): def sharing_info_page(): """Info page displayed when the user clicks the "Info" button next to the dataset selection""" logger.info("In sharing_info_page") - logger.error(request.url) + logger.info(request.url) fd = webqtlFormData.webqtlFormData(request.args) template_vars = SharingInfoPage.SharingInfoPage(fd) return template_vars @@ -780,7 +780,7 @@ def sharing_info_page(): # Take this out or secure it before putting into production @app.route("/get_temp_data") def get_temp_data(): - logger.error(request.url) + logger.info(request.url) temp_uuid = request.args['key'] return flask.jsonify(temp_data.TempData(temp_uuid).get_all()) -- cgit v1.2.3 From deaf64d3621285382621f4579db30e79f43651cb Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 26 Mar 2018 10:14:41 +0000 Subject: Remove flask_security module - not used --- wqflask/flask_security/__init__.py | 25 -- wqflask/flask_security/changeable.py | 45 --- wqflask/flask_security/confirmable.py | 83 ----- wqflask/flask_security/core.py | 382 --------------------- wqflask/flask_security/datastore.py | 261 -------------- wqflask/flask_security/decorators.py | 207 ----------- wqflask/flask_security/forms.py | 286 --------------- wqflask/flask_security/passwordless.py | 59 ---- wqflask/flask_security/recoverable.py | 80 ----- wqflask/flask_security/registerable.py | 44 --- wqflask/flask_security/script.py | 130 ------- wqflask/flask_security/signals.py | 29 -- wqflask/flask_security/templates/.DS_Store | Bin 6148 -> 0 bytes .../flask_security/templates/security/.DS_Store | Bin 6148 -> 0 bytes .../flask_security/templates/security/_macros.html | 16 - .../flask_security/templates/security/_menu.html | 15 - .../templates/security/_messages.html | 9 - .../templates/security/change_password.html | 11 - .../templates/security/email/change_notice.html | 4 - .../templates/security/email/change_notice.txt | 5 - .../security/email/confirmation_instructions.html | 3 - .../security/email/confirmation_instructions.txt | 3 - .../security/email/login_instructions.html | 5 - .../security/email/login_instructions.txt | 5 - .../security/email/reset_instructions.html | 1 - .../security/email/reset_instructions.txt | 3 - .../templates/security/email/reset_notice.html | 1 - .../templates/security/email/reset_notice.txt | 1 - .../templates/security/email/welcome.html | 7 - .../templates/security/email/welcome.txt | 7 - .../templates/security/forgot_password.html | 9 - .../templates/security/login_user.html | 12 - .../templates/security/register_user.html | 13 - .../templates/security/reset_password.html | 10 - .../templates/security/send_confirmation.html | 9 - .../templates/security/send_login.html | 9 - wqflask/flask_security/utils.py | 379 -------------------- wqflask/flask_security/views.py | 359 ------------------- 38 files changed, 2527 deletions(-) delete mode 100644 wqflask/flask_security/__init__.py delete mode 100644 wqflask/flask_security/changeable.py delete mode 100644 wqflask/flask_security/confirmable.py delete mode 100644 wqflask/flask_security/core.py delete mode 100644 wqflask/flask_security/datastore.py delete mode 100644 wqflask/flask_security/decorators.py delete mode 100644 wqflask/flask_security/forms.py delete mode 100644 wqflask/flask_security/passwordless.py delete mode 100644 wqflask/flask_security/recoverable.py delete mode 100644 wqflask/flask_security/registerable.py delete mode 100644 wqflask/flask_security/script.py delete mode 100644 wqflask/flask_security/signals.py delete mode 100644 wqflask/flask_security/templates/.DS_Store delete mode 100644 wqflask/flask_security/templates/security/.DS_Store delete mode 100644 wqflask/flask_security/templates/security/_macros.html delete mode 100644 wqflask/flask_security/templates/security/_menu.html delete mode 100644 wqflask/flask_security/templates/security/_messages.html delete mode 100644 wqflask/flask_security/templates/security/change_password.html delete mode 100644 wqflask/flask_security/templates/security/email/change_notice.html delete mode 100644 wqflask/flask_security/templates/security/email/change_notice.txt delete mode 100644 wqflask/flask_security/templates/security/email/confirmation_instructions.html delete mode 100644 wqflask/flask_security/templates/security/email/confirmation_instructions.txt delete mode 100644 wqflask/flask_security/templates/security/email/login_instructions.html delete mode 100644 wqflask/flask_security/templates/security/email/login_instructions.txt delete mode 100644 wqflask/flask_security/templates/security/email/reset_instructions.html delete mode 100644 wqflask/flask_security/templates/security/email/reset_instructions.txt delete mode 100644 wqflask/flask_security/templates/security/email/reset_notice.html delete mode 100644 wqflask/flask_security/templates/security/email/reset_notice.txt delete mode 100644 wqflask/flask_security/templates/security/email/welcome.html delete mode 100644 wqflask/flask_security/templates/security/email/welcome.txt delete mode 100644 wqflask/flask_security/templates/security/forgot_password.html delete mode 100644 wqflask/flask_security/templates/security/login_user.html delete mode 100644 wqflask/flask_security/templates/security/register_user.html delete mode 100644 wqflask/flask_security/templates/security/reset_password.html delete mode 100644 wqflask/flask_security/templates/security/send_confirmation.html delete mode 100644 wqflask/flask_security/templates/security/send_login.html delete mode 100644 wqflask/flask_security/utils.py delete mode 100644 wqflask/flask_security/views.py diff --git a/wqflask/flask_security/__init__.py b/wqflask/flask_security/__init__.py deleted file mode 100644 index 81e6c89e..00000000 --- a/wqflask/flask_security/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security - ~~~~~~~~~~~~~~~~~~ - - Flask-Security is a Flask extension that aims to add quick and simple - security via Flask-Login, Flask-Principal, Flask-WTF, and passlib. - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -__version__ = '1.6.0' - -from .core import Security, RoleMixin, UserMixin, AnonymousUser, current_user -from .datastore import SQLAlchemyUserDatastore, MongoEngineUserDatastore, PeeweeUserDatastore -from .decorators import auth_token_required, http_auth_required, \ - login_required, roles_accepted, roles_required -from .forms import ForgotPasswordForm, LoginForm, RegisterForm, \ - ResetPasswordForm, PasswordlessLoginForm, ConfirmRegisterForm -from .signals import confirm_instructions_sent, password_reset, \ - reset_password_instructions_sent, user_confirmed, user_registered -from .utils import login_user, logout_user, url_for_security - -print "Using our own flask.ext.security" \ No newline at end of file diff --git a/wqflask/flask_security/changeable.py b/wqflask/flask_security/changeable.py deleted file mode 100644 index 4447b655..00000000 --- a/wqflask/flask_security/changeable.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.changeable - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security recoverable module - - :copyright: (c) 2012 by Matt Wright. - :author: Eskil Heyn Olsen - :license: MIT, see LICENSE for more details. -""" - -from flask import current_app as app, request -from werkzeug.local import LocalProxy - -from .signals import password_changed -from .utils import send_mail, encrypt_password, url_for_security, \ - config_value - - -# Convenient references -_security = LocalProxy(lambda: app.extensions['security']) - -_datastore = LocalProxy(lambda: _security.datastore) - - -def send_password_changed_notice(user): - """Sends the password changed notice email for the specified user. - - :param user: The user to send the notice to - """ - send_mail(config_value('EMAIL_SUBJECT_PASSWORD_CHANGE_NOTICE'), user.email, - 'change_notice', user=user) - - -def change_user_password(user, password): - """Change the specified user's password - - :param user: The user to change_password - :param password: The unencrypted new password - """ - user.password = encrypt_password(password) - _datastore.put(user) - send_password_changed_notice(user) - password_changed.send(user, app=app._get_current_object()) diff --git a/wqflask/flask_security/confirmable.py b/wqflask/flask_security/confirmable.py deleted file mode 100644 index a7caf6cd..00000000 --- a/wqflask/flask_security/confirmable.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.confirmable - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security confirmable module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -from datetime import datetime - -from flask import current_app as app, request -from werkzeug.local import LocalProxy - -from .utils import send_mail, md5, url_for_security, get_token_status,\ - config_value -from .signals import user_confirmed, confirm_instructions_sent - - -# Convenient references -_security = LocalProxy(lambda: app.extensions['security']) - -_datastore = LocalProxy(lambda: _security.datastore) - - -def generate_confirmation_link(user): - token = generate_confirmation_token(user) - url = url_for_security('confirm_email', token=token) - return request.url_root[:-1] + url, token - - -def send_confirmation_instructions(user): - """Sends the confirmation instructions email for the specified user. - - :param user: The user to send the instructions to - :param token: The confirmation token - """ - - confirmation_link, token = generate_confirmation_link(user) - - send_mail(config_value('EMAIL_SUBJECT_CONFIRM'), user.email, - 'confirmation_instructions', user=user, - confirmation_link=confirmation_link) - - confirm_instructions_sent.send(user, app=app._get_current_object()) - return token - - -def generate_confirmation_token(user): - """Generates a unique confirmation token for the specified user. - - :param user: The user to work with - """ - data = [str(user.id), md5(user.email)] - return _security.confirm_serializer.dumps(data) - - -def requires_confirmation(user): - """Returns `True` if the user requires confirmation.""" - return _security.confirmable and user.confirmed_at == None - - -def confirm_email_token_status(token): - """Returns the expired status, invalid status, and user of a confirmation - token. For example:: - - expired, invalid, user = confirm_email_token_status('...') - - :param token: The confirmation token - """ - return get_token_status(token, 'confirm', 'CONFIRM_EMAIL') - - -def confirm_user(user): - """Confirms the specified user - - :param user: The user to confirm - """ - user.confirmed_at = datetime.utcnow() - _datastore.put(user) - user_confirmed.send(user, app=app._get_current_object()) diff --git a/wqflask/flask_security/core.py b/wqflask/flask_security/core.py deleted file mode 100644 index 0f3a231f..00000000 --- a/wqflask/flask_security/core.py +++ /dev/null @@ -1,382 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.core - ~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security core module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -from flask import current_app -from flask.ext.login import AnonymousUser as AnonymousUserBase, \ - UserMixin as BaseUserMixin, LoginManager, current_user -from flask.ext.principal import Principal, RoleNeed, UserNeed, Identity, \ - identity_loaded -from itsdangerous import URLSafeTimedSerializer -from passlib.context import CryptContext -from werkzeug.datastructures import ImmutableList -from werkzeug.local import LocalProxy - -from .utils import config_value as cv, get_config, md5, url_for_security -from .views import create_blueprint -from .forms import LoginForm, ConfirmRegisterForm, RegisterForm, \ - ForgotPasswordForm, ChangePasswordForm, ResetPasswordForm, \ - SendConfirmationForm, PasswordlessLoginForm - -# Convenient references -_security = LocalProxy(lambda: current_app.extensions['security']) - - -#: Default Flask-Security configuration -_default_config = { - 'BLUEPRINT_NAME': 'security', - 'URL_PREFIX': None, - 'SUBDOMAIN': None, - 'FLASH_MESSAGES': True, - 'PASSWORD_HASH': 'plaintext', - 'PASSWORD_SALT': None, - 'LOGIN_URL': '/login', - 'LOGOUT_URL': '/logout', - 'REGISTER_URL': '/register', - 'RESET_URL': '/reset', - 'CHANGE_URL': '/change', - 'CONFIRM_URL': '/confirm', - 'POST_LOGIN_VIEW': '/', - 'POST_LOGOUT_VIEW': '/', - 'CONFIRM_ERROR_VIEW': None, - 'POST_REGISTER_VIEW': None, - 'POST_CONFIRM_VIEW': None, - 'POST_RESET_VIEW': None, - 'POST_CHANGE_VIEW': None, - 'UNAUTHORIZED_VIEW': None, - 'FORGOT_PASSWORD_TEMPLATE': 'security/forgot_password.html', - 'LOGIN_USER_TEMPLATE': 'security/login_user.html', - 'REGISTER_USER_TEMPLATE': 'security/register_user.html', - 'RESET_PASSWORD_TEMPLATE': 'security/reset_password.html', - 'SEND_CONFIRMATION_TEMPLATE': 'security/send_confirmation.html', - 'SEND_LOGIN_TEMPLATE': 'security/send_login.html', - 'CONFIRMABLE': False, - 'REGISTERABLE': False, - 'RECOVERABLE': False, - 'TRACKABLE': False, - 'PASSWORDLESS': False, - 'CHANGEABLE': False, - 'SEND_REGISTER_EMAIL': True, - 'LOGIN_WITHIN': '1 days', - 'CONFIRM_EMAIL_WITHIN': '5 days', - 'RESET_PASSWORD_WITHIN': '5 days', - 'LOGIN_WITHOUT_CONFIRMATION': False, - 'EMAIL_SENDER': 'no-reply@localhost', - 'TOKEN_AUTHENTICATION_KEY': 'auth_token', - 'TOKEN_AUTHENTICATION_HEADER': 'Authentication-Token', - 'CONFIRM_SALT': 'confirm-salt', - 'RESET_SALT': 'reset-salt', - 'LOGIN_SALT': 'login-salt', - 'CHANGE_SALT': 'change-salt', - 'REMEMBER_SALT': 'remember-salt', - 'DEFAULT_HTTP_AUTH_REALM': 'Login Required', - 'EMAIL_SUBJECT_REGISTER': 'Welcome', - 'EMAIL_SUBJECT_CONFIRM': 'Please confirm your email', - 'EMAIL_SUBJECT_PASSWORDLESS': 'Login instructions', - 'EMAIL_SUBJECT_PASSWORD_NOTICE': 'Your password has been reset', - 'EMAIL_SUBJECT_PASSWORD_CHANGE_NOTICE': 'Your password has been changed', - 'EMAIL_SUBJECT_PASSWORD_RESET': 'Password reset instructions' -} - -#: Default Flask-Security messages -_default_messages = { - 'UNAUTHORIZED': ('You do not have permission to view this resource.', 'error'), - 'CONFIRM_REGISTRATION': ('Thank you. Confirmation instructions have been sent to %(email)s.', 'success'), - 'EMAIL_CONFIRMED': ('Thank you. Your email has been confirmed.', 'success'), - 'ALREADY_CONFIRMED': ('Your email has already been confirmed.', 'info'), - 'INVALID_CONFIRMATION_TOKEN': ('Invalid confirmation token.', 'error'), - 'EMAIL_ALREADY_ASSOCIATED': ('%(email)s is already associated with an account.', 'error'), - 'PASSWORD_MISMATCH': ('Password does not match', 'error'), - 'RETYPE_PASSWORD_MISMATCH': ('Passwords do not match', 'error'), - 'INVALID_REDIRECT': ('Redirections outside the domain are forbidden', 'error'), - 'PASSWORD_RESET_REQUEST': ('Instructions to reset your password have been sent to %(email)s.', 'info'), - 'PASSWORD_RESET_EXPIRED': ('You did not reset your password within %(within)s. New instructions have been sent to %(email)s.', 'error'), - 'INVALID_RESET_PASSWORD_TOKEN': ('Invalid reset password token.', 'error'), - 'CONFIRMATION_REQUIRED': ('Email requires confirmation.', 'error'), - 'CONFIRMATION_REQUEST': ('Confirmation instructions have been sent to %(email)s.', 'info'), - 'CONFIRMATION_EXPIRED': ('You did not confirm your email within %(within)s. New instructions to confirm your email have been sent to %(email)s.', 'error'), - 'LOGIN_EXPIRED': ('You did not login within %(within)s. New instructions to login have been sent to %(email)s.', 'error'), - 'LOGIN_EMAIL_SENT': ('Instructions to login have been sent to %(email)s.', 'success'), - 'INVALID_LOGIN_TOKEN': ('Invalid login token.', 'error'), - 'DISABLED_ACCOUNT': ('Account is disabled.', 'error'), - 'EMAIL_NOT_PROVIDED': ('Email not provided', 'error'), - 'INVALID_EMAIL_ADDRESS': ('Invalid email address', 'error'), - 'PASSWORD_NOT_PROVIDED': ('Password not provided', 'error'), - 'USER_DOES_NOT_EXIST': ('Specified user does not exist', 'error'), - 'INVALID_PASSWORD': ('Invalid password', 'error'), - 'PASSWORDLESS_LOGIN_SUCCESSFUL': ('You have successfuly logged in.', 'success'), - 'PASSWORD_RESET': ('You successfully reset your password and you have been logged in automatically.', 'success'), - 'PASSWORD_CHANGE': ('You successfully changed your password.', 'success'), - 'LOGIN': ('Please log in to access this page.', 'info'), - 'REFRESH': ('Please reauthenticate to access this page.', 'info'), -} - -_allowed_password_hash_schemes = [ - 'bcrypt', - 'des_crypt', - 'pbkdf2_sha256', - 'pbkdf2_sha512', - 'sha256_crypt', - 'sha512_crypt', - # And always last one... - 'plaintext' -] - -_default_forms = { - 'login_form': LoginForm, - 'confirm_register_form': ConfirmRegisterForm, - 'register_form': RegisterForm, - 'forgot_password_form': ForgotPasswordForm, - 'reset_password_form': ResetPasswordForm, - 'change_password_form': ChangePasswordForm, - 'send_confirmation_form': SendConfirmationForm, - 'passwordless_login_form': PasswordlessLoginForm, -} - - -def _user_loader(user_id): - return _security.datastore.find_user(id=user_id) - - -def _token_loader(token): - try: - data = _security.remember_token_serializer.loads(token) - user = _security.datastore.find_user(id=data[0]) - if user and md5(user.password) == data[1]: - return user - except: - pass - - return None - - -def _identity_loader(): - if not isinstance(current_user._get_current_object(), AnonymousUser): - identity = Identity(current_user.id) - return identity - - -def _on_identity_loaded(sender, identity): - if hasattr(current_user, 'id'): - identity.provides.add(UserNeed(current_user.id)) - - for role in current_user.roles: - identity.provides.add(RoleNeed(role.name)) - - identity.user = current_user - - -def _get_login_manager(app): - lm = LoginManager() - lm.anonymous_user = AnonymousUser - lm.login_view = '%s.login' % cv('BLUEPRINT_NAME', app=app) - lm.user_loader(_user_loader) - lm.token_loader(_token_loader) - lm.login_message, lm.login_message_category = cv('MSG_LOGIN', app=app) - lm.needs_refresh_message, lm.needs_refresh_message_category = cv('MSG_REFRESH', app=app) - lm.init_app(app) - return lm - - -def _get_principal(app): - p = Principal(app, use_sessions=False) - p.identity_loader(_identity_loader) - return p - - -def _get_pwd_context(app): - pw_hash = cv('PASSWORD_HASH', app=app) - if pw_hash not in _allowed_password_hash_schemes: - allowed = ', '.join(_allowed_password_hash_schemes[:-1]) + ' and ' + _allowed_password_hash_schemes[-1] - raise ValueError("Invalid hash scheme %r. Allowed values are %s" % (pw_hash, allowed)) - return CryptContext(schemes=_allowed_password_hash_schemes, default=pw_hash) - - -def _get_serializer(app, name): - secret_key = app.config.get('SECRET_KEY') - salt = app.config.get('SECURITY_%s_SALT' % name.upper()) - return URLSafeTimedSerializer(secret_key=secret_key, salt=salt) - - -def _get_state(app, datastore, **kwargs): - for key, value in get_config(app).items(): - print "in _get_state [{}]: {}".format(key, value) - kwargs[key.lower()] = value - - kwargs.update(dict( - app=app, - datastore=datastore, - login_manager=_get_login_manager(app), - principal=_get_principal(app), - pwd_context=_get_pwd_context(app), - remember_token_serializer=_get_serializer(app, 'remember'), - login_serializer=_get_serializer(app, 'login'), - reset_serializer=_get_serializer(app, 'reset'), - confirm_serializer=_get_serializer(app, 'confirm'), - _context_processors={}, - _send_mail_task=None - )) - - for key, value in _default_forms.items(): - if key not in kwargs or not kwargs[key]: - kwargs[key] = value - - return _SecurityState(**kwargs) - - -def _context_processor(): - return dict(url_for_security=url_for_security, security=_security) - - -class RoleMixin(object): - """Mixin for `Role` model definitions""" - def __eq__(self, other): - return (self.name == other or \ - self.name == getattr(other, 'name', None)) - - def __ne__(self, other): - return (self.name != other and - self.name != getattr(other, 'name', None)) - - -class UserMixin(BaseUserMixin): - """Mixin for `User` model definitions""" - - def is_active(self): - """Returns `True` if the user is active.""" - return self.active - - def get_auth_token(self): - """Returns the user's authentication token.""" - data = [str(self.id), md5(self.password)] - return _security.remember_token_serializer.dumps(data) - - def has_role(self, role): - """Returns `True` if the user identifies with the specified role. - - :param role: A role name or `Role` instance""" - return role in self.roles - - -class AnonymousUser(AnonymousUserBase): - """AnonymousUser definition""" - - def __init__(self): - super(AnonymousUser, self).__init__() - self.roles = ImmutableList() - - def has_role(self, *args): - """Returns `False`""" - return False - - -class _SecurityState(object): - - def __init__(self, **kwargs): - for key, value in kwargs.items(): - setattr(self, key.lower(), value) - - def _add_ctx_processor(self, endpoint, fn): - group = self._context_processors.setdefault(endpoint, []) - fn not in group and group.append(fn) - - def _run_ctx_processor(self, endpoint): - rv, fns = {}, [] - for g in [None, endpoint]: - for fn in self._context_processors.setdefault(g, []): - rv.update(fn()) - return rv - - def context_processor(self, fn): - self._add_ctx_processor(None, fn) - - def forgot_password_context_processor(self, fn): - self._add_ctx_processor('forgot_password', fn) - - def login_context_processor(self, fn): - self._add_ctx_processor('login', fn) - - def register_context_processor(self, fn): - self._add_ctx_processor('register', fn) - - def reset_password_context_processor(self, fn): - self._add_ctx_processor('reset_password', fn) - - def change_password_context_processor(self, fn): - self._add_ctx_processor('change_password', fn) - - def send_confirmation_context_processor(self, fn): - self._add_ctx_processor('send_confirmation', fn) - - def send_login_context_processor(self, fn): - self._add_ctx_processor('send_login', fn) - - def mail_context_processor(self, fn): - self._add_ctx_processor('mail', fn) - - def send_mail_task(self, fn): - self._send_mail_task = fn - - -class Security(object): - """The :class:`Security` class initializes the Flask-Security extension. - - :param app: The application. - :param datastore: An instance of a user datastore. - """ - def __init__(self, app=None, datastore=None, **kwargs): - self.app = app - self.datastore = datastore - - if app is not None and datastore is not None: - self._state = self.init_app(app, datastore, **kwargs) - - def init_app(self, app, datastore=None, register_blueprint=True, - login_form=None, confirm_register_form=None, - register_form=None, forgot_password_form=None, - reset_password_form=None, change_password_form=None, - send_confirmation_form=None, passwordless_login_form=None): - """Initializes the Flask-Security extension for the specified - application and datastore implentation. - - :param app: The application. - :param datastore: An instance of a user datastore. - :param register_blueprint: to register the Security blueprint or not. - """ - datastore = datastore or self.datastore - - for key, value in _default_config.items(): - app.config.setdefault('SECURITY_' + key, value) - - for key, value in _default_messages.items(): - app.config.setdefault('SECURITY_MSG_' + key, value) - - identity_loaded.connect_via(app)(_on_identity_loaded) - - state = _get_state(app, datastore, - login_form=login_form, - confirm_register_form=confirm_register_form, - register_form=register_form, - forgot_password_form=forgot_password_form, - reset_password_form=reset_password_form, - change_password_form=change_password_form, - send_confirmation_form=send_confirmation_form, - passwordless_login_form=passwordless_login_form) - - if register_blueprint: - app.register_blueprint(create_blueprint(state, __name__)) - app.context_processor(_context_processor) - - app.extensions['security'] = state - - return state - - def __getattr__(self, name): - return getattr(self._state, name, None) diff --git a/wqflask/flask_security/datastore.py b/wqflask/flask_security/datastore.py deleted file mode 100644 index 634399d9..00000000 --- a/wqflask/flask_security/datastore.py +++ /dev/null @@ -1,261 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.datastore - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - This module contains an user datastore classes. - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -class Datastore(object): - def __init__(self, db): - self.db = db - - def commit(self): - pass - - def put(self, model): - raise NotImplementedError - - def delete(self, model): - raise NotImplementedError - - -class SQLAlchemyDatastore(Datastore): - def commit(self): - self.db.session.commit() - - def put(self, model): - self.db.session.add(model) - return model - - def delete(self, model): - self.db.session.delete(model) - - -class MongoEngineDatastore(Datastore): - def put(self, model): - model.save() - return model - - def delete(self, model): - model.delete() - - -class PeeweeDatastore(Datastore): - def put(self, model): - model.save() - return model - - def delete(self, model): - model.delete_instance() - - -class UserDatastore(object): - """Abstracted user datastore. - - :param user_model: A user model class definition - :param role_model: A role model class definition - """ - - def __init__(self, user_model, role_model): - self.user_model = user_model - self.role_model = role_model - - def _prepare_role_modify_args(self, user, role): - if isinstance(user, basestring): - user = self.find_user(email=user) - if isinstance(role, basestring): - role = self.find_role(role) - return user, role - - def _prepare_create_user_args(self, **kwargs): - kwargs.setdefault('active', True) - roles = kwargs.get('roles', []) - for i, role in enumerate(roles): - rn = role.name if isinstance(role, self.role_model) else role - # see if the role exists - roles[i] = self.find_role(rn) - kwargs['roles'] = roles - return kwargs - - def find_user(self, *args, **kwargs): - """Returns a user matching the provided parameters.""" - raise NotImplementedError - - def find_role(self, *args, **kwargs): - """Returns a role matching the provided name.""" - raise NotImplementedError - - def add_role_to_user(self, user, role): - """Adds a role tp a user - - :param user: The user to manipulate - :param role: The role to add to the user - """ - rv = False - user, role = self._prepare_role_modify_args(user, role) - if role not in user.roles: - rv = True - user.roles.append(role) - return rv - - def remove_role_from_user(self, user, role): - """Removes a role from a user - - :param user: The user to manipulate - :param role: The role to remove from the user - """ - rv = False - user, role = self._prepare_role_modify_args(user, role) - if role in user.roles: - rv = True - user.roles.remove(role) - return rv - - def toggle_active(self, user): - """Toggles a user's active status. Always returns True.""" - user.active = not user.active - return True - - def deactivate_user(self, user): - """Deactivates a specified user. Returns `True` if a change was made. - - :param user: The user to deactivate - """ - if user.active: - user.active = False - return True - return False - - def activate_user(self, user): - """Activates a specified user. Returns `True` if a change was made. - - :param user: The user to activate - """ - if not user.active: - user.active = True - return True - return False - - def create_role(self, **kwargs): - """Creates and returns a new role from the given parameters.""" - - role = self.role_model(**kwargs) - return self.put(role) - - def find_or_create_role(self, name, **kwargs): - """Returns a role matching the given name or creates it with any - additionally provided parameters - """ - kwargs["name"] = name - return self.find_role(name) or self.create_role(**kwargs) - - def create_user(self, **kwargs): - """Creates and returns a new user from the given parameters.""" - - user = self.user_model(**self._prepare_create_user_args(**kwargs)) - print "in abstraced create_user, user is:", user - return self.put(user) - - def delete_user(self, user): - """Delete the specified user - - :param user: The user to delete - """ - self.delete(user) - - -class SQLAlchemyUserDatastore(SQLAlchemyDatastore, UserDatastore): - """A SQLAlchemy datastore implementation for Flask-Security that assumes the - use of the Flask-SQLAlchemy extension. - """ - def __init__(self, db, user_model, role_model): - SQLAlchemyDatastore.__init__(self, db) - UserDatastore.__init__(self, user_model, role_model) - - def find_user(self, **kwargs): - return self.user_model.query.filter_by(**kwargs).first() - - def find_role(self, role): - return self.role_model.query.filter_by(name=role).first() - - -class MongoEngineUserDatastore(MongoEngineDatastore, UserDatastore): - """A MongoEngine datastore implementation for Flask-Security that assumes - the use of the Flask-MongoEngine extension. - """ - def __init__(self, db, user_model, role_model): - MongoEngineDatastore.__init__(self, db) - UserDatastore.__init__(self, user_model, role_model) - - def find_user(self, **kwargs): - return self.user_model.objects(**kwargs).first() - - def find_role(self, role): - return self.role_model.objects(name=role).first() - - -class PeeweeUserDatastore(PeeweeDatastore, UserDatastore): - """A PeeweeD datastore implementation for Flask-Security that assumes - the use of the Flask-Peewee extension. - - :param user_model: A user model class definition - :param role_model: A role model class definition - :param role_link: A model implementing the many-to-many user-role relation - """ - def __init__(self, db, user_model, role_model, role_link): - PeeweeDatastore.__init__(self, db) - UserDatastore.__init__(self, user_model, role_model) - self.UserRole = role_link - - def find_user(self, **kwargs): - try: - return self.user_model.filter(**kwargs).get() - except self.user_model.DoesNotExist: - return None - - def find_role(self, role): - try: - return self.role_model.filter(name=role).get() - except self.role_model.DoesNotExist: - return None - - def create_user(self, **kwargs): - """Creates and returns a new user from the given parameters.""" - roles = kwargs.pop('roles', []) - user = self.user_model(**self._prepare_create_user_args(**kwargs)) - user = self.put(user) - for role in roles: - self.add_role_to_user(user, role) - return user - - - def add_role_to_user(self, user, role): - """Adds a role tp a user - - :param user: The user to manipulate - :param role: The role to add to the user - """ - user, role = self._prepare_role_modify_args(user, role) - if self.UserRole.select().where(self.UserRole.user==user, self.UserRole.role==role).count(): - return False - else: - self.UserRole.create(user=user, role=role) - return True - - def remove_role_from_user(self, user, role): - """Removes a role from a user - - :param user: The user to manipulate - :param role: The role to remove from the user - """ - user, role = self._prepare_role_modify_args(user, role) - if self.UserRole.select().where(self.UserRole.user==user, self.UserRole.role==role).count(): - self.UserRole.delete().where(self.UserRole.user==user, self.UserRole.role==role) - return True - else: - return False - diff --git a/wqflask/flask_security/decorators.py b/wqflask/flask_security/decorators.py deleted file mode 100644 index 0ea1105c..00000000 --- a/wqflask/flask_security/decorators.py +++ /dev/null @@ -1,207 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.decorators - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security decorators module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -from functools import wraps - -from flask import current_app, Response, request, redirect, _request_ctx_stack -from flask.ext.login import current_user, login_required -from flask.ext.principal import RoleNeed, Permission, Identity, identity_changed -from werkzeug.local import LocalProxy - -from . import utils - - -# Convenient references -_security = LocalProxy(lambda: current_app.extensions['security']) - - -_default_unauthorized_html = """ -

Unauthorized

-

The server could not verify that you are authorized to access the URL - requested. You either supplied the wrong credentials (e.g. a bad password), - or your browser doesn't understand how to supply the credentials required.

- """ - - -def _get_unauthorized_response(text=None, headers=None): - text = text or _default_unauthorized_html - headers = headers or {} - return Response(text, 401, headers) - - -def _get_unauthorized_view(): - cv = utils.get_url(utils.config_value('UNAUTHORIZED_VIEW')) - utils.do_flash(*utils.get_message('UNAUTHORIZED')) - return redirect(cv or request.referrer or '/') - - -def _check_token(): - header_key = _security.token_authentication_header - args_key = _security.token_authentication_key - header_token = request.headers.get(header_key, None) - token = request.args.get(args_key, header_token) - if request.json: - token = request.json.get(args_key, token) - serializer = _security.remember_token_serializer - - try: - data = serializer.loads(token) - except: - return False - - user = _security.datastore.find_user(id=data[0]) - - if utils.md5(user.password) == data[1]: - app = current_app._get_current_object() - _request_ctx_stack.top.user = user - identity_changed.send(app, identity=Identity(user.id)) - return True - - -def _check_http_auth(): - auth = request.authorization or dict(username=None, password=None) - user = _security.datastore.find_user(email=auth.username) - - if user and utils.verify_and_update_password(auth.password, user): - _security.datastore.commit() - app = current_app._get_current_object() - _request_ctx_stack.top.user = user - identity_changed.send(app, identity=Identity(user.id)) - return True - - return False - - -def http_auth_required(realm): - """Decorator that protects endpoints using Basic HTTP authentication. - The username should be set to the user's email address. - - :param realm: optional realm name""" - - def decorator(fn): - @wraps(fn) - def wrapper(*args, **kwargs): - if _check_http_auth(): - return fn(*args, **kwargs) - r = _security.default_http_auth_realm if callable(realm) else realm - h = {'WWW-Authenticate': 'Basic realm="%s"' % r} - return _get_unauthorized_response(headers=h) - return wrapper - - if callable(realm): - return decorator(realm) - return decorator - - -def auth_token_required(fn): - """Decorator that protects endpoints using token authentication. The token - should be added to the request by the client by using a query string - variable with a name equal to the configuration value of - `SECURITY_TOKEN_AUTHENTICATION_KEY` or in a request header named that of - the configuration value of `SECURITY_TOKEN_AUTHENTICATION_HEADER` - """ - - @wraps(fn) - def decorated(*args, **kwargs): - if _check_token(): - return fn(*args, **kwargs) - return _get_unauthorized_response() - return decorated - - -def auth_required(*auth_methods): - """ - Decorator that protects enpoints through multiple mechanisms - Example:: - - @app.route('/dashboard') - @auth_required('token', 'session') - def dashboard(): - return 'Dashboard' - - :param auth_methods: Specified mechanisms. - """ - login_mechanisms = { - 'token': lambda: _check_token(), - 'basic': lambda: _check_http_auth(), - 'session': lambda: current_user.is_authenticated() - } - - def wrapper(fn): - @wraps(fn) - def decorated_view(*args, **kwargs): - mechanisms = [login_mechanisms.get(method) for method in auth_methods] - for mechanism in mechanisms: - if mechanism and mechanism(): - return fn(*args, **kwargs) - return _get_unauthorized_response() - return decorated_view - return wrapper - - -def roles_required(*roles): - """Decorator which specifies that a user must have all the specified roles. - Example:: - - @app.route('/dashboard') - @roles_required('admin', 'editor') - def dashboard(): - return 'Dashboard' - - The current user must have both the `admin` role and `editor` role in order - to view the page. - - :param args: The required roles. - """ - def wrapper(fn): - @wraps(fn) - def decorated_view(*args, **kwargs): - perms = [Permission(RoleNeed(role)) for role in roles] - for perm in perms: - if not perm.can(): - return _get_unauthorized_view() - return fn(*args, **kwargs) - return decorated_view - return wrapper - - -def roles_accepted(*roles): - """Decorator which specifies that a user must have at least one of the - specified roles. Example:: - - @app.route('/create_post') - @roles_accepted('editor', 'author') - def create_post(): - return 'Create Post' - - The current user must have either the `editor` role or `author` role in - order to view the page. - - :param args: The possible roles. - """ - def wrapper(fn): - @wraps(fn) - def decorated_view(*args, **kwargs): - perm = Permission(*[RoleNeed(role) for role in roles]) - if perm.can(): - return fn(*args, **kwargs) - return _get_unauthorized_view() - return decorated_view - return wrapper - - -def anonymous_user_required(f): - @wraps(f) - def wrapper(*args, **kwargs): - if current_user.is_authenticated(): - return redirect(utils.get_url(_security.post_login_view)) - return f(*args, **kwargs) - return wrapper diff --git a/wqflask/flask_security/forms.py b/wqflask/flask_security/forms.py deleted file mode 100644 index 54677e77..00000000 --- a/wqflask/flask_security/forms.py +++ /dev/null @@ -1,286 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.forms - ~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security forms module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -import inspect -import urlparse - -import flask_wtf as wtf - -from flask import request, current_app -from flask_wtf import Form as BaseForm, TextField, PasswordField, \ - SubmitField, HiddenField, BooleanField, ValidationError, Field -from flask_login import current_user -from werkzeug.local import LocalProxy - -from .confirmable import requires_confirmation -from .utils import verify_and_update_password, get_message - -# Convenient reference -_datastore = LocalProxy(lambda: current_app.extensions['security'].datastore) - -_default_field_labels = { - 'email': 'Email Address', - 'password': 'Password', - 'remember_me': 'Remember Me', - 'login': 'Login', - 'retype_password': 'Retype Password', - 'register': 'Register', - 'send_confirmation': 'Resend Confirmation Instructions', - 'recover_password': 'Recover Password', - 'reset_password': 'Reset Password', - 'retype_password': 'Retype Password', - 'new_password': 'New Password', - 'change_password': 'Change Password', - 'send_login_link': 'Send Login Link' -} - - -class ValidatorMixin(object): - def __call__(self, form, field): - if self.message and self.message.isupper(): - self.message = get_message(self.message)[0] - return super(ValidatorMixin, self).__call__(form, field) - - -class EqualTo(ValidatorMixin, wtf.EqualTo): - pass - - -class Required(ValidatorMixin, wtf.Required): - pass - - -class Email(ValidatorMixin, wtf.Email): - pass - - -class Length(ValidatorMixin, wtf.Length): - pass - - -email_required = Required(message='EMAIL_NOT_PROVIDED') -email_validator = Email(message='INVALID_EMAIL_ADDRESS') -password_required = Required(message='PASSWORD_NOT_PROVIDED') - - -def get_form_field_label(key): - return _default_field_labels.get(key, '') - - -def unique_user_email(form, field): - if _datastore.find_user(email=field.data) is not None: - msg = get_message('EMAIL_ALREADY_ASSOCIATED', email=field.data)[0] - raise ValidationError(msg) - - -def valid_user_email(form, field): - form.user = _datastore.find_user(email=field.data) - if form.user is None: - raise ValidationError(get_message('USER_DOES_NOT_EXIST')[0]) - - -class Form(BaseForm): - def __init__(self, *args, **kwargs): - #print "importing tracer" - #from wqflask import tracer - #tracer.turn_on() - #print "imported tracer" - print "in Form, args:", args - print "in Form, kwargs:", kwargs - if current_app.testing: - self.TIME_LIMIT = None - super(Form, self).__init__(*args, **kwargs) - - -class EmailFormMixin(): - email = TextField(get_form_field_label('email'), - validators=[email_required, - email_validator]) - - -class UserEmailFormMixin(): - user = None - email = TextField(get_form_field_label('email'), - validators=[email_required, - email_validator, - valid_user_email]) - - -class UniqueEmailFormMixin(): - email = TextField(get_form_field_label('email'), - validators=[email_required, - email_validator, - unique_user_email]) - - -class PasswordFormMixin(): - password = PasswordField(get_form_field_label('password'), - validators=[password_required]) - - -class NewPasswordFormMixin(): - password = PasswordField(get_form_field_label('password'), - validators=[password_required, - Length(min=6, max=128)]) - - -class PasswordConfirmFormMixin(): - password_confirm = PasswordField( - get_form_field_label('retype_password'), - validators=[EqualTo('password', message='RETYPE_PASSWORD_MISMATCH')]) - - -class NextFormMixin(): - next = HiddenField() - - def validate_next(self, field): - url_next = urlparse.urlsplit(field.data) - url_base = urlparse.urlsplit(request.host_url) - if url_next.netloc and url_next.netloc != url_base.netloc: - field.data = '' - raise ValidationError(get_message('INVALID_REDIRECT')[0]) - - -class RegisterFormMixin(): - submit = SubmitField(get_form_field_label('register')) - - def to_dict(form): - def is_field_and_user_attr(member): - print "in ifaua:", member - return isinstance(member, Field) and \ - hasattr(_datastore.user_model, member.name) - - print("green:", vars(form)) - fields = inspect.getmembers(form, is_field_and_user_attr) - print("fields:" ,vars(form)) - return dict((key, value.data) for key, value in fields) - - -class SendConfirmationForm(Form, UserEmailFormMixin): - """The default forgot password form""" - - submit = SubmitField(get_form_field_label('send_confirmation')) - - def __init__(self, *args, **kwargs): - super(SendConfirmationForm, self).__init__(*args, **kwargs) - if request.method == 'GET': - self.email.data = request.args.get('email', None) - - def validate(self): - if not super(SendConfirmationForm, self).validate(): - return False - if self.user.confirmed_at is not None: - self.email.errors.append(get_message('ALREADY_CONFIRMED')[0]) - return False - return True - - -class ForgotPasswordForm(Form, UserEmailFormMixin): - """The default forgot password form""" - - submit = SubmitField(get_form_field_label('recover_password')) - - -class PasswordlessLoginForm(Form, UserEmailFormMixin): - """The passwordless login form""" - - submit = SubmitField(get_form_field_label('send_login_link')) - - def __init__(self, *args, **kwargs): - super(PasswordlessLoginForm, self).__init__(*args, **kwargs) - - def validate(self): - if not super(PasswordlessLoginForm, self).validate(): - return False - if not self.user.is_active(): - self.email.errors.append(get_message('DISABLED_ACCOUNT')[0]) - return False - return True - - -class LoginForm(Form, NextFormMixin): - """The default login form""" - - email = TextField(get_form_field_label('email')) - password = PasswordField(get_form_field_label('password')) - remember = BooleanField(get_form_field_label('remember_me')) - submit = SubmitField(get_form_field_label('login')) - - def __init__(self, *args, **kwargs): - super(LoginForm, self).__init__(*args, **kwargs) - - def validate(self): - if not super(LoginForm, self).validate(): - return False - - if self.email.data.strip() == '': - self.email.errors.append(get_message('EMAIL_NOT_PROVIDED')[0]) - return False - - if self.password.data.strip() == '': - self.password.errors.append(get_message('PASSWORD_NOT_PROVIDED')[0]) - return False - - self.user = _datastore.find_user(email=self.email.data) - - if self.user is None: - self.email.errors.append(get_message('USER_DOES_NOT_EXIST')[0]) - return False - if not verify_and_update_password(self.password.data, self.user): - self.password.errors.append(get_message('INVALID_PASSWORD')[0]) - return False - if requires_confirmation(self.user): - self.email.errors.append(get_message('CONFIRMATION_REQUIRED')[0]) - return False - if not self.user.is_active(): - self.email.errors.append(get_message('DISABLED_ACCOUNT')[0]) - return False - return True - - -class ConfirmRegisterForm(Form, RegisterFormMixin, - UniqueEmailFormMixin, NewPasswordFormMixin): - pass - - -class RegisterForm(ConfirmRegisterForm, PasswordConfirmFormMixin): - pass - - -class ResetPasswordForm(Form, NewPasswordFormMixin, PasswordConfirmFormMixin): - """The default reset password form""" - - submit = SubmitField(get_form_field_label('reset_password')) - - -class ChangePasswordForm(Form, PasswordFormMixin): - """The default change password form""" - - new_password = PasswordField(get_form_field_label('new_password'), - validators=[password_required, - Length(min=6, max=128)]) - - new_password_confirm = PasswordField(get_form_field_label('retype_password'), - validators=[EqualTo('new_password', message='RETYPE_PASSWORD_MISMATCH')]) - - submit = SubmitField(get_form_field_label('change_password')) - - def validate(self): - if not super(ChangePasswordForm, self).validate(): - return False - - if self.password.data.strip() == '': - self.password.errors.append(get_message('PASSWORD_NOT_PROVIDED')[0]) - return False - if not verify_and_update_password(self.password.data, current_user): - self.password.errors.append(get_message('INVALID_PASSWORD')[0]) - return False - return True diff --git a/wqflask/flask_security/passwordless.py b/wqflask/flask_security/passwordless.py deleted file mode 100644 index b0accb2c..00000000 --- a/wqflask/flask_security/passwordless.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.passwordless - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security passwordless module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -from flask import request, current_app as app -from werkzeug.local import LocalProxy - -from .signals import login_instructions_sent -from .utils import send_mail, url_for_security, get_token_status, \ - config_value - - -# Convenient references -_security = LocalProxy(lambda: app.extensions['security']) - -_datastore = LocalProxy(lambda: _security.datastore) - - -def send_login_instructions(user): - """Sends the login instructions email for the specified user. - - :param user: The user to send the instructions to - :param token: The login token - """ - token = generate_login_token(user) - url = url_for_security('token_login', token=token) - login_link = request.url_root[:-1] + url - - send_mail(config_value('EMAIL_SUBJECT_PASSWORDLESS'), user.email, - 'login_instructions', user=user, login_link=login_link) - - login_instructions_sent.send(dict(user=user, login_token=token), - app=app._get_current_object()) - - -def generate_login_token(user): - """Generates a unique login token for the specified user. - - :param user: The user the token belongs to - """ - return _security.login_serializer.dumps([str(user.id)]) - - -def login_token_status(token): - """Returns the expired status, invalid status, and user of a login token. - For example:: - - expired, invalid, user = login_token_status('...') - - :param token: The login token - """ - return get_token_status(token, 'login', 'LOGIN') diff --git a/wqflask/flask_security/recoverable.py b/wqflask/flask_security/recoverable.py deleted file mode 100644 index 6aafc111..00000000 --- a/wqflask/flask_security/recoverable.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.recoverable - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security recoverable module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -from flask import current_app as app, request -from werkzeug.local import LocalProxy - -from .signals import password_reset, reset_password_instructions_sent -from .utils import send_mail, md5, encrypt_password, url_for_security, \ - get_token_status, config_value - - -# Convenient references -_security = LocalProxy(lambda: app.extensions['security']) - -_datastore = LocalProxy(lambda: _security.datastore) - - -def send_reset_password_instructions(user): - """Sends the reset password instructions email for the specified user. - - :param user: The user to send the instructions to - """ - token = generate_reset_password_token(user) - url = url_for_security('reset_password', token=token) - reset_link = request.url_root[:-1] + url - - send_mail(config_value('EMAIL_SUBJECT_PASSWORD_RESET'), user.email, - 'reset_instructions', - user=user, reset_link=reset_link) - - reset_password_instructions_sent.send(dict(user=user, token=token), - app=app._get_current_object()) - - -def send_password_reset_notice(user): - """Sends the password reset notice email for the specified user. - - :param user: The user to send the notice to - """ - send_mail(config_value('EMAIL_SUBJECT_PASSWORD_NOTICE'), user.email, - 'reset_notice', user=user) - - -def generate_reset_password_token(user): - """Generates a unique reset password token for the specified user. - - :param user: The user to work with - """ - data = [str(user.id), md5(user.password)] - return _security.reset_serializer.dumps(data) - - -def reset_password_token_status(token): - """Returns the expired status, invalid status, and user of a password reset - token. For example:: - - expired, invalid, user = reset_password_token_status('...') - - :param token: The password reset token - """ - return get_token_status(token, 'reset', 'RESET_PASSWORD') - -def update_password(user, password): - """Update the specified user's password - - :param user: The user to update_password - :param password: The unencrypted new password - """ - user.password = encrypt_password(password) - _datastore.put(user) - send_password_reset_notice(user) - password_reset.send(user, app=app._get_current_object()) diff --git a/wqflask/flask_security/registerable.py b/wqflask/flask_security/registerable.py deleted file mode 100644 index 4606c7c6..00000000 --- a/wqflask/flask_security/registerable.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.registerable - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security registerable module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -from flask import current_app as app -from werkzeug.local import LocalProxy - -from .confirmable import generate_confirmation_link -from .signals import user_registered -from .utils import do_flash, get_message, send_mail, encrypt_password, \ - config_value - -# Convenient references -_security = LocalProxy(lambda: app.extensions['security']) - -_datastore = LocalProxy(lambda: _security.datastore) - - -def register_user(**kwargs): - print "in register_user kwargs:", kwargs - confirmation_link, token = None, None - kwargs['password'] = encrypt_password(kwargs['password']) - user = _datastore.create_user(**kwargs) - _datastore.commit() - - if _security.confirmable: - confirmation_link, token = generate_confirmation_link(user) - do_flash(*get_message('CONFIRM_REGISTRATION', email=user.email)) - - user_registered.send(dict(user=user, confirm_token=token), - app=app._get_current_object()) - - if config_value('SEND_REGISTER_EMAIL'): - send_mail(config_value('EMAIL_SUBJECT_REGISTER'), user.email, 'welcome', - user=user, confirmation_link=confirmation_link) - - return user diff --git a/wqflask/flask_security/script.py b/wqflask/flask_security/script.py deleted file mode 100644 index 9c9a2469..00000000 --- a/wqflask/flask_security/script.py +++ /dev/null @@ -1,130 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.script - ~~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security script module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" -try: - import simplejson as json -except ImportError: - import json - -import re - -from flask import current_app -from flask.ext.script import Command, Option -from werkzeug.local import LocalProxy - -from .utils import encrypt_password - - -_datastore = LocalProxy(lambda: current_app.extensions['security'].datastore) - - -def pprint(obj): - print json.dumps(obj, sort_keys=True, indent=4) - - -def commit(fn): - def wrapper(*args, **kwargs): - fn(*args, **kwargs) - _datastore.commit() - return wrapper - - -class CreateUserCommand(Command): - """Create a user""" - - option_list = ( - Option('-e', '--email', dest='email', default=None), - Option('-p', '--password', dest='password', default=None), - Option('-a', '--active', dest='active', default=''), - ) - - @commit - def run(self, **kwargs): - # sanitize active input - ai = re.sub(r'\s', '', str(kwargs['active'])) - kwargs['active'] = ai.lower() in ['', 'y', 'yes', '1', 'active'] - - from flask_security.forms import ConfirmRegisterForm - from werkzeug.datastructures import MultiDict - - form = ConfirmRegisterForm(MultiDict(kwargs), csrf_enabled=False) - - if form.validate(): - kwargs['password'] = encrypt_password(kwargs['password']) - _datastore.create_user(**kwargs) - print 'User created successfully.' - kwargs['password'] = '****' - pprint(kwargs) - else: - print 'Error creating user' - pprint(form.errors) - - -class CreateRoleCommand(Command): - """Create a role""" - - option_list = ( - Option('-n', '--name', dest='name', default=None), - Option('-d', '--desc', dest='description', default=None), - ) - - @commit - def run(self, **kwargs): - _datastore.create_role(**kwargs) - print 'Role "%(name)s" created successfully.' % kwargs - - -class _RoleCommand(Command): - option_list = ( - Option('-u', '--user', dest='user_identifier'), - Option('-r', '--role', dest='role_name'), - ) - - -class AddRoleCommand(_RoleCommand): - """Add a role to a user""" - - @commit - def run(self, user_identifier, role_name): - _datastore.add_role_to_user(user_identifier, role_name) - print "Role '%s' added to user '%s' successfully" % (role_name, user_identifier) - - -class RemoveRoleCommand(_RoleCommand): - """Add a role to a user""" - - @commit - def run(self, user_identifier, role_name): - _datastore.remove_role_from_user(user_identifier, role_name) - print "Role '%s' removed from user '%s' successfully" % (role_name, user_identifier) - - -class _ToggleActiveCommand(Command): - option_list = ( - Option('-u', '--user', dest='user_identifier'), - ) - - -class DeactivateUserCommand(_ToggleActiveCommand): - """Deactive a user""" - - @commit - def run(self, user_identifier): - _datastore.deactivate_user(user_identifier) - print "User '%s' has been deactivated" % user_identifier - - -class ActivateUserCommand(_ToggleActiveCommand): - """Deactive a user""" - - @commit - def run(self, user_identifier): - _datastore.activate_user(user_identifier) - print "User '%s' has been activated" % user_identifier diff --git a/wqflask/flask_security/signals.py b/wqflask/flask_security/signals.py deleted file mode 100644 index e1c29548..00000000 --- a/wqflask/flask_security/signals.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.signals - ~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security signals module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -import blinker - - -signals = blinker.Namespace() - -user_registered = signals.signal("user-registered") - -user_confirmed = signals.signal("user-confirmed") - -confirm_instructions_sent = signals.signal("confirm-instructions-sent") - -login_instructions_sent = signals.signal("login-instructions-sent") - -password_reset = signals.signal("password-reset") - -password_changed = signals.signal("password-changed") - -reset_password_instructions_sent = signals.signal("password-reset-instructions-sent") diff --git a/wqflask/flask_security/templates/.DS_Store b/wqflask/flask_security/templates/.DS_Store deleted file mode 100644 index b72f1d98..00000000 Binary files a/wqflask/flask_security/templates/.DS_Store and /dev/null differ diff --git a/wqflask/flask_security/templates/security/.DS_Store b/wqflask/flask_security/templates/security/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/wqflask/flask_security/templates/security/.DS_Store and /dev/null differ diff --git a/wqflask/flask_security/templates/security/_macros.html b/wqflask/flask_security/templates/security/_macros.html deleted file mode 100644 index 8575f3db..00000000 --- a/wqflask/flask_security/templates/security/_macros.html +++ /dev/null @@ -1,16 +0,0 @@ -{% macro render_field_with_errors(field) %} -

- {{ field.label }} {{ field(**kwargs)|safe }} - {% if field.errors %} -

    - {% for error in field.errors %} -
  • {{ error }}
    • - {% endfor %} -
- {% endif %} -

-{% endmacro %} - -{% macro render_field(field) %} -

{{ field(**kwargs)|safe }}

-{% endmacro %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/_menu.html b/wqflask/flask_security/templates/security/_menu.html deleted file mode 100644 index 5291f809..00000000 --- a/wqflask/flask_security/templates/security/_menu.html +++ /dev/null @@ -1,15 +0,0 @@ -{% if security.registerable or security.recoverable or security.confirmabled %} -

Menu

- -{% endif %} diff --git a/wqflask/flask_security/templates/security/_messages.html b/wqflask/flask_security/templates/security/_messages.html deleted file mode 100644 index 179d0636..00000000 --- a/wqflask/flask_security/templates/security/_messages.html +++ /dev/null @@ -1,9 +0,0 @@ -{%- with messages = get_flashed_messages(with_categories=true) -%} - {% if messages %} -
    - {% for category, message in messages %} -
  • {{ message }}
    • - {% endfor %} -
- {% endif %} -{%- endwith %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/change_password.html b/wqflask/flask_security/templates/security/change_password.html deleted file mode 100644 index 8ee3eb73..00000000 --- a/wqflask/flask_security/templates/security/change_password.html +++ /dev/null @@ -1,11 +0,0 @@ -{% from "security/_macros.html" import render_field_with_errors, render_field %} -{% include "security/_messages.html" %} -

Change password

-
- {{ change_password_form.hidden_tag() }} - {{ render_field_with_errors(change_password_form.password) }} - {{ render_field_with_errors(change_password_form.new_password) }} - {{ render_field_with_errors(change_password_form.new_password_confirm) }} - {{ render_field(change_password_form.submit) }} -
- diff --git a/wqflask/flask_security/templates/security/email/change_notice.html b/wqflask/flask_security/templates/security/email/change_notice.html deleted file mode 100644 index d1224cf5..00000000 --- a/wqflask/flask_security/templates/security/email/change_notice.html +++ /dev/null @@ -1,4 +0,0 @@ -

Your password has been changed.

-{% if security.recoverable %} -

If you did not change your password, click here to reset it.

-{% endif %} diff --git a/wqflask/flask_security/templates/security/email/change_notice.txt b/wqflask/flask_security/templates/security/email/change_notice.txt deleted file mode 100644 index e74bd80d..00000000 --- a/wqflask/flask_security/templates/security/email/change_notice.txt +++ /dev/null @@ -1,5 +0,0 @@ -Your password has been changed -{% if security.recoverable %} -If you did not change your password, click the link below to reset it. -{{ url_for_security('forgot_password', _external=True) }} -{% endif %} diff --git a/wqflask/flask_security/templates/security/email/confirmation_instructions.html b/wqflask/flask_security/templates/security/email/confirmation_instructions.html deleted file mode 100644 index 5082a9a8..00000000 --- a/wqflask/flask_security/templates/security/email/confirmation_instructions.html +++ /dev/null @@ -1,3 +0,0 @@ -

Please confirm your email through the link below:

- -

Confirm my account

\ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/confirmation_instructions.txt b/wqflask/flask_security/templates/security/email/confirmation_instructions.txt deleted file mode 100644 index fb435b55..00000000 --- a/wqflask/flask_security/templates/security/email/confirmation_instructions.txt +++ /dev/null @@ -1,3 +0,0 @@ -Please confirm your email through the link below: - -{{ confirmation_link }} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/login_instructions.html b/wqflask/flask_security/templates/security/email/login_instructions.html deleted file mode 100644 index 45a7cb57..00000000 --- a/wqflask/flask_security/templates/security/email/login_instructions.html +++ /dev/null @@ -1,5 +0,0 @@ -

Welcome {{ user.email }}!

- -

You can log into your through the link below:

- -

Login now

\ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/login_instructions.txt b/wqflask/flask_security/templates/security/email/login_instructions.txt deleted file mode 100644 index 1364ed65..00000000 --- a/wqflask/flask_security/templates/security/email/login_instructions.txt +++ /dev/null @@ -1,5 +0,0 @@ -Welcome {{ user.email }}! - -You can log into your through the link below: - -{{ login_link }} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/reset_instructions.html b/wqflask/flask_security/templates/security/email/reset_instructions.html deleted file mode 100644 index fd0b48d8..00000000 --- a/wqflask/flask_security/templates/security/email/reset_instructions.html +++ /dev/null @@ -1 +0,0 @@ -

Click here to reset your password

\ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/reset_instructions.txt b/wqflask/flask_security/templates/security/email/reset_instructions.txt deleted file mode 100644 index 91ac288e..00000000 --- a/wqflask/flask_security/templates/security/email/reset_instructions.txt +++ /dev/null @@ -1,3 +0,0 @@ -Click the link below to reset your password: - -{{ reset_link }} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/reset_notice.html b/wqflask/flask_security/templates/security/email/reset_notice.html deleted file mode 100644 index 536e2961..00000000 --- a/wqflask/flask_security/templates/security/email/reset_notice.html +++ /dev/null @@ -1 +0,0 @@ -

Your password has been reset

\ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/reset_notice.txt b/wqflask/flask_security/templates/security/email/reset_notice.txt deleted file mode 100644 index a3fa0b4b..00000000 --- a/wqflask/flask_security/templates/security/email/reset_notice.txt +++ /dev/null @@ -1 +0,0 @@ -Your password has been reset \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/welcome.html b/wqflask/flask_security/templates/security/email/welcome.html deleted file mode 100644 index 55eaed61..00000000 --- a/wqflask/flask_security/templates/security/email/welcome.html +++ /dev/null @@ -1,7 +0,0 @@ -

Welcome {{ user.email }}!

- -{% if security.confirmable %} -

You can confirm your email through the link below:

- -

Confirm my account

-{% endif %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/email/welcome.txt b/wqflask/flask_security/templates/security/email/welcome.txt deleted file mode 100644 index fb6ee5b5..00000000 --- a/wqflask/flask_security/templates/security/email/welcome.txt +++ /dev/null @@ -1,7 +0,0 @@ -Welcome {{ user.email }}! - -{% if security.confirmable %} -You can confirm your email through the link below: - -{{ confirmation_link }} -{% endif %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/forgot_password.html b/wqflask/flask_security/templates/security/forgot_password.html deleted file mode 100644 index 90fcaf66..00000000 --- a/wqflask/flask_security/templates/security/forgot_password.html +++ /dev/null @@ -1,9 +0,0 @@ -{% from "security/_macros.html" import render_field_with_errors, render_field %} -{% include "security/_messages.html" %} -

Send password reset instructions

-
- {{ forgot_password_form.hidden_tag() }} - {{ render_field_with_errors(forgot_password_form.email) }} - {{ render_field(forgot_password_form.submit) }} -
-{% include "security/_menu.html" %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/login_user.html b/wqflask/flask_security/templates/security/login_user.html deleted file mode 100644 index d781ce08..00000000 --- a/wqflask/flask_security/templates/security/login_user.html +++ /dev/null @@ -1,12 +0,0 @@ -{% from "security/_macros.html" import render_field_with_errors, render_field %} -{% include "security/_messages.html" %} -

Login

-
- {{ login_user_form.hidden_tag() }} - {{ render_field_with_errors(login_user_form.email) }} - {{ render_field_with_errors(login_user_form.password) }} - {{ render_field_with_errors(login_user_form.remember) }} - {{ render_field(login_user_form.next) }} - {{ render_field(login_user_form.submit) }} -
-{% include "security/_menu.html" %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/register_user.html b/wqflask/flask_security/templates/security/register_user.html deleted file mode 100644 index 87cf9b1d..00000000 --- a/wqflask/flask_security/templates/security/register_user.html +++ /dev/null @@ -1,13 +0,0 @@ -{% from "security/_macros.html" import render_field_with_errors, render_field %} -{% include "security/_messages.html" %} -

Register

-
- {{ register_user_form.hidden_tag() }} - {{ render_field_with_errors(register_user_form.email) }} - {{ render_field_with_errors(register_user_form.password) }} - {% if register_user_form.password_confirm %} - {{ render_field_with_errors(register_user_form.password_confirm) }} - {% endif %} - {{ render_field(register_user_form.submit) }} -
-{% include "security/_menu.html" %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/reset_password.html b/wqflask/flask_security/templates/security/reset_password.html deleted file mode 100644 index e6fc3f58..00000000 --- a/wqflask/flask_security/templates/security/reset_password.html +++ /dev/null @@ -1,10 +0,0 @@ -{% from "security/_macros.html" import render_field_with_errors, render_field %} -{% include "security/_messages.html" %} -

Reset password

-
- {{ reset_password_form.hidden_tag() }} - {{ render_field_with_errors(reset_password_form.password) }} - {{ render_field_with_errors(reset_password_form.password_confirm) }} - {{ render_field(reset_password_form.submit) }} -
-{% include "security/_menu.html" %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/send_confirmation.html b/wqflask/flask_security/templates/security/send_confirmation.html deleted file mode 100644 index 3e828407..00000000 --- a/wqflask/flask_security/templates/security/send_confirmation.html +++ /dev/null @@ -1,9 +0,0 @@ -{% from "security/_macros.html" import render_field_with_errors, render_field %} -{% include "security/_messages.html" %} -

Resend confirmation instructions

-
- {{ send_confirmation_form.hidden_tag() }} - {{ render_field_with_errors(send_confirmation_form.email) }} - {{ render_field(send_confirmation_form.submit) }} -
-{% include "security/_menu.html" %} \ No newline at end of file diff --git a/wqflask/flask_security/templates/security/send_login.html b/wqflask/flask_security/templates/security/send_login.html deleted file mode 100644 index 15611c57..00000000 --- a/wqflask/flask_security/templates/security/send_login.html +++ /dev/null @@ -1,9 +0,0 @@ -{% from "security/_macros.html" import render_field_with_errors, render_field %} -{% include "security/_messages.html" %} -

Login

-
- {{ send_login_form.hidden_tag() }} - {{ render_field_with_errors(send_login_form.email) }} - {{ render_field(send_login_form.submit) }} -
-{% include "security/_menu.html" %} \ No newline at end of file diff --git a/wqflask/flask_security/utils.py b/wqflask/flask_security/utils.py deleted file mode 100644 index 7397ab4f..00000000 --- a/wqflask/flask_security/utils.py +++ /dev/null @@ -1,379 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.utils - ~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security utils module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -import base64 -import blinker -import functools -import hashlib -import hmac -from contextlib import contextmanager -from datetime import datetime, timedelta - -from flask import url_for, flash, current_app, request, session, render_template -from flask.ext.login import login_user as _login_user, \ - logout_user as _logout_user -from flask.ext.mail import Message -from flask.ext.principal import Identity, AnonymousIdentity, identity_changed -from itsdangerous import BadSignature, SignatureExpired -from werkzeug.local import LocalProxy - -from .signals import user_registered, user_confirmed, \ - confirm_instructions_sent, login_instructions_sent, \ - password_reset, password_changed, reset_password_instructions_sent - -# Convenient references -_security = LocalProxy(lambda: current_app.extensions['security']) - -_datastore = LocalProxy(lambda: _security.datastore) - -_pwd_context = LocalProxy(lambda: _security.pwd_context) - - -def login_user(user, remember=True): - """Performs the login and sends the appropriate signal.""" - - if not _login_user(user, remember): - return False - - if _security.trackable: - old_current_login, new_current_login = user.current_login_at, datetime.utcnow() - remote_addr = request.remote_addr or 'untrackable' - old_current_ip, new_current_ip = user.current_login_ip, remote_addr - - user.last_login_at = old_current_login or new_current_login - user.current_login_at = new_current_login - user.last_login_ip = old_current_ip or new_current_ip - user.current_login_ip = new_current_ip - user.login_count = user.login_count + 1 if user.login_count else 1 - - _datastore.put(user) - - identity_changed.send(current_app._get_current_object(), - identity=Identity(user.id)) - return True - - -def logout_user(): - for key in ('identity.name', 'identity.auth_type'): - session.pop(key, None) - identity_changed.send(current_app._get_current_object(), - identity=AnonymousIdentity()) - _logout_user() - - -def get_hmac(password): - if _security.password_hash == 'plaintext': - return password - - if _security.password_salt is None: - raise RuntimeError('The configuration value `SECURITY_PASSWORD_SALT` ' - 'must not be None when the value of `SECURITY_PASSWORD_HASH` is ' - 'set to "%s"' % _security.password_hash) - - h = hmac.new(_security.password_salt, password.encode('utf-8'), hashlib.sha512) - return base64.b64encode(h.digest()) - - -def verify_password(password, password_hash): - return _pwd_context.verify(get_hmac(password), password_hash) - - -def verify_and_update_password(password, user): - verified, new_password = _pwd_context.verify_and_update(get_hmac(password), user.password) - if verified and new_password: - user.password = new_password - _datastore.put(user) - return verified - - -def encrypt_password(password): - return _pwd_context.encrypt(get_hmac(password)) - - -def md5(data): - return hashlib.md5(data).hexdigest() - - -def do_flash(message, category=None): - """Flash a message depending on if the `FLASH_MESSAGES` configuration - value is set. - - :param message: The flash message - :param category: The flash message category - """ - if config_value('FLASH_MESSAGES'): - flash(message, category) - - -def get_url(endpoint_or_url): - """Returns a URL if a valid endpoint is found. Otherwise, returns the - provided value. - - :param endpoint_or_url: The endpoint name or URL to default to - """ - try: - return url_for(endpoint_or_url) - except: - return endpoint_or_url - - -def get_security_endpoint_name(endpoint): - return '%s.%s' % (_security.blueprint_name, endpoint) - - -def url_for_security(endpoint, **values): - """Return a URL for the security blueprint - - :param endpoint: the endpoint of the URL (name of the function) - :param values: the variable arguments of the URL rule - :param _external: if set to `True`, an absolute URL is generated. Server - address can be changed via `SERVER_NAME` configuration variable which - defaults to `localhost`. - :param _anchor: if provided this is added as anchor to the URL. - :param _method: if provided this explicitly specifies an HTTP method. - """ - endpoint = get_security_endpoint_name(endpoint) - return url_for(endpoint, **values) - - -def get_post_login_redirect(): - """Returns the URL to redirect to after a user logs in successfully.""" - return (get_url(request.args.get('next')) or - get_url(request.form.get('next')) or - find_redirect('SECURITY_POST_LOGIN_VIEW')) - - -def find_redirect(key): - """Returns the URL to redirect to after a user logs in successfully. - - :param key: The session or application configuration key to search for - """ - rv = (get_url(session.pop(key.lower(), None)) or - get_url(current_app.config[key.upper()] or None) or '/') - return rv - - -def get_config(app): - """Conveniently get the security configuration for the specified - application without the annoying 'SECURITY_' prefix. - - :param app: The application to inspect - """ - items = app.config.items() - prefix = 'SECURITY_' - - def strip_prefix(tup): - return (tup[0].replace('SECURITY_', ''), tup[1]) - - return dict([strip_prefix(i) for i in items if i[0].startswith(prefix)]) - - -def get_message(key, **kwargs): - rv = config_value('MSG_' + key) - return rv[0] % kwargs, rv[1] - - -def config_value(key, app=None, default=None): - """Get a Flask-Security configuration value. - - :param key: The configuration key without the prefix `SECURITY_` - :param app: An optional specific application to inspect. Defaults to Flask's - `current_app` - :param default: An optional default value if the value is not set - """ - app = app or current_app - return get_config(app).get(key.upper(), default) - - -def get_max_age(key, app=None): - now = datetime.utcnow() - expires = now + get_within_delta(key + '_WITHIN', app) - td = (expires - now) - return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 1e6) / 1e6 - - -def get_within_delta(key, app=None): - """Get a timedelta object from the application configuration following - the internal convention of:: - - - - Examples of valid config values:: - - 5 days - 10 minutes - - :param key: The config value key without the 'SECURITY_' prefix - :param app: Optional application to inspect. Defaults to Flask's - `current_app` - """ - txt = config_value(key, app=app) - values = txt.split() - return timedelta(**{values[1]: int(values[0])}) - - -def send_mail(subject, recipient, template, **context): - """Send an email via the Flask-Mail extension. - - :param subject: Email subject - :param recipient: Email recipient - :param template: The name of the email template - :param context: The context to render the template with - """ - - context.setdefault('security', _security) - context.update(_security._run_ctx_processor('mail')) - - msg = Message(subject, - sender=_security.email_sender, - recipients=[recipient]) - - ctx = ('security/email', template) - msg.body = render_template('%s/%s.txt' % ctx, **context) - msg.html = render_template('%s/%s.html' % ctx, **context) - - if _security._send_mail_task: - _security._send_mail_task(msg) - return - - mail = current_app.extensions.get('mail') - mail.send(msg) - - -def get_token_status(token, serializer, max_age=None): - serializer = getattr(_security, serializer + '_serializer') - max_age = get_max_age(max_age) - user, data = None, None - expired, invalid = False, False - - try: - data = serializer.loads(token, max_age=max_age) - except SignatureExpired: - d, data = serializer.loads_unsafe(token) - expired = True - except BadSignature: - invalid = True - - if data: - user = _datastore.find_user(id=data[0]) - - expired = expired and (user is not None) - return expired, invalid, user - - -@contextmanager -def capture_passwordless_login_requests(): - login_requests = [] - - def _on(data, app): - login_requests.append(data) - - login_instructions_sent.connect(_on) - - try: - yield login_requests - finally: - login_instructions_sent.disconnect(_on) - - -@contextmanager -def capture_registrations(): - """Testing utility for capturing registrations. - - :param confirmation_sent_at: An optional datetime object to set the - user's `confirmation_sent_at` to - """ - registrations = [] - - def _on(data, app): - registrations.append(data) - - user_registered.connect(_on) - - try: - yield registrations - finally: - user_registered.disconnect(_on) - - -@contextmanager -def capture_reset_password_requests(reset_password_sent_at=None): - """Testing utility for capturing password reset requests. - - :param reset_password_sent_at: An optional datetime object to set the - user's `reset_password_sent_at` to - """ - reset_requests = [] - - def _on(request, app): - reset_requests.append(request) - - reset_password_instructions_sent.connect(_on) - - try: - yield reset_requests - finally: - reset_password_instructions_sent.disconnect(_on) - - -class CaptureSignals(object): - """Testing utility for capturing blinker signals. - - Context manager which mocks out selected signals and registers which are `sent` on and what - arguments were sent. Instantiate with a list of blinker `NamedSignals` to patch. Each signal - has it's `send` mocked out. - """ - def __init__(self, signals): - """Patch all given signals and make them available as attributes. - - :param signals: list of signals - """ - self._records = {} - self._receivers = {} - for signal in signals: - self._records[signal] = [] - self._receivers[signal] = functools.partial(self._record, signal) - - def __getitem__(self, signal): - """All captured signals are available via `ctxt[signal]`. - """ - if isinstance(signal, blinker.base.NamedSignal): - return self._records[signal] - else: - super(CaptureSignals, self).__setitem__(signal) - - def _record(self, signal, *args, **kwargs): - self._records[signal].append((args, kwargs)) - - def __enter__(self): - for signal, receiver in self._receivers.iteritems(): - signal.connect(receiver) - return self - - def __exit__(self, type, value, traceback): - for signal, receiver in self._receivers.iteritems(): - signal.disconnect(receiver) - - def signals_sent(self): - """Return a set of the signals sent. - :rtype: list of blinker `NamedSignals`. - """ - return set([signal for signal, _ in self._records.iteritems() if self._records[signal]]) - - -def capture_signals(): - """Factory method that creates a `CaptureSignals` with all the flask_security signals.""" - return CaptureSignals([user_registered, user_confirmed, - confirm_instructions_sent, login_instructions_sent, - password_reset, password_changed, - reset_password_instructions_sent]) - - diff --git a/wqflask/flask_security/views.py b/wqflask/flask_security/views.py deleted file mode 100644 index 1b8488d8..00000000 --- a/wqflask/flask_security/views.py +++ /dev/null @@ -1,359 +0,0 @@ -# -*- coding: utf-8 -*- -""" - flask.ext.security.views - ~~~~~~~~~~~~~~~~~~~~~~~~ - - Flask-Security views module - - :copyright: (c) 2012 by Matt Wright. - :license: MIT, see LICENSE for more details. -""" - -from flask import current_app, redirect, request, render_template, jsonify, \ - after_this_request, Blueprint -from flask_login import current_user -from werkzeug.datastructures import MultiDict -from werkzeug.local import LocalProxy - -from .confirmable import send_confirmation_instructions, \ - confirm_user, confirm_email_token_status -from .decorators import login_required, anonymous_user_required -from .passwordless import send_login_instructions, \ - login_token_status -from .recoverable import reset_password_token_status, \ - send_reset_password_instructions, update_password -from .changeable import change_user_password -from .registerable import register_user -from .utils import get_url, get_post_login_redirect, do_flash, \ - get_message, login_user, logout_user, url_for_security as url_for, \ - config_value - - -# Convenient references -_security = LocalProxy(lambda: current_app.extensions['security']) - -_datastore = LocalProxy(lambda: _security.datastore) - - -def _render_json(form, include_auth_token=False): - has_errors = len(form.errors) > 0 - - if has_errors: - code = 400 - response = dict(errors=form.errors) - else: - code = 200 - response = dict(user=dict(id=str(form.user.id))) - if include_auth_token: - token = form.user.get_auth_token() - response['user']['authentication_token'] = token - - return jsonify(dict(meta=dict(code=code), response=response)) - - -def _commit(response=None): - _datastore.commit() - return response - - -def _ctx(endpoint): - return _security._run_ctx_processor(endpoint) - - -@anonymous_user_required -def login(): - """View function for login view""" - - form_class = _security.login_form - - if request.json: - form = form_class(MultiDict(request.json)) - else: - form = form_class() - - if form.validate_on_submit(): - login_user(form.user, remember=form.remember.data) - after_this_request(_commit) - - if not request.json: - return redirect(get_post_login_redirect()) - - form.next.data = get_url(request.args.get('next')) \ - or get_url(request.form.get('next')) or '' - - if request.json: - return _render_json(form, True) - - return render_template(config_value('LOGIN_USER_TEMPLATE'), - login_user_form=form, - **_ctx('login')) - - -@login_required -def logout(): - """View function which handles a logout request.""" - - logout_user() - - return redirect(request.args.get('next', None) or - get_url(_security.post_logout_view)) - - -def register(): - """View function which handles a registration request.""" - - if _security.confirmable or request.json: - form_class = _security.confirm_register_form - else: - form_class = _security.register_form - - if request.json: - form_data = MultiDict(request.json) - else: - form_data = request.form - - form = form_class(form_data) - - if form.validate_on_submit(): - user = register_user(**form.to_dict()) - form.user = user - - if not _security.confirmable or _security.login_without_confirmation: - after_this_request(_commit) - login_user(user) - - if not request.json: - post_register_url = get_url(_security.post_register_view) - post_login_url = get_url(_security.post_login_view) - return redirect(post_register_url or post_login_url) - - if request.json: - return _render_json(form) - - return render_template(config_value('REGISTER_USER_TEMPLATE'), - register_user_form=form, - **_ctx('register')) - - -def send_login(): - """View function that sends login instructions for passwordless login""" - - form_class = _security.passwordless_login_form - - if request.json: - form = form_class(MultiDict(request.json)) - else: - form = form_class() - - if form.validate_on_submit(): - send_login_instructions(form.user) - if request.json is None: - do_flash(*get_message('LOGIN_EMAIL_SENT', email=form.user.email)) - - if request.json: - return _render_json(form) - - return render_template(config_value('SEND_LOGIN_TEMPLATE'), - send_login_form=form, - **_ctx('send_login')) - - -@anonymous_user_required -def token_login(token): - """View function that handles passwordless login via a token""" - - expired, invalid, user = login_token_status(token) - - if invalid: - do_flash(*get_message('INVALID_LOGIN_TOKEN')) - if expired: - send_login_instructions(user) - do_flash(*get_message('LOGIN_EXPIRED', email=user.email, - within=_security.login_within)) - if invalid or expired: - return redirect(url_for('login')) - - login_user(user, True) - after_this_request(_commit) - do_flash(*get_message('PASSWORDLESS_LOGIN_SUCCESSFUL')) - - return redirect(get_post_login_redirect()) - - -def send_confirmation(): - """View function which sends confirmation instructions.""" - - form_class = _security.send_confirmation_form - - if request.json: - form = form_class(MultiDict(request.json)) - else: - form = form_class() - - if form.validate_on_submit(): - send_confirmation_instructions(form.user) - if request.json is None: - do_flash(*get_message('CONFIRMATION_REQUEST', email=form.user.email)) - - if request.json: - return _render_json(form) - - return render_template(config_value('SEND_CONFIRMATION_TEMPLATE'), - send_confirmation_form=form, - **_ctx('send_confirmation')) - - -@anonymous_user_required -def confirm_email(token): - """View function which handles a email confirmation request.""" - - expired, invalid, user = confirm_email_token_status(token) - - if not user or invalid: - invalid = True - do_flash(*get_message('INVALID_CONFIRMATION_TOKEN')) - if expired: - send_confirmation_instructions(user) - do_flash(*get_message('CONFIRMATION_EXPIRED', email=user.email, - within=_security.confirm_email_within)) - if invalid or expired: - return redirect(get_url(_security.confirm_error_view) or - url_for('send_confirmation')) - - confirm_user(user) - login_user(user, True) - after_this_request(_commit) - do_flash(*get_message('EMAIL_CONFIRMED')) - - return redirect(get_url(_security.post_confirm_view) or - get_url(_security.post_login_view)) - - -def forgot_password(): - """View function that handles a forgotten password request.""" - - form_class = _security.forgot_password_form - - if request.json: - form = form_class(MultiDict(request.json)) - else: - form = form_class() - - if form.validate_on_submit(): - send_reset_password_instructions(form.user) - if request.json is None: - do_flash(*get_message('PASSWORD_RESET_REQUEST', email=form.user.email)) - - if request.json: - return _render_json(form) - - return render_template(config_value('FORGOT_PASSWORD_TEMPLATE'), - forgot_password_form=form, - **_ctx('forgot_password')) - - -@anonymous_user_required -def reset_password(token): - """View function that handles a reset password request.""" - - expired, invalid, user = reset_password_token_status(token) - - if invalid: - do_flash(*get_message('INVALID_RESET_PASSWORD_TOKEN')) - if expired: - do_flash(*get_message('PASSWORD_RESET_EXPIRED', email=user.email, - within=_security.reset_password_within)) - if invalid or expired: - return redirect(url_for('forgot_password')) - - form = _security.reset_password_form() - - if form.validate_on_submit(): - after_this_request(_commit) - update_password(user, form.password.data) - do_flash(*get_message('PASSWORD_RESET')) - login_user(user, True) - return redirect(get_url(_security.post_reset_view) or - get_url(_security.post_login_view)) - - return render_template(config_value('RESET_PASSWORD_TEMPLATE'), - reset_password_form=form, - reset_password_token=token, - **_ctx('reset_password')) - - -@login_required -def change_password(): - """View function which handles a change password request.""" - - form_class = _security.change_password_form - - if request.json: - form = form_class(MultiDict(request.json)) - else: - form = form_class() - - if form.validate_on_submit(): - after_this_request(_commit) - change_user_password(current_user, form.new_password.data) - if request.json is None: - do_flash(*get_message('PASSWORD_CHANGE')) - return redirect(get_url(_security.post_change_view) or - get_url(_security.post_login_view)) - - if request.json: - return _render_json(form) - - return render_template('security/change_password.html', - change_password_form=form, - **_ctx('change_password')) - - -def create_blueprint(state, import_name): - """Creates the security extension blueprint""" - - bp = Blueprint(state.blueprint_name, import_name, - url_prefix=state.url_prefix, - subdomain=state.subdomain, - template_folder='templates') - - bp.route(state.logout_url, endpoint='logout')(logout) - - if state.passwordless: - bp.route(state.login_url, - methods=['GET', 'POST'], - endpoint='login')(send_login) - bp.route(state.login_url + '/', - endpoint='token_login')(token_login) - else: - bp.route(state.login_url, - methods=['GET', 'POST'], - endpoint='login')(login) - - if state.registerable: - bp.route(state.register_url, - methods=['GET', 'POST'], - endpoint='register')(register) - - if state.recoverable: - bp.route(state.reset_url, - methods=['GET', 'POST'], - endpoint='forgot_password')(forgot_password) - bp.route(state.reset_url + '/', - methods=['GET', 'POST'], - endpoint='reset_password')(reset_password) - - if state.changeable: - bp.route(state.change_url, - methods=['GET', 'POST'], - endpoint='change_password')(change_password) - - if state.confirmable: - bp.route(state.confirm_url, - methods=['GET', 'POST'], - endpoint='send_confirmation')(send_confirmation) - bp.route(state.confirm_url + '/', - methods=['GET', 'POST'], - endpoint='confirm_email')(confirm_email) - - return bp -- cgit v1.2.3 From babe7424bfc06b2436b6e824389cf795afdf0a73 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 29 Mar 2018 07:46:14 +0000 Subject: Illegal search term corrected treatment and no longer shows as error in logs --- wqflask/wqflask/search_results.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 59e100d8..ca40f849 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -64,14 +64,23 @@ views.py). else: self.and_or = "and" self.search_terms = kw['search_terms_and'] - self.search_term_exists = True + if "http:" in self.search_terms: + self.search_term_exists = False + return + else: + self.search_term_exists = True + self.results = [] - if kw['type'] == "Phenotypes": # split datatype on type field + type = kw.get('type') + if type == "Phenotypes": # split datatype on type field dataset_type = "Publish" - elif kw['type'] == "Genotypes": + elif type == "Genotypes": dataset_type = "Geno" - else: + elif type == "ProbeSet": dataset_type = "ProbeSet" # ProbeSet is default + else: + self.search_term_exists = False + return self.dataset = create_dataset(kw['dataset'], dataset_type) logger.debug("search_terms:", self.search_terms) self.search() -- cgit v1.2.3 From f33afaa4df00aa0fef37ba9c5e70f3c19b96ce5c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 29 Mar 2018 07:58:11 +0000 Subject: Checking search error terms with regex --- wqflask/wqflask/search_results.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index ca40f849..25f69f3f 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -10,6 +10,7 @@ import time import math import datetime import collections +import re from pprint import pformat as pf @@ -64,7 +65,10 @@ views.py). else: self.and_or = "and" self.search_terms = kw['search_terms_and'] - if "http:" in self.search_terms: + search = self.search_terms + # check for dodgy search terms + regex = re.compile("http:|href|sql|select",re.IGNORECASE) + if regex.match(search): self.search_term_exists = False return else: -- cgit v1.2.3 From b43c4ceae959fdff685985acf33ac311e939b008 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 29 Mar 2018 08:00:55 +0000 Subject: Disable caching of search errors --- wqflask/wqflask/search_results.py | 2 +- wqflask/wqflask/views.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 25f69f3f..41fcf873 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -67,7 +67,7 @@ views.py). self.search_terms = kw['search_terms_and'] search = self.search_terms # check for dodgy search terms - regex = re.compile("http:|href|sql|select",re.IGNORECASE) + regex = re.compile("http:|href|sql|select|update",re.IGNORECASE) if regex.match(search): self.search_term_exists = False return diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3ebef046..a65924d8 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -199,14 +199,15 @@ def search_page(): logger.info("request.args is", request.args) the_search = search_results.SearchResultPage(request.args) result = the_search.__dict__ + valid_search = result['search_term_exists'] logger.debugf("result", result) - if USE_REDIS: + if USE_REDIS and valid_search: Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) Redis.expire(key, 60*60) - if result['search_term_exists']: + if valid_search: return render_template("search_result_page.html", **result) else: return render_template("search_error.html") -- cgit v1.2.3 From d215602d89ec6d929d489cd7f3507cbe3e8c29bf Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 29 Mar 2018 10:56:25 +0000 Subject: Search: improved regex and some debug info --- wqflask/wqflask/correlation/show_corr_results.py | 42 +----------------------- wqflask/wqflask/search_results.py | 16 +++++---- 2 files changed, 11 insertions(+), 47 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 9b048346..2c6c3a14 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -52,6 +52,7 @@ import utility.webqtlUtil #this is for parallel computing only. from wqflask.correlation import correlation_functions from utility.benchmark import Bench import utility.webqtlUtil +from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string from wqflask import user_manager from MySQLdb import escape_string as escape @@ -77,47 +78,6 @@ def print_mem(stage=""): mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss #print("{}: {}".format(stage, mem/1024)) -def is_float(value): - try: - float(value) - return True - except: - return False - -def is_int(value): - try: - int(value) - return True - except: - return False - -def is_str(value): - if value is None: - return False - try: - str(value) - return True - except: - return False - -def get_float(vars,name,default=None): - if name in vars: - if is_float(vars[name]): - return float(vars[name]) - return None - -def get_int(vars,name,default=None): - if name in vars: - if is_int(vars[name]): - return float(vars[name]) - return default - -def get_string(vars,name,default=None): - if name in vars: - if not vars[name] is None: - return str(vars[name]) - return default - class AuthException(Exception): pass diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 41fcf873..53c96591 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -26,6 +26,7 @@ from db import webqtlDatabaseFunction from flask import render_template from utility import formatting +from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string from utility.logger import getLogger logger = getLogger(__name__ ) @@ -67,8 +68,9 @@ views.py). self.search_terms = kw['search_terms_and'] search = self.search_terms # check for dodgy search terms - regex = re.compile("http:|href|sql|select|update",re.IGNORECASE) - if regex.match(search): + rx = re.compile(r'.*\W(href|http|sql|select|update)\W.*',re.IGNORECASE) + if rx.match(search): + logger.info("Regex failed search") self.search_term_exists = False return else: @@ -80,11 +82,10 @@ views.py). dataset_type = "Publish" elif type == "Genotypes": dataset_type = "Geno" - elif type == "ProbeSet": - dataset_type = "ProbeSet" # ProbeSet is default else: - self.search_term_exists = False - return + dataset_type = "ProbeSet" # ProbeSet is default + + assert(is_str(kw.get('dataset'))) self.dataset = create_dataset(kw['dataset'], dataset_type) logger.debug("search_terms:", self.search_terms) self.search() @@ -158,6 +159,7 @@ statement and executes else: combined_where_clause += "OR" else: + logger.debug("Search failed 1") self.search_term_exists = False if self.search_term_exists: combined_where_clause = "(" + combined_where_clause + ")" @@ -168,6 +170,7 @@ statement and executes else: logger.debug("len(search_terms)<=1") if self.search_terms == []: + logger.debug("Search failed 2") self.search_term_exists = False else: for a_search in self.search_terms: @@ -175,6 +178,7 @@ statement and executes if the_search != None: self.results.extend(the_search.run()) else: + logger.debug("Search failed 3") self.search_term_exists = False if self.search_term_exists: -- cgit v1.2.3 From b9ada0431b880e632b830a6e2169279ae1ecdd36 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 29 Mar 2018 11:00:41 +0000 Subject: Type checking in its own file --- wqflask/utility/type_checking.py | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 wqflask/utility/type_checking.py diff --git a/wqflask/utility/type_checking.py b/wqflask/utility/type_checking.py new file mode 100644 index 00000000..220e5f62 --- /dev/null +++ b/wqflask/utility/type_checking.py @@ -0,0 +1,42 @@ +# Type checking functions + +def is_float(value): + try: + float(value) + return True + except: + return False + +def is_int(value): + try: + int(value) + return True + except: + return False + +def is_str(value): + if value is None: + return False + try: + str(value) + return True + except: + return False + +def get_float(vars,name,default=None): + if name in vars: + if is_float(vars[name]): + return float(vars[name]) + return None + +def get_int(vars,name,default=None): + if name in vars: + if is_int(vars[name]): + return float(vars[name]) + return default + +def get_string(vars,name,default=None): + if name in vars: + if not vars[name] is None: + return str(vars[name]) + return default -- cgit v1.2.3 From e0c706c51c834caa836ecffd27a5d18fc23178ff Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 29 Mar 2018 11:18:13 +0000 Subject: Fixed gsearch blowout https://github.com/genenetwork/genenetwork2/issues/295 --- wqflask/wqflask/gsearch.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index e33e04e1..fe1e17d2 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -5,6 +5,7 @@ from base.data_set import create_dataset from base.trait import GeneralTrait from db import webqtlDatabaseFunction +from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string from utility.benchmark import Bench from utility.logger import getLogger @@ -13,8 +14,13 @@ logger = getLogger(__name__) class GSearch(object): def __init__(self, kw): + assert('type' in kw) + assert('terms' in kw) + self.type = kw['type'] self.terms = kw['terms'] + assert(is_str(self.type)) + if self.type == "gene": sql = """ SELECT -- cgit v1.2.3