From 178102c304e5ad1da2af5b2ab0af2484edb8a609 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Mar 2018 17:08:05 +0300 Subject: * Add mapping tests. Fix link-checker tests. * Add tests for the R/qtl and CIM mapping tools. * Fix tests for in-page links. --- test/requests/link_checker.py | 16 ++++++++++++---- test/requests/mapping_tests.py | 35 ++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index 256bf6ef..a75327f0 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -16,6 +16,10 @@ def is_internal_link(link): pattern = re.compile("^/.*") return pattern.match(link) +def is_in_page_link(link): + pattern = re.compile("^#.*") + return pattern.match(link) + def get_links(doc): return filter( lambda x: not ( @@ -32,17 +36,21 @@ def verify_link(link): else: print("ERROR: link `"+link+"` failed with status " , result.status_code) - except ConnectionError as ex: - print("ERROR: ", link, ex) + except Exception as ex: + print("ERROR: ("+link+")", ex) def check_page(host, start_url): print("") print("Checking links in page `"+start_url+"`") doc = parse(start_url).getroot() links = get_links(doc) + in_page_links = filter(is_in_page_link, links) internal_links = filter(is_internal_link, links) - external_links = filter(lambda x: not is_internal_link(x), links) - external_links.append("http://somenon-existentsite.brr") + external_links = filter(lambda x: not (is_internal_link(x) or is_in_page_link(x)), links) + + for link in in_page_links: + verify_link(start_url+link) + for link in internal_links: verify_link(host+link) diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py index fd20df11..8eb19de7 100644 --- a/test/requests/mapping_tests.py +++ b/test/requests/mapping_tests.py @@ -1,17 +1,10 @@ from __future__ import print_function import re +import copy import json import requests from lxml.html import fromstring -def get_data(list_item): - try: - value = list_item[1] - except: - value = None - #print("list_item:", list_item, "==>", value) - return value - def load_data_from_file(): filename = "../test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json" file_handle = open(filename, "r") @@ -19,6 +12,8 @@ def load_data_from_file(): return file_data def check_pylmm_tool_selection(host, data): + print("") + print("pylmm mapping tool selection") data["method"] = "pylmm" page = requests.post(host+"/marker_regression", data=data) doc = fromstring(page.text) @@ -27,10 +22,24 @@ def check_pylmm_tool_selection(host, data): assert form.fields["value:BXD1"] == "15.034" # Check value in the file def check_R_qtl_tool_selection(host, data): - pass + print("") + print("R/qtl mapping tool selection") + headers = {"Content-Type": "application/x-www-form-urlencoded"} + page = requests.post(host+"/marker_regression", data=data, headers=headers) + doc = fromstring(page.text) + form = doc.forms[1] + assert form.fields["dataset"] == "HC_M2_0606_P" + assert form.fields["value:BXD1"] == "15.034" def check_CIM_tool_selection(host, data): - pass + print("") + print("CIM mapping tool selection (using reaper)") + data["method"] = "reaper" + page = requests.post(host+"/marker_regression", data=data) + doc = fromstring(page.text) + form = doc.forms[1] + assert form.fields["dataset"] == "HC_M2_0606_P" + assert form.fields["value:BXD1"] == "15.034" def check_mapping(args_obj, parser): print("") @@ -38,6 +47,6 @@ def check_mapping(args_obj, parser): host = args_obj.host data = load_data_from_file() - check_pylmm_tool_selection(host, data) - check_R_qtl_tool_selection(host, data) - check_CIM_tool_selection(host, data) + check_pylmm_tool_selection(host, copy.deepcopy(data)) + check_R_qtl_tool_selection(host, copy.deepcopy(data)) ## Why does this fail? + check_CIM_tool_selection(host, copy.deepcopy(data)) -- cgit v1.2.3 From 883bcc257a38240de7de8888f78bac4406b5d027 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 19 Mar 2018 17:49:39 +0300 Subject: Initialise navigation tests. * Start working on navigation tests. --- test/requests/navigation_tests.py | 15 +++++++++++++++ test/requests/test-website.py | 7 ++++--- 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 test/requests/navigation_tests.py diff --git a/test/requests/navigation_tests.py b/test/requests/navigation_tests.py new file mode 100644 index 00000000..eda27324 --- /dev/null +++ b/test/requests/navigation_tests.py @@ -0,0 +1,15 @@ +from __future__ import print_function +import re +import requests +from lxml.html import parse + +def check_navigation(args_obj, parser): + print("") + print("Checking navigation.") + + host = args_obj.host + url = host + "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" + print("URL: ", url) + page = requests.get(url) + # Page is built by the javascript, hence using requests fails for this. + # Investigate use of selenium maybe? diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 2bef6eb1..dbf3b822 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -7,6 +7,7 @@ from __future__ import print_function import argparse from link_checker import check_links from mapping_tests import check_mapping +from navigation_tests import check_navigation from main_web_functionality import check_main_web_functionality print("Mechanical Rob firing up...") @@ -57,9 +58,9 @@ parser.add_argument("-m", "--mapping", dest="accumulate" , action="store_const", const=check_mapping, default=print_help , help="Checks for mapping.") -# parser.add_argument("-n", "--navigation", dest="accumulate" -# , action="store_const", const=check_navigation, default=print_help -# , help="Checks for navigation.") +parser.add_argument("-n", "--navigation", dest="accumulate" + , action="store_const", const=check_navigation, default=print_help + , help="Checks for navigation.") # parser.add_argument("-s", "--skip-broken", dest="accumulate" # , action="store_const", const=dummy, default=print_help -- cgit v1.2.3 From a20004e25d0143a14763180359e8266e31f9766f Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 21 Mar 2018 11:22:48 +0300 Subject: Deactivate navigation tests * Navigation depends on Javascript to render the elements being tested. That would require selenium and its ilk to run such tests, meaning such packages would need to be defined for Guix first. --- test/requests/test-website.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index dbf3b822..f03c148a 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -58,9 +58,10 @@ parser.add_argument("-m", "--mapping", dest="accumulate" , action="store_const", const=check_mapping, default=print_help , help="Checks for mapping.") -parser.add_argument("-n", "--navigation", dest="accumulate" - , action="store_const", const=check_navigation, default=print_help - , help="Checks for navigation.") +# Navigation tests deactivated since system relies on Javascript +# parser.add_argument("-n", "--navigation", dest="accumulate" +# , action="store_const", const=check_navigation, default=print_help +# , help="Checks for navigation.") # parser.add_argument("-s", "--skip-broken", dest="accumulate" # , action="store_const", const=dummy, default=print_help -- cgit v1.2.3 From 4c2ce5580a0891d89fc35c48da06d25fb4c8df76 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Thu, 22 Mar 2018 06:09:39 +0300 Subject: Fix 'def' statement * End the 'def' statement with a colon (:) * Add an empty line between lines at different indentation levels for clarity. --- wqflask/base/anon_collection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/base/anon_collection.py b/wqflask/base/anon_collection.py index 8ee73296..dd1aa27f 100644 --- a/wqflask/base/anon_collection.py +++ b/wqflask/base/anon_collection.py @@ -1,6 +1,6 @@ class AnonCollection(TraitCollection): - def __init__(self, anon_id) + def __init__(self, anon_id): self.anon_id = anon_id self.collection_members = Redis.smembers(self.anon_id) print("self.collection_members is:", self.collection_members) @@ -12,6 +12,7 @@ class AnonCollection(TraitCollection): print("traits_to_remove:", traits_to_remove) for trait in traits_to_remove: Redis.srem(self.anon_id, trait) + members_now = self.collection_members - traits_to_remove print("members_now:", members_now) print("Went from {} to {} members in set.".format(len(self.collection_members), len(members_now))) -- cgit v1.2.3 From cfce6d80be4fb38573c37d1943db2687d54cf2fc Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Thu, 22 Mar 2018 06:28:01 +0300 Subject: Change raise statement * Change the raise statement to use an Exception object rather than using a string for compatibility with Python3. --- wqflask/utility/svg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 512bc9e6..db13b9d1 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -1029,7 +1029,7 @@ class drawing: try: xv.feed(svg) except: - raise "SVG is not well formed, see messages above" + raise Exception("SVG is not well formed, see messages above") else: print "SVG well formed" if __name__=='__main__': -- cgit v1.2.3 From 55cc4c63478de9587e282522540334e5375aebf1 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 13 Apr 2018 15:33:39 +0300 Subject: Use single entry-point for tests * Remove the run-integration-tests.py, and use the test-website.py as the entry point for all tests. This simplifies running the tests for the site. --- test/requests/run-integration-tests.py | 34 ----------------------------- test/requests/test-website.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 34 deletions(-) delete mode 100644 test/requests/run-integration-tests.py diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py deleted file mode 100644 index 5e816549..00000000 --- a/test/requests/run-integration-tests.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys -from test_login_local import TestLoginLocal -from test_login_orcid import TestLoginOrcid -from test_login_github import TestLoginGithub -from test_registration import TestRegistration -from unittest import TestSuite, TextTestRunner, TestLoader - -test_cases = [ - TestRegistration - , TestLoginLocal - , TestLoginGithub - , TestLoginOrcid -] - -def suite(gn2_url, es_url): - the_suite = TestSuite() - for case in test_cases: - the_suite.addTests(initTest(case, gn2_url, es_url)) - return the_suite - -def initTest(klass, gn2_url, es_url): - loader = TestLoader() - methodNames = loader.getTestCaseNames(klass) - return [klass(mname, gn2_url, es_url) for mname in methodNames] - -def main(gn2_url, es_url): - runner = TextTestRunner() - runner.run(suite(gn2_url, es_url)) - -if __name__ == "__main__": - if len(sys.argv) < 3: - raise Exception("Required arguments missing:\n\tTry running `run-integration-test.py `") - else: - main(sys.argv[1], sys.argv[2]) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index a33fe708..b2e09bc4 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -12,6 +12,15 @@ from main_web_functionality import check_main_web_functionality import link_checker import sys +# Imports for integration tests +from wqflask import app +from test_login_local import TestLoginLocal +from test_login_orcid import TestLoginOrcid +from test_login_github import TestLoginGithub +from test_registration import TestRegistration +from test_forgot_password import TestForgotPassword +from unittest import TestSuite, TextTestRunner, TestLoader + print("Mechanical Rob firing up...") def run_all(args_obj, parser): @@ -30,6 +39,33 @@ def print_help(args_obj, parser): def dummy(args_obj, parser): print("Not implemented yet.") +def integration_tests(args_obj, parser): + gn2_url = args_obj.host + es_url = app.config.get("ELASTICSEARCH_HOST")+":"+str(app.config.get("ELASTICSEARCH_PORT")) + run_integration_tests(gn2_url, es_url) + +def initTest(klass, gn2_url, es_url): + loader = TestLoader() + methodNames = loader.getTestCaseNames(klass) + return [klass(mname, gn2_url, es_url) for mname in methodNames] + +def integration_suite(gn2_url, es_url): + test_cases = [ + TestRegistration + , TestLoginLocal + , TestLoginGithub + , TestLoginOrcid + , TestForgotPassword + ] + the_suite = TestSuite() + for case in test_cases: + the_suite.addTests(initTest(case, gn2_url, es_url)) + return the_suite + +def run_integration_tests(gn2_url, es_url): + runner = TextTestRunner() + runner.run(integration_suite(gn2_url, es_url)) + desc = """ This is Mechanical-Rob - an automated web server tester for @@ -64,6 +100,10 @@ parser.add_argument("-m", "--mapping", dest="accumulate" , action="store_const", const=check_mapping, default=print_help , help="Checks for mapping.") +parser.add_argument("-i", "--integration-tests", dest="accumulate" + , action="store_const", const=integration_tests, default=print_help + , help="Runs integration tests.") + # Navigation tests deactivated since system relies on Javascript # parser.add_argument("-n", "--navigation", dest="accumulate" # , action="store_const", const=check_navigation, default=print_help -- cgit v1.2.3 From 90da7aa5028d64437f3fcaf903075cbda293b575 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 13 Apr 2018 15:39:47 +0300 Subject: Fix logging in tests * Use the LOG_LEVEL provided by the application settings to enable easy control of logging when running tests. It helps avoid a lot of verbosity when running tests. --- test/requests/parametrized_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py index abf98fce..c585e910 100644 --- a/test/requests/parametrized_test.py +++ b/test/requests/parametrized_test.py @@ -1,5 +1,6 @@ import logging import unittest +from wqflask import app from elasticsearch import Elasticsearch, TransportError class ParametrizedTest(unittest.TestCase): @@ -14,6 +15,7 @@ class ParametrizedTest(unittest.TestCase): self.es_cleanup = [] es_logger = logging.getLogger("elasticsearch") + es_logger.setLevel(app.config.get("LOG_LEVEL")) es_logger.addHandler( logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) es_trace_logger = logging.getLogger("elasticsearch.trace") -- cgit v1.2.3 From 9396ba78aa204a7f9474c314ca5a519c48ba042c Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 13 Apr 2018 15:41:31 +0300 Subject: Check email provided on forgot password * Ensure that the user actually provides an email address when they try to use the forgot password feature. --- wqflask/wqflask/user_manager.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 5f6c818e..9d09dbf6 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -727,31 +727,33 @@ def logout(): return response -@app.route("/n/forgot_password") +@app.route("/n/forgot_password", methods=['GET']) def forgot_password(): """Entry point for forgotten password""" - return render_template("new_security/forgot_password.html") + print("ARGS: ", request.args) + errors = {"no-email": request.args.get("no-email")} + print("ERRORS: ", errors) + return render_template("new_security/forgot_password.html", errors=errors) @app.route("/n/forgot_password_submit", methods=('POST',)) def forgot_password_submit(): """When a forgotten password form is submitted we get here""" params = request.form email_address = params['email_address'] - logger.debug("Wants to send password E-mail to ",email_address) - es = get_elasticsearch_connection() - user_details = get_user_by_unique_column(es, "email_address", email_address) - if user_details: - ForgotPasswordEmail(user_details["email_address"]) - # try: - # user = model.User.query.filter_by(email_address=email_address).one() - # except orm.exc.NoResultFound: - # flash("Couldn't find a user associated with the email address {}. Sorry.".format( - # email_address)) - # return redirect(url_for("login")) - # ForgotPasswordEmail(user) - return render_template("new_security/forgot_password_step2.html", + next_page = None + if email_address != "": + logger.debug("Wants to send password E-mail to ",email_address) + es = get_elasticsearch_connection() + user_details = get_user_by_unique_column(es, "email_address", email_address) + if user_details: + ForgotPasswordEmail(user_details["email_address"]) + return render_template("new_security/forgot_password_step2.html", subject=ForgotPasswordEmail.subject) + else: + flash("You MUST provide an email", "alert-danger") + return redirect(url_for("forgot_password")) + @app.errorhandler(401) def unauthorized(error): return redirect(url_for('login')) -- cgit v1.2.3 From ea099c2820fbca3f935fb8dc657b88e71224cded Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 15 Apr 2018 09:53:21 +0300 Subject: Fix issue with indentation * Ensure the return is at the correct indentation level so that it is executed correctly. --- wqflask/wqflask/user_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 9d09dbf6..d652f2e9 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -747,8 +747,9 @@ def forgot_password_submit(): user_details = get_user_by_unique_column(es, "email_address", email_address) if user_details: ForgotPasswordEmail(user_details["email_address"]) - return render_template("new_security/forgot_password_step2.html", - subject=ForgotPasswordEmail.subject) + + return render_template("new_security/forgot_password_step2.html", + subject=ForgotPasswordEmail.subject) else: flash("You MUST provide an email", "alert-danger") -- cgit v1.2.3 From e3e98b0533460837c4ea2eac67c4281eb0ba0012 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 15 Apr 2018 11:48:06 +0300 Subject: Use existing code. Delay after delete. * Use existing code to get the elasticsearch connection. This should prevent tests from failing in case the way connections to elasticsearch are made change. * Delay a while after deleting to allow elasticsearch to re-index the data, thus preventing subtle bugs in the test. --- test/requests/parametrized_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py index c585e910..50003850 100644 --- a/test/requests/parametrized_test.py +++ b/test/requests/parametrized_test.py @@ -1,6 +1,7 @@ import logging import unittest from wqflask import app +from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column from elasticsearch import Elasticsearch, TransportError class ParametrizedTest(unittest.TestCase): @@ -11,7 +12,7 @@ class ParametrizedTest(unittest.TestCase): self.es_url = es_url def setUp(self): - self.es = Elasticsearch([self.es_url]) + self.es = get_elasticsearch_connection() self.es_cleanup = [] es_logger = logging.getLogger("elasticsearch") @@ -23,7 +24,9 @@ class ParametrizedTest(unittest.TestCase): logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) def tearDown(self): + from time import sleep self.es.delete_by_query( index="users" , doc_type="local" , body={"query":{"match":{"email_address":"test@user.com"}}}) + sleep(1) -- cgit v1.2.3 From 9ac9f94f3b1409ae3a47c8a9e890f578a69b020f Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 15 Apr 2018 11:56:00 +0300 Subject: Add tests for Forgot Password feature * Add tests to ensure the "Forgot Password" feature works appropriately. --- test/requests/test_forgot_password.py | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 test/requests/test_forgot_password.py diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py new file mode 100644 index 00000000..2bf34c5c --- /dev/null +++ b/test/requests/test_forgot_password.py @@ -0,0 +1,52 @@ +import requests +from wqflask import user_manager +from utility.elasticsearch_tools import get_user_by_unique_column +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +passwork_reset_link = '' +forgot_password_page = None + +class TestForgotPassword(ParametrizedTest): + + def setUp(self): + super(TestForgotPassword, self).setUp() + self.forgot_password_url = self.gn2_url+"/n/forgot_password_submit" + def send_email(to_addr, msg, fromaddr="no-reply@genenetwork.org"): + print("CALLING: send_email_mock()") + email_data = { + "to_addr": to_addr + , "msg": msg + , "fromaddr": from_addr} + + data = { + "es_connection": self.es, + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + user_manager.basic_info = lambda : { "basic_info": "basic" } + user_manager.RegisterUser(data) + + def testWithoutEmail(self): + data = {"email_address": ""} + error_notification = '
You MUST provide an email
' + result = requests.post(self.forgot_password_url, data=data) + self.assertEqual(result.url, self.gn2_url+"/n/forgot_password") + self.assertTrue( + result.content.find(error_notification) >= 0 + , "Error message should be displayed but was not") + + def testWithNonExistingEmail(self): + # Monkey patching doesn't work, so simply test that getting by email + # returns the correct data + user = get_user_by_unique_column(self.es, "email_address", "non-existent@domain.com") + self.assertTrue(user is None, "Should not find non-existent user") + + def testWithExistingEmail(self): + # Monkey patching doesn't work, so simply test that getting by email + # returns the correct data + user = get_user_by_unique_column(self.es, "email_address", "test@user.com") + self.assertTrue(user is not None, "Should find user") -- cgit v1.2.3 From fdd28defcaf3326f3c6b6507124708d83a1da119 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Sun, 15 Apr 2018 11:57:09 +0300 Subject: Deactivate analysis of email_address field * Prevent elasticsearch from analysing and tokenising the email_address field so as to avoid issue with getting back all email addresses with the same domain as the one being searched for. --- wqflask/utility/elasticsearch_tools.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index d35cb5ee..7d2ee8c9 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -24,6 +24,8 @@ def get_elasticsearch_connection(): "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + setup_users_index(es) + es_logger = logging.getLogger("elasticsearch") es_logger.setLevel(logging.INFO) es_logger.addHandler(logging.NullHandler()) @@ -33,6 +35,17 @@ def get_elasticsearch_connection(): return es +def setup_users_index(es_connection): + if es_connection: + index_settings = { + "properties": { + "email_address": { + "type": "string" + , "index": "not_analyzed"}}} + + es_connection.indices.create(index='users', ignore=400) + es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local") + def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) -- cgit v1.2.3 From dda4697505aea2cd950533dfb3a0dfb0e66ec018 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 16 Apr 2018 09:00:52 +0000 Subject: Docs on elasticsearch use --- README.md | 24 ++++++++++++++------ bin/test-website | 2 +- wqflask/utility/elasticsearch_tools.py | 41 ++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3e7e64d0..59645994 100644 --- a/README.md +++ b/README.md @@ -17,25 +17,35 @@ deploy GN2 and dependencies as a self contained unit on any machine. The database can be run separately as well as the source tree (for developers). See the [installation docs](doc/README.org). -## Test +## Run Once installed GN2 can be run online through a browser interface ```sh -./bin/genenetwork2 +genenetwork2 ``` -(default is http://localhost:5003/). For more examples, including running scripts and a Python REPL -see the startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). +(default is http://localhost:5003/). For full examples (you'll need to +set a number of environment variables), including running scripts and +a Python REPL, see the startup script +[./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). +## Testing -We are building up automated -testing using [mechanize](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) which can be run with +We are building 'Mechanical Rob' automated testing using Python +[requests](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) +which can be run with something like ```sh -./bin/test-website +env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 ``` +The GN2_PROFILE is the Guix profile that contains all +dependencies. The ./bin/genenetwork2 script sets up the environment +and executes test-website.py in a Python interpreter. The -a switch +says to run all tests and the URL points to the running GN2 http +server. + ## Documentation User documentation can be found diff --git a/bin/test-website b/bin/test-website index 5935f016..7fbcfd2f 100755 --- a/bin/test-website +++ b/bin/test-website @@ -2,6 +2,6 @@ if [ -z $GN2_PROFILE ]; then echo "Run request tests with something like" - echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 + echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 exit 1 fi diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 7d2ee8c9..4d4a9844 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,3 +1,44 @@ +# Elasticsearch support +# +# Some helpful commands to view the database: +# +# You can test the server being up with +# +# curl -H 'Content-Type: application/json' http://localhost:9200 +# +# List all indices +# +# curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v' +# +# To see the users index 'table' +# +# curl http://localhost:9200/users +# +# To list all user ids +# +# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' +# { +# "query" : { +# "match_all" : {} +# }, +# "stored_fields": [] +# }' +# +# To view a record +# +# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' +# { +# "query" : { +# "match" : { "email_address": "pjotr2017@thebird.nl"} +# } +# }' +# +# +# To delete the users index and data (dangerous!) +# +# curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users' + + from elasticsearch import Elasticsearch, TransportError import logging -- cgit v1.2.3 From bc1672f8617c56684ae3aeda7018362e818c46d6 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 16 Apr 2018 17:25:14 +0300 Subject: Update mappings for Elasticsearch 6.2. Update logger * Update the indexes mappings to be compatible with the newer Elasticsearch 6.2.* series. Close the index before updating it, and reopen it after to help with the re-indexing of the data. * Update the error logger to include the exception that was thrown. --- wqflask/utility/elasticsearch_tools.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 7d2ee8c9..0dc59d43 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -29,8 +29,8 @@ def get_elasticsearch_connection(): es_logger = logging.getLogger("elasticsearch") es_logger.setLevel(logging.INFO) es_logger.addHandler(logging.NullHandler()) - except: - logger.error("Failed to get elasticsearch connection") + except Exception as e: + logger.error("Failed to get elasticsearch connection", e) es = None return es @@ -40,11 +40,12 @@ def setup_users_index(es_connection): index_settings = { "properties": { "email_address": { - "type": "string" - , "index": "not_analyzed"}}} + "type": "keyword"}}} es_connection.indices.create(index='users', ignore=400) + es_connection.indices.close(index="users") es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local") + es_connection.indices.open(index="users") def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) -- cgit v1.2.3 From ebbc8c67c13b44df98ea3c2b355e86f77572543b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 16 Apr 2018 15:02:16 +0000 Subject: login messages improved --- wqflask/wqflask/templates/new_security/login_user.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 4a857c60..27b20ebf 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -31,16 +31,19 @@
{% if external_login["github"]: %} Login with Github + {% else %} +

Github login is not available right now

{% endif %} {% if external_login["orcid"]: %} Login with ORCID + {% else %} +

ORCID login is not available right now

{% endif %}
{% else: %}
-

You cannot login with external services at this time.
- Please try again later.

+

Sorry, you cannot login with Github or ORCID at this time.

{% endif %}
-- cgit v1.2.3