diff options
| -rw-r--r-- | README.md | 24 | ||||
| -rwxr-xr-x | bin/test-website | 2 | ||||
| -rw-r--r-- | test/requests/link_checker.py | 10 | ||||
| -rw-r--r-- | test/requests/mapping_tests.py | 35 | ||||
| -rw-r--r-- | test/requests/navigation_tests.py | 15 | ||||
| -rw-r--r-- | test/requests/parametrized_test.py | 7 | ||||
| -rw-r--r-- | test/requests/run-integration-tests.py | 34 | ||||
| -rwxr-xr-x | test/requests/test-website.py | 42 | ||||
| -rw-r--r-- | test/requests/test_forgot_password.py | 52 | ||||
| -rw-r--r-- | wqflask/base/anon_collection.py | 3 | ||||
| -rw-r--r-- | wqflask/utility/elasticsearch_tools.py | 61 | ||||
| -rw-r--r-- | wqflask/utility/svg.py | 2 | ||||
| -rw-r--r-- | wqflask/wqflask/templates/new_security/login_user.html | 7 | ||||
| -rw-r--r-- | wqflask/wqflask/user_manager.py | 35 |
14 files changed, 248 insertions, 81 deletions
@@ -17,25 +17,35 @@ deploy GN2 and dependencies as a self contained unit on any machine. The database can be run separately as well as the source tree (for developers). See the [installation docs](doc/README.org). -## Test +## Run Once installed GN2 can be run online through a browser interface ```sh -./bin/genenetwork2 +genenetwork2 ``` -(default is http://localhost:5003/). For more examples, including running scripts and a Python REPL -see the startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). +(default is http://localhost:5003/). For full examples (you'll need to +set a number of environment variables), including running scripts and +a Python REPL, see the startup script +[./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). +## Testing -We are building up automated -testing using [mechanize](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) which can be run with +We are building 'Mechanical Rob' automated testing using Python +[requests](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) +which can be run with something like ```sh -./bin/test-website +env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 ``` +The GN2_PROFILE is the Guix profile that contains all +dependencies. The ./bin/genenetwork2 script sets up the environment +and executes test-website.py in a Python interpreter. The -a switch +says to run all tests and the URL points to the running GN2 http +server. + ## Documentation User documentation can be found diff --git a/bin/test-website b/bin/test-website index 5935f016..7fbcfd2f 100755 --- a/bin/test-website +++ b/bin/test-website @@ -2,6 +2,6 @@ if [ -z $GN2_PROFILE ]; then echo "Run request tests with something like" - echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 + echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 exit 1 fi diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index 64553ed8..715f330c 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -18,6 +18,10 @@ def is_internal_link(link): pattern = re.compile("^/.*") return pattern.match(link) +def is_in_page_link(link): + pattern = re.compile("^#.*") + return pattern.match(link) + def get_links(doc): return filter( lambda x: not ( @@ -40,6 +44,7 @@ def verify_link(link): else: print("ERROR: link `"+link+"` failed with status " , result.status_code) + if DO_FAIL: raise Exception("Failed verify") except ConnectionError as ex: @@ -52,9 +57,10 @@ def check_page(host, start_url): print("Checking links host "+host+" in page `"+start_url+"`") doc = parse(start_url).getroot() links = get_links(doc) + in_page_links = filter(is_in_page_link, links) internal_links = filter(is_internal_link, links) - external_links = filter(lambda x: not is_internal_link(x), links) - # external_links.append("http://somenon-existentsite.brr") + external_links = filter(lambda x: not (is_internal_link(x) or is_in_page_link(x)), links) + for link in internal_links: verify_link(host+link) diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py index fd20df11..8eb19de7 100644 --- a/test/requests/mapping_tests.py +++ b/test/requests/mapping_tests.py @@ -1,17 +1,10 @@ from __future__ import print_function import re +import copy import json import requests from lxml.html import fromstring -def get_data(list_item): - try: - value = list_item[1] - except: - value = None - #print("list_item:", list_item, "==>", value) - return value - def load_data_from_file(): filename = "../test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json" file_handle = open(filename, "r") @@ -19,6 +12,8 @@ def load_data_from_file(): return file_data def check_pylmm_tool_selection(host, data): + print("") + print("pylmm mapping tool selection") data["method"] = "pylmm" page = requests.post(host+"/marker_regression", data=data) doc = fromstring(page.text) @@ -27,10 +22,24 @@ def check_pylmm_tool_selection(host, data): assert form.fields["value:BXD1"] == "15.034" # Check value in the file def check_R_qtl_tool_selection(host, data): - pass + print("") + print("R/qtl mapping tool selection") + headers = {"Content-Type": "application/x-www-form-urlencoded"} + page = requests.post(host+"/marker_regression", data=data, headers=headers) + doc = fromstring(page.text) + form = doc.forms[1] + assert form.fields["dataset"] == "HC_M2_0606_P" + assert form.fields["value:BXD1"] == "15.034" def check_CIM_tool_selection(host, data): - pass + print("") + print("CIM mapping tool selection (using reaper)") + data["method"] = "reaper" + page = requests.post(host+"/marker_regression", data=data) + doc = fromstring(page.text) + form = doc.forms[1] + assert form.fields["dataset"] == "HC_M2_0606_P" + assert form.fields["value:BXD1"] == "15.034" def check_mapping(args_obj, parser): print("") @@ -38,6 +47,6 @@ def check_mapping(args_obj, parser): host = args_obj.host data = load_data_from_file() - check_pylmm_tool_selection(host, data) - check_R_qtl_tool_selection(host, data) - check_CIM_tool_selection(host, data) + check_pylmm_tool_selection(host, copy.deepcopy(data)) + check_R_qtl_tool_selection(host, copy.deepcopy(data)) ## Why does this fail? + check_CIM_tool_selection(host, copy.deepcopy(data)) diff --git a/test/requests/navigation_tests.py b/test/requests/navigation_tests.py new file mode 100644 index 00000000..eda27324 --- /dev/null +++ b/test/requests/navigation_tests.py @@ -0,0 +1,15 @@ +from __future__ import print_function +import re +import requests +from lxml.html import parse + +def check_navigation(args_obj, parser): + print("") + print("Checking navigation.") + + host = args_obj.host + url = host + "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" + print("URL: ", url) + page = requests.get(url) + # Page is built by the javascript, hence using requests fails for this. + # Investigate use of selenium maybe? diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py index abf98fce..50003850 100644 --- a/test/requests/parametrized_test.py +++ b/test/requests/parametrized_test.py @@ -1,5 +1,7 @@ import logging import unittest +from wqflask import app +from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column from elasticsearch import Elasticsearch, TransportError class ParametrizedTest(unittest.TestCase): @@ -10,10 +12,11 @@ class ParametrizedTest(unittest.TestCase): self.es_url = es_url def setUp(self): - self.es = Elasticsearch([self.es_url]) + self.es = get_elasticsearch_connection() self.es_cleanup = [] es_logger = logging.getLogger("elasticsearch") + es_logger.setLevel(app.config.get("LOG_LEVEL")) es_logger.addHandler( logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) es_trace_logger = logging.getLogger("elasticsearch.trace") @@ -21,7 +24,9 @@ class ParametrizedTest(unittest.TestCase): logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) def tearDown(self): + from time import sleep self.es.delete_by_query( index="users" , doc_type="local" , body={"query":{"match":{"email_address":"test@user.com"}}}) + sleep(1) diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py deleted file mode 100644 index 5e816549..00000000 --- a/test/requests/run-integration-tests.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys -from test_login_local import TestLoginLocal -from test_login_orcid import TestLoginOrcid -from test_login_github import TestLoginGithub -from test_registration import TestRegistration -from unittest import TestSuite, TextTestRunner, TestLoader - -test_cases = [ - TestRegistration - , TestLoginLocal - , TestLoginGithub - , TestLoginOrcid -] - -def suite(gn2_url, es_url): - the_suite = TestSuite() - for case in test_cases: - the_suite.addTests(initTest(case, gn2_url, es_url)) - return the_suite - -def initTest(klass, gn2_url, es_url): - loader = TestLoader() - methodNames = loader.getTestCaseNames(klass) - return [klass(mname, gn2_url, es_url) for mname in methodNames] - -def main(gn2_url, es_url): - runner = TextTestRunner() - runner.run(suite(gn2_url, es_url)) - -if __name__ == "__main__": - if len(sys.argv) < 3: - raise Exception("Required arguments missing:\n\tTry running `run-integration-test.py <gn2-url> <es-url>`") - else: - main(sys.argv[1], sys.argv[2]) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 118c9df1..b2e09bc4 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -7,10 +7,20 @@ from __future__ import print_function import argparse from link_checker import check_links from mapping_tests import check_mapping +from navigation_tests import check_navigation from main_web_functionality import check_main_web_functionality import link_checker import sys +# Imports for integration tests +from wqflask import app +from test_login_local import TestLoginLocal +from test_login_orcid import TestLoginOrcid +from test_login_github import TestLoginGithub +from test_registration import TestRegistration +from test_forgot_password import TestForgotPassword +from unittest import TestSuite, TextTestRunner, TestLoader + print("Mechanical Rob firing up...") def run_all(args_obj, parser): @@ -29,6 +39,33 @@ def print_help(args_obj, parser): def dummy(args_obj, parser): print("Not implemented yet.") +def integration_tests(args_obj, parser): + gn2_url = args_obj.host + es_url = app.config.get("ELASTICSEARCH_HOST")+":"+str(app.config.get("ELASTICSEARCH_PORT")) + run_integration_tests(gn2_url, es_url) + +def initTest(klass, gn2_url, es_url): + loader = TestLoader() + methodNames = loader.getTestCaseNames(klass) + return [klass(mname, gn2_url, es_url) for mname in methodNames] + +def integration_suite(gn2_url, es_url): + test_cases = [ + TestRegistration + , TestLoginLocal + , TestLoginGithub + , TestLoginOrcid + , TestForgotPassword + ] + the_suite = TestSuite() + for case in test_cases: + the_suite.addTests(initTest(case, gn2_url, es_url)) + return the_suite + +def run_integration_tests(gn2_url, es_url): + runner = TextTestRunner() + runner.run(integration_suite(gn2_url, es_url)) + desc = """ This is Mechanical-Rob - an automated web server tester for @@ -63,6 +100,11 @@ parser.add_argument("-m", "--mapping", dest="accumulate" , action="store_const", const=check_mapping, default=print_help , help="Checks for mapping.") +parser.add_argument("-i", "--integration-tests", dest="accumulate" + , action="store_const", const=integration_tests, default=print_help + , help="Runs integration tests.") + +# Navigation tests deactivated since system relies on Javascript # parser.add_argument("-n", "--navigation", dest="accumulate" # , action="store_const", const=check_navigation, default=print_help # , help="Checks for navigation.") diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py new file mode 100644 index 00000000..2bf34c5c --- /dev/null +++ b/test/requests/test_forgot_password.py @@ -0,0 +1,52 @@ +import requests +from wqflask import user_manager +from utility.elasticsearch_tools import get_user_by_unique_column +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +passwork_reset_link = '' +forgot_password_page = None + +class TestForgotPassword(ParametrizedTest): + + def setUp(self): + super(TestForgotPassword, self).setUp() + self.forgot_password_url = self.gn2_url+"/n/forgot_password_submit" + def send_email(to_addr, msg, fromaddr="no-reply@genenetwork.org"): + print("CALLING: send_email_mock()") + email_data = { + "to_addr": to_addr + , "msg": msg + , "fromaddr": from_addr} + + data = { + "es_connection": self.es, + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + user_manager.basic_info = lambda : { "basic_info": "basic" } + user_manager.RegisterUser(data) + + def testWithoutEmail(self): + data = {"email_address": ""} + error_notification = '<div class="alert alert-danger">You MUST provide an email</div>' + result = requests.post(self.forgot_password_url, data=data) + self.assertEqual(result.url, self.gn2_url+"/n/forgot_password") + self.assertTrue( + result.content.find(error_notification) >= 0 + , "Error message should be displayed but was not") + + def testWithNonExistingEmail(self): + # Monkey patching doesn't work, so simply test that getting by email + # returns the correct data + user = get_user_by_unique_column(self.es, "email_address", "non-existent@domain.com") + self.assertTrue(user is None, "Should not find non-existent user") + + def testWithExistingEmail(self): + # Monkey patching doesn't work, so simply test that getting by email + # returns the correct data + user = get_user_by_unique_column(self.es, "email_address", "test@user.com") + self.assertTrue(user is not None, "Should find user") diff --git a/wqflask/base/anon_collection.py b/wqflask/base/anon_collection.py index 8ee73296..dd1aa27f 100644 --- a/wqflask/base/anon_collection.py +++ b/wqflask/base/anon_collection.py @@ -1,6 +1,6 @@ class AnonCollection(TraitCollection):
- def __init__(self, anon_id)
+ def __init__(self, anon_id):
self.anon_id = anon_id
self.collection_members = Redis.smembers(self.anon_id)
print("self.collection_members is:", self.collection_members)
@@ -12,6 +12,7 @@ class AnonCollection(TraitCollection): print("traits_to_remove:", traits_to_remove)
for trait in traits_to_remove:
Redis.srem(self.anon_id, trait)
+
members_now = self.collection_members - traits_to_remove
print("members_now:", members_now)
print("Went from {} to {} members in set.".format(len(self.collection_members), len(members_now)))
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 1dba357d..76dcaebf 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,3 +1,44 @@ +# Elasticsearch support +# +# Some helpful commands to view the database: +# +# You can test the server being up with +# +# curl -H 'Content-Type: application/json' http://localhost:9200 +# +# List all indices +# +# curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v' +# +# To see the users index 'table' +# +# curl http://localhost:9200/users +# +# To list all user ids +# +# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' +# { +# "query" : { +# "match_all" : {} +# }, +# "stored_fields": [] +# }' +# +# To view a record +# +# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' +# { +# "query" : { +# "match" : { "email_address": "pjotr2017@thebird.nl"} +# } +# }' +# +# +# To delete the users index and data (dangerous!) +# +# curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users' + + from elasticsearch import Elasticsearch, TransportError import logging @@ -7,7 +48,7 @@ logger = getLogger(__name__) from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT def test_elasticsearch_connection(): - es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+ELASTICSEARCH_PORT+'/'], verify_certs=True) + es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+str(ELASTICSEARCH_PORT)+'/'], verify_certs=True) if not es.ping(): logger.warning("Elasticsearch is DOWN") @@ -24,15 +65,29 @@ def get_elasticsearch_connection(): "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + setup_users_index(es) + es_logger = logging.getLogger("elasticsearch") es_logger.setLevel(logging.INFO) es_logger.addHandler(logging.NullHandler()) - except: - logger.error("Failed to get elasticsearch connection") + except Exception as e: + logger.error("Failed to get elasticsearch connection", e) es = None return es +def setup_users_index(es_connection): + if es_connection: + index_settings = { + "properties": { + "email_address": { + "type": "keyword"}}} + + es_connection.indices.create(index='users', ignore=400) + es_connection.indices.close(index="users") + es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local") + es_connection.indices.open(index="users") + def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 512bc9e6..db13b9d1 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -1029,7 +1029,7 @@ class drawing: try: xv.feed(svg) except: - raise "SVG is not well formed, see messages above" + raise Exception("SVG is not well formed, see messages above") else: print "SVG well formed" if __name__=='__main__': diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 4a857c60..27b20ebf 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -31,16 +31,19 @@ <div> {% if external_login["github"]: %} <a href="{{external_login['github']}}" title="Login with GitHub" class="btn btn-info btn-group">Login with Github</a> + {% else %} + <p>Github login is not available right now</p> {% endif %} {% if external_login["orcid"]: %} <a href="{{external_login['orcid']}}" title="Login with ORCID" class="btn btn-info btn-group">Login with ORCID</a> + {% else %} + <p>ORCID login is not available right now</p> {% endif %} </div> {% else: %} <div class="alert alert-warning"> - <p>You cannot login with external services at this time.<br /> - Please try again later.</p> + <p>Sorry, you cannot login with Github or ORCID at this time.</p> </div> {% endif %} <hr /> diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 5f6c818e..d652f2e9 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -727,30 +727,33 @@ def logout(): return response -@app.route("/n/forgot_password") +@app.route("/n/forgot_password", methods=['GET']) def forgot_password(): """Entry point for forgotten password""" - return render_template("new_security/forgot_password.html") + print("ARGS: ", request.args) + errors = {"no-email": request.args.get("no-email")} + print("ERRORS: ", errors) + return render_template("new_security/forgot_password.html", errors=errors) @app.route("/n/forgot_password_submit", methods=('POST',)) def forgot_password_submit(): """When a forgotten password form is submitted we get here""" params = request.form email_address = params['email_address'] - logger.debug("Wants to send password E-mail to ",email_address) - es = get_elasticsearch_connection() - user_details = get_user_by_unique_column(es, "email_address", email_address) - if user_details: - ForgotPasswordEmail(user_details["email_address"]) - # try: - # user = model.User.query.filter_by(email_address=email_address).one() - # except orm.exc.NoResultFound: - # flash("Couldn't find a user associated with the email address {}. Sorry.".format( - # email_address)) - # return redirect(url_for("login")) - # ForgotPasswordEmail(user) - return render_template("new_security/forgot_password_step2.html", - subject=ForgotPasswordEmail.subject) + next_page = None + if email_address != "": + logger.debug("Wants to send password E-mail to ",email_address) + es = get_elasticsearch_connection() + user_details = get_user_by_unique_column(es, "email_address", email_address) + if user_details: + ForgotPasswordEmail(user_details["email_address"]) + + return render_template("new_security/forgot_password_step2.html", + subject=ForgotPasswordEmail.subject) + + else: + flash("You MUST provide an email", "alert-danger") + return redirect(url_for("forgot_password")) @app.errorhandler(401) def unauthorized(error): |