From 3cc448e8ee95173dda7f2d50c0cf2ce743cd1926 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 10 Aug 2022 12:02:54 +0300 Subject: Fix formatting for legibility --- test/requests/correlation_tests.py | 44 ++++++++++++++++++++++++++++++++++++++ test/requests/test-website.py | 2 ++ 2 files changed, 46 insertions(+) create mode 100644 test/requests/correlation_tests.py (limited to 'test') diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py new file mode 100644 index 00000000..b0213ff5 --- /dev/null +++ b/test/requests/correlation_tests.py @@ -0,0 +1,44 @@ +import requests +from lxml.html import parse +from link_checker import check_page + +def do_request(host, data): + return request.get( + f"{host}/corr_compute", + params={ + **data, + "corr_dataset": "HC_M2_0606_P", + "corr_return_results": "100", + "corr_samples_group": "samples_primary",}) + +def check_sample_correlations(baseurl): + data = { + "corr_type": "sample", + "corr_sample_method": "pearson", + "location_type": "gene" + } + result = do_request(host, data) + assert result.status == 200 + assert (result.text.find("Values of record 1435464_at") >= 0), "" + +def check_tissue_correlations(baseurl): + data = { + "corr_type": "tissue" + } + result = do_request(host, data) + assert False, "Not implemented yet." + +def check_lit_correlations(baseurl): + data = { + "corr_type": "lit" + } + result = do_request(host, data) + assert False, "Not implemented yet." + +def check_correlations(args_obj, parser): + print("") + print("Checking the correlations...") + host = args_obj.host + check_sample_correlations(host) + check_tissue_correlations(host) + check_lit_correlations(host) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 71055fca..8062bff1 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -9,6 +9,7 @@ from link_checker import check_links from link_checker import check_packaged_js_files from mapping_tests import check_mapping from navigation_tests import check_navigation +from correlation_tests import check_correlations from main_web_functionality import check_main_web_functionality import link_checker import sys @@ -28,6 +29,7 @@ def run_all(args_obj, parser): check_links(args_obj, parser) check_packaged_js_files(args_obj, parser) check_mapping(args_obj, parser) + check_correlations(args_obj, parser) # TODO: Add other functions as they are created. -- cgit v1.2.3 From e55aadbf0f1dd4752cd68b572ce6c66bcb6caed8 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 10 Aug 2022 13:47:56 +0300 Subject: Check that the correlations work as expected. --- test/requests/correlation_tests.py | 61 +++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 17 deletions(-) (limited to 'test') diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py index b0213ff5..070698fd 100644 --- a/test/requests/correlation_tests.py +++ b/test/requests/correlation_tests.py @@ -2,43 +2,70 @@ import requests from lxml.html import parse from link_checker import check_page -def do_request(host, data): - return request.get( - f"{host}/corr_compute", - params={ - **data, +def sample_vals(): + return '{"C57BL/6J":"10.835","DBA/2J":"11.142","B6D2F1":"11.126","D2B6F1":"11.143","BXD1":"10.811","BXD2":"11.503","BXD5":"10.766","BXD6":"10.986","BXD8":"11.050","BXD9":"10.822","BXD11":"10.670","BXD12":"10.946","BXD13":"10.890","BXD14":"x","BXD15":"10.884","BXD16":"11.222","BXD18":"x","BXD19":"10.968","BXD20":"10.962","BXD21":"10.906","BXD22":"11.080","BXD23":"11.046","BXD24":"11.146","BXD24a":"x","BXD25":"x","BXD27":"11.078","BXD28":"11.034","BXD29":"10.808","BXD30":"x","BXD31":"11.087","BXD32":"11.029","BXD33":"10.662","BXD34":"11.482","BXD35":"x","BXD36":"x","BXD37":"x","BXD38":"10.836","BXD39":"10.926","BXD40":"10.638","BXD41":"x","BXD42":"10.974","BXD43":"10.828","BXD44":"10.900","BXD45":"11.358","BXD48":"11.042","BXD48a":"10.975","BXD49":"x","BXD50":"11.228","BXD51":"11.126","BXD52":"x","BXD53":"x","BXD54":"x","BXD55":"11.580","BXD56":"x","BXD59":"x","BXD60":"10.829","BXD61":"11.152","BXD62":"11.156","BXD63":"10.942","BXD64":"10.506","BXD65":"11.126","BXD65a":"11.272","BXD65b":"11.157","BXD66":"11.071","BXD67":"11.080","BXD68":"10.997","BXD69":"11.096","BXD70":"11.152","BXD71":"x","BXD72":"x","BXD73":"11.262","BXD73a":"11.444","BXD73b":"x","BXD74":"10.974","BXD75":"11.150","BXD76":"10.920","BXD77":"10.928","BXD78":"x","BXD79":"11.371","BXD81":"x","BXD83":"10.946","BXD84":"11.181","BXD85":"10.992","BXD86":"10.770","BXD87":"11.200","BXD88":"x","BXD89":"10.930","BXD90":"11.183","BXD91":"x","BXD93":"11.056","BXD94":"10.737","BXD95":"x","BXD98":"10.986","BXD99":"10.892","BXD100":"x","BXD101":"x","BXD102":"x","BXD104":"x","BXD105":"x","BXD106":"x","BXD107":"x","BXD108":"x","BXD109":"x","BXD110":"x","BXD111":"x","BXD112":"x","BXD113":"x","BXD114":"x","BXD115":"x","BXD116":"x","BXD117":"x","BXD119":"x","BXD120":"x","BXD121":"x","BXD122":"x","BXD123":"x","BXD124":"x","BXD125":"x","BXD126":"x","BXD127":"x","BXD128":"x","BXD128a":"x","BXD130":"x","BXD131":"x","BXD132":"x","BXD133":"x","BXD134":"x","BXD135":"x","BXD136":"x","BXD137":"x","BXD138":"x","BXD139":"x","BXD141":"x","BXD142":"x","BXD144":"x","BXD145":"x","BXD146":"x","BXD147":"x","BXD148":"x","BXD149":"x","BXD150":"x","BXD151":"x","BXD152":"x","BXD153":"x","BXD154":"x","BXD155":"x","BXD156":"x","BXD157":"x","BXD160":"x","BXD161":"x","BXD162":"x","BXD165":"x","BXD168":"x","BXD169":"x","BXD170":"x","BXD171":"x","BXD172":"x","BXD173":"x","BXD174":"x","BXD175":"x","BXD176":"x","BXD177":"x","BXD178":"x","BXD180":"x","BXD181":"x","BXD183":"x","BXD184":"x","BXD186":"x","BXD187":"x","BXD188":"x","BXD189":"x","BXD190":"x","BXD191":"x","BXD192":"x","BXD193":"x","BXD194":"x","BXD195":"x","BXD196":"x","BXD197":"x","BXD198":"x","BXD199":"x","BXD200":"x","BXD201":"x","BXD202":"x","BXD203":"x","BXD204":"x","BXD205":"x","BXD206":"x","BXD207":"x","BXD208":"x","BXD209":"x","BXD210":"x","BXD211":"x","BXD212":"x","BXD213":"x","BXD214":"x","BXD215":"x","BXD216":"x","BXD217":"x","BXD218":"x","BXD219":"x","BXD220":"x"}' + +def do_request(url, data): + return requests.post( + url, + data={ + "dataset": "HC_M2_0606_P", + "trait_id": "1435464_at", "corr_dataset": "HC_M2_0606_P", "corr_return_results": "100", - "corr_samples_group": "samples_primary",}) + "corr_samples_group": "samples_primary", + "sample_vals": sample_vals(), + **data, + }) def check_sample_correlations(baseurl): data = { "corr_type": "sample", "corr_sample_method": "pearson", - "location_type": "gene" + "location_type": "gene", + "corr_return_results": "200" } - result = do_request(host, data) - assert result.status == 200 - assert (result.text.find("Values of record 1435464_at") >= 0), "" + top_n_message = "The top 200 correlations ranked by the Genetic Correlation" + result = do_request(f"{baseurl}/corr_compute", data) + assert result.status_code == 200 + assert (result.text.find("Values of record 1435464_at") >= 0), result.text + assert (result.text.find(top_n_message) >= 0), result.text def check_tissue_correlations(baseurl): + print("\tChecking tissue correlations...", end="") data = { - "corr_type": "tissue" + "corr_type": "tissue", + "corr_sample_method": "pearson", + "location_type": "gene", } - result = do_request(host, data) - assert False, "Not implemented yet." + top_n_message = "The top 100 correlations ranked by the Genetic Correlation" + result = do_request(f"{baseurl}/corr_compute", data) + assert result.status_code == 200 + assert (result.text.find("Values of record 1435464_at") >= 0), result.text + assert (result.text.find(top_n_message) >= 0), result.text def check_lit_correlations(baseurl): + print("\tChecking lit correlations...", end="") data = { - "corr_type": "lit" + "corr_type": "lit", + "corr_return_results": "200" } - result = do_request(host, data) - assert False, "Not implemented yet." + top_n_message = "The top 500 correlations ranked by the Genetic Correlation" + result = do_request(f"{baseurl}/corr_compute", data) + assert result.status_code == 200 + assert (result.text.find("Values of record 1435464_at") >= 0), result.text + assert (result.text.find(top_n_message) >= 0), result.text def check_correlations(args_obj, parser): print("") print("Checking the correlations...") + corr_type_fns = { + "sample": check_sample_correlations,} host = args_obj.host - check_sample_correlations(host) + for corr_type, corr_type_fn in corr_type_fns.items(): + print(f"\tChecking {corr_type} correlations...", end="") + corr_type_fn(host) + print(" ok") check_tissue_correlations(host) check_lit_correlations(host) + print("OK") -- cgit v1.2.3 From 15a1ff0d5cf8562c16e14491edd24b0f3171e083 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Thu, 11 Aug 2022 11:01:05 +0300 Subject: Fix correlation tests: Fix the test message. --- test/requests/correlation_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py index 070698fd..0b8ba8cd 100644 --- a/test/requests/correlation_tests.py +++ b/test/requests/correlation_tests.py @@ -32,25 +32,23 @@ def check_sample_correlations(baseurl): assert (result.text.find(top_n_message) >= 0), result.text def check_tissue_correlations(baseurl): - print("\tChecking tissue correlations...", end="") data = { "corr_type": "tissue", "corr_sample_method": "pearson", "location_type": "gene", } - top_n_message = "The top 100 correlations ranked by the Genetic Correlation" + top_n_message = "The top 100 correlations ranked by the Tissue Correlation" result = do_request(f"{baseurl}/corr_compute", data) assert result.status_code == 200 assert (result.text.find("Values of record 1435464_at") >= 0), result.text assert (result.text.find(top_n_message) >= 0), result.text def check_lit_correlations(baseurl): - print("\tChecking lit correlations...", end="") data = { "corr_type": "lit", "corr_return_results": "200" } - top_n_message = "The top 500 correlations ranked by the Genetic Correlation" + top_n_message = "The top 200 correlations ranked by the Lit Correlation" result = do_request(f"{baseurl}/corr_compute", data) assert result.status_code == 200 assert (result.text.find("Values of record 1435464_at") >= 0), result.text -- cgit v1.2.3 From 4bc374aeeba3e9fac75dea266e35bedb633f950c Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Fri, 12 Aug 2022 12:52:38 +0300 Subject: Fix some issues with the tests. Add notes in code for improvements. --- test/requests/correlation_tests.py | 9 +++++---- wqflask/wqflask/correlation/rust_correlation.py | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py index 0b8ba8cd..2d314833 100644 --- a/test/requests/correlation_tests.py +++ b/test/requests/correlation_tests.py @@ -12,6 +12,7 @@ def do_request(url, data): "dataset": "HC_M2_0606_P", "trait_id": "1435464_at", "corr_dataset": "HC_M2_0606_P", + "corr_sample_method": "pearson", "corr_return_results": "100", "corr_samples_group": "samples_primary", "sample_vals": sample_vals(), @@ -34,7 +35,6 @@ def check_sample_correlations(baseurl): def check_tissue_correlations(baseurl): data = { "corr_type": "tissue", - "corr_sample_method": "pearson", "location_type": "gene", } top_n_message = "The top 100 correlations ranked by the Tissue Correlation" @@ -58,12 +58,13 @@ def check_correlations(args_obj, parser): print("") print("Checking the correlations...") corr_type_fns = { - "sample": check_sample_correlations,} + "sample": check_sample_correlations, + "tissue": check_tissue_correlations, + "tissue": check_lit_correlations + } host = args_obj.host for corr_type, corr_type_fn in corr_type_fns.items(): print(f"\tChecking {corr_type} correlations...", end="") corr_type_fn(host) print(" ok") - check_tissue_correlations(host) - check_lit_correlations(host) print("OK") diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py index b4435887..3628f549 100644 --- a/wqflask/wqflask/correlation/rust_correlation.py +++ b/wqflask/wqflask/correlation/rust_correlation.py @@ -135,6 +135,7 @@ def compute_correlation_rust( (this_dataset, this_trait, target_dataset, sample_data) = ( target_trait_info) + ## Replace this with `match ...` once we hit Python 3.10 corr_type_fns = { "sample": __compute_sample_corr__, "tissue": __compute_tissue_corr__, @@ -142,6 +143,7 @@ def compute_correlation_rust( } results = corr_type_fns[corr_type]( start_vars, corr_type, method, n_top, target_trait_info) + ## END: Replace this with `match ...` once we hit Python 3.10 top_tissue_results = {} top_lit_results = {} -- cgit v1.2.3 From 3cfd0ec3d38f4b460817606ea5d6749a1a85a1ea Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Fri, 12 Aug 2022 14:34:02 +0300 Subject: Update to newer lxml API --- test/requests/link_checker.py | 5 +++-- test/requests/main_web_functionality.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'test') diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index feae6526..86222254 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -1,6 +1,6 @@ import re import requests -from lxml.html import parse +from lxml.html import document_fromstring from requests.exceptions import ConnectionError DO_FAIL=False # fail on error @@ -66,7 +66,8 @@ def verify_static_file(link): def check_page(host, start_url): print("") print("Checking links host "+host+" in page `"+start_url+"`") - doc = parse(start_url).getroot() + req = requests.get(start_url) + doc = document_fromstring(start_url) links = get_links(doc) in_page_links = list(filter(is_in_page_link, links)) internal_links = list(filter(is_internal_link, links)) diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py index 28033ad5..79f37bb4 100644 --- a/test/requests/main_web_functionality.py +++ b/test/requests/main_web_functionality.py @@ -1,10 +1,11 @@ import requests -from lxml.html import parse +from lxml.html import document_fromstring from link_checker import check_page def check_home(url): - doc = parse(url).getroot() + results = requests.get(url) + doc = document_fromstring(results.text) search_button = doc.cssselect("#btsearch") assert(search_button[0].value == "Search") print("OK") @@ -18,7 +19,7 @@ def check_search_page(host): search_terms_or="", search_terms_and="MEAN=(15 16) LRS=(23 46)") result = requests.get(host+"/search", params=data) - found = result.text.find("records were found") + found = result.text.find("records found") assert(found >= 0) assert(result.status_code == 200) print("OK") @@ -27,7 +28,8 @@ def check_search_page(host): def check_traits_page(host, traits_url): - doc = parse(host+traits_url).getroot() + results = requests.get(host+traits_url) + doc = document_fromstring(results.text) traits_form = doc.forms[1] assert(traits_form.fields["corr_dataset"] == "HC_M2_0606_P") print("OK") -- cgit v1.2.3 From dd1039afb7b6a8fd0a219c88b5661155abf1d758 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 17 Aug 2022 10:06:50 +0300 Subject: Do incremental backoff for mechanical rob tests Sometimes the tests might be run before the service has began running. This commit delays the running of the tests for a while. --- test/requests/test-website.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'test') diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 8062bff1..36d85e98 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -19,11 +19,24 @@ from unittest import TestSuite, TextTestRunner, TestLoader print("Mechanical Rob firing up...") +def host_is_online(host): + import time + import requests + for count in range(1, 11): + try: + time.sleep(count) + requests.get(host) + return True + except Exception as cre: + print(f"Retrying in {count + 1} seconds ...") + + return False def run_all(args_obj, parser): print("") print("Running all tests.") print(args_obj) + assert host_is_online(args_obj.host), f"Could not connect to {host}" link_checker.DO_FAIL = args_obj.fail check_main_web_functionality(args_obj, parser) check_links(args_obj, parser) -- cgit v1.2.3 From d9b8dec4f8bd9f1f08bb1d1c55c4036b047f9051 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 17 Aug 2022 10:20:38 +0300 Subject: Reduce incremental backoff time --- test/requests/test-website.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test') diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 36d85e98..c0b36182 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -22,7 +22,7 @@ print("Mechanical Rob firing up...") def host_is_online(host): import time import requests - for count in range(1, 11): + for count in range(1, 5): try: time.sleep(count) requests.get(host) -- cgit v1.2.3 From 9f3bae5e4a59112eab7498acade0ed7011fcfffd Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Fri, 19 Aug 2022 16:13:54 +0300 Subject: Fix message and function key --- test/requests/correlation_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py index 2d314833..abb5794a 100644 --- a/test/requests/correlation_tests.py +++ b/test/requests/correlation_tests.py @@ -48,7 +48,7 @@ def check_lit_correlations(baseurl): "corr_type": "lit", "corr_return_results": "200" } - top_n_message = "The top 200 correlations ranked by the Lit Correlation" + top_n_message = "The top 200 correlations ranked by the Literature Correlation" result = do_request(f"{baseurl}/corr_compute", data) assert result.status_code == 200 assert (result.text.find("Values of record 1435464_at") >= 0), result.text @@ -60,7 +60,7 @@ def check_correlations(args_obj, parser): corr_type_fns = { "sample": check_sample_correlations, "tissue": check_tissue_correlations, - "tissue": check_lit_correlations + "lit": check_lit_correlations } host = args_obj.host for corr_type, corr_type_fn in corr_type_fns.items(): -- cgit v1.2.3