From 236ca06dc4c84baecb7b090b8724db997a5d988a Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 13 Mar 2021 13:04:33 +0300 Subject: Correlation api (#2) * add file for correlation api * register initial correlation api * add correlation package * add function for getting page data * delete loading page api * modify code for correlation * add tests folder for correlations * fix error in correlation api * add tests for correlation * add tests for correlation loading data * add module for correlation computations * modify api to return json when computing correlation * add tests for computing correlation * modify code for loading correlation data * modify tests for correlation computation * test loading correlation data using api endpoint * add tests for asserting error in creating Correlation object * add do correlation method * add dummy tests for do_correlation method * delete unused modules * add tests for creating trait and dataset * add intergration test for correlation api * add tests for correlation api * edit docorrelation method * modify integration tests for correlation api * modify tests for show_corr_results * add create dataset function * pep8 formatting and fix return value for api * add more test data for doing correlation * modify tests for correlation * pep8 formatting * add getting formatted corr type method * import json library add process samples method for correlation * fix issue with sample_vals key_error * create utility module for correlation * refactor endpoint for /corr_compute * add test and mocks for compute_correlation function * add compute correlation function and pep8 formatting * move get genofile samplelist to utility module * refactor code for CorrelationResults object * pep8 formatting for module * remove CorrelationResults from Api * add base package initialize data_set module with create_dataset,redis and Dataset_Getter * set dataset_structure if redis is empty * add callable for DatsetType * add set_dataset_key method If name is not in the object's dataset dictionary * add Dataset object and MrnaAssayDataSet * add db_tools * add mysql client * add DatasetGroup object * add species module * get mapping method * import helper functions and new dataset * add connection to db before request * add helper functions * add logger module * add get_group_samplelists module * add logger for debug * add code for adding sample_data * pep8 formatting * Add chunks module * add correlation helper module * add get_sample_r_and_p_values method add get_header_fields function * add generate corr json method * add function to retrieve_trait_info * remove comments and clean up code in show_corr_results * remove comments and clean up code for data_set module * pep8 formatting for helper_functions module * pep8 formatting for trait module * add module for species * add Temp Dataset Object * add Phenotype Dataset * add Genotype Dataset * add rettrieve sample_sample_data method * add webqtlUtil module * add do lit correlation for all traits * add webqtlCaseData:Settings not ported * return the_trait for create trait method * add correlation_test json data * add tests fore show corr results * add dictfier package * add tests for show_corr_results * add assertion for trait_id * refactor code for show_corr_results * add test file for compute_corr intergration tests * add scipy dependency * refactor show_corr_results object add do lit correlation for trait_list * add hmac module * add bunch module:Dictionary using object notation * add correlation functions * add rpy2 dependency * add hmac module * add MrnaAssayTissueData object and get_symbol_values_pairs function * add config module * add get json_results method * pep8 formatting remove comments * add config file * add db package * refactor correlatio compuatation module * add do tissue correlation for trait list * add do lit correlation for all traits * add do tissue correlation for all traits * add do_bicor for bicor method * raise error for when initital start vars is None * add support for both form and json data when for correlation input * remove print statement and pep8 formatting * add default settings file * add tools module for locate_ignore_error * refactor code remove comments for trait module * Add new test data for computing correlation * pep8 formatting and use pickle * refactor function for filtering form/json data * remove unused imports * remove mock functions in correlation_utility module * refactor tests for compute correlation and pep8 formatting * add tests for show_correlation results * modify tests for show_corr_results * add json files for tests * pep8 formatting for show_corr_results * Todo:Lint base files * pylint for intergration tests * add test module for test_corr_helpers * Add test chunk module * lint utility package * refactoring and pep8 formatting * implement simple metric for correlation * add hmac utility file * add correlation prefix * fix merge conflict * minor fixes for endpoints * import:python-scipy,python-sqlalchemy from guix * add python mysqlclient * remove pkg-resources from requirements * add python-rpy3 from guix * refactor code for species module * pep8 formatting and refactor code * add tests for genereating correlation results * lint correlation functions * fix failing tests for show_corr_results * add new correlation test data fix errors * fix issues related to getting group samplelists * refactor intergration tests for correlation * add todo for refactoring_wanted_inputs * replace custom Attribute setter with SimpleNamespace * comparison of sample r correlation results btwn genenenetwork2 and genenetwork3 * delete AttributeSetter * test request for /api/correlation/compute_correlation took 18.55710196495056 Seconds * refactor tests and show_correlation results * remove unneccessary comments and print statements * edit requirement txt file * api/correlation took 114.29814600944519 Seconds for correlation resullts:20000 - corr-type:lit - corr-method:pearson corr-dataset:corr_dataset:HC_M2_0606_P * capture SQL_URI and GENENETWORK FILES path * pep8 formatting edit && remove print statements * delete filter_input function update test and data for correlation * add docstring for required correlation_input * /api/correlation took 12.905632972717285 Seconds * pearson * lit *dataset:HX_M2_0606_P trait_id :1444666 p_range:(lower->-0.60,uppper->0.74) corr_return_results: 100 * update integration and unittest for correlation * add simple markdown docs for correlation * update docs * add tests and catch for invalid correlation_input * minor fix for api * Remove jupyter from deps * guix.scm: Remove duplicate entry * guix.scm: Add extra action items as comments * Trim requirements.txt file Co-authored-by: BonfaceKilz --- tests/unit/utility/__init__.py | 0 tests/unit/utility/test_chunks.py | 19 ++++++++++ tests/unit/utility/test_corr_result_helpers.py | 35 ++++++++++++++++++ tests/unit/utility/test_hmac.py | 51 ++++++++++++++++++++++++++ 4 files changed, 105 insertions(+) create mode 100644 tests/unit/utility/__init__.py create mode 100644 tests/unit/utility/test_chunks.py create mode 100644 tests/unit/utility/test_corr_result_helpers.py create mode 100644 tests/unit/utility/test_hmac.py (limited to 'tests/unit/utility') diff --git a/tests/unit/utility/__init__.py b/tests/unit/utility/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/utility/test_chunks.py b/tests/unit/utility/test_chunks.py new file mode 100644 index 0000000..7c42b44 --- /dev/null +++ b/tests/unit/utility/test_chunks.py @@ -0,0 +1,19 @@ +"""Test chunking""" + +import unittest + +from gn3.utility.chunks import divide_into_chunks + + +class TestChunks(unittest.TestCase): + "Test Utility method for chunking" + def test_divide_into_chunks(self): + "Check that a list is chunked correctly" + self.assertEqual(divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 3), + [[1, 2, 7], [3, 22, 8], [5, 22, 333]]) + self.assertEqual(divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 4), + [[1, 2, 7], [3, 22, 8], [5, 22, 333]]) + self.assertEqual(divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 5), + [[1, 2], [7, 3], [22, 8], [5, 22], [333]]) + self.assertEqual(divide_into_chunks([], 5), + [[]]) diff --git a/tests/unit/utility/test_corr_result_helpers.py b/tests/unit/utility/test_corr_result_helpers.py new file mode 100644 index 0000000..ce5891f --- /dev/null +++ b/tests/unit/utility/test_corr_result_helpers.py @@ -0,0 +1,35 @@ +""" Test correlation helper methods """ + +import unittest +from gn3.utility.corr_result_helpers import normalize_values +from gn3.utility.corr_result_helpers import common_keys +from gn3.utility.corr_result_helpers import normalize_values_with_samples + + +class TestCorrelationHelpers(unittest.TestCase): + """Test methods for normalising lists""" + + def test_normalize_values(self): + """Test that a list is normalised correctly""" + self.assertEqual( + normalize_values([2.3, None, None, 3.2, 4.1, 5],\ + [3.4, 7.2, 1.3, None, 6.2, 4.1]), + ([2.3, 4.1, 5], [3.4, 6.2, 4.1], 3) + ) + + def test_common_keys(self): + """Test that common keys are returned as a list""" + test_a = dict(BXD1=9.113, BXD2=9.825, BXD14=8.985, BXD15=9.300) + test_b = dict(BXD1=9.723, BXD3=9.825, BXD14=9.124, BXD16=9.300) + self.assertEqual(sorted(common_keys(test_a, test_b)), + ['BXD1', 'BXD14']) + + def test_normalize_values_with_samples(self): + """Test that a sample(dict) is normalised correctly""" + self.assertEqual( + normalize_values_with_samples( + dict(BXD1=9.113, BXD2=9.825, BXD14=8.985, + BXD15=9.300, BXD20=9.300), + dict(BXD1=9.723, BXD3=9.825, BXD14=9.124, BXD16=9.300)), + (({'BXD1': 9.113, 'BXD14': 8.985}, {'BXD1': 9.723, 'BXD14': 9.124}, 2)) + ) diff --git a/tests/unit/utility/test_hmac.py b/tests/unit/utility/test_hmac.py new file mode 100644 index 0000000..eba25a3 --- /dev/null +++ b/tests/unit/utility/test_hmac.py @@ -0,0 +1,51 @@ +"""Test hmac utility functions""" +# pylint: disable-all +import unittest +from unittest import mock + +from gn3.utility.hmac import data_hmac +from gn3.utility.hmac import url_for_hmac +from gn3.utility.hmac import hmac_creation + + +class TestHmacUtil(): + """Test Utility method for hmac creation""" + + @mock.patch("utility.hmac.app.config", {'SECRET_HMAC_CODE': "secret"}) + def test_hmac_creation(self): + """Test hmac creation with a utf-8 string""" + self.assertEqual(hmac_creation("ファイ"), "7410466338cfe109e946") + + @mock.patch("utility.hmac.app.config", + {'SECRET_HMAC_CODE': ('\x08\xdf\xfa\x93N\x80' + '\xd9\\H@\\\x9f`\x98d^' + '\xb4a;\xc6OM\x946a\xbc' + '\xfc\x80:*\xebc')}) + def test_hmac_creation_with_cookie(self): + """Test hmac creation with a cookie""" + cookie = "3f4c1dbf-5b56-4260-87d6-f35445bda37e:af4fcf5eace9e7c864ce" + uuid_, _, signature = cookie.partition(":") + self.assertEqual( + hmac_creation(uuid_), + "af4fcf5eace9e7c864ce") + + @mock.patch("utility.hmac.app.config", {'SECRET_HMAC_CODE': "secret"}) + def test_data_hmac(self): + """Test data_hmac fn with a utf-8 string""" + self.assertEqual(data_hmac("ファイ"), "ファイ:7410466338cfe109e946") + + @mock.patch("utility.hmac.app.config", {'SECRET_HMAC_CODE': "secret"}) + @mock.patch("utility.hmac.url_for") + def test_url_for_hmac_with_plain_url(self, mock_url): + """Test url_for_hmac without params""" + mock_url.return_value = "https://mock_url.com/ファイ/" + self.assertEqual(url_for_hmac("ファイ"), + "https://mock_url.com/ファイ/?hm=05bc39e659b1948f41e7") + + @mock.patch("utility.hmac.app.config", {'SECRET_HMAC_CODE': "secret"}) + @mock.patch("utility.hmac.url_for") + def test_url_for_hmac_with_param_in_url(self, mock_url): + """Test url_for_hmac with params""" + mock_url.return_value = "https://mock_url.com/?ファイ=1" + self.assertEqual(url_for_hmac("ファイ"), + "https://mock_url.com/?ファイ=1&hm=4709c1708270644aed79") -- cgit v1.2.3