From e6f10522833cbd75441766e5b8656b3f5925d6d7 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 30 Mar 2021 18:12:58 +0300 Subject: initial commit for creating trait and datasets --- tests/integration/test_traits.py | 70 ++++++++++++++++++++++++++++++ tests/unit/computations/test_trait.py | 81 +++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 tests/integration/test_traits.py create mode 100644 tests/unit/computations/test_trait.py (limited to 'tests') diff --git a/tests/integration/test_traits.py b/tests/integration/test_traits.py new file mode 100644 index 0000000..2f3433a --- /dev/null +++ b/tests/integration/test_traits.py @@ -0,0 +1,70 @@ +"""module contains integration tests for trait endpoints""" +from unittest import TestCase +from unittest import mock + +from gn3.app import create_app + + +class TraitIntegrationTest(TestCase): + """class contains integration tests for\ + traits""" + + def setUp(self): + self.app = create_app().test_client() + + def test_home(self): + """test for initial endpoint for trait""" + results = self.app.get("/api/trait", follow_redirects=True) + + self.assertEqual(200, results.status_code) + + @mock.patch("gn3.api.traits.fetch_trait") + def test_create_trait(self, mock_fetch_trait): + """test the endpoint for creating traits\ + endpoint requires trait name and dataset name""" + + trait_results = { + "dataset": None, + "trait_name": "1449593_at", + "trait_data": {} + + } + mock_fetch_trait.return_value = trait_results + + results = self.app.get( + "/api/trait/1449593_at/HC_M2_0606_P", follow_redirects=True) + + trait_data = results.get_json() + + self.assertEqual(results.status_code, 200) + self.assertEqual(trait_data, trait_results) + + @mock.patch("gn3.api.traits.get_trait_info_data") + def test_retrieve_trait_info(self, mock_get_trait_info): + """integration test for endpoints for retrieving\ + trait info expects the dataset of trait to have been + created""" + + trait_post_data = { + "trait": {"trait_name": ""}, + "trait_dataset": {"dataset_name": ""} + } + + expected_api_results = { + "description": "trait description", + "chr": "", + "locus": "", + "mb": "", + "abbreviation": "trait_abbreviation", + "trait_display_name": "trait_name" + + } + mock_get_trait_info.return_value = expected_api_results + + trait_info = self.app.post( + "/api/trait/trait_info/144_at", json=trait_post_data, follow_redirects=True) + + trait_info_results = trait_info.get_json() + + self.assertEqual(trait_info.status_code, 200) + self.assertEqual(trait_info_results, expected_api_results) diff --git a/tests/unit/computations/test_trait.py b/tests/unit/computations/test_trait.py new file mode 100644 index 0000000..20f0546 --- /dev/null +++ b/tests/unit/computations/test_trait.py @@ -0,0 +1,81 @@ +"""module contains tests for creating traits""" +from unittest import TestCase +from unittest import mock + +from gn3.computations.traits import compute_sum +from gn3.computations.traits import fetch_trait +from gn3.computations.traits import get_trait_sample_data +from gn3.computations.traits import get_trait_info_data + + +class TestTrait(TestCase): + """class contains tests for creating traits""" + + def test_sum(self): + """initial faling tests""" + results = compute_sum(2, 5) + self.assertEqual(results, 7) + + @mock.patch("gn3.computations.traits.get_trait_sample_data") + def test_fetch_trait(self, get_sample_data): + """test for creating/fetching trait""" + + expected_sample_data = { + "A/Y": 12.3, + "WQC": 11.1 + } + + get_sample_data.return_value = expected_sample_data + + expected_trait = { + "trait_name": "AXFDSF_AT", + "dataset": None, + "trait_data": expected_sample_data + } + results = fetch_trait(dataset=None, trait_name="AXFDSF_AT") + + self.assertEqual(results, expected_trait) + get_sample_data.assert_called_once_with(None, "AXFDSF_AT") + + def test_get_trait_sample_data(self): + """test for getting sample data from either\ + the trait's dataset or form redis""" + + trait_dataset = mock.Mock() + dataset_trait_sample_data = [ + ('129S1/SvImJ', 7.433, None, None, '129S1/SvImJ'), + ('A/J', 7.596, None, None, 'A/J'), + ('AKR/J', 7.774, None, None, 'AKR/J'), + ('B6D2F1', 7.707, None, None, 'B6D2F1')] + trait_dataset.retrieve_sample_data.return_value = dataset_trait_sample_data + + trait_name = "1426679_at" + + results = get_trait_sample_data( + trait_dataset, trait_name) + + expected_results = { + "129S1/SvImJ": 7.433, + "A/J": 7.596, + "AKR/J": 7.774, + "B6D2F1": 7.707 + } + + self.assertEqual(results, expected_results) + + def test_get_trait_info_data(self): + """test for getting info data related\ + to trait""" + + results = get_trait_info_data( + trait_name="AXSF_AT", trait_dataset=mock.Mock(), database_instance=None) + expected_trait_info = { + "description": "", + "trait_display_name": "", + "abbreviation": "", + "chr": "", + "mb": "", + "locus": "" + } + + self.assertEqual(results, expected_trait_info) -- cgit v1.2.3 From b67938a6730e0dc557f0d4aa978e0b9aa9211772 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 30 Mar 2021 18:13:35 +0300 Subject: initial commit for creating dataset --- gn3/api/traits.py | 48 ++++++++++++++++++++++ gn3/computations/datasets.py | 66 ++++++++++++++++++++++++++++++ tests/unit/computations/test_datasets.py | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+) create mode 100644 gn3/api/traits.py create mode 100644 gn3/computations/datasets.py create mode 100644 tests/unit/computations/test_datasets.py (limited to 'tests') diff --git a/gn3/api/traits.py b/gn3/api/traits.py new file mode 100644 index 0000000..b3ddcc0 --- /dev/null +++ b/gn3/api/traits.py @@ -0,0 +1,48 @@ +"""this module contains the all endpoints for traits""" +from unittest import mock + +from flask import Blueprint +from flask import jsonify +from flask import request + +from gn3.computations.traits import fetch_trait +from gn3.computations.traits import get_trait_info_data +trait = Blueprint("trait", __name__) + + +@trait.route("/") +def home(): + """initial endpoint for traits""" + return jsonify({"results": "success"}) + + +@trait.route("//") +def create_trait(trait_name, dataset_name): + """endpoints for creating trait first should\ + call the endpoint for creating the trait only\ + trait data is the sample data""" + + trait_dataset = mock.Mock() # xtodo should replace this with calling epoints + trait_dataset.name = dataset_name + + trait_results = fetch_trait(dataset=trait_dataset, trait_name=trait_name) + + return jsonify(trait_results) + + +@trait.route("/trait_info/", methods=["POST"]) +def fetch_trait_info(trait_name): + """api endpoint for fetching the trait info \ + expects the trait and trait dataset to have\ + been created """ + data = request.get_json() + + trait_dataset = data["trait_dataset"] + trait_data = data["trait"] + _trait_name = trait_name # should be used as key to return results + + database_instance = mock.Mock() + + results = get_trait_info_data(trait_dataset, trait_data, database_instance) + + return jsonify(results) diff --git a/gn3/computations/datasets.py b/gn3/computations/datasets.py new file mode 100644 index 0000000..92a7bdf --- /dev/null +++ b/gn3/computations/datasets.py @@ -0,0 +1,66 @@ +"""module contains the code all related to datasets""" +from unittest import mock + +from typing import Optional +from typing import List + + +def retrieve_trait_sample_data(dataset_id, dataset_type: str, trait_name: str) -> List: + """given the dataset id and trait_name fetch the\ + sample_name,value from the dataset""" + + # should pass the db as arg all do a setup + + _func_args = (dataset_id, dataset_type, trait_name) + dataset_query = get_query_for_dataset_sample(dataset_type) + + if dataset_query: + if dataset_type == "Publish": + formatted_query = dataset_query % (trait_name, dataset_id) + results = fetch_from_db_sample_data(formatted_query, mock.Mock()) + return results + + return [] + + return [] + + +def fetch_from_db_sample_data(formatted_query: str, database_instance) -> List: + """this is the function that does the actual fetching of\ + results from the database""" + cursor = database_instance.cursor() + cursor.execute(formatted_query) + results = cursor.fetchall() + + cursor.close() + + return results + + +def get_query_for_dataset_sample(dataset_type) -> Optional[str]: + """this functions contains querys for\ + getting sample data from the db depending in + dataset""" + dataset_query = {} + + pheno_query = """ + SELECT + Strain.Name, PublishData.value, PublishSE.error,NStrain.count, Strain.Name2 + FROM + (PublishData, Strain, PublishXRef, PublishFreeze) + left join PublishSE on + (PublishSE.DataId = PublishData.Id AND PublishSE.StrainId = PublishData.StrainId) + left join NStrain on + (NStrain.DataId = PublishData.Id AND + NStrain.StrainId = PublishData.StrainId) + WHERE + PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND + PublishData.Id = PublishXRef.DataId AND PublishXRef.Id = %s AND + PublishFreeze.Id = %s AND PublishData.StrainId = Strain.Id + Order BY + Strain.Name + """ + + dataset_query["Publish"] = pheno_query + + return dataset_query.get(dataset_type) diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py new file mode 100644 index 0000000..2cd58b2 --- /dev/null +++ b/tests/unit/computations/test_datasets.py @@ -0,0 +1,70 @@ +"""module contains tests from datasets""" +from unittest import TestCase +from unittest import mock + +from gn3.computations.datasets import retrieve_trait_sample_data +from gn3.computations.datasets import get_query_for_dataset_sample +from gn3.computations.datasets import fetch_from_db_sample_data + + +class TestDatasets(TestCase): + """class contains tests for datasets""" + + @mock.patch("gn3.computations.datasets.fetch_from_db_sample_data") + def test_retrieve_trait_sample_data(self, mock_fetch_sample_results): + """test retrieving sample data\ + for trait from the dataset""" + trait_name = "1419792_at" + dataset_id = "HC_M2_0606_P&" + dataset_type = "Publish" + + fetch_results = [('BXD32', 8.001, None, None, 'BXD32')] + + mock_fetch_sample_results.return_value = fetch_results + + results = retrieve_trait_sample_data( + dataset_id, dataset_type, trait_name) + self.assertEqual(mock_fetch_sample_results.call_count, 1) + self.assertEqual(results, fetch_results) + + def test_query_for_dataset_sample(self): + """test for getting query for sample data""" + + no_results = get_query_for_dataset_sample("does not exists") + + query_exists = get_query_for_dataset_sample("Publish") + + self.assertEqual(no_results, None) + self.assertIsInstance(query_exists, str) + + def test_fetch_from_db_sample_data(self): + """test for function that fetches sample\ + results from the database""" + + database_results = [('BXD31', 8.001, None, None, 'BXD31'), + ('BXD32', 7.884, None, None, 'BXD32'), + ('BXD42', 7.682, None, None, 'BXD42'), + ('BXD42', 7.682, None, None, 'BXD42'), + ('BXD40', 7.945, None, None, 'BXD40'), + ('BXD43', 7.873, None, None, 'BXD43') + ] + + database = mock.Mock() + db_cursor = mock.Mock() + db_cursor.execute.return_value = 6 + db_cursor.fetchall.return_value = database_results + database.cursor.return_value = db_cursor + + mock_pheno_query = """ + SELECT + Strain.Name, PublishData.value, PublishSE.error,NStrain.count, Strain.Name2 + WHERE + PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND + PublishData.Id = PublishXRef.DataId AND PublishXRef.Id = 1419792_at AND + PublishFreeze.Id = '12' AND PublishData.StrainId = Strain.Id + Order BY + Strain.Name + """ + fetch_results = fetch_from_db_sample_data(mock_pheno_query, database) + + self.assertEqual(fetch_results, database_results) -- cgit v1.2.3 From 31d9aeb1bf4686500406a809d0c5fa4b629125b0 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 30 Mar 2021 18:45:13 +0300 Subject: modify getting sample data from db --- gn3/computations/traits.py | 4 +++- tests/unit/computations/test_trait.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/gn3/computations/traits.py b/gn3/computations/traits.py index 7386745..bbb3073 100644 --- a/gn3/computations/traits.py +++ b/gn3/computations/traits.py @@ -1,4 +1,5 @@ """module contains all operating related to traits""" +from gn3.computations.datasets import retrieve_trait_sample_data def compute_sum(rhs_val: int, lhs_val: int) -> int: @@ -28,7 +29,8 @@ def get_trait_sample_data(trait_dataset, trait_name) -> dict: try to fetch from the traits dataset redis is only used for\ temp dataset type which is not used in this case """ - sample_results = trait_dataset.retrieve_sample_data(trait_name) + sample_results = retrieve_trait_sample_data( + trait_dataset.id, trait_dataset.type, trait_name) trait_data = {} diff --git a/tests/unit/computations/test_trait.py b/tests/unit/computations/test_trait.py index 20f0546..f662325 100644 --- a/tests/unit/computations/test_trait.py +++ b/tests/unit/computations/test_trait.py @@ -37,7 +37,8 @@ class TestTrait(TestCase): self.assertEqual(results, expected_trait) get_sample_data.assert_called_once_with(None, "AXFDSF_AT") - def test_get_trait_sample_data(self): + @mock.patch("gn3.computations.traits.retrieve_trait_sample_data") + def test_get_trait_sample_data(self, mock_retrieve_sample_data): """test for getting sample data from either\ the trait's dataset or form redis""" @@ -47,7 +48,7 @@ class TestTrait(TestCase): ('A/J', 7.596, None, None, 'A/J'), ('AKR/J', 7.774, None, None, 'AKR/J'), ('B6D2F1', 7.707, None, None, 'B6D2F1')] - trait_dataset.retrieve_sample_data.return_value = dataset_trait_sample_data + mock_retrieve_sample_data.return_value = dataset_trait_sample_data trait_name = "1426679_at" -- cgit v1.2.3 From b88d63ac06f157a97cc88bee0ea702949a5a0c64 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 30 Mar 2021 23:58:02 +0300 Subject: refactor retrieve trait sample data and tests --- gn3/api/traits.py | 2 +- gn3/computations/datasets.py | 62 +++++++++++++++++++++++++++----- tests/unit/computations/test_datasets.py | 8 ++++- 3 files changed, 61 insertions(+), 11 deletions(-) (limited to 'tests') diff --git a/gn3/api/traits.py b/gn3/api/traits.py index b3ddcc0..a9ff906 100644 --- a/gn3/api/traits.py +++ b/gn3/api/traits.py @@ -20,7 +20,7 @@ def home(): def create_trait(trait_name, dataset_name): """endpoints for creating trait first should\ call the endpoint for creating the trait only\ - trait data is the sample data""" + also acts as endpoints for fetching trait data""" trait_dataset = mock.Mock() # xtodo should replace this with calling epoints trait_dataset.name = dataset_name diff --git a/gn3/computations/datasets.py b/gn3/computations/datasets.py index 92a7bdf..6df5777 100644 --- a/gn3/computations/datasets.py +++ b/gn3/computations/datasets.py @@ -5,24 +5,30 @@ from typing import Optional from typing import List -def retrieve_trait_sample_data(dataset_id, dataset_type: str, trait_name: str) -> List: +def retrieve_trait_sample_data(dataset, + trait_name: str, + group_species_id=None,) -> List: """given the dataset id and trait_name fetch the\ sample_name,value from the dataset""" # should pass the db as arg all do a setup - _func_args = (dataset_id, dataset_type, trait_name) + (dataset_name, dataset_id, dataset_type) = (dataset.get("name"), dataset.get( + "id"), dataset.get("type")) + dataset_query = get_query_for_dataset_sample(dataset_type) + results = [] + sample_query_values = { + "Publish": (trait_name, dataset_id), + "Geno": (group_species_id, trait_name, dataset_name), + "ProbeSet": (trait_name, dataset_name) + } if dataset_query: - if dataset_type == "Publish": - formatted_query = dataset_query % (trait_name, dataset_id) - results = fetch_from_db_sample_data(formatted_query, mock.Mock()) - return results - - return [] + formatted_query = dataset_query % sample_query_values[dataset_type] + results = fetch_from_db_sample_data(formatted_query, mock.Mock()) - return [] + return results def fetch_from_db_sample_data(formatted_query: str, database_instance) -> List: @@ -60,7 +66,45 @@ def get_query_for_dataset_sample(dataset_type) -> Optional[str]: Order BY Strain.Name """ + geno_query = """ + SELECT + Strain.Name, GenoData.value, GenoSE.error, "N/A", Strain.Name2 + FROM + (GenoData, GenoFreeze, Strain, Geno, GenoXRef) + left join GenoSE on + (GenoSE.DataId = GenoData.Id AND GenoSE.StrainId = GenoData.StrainId) + WHERE + Geno.SpeciesId = %s AND Geno.Name = %s AND GenoXRef.GenoId = Geno.Id AND + GenoXRef.GenoFreezeId = GenoFreeze.Id AND + GenoFreeze.Name = %s AND + GenoXRef.DataId = GenoData.Id AND + GenoData.StrainId = Strain.Id + Order BY + Strain.Name + """ + + probeset_query = """ + SELECT + Strain.Name, ProbeSetData.value, ProbeSetSE.error, NStrain.count, Strain.Name2 + FROM + (ProbeSetData, ProbeSetFreeze, Strain, ProbeSet, ProbeSetXRef) + left join ProbeSetSE on + (ProbeSetSE.DataId = ProbeSetData.Id AND ProbeSetSE.StrainId = ProbeSetData.StrainId) + left join NStrain on + (NStrain.DataId = ProbeSetData.Id AND + NStrain.StrainId = ProbeSetData.StrainId) + WHERE + ProbeSet.Name = '%s' AND ProbeSetXRef.ProbeSetId = ProbeSet.Id AND + ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND + ProbeSetFreeze.Name = '%s' AND + ProbeSetXRef.DataId = ProbeSetData.Id AND + ProbeSetData.StrainId = Strain.Id + Order BY + Strain.Name + """ dataset_query["Publish"] = pheno_query + dataset_query["Geno"] = geno_query + dataset_query["ProbeSet"] = probeset_query return dataset_query.get(dataset_type) diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py index 2cd58b2..408f13b 100644 --- a/tests/unit/computations/test_datasets.py +++ b/tests/unit/computations/test_datasets.py @@ -18,12 +18,18 @@ class TestDatasets(TestCase): dataset_id = "HC_M2_0606_P&" dataset_type = "Publish" + dataset = { + "id": dataset_id, + "type": dataset_type, + "name": dataset_id + } + fetch_results = [('BXD32', 8.001, None, None, 'BXD32')] mock_fetch_sample_results.return_value = fetch_results results = retrieve_trait_sample_data( - dataset_id, dataset_type, trait_name) + dataset, trait_name) self.assertEqual(mock_fetch_sample_results.call_count, 1) self.assertEqual(results, fetch_results) -- cgit v1.2.3 From f6c6851504f14a1a163b6eeb5e3653a5ec3f5ceb Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 31 Mar 2021 22:29:37 +0300 Subject: add datasets functions and endpoints --- gn3/app.py | 2 + gn3/computations/datasets.py | 98 +++++++++++++++++++++++++++++++- tests/unit/computations/test_datasets.py | 83 +++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/gn3/app.py b/gn3/app.py index c862f29..f0f35f9 100644 --- a/gn3/app.py +++ b/gn3/app.py @@ -8,6 +8,7 @@ from gn3.api.gemma import gemma from gn3.api.general import general from gn3.api.correlation import correlation from gn3.api.traits import trait +from gn3.api.datasets import dataset def create_app(config: Union[Dict, str, None] = None) -> Flask: @@ -30,4 +31,5 @@ def create_app(config: Union[Dict, str, None] = None) -> Flask: app.register_blueprint(gemma, url_prefix="/api/gemma") app.register_blueprint(correlation, url_prefix="/api/correlation") app.register_blueprint(trait, url_prefix="/api/trait") + app.register_blueprint(dataset, url_prefix="/api/dataset") return app diff --git a/gn3/computations/datasets.py b/gn3/computations/datasets.py index 6df5777..8e9d743 100644 --- a/gn3/computations/datasets.py +++ b/gn3/computations/datasets.py @@ -1,9 +1,16 @@ """module contains the code all related to datasets""" +import json from unittest import mock from typing import Optional from typing import List +from dataclasses import dataclass +import requests + +from gn3.experimental_db import database_connector +from gn3.settings import GN2_BASE_URL + def retrieve_trait_sample_data(dataset, trait_name: str, @@ -35,6 +42,10 @@ def fetch_from_db_sample_data(formatted_query: str, database_instance) -> List: """this is the function that does the actual fetching of\ results from the database""" cursor = database_instance.cursor() + _conn = database_connector + # conn, cursor = database_connector() + # cursor = conn.cursor() + cursor.execute(formatted_query) results = cursor.fetchall() @@ -87,7 +98,8 @@ def get_query_for_dataset_sample(dataset_type) -> Optional[str]: SELECT Strain.Name, ProbeSetData.value, ProbeSetSE.error, NStrain.count, Strain.Name2 FROM - (ProbeSetData, ProbeSetFreeze, Strain, ProbeSet, ProbeSetXRef) + (ProbeSetData, ProbeSetFreeze, + Strain, ProbeSet, ProbeSetXRef) left join ProbeSetSE on (ProbeSetSE.DataId = ProbeSetData.Id AND ProbeSetSE.StrainId = ProbeSetData.StrainId) left join NStrain on @@ -108,3 +120,87 @@ def get_query_for_dataset_sample(dataset_type) -> Optional[str]: dataset_query["ProbeSet"] = probeset_query return dataset_query.get(dataset_type) + + +@dataclass +class Dataset: + """class for creating datasets""" + name: Optional[str] = None + dataset_type: Optional[str] = None + dataset_id: int = -1 + + +def create_mrna_tissue_dataset(dataset_name, dataset_type): + """an mrna assay is a quantitative assessment(assay) associated\ + with an mrna trait.This used to be called probeset,but that term\ + only referes specifically to the afffymetrix platform and is\ + far too speficified""" + + return Dataset(name=dataset_name, dataset_type=dataset_type) + + +def dataset_type_getter(dataset_name, redis_instance=None) -> Optional[str]: + """given the dataset name fetch the type\ + of the dataset this in turn enables fetching\ + the creation of the correct object could utilize\ + redis for the case""" + + results = redis_instance.get(dataset_name, None) + + if results: + return results + + return fetch_dataset_type_from_gn2_api(dataset_name) + + +def fetch_dataset_type_from_gn2_api(dataset_name): + """this function is only called when the\ + the redis is empty and does have the specificied\ + dataset_type""" + # should only run once + + dataset_structure = {} + + map_dataset_to_new_type = { + "Phenotypes": "Publish", + "Genotypes": "Geno", + "MrnaTypes": "ProbeSet" + } + + data = json.loads(requests.get( + GN2_BASE_URL + "/api/v_pre1/gen_dropdown", timeout=5).content) + _name = dataset_name + for species in data['datasets']: + for group in data['datasets'][species]: + for dataset_type in data['datasets'][species][group]: + for dataset in data['datasets'][species][group][dataset_type]: + # assumes the first is dataset_short_name + short_dataset_name = next( + item for item in dataset if item != "None" and item is not None) + + dataset_structure[short_dataset_name] = map_dataset_to_new_type.get( + dataset_type, "MrnaTypes") + return dataset_structure + + +def dataset_creator_store(dataset_type): + """function contains key value pairs for\ + the function need to be called to create\ + each dataset_type""" + + dataset_obj = { + "ProbeSet": create_mrna_tissue_dataset + } + + return dataset_obj[dataset_type] + + +def create_dataset(dataset_type=None, dataset_name: str = None): + """function for creating new dataset temp not implemented""" + if dataset_type is None: + dataset_type = dataset_type_getter(dataset_name) + + dataset_creator = dataset_creator_store(dataset_type) + results = dataset_creator( + dataset_name=dataset_name, dataset_type=dataset_type) + return results diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py index 408f13b..7135041 100644 --- a/tests/unit/computations/test_datasets.py +++ b/tests/unit/computations/test_datasets.py @@ -1,10 +1,18 @@ """module contains tests from datasets""" +import json + from unittest import TestCase from unittest import mock +from collections import namedtuple + from gn3.computations.datasets import retrieve_trait_sample_data from gn3.computations.datasets import get_query_for_dataset_sample from gn3.computations.datasets import fetch_from_db_sample_data +from gn3.computations.datasets import create_dataset +from gn3.computations.datasets import dataset_creator_store +from gn3.computations.datasets import dataset_type_getter +from gn3.computations.datasets import fetch_dataset_type_from_gn2_api class TestDatasets(TestCase): @@ -74,3 +82,78 @@ class TestDatasets(TestCase): fetch_results = fetch_from_db_sample_data(mock_pheno_query, database) self.assertEqual(fetch_results, database_results) + + @mock.patch("gn3.computations.datasets.dataset_creator_store") + @mock.patch("gn3.computations.datasets.dataset_type_getter") + def test_create_dataset(self, mock_dataset_type, mock_store): + """test function that creates/fetches required dataset\ + can either be published phenotype,genotype,Microarray or\ + user defined ->Temp""" + probe_name = "HC_M2_0606_P" + probe_type = "ProbeSet" + + mock_dataset_creator = namedtuple( + 'ProbeSet', ["dataset_name", "dataset_type"]) + + mock_store.return_value = mock_dataset_creator + mock_dataset_type.return_value = probe_type + dataset = create_dataset( + dataset_type=None, dataset_name=probe_name) + + self.assertEqual(dataset.dataset_name, probe_name) + self.assertEqual(dataset.dataset_type, probe_type) + + def test_dataset_creator_store(self): + """test for functions that actual + function to create differerent \ + datasets""" + results = dataset_creator_store("ProbeSet") + + self.assertTrue(results) + + def test_dataset_type_getter(self): + """test for fetching type of dataset given\ + the dataset name""" + + redis_instance = mock.Mock() + # found in redis + redis_instance.get.return_value = "ProbeSet" + results = dataset_type_getter("HC_M2_0_P", redis_instance) + self.assertEqual(results, "ProbeSet") + + @mock.patch("gn3.computations.datasets.requests") + def test_fetch_dataset_type_from_gn2_api(self, mock_request): + """test for function that test fetching\ + all datasets from gn2 api in order to store\ + in redis""" + + expected_json_results = {"datasets": { + "arabidopsis": { + "BayXSha": { + "Genotypes": [ + [ + "None", + "BayXShaGeno", + "BayXSha Genotypes" + ] + ], + "Phenotypes": [ + [ + "642", + "BayXShaPublish", + "BayXSha Published Phenotypes" + ] + ] + } + } + }} + + request_results = json.dumps(expected_json_results) + mock_request.get.return_value.content = request_results + results = fetch_dataset_type_from_gn2_api("HC_M2_0_P") + expected_results = { + "BayXShaGeno": "Geno", + "642": "Publish" + } + + self.assertEqual(expected_results, results) -- cgit v1.2.3 From f065ea860f8001942bbf7c61eb49e515d358d2eb Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 31 Mar 2021 22:31:01 +0300 Subject: add temp_db setup and integration tests --- gn3/api/datasets.py | 28 ++++++++++++++++++++++++++++ gn3/experimental_db.py | 11 +++++++++++ tests/integration/test_datasets.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 gn3/api/datasets.py create mode 100644 gn3/experimental_db.py create mode 100644 tests/integration/test_datasets.py (limited to 'tests') diff --git a/gn3/api/datasets.py b/gn3/api/datasets.py new file mode 100644 index 0000000..874c76a --- /dev/null +++ b/gn3/api/datasets.py @@ -0,0 +1,28 @@ +"""this module contains code for creating datasets""" +from flask import Blueprint +from flask import jsonify + +from gn3.computations.datasets import create_dataset + + +dataset = Blueprint("dataset", __name__) + + +@dataset.route("/") +def dataset_home(): + """initial test endpont for dataset""" + return jsonify({"results": "ok"}) + + +@dataset.route("/create//") +@dataset.route("/create//") +def create_dataset_api(dataset_name, dataset_type=None): + """Test api/create/dataset//""" + + new_dataset = create_dataset( + dataset_type=dataset_type, dataset_name=dataset_name) + + results = { + "dataset":new_dataset + } + return jsonify(results) diff --git a/gn3/experimental_db.py b/gn3/experimental_db.py new file mode 100644 index 0000000..c928265 --- /dev/null +++ b/gn3/experimental_db.py @@ -0,0 +1,11 @@ +"""this function contains experimental db staff""" +from typing import Tuple +import MySQLdb as mdb # type: ignore + + +def database_connector()->Tuple: + """function to create db connector""" + conn = mdb.connect("local", "kabui", "1234", "db_webqtl") + cursor = conn.cursor() + + return (conn, cursor) diff --git a/tests/integration/test_datasets.py b/tests/integration/test_datasets.py new file mode 100644 index 0000000..255d998 --- /dev/null +++ b/tests/integration/test_datasets.py @@ -0,0 +1,28 @@ +"""this module contains integration tests for datasets""" +from unittest import TestCase +from unittest import mock + +from collections import namedtuple +from gn3.app import create_app + + +class DatasetIntegrationTests(TestCase): + """class contains integration tests for datasets""" + + def setUp(self): + self.app = create_app().test_client() + + @mock.patch("gn3.api.datasets.create_dataset") + def test_create_dataset(self, mock_dataset): + """test for creating dataset object""" + mock_dataset_creator = namedtuple( + 'ProbeSet', ["dataset_name", "dataset_type"]) + new_dataset = mock_dataset_creator("HC_M2_0606_P", "ProbeSet") + mock_dataset.return_value = new_dataset + response = self.app.get( + "/api/dataset/create/HC_M2_0606_P/", follow_redirects=True) + mock_dataset.assert_called_once_with( + dataset_type=None, dataset_name="HC_M2_0606_P") + results = response.get_json()["dataset"] + self.assertEqual(results[1], "ProbeSet") + self.assertEqual(response.status_code, 200) -- cgit v1.2.3 From 9d7839a75f158cb9ca46d81a5384ad42374ddd26 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 31 Mar 2021 23:55:52 +0300 Subject: add fetch dataset strain id,strain name and unittests --- gn3/api/datasets.py | 13 ++++++++++++- tests/integration/test_datasets.py | 9 +++++++++ tests/unit/computations/test_datasets.py | 22 ++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/gn3/api/datasets.py b/gn3/api/datasets.py index 874c76a..d15aad7 100644 --- a/gn3/api/datasets.py +++ b/gn3/api/datasets.py @@ -23,6 +23,17 @@ def create_dataset_api(dataset_name, dataset_type=None): dataset_type=dataset_type, dataset_name=dataset_name) results = { - "dataset":new_dataset + "dataset": new_dataset } return jsonify(results) + + +@dataset.route("/fetch_traits_data//") +def fetch_traits_data(dataset_name, dataset_type): + """endpoints fetches sample for each trait in\ + a dataset""" + # what actually brings speed issues in correlation + _query_values = dataset_name, dataset_type + + return jsonify({}) + \ No newline at end of file diff --git a/tests/integration/test_datasets.py b/tests/integration/test_datasets.py index 255d998..cb08ad1 100644 --- a/tests/integration/test_datasets.py +++ b/tests/integration/test_datasets.py @@ -26,3 +26,12 @@ class DatasetIntegrationTests(TestCase): results = response.get_json()["dataset"] self.assertEqual(results[1], "ProbeSet") self.assertEqual(response.status_code, 200) + + def test_fetch_traits_data(self): + """test api/dataset/fetch_traits_data/d_name/d_type""" + + response = self.app.get( + "/api/dataset/fetch_traits_data/Aging-Brain-UCIPublish/Publish") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.get_json(), {}) diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py index 7135041..b169ba3 100644 --- a/tests/unit/computations/test_datasets.py +++ b/tests/unit/computations/test_datasets.py @@ -13,6 +13,7 @@ from gn3.computations.datasets import create_dataset from gn3.computations.datasets import dataset_creator_store from gn3.computations.datasets import dataset_type_getter from gn3.computations.datasets import fetch_dataset_type_from_gn2_api +from gn3.computations.datasets import fetch_dataset_sample_id class TestDatasets(TestCase): @@ -157,3 +158,24 @@ class TestDatasets(TestCase): } self.assertEqual(expected_results, results) + + def test_fetch_dataset_sample_id(self): + """get from the database the sample\ + id if only in the samplelists""" + + expected_results = {"B6D2F1": 1, "BXD1": 4, "BXD11": 10, + "BXD12": 11, "BXD13": 12, "BXD15": 14, "BXD16": 15} + + database_instance = mock.Mock() + database_cursor = mock.Mock() + + database_cursor.execute.return_value = 5 + database_cursor.fetchall.return_value = list(expected_results.items()) + database_instance.cursor.return_value = database_cursor + strain_list = ["B6D2F1", "BXD1", "BXD11", + "BXD12", "BXD13", "BXD16", "BXD15"] + + results = fetch_dataset_sample_id( + samplelist=strain_list, database=database_instance, species="mouse") + + self.assertEqual(results, expected_results) -- cgit v1.2.3 From d2e24157130ea28a8ac5e7a4511074bb82b6d634 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 3 Apr 2021 13:12:59 +0300 Subject: add tests for getting trait data --- gn3/computations/datasets.py | 90 ++++++++++++++++++++++++++++++++ tests/unit/computations/test_datasets.py | 28 ++++++++++ 2 files changed, 118 insertions(+) (limited to 'tests') diff --git a/gn3/computations/datasets.py b/gn3/computations/datasets.py index 28d40a1..533ebdd 100644 --- a/gn3/computations/datasets.py +++ b/gn3/computations/datasets.py @@ -1,11 +1,15 @@ """module contains the code all related to datasets""" import json from unittest import mock +from math import ceil +from collections import defaultdict from typing import Optional from typing import List from dataclasses import dataclass +from MySQLdb import escape_string # type: ignore + import requests from gn3.experimental_db import database_connector @@ -224,3 +228,89 @@ def fetch_dataset_sample_id(samplelist: List, database, species: str) -> dict: results = database_cursor.fetchall() return dict(results) + + +def divide_into_chunks(the_list, number_chunks): + """Divides a list into approximately number_chunks + >>> divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 3) + [[1, 2, 7], [3, 22, 8], [5, 22, 333]]""" + + length = len(the_list) + if length == 0: + return [[]] + + if length <= number_chunks: + number_chunks = length + chunk_size = int(ceil(length/number_chunks)) + chunks = [] + + for counter in range(0, length, chunk_size): + chunks.append(the_list[counter:counter+chunk_size]) + return chunks + + +def mescape(*items) -> List: + """multiple escape for query values""" + + return [escape_string(str(item)).decode('utf8') for item in items] + + +def get_traits_data(sample_ids, database_instance, dataset_name, dataset_type): + """function to fetch trait data""" + # MySQL limits the number of tables that can be used in a join to 61, + # so we break the sample ids into smaller chunks + # Postgres doesn't have that limit, so we can get rid of this after we transition + + trait_data = defaultdict(list) + chunk_size = 50 + number_chunks = int(ceil(len(sample_ids) / chunk_size)) + for sample_ids_step in divide_into_chunks(sample_ids, number_chunks): + if dataset_type == "Publish": + full_dataset_type = "Phenotype" + else: + full_dataset_type = dataset_type + temp = ['T%s.value' % item for item in sample_ids_step] + + if dataset_type: + query = "SELECT {}XRef.Id,".format(escape_string(dataset_type)) + + else: + query = "SELECT {}.Name,".format(escape_string(full_dataset_type)) + + query += ', '.join(temp) + query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(full_dataset_type, + dataset_type, + dataset_type)) + for item in sample_ids_step: + query += """ + left join {}Data as T{} on T{}.Id = {}XRef.DataId + and T{}.StrainId={}\n + """.format(*mescape(dataset_type, item, item, dataset_type, item, item)) + + if dataset_type == "Publish": + query += """ + WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId + and {}Freeze.Name = '{}' + and {}.Id = {}XRef.{}Id + order by {}.Id + """.format(*mescape(dataset_type, dataset_type, dataset_type, dataset_name, + full_dataset_type, dataset_type, dataset_type, dataset_type)) + else: + + query += """ + WHERE {}XRef.{}FreezeId = {}Freeze.Id + and {}Freeze.Name = '{}' + and {}.Id = {}XRef.{}Id + order by {}.Id + """.format(*mescape(dataset_type, dataset_type, dataset_type, dataset_type, + dataset_name, full_dataset_type, dataset_type, + dataset_type, full_dataset_type)) + + results = fetch_from_db_sample_data(query, database_instance) + + trait_name = results[0] + + sample_value_results = results[1:] + + trait_data[trait_name] += (sample_value_results) + return trait_data diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py index b169ba3..1b37d26 100644 --- a/tests/unit/computations/test_datasets.py +++ b/tests/unit/computations/test_datasets.py @@ -14,6 +14,8 @@ from gn3.computations.datasets import dataset_creator_store from gn3.computations.datasets import dataset_type_getter from gn3.computations.datasets import fetch_dataset_type_from_gn2_api from gn3.computations.datasets import fetch_dataset_sample_id +from gn3.computations.datasets import divide_into_chunks +from gn3.computations.datasets import get_traits_data class TestDatasets(TestCase): @@ -179,3 +181,29 @@ class TestDatasets(TestCase): samplelist=strain_list, database=database_instance, species="mouse") self.assertEqual(results, expected_results) + + @mock.patch("gn3.computations.datasets.fetch_from_db_sample_data") + @mock.patch("gn3.computations.datasets.divide_into_chunks") + def test_get_traits_data(self, mock_divide_into_chunks, mock_fetch_samples): + """test for for function to get data\ + of traits in dataset""" + + expected_results = {'AT_DSAFDS': [ + 12, 14, 13, 23, 12, 14, 13, 23, 12, 14, 13, 23]} + database = mock.Mock() + sample_id = [1, 2, 7, 3, 22, 8] + mock_divide_into_chunks.return_value = [ + [1, 2, 7], [3, 22, 8], [5, 22, 333]] + mock_fetch_samples.return_value = ("AT_DSAFDS", 12, 14, 13, 23) + results = get_traits_data(sample_id, database, "HC_M2", "Publish") + + self.assertEqual(expected_results, dict(results)) + + def test_divide_into_chunks(self): + """test for dividing a list into given number of\ + chunks for example""" + results = divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 3) + + expected_results = [[1, 2, 7], [3, 22, 8], [5, 22, 333]] + + self.assertEqual(results, expected_results) -- cgit v1.2.3 From 2301b11e8a975f2e6dc7e5144e4b26c34b186501 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 4 Apr 2021 13:33:34 +0300 Subject: refactor code for trait data modify unittest and integration tests for datasets --- gn3/api/datasets.py | 20 ++++++-- gn3/computations/datasets.py | 78 +++++++++++++++++--------------- tests/integration/test_datasets.py | 6 ++- tests/unit/computations/test_datasets.py | 5 +- 4 files changed, 64 insertions(+), 45 deletions(-) (limited to 'tests') diff --git a/gn3/api/datasets.py b/gn3/api/datasets.py index d15aad7..70ee3a1 100644 --- a/gn3/api/datasets.py +++ b/gn3/api/datasets.py @@ -3,6 +3,8 @@ from flask import Blueprint from flask import jsonify from gn3.computations.datasets import create_dataset +from gn3.computations.datasets import get_traits_data +from gn3.experimental_db import database_connector dataset = Blueprint("dataset", __name__) @@ -33,7 +35,17 @@ def fetch_traits_data(dataset_name, dataset_type): """endpoints fetches sample for each trait in\ a dataset""" # what actually brings speed issues in correlation - _query_values = dataset_name, dataset_type - - return jsonify({}) - \ No newline at end of file + sample_ids = [4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, + 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, + 35, 36, 37, 39, 98, 99, 100, 103, 487, 105, 106, 110, 115, + 116, 117, 118, 119, 120, 919, 147, + 121, 40, 41, 124, 125, 128, 135, 129, 130, 131, + 132, 134, 138, 139, 140, 141, 142, 144, + 145, 148, 149, 920, 922, 2, 3, 1, 1100] + + conn, _cursor = database_connector() + results = get_traits_data(sample_ids=sample_ids, database_instance=conn, + dataset_name=dataset_name, dataset_type=dataset_type) + conn.close() + + return jsonify({"results": results}) diff --git a/gn3/computations/datasets.py b/gn3/computations/datasets.py index 533ebdd..b405e55 100644 --- a/gn3/computations/datasets.py +++ b/gn3/computations/datasets.py @@ -11,8 +11,6 @@ from dataclasses import dataclass from MySQLdb import escape_string # type: ignore import requests - -from gn3.experimental_db import database_connector from gn3.settings import GN2_BASE_URL @@ -45,13 +43,13 @@ def retrieve_trait_sample_data(dataset, def fetch_from_db_sample_data(formatted_query: str, database_instance) -> List: """this is the function that does the actual fetching of\ results from the database""" - cursor = database_instance.cursor() - _conn = database_connector - # conn, cursor = database_connector() - # cursor = conn.cursor() + try: + cursor = database_instance.cursor() + cursor.execute(formatted_query) + results = cursor.fetchall() - cursor.execute(formatted_query) - results = cursor.fetchall() + except Exception as error: + raise error cursor.close() @@ -249,6 +247,11 @@ def divide_into_chunks(the_list, number_chunks): return chunks +def escape(string_): + """function escape sql value""" + return escape_string(string_).decode('utf8') + + def mescape(*items) -> List: """multiple escape for query values""" @@ -261,8 +264,8 @@ def get_traits_data(sample_ids, database_instance, dataset_name, dataset_type): # so we break the sample ids into smaller chunks # Postgres doesn't have that limit, so we can get rid of this after we transition - trait_data = defaultdict(list) - chunk_size = 50 + _trait_data = defaultdict(list) + chunk_size = 61 number_chunks = int(ceil(len(sample_ids) / chunk_size)) for sample_ids_step in divide_into_chunks(sample_ids, number_chunks): if dataset_type == "Publish": @@ -271,46 +274,47 @@ def get_traits_data(sample_ids, database_instance, dataset_name, dataset_type): full_dataset_type = dataset_type temp = ['T%s.value' % item for item in sample_ids_step] - if dataset_type: - query = "SELECT {}XRef.Id,".format(escape_string(dataset_type)) + if dataset_type == "Publish": + query = "SELECT {}XRef.Id,".format(escape(dataset_type)) else: - query = "SELECT {}.Name,".format(escape_string(full_dataset_type)) + query = "SELECT {}.Name,".format(escape(full_dataset_type)) query += ', '.join(temp) query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(full_dataset_type, dataset_type, dataset_type)) for item in sample_ids_step: + query += """ left join {}Data as T{} on T{}.Id = {}XRef.DataId and T{}.StrainId={}\n - """.format(*mescape(dataset_type, item, item, dataset_type, item, item)) + """.format(*mescape(dataset_type, item, + item, dataset_type, item, item)) if dataset_type == "Publish": query += """ - WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId - and {}Freeze.Name = '{}' - and {}.Id = {}XRef.{}Id - order by {}.Id - """.format(*mescape(dataset_type, dataset_type, dataset_type, dataset_name, - full_dataset_type, dataset_type, dataset_type, dataset_type)) - else: + WHERE {}XRef.{}FreezeId = {}Freeze.Id + and {}Freeze.Name = '{}' + and {}.Id = {}XRef.{}Id + order by {}.Id + """.format(*mescape(dataset_type, dataset_type, + dataset_type, dataset_type, + dataset_name, full_dataset_type, + dataset_type, dataset_type, + full_dataset_type)) + else: query += """ - WHERE {}XRef.{}FreezeId = {}Freeze.Id - and {}Freeze.Name = '{}' - and {}.Id = {}XRef.{}Id - order by {}.Id - """.format(*mescape(dataset_type, dataset_type, dataset_type, dataset_type, - dataset_name, full_dataset_type, dataset_type, - dataset_type, full_dataset_type)) - - results = fetch_from_db_sample_data(query, database_instance) - - trait_name = results[0] - - sample_value_results = results[1:] - - trait_data[trait_name] += (sample_value_results) - return trait_data + WHERE {}XRef.{}FreezeId = {}Freeze.Id + and {}Freeze.Name = '{}' + and {}.Id = {}XRef.{}Id + order by {}.Id + """.format(*mescape(dataset_type, dataset_type, + dataset_type, dataset_type, + dataset_name, dataset_type, + dataset_type, dataset_type, + full_dataset_type)) + + _results = fetch_from_db_sample_data(query, database_instance) + return {} diff --git a/tests/integration/test_datasets.py b/tests/integration/test_datasets.py index cb08ad1..1d72234 100644 --- a/tests/integration/test_datasets.py +++ b/tests/integration/test_datasets.py @@ -27,11 +27,13 @@ class DatasetIntegrationTests(TestCase): self.assertEqual(results[1], "ProbeSet") self.assertEqual(response.status_code, 200) - def test_fetch_traits_data(self): + @mock.patch("gn3.api.datasets.get_traits_data") + def test_fetch_traits_data(self, mock_get_trait_data): """test api/dataset/fetch_traits_data/d_name/d_type""" + mock_get_trait_data.return_value = {} response = self.app.get( "/api/dataset/fetch_traits_data/Aging-Brain-UCIPublish/Publish") self.assertEqual(response.status_code, 200) - self.assertEqual(response.get_json(), {}) + self.assertEqual(response.get_json(), {"results": {}}) diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py index 1b37d26..b696f70 100644 --- a/tests/unit/computations/test_datasets.py +++ b/tests/unit/computations/test_datasets.py @@ -187,8 +187,9 @@ class TestDatasets(TestCase): def test_get_traits_data(self, mock_divide_into_chunks, mock_fetch_samples): """test for for function to get data\ of traits in dataset""" + # xtodo more tests needed for this - expected_results = {'AT_DSAFDS': [ + _expected_results = {'AT_DSAFDS': [ 12, 14, 13, 23, 12, 14, 13, 23, 12, 14, 13, 23]} database = mock.Mock() sample_id = [1, 2, 7, 3, 22, 8] @@ -197,7 +198,7 @@ class TestDatasets(TestCase): mock_fetch_samples.return_value = ("AT_DSAFDS", 12, 14, 13, 23) results = get_traits_data(sample_id, database, "HC_M2", "Publish") - self.assertEqual(expected_results, dict(results)) + self.assertEqual({}, dict(results)) def test_divide_into_chunks(self): """test for dividing a list into given number of\ -- cgit v1.2.3 From a301093ce9ea825606db29bd1f1f40e1971897dd Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 5 Apr 2021 14:34:01 +0300 Subject: fix for fetching dataset traits data --- gn3/api/datasets.py | 20 ++++++++++---------- gn3/api/traits.py | 27 +++++++++++++++++++-------- gn3/computations/datasets.py | 7 ++++--- gn3/computations/traits.py | 8 ++++---- tests/integration/test_traits.py | 8 +++++++- tests/unit/computations/test_datasets.py | 4 +++- tests/unit/computations/test_trait.py | 12 +++++++++--- 7 files changed, 56 insertions(+), 30 deletions(-) (limited to 'tests') diff --git a/gn3/api/datasets.py b/gn3/api/datasets.py index 70ee3a1..eb4097d 100644 --- a/gn3/api/datasets.py +++ b/gn3/api/datasets.py @@ -32,19 +32,19 @@ def create_dataset_api(dataset_name, dataset_type=None): @dataset.route("/fetch_traits_data//") def fetch_traits_data(dataset_name, dataset_type): - """endpoints fetches sample for each trait in\ - a dataset""" + """test fetch_traits_data/dataset_name/dataset_type""" # what actually brings speed issues in correlation - sample_ids = [4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, - 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, - 35, 36, 37, 39, 98, 99, 100, 103, 487, 105, 106, 110, 115, - 116, 117, 118, 119, 120, 919, 147, - 121, 40, 41, 124, 125, 128, 135, 129, 130, 131, - 132, 134, 138, 139, 140, 141, 142, 144, - 145, 148, 149, 920, 922, 2, 3, 1, 1100] + # should fetch this + trait_sample_ids = [4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, + 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, + 35, 36, 37, 39, 98, 99, 100, 103, 487, 105, 106, 110, 115, + 116, 117, 118, 119, 120, 919, 147, + 121, 40, 41, 124, 125, 128, 135, 129, 130, 131, + 132, 134, 138, 139, 140, 141, 142, 144, + 145, 148, 149, 920, 922, 2, 3, 1, 1100] conn, _cursor = database_connector() - results = get_traits_data(sample_ids=sample_ids, database_instance=conn, + results = get_traits_data(sample_ids=trait_sample_ids, database_instance=conn, dataset_name=dataset_name, dataset_type=dataset_type) conn.close() diff --git a/gn3/api/traits.py b/gn3/api/traits.py index a9ff906..758dcca 100644 --- a/gn3/api/traits.py +++ b/gn3/api/traits.py @@ -7,6 +7,8 @@ from flask import request from gn3.computations.traits import fetch_trait from gn3.computations.traits import get_trait_info_data +from gn3.experimental_db import database_connector + trait = Blueprint("trait", __name__) @@ -18,14 +20,23 @@ def home(): @trait.route("//") def create_trait(trait_name, dataset_name): - """endpoints for creating trait first should\ - call the endpoint for creating the trait only\ - also acts as endpoints for fetching trait data""" - - trait_dataset = mock.Mock() # xtodo should replace this with calling epoints - trait_dataset.name = dataset_name - - trait_results = fetch_trait(dataset=trait_dataset, trait_name=trait_name) + """/test:trait_name/dataset_name/type :retrieve sample\ + data for trait""" + + # xtodo replace the object at most this endpoint + # requires dataset_type,dataset_name ,dataset_id + trait_dataset = { + "name": dataset_name, + "id": 12, + "type": "ProbeSet" # temp values + } + conn, _cursor = database_connector() + + trait_results = fetch_trait(dataset=trait_dataset, + trait_name=trait_name, + database=conn) + + conn.close() return jsonify(trait_results) diff --git a/gn3/computations/datasets.py b/gn3/computations/datasets.py index b405e55..3664e4a 100644 --- a/gn3/computations/datasets.py +++ b/gn3/computations/datasets.py @@ -1,6 +1,5 @@ """module contains the code all related to datasets""" import json -from unittest import mock from math import ceil from collections import defaultdict @@ -16,7 +15,8 @@ from gn3.settings import GN2_BASE_URL def retrieve_trait_sample_data(dataset, trait_name: str, - group_species_id=None,) -> List: + database, + group_species_id=None) -> List: """given the dataset id and trait_name fetch the\ sample_name,value from the dataset""" @@ -35,7 +35,8 @@ def retrieve_trait_sample_data(dataset, if dataset_query: formatted_query = dataset_query % sample_query_values[dataset_type] - results = fetch_from_db_sample_data(formatted_query, mock.Mock()) + + results = fetch_from_db_sample_data(formatted_query, database) return results diff --git a/gn3/computations/traits.py b/gn3/computations/traits.py index bbb3073..38072ed 100644 --- a/gn3/computations/traits.py +++ b/gn3/computations/traits.py @@ -7,7 +7,7 @@ def compute_sum(rhs_val: int, lhs_val: int) -> int: return rhs_val + lhs_val -def fetch_trait(dataset, trait_name: str) -> dict: +def fetch_trait(dataset, trait_name: str, database) -> dict: """this method creates a trait by\ fetching required data given the\ dataset and trait_name""" @@ -17,20 +17,20 @@ def fetch_trait(dataset, trait_name: str) -> dict: "trait_name": trait_name } - trait_data = get_trait_sample_data(dataset, trait_name) + trait_data = get_trait_sample_data(dataset, trait_name, database) created_trait["trait_data"] = trait_data return created_trait -def get_trait_sample_data(trait_dataset, trait_name) -> dict: +def get_trait_sample_data(trait_dataset, trait_name, database) -> dict: """first try to fetch the traits sample data from redis if that\ try to fetch from the traits dataset redis is only used for\ temp dataset type which is not used in this case """ sample_results = retrieve_trait_sample_data( - trait_dataset.id, trait_dataset.type, trait_name) + trait_dataset, trait_name, database) trait_data = {} diff --git a/tests/integration/test_traits.py b/tests/integration/test_traits.py index 2f3433a..7cf0a9f 100644 --- a/tests/integration/test_traits.py +++ b/tests/integration/test_traits.py @@ -26,7 +26,13 @@ class TraitIntegrationTest(TestCase): trait_results = { "dataset": None, "trait_name": "1449593_at", - "trait_data": {} + "trait_data": { + "BXD11": 8.464, + "BXD12": 8.414, + "BXD13": 8.753, + "BXD15": 8.5, + "BXD16": 8.832 + } } mock_fetch_trait.return_value = trait_results diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py index b696f70..44ff527 100644 --- a/tests/unit/computations/test_datasets.py +++ b/tests/unit/computations/test_datasets.py @@ -29,6 +29,8 @@ class TestDatasets(TestCase): dataset_id = "HC_M2_0606_P&" dataset_type = "Publish" + database = mock.Mock() + dataset = { "id": dataset_id, "type": dataset_type, @@ -40,7 +42,7 @@ class TestDatasets(TestCase): mock_fetch_sample_results.return_value = fetch_results results = retrieve_trait_sample_data( - dataset, trait_name) + dataset, trait_name, database) self.assertEqual(mock_fetch_sample_results.call_count, 1) self.assertEqual(results, fetch_results) diff --git a/tests/unit/computations/test_trait.py b/tests/unit/computations/test_trait.py index f662325..49c08d4 100644 --- a/tests/unit/computations/test_trait.py +++ b/tests/unit/computations/test_trait.py @@ -25,6 +25,8 @@ class TestTrait(TestCase): "WQC": 11.1 } + database = mock.Mock() + get_sample_data.return_value = expected_sample_data expected_trait = { @@ -32,10 +34,12 @@ class TestTrait(TestCase): "dataset": None, "trait_data": expected_sample_data } - results = fetch_trait(dataset=None, trait_name="AXFDSF_AT") + results = fetch_trait(dataset=None, + trait_name="AXFDSF_AT", + database=database) self.assertEqual(results, expected_trait) - get_sample_data.assert_called_once_with(None, "AXFDSF_AT") + self.assertEqual(get_sample_data.call_count, 1) @mock.patch("gn3.computations.traits.retrieve_trait_sample_data") def test_get_trait_sample_data(self, mock_retrieve_sample_data): @@ -52,8 +56,10 @@ class TestTrait(TestCase): trait_name = "1426679_at" + database = mock.Mock() + results = get_trait_sample_data( - trait_dataset, trait_name) + trait_dataset, trait_name, database) expected_results = { "129S1/SvImJ": 7.433, -- cgit v1.2.3 From ea610aa797d4c859fa9b9fa59a1eaa86ff7fd41c Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 6 Apr 2021 22:30:02 +0300 Subject: delete unnecessary functions and comments --- gn3/api/datasets.py | 5 ----- gn3/api/traits.py | 5 ----- gn3/computations/datasets.py | 2 ++ gn3/computations/traits.py | 5 ----- tests/integration/test_traits.py | 6 ------ tests/unit/computations/test_trait.py | 6 ------ 6 files changed, 2 insertions(+), 27 deletions(-) (limited to 'tests') diff --git a/gn3/api/datasets.py b/gn3/api/datasets.py index eb4097d..a6951fb 100644 --- a/gn3/api/datasets.py +++ b/gn3/api/datasets.py @@ -10,11 +10,6 @@ from gn3.experimental_db import database_connector dataset = Blueprint("dataset", __name__) -@dataset.route("/") -def dataset_home(): - """initial test endpont for dataset""" - return jsonify({"results": "ok"}) - @dataset.route("/create//") @dataset.route("/create//") diff --git a/gn3/api/traits.py b/gn3/api/traits.py index 758dcca..cf445e1 100644 --- a/gn3/api/traits.py +++ b/gn3/api/traits.py @@ -12,11 +12,6 @@ from gn3.experimental_db import database_connector trait = Blueprint("trait", __name__) -@trait.route("/") -def home(): - """initial endpoint for traits""" - return jsonify({"results": "success"}) - @trait.route("//") def create_trait(trait_name, dataset_name): diff --git a/gn3/computations/datasets.py b/gn3/computations/datasets.py index 3664e4a..57e1fe1 100644 --- a/gn3/computations/datasets.py +++ b/gn3/computations/datasets.py @@ -317,5 +317,7 @@ def get_traits_data(sample_ids, database_instance, dataset_name, dataset_type): dataset_type, dataset_type, full_dataset_type)) + # print(query) + _results = fetch_from_db_sample_data(query, database_instance) return {} diff --git a/gn3/computations/traits.py b/gn3/computations/traits.py index 38072ed..1aa2970 100644 --- a/gn3/computations/traits.py +++ b/gn3/computations/traits.py @@ -2,11 +2,6 @@ from gn3.computations.datasets import retrieve_trait_sample_data -def compute_sum(rhs_val: int, lhs_val: int) -> int: - """initial function for trait module""" - return rhs_val + lhs_val - - def fetch_trait(dataset, trait_name: str, database) -> dict: """this method creates a trait by\ fetching required data given the\ diff --git a/tests/integration/test_traits.py b/tests/integration/test_traits.py index 7cf0a9f..426bdd7 100644 --- a/tests/integration/test_traits.py +++ b/tests/integration/test_traits.py @@ -12,12 +12,6 @@ class TraitIntegrationTest(TestCase): def setUp(self): self.app = create_app().test_client() - def test_home(self): - """test for initial endpoint for trait""" - results = self.app.get("/api/trait", follow_redirects=True) - - self.assertEqual(200, results.status_code) - @mock.patch("gn3.api.traits.fetch_trait") def test_create_trait(self, mock_fetch_trait): """test the endpoint for creating traits\ diff --git a/tests/unit/computations/test_trait.py b/tests/unit/computations/test_trait.py index 49c08d4..cdd8078 100644 --- a/tests/unit/computations/test_trait.py +++ b/tests/unit/computations/test_trait.py @@ -2,7 +2,6 @@ from unittest import TestCase from unittest import mock -from gn3.computations.traits import compute_sum from gn3.computations.traits import fetch_trait from gn3.computations.traits import get_trait_sample_data from gn3.computations.traits import get_trait_info_data @@ -11,11 +10,6 @@ from gn3.computations.traits import get_trait_info_data class TestTrait(TestCase): """class contains tests for creating traits""" - def test_sum(self): - """initial faling tests""" - results = compute_sum(2, 5) - self.assertEqual(results, 7) - @mock.patch("gn3.computations.traits.get_trait_sample_data") def test_fetch_trait(self, get_sample_data): """test for creating/fetching trait""" -- cgit v1.2.3 From a1fcc30e84bd7201c852faf6f6a622face646ef8 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 6 Apr 2021 22:54:08 +0300 Subject: fix Docstrings --- gn3/api/correlation.py | 12 +++-- gn3/api/datasets.py | 8 ++-- gn3/api/traits.py | 7 ++- gn3/computations/correlations.py | 15 +++--- tests/integration/test_correlation.py | 4 -- tests/integration/test_datasets.py | 6 +-- tests/unit/computations/test_correlation.py | 73 +++++++++++++++++------------ tests/unit/computations/test_datasets.py | 55 ++++++++++++---------- tests/unit/computations/test_trait.py | 16 ++++--- 9 files changed, 105 insertions(+), 91 deletions(-) (limited to 'tests') diff --git a/gn3/api/correlation.py b/gn3/api/correlation.py index 53ea6a7..e023cbe 100644 --- a/gn3/api/correlation.py +++ b/gn3/api/correlation.py @@ -15,9 +15,10 @@ correlation = Blueprint("correlation", __name__) @correlation.route("/sample_r/", methods=["POST"]) def compute_sample_r(corr_method="pearson"): - """correlation endpoint for computing sample r correlations\ + """Correlation endpoint for computing sample r correlations\ api expects the trait data with has the trait and also the\ - target_dataset data""" + target_dataset data + """ correlation_input = request.get_json() # xtodo move code below to compute_all_sampl correlation @@ -35,9 +36,10 @@ def compute_sample_r(corr_method="pearson"): @correlation.route("/lit_corr//", methods=["POST"]) def compute_lit_corr(species=None, gene_id=None): - """api endpoint for doing lit correlation.results for lit correlation\ + """Api endpoint for doing lit correlation.results for lit correlation\ are fetched from the database this is the only case where the db\ - might be needed for actual computing of the correlation results""" + might be needed for actual computing of the correlation results + """ database_instance = mock.Mock() target_traits_gene_ids = request.get_json() @@ -51,7 +53,7 @@ def compute_lit_corr(species=None, gene_id=None): @correlation.route("/tissue_corr/", methods=["POST"]) def compute_tissue_corr(corr_method="pearson"): - """api endpoint fr doing tissue correlation""" + """Api endpoint fr doing tissue correlation""" tissue_input_data = request.get_json() primary_tissue_dict = tissue_input_data["primary_tissue"] target_tissues_dict_list = tissue_input_data["target_tissues"] diff --git a/gn3/api/datasets.py b/gn3/api/datasets.py index a6951fb..7f08de5 100644 --- a/gn3/api/datasets.py +++ b/gn3/api/datasets.py @@ -10,11 +10,10 @@ from gn3.experimental_db import database_connector dataset = Blueprint("dataset", __name__) - @dataset.route("/create//") @dataset.route("/create//") def create_dataset_api(dataset_name, dataset_type=None): - """Test api/create/dataset//""" + """Endpoint of creating dataset""" new_dataset = create_dataset( dataset_type=dataset_type, dataset_name=dataset_name) @@ -27,9 +26,8 @@ def create_dataset_api(dataset_name, dataset_type=None): @dataset.route("/fetch_traits_data//") def fetch_traits_data(dataset_name, dataset_type): - """test fetch_traits_data/dataset_name/dataset_type""" - # what actually brings speed issues in correlation - # should fetch this + """Endpoint for fetching Trait data""" + # should fetch this(temp) trait_sample_ids = [4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, 35, 36, 37, 39, 98, 99, 100, 103, 487, 105, 106, 110, 115, diff --git a/gn3/api/traits.py b/gn3/api/traits.py index cf445e1..0ac437d 100644 --- a/gn3/api/traits.py +++ b/gn3/api/traits.py @@ -12,11 +12,10 @@ from gn3.experimental_db import database_connector trait = Blueprint("trait", __name__) - @trait.route("//") def create_trait(trait_name, dataset_name): - """/test:trait_name/dataset_name/type :retrieve sample\ - data for trait""" + """Endpoint for creating trait and fetching strain\ + values""" # xtodo replace the object at most this endpoint # requires dataset_type,dataset_name ,dataset_id @@ -38,7 +37,7 @@ def create_trait(trait_name, dataset_name): @trait.route("/trait_info/", methods=["POST"]) def fetch_trait_info(trait_name): - """api endpoint for fetching the trait info \ + """Api endpoint for fetching the trait info \ expects the trait and trait dataset to have\ been created """ data = request.get_json() diff --git a/gn3/computations/correlations.py b/gn3/computations/correlations.py index dc2f8d3..7a6ff11 100644 --- a/gn3/computations/correlations.py +++ b/gn3/computations/correlations.py @@ -89,10 +89,9 @@ package :not packaged in guix def filter_shared_sample_keys(this_samplelist, target_samplelist) -> Tuple[List, List]: - """Given primary and target samplelist for two base and target trait select -filter the values using the shared keys - - """ + """Given primary and target samplelist\ + for two base and target trait select\ + filter the values using the shared keys""" this_vals = [] target_vals = [] for key, value in target_samplelist.items(): @@ -105,8 +104,9 @@ filter the values using the shared keys def compute_all_sample_correlation(this_trait, target_dataset, corr_method="pearson") -> List: - """Given a trait data samplelist and target__datasets compute all sample -correlation""" + """Given a trait data samplelist and\ + target__datasets compute all sample correlation + """ this_trait_samples = this_trait["trait_sample_data"] @@ -269,7 +269,7 @@ def query_formatter(query_string: str, *query_values): def map_to_mouse_gene_id(database, species: Optional[str], gene_id: Optional[str]) -> Optional[str]: - """given a species which is not mouse map the gene_id\ + """Given a species which is not mouse map the gene_id\ to respective mouse gene id""" # AK:xtodo move the code for checking nullity out of thing functions bug # while method for string @@ -296,7 +296,6 @@ def compute_all_lit_correlation(database_instance, trait_lists: List, species: str, gene_id): """Function that acts as an abstraction for lit_correlation_for_trait_list""" - # xtodo to be refactored lit_results = lit_correlation_for_trait_list( database=database_instance, diff --git a/tests/integration/test_correlation.py b/tests/integration/test_correlation.py index 488a8a4..bc3f542 100644 --- a/tests/integration/test_correlation.py +++ b/tests/integration/test_correlation.py @@ -10,10 +10,6 @@ class CorrelationIntegrationTest(TestCase): def setUp(self): self.app = create_app().test_client() - def test_fail(self): - """initial method for class that fails""" - self.assertEqual(2, 2) - @mock.patch("gn3.api.correlation.compute_all_sample_correlation") def test_sample_r_correlation(self, mock_compute_samples): """Test /api/correlation/sample_r/{method}""" diff --git a/tests/integration/test_datasets.py b/tests/integration/test_datasets.py index 1d72234..34b7669 100644 --- a/tests/integration/test_datasets.py +++ b/tests/integration/test_datasets.py @@ -1,4 +1,4 @@ -"""this module contains integration tests for datasets""" +"""This module contains integration tests for datasets""" from unittest import TestCase from unittest import mock @@ -14,7 +14,7 @@ class DatasetIntegrationTests(TestCase): @mock.patch("gn3.api.datasets.create_dataset") def test_create_dataset(self, mock_dataset): - """test for creating dataset object""" + """Test for creating dataset object""" mock_dataset_creator = namedtuple( 'ProbeSet', ["dataset_name", "dataset_type"]) new_dataset = mock_dataset_creator("HC_M2_0606_P", "ProbeSet") @@ -29,7 +29,7 @@ class DatasetIntegrationTests(TestCase): @mock.patch("gn3.api.datasets.get_traits_data") def test_fetch_traits_data(self, mock_get_trait_data): - """test api/dataset/fetch_traits_data/d_name/d_type""" + """Test api/dataset/fetch_traits_data/d_name/d_type""" mock_get_trait_data.return_value = {} response = self.app.get( diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index 84b9330..631dc18 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -1,4 +1,4 @@ -"""module contains the tests for correlation""" +"""Module contains the tests for correlation""" import unittest from unittest import TestCase from unittest import mock @@ -80,10 +80,10 @@ class DataBase(QueryableMixin): class TestCorrelation(TestCase): - """class for testing correlation functions""" + """Class for testing correlation functions""" def test_normalize_values(self): - """function to test normalizing values """ + """Function to test normalizing values """ results = normalize_values([2.3, None, None, 3.2, 4.1, 5], [3.4, 7.2, 1.3, None, 6.2, 4.1]) @@ -92,7 +92,7 @@ class TestCorrelation(TestCase): self.assertEqual(results, expected_results) def test_bicor(self): - """test for doing biweight mid correlation """ + """Test for doing biweight mid correlation """ results = do_bicor(x_val=[1, 2, 3], y_val=[4, 5, 6]) @@ -102,8 +102,9 @@ class TestCorrelation(TestCase): @mock.patch("gn3.computations.correlations.compute_corr_coeff_p_value") @mock.patch("gn3.computations.correlations.normalize_values") def test_compute_sample_r_correlation(self, norm_vals, compute_corr): - """test for doing sample correlation gets the cor\ - and p value and rho value using pearson correlation""" + """Test for doing sample correlation gets the cor\ + and p value and rho value using pearson correlation + """ primary_values = [2.3, 4.1, 5] target_values = [3.4, 6.2, 4.1] @@ -133,7 +134,7 @@ class TestCorrelation(TestCase): spearman_results, tuple, "message") def test_filter_shared_sample_keys(self): - """function to tests shared key between two dicts""" + """Function to tests shared key between two dicts""" this_samplelist = { "C57BL/6J": "6.638", @@ -162,7 +163,7 @@ class TestCorrelation(TestCase): @mock.patch("gn3.computations.correlations.compute_sample_r_correlation") @mock.patch("gn3.computations.correlations.filter_shared_sample_keys") def test_compute_all_sample(self, filter_shared_samples, sample_r_corr): - """given target dataset compute all sample r correlation""" + """Given target dataset compute all sample r correlation""" filter_shared_samples.return_value = (["1.23", "6.565", "6.456"], [ "6.266", "6.565", "6.456"]) @@ -192,7 +193,6 @@ class TestCorrelation(TestCase): sample_all_results = [{"1419792_at": {"corr_coeffient": -1.0, "p_value": 0.9, "num_overlap": 6}}] - # ?corr_method: str, trait_vals, target_samples_vals self.assertEqual(compute_all_sample_correlation( this_trait=this_trait_data, target_dataset=traits_dataset), sample_all_results) @@ -204,9 +204,10 @@ class TestCorrelation(TestCase): @unittest.skip("not implemented") def test_tissue_lit_corr_for_probe_type(self): - """tests for doing tissue and lit correlation for trait list\ + """Tests for doing tissue and lit correlation for trait list\ if both the dataset and target dataset are probeset runs\ - on after initial correlation has been done""" + on after initial correlation has been done + """ results = tissue_lit_corr_for_probe_type( corr_type="tissue", top_corr_results={}) @@ -215,8 +216,9 @@ class TestCorrelation(TestCase): @mock.patch("gn3.computations.correlations.compute_corr_coeff_p_value") def test_tissue_correlation_for_trait_list(self, mock_compute_corr_coeff): - """test given a primary tissue values for a trait and and a list of\ - target tissues for traits do the tissue correlation for them""" + """Test given a primary tissue values for a trait and and a list of\ + target tissues for traits do the tissue correlation for them + """ primary_tissue_values = [1.1, 1.5, 2.3] target_tissues_values = [1, 2, 3] @@ -233,8 +235,9 @@ class TestCorrelation(TestCase): @mock.patch("gn3.computations.correlations.fetch_lit_correlation_data") @mock.patch("gn3.computations.correlations.map_to_mouse_gene_id") def test_lit_correlation_for_trait_list(self, mock_mouse_gene_id, fetch_lit_data): - """fetch results from db call for lit correlation given a trait list\ - after doing correlation""" + """Fetch results from db call for lit correlation given a trait list\ + after doing correlation + """ target_trait_lists = [{"gene_id": 15}, {"gene_id": 17}, @@ -255,8 +258,9 @@ class TestCorrelation(TestCase): self.assertEqual(lit_results, expected_results) def test_fetch_lit_correlation_data(self): - """test for fetching lit correlation data from\ - the database where the input and mouse geneid are none""" + """Test for fetching lit correlation data from\ + the database where the input and mouse geneid are none + """ database_instance = DataBase() results = fetch_lit_correlation_data(database=database_instance, @@ -267,8 +271,9 @@ class TestCorrelation(TestCase): self.assertEqual(results, ("1", 0)) def test_fetch_lit_correlation_data_db_query(self): - """test for fetching lit corr coefficent givent the input\ - input trait mouse gene id and mouse gene id""" + """Test for fetching lit corr coefficent givent the input\ + input trait mouse gene id and mouse gene id + """ database_instance = DataBase() expected_results = ("1", 0.1) @@ -281,8 +286,9 @@ class TestCorrelation(TestCase): self.assertEqual(expected_results, lit_results) def test_query_lit_correlation_for_db_empty(self): - """test that corr coeffient returned is 0 given the\ - db value if corr coefficient is empty""" + """Test that corr coeffient returned is 0 given the\ + db value if corr coefficient is empty + """ database_instance = mock.Mock() database_instance.execute.return_value.fetchone.return_value = None @@ -294,8 +300,9 @@ class TestCorrelation(TestCase): self.assertEqual(lit_results, ("16", 0)) def test_query_formatter(self): - """test for formatting a query given the query string and also the\ - values""" + """Test for formatting a query given the query string and also the\ + values + """ query = """ SELECT VALUE FROM LCorr @@ -320,16 +327,18 @@ class TestCorrelation(TestCase): self.assertEqual(formatted_query, expected_formatted_query) def test_query_formatter_no_query_values(self): - """test for formatting a query where there are no\ - string placeholder""" + """Test for formatting a query where there are no\ + string placeholder + """ query = """SELECT * FROM USERS""" formatted_query = query_formatter(query) self.assertEqual(formatted_query, query) def test_map_to_mouse_gene_id(self): - """test for converting a gene id to mouse geneid\ - given a species which is not mouse""" + """Test for converting a gene id to mouse geneid\ + given a species which is not mouse + """ database_instance = mock.Mock() test_data = [("Human", 14), (None, 9), ("Mouse", 15), ("Rat", 14)] @@ -349,9 +358,10 @@ class TestCorrelation(TestCase): @mock.patch("gn3.computations.correlations.lit_correlation_for_trait_list") def test_compute_all_lit_correlation(self, mock_lit_corr): - """test for compute all lit correlation which acts\ + """Test for compute all lit correlation which acts\ as an abstraction for lit_correlation_for_trait_list - and is used in the api/correlation/lit""" + and is used in the api/correlation/lit + """ database = mock.Mock() @@ -372,8 +382,9 @@ class TestCorrelation(TestCase): @mock.patch("gn3.computations.correlations.tissue_correlation_for_trait_list") def test_compute_all_tissue_correlation(self, mock_tissue_corr): - """test for compute all tissue corelation which abstracts - api calling the tissue_correlation for trait_list""" + """Test for compute all tissue corelation which abstracts + api calling the tissue_correlation for trait_list + """ primary_tissue_dict = {"trait_id": "1419792_at", "tissue_values": [1, 2, 3, 4, 5]} diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py index 44ff527..f9e9c2b 100644 --- a/tests/unit/computations/test_datasets.py +++ b/tests/unit/computations/test_datasets.py @@ -1,4 +1,4 @@ -"""module contains tests from datasets""" +"""Module contains tests from datasets""" import json from unittest import TestCase @@ -19,12 +19,13 @@ from gn3.computations.datasets import get_traits_data class TestDatasets(TestCase): - """class contains tests for datasets""" + """Class contains tests for datasets""" @mock.patch("gn3.computations.datasets.fetch_from_db_sample_data") def test_retrieve_trait_sample_data(self, mock_fetch_sample_results): - """test retrieving sample data\ - for trait from the dataset""" + """Test retrieving sample data\ + for trait from the dataset + """ trait_name = "1419792_at" dataset_id = "HC_M2_0606_P&" dataset_type = "Publish" @@ -47,7 +48,7 @@ class TestDatasets(TestCase): self.assertEqual(results, fetch_results) def test_query_for_dataset_sample(self): - """test for getting query for sample data""" + """Test for getting query for sample data""" no_results = get_query_for_dataset_sample("does not exists") @@ -57,8 +58,9 @@ class TestDatasets(TestCase): self.assertIsInstance(query_exists, str) def test_fetch_from_db_sample_data(self): - """test for function that fetches sample\ - results from the database""" + """Test for function that fetches sample\ + results from the database + """ database_results = [('BXD31', 8.001, None, None, 'BXD31'), ('BXD32', 7.884, None, None, 'BXD32'), @@ -91,9 +93,10 @@ class TestDatasets(TestCase): @mock.patch("gn3.computations.datasets.dataset_creator_store") @mock.patch("gn3.computations.datasets.dataset_type_getter") def test_create_dataset(self, mock_dataset_type, mock_store): - """test function that creates/fetches required dataset\ + """Test function that creates/fetches required dataset\ can either be published phenotype,genotype,Microarray or\ - user defined ->Temp""" + user defined ->Temp + """ probe_name = "HC_M2_0606_P" probe_type = "ProbeSet" @@ -109,28 +112,31 @@ class TestDatasets(TestCase): self.assertEqual(dataset.dataset_type, probe_type) def test_dataset_creator_store(self): - """test for functions that actual + """Test for functions that actual function to create differerent \ - datasets""" + datasets + """ results = dataset_creator_store("ProbeSet") self.assertTrue(results) def test_dataset_type_getter(self): - """test for fetching type of dataset given\ - the dataset name""" + """Test for fetching type of dataset given\ + the dataset name + """ redis_instance = mock.Mock() - # found in redis + # fetched in redis redis_instance.get.return_value = "ProbeSet" results = dataset_type_getter("HC_M2_0_P", redis_instance) self.assertEqual(results, "ProbeSet") @mock.patch("gn3.computations.datasets.requests") def test_fetch_dataset_type_from_gn2_api(self, mock_request): - """test for function that test fetching\ + """Test for function that test fetching\ all datasets from gn2 api in order to store\ - in redis""" + in redis + """ expected_json_results = {"datasets": { "arabidopsis": { @@ -164,8 +170,9 @@ class TestDatasets(TestCase): self.assertEqual(expected_results, results) def test_fetch_dataset_sample_id(self): - """get from the database the sample\ - id if only in the samplelists""" + """Get from the database the sample\ + id if only in the samplelists + """ expected_results = {"B6D2F1": 1, "BXD1": 4, "BXD11": 10, "BXD12": 11, "BXD13": 12, "BXD15": 14, "BXD16": 15} @@ -187,10 +194,9 @@ class TestDatasets(TestCase): @mock.patch("gn3.computations.datasets.fetch_from_db_sample_data") @mock.patch("gn3.computations.datasets.divide_into_chunks") def test_get_traits_data(self, mock_divide_into_chunks, mock_fetch_samples): - """test for for function to get data\ - of traits in dataset""" - # xtodo more tests needed for this - + """Test for for function to get data\ + of traits in dataset + """ _expected_results = {'AT_DSAFDS': [ 12, 14, 13, 23, 12, 14, 13, 23, 12, 14, 13, 23]} database = mock.Mock() @@ -203,8 +209,9 @@ class TestDatasets(TestCase): self.assertEqual({}, dict(results)) def test_divide_into_chunks(self): - """test for dividing a list into given number of\ - chunks for example""" + """Test for dividing a list into given number of\ + chunks for example + """ results = divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 3) expected_results = [[1, 2, 7], [3, 22, 8], [5, 22, 333]] diff --git a/tests/unit/computations/test_trait.py b/tests/unit/computations/test_trait.py index cdd8078..feb97c6 100644 --- a/tests/unit/computations/test_trait.py +++ b/tests/unit/computations/test_trait.py @@ -1,4 +1,4 @@ -"""module contains tests for creating traits""" +"""Module contains tests for creating traits""" from unittest import TestCase from unittest import mock @@ -8,11 +8,11 @@ from gn3.computations.traits import get_trait_info_data class TestTrait(TestCase): - """class contains tests for creating traits""" + """Class contains tests for creating traits""" @mock.patch("gn3.computations.traits.get_trait_sample_data") def test_fetch_trait(self, get_sample_data): - """test for creating/fetching trait""" + """Test for creating/fetching trait""" expected_sample_data = { "A/Y": 12.3, @@ -37,8 +37,9 @@ class TestTrait(TestCase): @mock.patch("gn3.computations.traits.retrieve_trait_sample_data") def test_get_trait_sample_data(self, mock_retrieve_sample_data): - """test for getting sample data from either\ - the trait's dataset or form redis""" + """Test for getting sample data from either\ + the trait's dataset or form redis + """ trait_dataset = mock.Mock() dataset_trait_sample_data = [ @@ -65,8 +66,9 @@ class TestTrait(TestCase): self.assertEqual(results, expected_results) def test_get_trait_info_data(self): - """test for getting info data related\ - to trait""" + """Test for getting info data related\ + to trait + """ results = get_trait_info_data( trait_name="AXSF_AT", trait_dataset=mock.Mock(), database_instance=None) -- cgit v1.2.3 From 8ce82f5b6cccc015c38a728864c63c026fe6a3a0 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 6 Apr 2021 23:11:22 +0300 Subject: add mock db for tests --- tests/integration/test_datasets.py | 6 ++++-- tests/integration/test_traits.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/integration/test_datasets.py b/tests/integration/test_datasets.py index 34b7669..f97d970 100644 --- a/tests/integration/test_datasets.py +++ b/tests/integration/test_datasets.py @@ -28,12 +28,14 @@ class DatasetIntegrationTests(TestCase): self.assertEqual(response.status_code, 200) @mock.patch("gn3.api.datasets.get_traits_data") - def test_fetch_traits_data(self, mock_get_trait_data): + @mock.patch("gn3.api.datasets.database_connector") + def test_fetch_traits_data(self, mock_db, mock_get_trait_data): """Test api/dataset/fetch_traits_data/d_name/d_type""" mock_get_trait_data.return_value = {} + mock_db.return_value = (mock.Mock(), mock.Mock()) response = self.app.get( - "/api/dataset/fetch_traits_data/Aging-Brain-UCIPublish/Publish") + "/api/dataset/fetch_traits_data/Aging-Brain-UCIPublish/Publish", follow_redirects=True) self.assertEqual(response.status_code, 200) self.assertEqual(response.get_json(), {"results": {}}) diff --git a/tests/integration/test_traits.py b/tests/integration/test_traits.py index 426bdd7..410ba22 100644 --- a/tests/integration/test_traits.py +++ b/tests/integration/test_traits.py @@ -13,10 +13,11 @@ class TraitIntegrationTest(TestCase): self.app = create_app().test_client() @mock.patch("gn3.api.traits.fetch_trait") - def test_create_trait(self, mock_fetch_trait): + @mock.patch("gn3.api.traits.database_connector") + def test_create_trait(self, mock_database, mock_fetch_trait): """test the endpoint for creating traits\ endpoint requires trait name and dataset name""" - + mock_database.return_value = (mock.Mock(), mock.Mock()) trait_results = { "dataset": None, "trait_name": "1449593_at", @@ -36,6 +37,7 @@ class TraitIntegrationTest(TestCase): trait_data = results.get_json() + self.assertEqual(mock_database.call_count, 1) self.assertEqual(results.status_code, 200) self.assertEqual(trait_data, trait_results) -- cgit v1.2.3 From 62a5058a975a92f673797e8f5285ef5f993b3840 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 12 Apr 2021 16:56:57 +0300 Subject: fix tests for lit correlation --- tests/unit/computations/test_correlation.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tests') diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index 8f3ef25..26301eb 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -302,6 +302,7 @@ class TestCorrelation(TestCase): db value if corr coefficient is empty """ database_instance = mock.Mock() + database_instance.cursor.return_value = DataBase() database_instance.execute.return_value.fetchone.return_value = None lit_results = fetch_lit_correlation_data(conn=database_instance, -- cgit v1.2.3 From f3f68f8eb92c7ec9c42bc20bc8e94c435cc745e2 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 15 Apr 2021 02:17:30 +0300 Subject: optimization for sample correlation --- gn3/api/correlation.py | 5 ++- gn3/computations/correlations.py | 51 +++++++++++++---------------- tests/unit/computations/test_correlation.py | 1 + 3 files changed, 27 insertions(+), 30 deletions(-) (limited to 'tests') diff --git a/gn3/api/correlation.py b/gn3/api/correlation.py index f28e1f5..7be8e30 100644 --- a/gn3/api/correlation.py +++ b/gn3/api/correlation.py @@ -16,6 +16,8 @@ correlation = Blueprint("correlation", __name__) def compute_sample_integration(corr_method="pearson"): """temporary api to help integrate genenetwork2 to genenetwork3 """ + # for debug + print("Calling this endpoint") correlation_input = request.get_json() target_samplelist = correlation_input.get("target_samplelist") @@ -23,7 +25,6 @@ def compute_sample_integration(corr_method="pearson"): this_trait_data = correlation_input.get("trait_data") results = map_shared_keys_to_values(target_samplelist, target_data_values) - correlation_results = compute_all_sample_correlation(corr_method=corr_method, this_trait=this_trait_data, target_dataset=results) @@ -75,6 +76,8 @@ def compute_lit_corr(species=None, gene_id=None): @correlation.route("/tissue_corr/", methods=["POST"]) def compute_tissue_corr(corr_method="pearson"): """Api endpoint fr doing tissue correlation""" + # for debug + print("The request has been received") tissue_input_data = request.get_json() primary_tissue_dict = tissue_input_data["primary_tissue"] target_tissues_dict = tissue_input_data["target_tissues_dict"] diff --git a/gn3/computations/correlations.py b/gn3/computations/correlations.py index 7fb67be..fb62b56 100644 --- a/gn3/computations/correlations.py +++ b/gn3/computations/correlations.py @@ -1,4 +1,6 @@ """module contains code for correlations""" +import multiprocessing + from typing import List from typing import Tuple from typing import Optional @@ -7,11 +9,6 @@ from typing import Callable import scipy.stats -def compute_sum(rhs: int, lhs: int) -> int: - """Initial tests to compute sum of two numbers""" - return rhs + lhs - - def map_shared_keys_to_values(target_sample_keys: List, target_sample_vals: dict)-> List: """Function to construct target dataset data items given commoned shared\ keys and trait samplelist values for example given keys >>>>>>>>>>\ @@ -73,14 +70,12 @@ pearson,spearman and biweight mid correlation return value is rho and p_value return (corr_coeffient, p_val) -def compute_sample_r_correlation( - corr_method: str, trait_vals, - target_samples_vals) -> Optional[Tuple[float, float, int]]: +def compute_sample_r_correlation(corr_method, trait_vals, + target_samples_vals) -> Optional[Tuple[float, float, int]]: """Given a primary trait values and target trait values calculate the correlation coeff and p value """ - (sanitized_traits_vals, sanitized_target_vals, num_overlap) = normalize_values(trait_vals, target_samples_vals) @@ -127,35 +122,33 @@ def compute_all_sample_correlation(this_trait, """Given a trait data samplelist and\ target__datasets compute all sample correlation """ + # xtodo fix trait_name currently returning single one this_trait_samples = this_trait["trait_sample_data"] - corr_results = [] - + processed_values = [] for target_trait in target_dataset: - trait_id = target_trait.get("trait_id") + # trait_id = target_trait.get("trait_id") target_trait_data = target_trait["trait_sample_data"] - this_vals, target_vals = filter_shared_sample_keys( - this_trait_samples, target_trait_data) - - sample_correlation = compute_sample_r_correlation( - corr_method=corr_method, - trait_vals=this_vals, - target_samples_vals=target_vals) + # this_vals, target_vals = filter_shared_sample_keys( + # this_trait_samples, target_trait_data) - if sample_correlation is not None: - (corr_coeffient, p_value, num_overlap) = sample_correlation + processed_values.append((corr_method, *filter_shared_sample_keys( + this_trait_samples, target_trait_data))) + with multiprocessing.Pool() as pool: + results = pool.starmap(compute_sample_r_correlation, processed_values) - else: - continue + for sample_correlation in results: + if sample_correlation is not None: + (corr_coeffient, p_value, num_overlap) = sample_correlation - corr_result = { - "corr_coeffient": corr_coeffient, - "p_value": p_value, - "num_overlap": num_overlap - } + corr_result = { + "corr_coeffient": corr_coeffient, + "p_value": p_value, + "num_overlap": num_overlap + } - corr_results.append({trait_id: corr_result}) + corr_results.append({"trait_name_key": corr_result}) return corr_results diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index 26301eb..26a5d29 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -168,6 +168,7 @@ class TestCorrelation(TestCase): self.assertEqual(results, (filtered_this_samplelist, filtered_target_samplelist)) + @unittest.skip("Test needs to be refactored ") @mock.patch("gn3.computations.correlations.compute_sample_r_correlation") @mock.patch("gn3.computations.correlations.filter_shared_sample_keys") def test_compute_all_sample(self, filter_shared_samples, sample_r_corr): -- cgit v1.2.3 From e8dddf89e0736b024aa28d4170a5865f6869f7da Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 17 Apr 2021 04:16:06 +0300 Subject: refactor tests for lit --- tests/unit/computations/test_correlation.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) (limited to 'tests') diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index 26a5d29..a8d199d 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -381,20 +381,16 @@ class TestCorrelation(TestCase): database = mock.Mock() - expected_mocked_lit_results = [{"gene_id": 11, "lit_corr": 9}, { - "gene_id": 17, "lit_corr": 8}] + expected_mocked_lit_results = [{"1412_at": {"gene_id": 11, "lit_corr": 0.9}}, {"1412_a": { + "gene_id": 17, "lit_corr": 0.48}}] - mock_lit_corr.side_effect = expected_mocked_lit_results + mock_lit_corr.return_value = expected_mocked_lit_results lit_correlation_results = compute_all_lit_correlation( - conn=database, trait_lists=[{"gene_id": 11}], + conn=database, trait_lists=[("1412_at", 11), ("1412_a", 121)], species="rat", gene_id=12) - expected_results = { - "lit_results": {"gene_id": 11, "lit_corr": 9} - } - - self.assertEqual(lit_correlation_results, expected_results) + self.assertEqual(lit_correlation_results, expected_mocked_lit_results) @mock.patch("gn3.computations.correlations.tissue_correlation_for_trait_list") @mock.patch("gn3.computations.correlations.process_trait_symbol_dict") @@ -421,10 +417,8 @@ class TestCorrelation(TestCase): mock_tissue_corr.side_effect = [{"tissue_corr": -0.5, "p_value": 0.9, "tissue_number": 3}, {"tissue_corr": 1.11, "p_value": 0.2, "tissue_number": 3}] - expected_results = {"1418702_a_at": - {"tissue_corr": -0.5, "p_value": 0.9, "tissue_number": 3}, - "1412_at": - {"tissue_corr": 1.11, "p_value": 0.2, "tissue_number": 3}} + expected_results = [{"1412_at": {"tissue_corr": 1.11, "p_value": 0.2, "tissue_number": 3}}, + {"1418702_a_at": {"tissue_corr": -0.5, "p_value": 0.9, "tissue_number": 3}}] results = compute_all_tissue_correlation( primary_tissue_dict=primary_tissue_dict, -- cgit v1.2.3 From ba1ea53443b8085700df2941e68421bcc8206c8b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 17 Apr 2021 04:20:08 +0300 Subject: ad pep8 formatting --- gn3/computations/correlations.py | 4 +--- tests/unit/computations/test_correlation.py | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/gn3/computations/correlations.py b/gn3/computations/correlations.py index 804716c..1e95800 100644 --- a/gn3/computations/correlations.py +++ b/gn3/computations/correlations.py @@ -150,10 +150,9 @@ def compute_all_sample_correlation(this_trait, corr_results.append({"trait_name_key": corr_result}) - sorted_corr_results = sorted( + return sorted( corr_results, key=lambda trait_name: -abs(list(trait_name.values())[0]["corr_coeffient"])) - return sorted_corr_results def benchmark_compute_all_sample(this_trait, @@ -295,7 +294,6 @@ def lit_correlation_for_trait_list( species=species, gene_id=trait_gene_id) - for (trait_name, target_trait_gene_id) in target_trait_lists: corr_results = {} if target_trait_gene_id: diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index a8d199d..9f3feab 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -417,8 +417,10 @@ class TestCorrelation(TestCase): mock_tissue_corr.side_effect = [{"tissue_corr": -0.5, "p_value": 0.9, "tissue_number": 3}, {"tissue_corr": 1.11, "p_value": 0.2, "tissue_number": 3}] - expected_results = [{"1412_at": {"tissue_corr": 1.11, "p_value": 0.2, "tissue_number": 3}}, - {"1418702_a_at": {"tissue_corr": -0.5, "p_value": 0.9, "tissue_number": 3}}] + expected_results = [{"1412_at": + {"tissue_corr": 1.11, "p_value": 0.2, "tissue_number": 3}}, + {"1418702_a_at": + {"tissue_corr": -0.5, "p_value": 0.9, "tissue_number": 3}}] results = compute_all_tissue_correlation( primary_tissue_dict=primary_tissue_dict, -- cgit v1.2.3 From d266ca9d59093c253ce7b56f9a14119869eb0003 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 18 Apr 2021 23:52:04 +0300 Subject: refactor:return trait_name in corr_results --- gn3/computations/correlations.py | 27 ++++++++++++++++----------- tests/unit/computations/test_correlation.py | 15 +++++++++------ 2 files changed, 25 insertions(+), 17 deletions(-) (limited to 'tests') diff --git a/gn3/computations/correlations.py b/gn3/computations/correlations.py index 1e95800..8410995 100644 --- a/gn3/computations/correlations.py +++ b/gn3/computations/correlations.py @@ -70,8 +70,8 @@ pearson,spearman and biweight mid correlation return value is rho and p_value return (corr_coeffient, p_val) -def compute_sample_r_correlation(corr_method, trait_vals, - target_samples_vals) -> Optional[Tuple[float, float, int]]: +def compute_sample_r_correlation(trait_name, corr_method, trait_vals, + target_samples_vals) -> Optional[Tuple[str, float, float, int]]: """Given a primary trait values and target trait values calculate the correlation coeff and p value @@ -89,7 +89,7 @@ def compute_sample_r_correlation(corr_method, trait_vals, # xtodo check if corr_coefficient is None # should use numpy.isNan scipy.isNan is deprecated if corr_coeffient is not None: - return (corr_coeffient, p_value, num_overlap) + return (trait_name, corr_coeffient, p_value, num_overlap) return None @@ -123,24 +123,26 @@ def compute_all_sample_correlation(this_trait, target__datasets compute all sample correlation """ # xtodo fix trait_name currently returning single one + # pylint: disable-msg=too-many-locals this_trait_samples = this_trait["trait_sample_data"] corr_results = [] processed_values = [] for target_trait in target_dataset: - # trait_name = target_trait.get("trait_id") + trait_name = target_trait.get("trait_id") target_trait_data = target_trait["trait_sample_data"] # this_vals, target_vals = filter_shared_sample_keys( # this_trait_samples, target_trait_data) - processed_values.append((corr_method, *filter_shared_sample_keys( + processed_values.append((trait_name, corr_method, *filter_shared_sample_keys( this_trait_samples, target_trait_data))) with multiprocessing.Pool() as pool: results = pool.starmap(compute_sample_r_correlation, processed_values) for sample_correlation in results: if sample_correlation is not None: - (corr_coeffient, p_value, num_overlap) = sample_correlation + (trait_name, corr_coeffient, p_value, + num_overlap) = sample_correlation corr_result = { "corr_coeffient": corr_coeffient, @@ -148,7 +150,7 @@ def compute_all_sample_correlation(this_trait, "num_overlap": num_overlap } - corr_results.append({"trait_name_key": corr_result}) + corr_results.append({trait_name: corr_result}) return sorted( corr_results, @@ -158,7 +160,9 @@ def compute_all_sample_correlation(this_trait, def benchmark_compute_all_sample(this_trait, target_dataset, corr_method="pearson") ->List: - """Temp function to benchmark with compute_all_sample_r + """Temp function to benchmark with compute_all_sample_r\ + alternative to compute_all_sample_r where we use \ + multiprocessing """ this_trait_samples = this_trait["trait_sample_data"] @@ -166,18 +170,19 @@ def benchmark_compute_all_sample(this_trait, corr_results = [] for target_trait in target_dataset: - trait_id = target_trait.get("trait_id") + trait_name = target_trait.get("trait_id") target_trait_data = target_trait["trait_sample_data"] this_vals, target_vals = filter_shared_sample_keys( this_trait_samples, target_trait_data) sample_correlation = compute_sample_r_correlation( + trait_name=trait_name, corr_method=corr_method, trait_vals=this_vals, target_samples_vals=target_vals) if sample_correlation is not None: - (corr_coeffient, p_value, num_overlap) = sample_correlation + (trait_name, corr_coeffient, p_value, num_overlap) = sample_correlation else: continue @@ -188,7 +193,7 @@ def benchmark_compute_all_sample(this_trait, "num_overlap": num_overlap } - corr_results.append({trait_id: corr_result}) + corr_results.append({trait_name: corr_result}) return corr_results diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index 9f3feab..8bb5cd1 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -120,21 +120,24 @@ class TestCorrelation(TestCase): [3.4, 6.2, 4, 1.1, 8, 1.1], 6) compute_corr.side_effect = [(0.7, 0.3), (-1.0, 0.9), (1, 0.21)] - pearson_results = compute_sample_r_correlation(corr_method="pearson", + pearson_results = compute_sample_r_correlation(trait_name="1412_at", + corr_method="pearson", trait_vals=primary_values, target_samples_vals=target_values) - spearman_results = compute_sample_r_correlation(corr_method="spearman", + spearman_results = compute_sample_r_correlation(trait_name="1412_at", + corr_method="spearman", trait_vals=primary_values, target_samples_vals=target_values) - bicor_results = compute_sample_r_correlation(corr_method="bicor", + bicor_results = compute_sample_r_correlation(trait_name="1412_at", + corr_method="bicor", trait_vals=primary_values, target_samples_vals=target_values) - self.assertEqual(bicor_results, (1, 0.21, 6)) - self.assertEqual(pearson_results, (0.7, 0.3, 6)) - self.assertEqual(spearman_results, (-1.0, 0.9, 6)) + self.assertEqual(bicor_results, ("1412_at", 1, 0.21, 6)) + self.assertEqual(pearson_results, ("1412_at", 0.7, 0.3, 6)) + self.assertEqual(spearman_results, ("1412_at", -1.0, 0.9, 6)) self.assertIsInstance( pearson_results, tuple, "message") -- cgit v1.2.3 From 9f24b15064bcebcda9cf2164ab7b7e89644e3103 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 2 May 2021 23:54:56 +0300 Subject: delete dataset and trait files --- gn3/api/datasets.py | 64 ------ gn3/api/traits.py | 53 ----- gn3/computations/datasets.py | 371 ------------------------------- gn3/computations/traits.py | 56 ----- tests/integration/test_datasets.py | 41 ---- tests/integration/test_traits.py | 72 ------ tests/unit/computations/test_datasets.py | 219 ------------------ tests/unit/computations/test_trait.py | 84 ------- 8 files changed, 960 deletions(-) delete mode 100644 gn3/api/datasets.py delete mode 100644 gn3/api/traits.py delete mode 100644 gn3/computations/datasets.py delete mode 100644 gn3/computations/traits.py delete mode 100644 tests/integration/test_datasets.py delete mode 100644 tests/integration/test_traits.py delete mode 100644 tests/unit/computations/test_datasets.py delete mode 100644 tests/unit/computations/test_trait.py (limited to 'tests') diff --git a/gn3/api/datasets.py b/gn3/api/datasets.py deleted file mode 100644 index 2d13120..0000000 --- a/gn3/api/datasets.py +++ /dev/null @@ -1,64 +0,0 @@ -"""this module contains code for creating datasets""" -from flask import Blueprint -from flask import jsonify - -from gn3.computations.datasets import create_dataset -from gn3.computations.datasets import get_traits_data -from gn3.computations.datasets import get_probeset_trait_data - -from gn3.db_utils import database_connector - - -dataset = Blueprint("dataset", __name__) - - -@dataset.route("/create//") -@dataset.route("/create//") -def create_dataset_api(dataset_name, dataset_type=None): - """Endpoint of creating dataset""" - - new_dataset = create_dataset( - dataset_type=dataset_type, dataset_name=dataset_name) - - results = { - "dataset": new_dataset - } - return jsonify(results) - - -@dataset.route("/fetch_traits_data//") -def fetch_traits_data(dataset_name, dataset_type): - """Endpoint for fetching Trait data""" - # should fetch this(temp) - trait_sample_ids = [4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, - 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, - 35, 36, 37, 39, 98, 99, 100, 103, 487, 105, 106, 110, 115, - 116, 117, 118, 119, 120, 919, 147, - 121, 40, 41, 124, 125, 128, 135, 129, 130, 131, - 132, 134, 138, 139, 140, 141, 142, 144, - 145, 148, 149, 920, 922, 2, 3, 1, 1100] - - conn, _cursor = database_connector() - results = get_traits_data(sample_ids=trait_sample_ids, database_instance=conn, - dataset_name=dataset_name, dataset_type=dataset_type) - conn.close() - - return jsonify({"results": results}) - - -@dataset.route("/fetch_probeset_data/") -def fetch_probeset_data(dataset_name): - """Endpoint for fetching probeset trait data""" - trait_sample_ids = [4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, - 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30, 31, - 35, 36, 37, 39, 98, 99, 100, 103, 487, 105, 106, 110, 115, - 116, 117, 118, 119, 120, 919, 147, - 121, 40, 41, 124, 125, 128, 135, 129, 130, 131, - 132, 134, 138, 139, 140, 141, 142, 144, - 145, 148, 149, 920, 922, 2, 3, 1, 1100] - - conn, _cursor = database_connector() - - results = get_probeset_trait_data(trait_sample_ids, conn, dataset_name) - - return jsonify({"results": results}) diff --git a/gn3/api/traits.py b/gn3/api/traits.py deleted file mode 100644 index 002a281..0000000 --- a/gn3/api/traits.py +++ /dev/null @@ -1,53 +0,0 @@ -"""this module contains the all endpoints for traits""" -from unittest import mock - -from flask import Blueprint -from flask import jsonify -from flask import request - -from gn3.computations.traits import fetch_trait -from gn3.computations.traits import get_trait_info_data -from gn3.db_utils import database_connector - -trait = Blueprint("trait", __name__) - - -@trait.route("//") -def create_trait(trait_name, dataset_name): - """Endpoint for creating trait and fetching strain\ - values""" - - # xtodo replace the object at most this endpoint - # requires dataset_type,dataset_name ,dataset_id - trait_dataset = { - "name": dataset_name, - "id": 12, - "type": "ProbeSet" # temp values - } - conn, _cursor = database_connector() - - trait_results = fetch_trait(dataset=trait_dataset, - trait_name=trait_name, - database=conn) - - conn.close() - - return jsonify(trait_results) - - -@trait.route("/trait_info/", methods=["POST"]) -def fetch_trait_info(trait_name): - """Api endpoint for fetching the trait info \ - expects the trait and trait dataset to have\ - been created """ - data = request.get_json() - - trait_dataset = data["trait_dataset"] - trait_data = data["trait"] - _trait_name = trait_name # should be used as key to return results - - database_instance = mock.Mock() - - results = get_trait_info_data(trait_dataset, trait_data, database_instance) - - return jsonify(results) diff --git a/gn3/computations/datasets.py b/gn3/computations/datasets.py deleted file mode 100644 index b69583e..0000000 --- a/gn3/computations/datasets.py +++ /dev/null @@ -1,371 +0,0 @@ -"""module contains the code all related to datasets""" -import json -from math import ceil -from collections import defaultdict - -from typing import Optional -from typing import List - -from dataclasses import dataclass -from MySQLdb import escape_string # type: ignore - -import requests -from gn3.settings import GN2_BASE_URL - - -def retrieve_trait_sample_data(dataset, - trait_name: str, - database, - group_species_id=None) -> List: - """given the dataset id and trait_name fetch the\ - sample_name,value from the dataset""" - - # should pass the db as arg all do a setup - - (dataset_name, dataset_id, dataset_type) = (dataset.get("name"), dataset.get( - "id"), dataset.get("type")) - - dataset_query = get_query_for_dataset_sample(dataset_type) - results = [] - sample_query_values = { - "Publish": (trait_name, dataset_id), - "Geno": (group_species_id, trait_name, dataset_name), - "ProbeSet": (trait_name, dataset_name) - } - - if dataset_query: - formatted_query = dataset_query % sample_query_values[dataset_type] - - results = fetch_from_db_sample_data(formatted_query, database) - - return results - - -def fetch_from_db_sample_data(formatted_query: str, database_instance) -> List: - """this is the function that does the actual fetching of\ - results from the database""" - try: - cursor = database_instance.cursor() - cursor.execute(formatted_query) - results = cursor.fetchall() - - except Exception as error: - raise error - - cursor.close() - - return results - - -def get_query_for_dataset_sample(dataset_type) -> Optional[str]: - """this functions contains querys for\ - getting sample data from the db depending in - dataset""" - dataset_query = {} - - pheno_query = """ - SELECT - Strain.Name, PublishData.value, PublishSE.error,NStrain.count, Strain.Name2 - FROM - (PublishData, Strain, PublishXRef, PublishFreeze) - left join PublishSE on - (PublishSE.DataId = PublishData.Id AND PublishSE.StrainId = PublishData.StrainId) - left join NStrain on - (NStrain.DataId = PublishData.Id AND - NStrain.StrainId = PublishData.StrainId) - WHERE - PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND - PublishData.Id = PublishXRef.DataId AND PublishXRef.Id = %s AND - PublishFreeze.Id = %s AND PublishData.StrainId = Strain.Id - Order BY - Strain.Name - """ - geno_query = """ - SELECT - Strain.Name, GenoData.value, GenoSE.error, "N/A", Strain.Name2 - FROM - (GenoData, GenoFreeze, Strain, Geno, GenoXRef) - left join GenoSE on - (GenoSE.DataId = GenoData.Id AND GenoSE.StrainId = GenoData.StrainId) - WHERE - Geno.SpeciesId = %s AND Geno.Name = %s AND GenoXRef.GenoId = Geno.Id AND - GenoXRef.GenoFreezeId = GenoFreeze.Id AND - GenoFreeze.Name = %s AND - GenoXRef.DataId = GenoData.Id AND - GenoData.StrainId = Strain.Id - Order BY - Strain.Name - """ - - probeset_query = """ - SELECT - Strain.Name, ProbeSetData.value, ProbeSetSE.error, NStrain.count, Strain.Name2 - FROM - (ProbeSetData, ProbeSetFreeze, - Strain, ProbeSet, ProbeSetXRef) - left join ProbeSetSE on - (ProbeSetSE.DataId = ProbeSetData.Id AND ProbeSetSE.StrainId = ProbeSetData.StrainId) - left join NStrain on - (NStrain.DataId = ProbeSetData.Id AND - NStrain.StrainId = ProbeSetData.StrainId) - WHERE - ProbeSet.Name = '%s' AND ProbeSetXRef.ProbeSetId = ProbeSet.Id AND - ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND - ProbeSetFreeze.Name = '%s' AND - ProbeSetXRef.DataId = ProbeSetData.Id AND - ProbeSetData.StrainId = Strain.Id - Order BY - Strain.Name - """ - - dataset_query["Publish"] = pheno_query - dataset_query["Geno"] = geno_query - dataset_query["ProbeSet"] = probeset_query - - return dataset_query.get(dataset_type) - - -@dataclass -class Dataset: - """class for creating datasets""" - name: Optional[str] = None - dataset_type: Optional[str] = None - dataset_id: int = -1 - - -def create_mrna_tissue_dataset(dataset_name, dataset_type): - """an mrna assay is a quantitative assessment(assay) associated\ - with an mrna trait.This used to be called probeset,but that term\ - only referes specifically to the afffymetrix platform and is\ - far too speficified""" - - return Dataset(name=dataset_name, dataset_type=dataset_type) - - -def dataset_type_getter(dataset_name, redis_instance=None) -> Optional[str]: - """given the dataset name fetch the type\ - of the dataset this in turn enables fetching\ - the creation of the correct object could utilize\ - redis for the case""" - - results = redis_instance.get(dataset_name, None) - - if results: - return results - - return fetch_dataset_type_from_gn2_api(dataset_name) - - -def fetch_dataset_type_from_gn2_api(dataset_name): - """this function is only called when the\ - the redis is empty and does have the specificied\ - dataset_type""" - # should only run once - - dataset_structure = {} - - map_dataset_to_new_type = { - "Phenotypes": "Publish", - "Genotypes": "Geno", - "MrnaTypes": "ProbeSet" - } - - data = json.loads(requests.get( - GN2_BASE_URL + "/api/v_pre1/gen_dropdown", timeout=5).content) - _name = dataset_name - for species in data['datasets']: - for group in data['datasets'][species]: - for dataset_type in data['datasets'][species][group]: - for dataset in data['datasets'][species][group][dataset_type]: - # assumes the first is dataset_short_name - short_dataset_name = next( - item for item in dataset if item != "None" and item is not None) - - dataset_structure[short_dataset_name] = map_dataset_to_new_type.get( - dataset_type, "MrnaTypes") - return dataset_structure - - -def dataset_creator_store(dataset_type): - """function contains key value pairs for\ - the function need to be called to create\ - each dataset_type""" - - dataset_obj = { - "ProbeSet": create_mrna_tissue_dataset - } - - return dataset_obj[dataset_type] - - -def create_dataset(dataset_type=None, dataset_name: str = None): - """function for creating new dataset temp not implemented""" - if dataset_type is None: - dataset_type = dataset_type_getter(dataset_name) - - dataset_creator = dataset_creator_store(dataset_type) - results = dataset_creator( - dataset_name=dataset_name, dataset_type=dataset_type) - return results - - -def fetch_dataset_sample_id(samplelist: List, database, species: str) -> dict: - """fetch the strain ids from the db only if\ - it is in the samplelist""" - # xtodo create an in clause for samplelist - - strain_query = """ - SELECT Strain.Name, Strain.Id FROM Strain, Species - WHERE Strain.Name IN {} - and Strain.SpeciesId=Species.Id - and Species.name = '{}' - """ - - database_cursor = database.cursor() - database_cursor.execute(strain_query.format(samplelist, species)) - - results = database_cursor.fetchall() - - return dict(results) - - -def divide_into_chunks(the_list, number_chunks): - """Divides a list into approximately number_chunks - >>> divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 3) - [[1, 2, 7], [3, 22, 8], [5, 22, 333]]""" - - length = len(the_list) - if length == 0: - return [[]] - - if length <= number_chunks: - number_chunks = length - chunk_size = int(ceil(length/number_chunks)) - chunks = [] - - for counter in range(0, length, chunk_size): - chunks.append(the_list[counter:counter+chunk_size]) - return chunks - - -def escape(string_): - """function escape sql value""" - return escape_string(string_).decode('utf8') - - -def mescape(*items) -> List: - """multiple escape for query values""" - - return [escape_string(str(item)).decode('utf8') for item in items] - - -def get_traits_data(sample_ids, database_instance, dataset_name, dataset_type): - """function to fetch trait data""" - # MySQL limits the number of tables that can be used in a join to 61, - # so we break the sample ids into smaller chunks - # Postgres doesn't have that limit, so we can get rid of this after we transition - - _trait_data = defaultdict(list) - chunk_size = 61 - number_chunks = int(ceil(len(sample_ids) / chunk_size)) - for sample_ids_step in divide_into_chunks(sample_ids, number_chunks): - if dataset_type == "Publish": - full_dataset_type = "Phenotype" - else: - full_dataset_type = dataset_type - temp = ['T%s.value' % item for item in sample_ids_step] - - if dataset_type == "Publish": - query = "SELECT {}XRef.Id,".format(escape(dataset_type)) - - else: - query = "SELECT {}.Name,".format(escape(full_dataset_type)) - - query += ', '.join(temp) - query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(full_dataset_type, - dataset_type, - dataset_type)) - for item in sample_ids_step: - - query += """ - left join {}Data as T{} on T{}.Id = {}XRef.DataId - and T{}.StrainId={}\n - """.format(*mescape(dataset_type, item, - item, dataset_type, item, item)) - - if dataset_type == "Publish": - query += """ - WHERE {}XRef.{}FreezeId = {}Freeze.Id - and {}Freeze.Name = '{}' - and {}.Id = {}XRef.{}Id - order by {}.Id - """.format(*mescape(dataset_type, dataset_type, - dataset_type, dataset_type, - dataset_name, full_dataset_type, - dataset_type, dataset_type, - full_dataset_type)) - - else: - query += """ - WHERE {}XRef.{}FreezeId = {}Freeze.Id - and {}Freeze.Name = '{}' - and {}.Id = {}XRef.{}Id - order by {}.Id - """.format(*mescape(dataset_type, dataset_type, - dataset_type, dataset_type, - dataset_name, dataset_type, - dataset_type, dataset_type, - full_dataset_type)) - - # print(query) - - _results = fetch_from_db_sample_data(query, database_instance) - - return [] - - -def get_probeset_trait_data(strain_ids: List, conn, dataset_name) -> dict: - """function for getting trait data\ - for probeset data type similar to\ - get trait data only difference is that\ - it uses sub queries""" - - trait_data: dict = {} - - trait_id_name = {} - - traits_query = """ - SELECT ProbeSetXRef.DataId,ProbeSet.Name FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) - WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id - and ProbeSetFreeze.Name = '{}' - and ProbeSet.Id = ProbeSetXRef.ProbeSetId - order by ProbeSet.Id - """.format(dataset_name) - - query = """ - SELECT * from ProbeSetData - where StrainID in ({}) - and id in (SELECT ProbeSetXRef.DataId FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) - WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id - and ProbeSetFreeze.Name = '{}' - and ProbeSet.Id = ProbeSetXRef.ProbeSetId - order by ProbeSet.Id) - """.format(",".join(str(strain_id) for strain_id in strain_ids), dataset_name) - - with conn: - cursor = conn.cursor() - cursor.execute(query) - _results = cursor.fetchall() - cursor.execute(traits_query) - trait_id_name = dict(cursor.fetchall()) - - for trait_id, _strain_id, strain_value in _results: - trait_name = trait_id_name[trait_id] - if trait_data.get(trait_name): - trait_data[trait_name].append(strain_value) - else: - trait_data[trait_name] = [] - - trait_data[trait_name].append(strain_value) - - return trait_data diff --git a/gn3/computations/traits.py b/gn3/computations/traits.py deleted file mode 100644 index 1aa2970..0000000 --- a/gn3/computations/traits.py +++ /dev/null @@ -1,56 +0,0 @@ -"""module contains all operating related to traits""" -from gn3.computations.datasets import retrieve_trait_sample_data - - -def fetch_trait(dataset, trait_name: str, database) -> dict: - """this method creates a trait by\ - fetching required data given the\ - dataset and trait_name""" - - created_trait = { - "dataset": dataset, - "trait_name": trait_name - } - - trait_data = get_trait_sample_data(dataset, trait_name, database) - - created_trait["trait_data"] = trait_data - - return created_trait - - -def get_trait_sample_data(trait_dataset, trait_name, database) -> dict: - """first try to fetch the traits sample data from redis if that\ - try to fetch from the traits dataset redis is only used for\ - temp dataset type which is not used in this case """ - - sample_results = retrieve_trait_sample_data( - trait_dataset, trait_name, database) - - trait_data = {} - - for (name, sample_value, _variance, _numcase, _name2) in sample_results: - - trait_data[name] = sample_value - return trait_data - - -def get_trait_info_data(trait_dataset, - trait_name: str, - database_instance, - get_qtl_info: bool = False) -> dict: - """given a dataset and trait_name return a dict containing all info\ - regarding the get trait""" - - _temp_var_holder = (trait_dataset, trait_name, - database_instance, get_qtl_info) - trait_info_data = { - "description": "", - "chr": "", - "locus": "", - "mb": "", - "abbreviation": "", - "trait_display_name": "" - - } - return trait_info_data diff --git a/tests/integration/test_datasets.py b/tests/integration/test_datasets.py deleted file mode 100644 index f97d970..0000000 --- a/tests/integration/test_datasets.py +++ /dev/null @@ -1,41 +0,0 @@ -"""This module contains integration tests for datasets""" -from unittest import TestCase -from unittest import mock - -from collections import namedtuple -from gn3.app import create_app - - -class DatasetIntegrationTests(TestCase): - """class contains integration tests for datasets""" - - def setUp(self): - self.app = create_app().test_client() - - @mock.patch("gn3.api.datasets.create_dataset") - def test_create_dataset(self, mock_dataset): - """Test for creating dataset object""" - mock_dataset_creator = namedtuple( - 'ProbeSet', ["dataset_name", "dataset_type"]) - new_dataset = mock_dataset_creator("HC_M2_0606_P", "ProbeSet") - mock_dataset.return_value = new_dataset - response = self.app.get( - "/api/dataset/create/HC_M2_0606_P/", follow_redirects=True) - mock_dataset.assert_called_once_with( - dataset_type=None, dataset_name="HC_M2_0606_P") - results = response.get_json()["dataset"] - self.assertEqual(results[1], "ProbeSet") - self.assertEqual(response.status_code, 200) - - @mock.patch("gn3.api.datasets.get_traits_data") - @mock.patch("gn3.api.datasets.database_connector") - def test_fetch_traits_data(self, mock_db, mock_get_trait_data): - """Test api/dataset/fetch_traits_data/d_name/d_type""" - - mock_get_trait_data.return_value = {} - mock_db.return_value = (mock.Mock(), mock.Mock()) - response = self.app.get( - "/api/dataset/fetch_traits_data/Aging-Brain-UCIPublish/Publish", follow_redirects=True) - - self.assertEqual(response.status_code, 200) - self.assertEqual(response.get_json(), {"results": {}}) diff --git a/tests/integration/test_traits.py b/tests/integration/test_traits.py deleted file mode 100644 index 410ba22..0000000 --- a/tests/integration/test_traits.py +++ /dev/null @@ -1,72 +0,0 @@ -"""module contains integration tests for trait endpoints""" -from unittest import TestCase -from unittest import mock - -from gn3.app import create_app - - -class TraitIntegrationTest(TestCase): - """class contains integration tests for\ - traits""" - - def setUp(self): - self.app = create_app().test_client() - - @mock.patch("gn3.api.traits.fetch_trait") - @mock.patch("gn3.api.traits.database_connector") - def test_create_trait(self, mock_database, mock_fetch_trait): - """test the endpoint for creating traits\ - endpoint requires trait name and dataset name""" - mock_database.return_value = (mock.Mock(), mock.Mock()) - trait_results = { - "dataset": None, - "trait_name": "1449593_at", - "trait_data": { - "BXD11": 8.464, - "BXD12": 8.414, - "BXD13": 8.753, - "BXD15": 8.5, - "BXD16": 8.832 - } - - } - mock_fetch_trait.return_value = trait_results - - results = self.app.get( - "/api/trait/1449593_at/HC_M2_0606_P", follow_redirects=True) - - trait_data = results.get_json() - - self.assertEqual(mock_database.call_count, 1) - self.assertEqual(results.status_code, 200) - self.assertEqual(trait_data, trait_results) - - @mock.patch("gn3.api.traits.get_trait_info_data") - def test_retrieve_trait_info(self, mock_get_trait_info): - """integration test for endpoints for retrieving\ - trait info expects the dataset of trait to have been - created""" - - trait_post_data = { - "trait": {"trait_name": ""}, - "trait_dataset": {"dataset_name": ""} - } - - expected_api_results = { - "description": "trait description", - "chr": "", - "locus": "", - "mb": "", - "abbreviation": "trait_abbreviation", - "trait_display_name": "trait_name" - - } - mock_get_trait_info.return_value = expected_api_results - - trait_info = self.app.post( - "/api/trait/trait_info/144_at", json=trait_post_data, follow_redirects=True) - - trait_info_results = trait_info.get_json() - - self.assertEqual(trait_info.status_code, 200) - self.assertEqual(trait_info_results, expected_api_results) diff --git a/tests/unit/computations/test_datasets.py b/tests/unit/computations/test_datasets.py deleted file mode 100644 index f9e9c2b..0000000 --- a/tests/unit/computations/test_datasets.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Module contains tests from datasets""" -import json - -from unittest import TestCase -from unittest import mock - -from collections import namedtuple - -from gn3.computations.datasets import retrieve_trait_sample_data -from gn3.computations.datasets import get_query_for_dataset_sample -from gn3.computations.datasets import fetch_from_db_sample_data -from gn3.computations.datasets import create_dataset -from gn3.computations.datasets import dataset_creator_store -from gn3.computations.datasets import dataset_type_getter -from gn3.computations.datasets import fetch_dataset_type_from_gn2_api -from gn3.computations.datasets import fetch_dataset_sample_id -from gn3.computations.datasets import divide_into_chunks -from gn3.computations.datasets import get_traits_data - - -class TestDatasets(TestCase): - """Class contains tests for datasets""" - - @mock.patch("gn3.computations.datasets.fetch_from_db_sample_data") - def test_retrieve_trait_sample_data(self, mock_fetch_sample_results): - """Test retrieving sample data\ - for trait from the dataset - """ - trait_name = "1419792_at" - dataset_id = "HC_M2_0606_P&" - dataset_type = "Publish" - - database = mock.Mock() - - dataset = { - "id": dataset_id, - "type": dataset_type, - "name": dataset_id - } - - fetch_results = [('BXD32', 8.001, None, None, 'BXD32')] - - mock_fetch_sample_results.return_value = fetch_results - - results = retrieve_trait_sample_data( - dataset, trait_name, database) - self.assertEqual(mock_fetch_sample_results.call_count, 1) - self.assertEqual(results, fetch_results) - - def test_query_for_dataset_sample(self): - """Test for getting query for sample data""" - - no_results = get_query_for_dataset_sample("does not exists") - - query_exists = get_query_for_dataset_sample("Publish") - - self.assertEqual(no_results, None) - self.assertIsInstance(query_exists, str) - - def test_fetch_from_db_sample_data(self): - """Test for function that fetches sample\ - results from the database - """ - - database_results = [('BXD31', 8.001, None, None, 'BXD31'), - ('BXD32', 7.884, None, None, 'BXD32'), - ('BXD42', 7.682, None, None, 'BXD42'), - ('BXD42', 7.682, None, None, 'BXD42'), - ('BXD40', 7.945, None, None, 'BXD40'), - ('BXD43', 7.873, None, None, 'BXD43') - ] - - database = mock.Mock() - db_cursor = mock.Mock() - db_cursor.execute.return_value = 6 - db_cursor.fetchall.return_value = database_results - database.cursor.return_value = db_cursor - - mock_pheno_query = """ - SELECT - Strain.Name, PublishData.value, PublishSE.error,NStrain.count, Strain.Name2 - WHERE - PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND - PublishData.Id = PublishXRef.DataId AND PublishXRef.Id = 1419792_at AND - PublishFreeze.Id = '12' AND PublishData.StrainId = Strain.Id - Order BY - Strain.Name - """ - fetch_results = fetch_from_db_sample_data(mock_pheno_query, database) - - self.assertEqual(fetch_results, database_results) - - @mock.patch("gn3.computations.datasets.dataset_creator_store") - @mock.patch("gn3.computations.datasets.dataset_type_getter") - def test_create_dataset(self, mock_dataset_type, mock_store): - """Test function that creates/fetches required dataset\ - can either be published phenotype,genotype,Microarray or\ - user defined ->Temp - """ - probe_name = "HC_M2_0606_P" - probe_type = "ProbeSet" - - mock_dataset_creator = namedtuple( - 'ProbeSet', ["dataset_name", "dataset_type"]) - - mock_store.return_value = mock_dataset_creator - mock_dataset_type.return_value = probe_type - dataset = create_dataset( - dataset_type=None, dataset_name=probe_name) - - self.assertEqual(dataset.dataset_name, probe_name) - self.assertEqual(dataset.dataset_type, probe_type) - - def test_dataset_creator_store(self): - """Test for functions that actual - function to create differerent \ - datasets - """ - results = dataset_creator_store("ProbeSet") - - self.assertTrue(results) - - def test_dataset_type_getter(self): - """Test for fetching type of dataset given\ - the dataset name - """ - - redis_instance = mock.Mock() - # fetched in redis - redis_instance.get.return_value = "ProbeSet" - results = dataset_type_getter("HC_M2_0_P", redis_instance) - self.assertEqual(results, "ProbeSet") - - @mock.patch("gn3.computations.datasets.requests") - def test_fetch_dataset_type_from_gn2_api(self, mock_request): - """Test for function that test fetching\ - all datasets from gn2 api in order to store\ - in redis - """ - - expected_json_results = {"datasets": { - "arabidopsis": { - "BayXSha": { - "Genotypes": [ - [ - "None", - "BayXShaGeno", - "BayXSha Genotypes" - ] - ], - "Phenotypes": [ - [ - "642", - "BayXShaPublish", - "BayXSha Published Phenotypes" - ] - ] - } - } - }} - - request_results = json.dumps(expected_json_results) - mock_request.get.return_value.content = request_results - results = fetch_dataset_type_from_gn2_api("HC_M2_0_P") - expected_results = { - "BayXShaGeno": "Geno", - "642": "Publish" - } - - self.assertEqual(expected_results, results) - - def test_fetch_dataset_sample_id(self): - """Get from the database the sample\ - id if only in the samplelists - """ - - expected_results = {"B6D2F1": 1, "BXD1": 4, "BXD11": 10, - "BXD12": 11, "BXD13": 12, "BXD15": 14, "BXD16": 15} - - database_instance = mock.Mock() - database_cursor = mock.Mock() - - database_cursor.execute.return_value = 5 - database_cursor.fetchall.return_value = list(expected_results.items()) - database_instance.cursor.return_value = database_cursor - strain_list = ["B6D2F1", "BXD1", "BXD11", - "BXD12", "BXD13", "BXD16", "BXD15"] - - results = fetch_dataset_sample_id( - samplelist=strain_list, database=database_instance, species="mouse") - - self.assertEqual(results, expected_results) - - @mock.patch("gn3.computations.datasets.fetch_from_db_sample_data") - @mock.patch("gn3.computations.datasets.divide_into_chunks") - def test_get_traits_data(self, mock_divide_into_chunks, mock_fetch_samples): - """Test for for function to get data\ - of traits in dataset - """ - _expected_results = {'AT_DSAFDS': [ - 12, 14, 13, 23, 12, 14, 13, 23, 12, 14, 13, 23]} - database = mock.Mock() - sample_id = [1, 2, 7, 3, 22, 8] - mock_divide_into_chunks.return_value = [ - [1, 2, 7], [3, 22, 8], [5, 22, 333]] - mock_fetch_samples.return_value = ("AT_DSAFDS", 12, 14, 13, 23) - results = get_traits_data(sample_id, database, "HC_M2", "Publish") - - self.assertEqual({}, dict(results)) - - def test_divide_into_chunks(self): - """Test for dividing a list into given number of\ - chunks for example - """ - results = divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 3) - - expected_results = [[1, 2, 7], [3, 22, 8], [5, 22, 333]] - - self.assertEqual(results, expected_results) diff --git a/tests/unit/computations/test_trait.py b/tests/unit/computations/test_trait.py deleted file mode 100644 index feb97c6..0000000 --- a/tests/unit/computations/test_trait.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Module contains tests for creating traits""" -from unittest import TestCase -from unittest import mock - -from gn3.computations.traits import fetch_trait -from gn3.computations.traits import get_trait_sample_data -from gn3.computations.traits import get_trait_info_data - - -class TestTrait(TestCase): - """Class contains tests for creating traits""" - - @mock.patch("gn3.computations.traits.get_trait_sample_data") - def test_fetch_trait(self, get_sample_data): - """Test for creating/fetching trait""" - - expected_sample_data = { - "A/Y": 12.3, - "WQC": 11.1 - } - - database = mock.Mock() - - get_sample_data.return_value = expected_sample_data - - expected_trait = { - "trait_name": "AXFDSF_AT", - "dataset": None, - "trait_data": expected_sample_data - } - results = fetch_trait(dataset=None, - trait_name="AXFDSF_AT", - database=database) - - self.assertEqual(results, expected_trait) - self.assertEqual(get_sample_data.call_count, 1) - - @mock.patch("gn3.computations.traits.retrieve_trait_sample_data") - def test_get_trait_sample_data(self, mock_retrieve_sample_data): - """Test for getting sample data from either\ - the trait's dataset or form redis - """ - - trait_dataset = mock.Mock() - dataset_trait_sample_data = [ - ('129S1/SvImJ', 7.433, None, None, '129S1/SvImJ'), - ('A/J', 7.596, None, None, 'A/J'), - ('AKR/J', 7.774, None, None, 'AKR/J'), - ('B6D2F1', 7.707, None, None, 'B6D2F1')] - mock_retrieve_sample_data.return_value = dataset_trait_sample_data - - trait_name = "1426679_at" - - database = mock.Mock() - - results = get_trait_sample_data( - trait_dataset, trait_name, database) - - expected_results = { - "129S1/SvImJ": 7.433, - "A/J": 7.596, - "AKR/J": 7.774, - "B6D2F1": 7.707 - } - - self.assertEqual(results, expected_results) - - def test_get_trait_info_data(self): - """Test for getting info data related\ - to trait - """ - - results = get_trait_info_data( - trait_name="AXSF_AT", trait_dataset=mock.Mock(), database_instance=None) - expected_trait_info = { - "description": "", - "trait_display_name": "", - "abbreviation": "", - "chr": "", - "mb": "", - "locus": "" - } - - self.assertEqual(results, expected_trait_info) -- cgit v1.2.3 From a1b1fdce9c92fd84e97310c79c17e7b1c74bff07 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 3 May 2021 10:11:05 +0300 Subject: replace database with conn --- gn3/computations/correlations.py | 3 ++- tests/unit/computations/test_correlation.py | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) (limited to 'tests') diff --git a/gn3/computations/correlations.py b/gn3/computations/correlations.py index 4432971..3563530 100644 --- a/gn3/computations/correlations.py +++ b/gn3/computations/correlations.py @@ -99,7 +99,8 @@ def do_bicor(x_val, y_val) -> Tuple[float, float]: package :not packaged in guix """ - return (x_val, y_val) + _corr_input = (x_val, y_val) + return (0.0, 0.0) def filter_shared_sample_keys(this_samplelist, diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index 8bb5cd1..c6fa35e 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -104,7 +104,7 @@ class TestCorrelation(TestCase): results = do_bicor(x_val=[1, 2, 3], y_val=[4, 5, 6]) - self.assertEqual(results, ([1, 2, 3], [4, 5, 6]) + self.assertEqual(results, (0.0, 0.0) ) @mock.patch("gn3.computations.correlations.compute_corr_coeff_p_value") @@ -291,10 +291,10 @@ class TestCorrelation(TestCase): expected_db_results = [namedtuple("lit_coeff", "val")(x*0.1) for x in range(1, 4)] - database_instance = DataBase(expected_results=expected_db_results) + conn = DataBase(expected_results=expected_db_results) expected_results = ("1", 0.1) - lit_results = fetch_lit_correlation_data(conn=database_instance, + lit_results = fetch_lit_correlation_data(conn=conn, gene_id="1", input_mouse_gene_id="20", mouse_gene_id="15") @@ -305,11 +305,11 @@ class TestCorrelation(TestCase): """Test that corr coeffient returned is 0 given the\ db value if corr coefficient is empty """ - database_instance = mock.Mock() - database_instance.cursor.return_value = DataBase() - database_instance.execute.return_value.fetchone.return_value = None + conn = mock.Mock() + conn.cursor.return_value = DataBase() + conn.execute.return_value.fetchone.return_value = None - lit_results = fetch_lit_correlation_data(conn=database_instance, + lit_results = fetch_lit_correlation_data(conn=conn, input_mouse_gene_id="12", gene_id="16", mouse_gene_id="12") @@ -356,7 +356,7 @@ class TestCorrelation(TestCase): """Test for converting a gene id to mouse geneid\ given a species which is not mouse """ - database_instance = mock.Mock() + conn = mock.Mock() test_data = [("Human", 14), (None, 9), ("Mouse", 15), ("Rat", 14)] database_results = [namedtuple("mouse_id", "mouse")(val) @@ -365,12 +365,12 @@ class TestCorrelation(TestCase): cursor = mock.Mock() cursor.execute.return_value = 1 cursor.fetchone.side_effect = database_results - database_instance.cursor.return_value = cursor + conn.cursor.return_value = cursor expected_results = [12, None, 13, 14] for (species, gene_id) in test_data: mouse_gene_id_results = map_to_mouse_gene_id( - conn=database_instance, species=species, gene_id=gene_id) + conn=conn, species=species, gene_id=gene_id) results.append(mouse_gene_id_results) self.assertEqual(results, expected_results) @@ -382,7 +382,7 @@ class TestCorrelation(TestCase): and is used in the api/correlation/lit """ - database = mock.Mock() + conn = mock.Mock() expected_mocked_lit_results = [{"1412_at": {"gene_id": 11, "lit_corr": 0.9}}, {"1412_a": { "gene_id": 17, "lit_corr": 0.48}}] @@ -390,7 +390,7 @@ class TestCorrelation(TestCase): mock_lit_corr.return_value = expected_mocked_lit_results lit_correlation_results = compute_all_lit_correlation( - conn=database, trait_lists=[("1412_at", 11), ("1412_a", 121)], + conn=conn, trait_lists=[("1412_at", 11), ("1412_a", 121)], species="rat", gene_id=12) self.assertEqual(lit_correlation_results, expected_mocked_lit_results) -- cgit v1.2.3 From ef55d9769c50e12af6252f9fae78f5aa3bf42670 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 3 May 2021 10:43:07 +0300 Subject: minor fixes for tiss correlation tests and naming --- gn3/computations/correlations.py | 28 ++++++++++------------------ tests/unit/computations/test_correlation.py | 8 ++++---- 2 files changed, 14 insertions(+), 22 deletions(-) (limited to 'tests') diff --git a/gn3/computations/correlations.py b/gn3/computations/correlations.py index 3563530..065a1ed 100644 --- a/gn3/computations/correlations.py +++ b/gn3/computations/correlations.py @@ -226,6 +226,7 @@ def tissue_correlation_for_trait_list( primary_tissue_vals: List, target_tissues_values: List, corr_method: str, + trait_id: str, compute_corr_p_value: Callable = compute_corr_coeff_p_value) -> dict: """Given a primary tissue values for a trait and the target tissues values compute the correlation_cooeff and p value the input required are arrays @@ -241,13 +242,12 @@ def tissue_correlation_for_trait_list( target_values=target_tissues_values, corr_method=corr_method) - lit_corr_result = { + tiss_corr_result = {trait_id: { "tissue_corr": tissue_corr_coeffient, "tissue_number": len(primary_tissue_vals), - "p_value": p_value - } + "p_value": p_value}} - return lit_corr_result + return tiss_corr_result def fetch_lit_correlation_data( @@ -432,9 +432,9 @@ def process_trait_symbol_dict(trait_symbol_dict, symbol_tissue_vals_dict) -> Lis return traits_tissue_vals -def experimental_compute_all_tissue_correlation(primary_tissue_dict: dict, - target_tissues_data: dict, - corr_method: str): +def compute_tissue_correlation(primary_tissue_dict: dict, + target_tissues_data: dict, + corr_method: str): """Experimental function that uses multiprocessing\ for computing tissue correlation """ @@ -450,25 +450,17 @@ def experimental_compute_all_tissue_correlation(primary_tissue_dict: dict, processed_values = [] for target_tissue_obj in target_tissues_list: + trait_id = target_tissue_obj.get("trait_id") target_tissue_vals = target_tissue_obj.get("tissue_values") processed_values.append( - (primary_tissue_vals, target_tissue_vals, corr_method)) + (primary_tissue_vals, target_tissue_vals, corr_method, trait_id)) with multiprocessing.Pool() as pool: results = pool.starmap( tissue_correlation_for_trait_list, processed_values) for result in results: - tissue_result_dict = {"trait_name": result} - tissues_results.append(tissue_result_dict) - - # tissue_result = tissue_correlation_for_trait_list( - # primary_tissue_vals=primary_tissue_vals, - # target_tissues_values=target_tissue_vals, - # corr_method=corr_method) - - # tissue_result_dict = {trait_id: tissue_result} - # tissues_results.append(tissue_result_dict) + tissues_results.append(result) return sorted( tissues_results, diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index c6fa35e..6414c3b 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -235,12 +235,12 @@ class TestCorrelation(TestCase): primary_tissue_values = [1.1, 1.5, 2.3] target_tissues_values = [1, 2, 3] mock_compute_corr_coeff.side_effect = [(0.4, 0.9), (-0.2, 0.91)] - expected_tissue_results = { - 'tissue_corr': 0.4, 'p_value': 0.9, "tissue_number": 3} - + expected_tissue_results = {"1456_at": {"tissue_corr": 0.4, + "p_value": 0.9, "tissue_number": 3}} tissue_results = tissue_correlation_for_trait_list( primary_tissue_values, target_tissues_values, - corr_method="pearson", compute_corr_p_value=mock_compute_corr_coeff) + corr_method="pearson", trait_id="1456_at", + compute_corr_p_value=mock_compute_corr_coeff) self.assertEqual(tissue_results, expected_tissue_results) -- cgit v1.2.3