From 77312535e643e4c8fecd7c20b3381996808dea11 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Thu, 29 Jul 2021 14:09:49 +0300 Subject: Add partial type annotations for slink module Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi * Add some type annotations for the `nearest` function. * Leave some comments regarding the issues experienced when trying to add some typing annotations to the function to help with future endeavours of the same. --- gn3/computations/slink.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'gn3/computations') diff --git a/gn3/computations/slink.py b/gn3/computations/slink.py index 4aac6b3..23d3d88 100644 --- a/gn3/computations/slink.py +++ b/gn3/computations/slink.py @@ -7,6 +7,10 @@ slink: TODO: Describe what the function does... """ import logging +from typing import List, Tuple, Union, Sequence + +NumType = Union[int, float] +SeqOfNums = Sequence[NumType] class LengthError(BaseException): """Raised whenever child lists/tuples are not the same length as the parent @@ -73,7 +77,10 @@ raise an exception.""" def __flatten_list_of_lists(parent): return [item for child in parent for item in child] -def nearest(lists, i, j): +# i and j are Union[SeqOfNums, NumType], but that leads to errors where the +# values of i or j are indexed, since the NumType type is not indexable. +# I don't know how to type this so that it does not fail on running `mypy .` +def nearest(lists: Sequence[SeqOfNums], i, j) -> NumType: """ Computes shortest distance between member(s) in `i` and member(s) in `j`. @@ -126,6 +133,10 @@ def nearest(lists, i, j): raise ValueError("member values (i or j) should be lists/tuples of integers or integers") +# `lists` here could be Sequence[SeqOfNums], but that leads to errors I do not +# understand down the line +# Might have to re-implement the function especially since the errors are thrown +# where `listindexcopy` is involved def slink(lists): """ DESCRIPTION: -- cgit v1.2.3 From f712da630c1a3642cb44b62c4b2b857373cd78d7 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Wed, 4 Aug 2021 11:30:44 +0300 Subject: Fix issues caught by pylint * gn3/computations/slink.py: remove unused imports * gn3/db/traits.py: remove unnecessary `else` clauses * tests/unit/db/test_traits.py: add docstrings for functions --- gn3/computations/slink.py | 2 +- gn3/db/traits.py | 6 +++--- tests/unit/db/test_traits.py | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) (limited to 'gn3/computations') diff --git a/gn3/computations/slink.py b/gn3/computations/slink.py index 23d3d88..5953e6b 100644 --- a/gn3/computations/slink.py +++ b/gn3/computations/slink.py @@ -7,7 +7,7 @@ slink: TODO: Describe what the function does... """ import logging -from typing import List, Tuple, Union, Sequence +from typing import Union, Sequence NumType = Union[int, float] SeqOfNums = Sequence[NumType] diff --git a/gn3/db/traits.py b/gn3/db/traits.py index ea35d7e..29c91a6 100644 --- a/gn3/db/traits.py +++ b/gn3/db/traits.py @@ -166,8 +166,7 @@ def set_confidential_field(trait_info): "confidential": 1 if ( trait_info.get("pre_publication_description", None) and not trait_info.get("pubmed_id", None)) else 0} - else: - return trait_info + return trait_info def retrieve_probeset_trait_info(trait_data_source: Dict[str, Any], conn: Any): """Retrieve trait information for type `ProbeSet` traits. @@ -344,13 +343,14 @@ def set_riset_fields(trait_info, conn): def retrieve_trait_info( trait_type: str, trait_name: str, trait_dataset_id: int, - trait_dataset_name: str, conn: Any, QTL=None): + trait_dataset_name: str, conn: Any, qtl=None): """Retrieves the trait information. https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L397-L456 This function, or the dependent functions, might be incomplete as they are currently.""" + # pylint: disable=[R0913] trait_info_function_table = { "Publish": retrieve_publish_trait_info, "ProbeSet": retrieve_probeset_trait_info, diff --git a/tests/unit/db/test_traits.py b/tests/unit/db/test_traits.py index 2445d26..1c481a2 100644 --- a/tests/unit/db/test_traits.py +++ b/tests/unit/db/test_traits.py @@ -207,6 +207,7 @@ class TestTraitsDBFunctions(TestCase): ) def test_set_haveinfo_field(self): + """Test that the `haveinfo` field is set up correctly""" for trait_info, expected in [ [{}, {"haveinfo": 0}], [{"k1": "v1"}, {"k1": "v1", "haveinfo": 1}]]: @@ -214,6 +215,7 @@ class TestTraitsDBFunctions(TestCase): self.assertEqual(set_haveinfo_field(trait_info), expected) def test_set_homologene_id_field(self): + """Test that the `homologene_id` field is set up correctly""" for trait_info, expected in [ [{"type": "Publish"}, {"type": "Publish", "homologeneid": None}], @@ -229,6 +231,7 @@ class TestTraitsDBFunctions(TestCase): set_homologene_id_field(trait_info, db_mock), expected) def test_set_confidential_field(self): + """Test that the `confidential` field is set up correctly""" for trait_info, expected in [ [{"type": "Publish"}, {"type": "Publish", "confidential": 0}], [{"type": "ProbeSet"}, {"type": "ProbeSet"}], -- cgit v1.2.3 From 72ab476e5825c8c2b0d5102d6f1227ace8f7fa68 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 9 Aug 2021 18:34:46 +0300 Subject: Build up the heatmap data Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi * Add code to compute and organise the data that will be used to draw the final heatmap. This varies significantly in how it works from the original, but it still tries to retain the general flow of data. --- gn3/computations/heatmap.py | 173 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 gn3/computations/heatmap.py (limited to 'gn3/computations') diff --git a/gn3/computations/heatmap.py b/gn3/computations/heatmap.py new file mode 100644 index 0000000..98ea26f --- /dev/null +++ b/gn3/computations/heatmap.py @@ -0,0 +1,173 @@ +""" +This module will contain functions to be used in computation of the data used to +generate various kinds of heatmaps. +""" + +from gn3.computations.slink import slink +from gn3.computations.correlations2 import compute_correlation + +def export_trait_data( + trait_data: dict, strainlist: Sequence[str], dtype: str="val", + var_exists: bool=False, n_exists: bool=False): + """ + Export data according to `strainlist`. Mostly used in calculating + correlations. + + DESCRIPTION: + Migrated from + https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L166-L211 + + PARAMETERS + trait: (dict) + The dictionary of key-value pairs representing a trait + strainlist: (list) + A list of strain names + type: (str) + ... verify what this is ... + var_exists: (bool) + A flag indicating existence of variance + n_exists: (bool) + A flag indicating existence of ndata + """ + def __export_all_types(tdata, strain): + sample_data = [] + if tdata[strain]["val"]: + sample_data.append(tdata[strain]["val"]) + if var_exists: + if tdata[strain].var: + sample_data.append(tdata[strain]["var"]) + else: + sample_data.append(None) + if n_exists: + if tdata[strain]["ndata"]: + sample_data.append(tdata[strain]["ndata"]) + else: + sample_data.append(None) + else: + if var_exists and n_exists: + sample_data += [None, None, None] + elif var_exists or n_exists: + sample_data += [None, None] + else: + sample_data.append(None) + + return tuple(sample_data) + + def __exporter(accumulator, strain): + if tdata.has_key(strain): + if dtype == "val": + return accumulator + (tdata[strain]["val"], ) + if dtype == "var": + return accumulator + (tdata[strain]["var"], ) + if dtype == "N": + return tdata[strain]["ndata"] + if dtype == "all": + return accumulator + __export_all_types( + accumulator, tdata, strain) + else: + raise KeyError("Type `%s` is incorrect" % dtype) + else: + if var_exists and n_exists: + return accumulator + (None, None, None) + if var_exists or n_exists: + return accumulator + (None, None) + return accumulator + (None,) + + return reduce(__exporter(strain), strainlist, tuple()) + +def trait_display_name(trait: Dict): + """ + Given a trait, return a name to use to display the trait on a heatmap. + + DESCRIPTION + Migrated from + https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L141-L157 + """ + if trait.get("db", None) and trait.get("trait_name", None): + if trait["db"]["dataset_type"] == "Temp": + desc = trait["description"] + if desc.find("PCA") >= 0: + return "%s::%s" % ( + trait["db"]["displayname"], + desc[desc.rindex(':')+1:].strip()) + return "%s::%s" % ( + trait["db"]["displayname"], + desc[:desc.index('entered')].strip()) + else: + prefix = "%s::%s" % ( + trait["db"]["dataset_name"], trait["trait_name"]) + if trait["cellid"]: + return "%s::%s" % (prefix, trait["cellid"]) + return prefix + return trait["description"] + +def cluster_traits(traits_data_list: Sequence[Dict]): + """ + Clusters the trait values. + + DESCRIPTION + Attempts to replicate the clustering of the traits, as done at + https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/heatmap/Heatmap.py#L138-L162 + """ + def __compute_corr(tdata_i, tdata_j): + if tdata_j[0] < tdata_i[0]: + corr, nOverlap = compute_correlation(tdata_i, tdata_j) + if (1 - corr) < 0: + return 0.0 + return 1 - corr + return 0.0 + + def __cluster(tdata_i): + res2 = tuple( + __compute_corr(tdata_i, tdata_j) for tdata_j in enumerate(traits)) + + return tuple(__cluster(tdata_i) for tdata_i in enumerate(traits_data_list)) + +def heatmap_data( + fd, search_result, conn: Any, colorScheme=None, userPrivilege=None, + userName=None): + """ + heatmap function + + DESCRIPTION + This function is an attempt to reproduce the initialisation at + https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/heatmap/Heatmap.py#L46-L64 + and also the clustering and slink computations at + https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/heatmap/Heatmap.py#L138-L165 + with the help of the `gn3.computations.heatmap.cluster_traits` function. + + It does not try to actually draw the heatmap image. + + PARAMETERS: + TODO: Elaborate on the parameters here... + """ + cluster_checked = fd.formdata.getvalue("clusterCheck", "") + strainlist = [strain for strain in fd.strainlist if strain not in fd.parlist] + genotype = fd.genotype + + def __retrieve_traitlist_and_datalist(threshold, fullname): + trait = retrieve_trait_info(threshold, fullname, conn) + return (trait, export_trait_data(retrieve_trait_data(trait), strainlist)) + + traits_details = [ + __retrieve_traitlist_and_datalist(threshold, fullname) + for fullname in search_result] + traits_list = map(lambda x: x[0], traits_details) + traits_data_list = map(lambda x: x[1], traits_details) + + return { + "target_description_checked": fd.formdata.getvalue( + "targetDescriptionCheck", ""), + "cluster_checked": cluster_checked, + "slink_data": ( + slink(cluster_traits(traits_list, strainlist)) + if cluster_checked else False) + "sessionfile": fd.formdata.getvalue("session"), + "genotype": genotype, + "nLoci": sum(map(lambda x: len(x), genotype)) + "strainlist": strainlist, + "ppolar": fd.ppolar, + "mpolar":fd.mpolar, + "traits_list": traits_list + "traits_data_list": traits_data_list + } -- cgit v1.2.3 From d97e3141554c3a13a1ec163373b19b9552a79fb0 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Thu, 12 Aug 2021 13:08:05 +0300 Subject: Import missing definitions Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi * Import some missing definitions. --- gn3/computations/heatmap.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'gn3/computations') diff --git a/gn3/computations/heatmap.py b/gn3/computations/heatmap.py index 98ea26f..1b7dfb7 100644 --- a/gn3/computations/heatmap.py +++ b/gn3/computations/heatmap.py @@ -3,12 +3,15 @@ This module will contain functions to be used in computation of the data used to generate various kinds of heatmaps. """ +from functools import reduce +from typing import Any, Dict, Sequence from gn3.computations.slink import slink +from gn3.db.traits import retrieve_trait_data, retrieve_trait_info from gn3.computations.correlations2 import compute_correlation def export_trait_data( - trait_data: dict, strainlist: Sequence[str], dtype: str="val", - var_exists: bool=False, n_exists: bool=False): + trait_data: dict, strainlist: Sequence[str], dtype: str = "val", + var_exists: bool = False, n_exists: bool = False): """ Export data according to `strainlist`. Mostly used in calculating correlations. -- cgit v1.2.3 From 3420e378a614f1ecec85f633cd9f202764a54eda Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Thu, 12 Aug 2021 13:09:32 +0300 Subject: Fix linting errors Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi * Fix some errors caught by the linter. --- gn3/computations/heatmap.py | 79 +++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 39 deletions(-) (limited to 'gn3/computations') diff --git a/gn3/computations/heatmap.py b/gn3/computations/heatmap.py index 1b7dfb7..a0e778a 100644 --- a/gn3/computations/heatmap.py +++ b/gn3/computations/heatmap.py @@ -57,26 +57,24 @@ def export_trait_data( return tuple(sample_data) def __exporter(accumulator, strain): - if tdata.has_key(strain): + # pylint: disable=[R0911] + if trait_data.has_key(strain): if dtype == "val": - return accumulator + (tdata[strain]["val"], ) + return accumulator + (trait_data[strain]["val"], ) if dtype == "var": - return accumulator + (tdata[strain]["var"], ) + return accumulator + (trait_data[strain]["var"], ) if dtype == "N": - return tdata[strain]["ndata"] + return trait_data[strain]["ndata"] if dtype == "all": - return accumulator + __export_all_types( - accumulator, tdata, strain) - else: - raise KeyError("Type `%s` is incorrect" % dtype) - else: - if var_exists and n_exists: - return accumulator + (None, None, None) - if var_exists or n_exists: - return accumulator + (None, None) - return accumulator + (None,) + return accumulator + __export_all_types(trait_data, strain) + raise KeyError("Type `%s` is incorrect" % dtype) + if var_exists and n_exists: + return accumulator + (None, None, None) + if var_exists or n_exists: + return accumulator + (None, None) + return accumulator + (None,) - return reduce(__exporter(strain), strainlist, tuple()) + return reduce(__exporter, strainlist, tuple()) def trait_display_name(trait: Dict): """ @@ -96,12 +94,11 @@ def trait_display_name(trait: Dict): return "%s::%s" % ( trait["db"]["displayname"], desc[:desc.index('entered')].strip()) - else: - prefix = "%s::%s" % ( - trait["db"]["dataset_name"], trait["trait_name"]) - if trait["cellid"]: - return "%s::%s" % (prefix, trait["cellid"]) - return prefix + prefix = "%s::%s" % ( + trait["db"]["dataset_name"], trait["trait_name"]) + if trait["cellid"]: + return "%s::%s" % (prefix, trait["cellid"]) + return prefix return trait["description"] def cluster_traits(traits_data_list: Sequence[Dict]): @@ -114,21 +111,21 @@ def cluster_traits(traits_data_list: Sequence[Dict]): """ def __compute_corr(tdata_i, tdata_j): if tdata_j[0] < tdata_i[0]: - corr, nOverlap = compute_correlation(tdata_i, tdata_j) + corr_vals = compute_correlation(tdata_i, tdata_j) + corr = corr_vals[0] if (1 - corr) < 0: return 0.0 return 1 - corr return 0.0 def __cluster(tdata_i): - res2 = tuple( - __compute_corr(tdata_i, tdata_j) for tdata_j in enumerate(traits)) + return tuple( + __compute_corr(tdata_i, tdata_j) + for tdata_j in enumerate(traits_data_list)) return tuple(__cluster(tdata_i) for tdata_i in enumerate(traits_data_list)) -def heatmap_data( - fd, search_result, conn: Any, colorScheme=None, userPrivilege=None, - userName=None): +def heatmap_data(formd, search_result, conn: Any): """ heatmap function @@ -144,13 +141,17 @@ def heatmap_data( PARAMETERS: TODO: Elaborate on the parameters here... """ - cluster_checked = fd.formdata.getvalue("clusterCheck", "") - strainlist = [strain for strain in fd.strainlist if strain not in fd.parlist] - genotype = fd.genotype + threshold = 0 # webqtlConfig.PUBLICTHRESH + cluster_checked = formd.formdata.getvalue("clusterCheck", "") + strainlist = [ + strain for strain in formd.strainlist if strain not in formd.parlist] + genotype = formd.genotype def __retrieve_traitlist_and_datalist(threshold, fullname): trait = retrieve_trait_info(threshold, fullname, conn) - return (trait, export_trait_data(retrieve_trait_data(trait), strainlist)) + return ( + trait, + export_trait_data(retrieve_trait_data(trait, conn), strainlist)) traits_details = [ __retrieve_traitlist_and_datalist(threshold, fullname) @@ -159,18 +160,18 @@ def heatmap_data( traits_data_list = map(lambda x: x[1], traits_details) return { - "target_description_checked": fd.formdata.getvalue( + "target_description_checked": formd.formdata.getvalue( "targetDescriptionCheck", ""), "cluster_checked": cluster_checked, "slink_data": ( - slink(cluster_traits(traits_list, strainlist)) - if cluster_checked else False) - "sessionfile": fd.formdata.getvalue("session"), + slink(cluster_traits(traits_data_list)) + if cluster_checked else False), + "sessionfile": formd.formdata.getvalue("session"), "genotype": genotype, - "nLoci": sum(map(lambda x: len(x), genotype)) + "nLoci": sum(map(len, genotype)), "strainlist": strainlist, - "ppolar": fd.ppolar, - "mpolar":fd.mpolar, - "traits_list": traits_list + "ppolar": formd.ppolar, + "mpolar":formd.mpolar, + "traits_list": traits_list, "traits_data_list": traits_data_list } -- cgit v1.2.3