"""This class contains functions relating to trait data manipulation"""
import os
from functools import reduce
from typing import Any, Dict, Sequence
from gn3.settings import TMPDIR
from gn3.chancy import random_string
from gn3.function_helpers import compose
from gn3.db.datasets import retrieve_trait_dataset
def export_trait_data(
trait_data: dict, samplelist: Sequence[str], dtype: str = "val",
var_exists: bool = False, n_exists: bool = False):
"""
Export data according to `samplelist`. Mostly used in calculating
correlations.
DESCRIPTION:
Migrated from
https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L166-L211
PARAMETERS
trait: (dict)
The dictionary of key-value pairs representing a trait
samplelist: (list)
A list of sample names
dtype: (str)
... verify what this is ...
var_exists: (bool)
A flag indicating existence of variance
n_exists: (bool)
A flag indicating existence of ndata
"""
def __export_all_types(tdata, sample):
sample_data = []
if tdata[sample]["value"]:
sample_data.append(tdata[sample]["value"])
if var_exists:
if tdata[sample]["variance"]:
sample_data.append(tdata[sample]["variance"])
else:
sample_data.append(None)
if n_exists:
if tdata[sample]["ndata"]:
sample_data.append(tdata[sample]["ndata"])
else:
sample_data.append(None)
else:
if var_exists and n_exists:
sample_data += [None, None, None]
elif var_exists or n_exists:
sample_data += [None, None]
else:
sample_data.append(None)
return tuple(sample_data)
def __exporter(accumulator, sample):
# pylint: disable=[R0911]
if sample in trait_data["data"]:
if dtype == "val":
return accumulator + (trait_data["data"][sample]["value"], )
if dtype == "var":
return accumulator + (trait_data["data"][sample]["variance"], )
if dtype == "N":
return accumulator + (trait_data["data"][sample]["ndata"], )
if dtype == "all":
return accumulator + __export_all_types(trait_data["data"], sample)
raise KeyError(f"Type `{dtype}` is incorrect")
if var_exists and n_exists:
return accumulator + (None, None, None)
if var_exists or n_exists:
return accumulator + (None, None)
return accumulator + (None,)
return reduce(__exporter, samplelist, tuple())
def retrieve_publish_trait_info(trait_data_source: Dict[str, Any], conn: Any):
"""Retrieve trait information for type `Publish` traits.
https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L399-L421"""
keys = (
"Id", "PubMed_ID", "Pre_publication_description",
"Post_publication_description", "Original_description",
"Pre_publication_abbreviation", "Post_publication_abbreviation",
"Lab_code", "Submitter", "Owner", "Authorized_Users", "Authors",
"Title", "Abstract", "Journal", "Volume", "Pages", "Month", "Year",
"Sequence", "Units", "comments")
columns = (
"PublishXRef.Id, Publication.PubMed_ID, "
"Phenotype.Pre_publication_description, "
"Phenotype.Post_publication_description, "
"Phenotype.Original_description, "
"Phenotype.Pre_publication_abbreviation, "
"Phenotype.Post_publication_abbreviation, "
"Phenotype.Lab_code, Phenotype.Submitter, Phenotype.Owner, "
"Phenotype.Authorized_Users, CAST(Publication.Authors AS BINARY), "
"Publication.Title, Publication.Abstract, Publication.Journal, "
"Publication.Volume, Publication.Pages, Publication.Month, "
"Publication.Year, PublishXRef.Sequence, Phenotype.Units, "
"PublishXRef.comments")
query = (
"SELECT "
f"{columns} "
"FROM "
"PublishXRef, Publication, Phenotype "
"WHERE "
"PublishXRef.Id = %(trait_name)s AND "
"Phenotype.Id = PublishXRef.PhenotypeId AND "
"Publication.Id = PublishXRef.PublicationId AND "
"PublishXRef.InbredSetId = %(trait_dataset_id)s")
with conn.cursor() as cursor:
cursor.execute(
query,
{
k: v for k, v in trait_data_source.items()
if k in ["trait_name", "trait_dataset_id"]
})
return dict(zip([k.lower() for k in keys], cursor.fetchone()))
def set_confidential_field(trait_type, trait_info):
"""Post processing function for 'Publish' trait types.
It sets the value for the 'confidential' key."""
if trait_type == "Publish":
return {
**trait_info,
"confidential": 1 if (
trait_info.get("pre_publication_description", None)
and not trait_info.get("pubmed_id", None)) else 0}
return trait_info
def retrieve_probeset_trait_info(trait_data_source: Dict[str, Any], conn: Any):
"""Retrieve trait information for type `ProbeSet` traits.
https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L424-L435"""
keys = (
"name", "symbol", "description", "probe_target_description", "chr",
"mb", "alias", "geneid", "genbankid", "unigeneid", "omim",
"refseq_transcriptid", "blatseq", "targetseq", "chipid", "comments",
"strand_probe", "strand_gene", "probe_set_target_region", "proteinid",
"probe_set_specificity", "probe_set_blat_score",
"probe_set_blat_mb_start", "probe_set_blat_mb_end", "probe_set_strand",
"probe_set_note_by_rw", "flag")
columns = (f"ProbeSet.{x}" for x in keys)
query = (
f"SELECT {', '.join(columns)} "
"FROM "
"ProbeSet, ProbeSetFreeze, ProbeSetXRef "
"WHERE "
"ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND "
"ProbeSetXRef.ProbeSetId = ProbeSet.Id AND "
"ProbeSetFreeze.Name = %(trait_dataset_name)s AND "
"ProbeSet.Name = %(trait_name)s")
with conn.cursor() as cursor:
cursor.execute(
query,
{
k: v for k, v in trait_data_source.items()
if k in ["trait_name", "trait_dataset_name"]
})
return dict(zip(keys, cursor.fetchone()))
def retrieve_geno_trait_info(trait_data_source: Dict[str, Any], conn: Any):
"""Retrieve trait information for type `Geno` traits.
https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L438-L449"""
keys = ("name", "chr", "mb", "source2", "sequence")
columns = ", ".join(f"Geno.{x}" for x in keys)
query = (
f"SELECT {columns} "
"FROM "
"Geno INNER JOIN GenoXRef ON GenoXRef.GenoId = Geno.Id "
"INNER JOIN GenoFreeze ON GenoFreeze.Id = GenoXRef.GenoFreezeId "
"WHERE "
"GenoFreeze.Name = %(trait_dataset_name)s AND "
"Geno.Name = %(trait_name)s")
with conn.cursor() as cursor:
cursor.execute(
query,
{
k: v for k, v in trait_data_source.items()
if k in ["trait_name", "trait_dataset_name"]
})
return dict(zip(keys, cursor.fetchone()))
def retrieve_temp_trait_info(trait_data_source: Dict[str, Any], conn: Any):
"""Retrieve trait information for type `Temp` traits.
https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L450-452"""
keys = ("name", "description")
query = (
f"SELECT {', '.join(keys)} FROM Temp "
"WHERE Name = %(trait_name)s")
with conn.cursor() as cursor:
cursor.execute(
query,
{
k: v for k, v in trait_data_source.items()
if k in ["trait_name"]
})
return dict(zip(keys, cursor.fetchone()))
def set_haveinfo_field(trait_info):
"""
Common postprocessing function for all trait types.
Sets the value for the 'haveinfo' field."""
return {**trait_info, "haveinfo": 1 if trait_info else 0}
def set_homologene_id_field_probeset(trait_info, conn):
"""
Postprocessing function for 'ProbeSet' traits.
Sets the value for the 'homologene' key.
"""
query = (
"SELECT HomologeneId FROM Homologene, Species, InbredSet"
" WHERE Homologene.GeneId = %(geneid)s AND InbredSet.Name = %(group)s"
" AND InbredSet.SpeciesId = Species.Id AND"
" Species.TaxonomyId = Homologene.TaxonomyId")
with conn.cursor() as cursor:
cursor.execute(
query,
{
k: v for k, v in trait_info.items()
if k in ["geneid", "group"]
})
res = cursor.fetchone()
if res:
return {**trait_info, "homologeneid": res[0]}
return {**trait_info, "homologeneid": None}
def set_homologene_id_field(trait_type, trait_info, conn):
"""
Common postprocessing function for all trait types.
Sets the value for the 'homologene' key."""
def set_to_null(ti): return {**ti, "homologeneid": None} # pylint: disable=[C0103, C0321]
functions_table = {
"Temp": set_to_null,
"Geno": set_to_null,
"Publish": set_to_null,
"ProbeSet": lambda ti: set_homologene_id_field_probeset(ti, conn)
}
return functions_table[trait_type](trait_info)
def load_publish_qtl_info(trait_info, conn):
"""
Load extra QTL information for `Publish` traits
"""
query = (
"SELECT PublishXRef.Locus, PublishXRef.LRS, PublishXRef.additive "
"FROM PublishXRef, PublishFreeze "
"WHERE PublishXRef.Id = %(trait_name)s "
"AND PublishXRef.InbredSetId = PublishFreeze.InbredSetId "
"AND PublishFreeze.Id = %(dataset_id)s")
with conn.cursor() as cursor:
cursor.execute(
query,
{
"trait_name": trait_info["trait_name"],
"dataset_id": trait_info["db"]["dataset_id"]
})
return dict(zip(["locus", "lrs", "additive"], cursor.fetchone()))
return {"locus": "", "lrs": "", "additive": ""}
def load_probeset_qtl_info(trait_info, conn):
"""
Load extra QTL information for `ProbeSet` traits
"""
query = (
"SELECT ProbeSetXRef.Locus, ProbeSetXRef.LRS, ProbeSetXRef.pValue, "
"ProbeSetXRef.mean, ProbeSetXRef.additive "
"FROM ProbeSetXRef, ProbeSet "
"WHERE ProbeSetXRef.ProbeSetId = ProbeSet.Id "
" AND ProbeSet.Name = %(trait_name)s "
"AND ProbeSetXRef.ProbeSetFreezeId = %(dataset_id)s")
with conn.cursor() as cursor:
cursor.execute(
query,
{
"trait_name": trait_info["trait_name"],
"dataset_id": trait_info["db"]["dataset_id"]
})
return dict(zip(
["locus", "lrs", "pvalue", "mean", "additive"], cursor.fetchone()))
return {"locus": "", "lrs": "", "pvalue": "", "mean": "", "additive": ""}
def load_qtl_info(qtl, trait_type, trait_info, conn):
"""
Load extra QTL information for traits
DESCRIPTION:
Migrated from
https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L500-L534
PARAMETERS:
qtl: boolean
trait_type: string
The type of the trait in consideration
trait_info: map/dictionary
A dictionary of the trait's key-value pairs
conn:
A database connection object
"""
if not qtl:
return trait_info
qtl_info_functions = {
"Publish": load_publish_qtl_info,
"ProbeSet": load_probeset_qtl_info
}
if trait_info["name"] not in qtl_info_functions:
return trait_info
return qtl_info_functions[trait_type](trait_info, conn)
def build_trait_name(trait_fullname):
"""
Initialises the trait's name, and other values from the search data provided
"""
def dataset_type(dset_name):
if dset_name.find('Temp') >= 0:
return "Temp"
if dset_name.find('Geno') >= 0:
return "Geno"
if dset_name.find('Publish') >= 0:
return "Publish"
return "ProbeSet"
name_parts = trait_fullname.split("::")
assert len(name_parts) >= 2, f"Name format error: '{trait_fullname}'"
dataset_name = name_parts[0]
dataset_type = dataset_type(dataset_name)
return {
"db": {
"dataset_name": dataset_name,
"dataset_type": dataset_type},
"trait_fullname": trait_fullname,
"trait_name": name_parts[1],
"cellid": name_parts[2] if len(name_parts) == 3 else ""
}
def retrieve_probeset_sequence(trait, conn):
"""
Retrieve a 'ProbeSet' trait's sequence information
"""
query = (
"SELECT ProbeSet.BlatSeq "
"FROM ProbeSet, ProbeSetFreeze, ProbeSetXRef "
"WHERE ProbeSet.Id=ProbeSetXRef.ProbeSetId "
"AND ProbeSetFreeze.Id = ProbeSetXRef.ProbeSetFreezeId "
"AND ProbeSet.Name = %(trait_name)s "
"AND ProbeSetFreeze.Name = %(dataset_name)s")
with conn.cursor() as cursor:
cursor.execute(
query,
{
"trait_name": trait["trait_name"],
"dataset_name": trait["db"]["dataset_name"]
})
seq = cursor.fetchone()
return {**trait, "sequence": seq[0] if seq else ""}
def retrieve_trait_info(
threshold: int, trait_full_name: str, conn: Any,
qtl=None):
"""Retrieves the trait information.
https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L397-L456
This function, or the dependent functions, might be incomplete as they are
currently."""
trait = build_trait_name(trait_full_name)
trait_dataset_type = trait["db"]["dataset_type"]
trait_info_function_table = {
"Publish": retrieve_publish_trait_info,
"ProbeSet": retrieve_probeset_trait_info,
"Geno": retrieve_geno_trait_info,
"Temp": retrieve_temp_trait_info
}
common_post_processing_fn = compose(
lambda ti: load_qtl_info(qtl, trait_dataset_type, ti, conn),
lambda ti: set_homologene_id_field(trait_dataset_type, ti, conn),
lambda ti: {"trait_type": trait_dataset_type, **ti},
lambda ti: {**trait, **ti})
trait_post_processing_functions_table = {
"Publish": compose(
lambda ti: set_confidential_field(trait_dataset_type, ti),
common_post_processing_fn),
"ProbeSet": compose(
lambda ti: retrieve_probeset_sequence(ti, conn),
common_post_processing_fn),
"Geno": common_post_processing_fn,
"Temp": common_post_processing_fn
}
retrieve_info = compose(
set_haveinfo_field, trait_info_function_table[trait_dataset_type])
trait_dataset = retrieve_trait_dataset(
trait_dataset_type, trait, threshold, conn)
trait_info = retrieve_info(
{
"trait_name": trait["trait_name"],
"trait_dataset_id": trait_dataset["dataset_id"],
"trait_dataset_name": trait_dataset["dataset_name"]
},
conn)
if trait_info["haveinfo"]:
return {
**trait_post_processing_functions_table[trait_dataset_type](
{**trait_info, "group": trait_dataset["group"]}),
"db": {**trait["db"], **trait_dataset}
}
return trait_info
def retrieve_temp_trait_data(trait_info: dict, conn: Any):
"""
Retrieve trait data for `Temp` traits.
"""
query = (
"SELECT "
"Strain.Name, TempData.value, TempData.SE, TempData.NStrain, "
"TempData.Id "
"FROM TempData, Temp, Strain "
"WHERE TempData.StrainId = Strain.Id "
"AND TempData.Id = Temp.DataId "
"AND Temp.name = %(trait_name)s "
"ORDER BY Strain.Name")
with conn.cursor() as cursor:
cursor.execute(
query,
{"trait_name": trait_info["trait_name"]})
return [dict(zip(
["sample_name", "value", "se_error", "nstrain", "id"],
row))
for row in cursor.fetchall()]
return []
def retrieve_species_id(group, conn: Any):
"""
Retrieve a species id given the Group value
"""
with conn.cursor as cursor:
cursor.execute(
"SELECT SpeciesId from InbredSet WHERE Name = %(group)s",
{"group": group})
return cursor.fetchone()[0]
return None
def retrieve_geno_trait_data(trait_info: Dict, conn: Any):
"""
Retrieve trait data for `Geno` traits.
"""
query = (
"SELECT Strain.Name, GenoData.value, GenoSE.error, GenoData.Id "
"FROM (GenoData, GenoFreeze, Strain, Geno, GenoXRef) "
"LEFT JOIN GenoSE ON "
"(GenoSE.DataId = GenoData.Id AND GenoSE.StrainId = GenoData.StrainId) "
"WHERE Geno.SpeciesId = %(species_id)s "
"AND Geno.Name = %(trait_name)s AND GenoXRef.GenoId = Geno.Id "
"AND GenoXRef.GenoFreezeId = GenoFreeze.Id "
"AND GenoFreeze.Name = %(dataset_name)s "
"AND GenoXRef.DataId = GenoData.Id "
"AND GenoData.StrainId = Strain.Id "
"ORDER BY Strain.Name")
with conn.cursor() as cursor:
cursor.execute(
query,
{"trait_name": trait_info["trait_name"],
"dataset_name": trait_info["db"]["dataset_name"],
"species_id": retrieve_species_id(
trait_info["db"]["group"], conn)})
return [
dict(zip(
["sample_name", "value", "se_error", "id"],
row))
for row in cursor.fetchall()]
return []
def retrieve_publish_trait_data(trait_info: Dict, conn: Any):
"""
Retrieve trait data for `Publish` traits.
"""
query = (
"SELECT "
"Strain.Name, PublishData.value, PublishSE.error, NStrain.count, "
"PublishData.Id "
"FROM (PublishData, Strain, PublishXRef) "
"LEFT JOIN PublishSE ON "
"(PublishSE.DataId = PublishData.Id "
"AND PublishSE.StrainId = PublishData.StrainId) "
"LEFT JOIN NStrain ON "
"(NStrain.DataId = PublishData.Id "
"AND NStrain.StrainId = PublishData.StrainId) "
"WHERE PublishData.Id = PublishXRef.DataId "
"AND PublishXRef.Id = %(trait_name)s "
"AND PublishXRef.InbredSetId = %(dataset_id)s "
"AND PublishData.StrainId = Strain.Id "
"ORDER BY Strain.Name")
with conn.cursor() as cursor:
cursor.execute(
query,
{"trait_name": trait_info["trait_name"],
"dataset_id": trait_info["db"]["dataset_id"]})
return [
dict(zip(
["sample_name", "value", "se_error", "nstrain", "id"], row))
for row in cursor.fetchall()]
return []
def retrieve_cellid_trait_data(trait_info: Dict, conn: Any):
"""
Retrieve trait data for `Probe Data` types.
"""
query = (
"SELECT "
"Strain.Name, ProbeData.value, ProbeSE.error, ProbeData.Id "
"FROM (ProbeData, ProbeFreeze, ProbeSetFreeze, ProbeXRef, Strain,"
" Probe, ProbeSet) "
"LEFT JOIN ProbeSE ON "
"(ProbeSE.DataId = ProbeData.Id "
" AND ProbeSE.StrainId = ProbeData.StrainId) "
"WHERE Probe.Name = %(cellid)s "
"AND ProbeSet.Name = %(trait_name)s "
"AND Probe.ProbeSetId = ProbeSet.Id "
"AND ProbeXRef.ProbeId = Probe.Id "
"AND ProbeXRef.ProbeFreezeId = ProbeFreeze.Id "
"AND ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id "
"AND ProbeSetFreeze.Name = %(dataset_name)s "
"AND ProbeXRef.DataId = ProbeData.Id "
"AND ProbeData.StrainId = Strain.Id "
"ORDER BY Strain.Name")
with conn.cursor() as cursor:
cursor.execute(
query,
{"cellid": trait_info["cellid"],
"trait_name": trait_info["trait_name"],
"dataset_id": trait_info["db"]["dataset_id"]})
return [
dict(zip(
["sample_name", "value", "se_error", "id"], row))
for row in cursor.fetchall()]
return []
def retrieve_probeset_trait_data(trait_info: Dict, conn: Any):
"""
Retrieve trait data for `ProbeSet` traits.
"""
query = (
"SELECT Strain.Name, ProbeSetData.value, ProbeSetSE.error, "
"ProbeSetData.Id "
"FROM (ProbeSetData, ProbeSetFreeze, Strain, ProbeSet, ProbeSetXRef) "
"LEFT JOIN ProbeSetSE ON "
"(ProbeSetSE.DataId = ProbeSetData.Id "
"AND ProbeSetSE.StrainId = ProbeSetData.StrainId) "
"WHERE ProbeSet.Name = %(trait_name)s "
"AND ProbeSetXRef.ProbeSetId = ProbeSet.Id "
"AND ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id "
"AND ProbeSetFreeze.Name = %(dataset_name)s "
"AND ProbeSetXRef.DataId = ProbeSetData.Id "
"AND ProbeSetData.StrainId = Strain.Id "
"ORDER BY Strain.Name")
with conn.cursor() as cursor:
cursor.execute(
query,
{"trait_name": trait_info["trait_name"],
"dataset_name": trait_info["db"]["dataset_name"]})
return [
dict(zip(
["sample_name", "value", "se_error", "id"], row))
for row in cursor.fetchall()]
return []
def with_samplelist_data_setup(samplelist: Sequence[str]):
"""
Build function that computes the trait data from provided list of samples.
PARAMETERS
samplelist: (list)
A list of sample names
RETURNS:
Returns a function that given some data from the database, computes the
sample's value, variance and ndata values, only if the sample is present
in the provided `samplelist` variable.
"""
def setup_fn(tdata):
if tdata["sample_name"] in samplelist:
val = tdata["value"]
if val is not None:
return {
"sample_name": tdata["sample_name"],
"value": val,
"variance": tdata["se_error"],
"ndata": tdata.get("nstrain", None)
}
return None
return setup_fn
def without_samplelist_data_setup():
"""
Build function that computes the trait data.
RETURNS:
Returns a function that given some data from the database, computes the
sample's value, variance and ndata values.
"""
def setup_fn(tdata):
val = tdata["value"]
if val is not None:
return {
"sample_name": tdata["sample_name"],
"value": val,
"variance": tdata["se_error"],
"ndata": tdata.get("nstrain", None)
}
return None
return setup_fn
def retrieve_trait_data(trait: dict, conn: Any, samplelist: Sequence[str] = tuple()):
"""
Retrieve trait data
DESCRIPTION
Retrieve trait data as is done in
https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/base/webqtlTrait.py#L258-L386
"""
# I do not like this section, but it retains the flow in the old codebase
if trait["db"]["dataset_type"] == "Temp":
results = retrieve_temp_trait_data(trait, conn)
elif trait["db"]["dataset_type"] == "Publish":
results = retrieve_publish_trait_data(trait, conn)
elif trait["cellid"]:
results = retrieve_cellid_trait_data(trait, conn)
elif trait["db"]["dataset_type"] == "ProbeSet":
results = retrieve_probeset_trait_data(trait, conn)
else:
results = retrieve_geno_trait_data(trait, conn)
if results:
# do something with mysqlid
mysqlid = results[0]["id"]
if samplelist:
data = [
item for item in
map(with_samplelist_data_setup(samplelist), results)
if item is not None]
else:
data = [
item for item in
map(without_samplelist_data_setup(), results)
if item is not None]
return {
"mysqlid": mysqlid,
"data": dict(map(
lambda x: (
x["sample_name"],
{k: v for k, v in x.items() if x != "sample_name"}),
data))}
return {}
def generate_traits_filename(base_path: str = TMPDIR):
"""Generate a unique filename for use with generated traits files."""
return (
f"{os.path.abspath(base_path)}/traits_test_file_{random_string(10)}.txt")
def export_informative(trait_data: dict, inc_var: bool = False) -> tuple:
"""
Export informative strain
This is a migration of the `exportInformative` function in
web/webqtl/base/webqtlTrait.py module in GeneNetwork1.
There is a chance that the original implementation has a bug, especially
dealing with the `inc_var` value. It the `inc_var` value is meant to control
the inclusion of the `variance` value, then the current implementation, and
that one in GN1 have a bug.
"""
def __exporter__(acc, data_item):
if not inc_var or data_item["variance"] is not None:
return (
acc[0] + (data_item["sample_name"],),
acc[1] + (data_item["value"],),
acc[2] + (data_item["variance"],))
return acc
return reduce(
__exporter__,
filter(lambda td: td["value"] is not None,
trait_data["data"].values()),
(tuple(), tuple(), tuple()))
