path: root/wqflask/base/trait.py

from __future__ import absolute_import, division, print_function

import string
import resource
import codecs

from htmlgen import HTMLgen2 as HT

from base import webqtlConfig
from base.webqtlCaseData import webqtlCaseData
from base.data_set import create_dataset
from db import webqtlDatabaseFunction
from utility import webqtlUtil

from wqflask import app

import simplejson as json
from MySQLdb import escape_string as escape
from pprint import pformat as pf

from flask import Flask, g, request

from utility.logger import getLogger
logger = getLogger(__name__ )

from wqflask import user_manager

def print_mem(stage=""):
    mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    print("{}: {}".format(stage, mem/1024))

class GeneralTrait(object):
    """
    Trait class defines a trait in webqtl, can be either Microarray,
    Published phenotype, genotype, or user input trait

    """

    def __init__(self, get_qtl_info=False, get_sample_info=True, **kw):
        # xor assertion
        assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name";
        if kw.get('dataset_name'):
            self.dataset = create_dataset(kw.get('dataset_name'))
            #print(" in GeneralTrait created dataset:", self.dataset)
        else:
            self.dataset = kw.get('dataset')
        self.name = kw.get('name')                 # Trait ID, ProbeSet ID, Published ID, etc.
        #print("THE NAME IS:", self.name)
        self.cellid = kw.get('cellid')
        self.identification = kw.get('identification', 'un-named trait')
        self.haveinfo = kw.get('haveinfo', False)
        self.sequence = kw.get('sequence')         # Blat sequence, available for ProbeSet
        self.data = kw.get('data', {})

        # Sets defaults
        self.locus = None
        self.lrs = None
        self.pvalue = None
        self.mean = None
        self.additive = None
        self.num_overlap = None
        self.strand_probe = None
        self.symbol = None

        if kw.get('fullname'):
            name2 = value.split("::")
            if len(name2) == 2:
                self.dataset, self.name = name2
                # self.cellid is set to None above
            elif len(name2) == 3:
                self.dataset, self.name, self.cellid = name2

        # Todo: These two lines are necessary most of the time, but perhaps not all of the time
        # So we could add a simple if statement to short-circuit this if necessary
        self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info)
        if get_sample_info != False:
            self = retrieve_sample_data(self, self.dataset)


    def get_name(self):
        stringy = ""
        if self.dataset and self.name:
            stringy = "%s::%s" % (self.dataset, self.name)
            if self.cellid:
                stringy += "::" + self.cellid
        else:
            stringy = self.description
        return stringy


    def get_given_name(self):
        """
         when user enter a trait or GN generate a trait, user want show the name
         not the name that generated by GN randomly, the two follow function are
         used to give the real name and the database. displayName() will show the
         database also, getGivenName() just show the name.
         For other trait, displayName() as same as getName(), getGivenName() as
         same as self.name

         Hongqiang 11/29/07

        """
        stringy = self.name
        if self.dataset and self.name:
            desc = self.dataset.get_desc()
            if desc:
                #desc = self.handle_pca(desc)
                stringy = desc
        return stringy


    def display_name(self):
        stringy = ""
        if self.dataset and self.name:
            desc = self.dataset.get_desc()
            #desc = self.handle_pca(desc)
            if desc:
                #desc = self.handle_pca(desc)
                #stringy = desc
                #if desc.__contains__('PCA'):
                #    desc = desc[desc.rindex(':')+1:].strip()
                #else:
                #    desc = desc[:desc.index('entered')].strip()
                #desc = self.handle_pca(desc)
                stringy = "%s::%s" % (self.dataset, desc)
            else:
                stringy = "%s::%s" % (self.dataset, self.name)
                if self.cellid:
                    stringy += "::" + self.cellid
        else:
            stringy = self.description

        return stringy


    #def __str__(self):
    #       #return "%s %s" % (self.getName(), self.group)
    #       return self.getName()
    #__str__ = getName
    #__repr__ = __str__

    def export_data(self, samplelist, the_type="val"):
        """
        export data according to samplelist
        mostly used in calculating correlation

        """
        result = []
        for sample in samplelist:
            if self.data.has_key(sample):
                if the_type=='val':
                    result.append(self.data[sample].val)
                elif the_type=='var':
                    result.append(self.data[sample].var)
                elif the_type=='N':
                    result.append(self.data[sample].N)
                else:
                    raise KeyError, `the_type`+' the_type is incorrect.'
            else:
                result.append(None)
        return result

    def export_informative(self, include_variance=0):
        """
        export informative sample
        mostly used in qtl regression

        """
        samples = []
        vals = []
        the_vars = []
        sample_aliases = []
        for sample_name, sample_data in self.data.items():
            if sample_data.value != None:
                if not include_variance or sample_data.variance != None:
                    samples.append(sample_name)
                    vals.append(sample_data.value)
                    the_vars.append(sample_data.variance)
                    sample_aliases.append(sample_data.name2)
        return  samples, vals, the_vars, sample_aliases


    @property
    def name_header_fmt(self):
        '''Return a human-readable name for use in page header'''
        if self.dataset.type == 'ProbeSet':
            return self.symbol
        elif self.dataset.type == 'Geno':
            return self.name
        elif self.dataset.type == 'Publish':
            return self.post_publication_abbreviation
        else:
            return "unnamed"

    @property
    def description_fmt(self):
        '''Return a text formated description'''
        if self.dataset.type == 'ProbeSet':
            if self.description:
                formatted = self.description
                if self.probe_target_description:
                    formatted += "; " + self.probe_target_description
        elif self.dataset.type == 'Publish':
            if self.confidential:
                formatted = self.pre_publication_description
            else:
                formatted = self.post_publication_description
        else:
            formatted = "Not available"
        return formatted.capitalize()

    @property
    def alias_fmt(self):
        '''Return a text formatted alias'''
        if self.alias:
            alias = string.replace(self.alias, ";", " ")
            alias = string.join(string.split(alias), ", ")
        else:
            alias = 'Not available'

        return alias


    @property
    def location_fmt(self):
        '''Return a text formatted location

        While we're at it we set self.location in case we need it later (do we?)

        '''

        if self.chr and self.mb:
            self.location = 'Chr %s @ %s Mb'  % (self.chr,self.mb)
        elif self.chr:
            self.location = 'Chr %s @ Unknown position' % (self.chr)
        else:
            self.location = 'Not available'

        fmt = self.location
        ##XZ: deal with direction
        if self.strand_probe == '+':
            fmt += (' on the plus strand ')
        elif self.strand_probe == '-':
            fmt += (' on the minus strand ')

        return fmt

# In ProbeSet, there are maybe several annotations match one sequence
# so we need use sequence(BlatSeq) as the identification, when we update
# one annotation, we update the others who match the sequence also.
#
# Hongqiang Li, 3/3/2008
def getSequence(trait, dataset_name):
    dataset = create_dataset(dataset_name)
     
    if dataset.type == 'ProbeSet':
        results = g.db.execute('''
                       SELECT
                               ProbeSet.BlatSeq
                       FROM
                               ProbeSet, ProbeSetFreeze, ProbeSetXRef
                       WHERE
                               ProbeSet.Id=ProbeSetXRef.ProbeSetId and
                               ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and
                               ProbeSet.Name = %s
                               ProbeSetFreeze.Name = %s
               ''', trait.name, dataset.name).fetchone()

        return results[0]        
        
def retrieve_sample_data(trait, dataset_name, samplelist=None):

    dataset = create_dataset(dataset_name)

    if samplelist == None:
        samplelist = []

    results = dataset.retrieve_sample_data(trait.name)

    # Todo: is this necessary? If not remove
    trait.data.clear()

    all_samples_ordered = dataset.group.all_samples_ordered()

    if results:
        for item in results:
            name, value, variance, num_cases, name2 = item
            if not samplelist or (samplelist and name in samplelist):
                trait.data[name] = webqtlCaseData(*item)   #name, value, variance, num_cases)
                
    return trait
        
def convert_location_to_value(chromosome, mb):
    try:
        location_value = int(chromosome)*1000 + float(mb)
    except ValueError:
        if chromosome.upper() == 'X':
            location_value = 20*1000 + float(mb)
        else:
            location_value = (ord(str(chromosome).upper()[0])*1000 +
                              float(mb))

    return location_value

@app.route("/trait/get_sample_data")
def get_sample_data():
    params = request.args
    trait = params['trait']
    dataset = params['dataset']

    trait_ob = GeneralTrait(name=trait, dataset_name=dataset)

    return json.dumps([trait, {key: value.value for key, value in trait_ob.data.iteritems() }])

    #jsonable_sample_data = {}
    #for sample in trait_ob.data.iteritems():
    #    jsonable_sample_data[sample] = trait_ob.data[sample].value
    #
    #return jsonable_sample_data
    
def jsonable(trait, dataset_name):
    """Return a dict suitable for using as json

    Actual turning into json doesn't happen here though"""

    dataset = create_dataset(dataset_name)
    
    if dataset.type == "ProbeSet":
        return dict(name=trait.name,
                    symbol=trait.symbol,
                    dataset=dataset.name,
                    description=trait.description_display,
                    mean=trait.mean,
                    location=trait.location_repr,
                    lrs_score=trait.LRS_score_repr,
                    lrs_location=trait.LRS_location_repr,
                    additive=trait.additive
                    )
    elif dataset.type == "Publish":
        if trait.pubmed_id:
            return dict(name=trait.name,
                        dataset=dataset.name,
                        description=trait.description_display,
                        authors=trait.authors,
                        pubmed_text=trait.pubmed_text,
                        pubmed_link=trait.pubmed_link,
                        lrs_score=trait.LRS_score_repr,
                        lrs_location=trait.LRS_location_repr,
                        additive=trait.additive
                        )
        else:
            return dict(name=trait.name,
                        dataset=dataset.name,
                        description=trait.description_display,
                        authors=trait.authors,
                        pubmed_text=trait.pubmed_text,
                        lrs_score=trait.LRS_score_repr,
                        lrs_location=trait.LRS_location_repr,
                        additive=trait.additive
                        )
    elif dataset.type == "Geno":
        return dict(name=trait.name,
                    dataset=dataset.name,
                    location=trait.location_repr
                    )
    else:
        return dict()

def jsonable_table_row(trait, dataset_name, index):
    """Return a list suitable for json and intended to be displayed in a table

    Actual turning into json doesn't happen here though"""

    dataset = create_dataset(dataset_name)
    
    if dataset.type == "ProbeSet":
        if trait.mean == "":
            mean = "N/A"
        else:
            mean = "%.3f" % round(float(trait.mean), 2)
        if trait.additive == "":
            additive = "N/A"
        else:
            additive = "%.3f" % round(float(trait.additive), 2)
        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" style="transform: scale(1.5);" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                index,
                '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                trait.symbol,
                trait.description_display,
                trait.location_repr,
                mean, 
                trait.LRS_score_repr,
                trait.LRS_location_repr,
                additive]
    elif dataset.type == "Publish":
        if trait.additive == "":
            additive = "N/A"
        else:
            additive = "%.2f" % round(float(trait.additive), 2)
        if trait.pubmed_id:
            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" style="transform: scale(1.5);" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                    index,
                    '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                    trait.description_display,
                    trait.authors,
                    '<a href="' + trait.pubmed_link + '">' + trait.pubmed_text + '</href>',
                    trait.LRS_score_repr,
                    trait.LRS_location_repr,
                    additive]
        else:
            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" style="transform: scale(1.5);" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                    index,
                    '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                    trait.description_display,
                    trait.authors,
                    trait.pubmed_text,
                    trait.LRS_score_repr,
                    trait.LRS_location_repr,
                    additive]
    elif dataset.type == "Geno":
        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" style="transform: scale(1.5);" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                index,
                '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                trait.location_repr]
    else:
        return dict()

def retrieve_trait_info(trait, dataset, get_qtl_info=False):
    assert dataset, "Dataset doesn't exist"
    
    if dataset.type == 'Publish':
        query = """
                SELECT
                        PublishXRef.Id, Publication.PubMed_ID,
                        Phenotype.Pre_publication_description, Phenotype.Post_publication_description, Phenotype.Original_description,
                        Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation,
                        Phenotype.Lab_code, Phenotype.Submitter, Phenotype.Owner, Phenotype.Authorized_Users,
                        Publication.Authors, Publication.Title, Publication.Abstract,
                        Publication.Journal, Publication.Volume, Publication.Pages,
                        Publication.Month, Publication.Year, PublishXRef.Sequence,
                        Phenotype.Units, PublishXRef.comments
                FROM
                        PublishXRef, Publication, Phenotype, PublishFreeze
                WHERE
                        PublishXRef.Id = %s AND
                        Phenotype.Id = PublishXRef.PhenotypeId AND
                        Publication.Id = PublishXRef.PublicationId AND
                        PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND
                        PublishFreeze.Id = %s
                """ % (trait.name, dataset.id)

        logger.sql(query)
        trait_info = g.db.execute(query).fetchone()


    #XZ, 05/08/2009: Xiaodong add this block to use ProbeSet.Id to find the probeset instead of just using ProbeSet.Name
    #XZ, 05/08/2009: to avoid the problem of same probeset name from different platforms.
    elif dataset.type == 'ProbeSet':
        display_fields_string = ', ProbeSet.'.join(dataset.display_fields)
        display_fields_string = 'ProbeSet.' + display_fields_string
        query = """
                SELECT %s
                FROM ProbeSet, ProbeSetFreeze, ProbeSetXRef
                WHERE
                        ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND
                        ProbeSetXRef.ProbeSetId = ProbeSet.Id AND
                        ProbeSetFreeze.Name = '%s' AND
                        ProbeSet.Name = '%s'
                """ % (escape(display_fields_string),
                       escape(dataset.name),
                       escape(str(trait.name)))
        logger.sql(query)
        trait_info = g.db.execute(query).fetchone()
    #XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name
    # to avoid the problem of same marker name from different species.
    elif dataset.type == 'Geno':
        display_fields_string = string.join(dataset.display_fields,',Geno.')
        display_fields_string = 'Geno.' + display_fields_string
        query = """
                SELECT %s
                FROM Geno, GenoFreeze, GenoXRef
                WHERE
                        GenoXRef.GenoFreezeId = GenoFreeze.Id AND
                        GenoXRef.GenoId = Geno.Id AND
                        GenoFreeze.Name = '%s' AND
                        Geno.Name = '%s'
                """ % (escape(display_fields_string),
                       escape(dataset.name),
                       escape(trait.name))
        logger.sql(query)
        trait_info = g.db.execute(query).fetchone()
    else: #Temp type
        query = """SELECT %s FROM %s WHERE Name = %s"""
        logger.sql(query)
        trait_info = g.db.execute(query,
                                  (string.join(dataset.display_fields,','),
                                               dataset.type, trait.name)).fetchone()
    if trait_info:
        trait.haveinfo = True

        #XZ: assign SQL query result to trait attributes.
        for i, field in enumerate(dataset.display_fields):
            holder = trait_info[i]
            if isinstance(trait_info[i], basestring):
                holder = unicode(trait_info[i], "utf8", "ignore")
            setattr(trait, field, holder)

        if dataset.type == 'Publish':
            trait.confidential = 0
            if trait.pre_publication_description and not trait.pubmed_id:
                trait.confidential = 1

            description = trait.post_publication_description

            #If the dataset is confidential and the user has access to confidential
            #phenotype traits, then display the pre-publication description instead
            #of the post-publication description
            if trait.confidential:
                trait.description_display = trait.pre_publication_description

                #if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(
                #        privilege=self.dataset.privilege,
                #        userName=self.dataset.userName,
                #        authorized_users=self.authorized_users):
                #
                #    description = self.pre_publication_description
            else:
                if description:
                    trait.description_display = description.strip()
                else:
                    trait.description_display = ""

            if not trait.year.isdigit():
                trait.pubmed_text = "N/A"
            else:
                trait.pubmed_text = trait.year

            if trait.pubmed_id:
                trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id


        trait.homologeneid = None
        if dataset.type == 'ProbeSet' and dataset.group:
            if trait.geneid:
                #XZ, 05/26/2010: From time to time, this query get error message because some geneid values in database are not number.
                #XZ: So I have to test if geneid is number before execute the query.
                #XZ: The geneid values in database should be cleaned up.
                #try:
                #    float(self.geneid)
                #    geneidIsNumber = True
                #except ValueError:
                #    geneidIsNumber = False
                #if geneidIsNumber:
                query = """
                        SELECT
                                HomologeneId
                        FROM
                                Homologene, Species, InbredSet
                        WHERE
                                Homologene.GeneId ='%s' AND
                                InbredSet.Name = '%s' AND
                                InbredSet.SpeciesId = Species.Id AND
                                Species.TaxonomyId = Homologene.TaxonomyId
                        """ % (escape(str(trait.geneid)), escape(dataset.group.name))
                logger.sql(query)
                result = g.db.execute(query).fetchone()
                #else:
                #    result = None

                if result:
                    trait.homologeneid = result[0]

            description_string = unicode(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8')
            target_string = unicode(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8')

            if len(description_string) > 1 and description_string != 'None':
                description_display = description_string
            else:
                description_display = trait.symbol

            if (len(description_display) > 1 and description_display != 'N/A' and
                    len(target_string) > 1 and target_string != 'None'):
                description_display = description_display + '; ' + target_string.strip()

            # Save it for the jinja2 template
            trait.description_display = description_display

            #XZ: trait_location_value is used for sorting
            trait.location_repr = 'N/A'
            trait.location_value = 1000000

            if trait.chr and trait.mb:
                #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y")
                #This is so we can convert the location to a number used for sorting
                trait_location_value = convert_location_to_value(trait.chr, trait.mb)
                 #try:
                #    trait_location_value = int(self.chr)*1000 + self.mb
                #except ValueError:
                #    if self.chr.upper() == 'X':
                #        trait_location_value = 20*1000 + self.mb
                #    else:
                #        trait_location_value = (ord(str(self.chr).upper()[0])*1000 +
                #                               self.mb)

                #ZS: Put this in function currently called "convert_location_to_value"
                trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb))
                trait.location_value = trait_location_value

        elif dataset.type == "Geno":
            trait.location_repr = 'N/A'
            trait.location_value = 1000000

            if trait.chr and trait.mb:
                #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y")
                #This is so we can convert the location to a number used for sorting
                trait_location_value = convert_location_to_value(trait.chr, trait.mb)

                #ZS: Put this in function currently called "convert_location_to_value"
                trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb))
                trait.location_value = trait_location_value

        if get_qtl_info:
            #LRS and its location
            trait.LRS_score_repr = "N/A"
            trait.LRS_score_value = 0
            trait.LRS_location_repr = "N/A"
            trait.LRS_location_value = 1000000
            if dataset.type == 'ProbeSet' and not trait.cellid:
                query = """
                        SELECT
                                ProbeSetXRef.Locus, ProbeSetXRef.LRS, ProbeSetXRef.pValue, ProbeSetXRef.mean, ProbeSetXRef.additive
                        FROM
                                ProbeSetXRef, ProbeSet
                        WHERE
                                ProbeSetXRef.ProbeSetId = ProbeSet.Id AND
                                ProbeSet.Name = "{}" AND
                                ProbeSetXRef.ProbeSetFreezeId ={}
                        """.format(trait.name, dataset.id)
                logger.sql(query)
                trait_qtl = g.db.execute(query).fetchone()
                if trait_qtl:
                    trait.locus, trait.lrs, trait.pvalue, trait.mean, trait.additive = trait_qtl
                    if trait.locus:
                        query = """
                            select Geno.Chr, Geno.Mb from Geno, Species
                            where Species.Name = '{}' and
                            Geno.Name = '{}' and
                            Geno.SpeciesId = Species.Id
                            """.format(dataset.group.species, trait.locus)
                        logger.sql(query)
                        result = g.db.execute(query).fetchone()
                        if result:
                            trait.locus_chr = result[0]
                            trait.locus_mb = result[1]
                        else:
                            trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = ""
                    else:
                        trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = ""
                else:
                    trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.mean = trait.additive = ""


            if dataset.type == 'Publish':
                query = """
                        SELECT
                                PublishXRef.Locus, PublishXRef.LRS, PublishXRef.additive
                        FROM
                                PublishXRef, PublishFreeze
                        WHERE
                                PublishXRef.Id = %s AND
                                PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND
                                PublishFreeze.Id =%s
                """ % (trait.name, dataset.id)
                logger.sql(query)
                trait_qtl = g.db.execute(query).fetchone()
                if trait_qtl:
                    trait.locus, trait.lrs, trait.additive = trait_qtl
                    if trait.locus:
                        query = """
                            select Geno.Chr, Geno.Mb from Geno, Species
                            where Species.Name = '{}' and
                            Geno.Name = '{}' and
                            Geno.SpeciesId = Species.Id
                            """.format(dataset.group.species, trait.locus)
                        logger.sql(query)
                        result = g.db.execute(query).fetchone()
                        if result:
                            trait.locus_chr = result[0]
                            trait.locus_mb = result[1]
                        else:
                            trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = ""
                    else:
                        trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = ""
                else:
                    trait.locus = trait.lrs = trait.additive = ""

            if (dataset.type == 'Publish' or dataset.type == "ProbeSet") and trait.locus_chr != "" and trait.locus_mb != "":
                #XZ: LRS_location_value is used for sorting
                try:
                    LRS_location_value = int(trait.locus_chr)*1000 + float(trait.locus_mb)
                except:
                    if trait.locus_chr.upper() == 'X':
                        LRS_location_value = 20*1000 + float(trait.locus_mb)
                    else:
                        LRS_location_value = ord(str(trait.locus_chr).upper()[0])*1000 + float(trait.locus_mb)

                trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (trait.locus_chr, float(trait.locus_mb))
                if trait.lrs != "":
                    trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs
                    trait.LRS_score_value = LRS_score_value = trait.lrs
    else:
        raise KeyError, `trait.name`+' information is not found in the database.'
        
    return trait