Fixed issue where too much memory was used as a result of creating a

dataset object for each trait in the correlation results

Added new fields/columns for each trait in the correlation result table
(max LRS, max LRS location, mean expression)

Fixed error if trait doesn't have these fields

2 files changed, 37 insertions, 20 deletions

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 0c7676c4..0903bf16 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -90,8 +90,8 @@ class Markers(object):
         self.markers = json.load(json_data_fh)
     
     def add_pvalues(self, p_values):
-        print("length of self.markers:", len(self.markers))
-        print("length of p_values:", len(p_values))
+        #print("length of self.markers:", len(self.markers))
+        #print("length of p_values:", len(p_values))
         
         # THIS IS only needed for the case when we are limiting the number of p-values calculated
         if len(self.markers) < len(p_values):
@@ -161,7 +161,7 @@ class DatasetGroup(object):
         self.f1list = None
         self.parlist = None        
         self.get_f1_parent_strains()
-        print("parents/f1s: {}:{}".format(self.parlist, self.f1list))
+        #print("parents/f1s: {}:{}".format(self.parlist, self.f1list))
         
         self.species = webqtlDatabaseFunction.retrieve_species(self.name)
         
@@ -170,7 +170,7 @@ class DatasetGroup(object):
         
         
     def get_markers(self):
-        print("self.species is:", self.species)
+        #print("self.species is:", self.species)
         if self.species == "human":
             marker_class = HumanMarkers 
         else:
@@ -293,14 +293,14 @@ class DataSet(object):
             self.name,
             self.name,
             self.name))
-        print("query_args are:", query_args)
+        #print("query_args are:", query_args)
 
-        print("""
-                SELECT Id, Name, FullName, ShortName
-                FROM %s
-                WHERE public > %s AND
-                     (Name = '%s' OR FullName = '%s' OR ShortName = '%s')
-          """ % (query_args))
+        #print("""
+        #        SELECT Id, Name, FullName, ShortName
+        #        FROM %s
+        #        WHERE public > %s AND
+        #             (Name = '%s' OR FullName = '%s' OR ShortName = '%s')
+        #  """ % (query_args))
 
         self.id, self.name, self.fullname, self.shortname = g.db.execute("""
                 SELECT Id, Name, FullName, ShortName
@@ -624,12 +624,12 @@ class MrnaAssayDataSet(DataSet):
             and ProbeSetFreezeId = {}
             """.format(escape(str(self.id)))
         results = g.db.execute(query).fetchall()
-        print("After get_trait_list query")
+        #print("After get_trait_list query")
         trait_data = {}
         for trait in results:
             print("Retrieving sample_data for ", trait[0])
             trait_data[trait[0]] = self.retrieve_sample_data(trait[0])
-        print("After retrieve_sample_data")
+        #print("After retrieve_sample_data")
         return trait_data
     
     def get_trait_data(self):
@@ -763,7 +763,7 @@ class MrnaAssayDataSet(DataSet):
             """ % (escape(str(this_trait.dataset.id)),
                    escape(this_trait.name)))
 
-            print("query is:", pf(query))
+            #print("query is:", pf(query))
 
             result = g.db.execute(query).fetchone()
             
@@ -926,7 +926,7 @@ class TempDataSet(DataSet):
 
 def geno_mrna_confidentiality(ob):
     dataset_table = ob.type + "Freeze"
-    print("dataset_table [%s]: %s" % (type(dataset_table), dataset_table))
+    #print("dataset_table [%s]: %s" % (type(dataset_table), dataset_table))
 
     query = '''SELECT Id, Name, FullName, confidentiality,
                         AuthorisedUsers FROM %s WHERE Name = %%s''' % (dataset_table)
diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index 53f41779..f333d5a7 100755
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -1,6 +1,8 @@
 from __future__ import absolute_import, division, print_function
 
 import string
+import resource
+
 
 from htmlgen import HTMLgen2 as HT
 
@@ -15,6 +17,10 @@ from pprint import pformat as pf
 
 from flask import Flask, g
 
+def print_mem(stage=""):
+    mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    print("{}: {}".format(stage, mem/1024))
+
 class GeneralTrait(object):
     """
     Trait class defines a trait in webqtl, can be either Microarray,
@@ -23,8 +29,12 @@ class GeneralTrait(object):
     """
 
     def __init__(self, **kw):
-        #print("in GeneralTrait")
-        self.dataset = kw.get('dataset')           # database name
+        # xor assertion
+        assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. xor name";
+        if kw.get('dataset_name'):
+            self.dataset = create_dataset(kw.get('dataset_name'))
+        else:
+            self.dataset = kw.get('dataset')
         self.name = kw.get('name')                 # Trait ID, ProbeSet ID, Published ID, etc.
         self.cellid = kw.get('cellid')
         self.identification = kw.get('identification', 'un-named trait')
@@ -39,8 +49,6 @@ class GeneralTrait(object):
                 # self.cellid is set to None above
             elif len(name2) == 3:
                 self.dataset, self.name, self.cellid = name2
-
-        self.dataset = create_dataset(self.dataset)
         
         # Todo: These two lines are necessary most of the time, but perhaps not all of the time
         # So we could add a simple if statement to short-circuit this if necessary
@@ -355,8 +363,17 @@ class GeneralTrait(object):
                     #traitQTL = self.cursor.fetchone()
                     if traitQTL:
                         self.locus, self.lrs, self.pvalue, self.mean = traitQTL
+                        if self.locus:
+                            result = g.db.execute("""
+                                select Geno.Chr, Geno.Mb from Geno, Species
+                                where Species.Name = '%s' and
+                                Geno.Name = '%s' and
+                                Geno.SpeciesId = Species.Id
+                                """, (species, self.locus)).fetchone()
+                            self.locus_chr = result[0]
+                            self.locus_mb = result[1]
                     else:
-                        self.locus = self.lrs = self.pvalue = self.mean = ""
+                        self.locus = self.locus_chr = self.locus_mb = self.lrs = self.pvalue = self.mean = ""
                 if self.dataset.type == 'Publish':
                     traitQTL = g.db.execute("""
                             SELECT