diff options
| -rw-r--r-- | misc/notes.txt | 3 | ||||
| -rwxr-xr-x | wqflask/base/data_set.py | 2 | ||||
| -rwxr-xr-x | wqflask/base/trait.py | 6 | ||||
| -rw-r--r-- | wqflask/wqflask/correlation/show_corr_results.py | 71 | ||||
| -rw-r--r-- | wqflask/wqflask/templates/correlation_page.html | 15 | ||||
| -rw-r--r-- | wqflask/wqflask/views.py | 9 |
6 files changed, 91 insertions, 15 deletions
diff --git a/misc/notes.txt b/misc/notes.txt index 4e478345..678e71b8 100644 --- a/misc/notes.txt +++ b/misc/notes.txt @@ -190,7 +190,8 @@ tidyp -m -i -w 100 index_page.html =========================================== -ps -ax - View processes +ps ax - View processes +ps ux (the "u" is for user) kill (process #) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 4c5c46a5..0c7676c4 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -809,7 +809,7 @@ class MrnaAssayDataSet(DataSet): this_trait.LRS_location_repr = 'Chr %s: %.4f Mb' % (lrs_chr, float(lrs_mb)) - def convert_location_to_value(chromosome, mb): + def convert_location_to_value(self, chromosome, mb): try: location_value = int(chromosome)*1000 + float(mb) except ValueError: diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 5fde114f..53f41779 100755 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -23,7 +23,7 @@ class GeneralTrait(object): """ def __init__(self, **kw): - print("in GeneralTrait") + #print("in GeneralTrait") self.dataset = kw.get('dataset') # database name self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. self.cellid = kw.get('cellid') @@ -269,7 +269,7 @@ class GeneralTrait(object): escape(self.dataset.name), escape(self.name)) traitInfo = g.db.execute(query).fetchone() - print("traitInfo is: ", pf(traitInfo)) + #print("traitInfo is: ", pf(traitInfo)) #XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name # to avoid the problem of same marker name from different species. elif self.dataset.type == 'Geno': @@ -287,7 +287,7 @@ class GeneralTrait(object): escape(self.dataset.name), escape(self.name)) traitInfo = g.db.execute(query).fetchone() - print("traitInfo is: ", pf(traitInfo)) + #print("traitInfo is: ", pf(traitInfo)) else: #Temp type query = """SELECT %s FROM %s WHERE Name = %s """ % (string.join(self.dataset.display_fields,','), diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 5d40c835..96c0155b 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -29,6 +29,7 @@ from __future__ import absolute_import, print_function, division +import gc import string import cPickle import os @@ -37,6 +38,7 @@ import time import pp import math import collections +import resource import scipy @@ -69,7 +71,13 @@ TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] TISSUE_MOUSE_DB = 1 -class AuthException(Exception): pass +def print_mem(stage=""): + mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + print("{}: {}".format(stage, mem/1024)) + + +class AuthException(Exception): + pass class CorrelationResults(object): @@ -99,6 +107,7 @@ class CorrelationResults(object): #print("start_vars: ", pf(start_vars)) with Bench("Doing correlations"): + print_mem("At beginning") helper_functions.get_species_dataset_trait(self, start_vars) self.dataset.group.read_genotype_file() @@ -126,8 +135,10 @@ class CorrelationResults(object): self.target_dataset = data_set.create_dataset(start_vars['corr_dataset']) self.target_dataset.get_trait_data() - + + self.correlation_data = {} + print_mem("Before calculating correlations") for trait, values in self.target_dataset.trait_data.iteritems(): this_trait_values = [] target_values = [] @@ -139,15 +150,63 @@ class CorrelationResults(object): target_values.append(target_sample_value) this_trait_values, target_values = normalize_values(this_trait_values, target_values) + if self.corr_method == 'pearson': sample_r, sample_p = scipy.stats.pearsonr(this_trait_values, target_values) else: sample_r, sample_p = scipy.stats.spearmanr(this_trait_values, target_values) + self.correlation_data[trait] = [sample_r, sample_p] - self.correlation_data = collections.OrderedDict( - sorted(self.correlation_data.items(), - key=lambda t: -abs(t[1][0]))) - + + print_mem("After calculating correlations") + + self.correlation_data = collections.OrderedDict(sorted(self.correlation_data.items(), + key=lambda t: -abs(t[1][0]))) + + self.correlation_data_slice = collections.OrderedDict() + + old_memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + + for trait_counter, trait in enumerate(self.correlation_data.keys()[:300]): + print_mem("In trait info loop") + print("\nTrait #:", trait_counter) + print_mem("Before trait_object") + trait_object = GeneralTrait(dataset=self.dataset.name, name=trait) + print_mem("After trait object") + trait_info = dict( + correlation = float(self.correlation_data[trait][0]), + p_value = float(self.correlation_data[trait][1]), + symbol = trait_object.symbol, + alias = trait_object.alias, + description = trait_object.description, + chromosome = trait_object.chr, + mb = trait_object.mb + ) + print_mem("Before deleting trait object") + del trait_object + print_mem("After deleting trait object") + gc.collect() + print_mem("After colleting garabage") + print("** trait_info:", pf(trait_info)) + print("\n** Start trait_info") + counter = 1 + for key, value in trait_info.iteritems(): + print(" <{}> [{}] {}: [{}] {}\n".format( + counter, type(key), key, type(value), value)) + counter += 1 + print("** Done trait_info") + self.correlation_data_slice[trait] = trait_info + #self.correlation_data_slice[trait].append(trait_object) + + new_memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + print("Memory difference:", new_memory_usage-old_memory_usage) + old_memory_usage = new_memory_usage + print_mem("End of purple loop") + print("*************************** End purple ******** ") + + print_mem("After getting trait info") + print("Garbage colleting...") + gc.collect() #XZ, 09/18/2008: get all information about the user selected database. #target_db_name = fd.corr_dataset diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 68fe81ed..b06f7096 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -24,14 +24,23 @@ <th>Sample rho</th> <th>Sample p(rho)</th> {% endif %} + <th>Symbol</th> + <th>Alias</th> + <th>Description</th> + <th>Location</th> </tr> </thead> <tbody> - {% for trait in correlation_data %} + {% for trait in correlation_data_slice %} <tr> <td>{{ trait }}</td> - <td>{{ correlation_data[trait][0] }}</td> - <td>{{ correlation_data[trait][1] }}</td> + <td>{{ correlation_data_slice[trait].correlation }}</td> + <td>{{ correlation_data_slice[trait].p_value }}</td> + <td>{{ correlation_data_slice[trait].symbol }}</td> + <td>{{ correlation_data_slice[trait].alias }}</td> + <td>{{ correlation_data_slice[trait].description }}</td> + <td>Chr{{ correlation_data_slice[trait].chromosome }}: {{ correlation_data_slice[trait].mb }}</td> + {# <td>{{ correlation_data_slice[trait][2].__dict__ }}</td> #} </tr> {% endfor %} </tbody> diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 8531561a..102863f0 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -3,6 +3,8 @@ from __future__ import absolute_import, division, print_function import csv import StringIO # Todo: Use cStringIO? +import gc + import cPickle as pickle import simplejson as json @@ -251,4 +253,9 @@ def json_default_handler(obj): # return None else: raise TypeError, 'Object of type %s with value of %s is not JSON serializable' % ( - type(obj), repr(obj))
\ No newline at end of file + type(obj), repr(obj)) + + +#@app.after_request +#def after_request(response): +# gc.collect()
\ No newline at end of file |