diff options
Diffstat (limited to 'wqflask/base')
| -rw-r--r-- | wqflask/base/GeneralObject.py | 77 | ||||
| -rw-r--r-- | wqflask/base/anon_collection.py | 21 | ||||
| -rw-r--r-- | wqflask/base/data_set.py | 683 | ||||
| -rw-r--r-- | wqflask/base/mrna_assay_tissue_data.py | 66 | ||||
| -rw-r--r-- | wqflask/base/species.py | 107 | ||||
| -rw-r--r-- | wqflask/base/template.py | 123 | ||||
| -rw-r--r-- | wqflask/base/trait.py | 537 | ||||
| -rw-r--r-- | wqflask/base/trait_collection.py | 53 | ||||
| -rw-r--r-- | wqflask/base/webqtlCaseData.py | 42 | ||||
| -rw-r--r-- | wqflask/base/webqtlConfig.py | 66 | ||||
| -rw-r--r-- | wqflask/base/webqtlFormData.py | 352 |
11 files changed, 662 insertions, 1465 deletions
diff --git a/wqflask/base/GeneralObject.py b/wqflask/base/GeneralObject.py index 02a1ef06..0fccaab3 100644 --- a/wqflask/base/GeneralObject.py +++ b/wqflask/base/GeneralObject.py @@ -25,44 +25,41 @@ # Last updated by GeneNetwork Core Team 2010/10/20 class GeneralObject: - """ - Base class to define an Object. - a = [Spam(1, 4), Spam(9, 3), Spam(4,6)] - a.sort(lambda x, y: cmp(x.eggs, y.eggs)) - """ + """ + Base class to define an Object. + a = [Spam(1, 4), Spam(9, 3), Spam(4,6)] + a.sort(lambda x, y: cmp(x.eggs, y.eggs)) + """ - def __init__(self, *args, **kw): - self.contents = list(args) - for name, value in kw.items(): - setattr(self, name, value) - - def __setitem__(self, key, value): - setattr(self, key, value) - - def __getitem__(self, key): - return getattr(self, key) - - def __getattr__(self, key): - if key in self.__dict__.keys(): - return self.__dict__[key] - else: - return eval("self.__dict__.%s" % key) - - def __len__(self): - return len(self.__dict__) - 1 - - def __str__(self): - s = '' - for key in self.__dict__.keys(): - if key != 'contents': - s += '%s = %s\n' % (key,self.__dict__[key]) - return s - - def __repr__(self): - s = '' - for key in self.__dict__.keys(): - s += '%s = %s\n' % (key,self.__dict__[key]) - return s - - def __cmp__(self,other): - return len(self.__dict__.keys()).__cmp__(len(other.__dict__.keys())) \ No newline at end of file + def __init__(self, *args, **kw): + self.contents = list(args) + for name, value in kw.items(): + setattr(self, name, value) + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def __getattr__(self, key): + return eval("self.__dict__.%s" % key) + + def __len__(self): + return len(self.__dict__) - 1 + + def __str__(self): + s = '' + for key in self.__dict__.keys(): + if key != 'contents': + s += '%s = %s\n' % (key, self.__dict__[key]) + return s + + def __repr__(self): + s = '' + for key in self.__dict__.keys(): + s += '%s = %s\n' % (key, self.__dict__[key]) + return s + + def __cmp__(self, other): + return len(self.__dict__.keys()).__cmp__(len(other.__dict__.keys())) diff --git a/wqflask/base/anon_collection.py b/wqflask/base/anon_collection.py deleted file mode 100644 index 8ee73296..00000000 --- a/wqflask/base/anon_collection.py +++ /dev/null @@ -1,21 +0,0 @@ -class AnonCollection(TraitCollection): - - def __init__(self, anon_id) - self.anon_id = anon_id - self.collection_members = Redis.smembers(self.anon_id) - print("self.collection_members is:", self.collection_members) - self.num_members = len(self.collection_members) - - - @app.route("/collections/remove", methods=('POST',)) - def remove_traits(traits_to_remove): - print("traits_to_remove:", traits_to_remove) - for trait in traits_to_remove: - Redis.srem(self.anon_id, trait) - members_now = self.collection_members - traits_to_remove - print("members_now:", members_now) - print("Went from {} to {} members in set.".format(len(self.collection_members), len(members_now))) - - # We need to return something so we'll return this...maybe in the future - # we can use it to check the results - return str(len(members_now)) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 4959457a..afffe780 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -19,6 +19,23 @@ # This module is used by GeneNetwork project (www.genenetwork.org) from __future__ import absolute_import, print_function, division +from db.call import fetchall, fetchone, fetch1 +from utility.logger import getLogger +from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL +from db.gn_server import menu_main +from pprint import pformat as pf +from MySQLdb import escape_string as escape +from maintenance import get_group_samplelists +from utility.tools import locate, locate_ignore_error, flat_files +from utility import gen_geno_ob +from utility import chunks +from utility.benchmark import Bench +from utility import webqtlUtil +from db import webqtlDatabaseFunction +from base import species +from base import webqtlConfig +import reaper +from flask import Flask, g import os import math import string @@ -26,45 +43,28 @@ import collections import codecs import json +import requests import gzip import cPickle as pickle import itertools -from operator import itemgetter from redis import Redis -Redis = Redis() -from flask import Flask, g +r = Redis() -import reaper - -from base import webqtlConfig -from base import species -from db import webqtlDatabaseFunction -from utility import webqtlUtil -from utility.benchmark import Bench -from utility import chunks -from utility.tools import locate, locate_ignore_error, flat_files - -from maintenance import get_group_samplelists - -from MySQLdb import escape_string as escape -from pprint import pformat as pf -from db.gn_server import menu_main -from db.call import fetchall,fetchone,fetch1 - -from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists -from utility.logger import getLogger -logger = getLogger(__name__ ) +logger = getLogger(__name__) # Used by create_database to instantiate objects # Each subclass will add to this DS_NAME_MAP = {} -def create_dataset(dataset_name, dataset_type = None, get_samplelist = True, group_name = None): + +def create_dataset(dataset_name, dataset_type=None, get_samplelist=True, group_name=None): + if dataset_name == "Temp": + dataset_type = "Temp" + if not dataset_type: dataset_type = Dataset_Getter(dataset_name) - logger.debug("dataset_type", dataset_type) dataset_ob = DS_NAME_MAP[dataset_type] dataset_class = globals()[dataset_ob] @@ -73,9 +73,10 @@ def create_dataset(dataset_name, dataset_type = None, get_samplelist = True, gro else: return dataset_class(dataset_name, get_samplelist) -class Dataset_Types(object): - def __init__(self): +class DatasetType: + + def __init__(self, redis_instance): """Create a dictionary of samples where the value is set to Geno, Publish or ProbeSet. E.g. @@ -90,39 +91,93 @@ Publish or ProbeSet. E.g. 'B139_K_1206_R': 'ProbeSet' ... """ + self.redis_instance = redis_instance self.datasets = {} - if USE_GN_SERVER: - data = menu_main() - else: - file_name = "wqflask/static/new/javascript/dataset_menu_structure.json" - with open(file_name, 'r') as fh: - data = json.load(fh) - - for species in data['datasets']: - for group in data['datasets'][species]: - for dataset_type in data['datasets'][species][group]: - for dataset in data['datasets'][species][group][dataset_type]: - short_dataset_name = dataset[1] - if dataset_type == "Phenotypes": - new_type = "Publish" - elif dataset_type == "Genotypes": - new_type = "Geno" - else: - new_type = "ProbeSet" - self.datasets[short_dataset_name] = new_type - # Set LOG_LEVEL_DEBUG=5 to see the following: - logger.debugf(5,"datasets",self.datasets) + data = self.redis_instance.get("dataset_structure") + if data: + self.datasets = json.loads(data) + else: # ZS: I don't think this should ever run unless Redis is emptied + try: + data = json.loads(requests.get( + GN2_BASE_URL + "/api/v_pre1/gen_dropdown", timeout=5).content) + for species in data['datasets']: + for group in data['datasets'][species]: + for dataset_type in data['datasets'][species][group]: + for dataset in data['datasets'][species][group][dataset_type]: + short_dataset_name = dataset[1] + if dataset_type == "Phenotypes": + new_type = "Publish" + elif dataset_type == "Genotypes": + new_type = "Geno" + else: + new_type = "ProbeSet" + self.datasets[short_dataset_name] = new_type + except: + pass + + self.redis_instance.set("dataset_structure", json.dumps(self.datasets)) + + def set_dataset_key(self, t, name): + """If name is not in the object's dataset dictionary, set it, and update + dataset_structure in Redis + + args: + t: Type of dataset structure which can be: 'mrna_expr', 'pheno', + 'other_pheno', 'geno' + name: The name of the key to inserted in the datasets dictionary + + """ + sql_query_mapping = { + 'mrna_expr': ("""SELECT ProbeSetFreeze.Id FROM """ + + """ProbeSetFreeze WHERE ProbeSetFreeze.Name = "{}" """), + 'pheno': ("""SELECT InfoFiles.GN_AccesionId """ + + """FROM InfoFiles, PublishFreeze, InbredSet """ + + """WHERE InbredSet.Name = '{}' AND """ + + """PublishFreeze.InbredSetId = InbredSet.Id AND """ + + """InfoFiles.InfoPageName = PublishFreeze.Name"""), + 'other_pheno': ("""SELECT PublishFreeze.Name """ + + """FROM PublishFreeze, InbredSet """ + + """WHERE InbredSet.Name = '{}' AND """ + + """PublishFreeze.InbredSetId = InbredSet.Id"""), + 'geno': ("""SELECT GenoFreeze.Id FROM GenoFreeze WHERE """ + + """GenoFreeze.Name = "{}" """) + } + + dataset_name_mapping = { + "mrna_expr": "ProbeSet", + "pheno": "Publish", + "other_pheno": "Publish", + "geno": "Geno", + } + + if t in ['pheno', 'other_pheno']: + name = name.replace("Publish", "") + + if bool(len(g.db.execute(sql_query_mapping[t].format(name)).fetchone())): + self.datasets[name] = dataset_name_mapping[t] + self.redis_instance.set("dataset_structure", json.dumps(self.datasets)) + return True + + return None def __call__(self, name): - return self.datasets[name] + + if name not in self.datasets: + for t in ["mrna_expr", "pheno", "other_pheno", "geno"]: + # This has side-effects, with the end result being a truth-y value + if(self.set_dataset_key(t, name)): + break + return self.datasets.get(name, None) # Return None if name has not been set + # Do the intensive work at startup one time only -Dataset_Getter = Dataset_Types() +Dataset_Getter = DatasetType(r) + def create_datasets_list(): if USE_REDIS: key = "all_datasets" - result = Redis.get(key) + result = r.get(key) if result: logger.debug("Redis cache hit") @@ -138,16 +193,16 @@ def create_datasets_list(): for dataset_type in type_dict: query = "SELECT Name FROM {}".format(type_dict[dataset_type]) for result in fetchall(query): - #The query at the beginning of this function isn't - #necessary here, but still would rather just reuse - #it logger.debug("type: {}\tname: - #{}".format(dataset_type, result.Name)) + # The query at the beginning of this function isn't + # necessary here, but still would rather just reuse + # it logger.debug("type: {}\tname: + # {}".format(dataset_type, result.Name)) dataset = create_dataset(result.Name, dataset_type) datasets.append(dataset) if USE_REDIS: - Redis.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL)) - Redis.expire(key, 60*60) + r.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL)) + r.expire(key, 60*60) return datasets @@ -168,21 +223,33 @@ def mescape(*items): class Markers(object): """Todo: Build in cacheing so it saves us reading the same file more than once""" + def __init__(self, name): - json_data_fh = open(locate(name + '.json','genotype/json')) - try: - markers = json.load(json_data_fh) - except: - markers = [] + json_data_fh = open(locate(name + ".json", 'genotype/json')) + + markers = [] + with open("%s/%s_snps.txt" % (flat_files('genotype/bimbam'), name), 'r') as bimbam_fh: + if len(bimbam_fh.readline().split(", ")) > 2: + delimiter = ", " + elif len(bimbam_fh.readline().split(",")) > 2: + delimiter = "," + elif len(bimbam_fh.readline().split("\t")) > 2: + delimiter = "\t" + else: + delimiter = " " + for line in bimbam_fh: + marker = {} + marker['name'] = line.split(delimiter)[0].rstrip() + marker['Mb'] = float(line.split(delimiter)[1].rstrip())/1000000 + marker['chr'] = line.split(delimiter)[2].rstrip() + markers.append(marker) for marker in markers: - if (marker['chr'] != "X") and (marker['chr'] != "Y"): + if (marker['chr'] != "X") and (marker['chr'] != "Y") and (marker['chr'] != "M"): marker['chr'] = int(marker['chr']) marker['Mb'] = float(marker['Mb']) self.markers = markers - #logger.debug("self.markers:", self.markers) - def add_pvalues(self, p_values): logger.debug("length of self.markers:", len(self.markers)) @@ -190,7 +257,7 @@ class Markers(object): if type(p_values) is list: # THIS IS only needed for the case when we are limiting the number of p-values calculated - #if len(self.markers) > len(p_values): + # if len(self.markers) > len(p_values): # self.markers = self.markers[:len(p_values)] for marker, p_value in itertools.izip(self.markers, p_values): @@ -202,7 +269,7 @@ class Markers(object): marker['lrs_value'] = 0 else: marker['lod_score'] = -math.log10(marker['p_value']) - #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values + # Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 elif type(p_values) is dict: filtered_markers = [] @@ -217,18 +284,20 @@ class Markers(object): marker['lrs_value'] = 0 else: marker['lod_score'] = -math.log10(marker['p_value']) - #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values - marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 + # Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values + marker['lrs_value'] = - \ + math.log10(marker['p_value']) * 4.61 filtered_markers.append(marker) - #else: + # else: #logger.debug("marker {} NOT in p_values".format(i)) - #self.markers.remove(marker) + # self.markers.remove(marker) #del self.markers[i] self.markers = filtered_markers + class HumanMarkers(Markers): - def __init__(self, name, specified_markers = []): + def __init__(self, name, specified_markers=[]): marker_data_fh = open(flat_files('mapping') + '/' + name + '.bim') self.markers = [] for line in marker_data_fh: @@ -251,7 +320,6 @@ class HumanMarkers(Markers): #logger.debug("markers is: ", pf(self.markers)) - def add_pvalues(self, p_values): super(HumanMarkers, self).add_pvalues(p_values) @@ -264,13 +332,15 @@ class DatasetGroup(object): has multiple datasets associated with it. """ + def __init__(self, dataset, name=None): """This sets self.group and self.group_id""" - #logger.debug("DATASET NAME2:", dataset.name) if name == None: - self.name, self.id, self.genetic_type = fetchone(dataset.query_for_group) + self.name, self.id, self.genetic_type = fetchone( + dataset.query_for_group) else: - self.name, self.id, self.genetic_type = fetchone("SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType FROM InbredSet where Name='%s'" % name) + self.name, self.id, self.genetic_type = fetchone( + "SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType FROM InbredSet where Name='%s'" % name) if self.name == 'BXD300': self.name = "BXD" @@ -278,7 +348,6 @@ class DatasetGroup(object): self.parlist = None self.get_f1_parent_strains() - self.accession_id = self.get_accession_id() self.mapping_id, self.mapping_names = self.get_mapping_methods() self.species = webqtlDatabaseFunction.retrieve_species(self.name) @@ -288,27 +357,16 @@ class DatasetGroup(object): self._datasets = None self.genofile = None - def get_accession_id(self): - results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where - InbredSet.Name = %s and - PublishFreeze.InbredSetId = InbredSet.Id and - InfoFiles.InfoPageName = PublishFreeze.Name and - PublishFreeze.public > 0 and - PublishFreeze.confidentiality < 1 order by - PublishFreeze.CreateTime desc""", (self.name)).fetchone() - - if results != None: - return str(results[0]) - else: - return "None" - def get_mapping_methods(self): - mapping_id = g.db.execute("select MappingMethodId from InbredSet where Name= '%s'" % self.name).fetchone()[0] + mapping_id = g.db.execute( + "select MappingMethodId from InbredSet where Name= '%s'" % self.name).fetchone()[0] if mapping_id == "1": - mapping_names = ["QTLReaper", "PYLMM", "R/qtl"] + mapping_names = ["GEMMA", "QTLReaper", "R/qtl"] elif mapping_id == "2": mapping_names = ["GEMMA"] + elif mapping_id == "3": + mapping_names = ["R/qtl"] elif mapping_id == "4": mapping_names = ["GEMMA", "PLINK"] else: @@ -316,12 +374,7 @@ class DatasetGroup(object): return mapping_id, mapping_names - def get_specified_markers(self, markers = []): - self.markers = HumanMarkers(self.name, markers) - def get_markers(self): - logger.debug("self.species is:", self.species) - def check_plink_gemma(): if flat_file_exists("mapping"): MAPPING_PATH = flat_files("mapping")+"/" @@ -332,9 +385,12 @@ class DatasetGroup(object): if check_plink_gemma(): marker_class = HumanMarkers else: - marker_class = Markers + marker_class = Markers - self.markers = marker_class(self.name) + if self.genofile: + self.markers = marker_class(self.genofile[:-5]) + else: + self.markers = marker_class(self.name) def get_f1_parent_strains(self): try: @@ -348,33 +404,36 @@ class DatasetGroup(object): if maternal and paternal: self.parlist = [maternal, paternal] + def get_genofiles(self): + jsonfile = "%s/%s.json" % (webqtlConfig.GENODIR, self.name) + try: + f = open(jsonfile) + except: + return None + jsondata = json.load(f) + return jsondata['genofile'] + def get_samplelist(self): result = None - key = "samplelist:v2:" + self.name + key = "samplelist:v3:" + self.name if USE_REDIS: - result = Redis.get(key) + result = r.get(key) if result is not None: - #logger.debug("Sample List Cache hit!!!") - #logger.debug("Before unjsonifying {}: {}".format(type(result), result)) self.samplelist = json.loads(result) - #logger.debug(" type: ", type(self.samplelist)) - #logger.debug(" self.samplelist: ", self.samplelist) else: logger.debug("Cache not hit") - genotype_fn = locate_ignore_error(self.name+".geno",'genotype') - mapping_fn = locate_ignore_error(self.name+".fam",'mapping') - if mapping_fn: - self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn) - elif genotype_fn: - self.samplelist = get_group_samplelists.get_samplelist("geno", genotype_fn) + genotype_fn = locate_ignore_error(self.name+".geno", 'genotype') + if genotype_fn: + self.samplelist = get_group_samplelists.get_samplelist( + "geno", genotype_fn) else: self.samplelist = None - logger.debug("Sample list: ",self.samplelist) + if USE_REDIS: - Redis.set(key, json.dumps(self.samplelist)) - Redis.expire(key, 60*5) + r.set(key, json.dumps(self.samplelist)) + r.expire(key, 60*5) def all_samples_ordered(self): result = [] @@ -382,26 +441,36 @@ class DatasetGroup(object): [result.extend(l) for l in lists if l] return result - def read_genotype_file(self): + def read_genotype_file(self, use_reaper=False): '''Read genotype from .geno file instead of database''' - #genotype_1 is Dataset Object without parents and f1 - #genotype_2 is Dataset Object with parents and f1 (not for intercross) + # genotype_1 is Dataset Object without parents and f1 + # genotype_2 is Dataset Object with parents and f1 (not for intercross) - genotype_1 = reaper.Dataset() + #genotype_1 = reaper.Dataset() # reaper barfs on unicode filenames, so here we ensure it's a string if self.genofile: - full_filename = str(locate(self.genofile, 'genotype')) + if "RData" in self.genofile: # ZS: This is a temporary fix; I need to change the way the JSON files that point to multiple genotype files are structured to point to other file types like RData + full_filename = str( + locate(self.genofile.split(".")[0] + ".geno", 'genotype')) + else: + full_filename = str(locate(self.genofile, 'genotype')) else: full_filename = str(locate(self.name + '.geno', 'genotype')) - genotype_1.read(full_filename) + + if use_reaper: + genotype_1 = reaper.Dataset() + genotype_1.read(full_filename) + else: + genotype_1 = gen_geno_ob.genotype(full_filename) if genotype_1.type == "group" and self.parlist: - genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1]) #, F1=_f1) + genotype_2 = genotype_1.add( + Mat=self.parlist[0], Pat=self.parlist[1]) # , F1=_f1) else: genotype_2 = genotype_1 - #determine default genotype object + # determine default genotype object if self.incparentsf1 and genotype_1.type != "intercross": genotype = genotype_2 else: @@ -412,24 +481,21 @@ class DatasetGroup(object): return genotype -def datasets(group_name, this_group = None): + +def datasets(group_name, this_group=None): key = "group_dataset_menu:v2:" + group_name - logger.debug("key is2:", key) dataset_menu = [] - logger.debug("[tape4] webqtlConfig.PUBLICTHRESH:", webqtlConfig.PUBLICTHRESH) - logger.debug("[tape4] type webqtlConfig.PUBLICTHRESH:", type(webqtlConfig.PUBLICTHRESH)) the_results = fetchall(''' (SELECT '#PublishFreeze',PublishFreeze.FullName,PublishFreeze.Name FROM PublishFreeze,InbredSet WHERE PublishFreeze.InbredSetId = InbredSet.Id and InbredSet.Name = '%s' - and PublishFreeze.public > %s) + ORDER BY PublishFreeze.Id ASC) UNION (SELECT '#GenoFreeze',GenoFreeze.FullName,GenoFreeze.Name FROM GenoFreeze, InbredSet WHERE GenoFreeze.InbredSetId = InbredSet.Id - and InbredSet.Name = '%s' - and GenoFreeze.public > %s) + and InbredSet.Name = '%s') UNION (SELECT Tissue.Name, ProbeSetFreeze.FullName,ProbeSetFreeze.Name FROM ProbeSetFreeze, ProbeFreeze, InbredSet, Tissue @@ -437,28 +503,37 @@ def datasets(group_name, this_group = None): and ProbeFreeze.TissueId = Tissue.Id and ProbeFreeze.InbredSetId = InbredSet.Id and InbredSet.Name like %s - and ProbeSetFreeze.public > %s - ORDER BY Tissue.Name, ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId) - ''' % (group_name, webqtlConfig.PUBLICTHRESH, - group_name, webqtlConfig.PUBLICTHRESH, - "'" + group_name + "'", webqtlConfig.PUBLICTHRESH)) - - #for tissue_name, dataset in itertools.groupby(the_results, itemgetter(0)): - for dataset_item in the_results: + ORDER BY Tissue.Name, ProbeSetFreeze.OrderList DESC) + ''' % (group_name, + group_name, + "'" + group_name + "'")) + + sorted_results = sorted(the_results, key=lambda kv: kv[0]) + + # ZS: This is kind of awkward, but need to ensure Phenotypes show up before Genotypes in dropdown + pheno_inserted = False + geno_inserted = False + for dataset_item in sorted_results: tissue_name = dataset_item[0] dataset = dataset_item[1] dataset_short = dataset_item[2] if tissue_name in ['#PublishFreeze', '#GenoFreeze']: - dataset_menu.append(dict(tissue=None, datasets=[(dataset, dataset_short)])) + if tissue_name == '#PublishFreeze' and (dataset_short == group_name + 'Publish'): + dataset_menu.insert( + 0, dict(tissue=None, datasets=[(dataset, dataset_short)])) + pheno_inserted = True + elif pheno_inserted and tissue_name == '#GenoFreeze': + dataset_menu.insert( + 1, dict(tissue=None, datasets=[(dataset, dataset_short)])) + geno_inserted = True + else: + dataset_menu.append( + dict(tissue=None, datasets=[(dataset, dataset_short)])) else: - dataset_sub_menu = [item[1:] for item in dataset] - tissue_already_exists = False - tissue_position = None for i, tissue_dict in enumerate(dataset_menu): if tissue_dict['tissue'] == tissue_name: tissue_already_exists = True - tissue_position = i break if tissue_already_exists: @@ -466,11 +541,11 @@ def datasets(group_name, this_group = None): dataset_menu[i]['datasets'].append((dataset, dataset_short)) else: dataset_menu.append(dict(tissue=tissue_name, - datasets=[(dataset, dataset_short)])) + datasets=[(dataset, dataset_short)])) if USE_REDIS: - Redis.set(key, pickle.dumps(dataset_menu, pickle.HIGHEST_PROTOCOL)) - Redis.expire(key, 60*5) + r.set(key, pickle.dumps(dataset_menu, pickle.HIGHEST_PROTOCOL)) + r.expire(key, 60*5) if this_group != None: this_group._datasets = dataset_menu @@ -478,6 +553,7 @@ def datasets(group_name, this_group = None): else: return dataset_menu + class DataSet(object): """ DataSet class defines a dataset in webqtl, can be either Microarray, @@ -485,7 +561,7 @@ class DataSet(object): """ - def __init__(self, name, get_samplelist = True, group_name = None): + def __init__(self, name, get_samplelist=True, group_name=None): assert name, "Need a name" self.name = name @@ -493,21 +569,23 @@ class DataSet(object): self.shortname = None self.fullname = None self.type = None - self.data_scale = None #ZS: For example log2 + self.data_scale = None # ZS: For example log2 self.setup() - if self.type == "Temp": #Need to supply group name as input if temp trait - self.group = DatasetGroup(self, group_name) # sets self.group and self.group_id and gets genotype + if self.type == "Temp": # Need to supply group name as input if temp trait + # sets self.group and self.group_id and gets genotype + self.group = DatasetGroup(self, name=group_name) else: self.check_confidentiality() self.retrieve_other_names() - self.group = DatasetGroup(self) # sets self.group and self.group_id and gets genotype + # sets self.group and self.group_id and gets genotype + self.group = DatasetGroup(self) + self.accession_id = self.get_accession_id() if get_samplelist == True: - self.group.get_samplelist() + self.group.get_samplelist() self.species = species.TheSpecies(self) - def get_desc(self): """Gets overridden later, at least for Temp...used by trait's get_given_name""" return None @@ -517,6 +595,31 @@ class DataSet(object): def riset(): Weve_Renamed_This_As_Group + def get_accession_id(self): + if self.type == "Publish": + results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where + InbredSet.Name = %s and + PublishFreeze.InbredSetId = InbredSet.Id and + InfoFiles.InfoPageName = PublishFreeze.Name and + PublishFreeze.public > 0 and + PublishFreeze.confidentiality < 1 order by + PublishFreeze.CreateTime desc""", (self.group.name)).fetchone() + elif self.type == "Geno": + results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, GenoFreeze, InbredSet where + InbredSet.Name = %s and + GenoFreeze.InbredSetId = InbredSet.Id and + InfoFiles.InfoPageName = GenoFreeze.ShortName and + GenoFreeze.public > 0 and + GenoFreeze.confidentiality < 1 order by + GenoFreeze.CreateTime desc""", (self.group.name)).fetchone() + else: + results = None + + if results != None: + return str(results[0]) + else: + return "None" + def retrieve_other_names(self): """This method fetches the the dataset names in search_result. @@ -529,29 +632,26 @@ class DataSet(object): """ - try: if self.type == "ProbeSet": query_args = tuple(escape(x) for x in ( - str(webqtlConfig.PUBLICTHRESH), self.name, self.name, self.name)) self.id, self.name, self.fullname, self.shortname, self.data_scale, self.tissue = fetch1(""" -SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, ProbeSetFreeze.DataScale, Tissue.Name -FROM ProbeSetFreeze, ProbeFreeze, Tissue -WHERE ProbeSetFreeze.public > %s -AND ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id -AND ProbeFreeze.TissueId = Tissue.Id -AND (ProbeSetFreeze.Name = '%s' OR ProbeSetFreeze.FullName = '%s' OR ProbeSetFreeze.ShortName = '%s') - """ % (query_args),"/dataset/"+self.name+".json", - lambda r: (r["id"],r["name"],r["full_name"],r["short_name"],r["data_scale"],r["tissue"]) + SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, ProbeSetFreeze.DataScale, Tissue.Name + FROM ProbeSetFreeze, ProbeFreeze, Tissue + WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id + AND ProbeFreeze.TissueId = Tissue.Id + AND (ProbeSetFreeze.Name = '%s' OR ProbeSetFreeze.FullName = '%s' OR ProbeSetFreeze.ShortName = '%s') + """ % (query_args), "/dataset/"+self.name+".json", + lambda r: (r["id"], r["name"], r["full_name"], + r["short_name"], r["data_scale"], r["tissue"]) ) else: query_args = tuple(escape(x) for x in ( (self.type + "Freeze"), - str(webqtlConfig.PUBLICTHRESH), self.name, self.name, self.name)) @@ -560,12 +660,12 @@ AND (ProbeSetFreeze.Name = '%s' OR ProbeSetFreeze.FullName = '%s' OR ProbeSetFre self.id, self.name, self.fullname, self.shortname = fetchone(""" SELECT Id, Name, FullName, ShortName FROM %s - WHERE public > %s AND - (Name = '%s' OR FullName = '%s' OR ShortName = '%s') - """ % (query_args)) + WHERE (Name = '%s' OR FullName = '%s' OR ShortName = '%s') + """ % (query_args)) except TypeError: - logger.debug("Dataset {} is not yet available in GeneNetwork.".format(self.name)) + logger.debug( + "Dataset {} is not yet available in GeneNetwork.".format(self.name)) pass def get_trait_data(self, sample_list=None): @@ -623,7 +723,7 @@ AND (ProbeSetFreeze.Name = '%s' OR ProbeSetFreeze.FullName = '%s' OR ProbeSetFre and {}.Id = {}XRef.{}Id order by {}.Id """.format(*mescape(self.type, self.type, self.type, self.name, - dataset_type, self.type, dataset_type, dataset_type)) + dataset_type, self.type, dataset_type, dataset_type)) else: query += """ WHERE {}XRef.{}FreezeId = {}Freeze.Id @@ -631,7 +731,7 @@ AND (ProbeSetFreeze.Name = '%s' OR ProbeSetFreeze.FullName = '%s' OR ProbeSetFre and {}.Id = {}XRef.{}Id order by {}.Id """.format(*mescape(self.type, self.type, self.type, self.type, - self.name, dataset_type, self.type, self.type, dataset_type)) + self.name, dataset_type, self.type, self.type, dataset_type)) #logger.debug("trait data query: ", query) @@ -651,6 +751,7 @@ AND (ProbeSetFreeze.Name = '%s' OR ProbeSetFreeze.FullName = '%s' OR ProbeSetFre self.trait_data[trait_name] += ( trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) + class PhenotypeDataSet(DataSet): DS_NAME_MAP['Publish'] = 'PhenotypeDataSet' @@ -660,24 +761,26 @@ class PhenotypeDataSet(DataSet): # Fields in the database table self.search_fields = ['Phenotype.Post_publication_description', - 'Phenotype.Pre_publication_description', - 'Phenotype.Pre_publication_abbreviation', - 'Phenotype.Post_publication_abbreviation', - 'Phenotype.Lab_code', - 'Publication.PubMed_ID', - 'Publication.Abstract', - 'Publication.Title', - 'Publication.Authors', - 'PublishXRef.Id'] + 'Phenotype.Pre_publication_description', + 'Phenotype.Pre_publication_abbreviation', + 'Phenotype.Post_publication_abbreviation', + 'PublishXRef.mean', + 'Phenotype.Lab_code', + 'Publication.PubMed_ID', + 'Publication.Abstract', + 'Publication.Title', + 'Publication.Authors', + 'PublishXRef.Id'] # Figure out what display_fields is - self.display_fields = ['name', + self.display_fields = ['name', 'group_code', 'pubmed_id', 'pre_publication_description', 'post_publication_description', 'original_description', 'pre_publication_abbreviation', 'post_publication_abbreviation', + 'mean', 'lab_code', 'submitter', 'owner', 'authorized_users', @@ -689,13 +792,13 @@ class PhenotypeDataSet(DataSet): # Fields displayed in the search results table header self.header_fields = ['Index', - 'Record', - 'Description', - 'Authors', - 'Year', - 'Max LRS', - 'Max LRS Location', - 'Additive Effect'] + 'Record', + 'Description', + 'Authors', + 'Year', + 'Max LRS', + 'Max LRS Location', + 'Additive Effect'] self.type = 'Publish' @@ -713,21 +816,7 @@ class PhenotypeDataSet(DataSet): # (Urgently?) Need to write this pass - def get_trait_list(self): - query = """ - select PublishXRef.Id - from PublishXRef, PublishFreeze - where PublishFreeze.InbredSetId=PublishXRef.InbredSetId - and PublishFreeze.Id = {} - """.format(escape(str(self.id))) - logger.sql(query) - results = g.db.execute(query).fetchall() - trait_data = {} - for trait in results: - trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) - return trait_data - - def get_trait_info(self, trait_list, species = ''): + def get_trait_info(self, trait_list, species=''): for this_trait in trait_list: if not this_trait.haveinfo: @@ -735,12 +824,12 @@ class PhenotypeDataSet(DataSet): description = this_trait.post_publication_description - #If the dataset is confidential and the user has access to confidential - #phenotype traits, then display the pre-publication description instead - #of the post-publication description + # If the dataset is confidential and the user has access to confidential + # phenotype traits, then display the pre-publication description instead + # of the post-publication description if this_trait.confidential: this_trait.description_display = "" - continue # for now + continue # for now, because no authorization features if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait( privilege=self.privilege, @@ -762,11 +851,9 @@ class PhenotypeDataSet(DataSet): if this_trait.pubmed_id: this_trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % this_trait.pubmed_id - #LRS and its location + # LRS and its location this_trait.LRS_score_repr = "N/A" - this_trait.LRS_score_value = 0 this_trait.LRS_location_repr = "N/A" - this_trait.LRS_location_value = 1000000 if this_trait.lrs: query = """ @@ -783,18 +870,9 @@ class PhenotypeDataSet(DataSet): LRS_Chr = result[0] LRS_Mb = result[1] - #XZ: LRS_location_value is used for sorting - try: - LRS_location_value = int(LRS_Chr)*1000 + float(LRS_Mb) - except: - if LRS_Chr.upper() == 'X': - LRS_location_value = 20*1000 + float(LRS_Mb) - else: - LRS_location_value = ord(str(LRS_chr).upper()[0])*1000 + float(LRS_Mb) - this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs - this_trait.LRS_score_value = LRS_score_value = this_trait.lrs - this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb)) + this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % ( + LRS_Chr, float(LRS_Mb)) def retrieve_sample_data(self, trait): query = """ @@ -855,45 +933,19 @@ class GenotypeDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - def get_trait_list(self): - query = """ - select Geno.Name - from Geno, GenoXRef - where GenoXRef.GenoId = Geno.Id - and GenoFreezeId = {} - """.format(escape(str(self.id))) - logger.sql(query) - results = g.db.execute(query).fetchall() - trait_data = {} - for trait in results: - trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) - return trait_data - def get_trait_info(self, trait_list, species=None): for this_trait in trait_list: if not this_trait.haveinfo: this_trait.retrieveInfo() - #XZ: trait_location_value is used for sorting - trait_location_repr = 'N/A' - trait_location_value = 1000000 - if this_trait.chr and this_trait.mb: - try: - trait_location_value = int(this_trait.chr)*1000 + this_trait.mb - except: - if this_trait.chr.upper() == 'X': - trait_location_value = 20*1000 + this_trait.mb - else: - trait_location_value = ord(str(this_trait.chr).upper()[0])*1000 + this_trait.mb - - this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb) ) - this_trait.location_value = trait_location_value + this_trait.location_repr = 'Chr%s: %.6f' % ( + this_trait.chr, float(this_trait.mb)) def retrieve_sample_data(self, trait): query = """ SELECT - Strain.Name, GenoData.value, GenoSE.error, GenoData.Id, Strain.Name2 + Strain.Name, GenoData.value, GenoSE.error, "N/A", Strain.Name2 FROM (GenoData, GenoFreeze, Strain, Geno, GenoXRef) left join GenoSE on @@ -945,6 +997,7 @@ class MrnaAssayDataSet(DataSet): 'blatseq', 'targetseq', 'chipid', 'comments', 'strand_probe', 'strand_gene', + 'proteinid', 'uniprotid', 'probe_set_target_region', 'probe_set_specificity', 'probe_set_blat_score', @@ -956,14 +1009,14 @@ class MrnaAssayDataSet(DataSet): # Fields displayed in the search results table header self.header_fields = ['Index', - 'Record', - 'Symbol', - 'Description', - 'Location', - 'Mean', - 'Max LRS', - 'Max LRS Location', - 'Additive Effect'] + 'Record', + 'Symbol', + 'Description', + 'Location', + 'Mean', + 'Max LRS', + 'Max LRS Location', + 'Additive Effect'] # Todo: Obsolete or rename this field self.type = 'ProbeSet' @@ -979,24 +1032,9 @@ class MrnaAssayDataSet(DataSet): ProbeSetFreeze.Name = "%s" ''' % escape(self.name) - def check_confidentiality(self): return geno_mrna_confidentiality(self) - def get_trait_list_1(self): - query = """ - select ProbeSet.Name - from ProbeSet, ProbeSetXRef - where ProbeSetXRef.ProbeSetId = ProbeSet.Id - and ProbeSetFreezeId = {} - """.format(escape(str(self.id))) - logger.sql(query) - results = g.db.execute(query).fetchall() - trait_data = {} - for trait in results: - trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) - return trait_data - def get_trait_info(self, trait_list=None, species=''): # Note: setting trait_list to [] is probably not a great idea. @@ -1011,10 +1049,12 @@ class MrnaAssayDataSet(DataSet): if not this_trait.symbol: this_trait.symbol = "N/A" - #XZ, 12/08/2008: description - #XZ, 06/05/2009: Rob asked to add probe target description - description_string = unicode(str(this_trait.description).strip(codecs.BOM_UTF8), 'utf-8') - target_string = unicode(str(this_trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') + # XZ, 12/08/2008: description + # XZ, 06/05/2009: Rob asked to add probe target description + description_string = unicode( + str(this_trait.description).strip(codecs.BOM_UTF8), 'utf-8') + target_string = unicode( + str(this_trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') if len(description_string) > 1 and description_string != 'None': description_display = description_string @@ -1028,31 +1068,13 @@ class MrnaAssayDataSet(DataSet): # Save it for the jinja2 template this_trait.description_display = description_display - #XZ: trait_location_value is used for sorting - trait_location_repr = 'N/A' - trait_location_value = 1000000 - if this_trait.chr and this_trait.mb: - #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") - #This is so we can convert the location to a number used for sorting - trait_location_value = self.convert_location_to_value(this_trait.chr, this_trait.mb) - #try: - # trait_location_value = int(this_trait.chr)*1000 + this_trait.mb - #except ValueError: - # if this_trait.chr.upper() == 'X': - # trait_location_value = 20*1000 + this_trait.mb - # else: - # trait_location_value = (ord(str(this_trait.chr).upper()[0])*1000 + - # this_trait.mb) - - #ZS: Put this in function currently called "convert_location_to_value" - this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, - float(this_trait.mb)) - this_trait.location_value = trait_location_value - - #Get mean expression value + this_trait.location_repr = 'Chr%s: %.6f' % ( + this_trait.chr, float(this_trait.mb)) + + # Get mean expression value query = ( - """select ProbeSetXRef.mean from ProbeSetXRef, ProbeSet + """select ProbeSetXRef.mean from ProbeSetXRef, ProbeSet where ProbeSetXRef.ProbeSetFreezeId = %s and ProbeSet.Id = ProbeSetXRef.ProbeSetId and ProbeSet.Name = '%s' @@ -1068,13 +1090,11 @@ class MrnaAssayDataSet(DataSet): if mean: this_trait.mean = "%2.3f" % mean - #LRS and its location + # LRS and its location this_trait.LRS_score_repr = 'N/A' - this_trait.LRS_score_value = 0 this_trait.LRS_location_repr = 'N/A' - this_trait.LRS_location_value = 1000000 - #Max LRS and its Locus location + # Max LRS and its Locus location if this_trait.lrs and this_trait.locus: query = """ select Geno.Chr, Geno.Mb from Geno, Species @@ -1087,49 +1107,23 @@ class MrnaAssayDataSet(DataSet): if result: lrs_chr, lrs_mb = result - #XZ: LRS_location_value is used for sorting - lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb) this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs - this_trait.LRS_score_value = this_trait.lrs - this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb)) - - - def convert_location_to_value(self, chromosome, mb): - try: - location_value = int(chromosome)*1000 + float(mb) - except ValueError: - if chromosome.upper() == 'X': - location_value = 20*1000 + float(mb) - else: - location_value = (ord(str(chromosome).upper()[0])*1000 + - float(mb)) - - return location_value + this_trait.LRS_location_repr = 'Chr%s: %.6f' % ( + lrs_chr, float(lrs_mb)) - def get_sequence(self): - query = """ - SELECT - ProbeSet.BlatSeq - FROM - ProbeSet, ProbeSetFreeze, ProbeSetXRef - WHERE - ProbeSet.Id=ProbeSetXRef.ProbeSetId and - ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and - ProbeSet.Name = %s - ProbeSetFreeze.Name = %s - """ % (escape(self.name), escape(self.dataset.name)) - logger.sql(query) - results = g.db.execute(query).fetchone() - return results[0] + return trait_list def retrieve_sample_data(self, trait): query = """ SELECT - Strain.Name, ProbeSetData.value, ProbeSetSE.error, ProbeSetData.Id, Strain.Name2 + Strain.Name, ProbeSetData.value, ProbeSetSE.error, NStrain.count, Strain.Name2 FROM (ProbeSetData, ProbeSetFreeze, Strain, ProbeSet, ProbeSetXRef) left join ProbeSetSE on (ProbeSetSE.DataId = ProbeSetData.Id AND ProbeSetSE.StrainId = ProbeSetData.StrainId) + left join NStrain on + (NStrain.DataId = ProbeSetData.Id AND + NStrain.StrainId = ProbeSetData.StrainId) WHERE ProbeSet.Name = '%s' AND ProbeSetXRef.ProbeSetId = ProbeSet.Id AND ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND @@ -1144,7 +1138,6 @@ class MrnaAssayDataSet(DataSet): #logger.debug("RETRIEVED RESULTS HERE:", results) return results - def retrieve_genes(self, column_name): query = """ select ProbeSet.Name, ProbeSet.%s @@ -1180,7 +1173,6 @@ class TempDataSet(DataSet): self.fullname = 'Temporary Storage' self.shortname = 'Temp' - @staticmethod def handle_pca(desc): if 'PCA' in desc: @@ -1198,19 +1190,6 @@ class TempDataSet(DataSet): desc = self.handle_pca(desc) return desc - def get_group(self): - query = """ - SELECT - InbredSet.Name, InbredSet.Id - FROM - InbredSet, Temp - WHERE - Temp.InbredSetId = InbredSet.Id AND - Temp.Name = "%s" - """ % self.name - logger.sql(query) - self.group, self.group_id = g.db.execute(query).fetchone() - def retrieve_sample_data(self, trait): query = """ SELECT @@ -1234,7 +1213,7 @@ def geno_mrna_confidentiality(ob): #logger.debug("dataset_table [%s]: %s" % (type(dataset_table), dataset_table)) query = '''SELECT Id, Name, FullName, confidentiality, - AuthorisedUsers FROM %s WHERE Name = "%s"''' % (dataset_table,ob.name) + AuthorisedUsers FROM %s WHERE Name = "%s"''' % (dataset_table, ob.name) logger.sql(query) result = g.db.execute(query) diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index eb836e6c..6fec5dcd 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -18,21 +18,11 @@ class MrnaAssayTissueData(object): def __init__(self, gene_symbols=None): self.gene_symbols = gene_symbols - self.have_data = False if self.gene_symbols == None: self.gene_symbols = [] - #print("self.gene_symbols:", self.gene_symbols) - self.data = collections.defaultdict(Bunch) - #self.gene_id_dict ={} - #self.data_id_dict = {} - #self.chr_dict = {} - #self.mb_dict = {} - #self.desc_dict = {} - #self.probe_target_desc_dict = {} - query = '''select t.Symbol, t.GeneId, t.DataId, t.Chr, t.Mb, t.description, t.Probe_Target_Description from ( select Symbol, max(Mean) as maxmean @@ -52,7 +42,6 @@ class MrnaAssayTissueData(object): in_clause = db_tools.create_in_clause(gene_symbols) #ZS: This was in the query, not sure why: http://docs.python.org/2/library/string.html?highlight=lower#string.lower - query += ''' Symbol in {} group by Symbol) as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol and t.Mean = x.maxmean; @@ -67,9 +56,7 @@ class MrnaAssayTissueData(object): for result in results: symbol = result[0] - #if symbol.lower() in [gene_symbol.lower() for gene_symbol in gene_symbols]: if symbol.lower() in lower_symbols: - #gene_symbols.append(symbol) symbol = symbol.lower() self.data[symbol].gene_id = result.GeneId @@ -79,8 +66,6 @@ class MrnaAssayTissueData(object): self.data[symbol].description = result.description self.data[symbol].probe_target_description = result.Probe_Target_Description - print("self.data: ", pf(self.data)) - ########################################################################### #Input: cursor, symbolList (list), dataIdDict(Dict) #output: symbolValuepairDict (dictionary):one dictionary of Symbol and Value Pair, @@ -107,53 +92,4 @@ class MrnaAssayTissueData(object): else: symbol_values_dict[result.Symbol.lower()].append(result.value) - #for symbol in self.data: - # data_id = self.data[symbol].data_id - # symbol_values_dict[symbol] = self.get_tissue_values(data_id) - - - return symbol_values_dict - - - #def get_tissue_values(self, data_id): - # """Gets the tissue values for a particular gene""" - # - # tissue_values=[] - # - # query = """SELECT value, id - # FROM TissueProbeSetData - # WHERE Id IN {}""".format(db_tools.create_in_clause(data_id)) - # - # #try : - # results = g.db.execute(query).fetchall() - # for result in results: - # tissue_values.append(result.value) - # #symbol_values_dict[symbol] = value_list - # #except: - # # symbol_values_pairs[symbol] = None - # - # return tissue_values - -######################################################################################################## -#input: cursor, symbolList (list), dataIdDict(Dict): key is symbol -#output: SymbolValuePairDict(dictionary):one dictionary of Symbol and Value Pair. -# key is symbol, value is one list of expression values of one probeSet. -#function: wrapper function for getSymbolValuePairDict function -# build gene symbol list if necessary, cut it into small lists if necessary, -# then call getSymbolValuePairDict function and merge the results. -######################################################################################################## - -#def get_trait_symbol_and_tissue_values(symbol_list=None): -# tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list) -# -# #symbolList, -# #geneIdDict, -# #dataIdDict, -# #ChrDict, -# #MbDict, -# #descDict, -# #pTargetDescDict = getTissueProbeSetXRefInfo( -# # GeneNameLst=GeneNameLst,TissueProbeSetFreezeId=TissueProbeSetFreezeId) -# -# if len(tissue_data.gene_symbols): -# return get_symbol_values_pairs(tissue_data) + return symbol_values_dict \ No newline at end of file diff --git a/wqflask/base/species.py b/wqflask/base/species.py index ce763fc3..6d99af65 100644 --- a/wqflask/base/species.py +++ b/wqflask/base/species.py @@ -14,23 +14,13 @@ from utility.logger import getLogger logger = getLogger(__name__ ) class TheSpecies(object): - def __init__(self, dataset): - self.dataset = dataset - #print("self.dataset is:", pf(self.dataset.__dict__)) - self.chromosomes = Chromosomes(self.dataset) - self.genome_mb_length = self.chromosomes.get_genome_mb_length() - - #@property - #def chromosomes(self): - # chromosomes = [("All", -1)] - # - # for counter, genotype in enumerate(self.dataset.group.genotype): - # if len(genotype) > 1: - # chromosomes.append((genotype.name, counter)) - # - # print("chromosomes is: ", pf(chromosomes)) - # - # return chromosomes + def __init__(self, dataset=None, species_name=None): + if species_name != None: + self.name = species_name + self.chromosomes = Chromosomes(species=self.name) + else: + self.dataset = dataset + self.chromosomes = Chromosomes(dataset=self.dataset) class IndChromosome(object): def __init__(self, name, length): @@ -42,17 +32,22 @@ class IndChromosome(object): """Chromosome length in megabases""" return self.length / 1000000 - def set_cm_length(self, genofile_chr): - self.cm_length = genofile_chr[-1].cM - genofile_chr[0].cM - - class Chromosomes(object): - def __init__(self, dataset): - self.dataset = dataset + def __init__(self, dataset=None, species=None): self.chromosomes = collections.OrderedDict() + if species != None: + query = """ + Select + Chr_Length.Name, Chr_Length.OrderId, Length from Chr_Length, Species + where + Chr_Length.SpeciesId = Species.SpeciesId AND + Species.Name = '%s' + Order by OrderId + """ % species.capitalize() + else: + self.dataset = dataset - - query = """ + query = """ Select Chr_Length.Name, Chr_Length.OrderId, Length from Chr_Length, InbredSet where @@ -64,64 +59,4 @@ class Chromosomes(object): results = g.db.execute(query).fetchall() for item in results: - self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length) - - self.set_mb_graph_interval() - #self.get_cm_length_list() - - - def set_mb_graph_interval(self): - """Empirical megabase interval""" - - if self.chromosomes: - self.mb_graph_interval = self.get_genome_mb_length()/(len(self.chromosomes)*12) - else: - self.mb_graph_interval = 1 - - #if self.chromosomes: - #assert self.chromosomes, "Have to add some code back in apparently to set it to 1" - #self.mb_graph_interval = self.get_genome_mb_length()/(len(self.chromosomes)*12) - #else: - #self.mb_graph_interval = 1 - - - def get_genome_mb_length(self): - """Gets the sum of each chromosome's length in megabases""" - - return sum([ind_chromosome.mb_length for ind_chromosome in self.chromosomes.values()]) - - - def get_genome_cm_length(self): - """Gets the sum of each chromosome's length in centimorgans""" - - return sum([ind_chromosome.cm_length for ind_chromosome in self.chromosomes.values()]) - - def get_cm_length_list(self): - """Chromosome length in centimorgans - - Calculates the length in centimorgans by subtracting the centimorgan position - of the last marker in a chromosome by the position of the first marker - - """ - - self.dataset.group.read_genotype_file() - - self.cm_length_list = [] - - for chromosome in self.dataset.group.genotype: - self.cm_length_list.append(chromosome[-1].cM - chromosome[0].cM) - - print("self.cm_length_list:", pf(self.cm_length_list)) - - assert len(self.cm_length_list) == len(self.chromosomes), "Uh-oh lengths should be equal!" - for counter, chromosome in enumerate(self.chromosomes.values()): - chromosome.cm_length = self.cm_length_list[counter] - #self.chromosomes[counter].cm_length = item - - for key, value in self.chromosomes.items(): - print("bread - %s: %s" % (key, pf(vars(value)))) - - -# Testing -#if __name__ == '__main__': -# foo = dict(bar=dict(length)) + self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length) \ No newline at end of file diff --git a/wqflask/base/template.py b/wqflask/base/template.py deleted file mode 100644 index aa8f90dc..00000000 --- a/wqflask/base/template.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# This program is available from Source Forge: at GeneNetwork Project -# (sourceforge.net/projects/genenetwork/). -# -# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) -# at rwilliams@uthsc.edu and xzhou15@uthsc.edu -# -# -# -# This module is used by GeneNetwork project (www.genenetwork.org) -# -# Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 - -template = """ -<?XML VERSION="1.0" ENCODING="UTF-8"> -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<HTML> -<HEAD> -<TITLE>%s</TITLE> - -<META http-equiv=Content-Type content="text/html; charset=iso-8859-1"> -<META NAME="keywords" CONTENT="genetics, bioinformatics, genome, phenome, gene expression, complex trait analysis, gene mapping, SNP, quantitative trait locus QTL, expression eQTL, WebQTL, Traitnet, Traitnetwork, personalized medicine"> -<META NAME="description" CONTENT ="GeneNetwork is a free scientific web resource used to study relationships between differences in genes, environmental factors, phenotypes, and disease risk." > -<META NAME="author" CONTENT ="GeneNetwork developers" > -<META NAME="geo.placename" CONTENT ="Memphis, TN" > -<META NAME="geo.region" CONTENT="US-TN"> -%s -<LINK REL="stylesheet" TYPE="text/css" HREF='/css/general.css'> -<LINK REL="stylesheet" TYPE="text/css" HREF='/css/menu.css'> -<link rel="stylesheet" media="all" type="text/css" href="/css/tabbed_pages.css" /> -<LINK REL="apple-touch-icon" href="/images/ipad_icon3.png" /> -<link type="text/css" href='/css/custom-theme/jquery-ui-1.8.12.custom.css' rel='Stylesheet' /> -<link type="text/css" href='/css/tab_style.css' rel='Stylesheet' /> - -<script type="text/javascript" src="/javascript/jquery-1.5.2.min.js"></script> -<SCRIPT SRC="/javascript/webqtl.js"></SCRIPT> -<SCRIPT SRC="/javascript/dhtml.js"></SCRIPT> -<SCRIPT SRC="/javascript/tablesorter.js"></SCRIPT> -<SCRIPT SRC="/javascript/jqueryFunction.js"></SCRIPT> -<script src="/javascript/tabbed_pages.js" type="text/javascript"></script> -<script src="/javascript/jquery-ui-1.8.12.custom.min.js" type="text/javascript"></script> -%s - -<script type="text/javascript"> - var _gaq = _gaq || []; - _gaq.push(['_setAccount', 'UA-3782271-1']); - _gaq.push(['_trackPageview']); - (function() { - var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; - ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; - var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); - })(); -</script> -</HEAD> -<BODY bottommargin="2" leftmargin="2" rightmargin="2" topmargin="2" text=#000000 bgColor=#ffffff %s> -%s -<TABLE cellSpacing=5 cellPadding=4 width="100%%" border=0> - <TBODY> - <!-- Start of header --> - <TR> - %s - </TR> - <!-- End of header --> - - <!-- Start of body --> - <TR> - <TD bgColor=#eeeeee class="solidBorder"> - <Table width= "100%%" cellSpacing=0 cellPadding=5> - <TR> - %s - </TR> - </TABLE> - </TD> - </TR> - <!-- End of body --> - - <!-- Start of footer --> - <TR> - <TD align=center bgColor=#ddddff class="solidBorder"> - <TABLE width="90%%">%s</table> - </td> - </TR> - <!-- End of footer --> -</TABLE> - -<!-- menu script itself. you should not modify this file --> -<script language="JavaScript" src="/javascript/menu_new.js"></script> -<!-- items structure. menu hierarchy and links are stored there --> -<script language="JavaScript" src="/javascript/menu_items.js"></script> -<!-- files with geometry and styles structures --> -<script language="JavaScript" src="/javascript/menu_tpl.js"></script> -<script language="JavaScript"> - <!--// - // Note where menu initialization block is located in HTML document. - // Don't try to position menu locating menu initialization block in - // some table cell or other HTML element. Always put it before </body> - // each menu gets two parameters (see demo files) - // 1. items structure - // 2. geometry structure - new menu (MENU_ITEMS, MENU_POS); - // make sure files containing definitions for these variables are linked to the document - // if you got some javascript error like "MENU_POS is not defined", then you've made syntax - // error in menu_tpl.js file or that file isn't linked properly. - - // also take a look at stylesheets loaded in header in order to set styles - //--> -</script> -</BODY> -</HTML> -""" diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index e22a51e4..7666348e 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -1,16 +1,23 @@ from __future__ import absolute_import, division, print_function +import os import string import resource import codecs - -from htmlgen import HTMLgen2 as HT +import requests +import random +import urllib from base import webqtlConfig from base.webqtlCaseData import webqtlCaseData from base.data_set import create_dataset from db import webqtlDatabaseFunction from utility import webqtlUtil +from utility import hmac +from utility.authentication_tools import check_resource_availability +from utility.tools import GN2_BASE_URL, GN_VERSION +from utility.redis_tools import get_redis_conn, get_resource_id, get_resource_info +Redis = get_redis_conn() from wqflask import app @@ -18,16 +25,35 @@ import simplejson as json from MySQLdb import escape_string as escape from pprint import pformat as pf -from flask import Flask, g, request +from flask import Flask, g, request, url_for, redirect, make_response, render_template from utility.logger import getLogger logger = getLogger(__name__ ) -from wqflask import user_manager +def create_trait(**kw): + assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name"; + + permitted = True + if kw.get('name'): + if kw.get('dataset_name'): + if kw.get('dataset_name') != "Temp": + dataset = create_dataset(kw.get('dataset_name')) + else: + dataset = kw.get('dataset') + + if kw.get('dataset_name') != "Temp": + if dataset.type == 'Publish': + permissions = check_resource_availability(dataset, kw.get('name')) + else: + permissions = check_resource_availability(dataset) -def print_mem(stage=""): - mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - print("{}: {}".format(stage, mem/1024)) + if "view" in permissions['data']: + the_trait = GeneralTrait(**kw) + if the_trait.dataset.type != "Temp": + the_trait = retrieve_trait_info(the_trait, the_trait.dataset, get_qtl_info=kw.get('get_qtl_info')) + return the_trait + else: + return None class GeneralTrait(object): """ @@ -39,18 +65,21 @@ class GeneralTrait(object): def __init__(self, get_qtl_info=False, get_sample_info=True, **kw): # xor assertion assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name"; + self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. if kw.get('dataset_name'): - self.dataset = create_dataset(kw.get('dataset_name')) - #print(" in GeneralTrait created dataset:", self.dataset) + if kw.get('dataset_name') == "Temp": + temp_group = self.name.split("_")[2] + self.dataset = create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = temp_group) + else: + self.dataset = create_dataset(kw.get('dataset_name')) else: self.dataset = kw.get('dataset') - self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. - #print("THE NAME IS:", self.name) self.cellid = kw.get('cellid') self.identification = kw.get('identification', 'un-named trait') self.haveinfo = kw.get('haveinfo', False) self.sequence = kw.get('sequence') # Blat sequence, available for ProbeSet self.data = kw.get('data', {}) + self.view = True # Sets defaults self.locus = None @@ -61,11 +90,10 @@ class GeneralTrait(object): self.num_overlap = None self.strand_probe = None self.symbol = None + self.display_name = self.name self.LRS_score_repr = "N/A" - self.LRS_score_value = 0 self.LRS_location_repr = "N/A" - self.LRS_location_value = 1000000 if kw.get('fullname'): name2 = value.split("::") @@ -77,94 +105,8 @@ class GeneralTrait(object): # Todo: These two lines are necessary most of the time, but perhaps not all of the time # So we could add a simple if statement to short-circuit this if necessary - if self.dataset.type != "Temp": - self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info) - if get_sample_info != False: - self = retrieve_sample_data(self, self.dataset) - - - def get_name(self): - stringy = "" - if self.dataset and self.name: - stringy = "%s::%s" % (self.dataset, self.name) - if self.cellid: - stringy += "::" + self.cellid - else: - stringy = self.description - return stringy - - - def get_given_name(self): - """ - when user enter a trait or GN generate a trait, user want show the name - not the name that generated by GN randomly, the two follow function are - used to give the real name and the database. displayName() will show the - database also, getGivenName() just show the name. - For other trait, displayName() as same as getName(), getGivenName() as - same as self.name - - Hongqiang 11/29/07 - - """ - stringy = self.name - if self.dataset and self.name: - desc = self.dataset.get_desc() - if desc: - #desc = self.handle_pca(desc) - stringy = desc - return stringy - - - def display_name(self): - stringy = "" - if self.dataset and self.name: - desc = self.dataset.get_desc() - #desc = self.handle_pca(desc) - if desc: - #desc = self.handle_pca(desc) - #stringy = desc - #if desc.__contains__('PCA'): - # desc = desc[desc.rindex(':')+1:].strip() - #else: - # desc = desc[:desc.index('entered')].strip() - #desc = self.handle_pca(desc) - stringy = "%s::%s" % (self.dataset, desc) - else: - stringy = "%s::%s" % (self.dataset, self.name) - if self.cellid: - stringy += "::" + self.cellid - else: - stringy = self.description - - return stringy - - - #def __str__(self): - # #return "%s %s" % (self.getName(), self.group) - # return self.getName() - #__str__ = getName - #__repr__ = __str__ - - def export_data(self, samplelist, the_type="val"): - """ - export data according to samplelist - mostly used in calculating correlation - - """ - result = [] - for sample in samplelist: - if self.data.has_key(sample): - if the_type=='val': - result.append(self.data[sample].val) - elif the_type=='var': - result.append(self.data[sample].var) - elif the_type=='N': - result.append(self.data[sample].N) - else: - raise KeyError, `the_type`+' the_type is incorrect.' - else: - result.append(None) - return result + if get_sample_info != False: + self = retrieve_sample_data(self, self.dataset) def export_informative(self, include_variance=0): """ @@ -185,19 +127,6 @@ class GeneralTrait(object): sample_aliases.append(sample_data.name2) return samples, vals, the_vars, sample_aliases - - @property - def name_header_fmt(self): - '''Return a human-readable name for use in page header''' - if self.dataset.type == 'ProbeSet': - return self.symbol - elif self.dataset.type == 'Geno': - return self.name - elif self.dataset.type == 'Publish': - return self.post_publication_abbreviation - else: - return "unnamed" - @property def description_fmt(self): '''Return a text formated description''' @@ -206,6 +135,8 @@ class GeneralTrait(object): formatted = self.description if self.probe_target_description: formatted += "; " + self.probe_target_description + else: + formatted = "Not available" elif self.dataset.type == 'Publish': if self.confidential: formatted = self.pre_publication_description @@ -213,16 +144,42 @@ class GeneralTrait(object): formatted = self.post_publication_description else: formatted = "Not available" + return formatted @property def alias_fmt(self): '''Return a text formatted alias''' - if self.alias: + + alias = 'Not available' + if getattr(self, "alias", None): alias = string.replace(self.alias, ";", " ") alias = string.join(string.split(alias), ", ") - else: - alias = 'Not available' + + return alias + + @property + def wikidata_alias_fmt(self): + '''Return a text formatted alias''' + + alias = 'Not available' + if self.symbol: + human_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.upper()) + mouse_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.capitalize()) + other_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.lower()) + + if human_response and mouse_response and other_response: + alias_list = json.loads(human_response.content) + json.loads(mouse_response.content) + json.loads(other_response.content) + + filtered_aliases = [] + seen = set() + for item in alias_list: + if item in seen: + continue + else: + filtered_aliases.append(item) + seen.add(item) + alias = "; ".join(filtered_aliases) return alias @@ -250,60 +207,33 @@ class GeneralTrait(object): fmt += (' on the minus strand ') return fmt - -# In ProbeSet, there are maybe several annotations match one sequence -# so we need use sequence(BlatSeq) as the identification, when we update -# one annotation, we update the others who match the sequence also. -# -# Hongqiang Li, 3/3/2008 -def getSequence(trait, dataset_name): - dataset = create_dataset(dataset_name) - - if dataset.type == 'ProbeSet': - results = g.db.execute(''' - SELECT - ProbeSet.BlatSeq - FROM - ProbeSet, ProbeSetFreeze, ProbeSetXRef - WHERE - ProbeSet.Id=ProbeSetXRef.ProbeSetId and - ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and - ProbeSet.Name = %s - ProbeSetFreeze.Name = %s - ''', trait.name, dataset.name).fetchone() - - return results[0] def retrieve_sample_data(trait, dataset, samplelist=None): if samplelist == None: samplelist = [] - results = dataset.retrieve_sample_data(trait.name) + if dataset.type == "Temp": + results = Redis.get(trait.name).split() + else: + results = dataset.retrieve_sample_data(trait.name) # Todo: is this necessary? If not remove trait.data.clear() - all_samples_ordered = dataset.group.all_samples_ordered() - if results: - for item in results: - name, value, variance, num_cases, name2 = item - if not samplelist or (samplelist and name in samplelist): - trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) - - return trait - -def convert_location_to_value(chromosome, mb): - try: - location_value = int(chromosome)*1000 + float(mb) - except ValueError: - if chromosome.upper() == 'X': - location_value = 20*1000 + float(mb) + if dataset.type == "Temp": + all_samples_ordered = dataset.group.all_samples_ordered() + for i, item in enumerate(results): + try: + trait.data[all_samples_ordered[i]] = webqtlCaseData(all_samples_ordered[i], float(item)) + except: + pass else: - location_value = (ord(str(chromosome).upper()[0])*1000 + - float(mb)) - - return location_value + for item in results: + name, value, variance, num_cases, name2 = item + if not samplelist or (samplelist and name in samplelist): + trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) + return trait @app.route("/trait/get_sample_data") def get_sample_data(): @@ -311,15 +241,28 @@ def get_sample_data(): trait = params['trait'] dataset = params['dataset'] - trait_ob = GeneralTrait(name=trait, dataset_name=dataset) - - return json.dumps([trait, {key: value.value for key, value in trait_ob.data.iteritems() }]) - - #jsonable_sample_data = {} - #for sample in trait_ob.data.iteritems(): - # jsonable_sample_data[sample] = trait_ob.data[sample].value - # - #return jsonable_sample_data + trait_ob = create_trait(name=trait, dataset_name=dataset) + if trait_ob: + trait_dict = {} + trait_dict['name'] = trait + trait_dict['db'] = dataset + trait_dict['type'] = trait_ob.dataset.type + trait_dict['group'] = trait_ob.dataset.group.name + trait_dict['tissue'] = trait_ob.dataset.tissue + trait_dict['species'] = trait_ob.dataset.group.species + trait_dict['url'] = url_for('show_trait_page', trait_id = trait, dataset = dataset) + trait_dict['description'] = trait_ob.description_display + if trait_ob.dataset.type == "ProbeSet": + trait_dict['symbol'] = trait_ob.symbol + trait_dict['location'] = trait_ob.location_repr + elif trait_ob.dataset.type == "Publish": + if trait_ob.pubmed_id: + trait_dict['pubmed_link'] = trait_ob.pubmed_link + trait_dict['pubmed_text'] = trait_ob.pubmed_text + + return json.dumps([trait_dict, {key: value.value for key, value in trait_ob.data.iteritems() }]) + else: + return None def jsonable(trait): """Return a dict suitable for using as json @@ -332,6 +275,7 @@ def jsonable(trait): return dict(name=trait.name, symbol=trait.symbol, dataset=dataset.name, + dataset_name = dataset.shortname, description=trait.description_display, mean=trait.mean, location=trait.location_repr, @@ -343,7 +287,9 @@ def jsonable(trait): if trait.pubmed_id: return dict(name=trait.name, dataset=dataset.name, + dataset_name = dataset.shortname, description=trait.description_display, + abbreviation=trait.abbreviation, authors=trait.authors, pubmed_text=trait.pubmed_text, pubmed_link=trait.pubmed_link, @@ -354,7 +300,9 @@ def jsonable(trait): else: return dict(name=trait.name, dataset=dataset.name, + dataset_name = dataset.shortname, description=trait.description_display, + abbreviation=trait.abbreviation, authors=trait.authors, pubmed_text=trait.pubmed_text, lrs_score=trait.LRS_score_repr, @@ -364,6 +312,7 @@ def jsonable(trait): elif dataset.type == "Geno": return dict(name=trait.name, dataset=dataset.name, + dataset_name = dataset.shortname, location=trait.location_repr ) else: @@ -385,7 +334,7 @@ def jsonable_table_row(trait, dataset_name, index): additive = "N/A" else: additive = "%.3f" % round(float(trait.additive), 2) - return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', + return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', index, '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>', trait.symbol, @@ -401,7 +350,7 @@ def jsonable_table_row(trait, dataset_name, index): else: additive = "%.2f" % round(float(trait.additive), 2) if trait.pubmed_id: - return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', + return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', index, '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>', trait.description_display, @@ -411,7 +360,7 @@ def jsonable_table_row(trait, dataset_name, index): trait.LRS_location_repr, additive] else: - return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', + return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', index, '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>', trait.description_display, @@ -421,94 +370,108 @@ def jsonable_table_row(trait, dataset_name, index): trait.LRS_location_repr, additive] elif dataset.type == "Geno": - return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', + return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', index, '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>', trait.location_repr] else: return dict() + def retrieve_trait_info(trait, dataset, get_qtl_info=False): assert dataset, "Dataset doesn't exist" - + + resource_id = get_resource_id(dataset, trait.name) if dataset.type == 'Publish': - query = """ - SELECT - PublishXRef.Id, Publication.PubMed_ID, - Phenotype.Pre_publication_description, Phenotype.Post_publication_description, Phenotype.Original_description, - Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation, - Phenotype.Lab_code, Phenotype.Submitter, Phenotype.Owner, Phenotype.Authorized_Users, - Publication.Authors, Publication.Title, Publication.Abstract, - Publication.Journal, Publication.Volume, Publication.Pages, - Publication.Month, Publication.Year, PublishXRef.Sequence, - Phenotype.Units, PublishXRef.comments - FROM - PublishXRef, Publication, Phenotype, PublishFreeze - WHERE - PublishXRef.Id = %s AND - Phenotype.Id = PublishXRef.PhenotypeId AND - Publication.Id = PublishXRef.PublicationId AND - PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND - PublishFreeze.Id = %s - """ % (trait.name, dataset.id) + the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view".format(resource_id, g.user_session.user_id) + else: + the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view&trait={}".format(resource_id, g.user_session.user_id, trait.name) + + try: + response = requests.get(the_url).content + trait_info = json.loads(response) + except: #ZS: I'm assuming the trait is viewable if the try fails for some reason; it should never reach this point unless the user has privileges, since that's dealt with in create_trait + if dataset.type == 'Publish': + query = """ + SELECT + PublishXRef.Id, InbredSet.InbredSetCode, Publication.PubMed_ID, + CAST(Phenotype.Pre_publication_description AS BINARY), + CAST(Phenotype.Post_publication_description AS BINARY), + CAST(Phenotype.Original_description AS BINARY), + CAST(Phenotype.Pre_publication_abbreviation AS BINARY), + CAST(Phenotype.Post_publication_abbreviation AS BINARY), PublishXRef.mean, + Phenotype.Lab_code, Phenotype.Submitter, Phenotype.Owner, Phenotype.Authorized_Users, + CAST(Publication.Authors AS BINARY), CAST(Publication.Title AS BINARY), CAST(Publication.Abstract AS BINARY), + CAST(Publication.Journal AS BINARY), Publication.Volume, Publication.Pages, + Publication.Month, Publication.Year, PublishXRef.Sequence, + Phenotype.Units, PublishXRef.comments + FROM + PublishXRef, Publication, Phenotype, PublishFreeze, InbredSet + WHERE + PublishXRef.Id = %s AND + Phenotype.Id = PublishXRef.PhenotypeId AND + Publication.Id = PublishXRef.PublicationId AND + PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND + PublishXRef.InbredSetId = InbredSet.Id AND + PublishFreeze.Id = %s + """ % (trait.name, dataset.id) + + logger.sql(query) + trait_info = g.db.execute(query).fetchone() + + #XZ, 05/08/2009: Xiaodong add this block to use ProbeSet.Id to find the probeset instead of just using ProbeSet.Name + #XZ, 05/08/2009: to avoid the problem of same probeset name from different platforms. + elif dataset.type == 'ProbeSet': + display_fields_string = ', ProbeSet.'.join(dataset.display_fields) + display_fields_string = 'ProbeSet.' + display_fields_string + query = """ + SELECT %s + FROM ProbeSet, ProbeSetFreeze, ProbeSetXRef + WHERE + ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND + ProbeSetXRef.ProbeSetId = ProbeSet.Id AND + ProbeSetFreeze.Name = '%s' AND + ProbeSet.Name = '%s' + """ % (escape(display_fields_string), + escape(dataset.name), + escape(str(trait.name))) + logger.sql(query) + trait_info = g.db.execute(query).fetchone() + #XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name + # to avoid the problem of same marker name from different species. + elif dataset.type == 'Geno': + display_fields_string = string.join(dataset.display_fields,',Geno.') + display_fields_string = 'Geno.' + display_fields_string + query = """ + SELECT %s + FROM Geno, GenoFreeze, GenoXRef + WHERE + GenoXRef.GenoFreezeId = GenoFreeze.Id AND + GenoXRef.GenoId = Geno.Id AND + GenoFreeze.Name = '%s' AND + Geno.Name = '%s' + """ % (escape(display_fields_string), + escape(dataset.name), + escape(trait.name)) + logger.sql(query) + trait_info = g.db.execute(query).fetchone() + else: #Temp type + query = """SELECT %s FROM %s WHERE Name = %s""" + logger.sql(query) + trait_info = g.db.execute(query, + (string.join(dataset.display_fields,','), + dataset.type, trait.name)).fetchone() - logger.sql(query) - trait_info = g.db.execute(query).fetchone() - - - #XZ, 05/08/2009: Xiaodong add this block to use ProbeSet.Id to find the probeset instead of just using ProbeSet.Name - #XZ, 05/08/2009: to avoid the problem of same probeset name from different platforms. - elif dataset.type == 'ProbeSet': - display_fields_string = ', ProbeSet.'.join(dataset.display_fields) - display_fields_string = 'ProbeSet.' + display_fields_string - query = """ - SELECT %s - FROM ProbeSet, ProbeSetFreeze, ProbeSetXRef - WHERE - ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND - ProbeSetXRef.ProbeSetId = ProbeSet.Id AND - ProbeSetFreeze.Name = '%s' AND - ProbeSet.Name = '%s' - """ % (escape(display_fields_string), - escape(dataset.name), - escape(str(trait.name))) - logger.sql(query) - trait_info = g.db.execute(query).fetchone() - #XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name - # to avoid the problem of same marker name from different species. - elif dataset.type == 'Geno': - display_fields_string = string.join(dataset.display_fields,',Geno.') - display_fields_string = 'Geno.' + display_fields_string - query = """ - SELECT %s - FROM Geno, GenoFreeze, GenoXRef - WHERE - GenoXRef.GenoFreezeId = GenoFreeze.Id AND - GenoXRef.GenoId = Geno.Id AND - GenoFreeze.Name = '%s' AND - Geno.Name = '%s' - """ % (escape(display_fields_string), - escape(dataset.name), - escape(trait.name)) - logger.sql(query) - trait_info = g.db.execute(query).fetchone() - else: #Temp type - query = """SELECT %s FROM %s WHERE Name = %s""" - logger.sql(query) - trait_info = g.db.execute(query, - (string.join(dataset.display_fields,','), - dataset.type, trait.name)).fetchone() if trait_info: trait.haveinfo = True - - #XZ: assign SQL query result to trait attributes. for i, field in enumerate(dataset.display_fields): - holder = trait_info[i] - if isinstance(trait_info[i], basestring): - holder = unicode(trait_info[i], "utf-8", "ignore") + holder = trait_info[i] setattr(trait, field, holder) if dataset.type == 'Publish': + if trait.group_code: + trait.display_name = trait.group_code + "_" + str(trait.name) + trait.confidential = 0 if trait.pre_publication_description and not trait.pubmed_id: trait.confidential = 1 @@ -519,20 +482,19 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): #phenotype traits, then display the pre-publication description instead #of the post-publication description if trait.confidential: + trait.abbreviation = trait.pre_publication_abbreviation trait.description_display = trait.pre_publication_description - - #if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait( - # privilege=self.dataset.privilege, - # userName=self.dataset.userName, - # authorized_users=self.authorized_users): - # - # description = self.pre_publication_description else: + trait.abbreviation = trait.post_publication_abbreviation if description: trait.description_display = description.strip() else: trait.description_display = "" + trait.abbreviation = unicode(str(trait.abbreviation).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + trait.description_display = unicode(str(trait.description_display).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + trait.authors = unicode(str(trait.authors).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + if not trait.year.isdigit(): trait.pubmed_text = "N/A" else: @@ -541,38 +503,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if trait.pubmed_id: trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id - - trait.homologeneid = None if dataset.type == 'ProbeSet' and dataset.group: - if trait.geneid: - #XZ, 05/26/2010: From time to time, this query get error message because some geneid values in database are not number. - #XZ: So I have to test if geneid is number before execute the query. - #XZ: The geneid values in database should be cleaned up. - #try: - # float(self.geneid) - # geneidIsNumber = True - #except ValueError: - # geneidIsNumber = False - #if geneidIsNumber: - query = """ - SELECT - HomologeneId - FROM - Homologene, Species, InbredSet - WHERE - Homologene.GeneId ='%s' AND - InbredSet.Name = '%s' AND - InbredSet.SpeciesId = Species.Id AND - Species.TaxonomyId = Homologene.TaxonomyId - """ % (escape(str(trait.geneid)), escape(dataset.group.name)) - logger.sql(query) - result = g.db.execute(query).fetchone() - #else: - # result = None - - if result: - trait.homologeneid = result[0] - description_string = unicode(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') target_string = unicode(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') @@ -588,47 +519,22 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): # Save it for the jinja2 template trait.description_display = description_display - #XZ: trait_location_value is used for sorting trait.location_repr = 'N/A' - trait.location_value = 1000000 - if trait.chr and trait.mb: - #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") - #This is so we can convert the location to a number used for sorting - trait_location_value = convert_location_to_value(trait.chr, trait.mb) - #try: - # trait_location_value = int(self.chr)*1000 + self.mb - #except ValueError: - # if self.chr.upper() == 'X': - # trait_location_value = 20*1000 + self.mb - # else: - # trait_location_value = (ord(str(self.chr).upper()[0])*1000 + - # self.mb) - - #ZS: Put this in function currently called "convert_location_to_value" trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) - trait.location_value = trait_location_value elif dataset.type == "Geno": trait.location_repr = 'N/A' - trait.location_value = 1000000 - if trait.chr and trait.mb: - #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") - #This is so we can convert the location to a number used for sorting - trait_location_value = convert_location_to_value(trait.chr, trait.mb) - - #ZS: Put this in function currently called "convert_location_to_value" trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) - trait.location_value = trait_location_value if get_qtl_info: #LRS and its location trait.LRS_score_repr = "N/A" - trait.LRS_score_value = 0 trait.LRS_location_repr = "N/A" - trait.LRS_location_value = 1000000 + trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.additive = "" if dataset.type == 'ProbeSet' and not trait.cellid: + trait.mean = "" query = """ SELECT ProbeSetXRef.Locus, ProbeSetXRef.LRS, ProbeSetXRef.pValue, ProbeSetXRef.mean, ProbeSetXRef.additive @@ -659,9 +565,6 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = "" else: trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = "" - else: - trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.mean = trait.additive = "" - if dataset.type == 'Publish': query = """ @@ -698,19 +601,9 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.locus = trait.lrs = trait.additive = "" if (dataset.type == 'Publish' or dataset.type == "ProbeSet") and trait.locus_chr != "" and trait.locus_mb != "": - #XZ: LRS_location_value is used for sorting - try: - LRS_location_value = int(trait.locus_chr)*1000 + float(trait.locus_mb) - except: - if trait.locus_chr.upper() == 'X': - LRS_location_value = 20*1000 + float(trait.locus_mb) - else: - LRS_location_value = ord(str(trait.locus_chr).upper()[0])*1000 + float(trait.locus_mb) - trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (trait.locus_chr, float(trait.locus_mb)) if trait.lrs != "": trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs - trait.LRS_score_value = LRS_score_value = trait.lrs else: raise KeyError, `trait.name`+' information is not found in the database.' diff --git a/wqflask/base/trait_collection.py b/wqflask/base/trait_collection.py deleted file mode 100644 index d388a3af..00000000 --- a/wqflask/base/trait_collection.py +++ /dev/null @@ -1,53 +0,0 @@ -class TraitCollection(object): - - def __init__(self, is_anon=False): - self.is_anon = is_anon - - - @app.route("/collections/remove", methods=('POST',)) - def remove_traits(): - if is_anon: - AnonCollection.remove_traits() - else: - UserCollection.remove_traits() - - params = request.form - print("params are:", params) - uc_id = params['uc_id'] - uc = model.UserCollection.query.get(uc_id) - traits_to_remove = params.getlist('traits[]') - print("traits_to_remove are:", traits_to_remove) - traits_to_remove = process_traits(traits_to_remove) - print("\n\n after processing, traits_to_remove:", traits_to_remove) - all_traits = uc.members_as_set() - print(" all_traits:", all_traits) - members_now = all_traits - traits_to_remove - print(" members_now:", members_now) - print("Went from {} to {} members in set.".format(len(all_traits), len(members_now))) - uc.members = json.dumps(list(members_now)) - uc.changed_timestamp = datetime.datetime.utcnow() - db_session.commit() - - # We need to return something so we'll return this...maybe in the future - # we can use it to check the results - return str(len(members_now)) - - def __init__(self, anon_id) - self.anon_key = anon_key - self.collection_members = Redis.smembers(self.anon_id) - print("self.collection_members is:", self.collection_members) - self.num_members = len(self.collection_members) - - - @app.route("/collections/remove", methods=('POST',)) - def remove_traits(traits_to_remove): - print("traits_to_remove:", traits_to_remove) - for trait in traits_to_remove: - Redis.srem(self.anon_id, trait) - members_now = self.collection_members - traits_to_remove - print("members_now:", members_now) - print("Went from {} to {} members in set.".format(len(self.collection_members), len(members_now))) - - # We need to return something so we'll return this...maybe in the future - # we can use it to check the results - return str(len(members_now)) diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py index 845a7224..2844cedd 100644 --- a/wqflask/base/webqtlCaseData.py +++ b/wqflask/base/webqtlCaseData.py @@ -19,8 +19,7 @@ # This module is used by GeneNetwork project (www.genenetwork.org) # # Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 + from utility.logger import getLogger logger = getLogger(__name__) @@ -29,7 +28,7 @@ import utility.tools utility.tools.show_settings() -class webqtlCaseData(object): +class webqtlCaseData: """one case data in one trait""" def __init__(self, name, value=None, variance=None, num_cases=None, name2=None): @@ -43,37 +42,40 @@ class webqtlCaseData(object): self.outlier = None # Not set to True/False until later def __repr__(self): - str = "<webqtlCaseData> " - if self.value: - str += "value=%2.3f" % self.value - if self.variance: - str += " variance=%2.3f" % self.variance + case_data_string = "<webqtlCaseData> " + if self.value is not None: + case_data_string += "value=%2.3f" % self.value + if self.variance is not None: + case_data_string += " variance=%2.3f" % self.variance if self.num_cases: - str += " ndata=%d" % self.num_cases + case_data_string += " ndata=%s" % self.num_cases if self.name: - str += " name=%s" % self.name + case_data_string += " name=%s" % self.name if self.name2: - str += " name2=%s" % self.name2 - return str + case_data_string += " name2=%s" % self.name2 + return case_data_string @property def class_outlier(self): """Template helper""" if self.outlier: return "outlier" - else: - return "" + return "" @property def display_value(self): - if self.value: + if self.value is not None: return "%2.3f" % self.value - else: - return "x" + return "x" @property def display_variance(self): - if self.variance: + if self.variance is not None: return "%2.3f" % self.variance - else: - return "x" + return "x" + + @property + def display_num_cases(self): + if self.num_cases is not None: + return "%s" % self.num_cases + return "x" diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 1e47e183..862ac881 100644 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -17,61 +17,65 @@ DEBUG = 1 #USER privilege USERDICT = {'guest':1,'user':2, 'admin':3, 'root':4} +#Set privileges +SUPER_PRIVILEGES = {'data': ['no-access', 'view', 'edit'], 'metadata': ['no-access', 'view', 'edit'], 'admin': ['not-admin', 'edit-access', 'edit-admins']} +DEFAULT_PRIVILEGES = {'data': ['no-access', 'view'], 'metadata': ['no-access', 'view'], 'admin': ['not-admin']} + #minimum number of informative strains KMININFORMATIVE = 5 -#maximum number of traits for interval mapping -MULTIPLEMAPPINGLIMIT = 11 - -#maximum number of traits for correlation -MAXCORR = 100 - #Daily download limit from one IP DAILYMAXIMUM = 1000 #maximum LRS value MAXLRS = 460.0 -#temporary data life span -MAXLIFE = 86400 - #MINIMUM Database public value PUBLICTHRESH = 0 -#NBCI address -NCBI_LOCUSID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" -UCSC_REFSEQ = "http://genome.cse.ucsc.edu/cgi-bin/hgGene?db=%s&hgg_gene=%s&hgg_chrom=chr%s&hgg_start=%s&hgg_end=%s" -GENBANK_ID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=Nucleotide&cmd=search&doptcmdl=DocSum&term=%s" -OMIM_ID = "http://www.ncbi.nlm.nih.gov/omim/%s" -UNIGEN_ID = "http://www.ncbi.nlm.nih.gov/UniGene/clust.cgi?ORG=%s&CID=%s"; -HOMOLOGENE_ID = "http://www.ncbi.nlm.nih.gov/sites/entrez?Db=homologene&Cmd=DetailsSearch&Term=%s" +#EXTERNAL LINK ADDRESSES PUBMEDLINK_URL = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids=%s&dopt=Abstract" -UCSC_POS = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=mammal&org=%s&db=%s&position=chr%s:%s-%s&pix=800&Submit=submit" UCSC_BLAT = 'http://genome.ucsc.edu/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' UTHSC_BLAT = 'http://ucscbrowser.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' -UCSC_GENOME = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=http://web2qtl.utmem.edu:88/snp/chr%s" -ENSEMBLE_BLAT = 'http://www.ensembl.org/Mus_musculus/featureview?type=AffyProbe&id=%s' -DBSNP = 'http://www.ncbi.nlm.nih.gov/SNP/snp_ref.cgi?type=rs&rs=%s' -UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s&hgt.customText=http://gbic.biol.rug.nl/~ralberts/tracks/%s/%s" -GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit" -ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s" - -# The following paths are no longer in use! -# HTMLPATH is replaced by GENODIR -# IMGDIR is replaced by GENERATED_IMAGE_DIR +UTHSC_BLAT2 = 'http://ucscbrowserbeta.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' +GENOMEBROWSER_URL="https://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=%s" +NCBI_LOCUSID = "http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=%s" +GENBANK_ID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=Nucleotide&cmd=search&doptcmdl=DocSum&term=%s" +OMIM_ID = "http://www.ncbi.nlm.nih.gov/omim/%s" +UNIGEN_ID = "http://www.ncbi.nlm.nih.gov/UniGene/clust.cgi?ORG=%s&CID=%s" +HOMOLOGENE_ID = "http://www.ncbi.nlm.nih.gov/homologene/?term=%s" +GENOTATION_URL = "http://www.genotation.org/Getd2g.pl?gene_list=%s" +GTEX_URL = "https://www.gtexportal.org/home/gene/%s" +GENEBRIDGE_URL = "https://www.systems-genetics.org/modules_by_gene/%s?organism=%s" +GENEMANIA_URL = "https://genemania.org/search/%s/%s" +UCSC_REFSEQ = "http://genome.cse.ucsc.edu/cgi-bin/hgTracks?db=%s&hgg_gene=%s&hgg_chrom=chr%s&hgg_start=%s&hgg_end=%s" +BIOGPS_URL = "http://biogps.org/?org=%s#goto=genereport&id=%s" +STRING_URL = "http://string-db.org/newstring_cgi/show_network_section.pl?identifier=%s" +PANTHER_URL = "http://www.pantherdb.org/genes/geneList.do?searchType=basic&fieldName=all&organism=all&listType=1&fieldValue=%s" +GEMMA_URL = "http://www.chibi.ubc.ca/Gemma/gene/showGene.html?ncbiid=%s" +ABA_URL = "http://mouse.brain-map.org/search/show?search_type=gene&search_term=%s" +EBIGWAS_URL = "https://www.ebi.ac.uk/gwas/search?query=%s" +WIKI_PI_URL = "http://severus.dbmi.pitt.edu/wiki-pi/index.php/search?q=%s" +ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Transcript/Idhistory?t=%s" +DBSNP = 'http://ensembl.org/Mus_musculus/Variation/Population?v=%s' +PROTEIN_ATLAS_URL = "http://www.proteinatlas.org/search/%s" +OPEN_TARGETS_URL = "https://genetics.opentargets.org/gene/%s" +UNIPROT_URL = "https://www.uniprot.org/uniprot/%s" +RGD_URL = "https://rgd.mcw.edu/rgdweb/elasticResults.html?term=%s&category=Gene&species=%s" +PHENOGEN_URL = "https://phenogen.org/gene.jsp?speciesCB=Rn&auto=Y&geneTxt=%s&genomeVer=rn6§ion=geneEQTL" # Temporary storage (note that this TMPDIR can be set as an # environment variable - use utility.tools.TEMPDIR when you # want to reach this base dir assert_writable_dir(TEMPDIR) -TMPDIR = mk_dir(TEMPDIR+'gn2') +TMPDIR = mk_dir(TEMPDIR+'/gn2/') assert_writable_dir(TMPDIR) CACHEDIR = mk_dir(TMPDIR+'/cache/') # We can no longer write into the git tree: -GENERATED_IMAGE_DIR = mk_dir(TMPDIR+'/generated/') -GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generated_text/') +GENERATED_IMAGE_DIR = mk_dir(TMPDIR+'generated/') +GENERATED_TEXT_DIR = mk_dir(TMPDIR+'generated_text/') # Make sure we have permissions to access these assert_writable_dir(CACHEDIR) @@ -81,6 +85,7 @@ assert_writable_dir(GENERATED_TEXT_DIR) # Flat file directories GENODIR = flat_files('genotype')+'/' assert_dir(GENODIR) +# assert_dir(GENODIR+'bimbam') # for gemma # JSON genotypes are OBSOLETE JSON_GENODIR = flat_files('genotype/json')+'/' @@ -92,4 +97,3 @@ if not valid_path(JSON_GENODIR): PORTADDR = "http://50.16.251.170" INFOPAGEHREF = '/dbdoc/%s.html' CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR' -SCRIPTFILE = 'main.py' diff --git a/wqflask/base/webqtlFormData.py b/wqflask/base/webqtlFormData.py deleted file mode 100644 index 10251756..00000000 --- a/wqflask/base/webqtlFormData.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# This program is available from Source Forge: at GeneNetwork Project -# (sourceforge.net/projects/genenetwork/). -# -# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) -# at rwilliams@uthsc.edu and xzhou15@uthsc.edu -# -# -# -# This module is used by GeneNetwork project (www.genenetwork.org) -# -# Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 - -#from mod_python import Cookie - -from __future__ import print_function -from pprint import pformat as pf - -import string -import os - -import reaper - -import webqtlConfig -from webqtlCaseData import webqtlCaseData -from utility import webqtlUtil - -class webqtlFormData(object): - 'Represents data from a WebQTL form page, needed to generate the next page' - - attrs = ('formID','group','genotype','samplelist','allsamplelist', 'display_variance' - 'suggestive','significance','submitID','identification', 'enablevariance', - 'nperm','nboot','email','incparentsf1','genotype_1','genotype_2','traitInfo') - - #XZ: Attention! All attribute values must be picklable! - - def __init__(self, - start_vars = None, - req = None, - mod_python_session=None, - FieldStorage_formdata=None): - # Todo: rework this whole thing - print("in webqtlFormData start_vars are:", pf(start_vars)) - for item in webqtlFormData.attrs: - self.__dict__[item] = None - - #ZS: This is only used in DataEditingPage.py (as far as I know) - self.varianceDispName = None - - for item in start_vars: - self.__dict__[item] = start_vars[item] - print(" Now self.dict is:", pf(self.__dict__)) - - #Todo: This can't be good below...rework - try: - self.remote_ip = req.connection.remote_ip - except: - self.remote_ip = '1.2.3.4' - - if req and req.headers_in.has_key('referer'): - self.refURL = req.headers_in['referer'] - else: - self.refURL = None - - # For now let's just comment all this out - Sam - - #self.cookies = cookieData.cookieData(Cookie.get_cookies(req)) #XZ: dictionary type. To hold values transfered from mod_python Cookie. - # - ##XZ: dictionary type. To hold values transfered from mod_python Session object. We assume that it is always picklable. - #self.input_session_data = sessionData.sessionData( mod_python_session ) - # - ##XZ: FieldStorage_formdata may contain item that can't be pickled. Must convert to picklable data. - #self.formdata = cgiData( FieldStorage_formdata ) - # - ##get Form ID - #self.formID = self.formdata.getfirst('FormID') - # - ##get rest of the attributes - #if self.formID: - # for item in self.attrs: - # value = self.formdata.getfirst(item) - # if value != None: - # setattr(self,item,string.strip(value)) - - self.ppolar = None - self.mpolar = None - - print("[yellow] self.group is:", self.group) - if self.group: - #try: - # # NL, 07/27/2010. ParInfo has been moved from webqtlForm.py to webqtlUtil.py; - _f1, _f12, self.mpolar, self.ppolar = webqtlUtil.ParInfo[self.group] - #except: - # f1 = f12 = self.mpolar = self.ppolar = None - - - def set_number(stringy): - return int(stringy) if stringy else 2000 # Rob asked to change the default value to 2000 - - self.nperm = set_number(self.nperm) - self.nboot = set_number(self.nboot) - - - #if self.allsamplelist: - # self.allsamplelist = map(string.strip, string.split(self.allsamplelist)) - print("self.allsamplelist is:", self.allsamplelist) - if self.allsamplelist: - self.allsamplelist = self.allsamplelist.split() - print("now self.allsamplelist is:", self.allsamplelist) - #self.readGenotype() - #self.readData() - - if self.group == 'BXD300': - self.group = 'BXD' - - - def __getitem__(self, key): - print("in __getitem__") - return self.__dict__[key] - - def get(self, key, default=None): - if key in self.__dict__: - return self.__dict__[key] - else: - return default - - def __str__(self): - rstr = '' - for item in self.attrs: - if item != 'genotype': - rstr += '%s:%s\n' % (item,str(getattr(self,item))) - return rstr - - - def readGenotype(self): - '''read genotype from .geno file''' - if self.group == 'BXD300': - self.group = 'BXD' - - assert self.group, "self.group needs to be set" - - #genotype_1 is Dataset Object without parents and f1 - #genotype_2 is Dataset Object with parents and f1 (not for intercross) - - self.genotype_1 = reaper.Dataset() - - full_filename = locate(self.group + '.geno','genotype') - - # reaper barfs on unicode filenames, so here we ensure it's a string - full_filename = str(full_filename) - self.genotype_1.read(full_filename) - - print("Got to after read") - - try: - # NL, 07/27/2010. ParInfo has been moved from webqtlForm.py to webqtlUtil.py; - _f1, _f12, _mat, _pat = webqtlUtil.ParInfo[self.group] - except KeyError: - _f1 = _f12 = _mat = _pat = None - - self.genotype_2 = self.genotype_1 - if self.genotype_1.type == "group" and _mat and _pat: - self.genotype_2 = self.genotype_1.add(Mat=_mat, Pat=_pat) #, F1=_f1) - - #determine default genotype object - if self.incparentsf1 and self.genotype_1.type != "intercross": - self.genotype = self.genotype_2 - else: - self.incparentsf1 = 0 - self.genotype = self.genotype_1 - - self.samplelist = list(self.genotype.prgy) - self.f1list = [] - self.parlist = [] - - if _f1 and _f12: - self.f1list = [_f1, _f12] - if _mat and _pat: - self.parlist = [_mat, _pat] - - - def readData(self, samplelist, incf1=None): - '''read user input data or from trait data and analysis form''' - - if incf1 == None: - incf1 = [] - - if not self.genotype: - self.readGenotype() - if not samplelist: - if incf1: - samplelist = self.f1list + self.samplelist - else: - samplelist = self.samplelist - - #print("before traitfiledata self.traitfile is:", pf(self.traitfile)) - - traitfiledata = getattr(self, "traitfile", None) - traitpastedata = getattr(self, "traitpaste", None) - variancefiledata = getattr(self, "variancefile", None) - variancepastedata = getattr(self, "variancepaste", None) - Nfiledata = getattr(self, "Nfile", None) - - #### Todo: Rewrite below when we get to someone submitting their own trait ##### - - def to_float(item): - try: - return float(item) - except ValueError: - return None - - print("bottle samplelist is:", samplelist) - if traitfiledata: - tt = traitfiledata.split() - values = map(webqtlUtil.StringAsFloat, tt) - elif traitpastedata: - tt = traitpastedata.split() - values = map(webqtlUtil.StringAsFloat, tt) - else: - print("mapping formdataasfloat") - #values = map(self.FormDataAsFloat, samplelist) - values = [to_float(getattr(self, key)) for key in samplelist] - print("rocket values is:", values) - - - if len(values) < len(samplelist): - values += [None] * (len(samplelist) - len(values)) - elif len(values) > len(samplelist): - values = values[:len(samplelist)] - print("now values is:", values) - - - if variancefiledata: - tt = variancefiledata.split() - variances = map(webqtlUtil.StringAsFloat, tt) - elif variancepastedata: - tt = variancepastedata.split() - variances = map(webqtlUtil.StringAsFloat, tt) - else: - variances = map(self.FormVarianceAsFloat, samplelist) - - if len(variances) < len(samplelist): - variances += [None]*(len(samplelist) - len(variances)) - elif len(variances) > len(samplelist): - variances = variances[:len(samplelist)] - - if Nfiledata: - tt = string.split(Nfiledata) - nsamples = map(webqtlUtil.IntAsFloat, tt) - if len(nsamples) < len(samplelist): - nsamples += [None]*(len(samplelist) - len(nsamples)) - else: - nsamples = map(self.FormNAsFloat, samplelist) - - ##values, variances, nsamples is obsolete - self.allTraitData = {} - for i, _sample in enumerate(samplelist): - if values[i] != None: - self.allTraitData[_sample] = webqtlCaseData( - _sample, values[i], variances[i], nsamples[i]) - print("allTraitData is:", pf(self.allTraitData)) - - - - def informativeStrains(self, samplelist=None, include_variances = None): - '''if readData was called, use this to output informative samples (sample with values)''' - - if not samplelist: - samplelist = self.samplelist - - samples = [] - values = [] - variances = [] - - #print("self.allTraitData is:", pf(self.allTraitData)) - - for sample in samplelist: - if sample in self.allTraitData: - _val, _var = self.allTraitData[sample].value, self.allTraitData[sample].variance - if _val != None: - if include_variances: - if _var != None: - samples.append(sample) - values.append(_val) - variances.append(_var) - else: - samples.append(sample) - values.append(_val) - variances.append(None) - - return samples, values, variances, len(samples) - - - - #def FormDataAsFloat(self, key): - # - # #try: - # # return float(self.key) - # #except: - # # return None - - - def FormVarianceAsFloat(self, key): - try: - return float(self.formdata.getfirst('V' + key)) - except: - return None - - def FormNAsFloat(self, key): - try: - return int(self.formdata.getfirst('N' + key)) - except: - return None - - def Sample(self): - 'Create some dummy data for testing' - self.group = 'BXD' - self.incparentsf1 = 'on' - #self.display = 9.2 - #self.significance = 16.1 - self.readGenotype() - self.identification = 'BXD : Coat color example by Lu Lu, et al' - #self.readGenotype() - #self.genotype.ReadMM('AXBXAforQTL') - #self.samplelist = map((lambda x, y='': '%s%s' % (y,x)), self.genotype.prgy) - #self.samplelist.sort() - self.allTraitData = {'BXD29': webqtlCaseData(3), 'BXD28': webqtlCaseData(2), - 'BXD25': webqtlCaseData(2), 'BXD24': webqtlCaseData(2), 'BXD27': webqtlCaseData(2), - 'BXD21': webqtlCaseData(1), 'BXD20': webqtlCaseData(4), 'BXD23': webqtlCaseData(4), - 'BXD22': webqtlCaseData(3), 'BXD14': webqtlCaseData(4), 'BXD15': webqtlCaseData(2), - 'BXD16': webqtlCaseData(3), 'BXD11': webqtlCaseData(4), 'BXD12': webqtlCaseData(3), - 'BXD13': webqtlCaseData(2), 'BXD18': webqtlCaseData(3), 'BXD19': webqtlCaseData(3), - 'BXD38': webqtlCaseData(3), 'BXD39': webqtlCaseData(3), 'BXD36': webqtlCaseData(2), - 'BXD34': webqtlCaseData(4), 'BXD35': webqtlCaseData(4), 'BXD32': webqtlCaseData(4), - 'BXD33': webqtlCaseData(3), 'BXD30': webqtlCaseData(1), 'BXD31': webqtlCaseData(4), - 'DBA/2J': webqtlCaseData(1), 'BXD8': webqtlCaseData(3), 'BXD9': webqtlCaseData(1), - 'BXD6': webqtlCaseData(3), 'BXD5': webqtlCaseData(3), 'BXD2': webqtlCaseData(4), - 'BXD1': webqtlCaseData(1), 'C57BL/6J': webqtlCaseData(4), 'B6D2F1': webqtlCaseData(4), - 'BXD42': webqtlCaseData(4), 'BXD40': webqtlCaseData(3)} |