From af7d0bca229f3ebaa80a16d1ce3a2bf1a8abd5df Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:37:55 +0000 Subject: [PATCH 023/100] WIP fixing all paths --- wqflask/base/data_set.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index e37a838f..0e5a3ac1 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -405,10 +405,11 @@ class DatasetGroup(object): #print("Cache not hit") from utility.tools import plink_command - PLINK_PATH,PLINK_COMMAND = plink_command() + PLINK_RUN = plink_command() geno_file_path = webqtlConfig.GENODIR+self.name+".geno" plink_file_path = PLINK_PATH+"/"+self.name+".fam" + # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES if os.path.isfile(plink_file_path): self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path) -- cgit v1.2.3 From 4b80bbfe261e8d0587a637b35416834e027f0999 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:46:23 +0000 Subject: [PATCH 024/100] Sanitizing file handling --- wqflask/base/data_set.py | 29 ++++++++++------------------- wqflask/utility/tools.py | 20 +++++++++++++++----- wqflask/wqflask/show_trait/show_trait.py | 11 ++++------- 3 files changed, 29 insertions(+), 31 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 0e5a3ac1..c6f34143 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,12 +44,15 @@ from dbFunction import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks +from utility.tools import flat_files from maintenance import get_group_samplelists from MySQLdb import escape_string as escape from pprint import pformat as pf +MAPPING_PATH = flat_files("mapping") + # Used by create_database to instantiate objects # Each subclass will add to this DS_NAME_MAP = {} @@ -404,15 +407,11 @@ class DatasetGroup(object): else: #print("Cache not hit") - from utility.tools import plink_command - PLINK_RUN = plink_command() - geno_file_path = webqtlConfig.GENODIR+self.name+".geno" - plink_file_path = PLINK_PATH+"/"+self.name+".fam" - # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES - if os.path.isfile(plink_file_path): - self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path) + mapping_file_path = MAPPING_PATH+"/"+self.name+".fam" + if os.path.isfile(mapping_file_path): + self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path) elif os.path.isfile(geno_file_path): self.samplelist = get_group_samplelists.get_samplelist("geno", geno_file_path) else: @@ -441,18 +440,10 @@ class DatasetGroup(object): # reaper barfs on unicode filenames, so here we ensure it's a string full_filename = str(os.path.join(webqtlConfig.GENODIR, self.name + '.geno')) - if os.path.isfile(full_filename): - #print("Reading file: ", full_filename) - genotype_1.read(full_filename) - #print("File read") - else: - try: - full_filename = str(os.path.join(webqtlConfig.TMPDIR, self.name + '.geno')) - #print("Reading file") - genotype_1.read(full_filename) - #print("File read") - except IOError: - print("File doesn't exist!") + if not os.path.isfile(full_filename): + raise SystemError("File "+full_filename+" does not exist") + print("Reading file: ", full_filename) + genotype_1.read(full_filename) if genotype_1.type == "group" and self.parlist: genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1]) #, F1=_f1) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index c3c9b292..44bbb1b0 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -52,17 +52,27 @@ def get_setting(command_id,guess=None): raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path') return command +def valid_bin(bin): + if os.path.islink(bin) or os.path.isfile(bin): + return bin + return None + +def valid_path(dir): + if os.path.isdir(dir): + return dir + return None + def pylmm_command(guess=None): - return get_setting("PYLMM_RUN",guess) + return valid_bin(get_setting("PYLMM_RUN",guess)) def gemma_command(guess=None): - return get_setting("GEMMA_RUN",guess) + return valid_bin(get_setting("GEMMA_RUN",guess)) def plink_command(guess=None): - return get_setting("PLINK_RUN",guess) + return valid_bin(get_setting("PLINK_RUN",guess)) def flat_files(subdir=None): base = get_setting("GENENETWORK_FILES") if subdir: - return base+"/"+subdir - return base + return valid_path(base+"/"+subdir) + return valid_path(base) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 458e48da..35f7fe5f 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -16,8 +16,6 @@ from base import webqtlConfig from base import webqtlCaseData from wqflask.show_trait.SampleList import SampleList from utility import webqtlUtil, Plot, Bunch, helper_functions -# from utility.tools import plink_command -from utility.tools import flat_files from base.trait import GeneralTrait from base import data_set from dbFunction import webqtlDatabaseFunction @@ -25,7 +23,8 @@ from basicStatistics import BasicStatisticsFunctions from pprint import pformat as pf -MAPPING_FILES = flat_files("mapping") +from utility.tools import flat_files +MAPPING_PATH = flat_files("mapping") ############################################### # @@ -34,8 +33,6 @@ MAPPING_FILES = flat_files("mapping") # ############################################## - - class ShowTrait(object): def __init__(self, kw): @@ -162,8 +159,8 @@ class ShowTrait(object): def get_mapping_methods(self): '''Only display mapping methods when the dataset group's genotype file exists''' def check_plink_gemma(): - if (os.path.isfile(MAPPYING_FILES+"/"+self.dataset.group.name+".bed") and - os.path.isfile(MAPPING_FILES+"/"+self.dataset.group.name+".map")): + if (os.path.isfile(MAPPYING_PATH+"/"+self.dataset.group.name+".bed") and + os.path.isfile(MAPPING_PATH+"/"+self.dataset.group.name+".map")): return True else: return False -- cgit v1.2.3 From a5ff6520ad29266e1647ce54ad6b2c0f200dfe9a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:55:56 +0000 Subject: [PATCH 032/100] file locating: error checking is built-in --- wqflask/base/data_set.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index c6f34143..9ffc09ac 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,15 +44,13 @@ from dbFunction import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks -from utility.tools import flat_files +from utility.tools import locate from maintenance import get_group_samplelists from MySQLdb import escape_string as escape from pprint import pformat as pf -MAPPING_PATH = flat_files("mapping") - # Used by create_database to instantiate objects # Each subclass will add to this DS_NAME_MAP = {} @@ -405,11 +403,10 @@ class DatasetGroup(object): #print(" type: ", type(self.samplelist)) #print(" self.samplelist: ", self.samplelist) else: - #print("Cache not hit") - - geno_file_path = webqtlConfig.GENODIR+self.name+".geno" + print("Cache not hit") - mapping_file_path = MAPPING_PATH+"/"+self.name+".fam" + geno_file_path = locate(self.name+".geno",'genotype') + mapping_file_path = locate(self.name+".fam",'mapping') if os.path.isfile(mapping_file_path): self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path) elif os.path.isfile(geno_file_path): @@ -439,10 +436,7 @@ class DatasetGroup(object): genotype_1 = reaper.Dataset() # reaper barfs on unicode filenames, so here we ensure it's a string - full_filename = str(os.path.join(webqtlConfig.GENODIR, self.name + '.geno')) - if not os.path.isfile(full_filename): - raise SystemError("File "+full_filename+" does not exist") - print("Reading file: ", full_filename) + full_filename = str(locate(self.name+'.geno','genotype')) genotype_1.read(full_filename) if genotype_1.type == "group" and self.parlist: -- cgit v1.2.3 From 218fd0c7666583748098f0b7b4286b6d1cbf6838 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 23 Feb 2016 11:16:33 +0000 Subject: [PATCH 033/100] Refactored file searching --- wqflask/base/data_set.py | 16 ++++++++-------- wqflask/utility/tools.py | 4 +++- 2 files changed, 11 insertions(+), 9 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 9ffc09ac..ce13dd77 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,7 +44,7 @@ from dbFunction import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks -from utility.tools import locate +from utility.tools import locate, locate_without_error from maintenance import get_group_samplelists @@ -405,15 +405,15 @@ class DatasetGroup(object): else: print("Cache not hit") - geno_file_path = locate(self.name+".geno",'genotype') - mapping_file_path = locate(self.name+".fam",'mapping') - if os.path.isfile(mapping_file_path): - self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path) - elif os.path.isfile(geno_file_path): - self.samplelist = get_group_samplelists.get_samplelist("geno", geno_file_path) + genotype_fn = locate_without_error(self.name+".geno",'genotype') + mapping_fn = locate_without_error(self.name+".fam",'mapping') + if mapping_fn: + self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn) + elif genotype_fn: + self.samplelist = get_group_samplelists.get_samplelist("geno", genotype_fn) else: self.samplelist = None - #print("after get_samplelist") + print("Sample list: ",self.samplelist) Redis.set(key, json.dumps(self.samplelist)) Redis.expire(key, 60*5) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index dd40be99..d4c10c68 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -36,7 +36,7 @@ def get_setting(command_id,guess=None): """ def value(command): if command: - sys.stderr.write("Found value "+command+"\n") + sys.stderr.write("Found path "+command+"\n") return command else: return None @@ -95,6 +95,7 @@ def locate(name, subdir=None): if valid_path(base): lookfor = base + "/" + name if valid_file(lookfor): + print("Found: file "+lookfor+"\n") return lookfor else: raise IOError("Can not locate "+lookfor) @@ -114,6 +115,7 @@ def locate_without_error(name, subdir=None): if valid_path(base): lookfor = base + "/" + name if valid_file(lookfor): + print("Found: file "+name+"\n") return lookfor sys.stderr.write("WARNING: file "+name+" not found\n") return None -- cgit v1.2.3 From 5fffa1aa4260af6663c1f9b4cf7494d3ebc6995d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 11:01:29 +0000 Subject: [PATCH 037/100] WebQtlConfig: sanitizing naming and used vars --- bin/genenetwork2 | 2 +- wqflask/base/data_set.py | 194 ++++++++++----------- wqflask/base/webqtlFormData.py | 2 +- .../basicStatistics/BasicStatisticsFunctions.py | 6 +- wqflask/maintenance/get_group_samplelists.py | 3 +- wqflask/utility/tools.py | 11 +- wqflask/wqflask/correlation/show_corr_results.py | 2 +- .../wqflask/correlation_matrix/show_corr_matrix.py | 1 - wqflask/wqflask/heatmap/heatmap.py | 2 - .../interval_analyst/IntervalAnalystPage.py | 138 +++++++-------- .../marker_regression/MarkerRegressionPage.py | 6 +- .../wqflask/marker_regression/marker_regression.py | 2 +- .../marker_regression/marker_regression_gn1.py | 116 ++++++------ wqflask/wqflask/search_results.py | 1 - wqflask/wqflask/show_trait/show_trait.py | 2 +- 15 files changed, 246 insertions(+), 242 deletions(-) (limited to 'wqflask/base') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 54eee02e..00ee09bf 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,6 +1,6 @@ #! /bin/bash # -# This will run the server with default settings +# This will run the server with default settings. # Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index ce13dd77..6527657a 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -76,22 +76,22 @@ def create_dataset(dataset_name, dataset_type = None, get_samplelist = True): #def get_dataset_type_from_json(dataset_name): - + class Dataset_Types(object): - + def __init__(self): self.datasets = {} file_name = "wqflask/static/new/javascript/dataset_menu_structure.json" with open(file_name, 'r') as fh: data = json.load(fh) - + print("*" * 70) for species in data['datasets']: for group in data['datasets'][species]: for dataset_type in data['datasets'][species][group]: for dataset in data['datasets'][species][group][dataset_type]: #print("dataset is:", dataset) - + short_dataset_name = dataset[1] if dataset_type == "Phenotypes": new_type = "Publish" @@ -100,32 +100,32 @@ class Dataset_Types(object): else: new_type = "ProbeSet" self.datasets[short_dataset_name] = new_type - + def __call__(self, name): return self.datasets[name] - + # Do the intensive work at startup one time only Dataset_Getter = Dataset_Types() # #print("Running at startup:", get_dataset_type_from_json("HBTRC-MLPFC_0611")) - + def create_datasets_list(): key = "all_datasets" result = Redis.get(key) - + if result: print("Cache hit!!!") datasets = pickle.loads(result) - + else: datasets = list() with Bench("Creating DataSets object"): type_dict = {'Publish': 'PublishFreeze', 'ProbeSet': 'ProbeSetFreeze', 'Geno': 'GenoFreeze'} - + for dataset_type in type_dict: query = "SELECT Name FROM {}".format(type_dict[dataset_type]) for result in g.db.execute(query).fetchall(): @@ -134,10 +134,10 @@ def create_datasets_list(): #print("type: {}\tname: {}".format(dataset_type, result.Name)) dataset = create_dataset(result.Name, dataset_type) datasets.append(dataset) - + Redis.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL)) Redis.expire(key, 60*60) - + return datasets @@ -158,30 +158,30 @@ def mescape(*items): class Markers(object): """Todo: Build in cacheing so it saves us reading the same file more than once""" def __init__(self, name): - json_data_fh = open(os.path.join(webqtlConfig.NEWGENODIR + name + '.json')) + json_data_fh = open(locate(name + '.json','genotype/json')) try: markers = json.load(json_data_fh) except: markers = [] - + for marker in markers: if (marker['chr'] != "X") and (marker['chr'] != "Y"): marker['chr'] = int(marker['chr']) marker['Mb'] = float(marker['Mb']) - + self.markers = markers #print("self.markers:", self.markers) - - + + def add_pvalues(self, p_values): print("length of self.markers:", len(self.markers)) print("length of p_values:", len(p_values)) - + if type(p_values) is list: # THIS IS only needed for the case when we are limiting the number of p-values calculated #if len(self.markers) > len(p_values): # self.markers = self.markers[:len(p_values)] - + for marker, p_value in itertools.izip(self.markers, p_values): if not p_value: continue @@ -214,7 +214,7 @@ class Markers(object): #self.markers.remove(marker) #del self.markers[i] self.markers = filtered_markers - + #for i, marker in enumerate(self.markers): # if not 'p_value' in marker: @@ -223,9 +223,9 @@ class Markers(object): # #self.markers.remove(self.markers[i]) class HumanMarkers(Markers): - + def __init__(self, name, specified_markers = []): - marker_data_fh = open(os.path.join(webqtlConfig.PYLMM_PATH + name + '.bim')) + marker_data_fh = open(locate('genotype') + '/' + name + '.bim') self.markers = [] for line in marker_data_fh: splat = line.strip().split() @@ -244,7 +244,7 @@ class HumanMarkers(Markers): marker['name'] = splat[1] marker['Mb'] = float(splat[3]) / 1000000 self.markers.append(marker) - + #print("markers is: ", pf(self.markers)) @@ -257,26 +257,26 @@ class HumanMarkers(Markers): # marker['lod_score'] = -math.log10(marker['p_value']) # #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values # marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 - + #print("p_values2:", pf(p_values)) super(HumanMarkers, self).add_pvalues(p_values) - + #with Bench("deleting markers"): # markers = [] # for marker in self.markers: # if not marker['Mb'] <= 0 and not marker['chr'] == 0: # markers.append(marker) # self.markers = markers - - + + class DatasetGroup(object): """ Each group has multiple datasets; each species has multiple groups. - + For example, Mouse has multiple groups (BXD, BXA, etc), and each group has multiple datasets associated with it. - + """ def __init__(self, dataset): """This sets self.group and self.group_id""" @@ -284,14 +284,14 @@ class DatasetGroup(object): self.name, self.id = g.db.execute(dataset.query_for_group).fetchone() if self.name == 'BXD300': self.name = "BXD" - + self.f1list = None self.parlist = None self.get_f1_parent_strains() #print("parents/f1s: {}:{}".format(self.parlist, self.f1list)) - + self.species = webqtlDatabaseFunction.retrieve_species(self.name) - + self.incparentsf1 = False self.allsamples = None self._datasets = None @@ -302,7 +302,7 @@ class DatasetGroup(object): def get_markers(self): #print("self.species is:", self.species) if self.species == "human": - marker_class = HumanMarkers + marker_class = HumanMarkers else: marker_class = Markers @@ -356,7 +356,7 @@ class DatasetGroup(object): dataset_menu.append(dict(tissue=None, datasets=[(dataset, dataset_short)])) else: dataset_sub_menu = [item[1:] for item in dataset] - + tissue_already_exists = False tissue_position = None for i, tissue_dict in enumerate(dataset_menu): @@ -384,7 +384,7 @@ class DatasetGroup(object): f1, f12, maternal, paternal = webqtlUtil.ParInfo[self.name] except KeyError: f1 = f12 = maternal = paternal = None - + if f1 and f12: self.f1list = [f1, f12] if maternal and paternal: @@ -455,18 +455,18 @@ class DatasetGroup(object): #self.samplelist = list(self.genotype.prgy) self.samplelist = list(genotype.prgy) - + return genotype #class DataSets(object): # """Builds a list of DataSets""" -# +# # def __init__(self): # self.datasets = list() -# +# + - #query = """SELECT Name FROM ProbeSetFreeze # UNION # SELECT Name From PublishFreeze @@ -501,7 +501,7 @@ class DataSet(object): self.check_confidentiality() self.retrieve_other_names() - + self.group = DatasetGroup(self) # sets self.group and self.group_id and gets genotype if get_samplelist == True: self.group.get_samplelist() @@ -511,30 +511,30 @@ class DataSet(object): def get_desc(self): """Gets overridden later, at least for Temp...used by trait's get_given_name""" return None - + #@staticmethod #def get_by_trait_id(trait_id): # """Gets the dataset object given the trait id""" - # - # # - # name = g.db.execute(""" SELECT - # + # + # + # name = g.db.execute(""" SELECT + # # """) - # + # # return DataSet(name) # Delete this eventually @property def riset(): Weve_Renamed_This_As_Group - - + + #@property #def group(self): # if not self._group: # self.get_group() - # + # # return self._group @@ -546,7 +546,7 @@ class DataSet(object): This is not meant to retrieve the data set info if no name at all is passed. """ - + try: if self.type == "ProbeSet": query_args = tuple(escape(x) for x in ( @@ -582,17 +582,17 @@ class DataSet(object): except TypeError: print("Dataset {} is not yet available in GeneNetwork.".format(self.name)) pass - + def get_trait_data(self, sample_list=None): if sample_list: self.samplelist = sample_list else: self.samplelist = self.group.samplelist - + if self.group.parlist != None and self.group.f1list != None: if (self.group.parlist + self.group.f1list) in self.samplelist: self.samplelist += self.group.parlist + self.group.f1list - + query = """ SELECT Strain.Name, Strain.Id FROM Strain, Species WHERE Strain.Name IN {} @@ -610,9 +610,9 @@ class DataSet(object): trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId + #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId #tempTable = None - #if GeneId and db.type == "ProbeSet": + #if GeneId and db.type == "ProbeSet": # if method == "3": # tempTable = self.getTempLiteratureTable(species=species, # input_species_geneid=GeneId, @@ -623,7 +623,7 @@ class DataSet(object): # TissueProbeSetFreezeId=tissueProbeSetFreezeId, # method=method, # returnNumber=returnNumber) - + if self.type == "Publish": dataset_type = "Phenotype" else: @@ -644,7 +644,7 @@ class DataSet(object): left join {}Data as T{} on T{}.Id = {}XRef.DataId and T{}.StrainId={}\n """.format(*mescape(self.type, item, item, self.type, item, item)) - + if self.type == "Publish": query += """ WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId @@ -661,16 +661,16 @@ class DataSet(object): order by {}.Id """.format(*mescape(self.type, self.type, self.type, self.type, self.name, dataset_type, self.type, self.type, dataset_type)) - + #print("trait data query: ", query) - + results = g.db.execute(query).fetchall() #print("query results:", results) trait_sample_data.append(results) trait_count = len(trait_sample_data[0]) self.trait_data = collections.defaultdict(list) - + # put all of the separate data together into a dictionary where the keys are # trait names and values are lists of sample values for trait_counter in range(trait_count): @@ -683,9 +683,9 @@ class PhenotypeDataSet(DataSet): DS_NAME_MAP['Publish'] = 'PhenotypeDataSet' def setup(self): - + #print("IS A PHENOTYPEDATASET") - + # Fields in the database table self.search_fields = ['Phenotype.Post_publication_description', 'Phenotype.Pre_publication_description', @@ -756,26 +756,26 @@ class PhenotypeDataSet(DataSet): def get_trait_info(self, trait_list, species = ''): for this_trait in trait_list: - + if not this_trait.haveinfo: this_trait.retrieve_info(get_qtl_info=True) description = this_trait.post_publication_description - + #If the dataset is confidential and the user has access to confidential #phenotype traits, then display the pre-publication description instead #of the post-publication description if this_trait.confidential: this_trait.description_display = "" continue # for now - + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait( privilege=self.privilege, userName=self.userName, authorized_users=this_trait.authorized_users): - + description = this_trait.pre_publication_description - + if len(description) > 0: this_trait.description_display = description.strip() else: @@ -820,7 +820,7 @@ class PhenotypeDataSet(DataSet): this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs this_trait.LRS_score_value = LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb)) - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -878,7 +878,7 @@ class GenotypeDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - + def get_trait_list(self): query = """ select Geno.Name @@ -912,7 +912,7 @@ class GenotypeDataSet(DataSet): this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb) ) this_trait.location_value = trait_location_value - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1004,7 +1004,7 @@ class MrnaAssayDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - + def get_trait_list_1(self): query = """ select ProbeSet.Name @@ -1020,7 +1020,7 @@ class MrnaAssayDataSet(DataSet): trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) #print("After retrieve_sample_data") return trait_data - + #def get_trait_data(self): # self.samplelist = self.group.samplelist + self.group.parlist + self.group.f1list # query = """ @@ -1040,9 +1040,9 @@ class MrnaAssayDataSet(DataSet): # trait_sample_data = [] # for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): # - # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId + # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId # #tempTable = None - # #if GeneId and db.type == "ProbeSet": + # #if GeneId and db.type == "ProbeSet": # # if method == "3": # # tempTable = self.getTempLiteratureTable(species=species, # # input_species_geneid=GeneId, @@ -1053,7 +1053,7 @@ class MrnaAssayDataSet(DataSet): # # TissueProbeSetFreezeId=tissueProbeSetFreezeId, # # method=method, # # returnNumber=returnNumber) - # + # # temp = ['T%s.value' % item for item in sample_ids_step] # query = "SELECT {}.Name,".format(escape(self.type)) # data_start_pos = 1 @@ -1067,7 +1067,7 @@ class MrnaAssayDataSet(DataSet): # left join {}Data as T{} on T{}.Id = {}XRef.DataId # and T{}.StrainId={}\n # """.format(*mescape(self.type, item, item, self.type, item, item)) - # + # # query += """ # WHERE {}XRef.{}FreezeId = {}Freeze.Id # and {}Freeze.Name = '{}' @@ -1080,7 +1080,7 @@ class MrnaAssayDataSet(DataSet): # # trait_count = len(trait_sample_data[0]) # self.trait_data = collections.defaultdict(list) - # + # # # put all of the separate data together into a dictionary where the keys are # # trait names and values are lists of sample values # for trait_counter in range(trait_count): @@ -1088,11 +1088,11 @@ class MrnaAssayDataSet(DataSet): # for chunk_counter in range(int(number_chunks)): # self.trait_data[trait_name] += ( # trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) - + def get_trait_info(self, trait_list=None, species=''): - # Note: setting trait_list to [] is probably not a great idea. + # Note: setting trait_list to [] is probably not a great idea. if not trait_list: trait_list = [] @@ -1155,7 +1155,7 @@ class MrnaAssayDataSet(DataSet): #print("query is:", pf(query)) result = g.db.execute(query).fetchone() - + mean = result[0] if result else 0 if mean: @@ -1176,7 +1176,7 @@ class MrnaAssayDataSet(DataSet): Geno.SpeciesId = Species.Id """.format(species, this_trait.locus) result = g.db.execute(query).fetchone() - + if result: #if result[0] and result[1]: # lrs_chr = result[0] @@ -1184,7 +1184,7 @@ class MrnaAssayDataSet(DataSet): lrs_chr, lrs_mb = result #XZ: LRS_location_value is used for sorting lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb) - + #try: # lrs_location_value = int(lrs_chr)*1000 + float(lrs_mb) #except: @@ -1197,7 +1197,7 @@ class MrnaAssayDataSet(DataSet): this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs this_trait.LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb)) - + def convert_location_to_value(self, chromosome, mb): try: @@ -1208,7 +1208,7 @@ class MrnaAssayDataSet(DataSet): else: location_value = (ord(str(chromosome).upper()[0])*1000 + float(mb)) - + return location_value def get_sequence(self): @@ -1225,7 +1225,7 @@ class MrnaAssayDataSet(DataSet): """ % (escape(self.name), escape(self.dataset.name)) results = g.db.execute(query).fetchone() return results[0] - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1246,8 +1246,8 @@ class MrnaAssayDataSet(DataSet): results = g.db.execute(query).fetchall() #print("RETRIEVED RESULTS HERE:", results) return results - - + + def retrieve_genes(self, column_name): query = """ select ProbeSet.Name, ProbeSet.%s @@ -1256,7 +1256,7 @@ class MrnaAssayDataSet(DataSet): ProbeSetXRef.ProbeSetId=ProbeSet.Id; """ % (column_name, escape(str(self.id))) results = g.db.execute(query).fetchall() - + return dict(results) #def retrieve_gene_symbols(self): @@ -1285,8 +1285,8 @@ class MrnaAssayDataSet(DataSet): # for item in results: # symbol_dict[item[0]] = item[1] # return symbol_dict - - + + class TempDataSet(DataSet): @@ -1308,8 +1308,8 @@ class TempDataSet(DataSet): self.id = 1 self.fullname = 'Temporary Storage' self.shortname = 'Temp' - - + + @staticmethod def handle_pca(desc): if 'PCA' in desc: @@ -1318,13 +1318,13 @@ class TempDataSet(DataSet): else: desc = desc[:desc.index('entered')].strip() return desc - + def get_desc(self): g.db.execute('SELECT description FROM Temp WHERE Name=%s', self.name) desc = g.db.fetchone()[0] desc = self.handle_pca(desc) - return desc - + return desc + def get_group(self): self.cursor.execute(""" SELECT @@ -1337,7 +1337,7 @@ class TempDataSet(DataSet): """, self.name) self.group, self.group_id = self.cursor.fetchone() #return self.group - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1351,7 +1351,7 @@ class TempDataSet(DataSet): Order BY Strain.Name """ % escape(trait.name) - + results = g.db.execute(query).fetchall() diff --git a/wqflask/base/webqtlFormData.py b/wqflask/base/webqtlFormData.py index 44fdcc3f..10251756 100755 --- a/wqflask/base/webqtlFormData.py +++ b/wqflask/base/webqtlFormData.py @@ -157,7 +157,7 @@ class webqtlFormData(object): self.genotype_1 = reaper.Dataset() - full_filename = os.path.join(webqtlConfig.GENODIR, self.group + '.geno') + full_filename = locate(self.group + '.geno','genotype') # reaper barfs on unicode filenames, so here we ensure it's a string full_filename = str(full_filename) diff --git a/wqflask/basicStatistics/BasicStatisticsFunctions.py b/wqflask/basicStatistics/BasicStatisticsFunctions.py index 74784853..e748a822 100755 --- a/wqflask/basicStatistics/BasicStatisticsFunctions.py +++ b/wqflask/basicStatistics/BasicStatisticsFunctions.py @@ -118,7 +118,7 @@ def plotNormalProbability(vals=None, RISet='', title=None, showstrains=0, specia Plot.plotXY(c, dataZ, dataX, dataLabel = dataLabel, XLabel='Expected Z score', connectdot=0, YLabel='Trait value', title=title, specialCases=specialStrains, showLabel = showLabel) filename= webqtlUtil.genRandStr("nP_") - c.save(webqtlConfig.IMGDIR+filename, format='gif') + c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) @@ -145,7 +145,7 @@ def plotBoxPlot(vals): Plot.plotBoxPlot(canvas, XXX, offset=(xLeftOffset, xRightOffset, yTopOffset, yBottomOffset), XLabel= "Trait") filename= webqtlUtil.genRandStr("Box_") - canvas.save(webqtlConfig.IMGDIR+filename, format='gif') + canvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) plotLink = HT.Span("More about ", HT.Href(text="Box Plots", url="http://davidmlane.com/hyperstat/A37797.html", target="_blank", Class="fs13")) @@ -201,7 +201,7 @@ def plotBarGraph(identification='', RISet='', vals=None, type="name"): Plot.plotBarText(c, tvals, tnames, variance=tvars, YLabel='Value', title=title, sLabel = sLabel, barSpace = sw) filename= webqtlUtil.genRandStr("Bar_") - c.save(webqtlConfig.IMGDIR+filename, format='gif') + c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) return img diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py index b8397b47..a9059fad 100755 --- a/wqflask/maintenance/get_group_samplelists.py +++ b/wqflask/maintenance/get_group_samplelists.py @@ -6,7 +6,6 @@ import gzip from base import webqtlConfig - def process_genofiles(geno_dir=webqtlConfig.GENODIR): print("Yabba") #sys.exit("Dabba") @@ -54,4 +53,4 @@ def get_samplelist_from_plink(genofilename): line = line.split(" ") samplelist.append(line[0]) - return samplelist \ No newline at end of file + return samplelist diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 0f2e4d88..c0f6a49a 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -79,6 +79,15 @@ def flat_files(subdir=None): return valid_path(base+"/"+subdir) return valid_path(base) +def assert_dir(dir): + if not valid_path(dir): + raise Exception("ERROR: can not find directory "+dir) + return dir + +def mk_dir(dir): + os.makedirs(dir) + return assert_dir(dir) + def locate(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -98,7 +107,7 @@ def locate(name, subdir=None): if subdir: sys.stderr.write(subdir) raise IOError("Can not locate "+name+" in "+base) -def locate_without_error(name, subdir=None): +def locate_ignore_error(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 21a2c26c..06b4860e 100755 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -943,7 +943,7 @@ class CorrelationResults(object): use_tissue_corr = True DatabaseFileName = self.getFileName( target_db_name=self.target_db_name ) - datasetFile = open(webqtlConfig.TEXTDIR+DatabaseFileName,'r') + datasetFile = open(webqtlConfig.CACHEDIR+DatabaseFileName,'r') #XZ, 01/08/2009: read the first line line = datasetFile.readline() diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 6bc0ef77..f74e655d 100755 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -43,7 +43,6 @@ from pprint import pformat as pf from htmlgen import HTMLgen2 as HT import reaper -from base import webqtlConfig from utility.THCell import THCell from utility.TDCell import TDCell from base.trait import GeneralTrait diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 40f518f0..61847bc3 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -26,8 +26,6 @@ import reaper from base.trait import GeneralTrait from base import data_set from base import species -from base import webqtlConfig -from utility import webqtlUtil # from wqflask.my_pylmm.pyLMM import lmm # from wqflask.my_pylmm.pyLMM import input from utility import helper_functions diff --git a/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py b/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py index ec9aa29c..f45ec0c4 100755 --- a/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py +++ b/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py @@ -45,40 +45,40 @@ class IntervalAnalystPage(templatePage): #A dictionary that lets us map the html form names "txStart_mm6" -> "Mb Start (mm8)" #the first item is the short name (column headers) and the second item is the long name (dropdown list) # [short name, long name, category] - columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'], + columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'], "GeneDescription" : ["Description", "Gene Description", 'species'], - 'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'], - 'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'], - 'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'], + 'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'], + 'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'], + 'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'], "ProteinID" : ["Prot ID", "Protein ID", 'protein'], - "Chromosome" : ["Chr", "Chromosome", 'species'], - "TxStart" : ["Start", "Mb Start", 'species'], - "TxEnd" : ["End", "Mb End", 'species'], - "GeneLength" : ["Length", "Kb Length", 'species'], - "cdsStart" : ["CDS Start", "Mb CDS Start", 'species'], + "Chromosome" : ["Chr", "Chromosome", 'species'], + "TxStart" : ["Start", "Mb Start", 'species'], + "TxEnd" : ["End", "Mb End", 'species'], + "GeneLength" : ["Length", "Kb Length", 'species'], + "cdsStart" : ["CDS Start", "Mb CDS Start", 'species'], "cdsEnd" : ["CDS End", "Mb CDS End", 'species'], - "exonCount" : ["Num Exons", "Exon Count", 'species'], - "exonStarts" : ["Exon Starts", "Exon Starts", 'species'], - "exonEnds" : ["Exon Ends", "Exon Ends", 'species'], - "Strand" : ["Strand", "Strand", 'species'], + "exonCount" : ["Num Exons", "Exon Count", 'species'], + "exonStarts" : ["Exon Starts", "Exon Starts", 'species'], + "exonEnds" : ["Exon Ends", "Exon Ends", 'species'], + "Strand" : ["Strand", "Strand", 'species'], "GeneID" : ["Gene ID", "Gene ID", 'species'], - "GenBankID" : ["GenBank", "GenBank ID", 'species'], + "GenBankID" : ["GenBank", "GenBank ID", 'species'], "UnigenID" : ["Unigen", "Unigen ID", 'species'], - "NM_ID" : ["NM ID", "NM ID", 'species'], + "NM_ID" : ["NM ID", "NM ID", 'species'], "kgID" : ["kg ID", "kg ID", 'species'], - "snpCount" : ["SNPs", "SNP Count", 'species'], - "snpDensity" : ["SNP Density", "SNP Density", 'species'], - "lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'], - "lod" : ["LOD", "Likelihood Odds Ratio", 'misc'], - "pearson" : ["Pearson", "Pearson Product Moment", 'misc'], - "literature" : ["Lit Corr", "Literature Correlation", 'misc'], + "snpCount" : ["SNPs", "SNP Count", 'species'], + "snpDensity" : ["SNP Density", "SNP Density", 'species'], + "lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'], + "lod" : ["LOD", "Likelihood Odds Ratio", 'misc'], + "pearson" : ["Pearson", "Pearson Product Moment", 'misc'], + "literature" : ["Lit Corr", "Literature Correlation", 'misc'], } ###Species Freeze speciesFreeze = {'mouse':'mm9', 'rat':'rn3', 'human':'hg19'} for key in speciesFreeze.keys(): speciesFreeze[speciesFreeze[key]] = key - + def __init__(self, fd): templatePage.__init__(self, fd) @@ -86,7 +86,7 @@ class IntervalAnalystPage(templatePage): fd.formdata['remote_ip'] = fd.remote_ip if not self.openMysql(): return - + self.species = fd.formdata.getvalue("species", "mouse") try: self.startMb = float(fd.formdata.getvalue("startMb")) @@ -96,7 +96,7 @@ class IntervalAnalystPage(templatePage): self.endMb = float(fd.formdata.getvalue("endMb")) except: self.endMb = self.startMb + 10 - + self.Chr = fd.formdata.getvalue("chromosome", "1") self.xls = fd.formdata.getvalue("xls", "1") try: @@ -107,38 +107,38 @@ class IntervalAnalystPage(templatePage): self.diffColDefault = self.diffCol = [] if self.species != 'mouse': self.diffColDefault = [2, 3]#default is B6 and D2 for other species - + controlFrm, dispFields = self.genControlForm(fd) geneTable, filename = self.genGeneTable(fd, dispFields) - + infoTD = HT.TD(width=400, valign= "top") - infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"), - HT.Strong("Species : "), self.species.title(), HT.BR(), - HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(), - HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(), + infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"), + HT.Strong("Species : "), self.species.title(), HT.BR(), + HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(), + HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(), ) if filename: infoTD.append(HT.BR(), HT.BR(), HT.Href(text="Download", url = "/tmp/" + filename, Class="normalsize") , " output in MS excel format.") - + mainTable = HT.TableLite(HT.TR(infoTD, HT.TD(controlFrm, Class="doubleBorder", width=400), HT.TD(" ", width="")), cellpadding=10) mainTable.append(HT.TR(HT.TD(geneTable, colspan=3))) self.dict['body'] = HT.TD(mainTable) self.dict['title'] = "Interval Analyst" - + def genGeneTable(self, fd, dispFields): filename = "" if self.xls: #import pyXLWriter as xl filename = "IntAn_Chr%s_%2.6f-%2.6f" % (self.Chr, self.startMb, self.endMb) filename += ".xls" - + # Create a new Excel workbook workbook = xl.Writer(os.path.join(webqtlConfig.TMPDIR, filename)) worksheet = workbook.add_worksheet() titleStyle = workbook.add_format(align = 'left', bold = 0, size=18, border = 1, border_color="gray") headingStyle = workbook.add_format(align = 'center', bold = 1, size=13, fg_color = 0x1E, color="white", border = 1, border_color="gray") - + ##Write title Info worksheet.write([0, 0], "GeneNetwork Interval Analyst Table", titleStyle) worksheet.write([1, 0], "%s%s" % (webqtlConfig.PORTADDR, os.path.join(webqtlConfig.CGIDIR, self._scriptfile))) @@ -148,12 +148,12 @@ class IntervalAnalystPage(templatePage): worksheet.write([4, 0], "Search by : %s" % fd.formdata['remote_ip']) worksheet.write([5, 0], "view region : Chr %s %2.6f - %2.6f Mb" % (self.Chr, self.startMb, self.endMb)) nTitleRow = 7 - + geneTable = HT.TableLite(Class="collap", cellpadding=5) headerRow = HT.TR(HT.TD(" ", Class="fs13 fwb ffl b1 cw cbrb", width="1")) if self.xls: worksheet.write([nTitleRow, 0], "Index", headingStyle) - + for ncol, column in enumerate(dispFields): if len(column) == 1: headerRow.append(HT.TD(self.columnNames[column[0]][0], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1,align="Center")) @@ -162,24 +162,24 @@ class IntervalAnalystPage(templatePage): worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle) worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle)) else: - headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]], + headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1, align="Center")) if self.xls: colTitle = self.columnNames[column[0]][0] + " (%s)" % self.speciesFreeze[column[1]] worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle) worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle)) - #headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), - # "(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]), + #headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), + # "(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]), # Class="colorBlue", NOWRAP=1, align="Center")) geneTable.append(headerRow) - + geneCol = GeneUtil.loadGenes(self.cursor, self.Chr, self.diffColDefault, self.startMb, self.endMb, species=self.species) for gIndex, theGO in enumerate(geneCol): geneRow = HT.TR(HT.TD(gIndex+1, Class="fs12 fwn b1", align="right")) if self.xls: nTitleRow += 1 worksheet.write([nTitleRow, 0], gIndex + 1) - + for ncol, column in enumerate(dispFields): if len(column) == 1 or column[1]== self.species: keyValue = "" @@ -196,17 +196,17 @@ class IntervalAnalystPage(templatePage): curGO = theGO[subGO] if theGO[subGO].has_key(fieldName): keyValue = theGO[subGO][fieldName] - + if self.xls: worksheet.write([nTitleRow, ncol+1], keyValue) geneRow.append(self.formatTD(keyValue, fieldName, curSpecies, curGO)) - + geneTable.append(geneRow) - + if self.xls: workbook.close() return geneTable, filename - + def formatTD(self, keyValue, fieldName, Species, theGO): if keyValue is None: keyValue = "" @@ -219,7 +219,7 @@ class IntervalAnalystPage(templatePage): keyValue = "" return HT.TD(keyValue, Class="fs12 fwn b1", width=300) elif fieldName in ("GeneSymbol"): - webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), + webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), HT.Image("/images/webqtl_search.gif", border=0, valign="top"), target="_blank") if theGO['GeneID']: geneSymbolLink = HT.Href(webqtlConfig.NCBI_LOCUSID % theGO['GeneID'], keyValue, Class="normalsize", target="_blank") @@ -236,8 +236,8 @@ class IntervalAnalystPage(templatePage): return HT.TD(keyValue, Class="fs12 fwn b1",align="right") elif fieldName in ("snpCount"): if keyValue: - snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'), - theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]), + snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'), + theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]), text=theGO["snpCount"], target="_blank", Class="normalsize") else: snpString = keyValue @@ -252,13 +252,13 @@ class IntervalAnalystPage(templatePage): return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1) else: return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1,align="right") - + def genControlForm(self, fd): ##desc GeneList self.cursor.execute("Desc GeneList") GeneListFields = self.cursor.fetchall() GeneListFields = map(lambda X: X[0], GeneListFields) - + #group columns by category--used for creating the dropdown list of possible columns categories = {} for item in self.columnNames.keys(): @@ -267,7 +267,7 @@ class IntervalAnalystPage(templatePage): categories[category[-1]] = [item ] else: categories[category[-1]] = categories[category[-1]]+[item] - + ##List All Species in the Gene Table speciesDict = {} self.cursor.execute("select Species.Name, GeneList.SpeciesId from Species, GeneList where \ @@ -292,34 +292,34 @@ class IntervalAnalystPage(templatePage): pass AppliedField.append(item2) categories[specName] = AppliedField - + categoriesOrder += ['misc'] - + ############################################################ ## Create the list of possible columns for the dropdown list ############################################################ allColumnsList = HT.Select(name="allColumns", Class="snpBrowserDropBox") - + for category in categoriesOrder: allFields = categories[category] if allFields: geneOpt = HT.Optgroup(label=category.title()) for item in allFields: if category in self.speciesFreeze.keys(): - geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]), + geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]), "%s__%s" % (item, self.speciesFreeze[category]))) else: geneOpt.append((self.columnNames[item][1], item)) geneOpt.sort() allColumnsList.append(geneOpt) - + ###################################### ## Create the list of selected columns ###################################### - + #cols contains the value of all the selected columns submitCols = cols = fd.formdata.getvalue("columns", "default") - + if cols == "default": if self.species=="mouse": #these are the same columns that are shown on intervalPage.py cols = ['GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'Strand', 'GeneLength', 'GeneID', 'NM_ID', 'snpCount', 'snpDensity'] @@ -331,12 +331,12 @@ class IntervalAnalystPage(templatePage): else: if type(cols)==type(""): cols = [cols] - + colsLst = [] dispFields = [] for column in cols: if submitCols == "default" and column not in ('GeneSymbol') and (column in GeneListFields or column in speciesField): - colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]), + colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]), "%s__%s" % (column, self.speciesFreeze[self.species]))) dispFields.append([column, self.species]) else: @@ -346,17 +346,17 @@ class IntervalAnalystPage(templatePage): dispFields.append([column]) else: thisSpecies = self.speciesFreeze[column2[1]] - colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]), + colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]), column)) dispFields.append((column2[0], thisSpecies)) selectedColumnsList = HT.Select(name="columns", Class="snpBrowserSelectBox", multiple="true", data=colsLst, size=6) - + ########################## ## Create the columns form - ########################## + ########################## columnsForm = HT.Form(name="columnsForm", submit=HT.Input(type='hidden'), cgi=os.path.join(webqtlConfig.CGIDIR, self._scriptfile), enctype="multipart/form-data") columnsForm.append(HT.Input(type="hidden", name="fromdatabase", value= fd.formdata.getvalue("fromdatabase", "unknown"))) - columnsForm.append(HT.Input(type="hidden", name="species", value=self.species)) + columnsForm.append(HT.Input(type="hidden", name="species", value=self.species)) if self.diffCol: columnsForm.append(HT.Input(type="hidden", name="s1", value=self.diffCol[0])) columnsForm.append(HT.Input(type="hidden", name="s2", value=self.diffCol[1])) @@ -366,8 +366,8 @@ class IntervalAnalystPage(templatePage): removeButton = HT.Input(type="button", name="remove", value="Remove", Class="button", onClick="removeFromList(this.form.columns.selectedIndex, this.form.columns)") upButton = HT.Input(type="button", name="up", value="Up", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex-1, this.form.columns)") downButton = HT.Input(type="button", name="down", value="Down", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex+1, this.form.columns)") - clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)") - submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)") + clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)") + submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)") selectChrBox = HT.Select(name="chromosome") self.cursor.execute(""" @@ -375,11 +375,11 @@ class IntervalAnalystPage(templatePage): Chr_Length.Name, Length from Chr_Length, Species where Chr_Length.SpeciesId = Species.Id AND - Species.Name = '%s' + Species.Name = '%s' Order by Chr_Length.OrderId """ % self.species) - + results = self.cursor.fetchall() for chrInfo in results: selectChrBox.append((chrInfo[0], chrInfo[0])) @@ -401,5 +401,5 @@ class IntervalAnalystPage(templatePage): #columnsForm.append(HT.Input(type="hidden", name="sort", value=diffCol), # HT.Input(type="hidden", name="identification", value=identification), # HT.Input(type="hidden", name="traitInfo", value=traitInfo)) - + return columnsForm, dispFields diff --git a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py index d02d80b3..455fcf95 100755 --- a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py +++ b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py @@ -140,7 +140,7 @@ class MarkerRegressionPage(templatePage): intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight)) gifmap = self.plotIntMappingForPLINK(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, plinkResultDict=plinkResultDict) - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png') intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap') TD_LR = HT.TR(HT.TD(HT.Blockquote(gifmap,intImg, HT.P()), bgColor='#eeeeee', height = 200)) @@ -249,7 +249,7 @@ class MarkerRegressionPage(templatePage): intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight)) gifmap = self.plotIntMapping(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= "") filename= webqtlUtil.genRandStr("Itvl_") - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png') intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap') ################################################################ @@ -458,7 +458,7 @@ class MarkerRegressionPage(templatePage): #plotBar(myCanvas,10,10,390,290,LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test',identification=fd.identification) Plot.plotBar(myCanvas, LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test') filename= webqtlUtil.genRandStr("Reg_") - myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif') + myCanvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test') if fd.suggestive == None: diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 36334317..265f9473 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -30,7 +30,7 @@ from flask import Flask, g from base.trait import GeneralTrait from base import data_set from base import species -from base import webqtlConfig +# from base import webqtlConfig from utility import webqtlUtil from utility import helper_functions from utility import Plot, Bunch diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 01303b0f..decde579 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -178,8 +178,8 @@ class MarkerRegression(object): self.species = start_vars['species'] #Needing for form submission when doing single chr mapping or remapping after changing options - self.vals = start_vars['vals'] - self.mapping_method = start_vars['mapping_method'] + self.vals = start_vars['vals'] + self.mapping_method = start_vars['mapping_method'] if self.mapping_method == "rqtl_geno": self.mapmethod_rqtl_geno = start_vars['method'] self.mapmodel_rqtl_geno = start_vars['model'] @@ -232,7 +232,7 @@ class MarkerRegression(object): self.significant = start_vars['significant'] else: self.nperm = 0 - + if 'bootCheck' in start_vars.keys(): self.bootChecked = start_vars['bootCheck'] else: @@ -308,7 +308,7 @@ class MarkerRegression(object): if 'showSNP' in start_vars.keys(): self.SNPChecked = start_vars['showSNP'] else: - self.SNPChecked = False + self.SNPChecked = False if 'showGenes' in start_vars.keys(): self.geneChecked = start_vars['showGenes'] else: @@ -321,7 +321,7 @@ class MarkerRegression(object): self.endMb = float(start_vars['endMb']) except: self.endMb = -1 - try: + try: self.lrsMax = float(start_vars['lrsMax']) except: self.lrsMax = 0 @@ -363,7 +363,7 @@ class MarkerRegression(object): self.ChrList.append((indChr.name, i)) - + self.ChrLengthMbList = g.db.execute(""" Select Length from Chr_Length, InbredSet @@ -517,7 +517,7 @@ class MarkerRegression(object): chrName = self.selectedChr # Draw the genes for this chromosome / region of this chromosome webqtldatabase = self.dataset.name - + if self.dataset.group.species == "mouse": self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, webqtldatabase, "mouse") elif self.dataset.group.species == "rat": @@ -533,11 +533,11 @@ class MarkerRegression(object): # # #GENEID = fd.formdata.getvalue('GeneId') or None # GENEID = None - # + # # geneTableContainer = HT.Div(Id="sortable") #Div to hold table # geneTable = self.geneTable(self.geneCol,GENEID) # geneTableContainer.append(geneTable) - # + # # mainfmName = webqtlUtil.genRandStr("fm_") # tableForm = HT.Form(cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name=mainfmName, submit=HT.Input(type='hidden')) # tableForm.append(HT.Input(name='FormID', value='', type='hidden')) @@ -552,22 +552,22 @@ class MarkerRegression(object): #else: showLocusForm = "" intCanvas = pid.PILCanvas(size=(self.graphWidth, self.graphHeight)) - gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) + gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) self.gifmap = gifmap.__str__() #print("GIFMAP:", gifmap.__str__()) self.filename= webqtlUtil.genRandStr("Itvl_") - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, self.filename), format='jpeg') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename), format='jpeg') intImg=HT.Image('/image/'+self.filename+'.png', border=0, usemap='#WebQTLImageMap') #Scales plot differently for high resolution if self.draw2X: intCanvasX2 = pid.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2)) gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2) - intCanvasX2.save(os.path.join(webqtlConfig.IMGDIR, self.filename+"X2"), format='png') + intCanvasX2.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename+"X2"), format='png') #DLintImgX2=HT.Href(text='Download',url = '/image/'+self.filename+'X2.png', Class='smallsize', target='_blank') - + #textUrl = self.writeQTL2Text(fd, self.filename) ################################################################ @@ -597,7 +597,7 @@ class MarkerRegression(object): showLocusForm.append(intImg) else: showLocusForm = intImg - + if self.permChecked and self.nperm > 0 and not self.multipleInterval and 0 < self.nperm: self.perm_filename = self.drawPermutationHistogram() #perm_text_file = self.permutationTextFile() @@ -667,7 +667,7 @@ class MarkerRegression(object): TD_LR.append(HT.Blockquote(tableForm)) self.body = TD_LR - + #self.dict['body'] = TD_LR #self.dict['title'] = "Mapping" @@ -858,7 +858,7 @@ class MarkerRegression(object): BootCoord = [] i = 0 startX = xLeftOffset - + if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes for j, _chr in enumerate(self.genotype): BootCoord.append( []) @@ -881,8 +881,8 @@ class MarkerRegression(object): else: Xc = startX + (_locus.cM-_chr[0].cM)*plotXScale BootCoord[-1].append([Xc, self.bootResult[i]]) - i += 1 - + i += 1 + #reduce bootResult if self.selectedChr > -1: maxBootBar = 80.0 @@ -1411,7 +1411,7 @@ class MarkerRegression(object): if _strains[ii] in self.dataset.group.samplelist: temp = GeneralObject(name=_strains[ii], value=_val) smd.append(temp) - + smd.sort(lambda A, B: cmp(A.value, B.value)) smd.reverse() @@ -1566,14 +1566,14 @@ class MarkerRegression(object): firstGene = 0 else: lastGene = 0 - + for j, _geno in enumerate (self.genotype[0][1].genotype): plotbxd=0 for item in smd: if item.name == samplelist[j]: - plotbxd=1 - + plotbxd=1 + if (plotbxd == 1): ind = 0 @@ -1620,28 +1620,28 @@ class MarkerRegression(object): currentChromosome = self.genotype[0].name i = 0 - + paddingTop = yTopOffset ucscPaddingTop = paddingTop + self.WEBQTL_BAND_HEIGHT + self.BAND_SPACING ensemblPaddingTop = ucscPaddingTop + self.UCSC_BAND_HEIGHT + self.BAND_SPACING - + if zoom == 1: for pixel in range(xLeftOffset, xLeftOffset + plotWidth, pixelStep): - + calBase = self.kONE_MILLION*(startMb + (endMb-startMb)*(pixel-xLeftOffset-0.0)/plotWidth) - + xBrowse1 = pixel xBrowse2 = min(xLeftOffset + plotWidth, (pixel + pixelStep - 1)) - + WEBQTL_COORDS = "%d, %d, %d, %d" % (xBrowse1, paddingTop, xBrowse2, (paddingTop+self.WEBQTL_BAND_HEIGHT)) bandWidth = xBrowse2 - xBrowse1 WEBQTL_HREF = "javascript:rangeView('%s', %f, %f)" % (self.selectedChr - 1, max(0, (calBase-webqtlZoomWidth))/1000000.0, (calBase+webqtlZoomWidth)/1000000.0) - + WEBQTL_TITLE = "Click to view this section of the genome in WebQTL" gifmap.areas.append(HT.Area(shape='rect',coords=WEBQTL_COORDS,href=WEBQTL_HREF, title=WEBQTL_TITLE)) canvas.drawRect(xBrowse1, paddingTop, xBrowse2, (paddingTop + self.WEBQTL_BAND_HEIGHT), edgeColor=self.CLICKABLE_WEBQTL_REGION_COLOR, fillColor=self.CLICKABLE_WEBQTL_REGION_COLOR) canvas.drawLine(xBrowse1, paddingTop, xBrowse1, (paddingTop + self.WEBQTL_BAND_HEIGHT), color=self.CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR) - + UCSC_COORDS = "%d, %d, %d, %d" %(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT)) if self.species == "mouse": UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=%s/snp/chr%s" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases, webqtlConfig.PORTADDR, self.selectedChr) @@ -1651,7 +1651,7 @@ class MarkerRegression(object): gifmap.areas.append(HT.Area(shape='rect',coords=UCSC_COORDS,href=UCSC_HREF, title=UCSC_TITLE)) canvas.drawRect(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), edgeColor=self.CLICKABLE_UCSC_REGION_COLOR, fillColor=self.CLICKABLE_UCSC_REGION_COLOR) canvas.drawLine(xBrowse1, ucscPaddingTop, xBrowse1, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), color=self.CLICKABLE_UCSC_REGION_OUTLINE_COLOR) - + ENSEMBL_COORDS = "%d, %d, %d, %d" %(xBrowse1, ensemblPaddingTop, xBrowse2, (ensemblPaddingTop+self.ENSEMBL_BAND_HEIGHT)) if self.species == "mouse": ENSEMBL_HREF = "http://www.ensembl.org/Mus_musculus/contigview?highlight=&chr=%s&vc_start=%d&vc_end=%d&x=35&y=12" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) @@ -1766,7 +1766,7 @@ class MarkerRegression(object): ChrAInfo = [] preLpos = -1 distinctCount = 0.0 - + #if len(self.genotype) > 1: if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes for i, _chr in enumerate(self.genotype): @@ -1809,7 +1809,7 @@ class MarkerRegression(object): offsetA = -stepA lineColor = pid.lightblue startPosX = xLeftOffset - + for j, ChrInfo in enumerate(ChrAInfo): preLpos = -1 for i, item in enumerate(ChrInfo): @@ -1877,7 +1877,7 @@ class MarkerRegression(object): # lodm = self.LODFACTOR # else: # lodm = 1.0 - + #ZS: This is a mess, but I don't know a better way to account for different mapping methods returning results in different formats + the option to change between LRS and LOD if self.lrsMax <= 0: #sliding scale if "lrs_value" in self.qtlresults[0]: @@ -1890,7 +1890,7 @@ class MarkerRegression(object): else: if self.permChecked and self.nperm > 0 and not self.multipleInterval: self.significant = min(self.significant, webqtlConfig.MAXLRS) - self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) + self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) else: pass else: @@ -1903,10 +1903,10 @@ class MarkerRegression(object): else: if self.permChecked and self.nperm > 0 and not self.multipleInterval: self.significant = min(self.significant, webqtlConfig.MAXLRS) - self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) + self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) else: pass - + if self.permChecked and self.nperm > 0 and not self.multipleInterval: LRS_LOD_Max = max(self.significant, LRS_LOD_Max) @@ -1923,7 +1923,7 @@ class MarkerRegression(object): LRSScale = 2.5 else: LRSScale = 1.0 - + LRSAxisList = Plot.frange(LRSScale, LRS_LOD_Max, LRSScale) #make sure the user's value appears on the y-axis #update by NL 6-21-2011: round the LOD value to 100 when LRS_LOD_Max is equal to 460 @@ -1953,7 +1953,7 @@ class MarkerRegression(object): #"Significant" and "Suggestive" Drawing Routine # ======= Draw the thick lines for "Significant" and "Suggestive" ===== (crowell: I tried to make the SNPs draw over these lines, but piddle wouldn't have it...) - + #ZS: I don't know if what I did here with this inner function is clever or overly complicated, but it's the only way I could think of to avoid duplicating the code inside this function def add_suggestive_significant_lines_and_legend(start_pos_x, chr_length_dist): rightEdge = int(start_pos_x + chr_length_dist*plotXScale - self.SUGGESTIVE_WIDTH/1.5) @@ -1976,13 +1976,13 @@ class MarkerRegression(object): start_pos_x += (chr_length_dist+self.GraphInterval)*plotXScale return start_pos_x - + for i, _chr in enumerate(self.genotype): if self.selectedChr != -1: if _chr.name == self.ChrList[self.selectedChr][0]: startPosX = add_suggestive_significant_lines_and_legend(startPosX, self.ChrLengthDistList[0]) break - else: + else: continue else: startPosX = add_suggestive_significant_lines_and_legend(startPosX, self.ChrLengthDistList[i]) @@ -1997,7 +1997,7 @@ class MarkerRegression(object): #else: # dominanceMax = -1 lrsEdgeWidth = 2 - + if zoom == 2: lrsEdgeWidth = 2 * lrsEdgeWidth @@ -2018,7 +2018,7 @@ class MarkerRegression(object): if qtlresult['chr'] != previous_chr and self.selectedChr == -1: if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and self.additiveChecked: plusColor = self.ADDITIVE_COLOR_POSITIVE minusColor = self.ADDITIVE_COLOR_NEGATIVE @@ -2048,7 +2048,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + LRSCoordXY = [] AdditiveCoordXY = [] previous_chr = qtlresult['chr'] @@ -2061,7 +2061,7 @@ class MarkerRegression(object): #startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale - #for j, _chr in enumerate(self.genotype): + #for j, _chr in enumerate(self.genotype): #ZS: This is beause the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used if self.plotScale == "physic": this_chr = str(self.ChrList[self.selectedChr][0]) @@ -2126,7 +2126,7 @@ class MarkerRegression(object): if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and self.additiveChecked: plusColor = self.ADDITIVE_COLOR_POSITIVE minusColor = self.ADDITIVE_COLOR_NEGATIVE @@ -2156,7 +2156,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and INTERCROSS and self.dominanceChecked: plusColor = self.DOMINANCE_COLOR_POSITIVE minusColor = self.DOMINANCE_COLOR_NEGATIVE @@ -2186,7 +2186,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + ###draw additive scale if not self.multipleInterval and self.additiveChecked: @@ -2222,7 +2222,7 @@ class MarkerRegression(object): if zoom == 2: fontZoom = 1.5 yTopOffset += 30 - + #calculate plot scale if self.plotScale != 'physic': self.ChrLengthDistList = self.ChrLengthCMList @@ -2589,13 +2589,13 @@ class MarkerRegression(object): perm_output = [value/4.16 for value in self.perm_output] else: perm_output = self.perm_output - + Plot.plotBar(myCanvas, perm_output, XLabel=self.LRS_LOD, YLabel='Frequency', title=' Histogram of Permutation Test') filename= webqtlUtil.genRandStr("Reg_") myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif') - + return filename - + # img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test') # self.suggestive = self.perm_output[int(self.nperm*0.37-1)] @@ -2609,9 +2609,9 @@ class MarkerRegression(object): # permutation.append(HT.TR(HT.TD(img)), # HT.TR(HT.TD('')), # HT.TR(HT.TD('Total of %d permutations'%self.nperm))) - + # return permutation - + def permutationTextFile(self): filename= webqtlUtil.genRandStr("Reg_") fpText = open('%s.txt' % (webqtlConfig.TMPDIR+filename), 'wb') @@ -2624,12 +2624,12 @@ class MarkerRegression(object): '    Significant LRS =%3.2f\n'%self.significant, HT.BR(), '    Highly Significant LRS =%3.2f\n' % self.highlysignificant) - + for lrs_value in self.perm_output: fpText.write(str(lrs_value) + "\n") - + textUrl = HT.Href(text = 'Download Permutation Results', url= '/tmp/'+filename+'.txt', target = "_blank", Class='fs12 fwn') - + return textUrl def geneTable(self, geneCol, refGene=None): @@ -2654,7 +2654,7 @@ class MarkerRegression(object): else: gene_table = "" - return gene_table + return gene_table def getLiteratureCorrelation(cursor,geneId1=None,geneId2=None): if not geneId1 or not geneId2: @@ -2954,4 +2954,4 @@ And by voluntary, according to HFG when I talked to him, they have a choice betw sortby = ("", "") - return sortby \ No newline at end of file + return sortby diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 52fe2e34..9941a4d3 100755 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -26,7 +26,6 @@ from MySQLdb import escape_string as escape # Instead of importing HT we're going to build a class below until we can eliminate it # from htmlgen import HTMLgen2 as HT -from base import webqtlConfig from utility.benchmark import Bench from base.data_set import create_dataset from base.trait import GeneralTrait diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 35f7fe5f..e80cf191 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -166,7 +166,7 @@ class ShowTrait(object): return False def check_pylmm_rqtl(): - if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.NEWGENODIR+self.dataset.group.name+".json") > 0): + if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.JSON_GENODIR+self.dataset.group.name+".json") > 0): return True else: return False -- cgit v1.2.3 From 31a1c126185adb747028d035717ad72728a52b0e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 11:25:10 +0000 Subject: -a [PATCH 038/100] Fix compile errors --- wqflask/base/data_set.py | 6 +++--- wqflask/utility/tools.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 6527657a..8b2a9f01 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,7 +44,7 @@ from dbFunction import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks -from utility.tools import locate, locate_without_error +from utility.tools import locate, locate_ignore_error from maintenance import get_group_samplelists @@ -405,8 +405,8 @@ class DatasetGroup(object): else: print("Cache not hit") - genotype_fn = locate_without_error(self.name+".geno",'genotype') - mapping_fn = locate_without_error(self.name+".fam",'mapping') + genotype_fn = locate_ignore_error(self.name+".geno",'genotype') + mapping_fn = locate_ignore_error(self.name+".fam",'mapping') if mapping_fn: self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn) elif genotype_fn: diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index c0f6a49a..67c5128a 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -1,3 +1,4 @@ + # Tools/paths finder resolves external paths from settings and/or environment # variables @@ -83,9 +84,10 @@ def assert_dir(dir): if not valid_path(dir): raise Exception("ERROR: can not find directory "+dir) return dir - + def mk_dir(dir): - os.makedirs(dir) + if not valid_path(dir): + os.makedirs(dir) return assert_dir(dir) def locate(name, subdir=None): -- cgit v1.2.3 From c5dbe982272ef95c6598c36172444b1e8263c01a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 11:39:05 +0000 Subject: -a [PATCH 039/100] Removing commented code --- wqflask/base/data_set.py | 228 ----------------------------------------------- 1 file changed, 228 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8b2a9f01..52ac95f0 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -58,25 +58,13 @@ DS_NAME_MAP = {} def create_dataset(dataset_name, dataset_type = None, get_samplelist = True): if not dataset_type: dataset_type = Dataset_Getter(dataset_name) - #dataset_type = get_dataset_type_from_json(dataset_name) print("dataset_type is:", dataset_type) - #query = """ - # SELECT DBType.Name - # FROM DBList, DBType - # WHERE DBList.Name = '{}' and - # DBType.Id = DBList.DBTypeId - # """.format(escape(dataset_name)) - #dataset_type = g.db.execute(query).fetchone().Name - dataset_ob = DS_NAME_MAP[dataset_type] dataset_class = globals()[dataset_ob] return dataset_class(dataset_name, get_samplelist) - -#def get_dataset_type_from_json(dataset_name): - class Dataset_Types(object): def __init__(self): @@ -90,8 +78,6 @@ class Dataset_Types(object): for group in data['datasets'][species]: for dataset_type in data['datasets'][species][group]: for dataset in data['datasets'][species][group][dataset_type]: - #print("dataset is:", dataset) - short_dataset_name = dataset[1] if dataset_type == "Phenotypes": new_type = "Publish" @@ -107,10 +93,6 @@ class Dataset_Types(object): # Do the intensive work at startup one time only Dataset_Getter = Dataset_Types() -# -#print("Running at startup:", get_dataset_type_from_json("HBTRC-MLPFC_0611")) - - def create_datasets_list(): key = "all_datasets" result = Redis.get(key) @@ -215,13 +197,6 @@ class Markers(object): #del self.markers[i] self.markers = filtered_markers - - #for i, marker in enumerate(self.markers): - # if not 'p_value' in marker: - # #print("self.markers[i]", self.markers[i]) - # del self.markers[i] - # #self.markers.remove(self.markers[i]) - class HumanMarkers(Markers): def __init__(self, name, specified_markers = []): @@ -249,26 +224,8 @@ class HumanMarkers(Markers): def add_pvalues(self, p_values): - #for marker, p_value in itertools.izip(self.markers, p_values): - # if marker['Mb'] <= 0 and marker['chr'] == 0: - # continue - # marker['p_value'] = p_value - # print("p_value is:", marker['p_value']) - # marker['lod_score'] = -math.log10(marker['p_value']) - # #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values - # marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 - - #print("p_values2:", pf(p_values)) super(HumanMarkers, self).add_pvalues(p_values) - #with Bench("deleting markers"): - # markers = [] - # for marker in self.markers: - # if not marker['Mb'] <= 0 and not marker['chr'] == 0: - # markers.append(marker) - # self.markers = markers - - class DatasetGroup(object): """ @@ -311,12 +268,6 @@ class DatasetGroup(object): def datasets(self): key = "group_dataset_menu:v2:" + self.name print("key is2:", key) - #with Bench("Loading cache"): - # result = Redis.get(key) - #if result: - # self._datasets = pickle.loads(result) - # return self._datasets - dataset_menu = [] print("[tape4] webqtlConfig.PUBLICTHRESH:", webqtlConfig.PUBLICTHRESH) print("[tape4] type webqtlConfig.PUBLICTHRESH:", type(webqtlConfig.PUBLICTHRESH)) @@ -425,11 +376,6 @@ class DatasetGroup(object): def read_genotype_file(self): '''Read genotype from .geno file instead of database''' - #if self.group == 'BXD300': - # self.group = 'BXD' - # - #assert self.group, "self.group needs to be set" - #genotype_1 is Dataset Object without parents and f1 #genotype_2 is Dataset Object with parents and f1 (not for intercross) @@ -446,40 +392,16 @@ class DatasetGroup(object): #determine default genotype object if self.incparentsf1 and genotype_1.type != "intercross": - #self.genotype = genotype_2 genotype = genotype_2 else: self.incparentsf1 = 0 - #self.genotype = genotype_1 genotype = genotype_1 - #self.samplelist = list(self.genotype.prgy) self.samplelist = list(genotype.prgy) return genotype -#class DataSets(object): -# """Builds a list of DataSets""" -# -# def __init__(self): -# self.datasets = list() -# - - - #query = """SELECT Name FROM ProbeSetFreeze - # UNION - # SELECT Name From PublishFreeze - # UNION - # SELECT Name From GenoFreeze""" - # - #for result in g.db.execute(query).fetchall(): - # dataset = DataSet(result.Name) - # self.datasets.append(dataset) - -#ds = DataSets() -#print("[orange] ds:", ds.datasets) - class DataSet(object): """ DataSet class defines a dataset in webqtl, can be either Microarray, @@ -512,32 +434,11 @@ class DataSet(object): """Gets overridden later, at least for Temp...used by trait's get_given_name""" return None - #@staticmethod - #def get_by_trait_id(trait_id): - # """Gets the dataset object given the trait id""" - # - # - # - # name = g.db.execute(""" SELECT - # - # """) - # - # return DataSet(name) - # Delete this eventually @property def riset(): Weve_Renamed_This_As_Group - - #@property - #def group(self): - # if not self._group: - # self.get_group() - # - # return self._group - - def retrieve_other_names(self): """ If the data set name parameter is not found in the 'Name' field of the data set table, @@ -609,21 +510,6 @@ class DataSet(object): number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - - #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId - #tempTable = None - #if GeneId and db.type == "ProbeSet": - # if method == "3": - # tempTable = self.getTempLiteratureTable(species=species, - # input_species_geneid=GeneId, - # returnNumber=returnNumber) - # - # if method == "4" or method == "5": - # tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol, - # TissueProbeSetFreezeId=tissueProbeSetFreezeId, - # method=method, - # returnNumber=returnNumber) - if self.type == "Publish": dataset_type = "Phenotype" else: @@ -1013,83 +899,11 @@ class MrnaAssayDataSet(DataSet): and ProbeSetFreezeId = {} """.format(escape(str(self.id))) results = g.db.execute(query).fetchall() - #print("After get_trait_list query") trait_data = {} for trait in results: - #print("Retrieving sample_data for ", trait[0]) trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) - #print("After retrieve_sample_data") return trait_data - #def get_trait_data(self): - # self.samplelist = self.group.samplelist + self.group.parlist + self.group.f1list - # query = """ - # SELECT Strain.Name, Strain.Id FROM Strain, Species - # WHERE Strain.Name IN {} - # and Strain.SpeciesId=Species.Id - # and Species.name = '{}' - # """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) - # results = dict(g.db.execute(query).fetchall()) - # sample_ids = [results[item] for item in self.samplelist] - # - # # MySQL limits the number of tables that can be used in a join to 61, - # # so we break the sample ids into smaller chunks - # # Postgres doesn't have that limit, so we can get rid of this after we transition - # chunk_size = 50 - # number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - # trait_sample_data = [] - # for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - # - # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId - # #tempTable = None - # #if GeneId and db.type == "ProbeSet": - # # if method == "3": - # # tempTable = self.getTempLiteratureTable(species=species, - # # input_species_geneid=GeneId, - # # returnNumber=returnNumber) - # # - # # if method == "4" or method == "5": - # # tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol, - # # TissueProbeSetFreezeId=tissueProbeSetFreezeId, - # # method=method, - # # returnNumber=returnNumber) - # - # temp = ['T%s.value' % item for item in sample_ids_step] - # query = "SELECT {}.Name,".format(escape(self.type)) - # data_start_pos = 1 - # query += string.join(temp, ', ') - # query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(self.type, - # self.type, - # self.type)) - # - # for item in sample_ids_step: - # query += """ - # left join {}Data as T{} on T{}.Id = {}XRef.DataId - # and T{}.StrainId={}\n - # """.format(*mescape(self.type, item, item, self.type, item, item)) - # - # query += """ - # WHERE {}XRef.{}FreezeId = {}Freeze.Id - # and {}Freeze.Name = '{}' - # and {}.Id = {}XRef.{}Id - # order by {}.Id - # """.format(*mescape(self.type, self.type, self.type, self.type, - # self.name, self.type, self.type, self.type, self.type)) - # results = g.db.execute(query).fetchall() - # trait_sample_data.append(results) - # - # trait_count = len(trait_sample_data[0]) - # self.trait_data = collections.defaultdict(list) - # - # # put all of the separate data together into a dictionary where the keys are - # # trait names and values are lists of sample values - # for trait_counter in range(trait_count): - # trait_name = trait_sample_data[0][trait_counter][0] - # for chunk_counter in range(int(number_chunks)): - # self.trait_data[trait_name] += ( - # trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) - - def get_trait_info(self, trait_list=None, species=''): # Note: setting trait_list to [] is probably not a great idea. @@ -1178,22 +992,9 @@ class MrnaAssayDataSet(DataSet): result = g.db.execute(query).fetchone() if result: - #if result[0] and result[1]: - # lrs_chr = result[0] - # lrs_mb = result[1] lrs_chr, lrs_mb = result #XZ: LRS_location_value is used for sorting lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb) - - #try: - # lrs_location_value = int(lrs_chr)*1000 + float(lrs_mb) - #except: - # if lrs_chr.upper() == 'X': - # lrs_location_value = 20*1000 + float(lrs_mb) - # else: - # lrs_location_value = (ord(str(LRS_chr).upper()[0])*1000 + - # float(lrs_mb)) - this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs this_trait.LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb)) @@ -1259,35 +1060,6 @@ class MrnaAssayDataSet(DataSet): return dict(results) - #def retrieve_gene_symbols(self): - # query = """ - # select ProbeSet.Name, ProbeSet.Symbol, ProbeSet.GeneId - # from ProbeSet,ProbeSetXRef - # where ProbeSetXRef.ProbeSetFreezeId = %s and - # ProbeSetXRef.ProbeSetId=ProbeSet.Id; - # """ % (self.id) - # results = g.db.execute(query).fetchall() - # symbol_dict = {} - # for item in results: - # symbol_dict[item[0]] = item[1] - # return symbol_dict - # - #def retrieve_gene_ids(self): - # query = """ - # select ProbeSet.Name, ProbeSet.GeneId - # from ProbeSet,ProbeSetXRef - # where ProbeSetXRef.ProbeSetFreezeId = %s and - # ProbeSetXRef.ProbeSetId=ProbeSet.Id; - # """ % (self.id) - # return process_and_run_query(query) - # results = g.db.execute(query).fetchall() - # symbol_dict = {} - # for item in results: - # symbol_dict[item[0]] = item[1] - # return symbol_dict - - - class TempDataSet(DataSet): '''Temporary user-generated data set''' -- cgit v1.2.3 From 9164ffc23879a8f9338939c06c36d3d49c3c899a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 10:26:54 +0000 Subject: Add file --- wqflask/base/webqtlConfig.py | 84 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 wqflask/base/webqtlConfig.py (limited to 'wqflask/base') diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py new file mode 100755 index 00000000..cdce119a --- /dev/null +++ b/wqflask/base/webqtlConfig.py @@ -0,0 +1,84 @@ +#########################################' +# Environment Variables - public +# +# Note: much of this needs to handled by the settings/environment +# scripts. But rather than migrating everything in one go, we'll +# take it a step at a time. First the hard coded paths get replaced +# with those in utility/tools.py +# +######################################### + +from utility.tools import mk_dir, assert_dir, flat_files, TEMPDIR + +#Debug Level +#1 for debug, mod python will reload import each time +DEBUG = 1 + +#USER privilege +USERDICT = {'guest':1,'user':2, 'admin':3, 'root':4} + +#minimum number of informative strains +KMININFORMATIVE = 5 + +#maximum number of traits for interval mapping +MULTIPLEMAPPINGLIMIT = 11 + +#maximum number of traits for correlation +MAXCORR = 100 + +#Daily download limit from one IP +DAILYMAXIMUM = 1000 + +#maximum LRS value +MAXLRS = 460.0 + +#temporary data life span +MAXLIFE = 86400 + +#MINIMUM Database public value +PUBLICTHRESH = 0 + +#NBCI address +NCBI_LOCUSID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" +UCSC_REFSEQ = "http://genome.cse.ucsc.edu/cgi-bin/hgGene?db=%s&hgg_gene=%s&hgg_chrom=chr%s&hgg_start=%s&hgg_end=%s" +GENBANK_ID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=Nucleotide&cmd=search&doptcmdl=DocSum&term=%s" +OMIM_ID = "http://www.ncbi.nlm.nih.gov/omim/%s" +UNIGEN_ID = "http://www.ncbi.nlm.nih.gov/UniGene/clust.cgi?ORG=%s&CID=%s"; +HOMOLOGENE_ID = "http://www.ncbi.nlm.nih.gov/sites/entrez?Db=homologene&Cmd=DetailsSearch&Term=%s" +PUBMEDLINK_URL = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids=%s&dopt=Abstract" +UCSC_POS = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=mammal&org=%s&db=%s&position=chr%s:%s-%s&pix=800&Submit=submit" +UCSC_BLAT = 'http://genome.ucsc.edu/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' +UTHSC_BLAT = 'http://ucscbrowser.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' +UCSC_GENOME = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=http://web2qtl.utmem.edu:88/snp/chr%s" +ENSEMBLE_BLAT = 'http://www.ensembl.org/Mus_musculus/featureview?type=AffyProbe&id=%s' +DBSNP = 'http://www.ncbi.nlm.nih.gov/SNP/snp_ref.cgi?type=rs&rs=%s' +UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s&hgt.customText=http://gbic.biol.rug.nl/~ralberts/tracks/%s/%s" +GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit" +ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s" + +# HTMLPATH = GNROOT + 'genotype_files/' +# PYLMM_PATH +# IMGDIR = GNROOT + '/wqflask/wqflask/static/output/' + +# Temporary storage: +TMPDIR = mk_dir(TEMPDIR+'/gn2/') +CACHEDIR = mk_dir(TEMPDIR+'/cache/') +# We can no longer write into the git tree: +GENERATED_IMAGE_DIR = mk_dir(TMPDIR+'/generate/') +GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generate_text/') + +# Flat file directories +GENODIR = flat_files('genotype')+'/' +JSON_GENODIR = assert_dir(GENODIR+'json/') + +# SITENAME = 'GN' +# PORTADDR = "http://50.16.251.170" +# BASEHREF = '' + +INFOPAGEHREF = '/dbdoc/%s.html' +CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR' +SCRIPTFILE = 'main.py' + +# GLOSSARYFILE = "/glossary.html" +# REFRESHSTR = '' +# REFRESHDIR = '%s' + SCRIPTFILE +'?sid=%s' -- cgit v1.2.3 From 6e6482db14c3840328d786c551feea6e34a3ef06 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 28 Apr 2016 18:42:20 +0000 Subject: Added option to export permutation results for mapping page Added data_scale to dataset objects and basic stats table will now check data scale when calculating range Made interval analyst results table work with datatables Changed the appearance of the basic stats table some by giving it a border --- wqflask/base/data_set.py | 74 +--------------------- .../wqflask/marker_regression/marker_regression.py | 12 +++- wqflask/wqflask/show_trait/show_trait.py | 1 + .../wqflask/static/new/javascript/show_trait.js | 4 +- .../wqflask/templates/marker_regression_gn1.html | 31 ++++++++- wqflask/wqflask/templates/show_trait.html | 1 - .../wqflask/templates/show_trait_statistics.html | 2 +- wqflask/wqflask/views.py | 23 +++++++ 8 files changed, 68 insertions(+), 80 deletions(-) mode change 100755 => 100644 wqflask/base/data_set.py (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py old mode 100755 new mode 100644 index e37a838f..379e5906 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -509,6 +509,7 @@ class DataSet(object): self.shortname = None self.fullname = None self.type = None + self.data_scale = None #ZS: For example log2 self.setup() @@ -569,8 +570,8 @@ class DataSet(object): self.name, self.name)) - self.id, self.name, self.fullname, self.shortname, self.tissue = g.db.execute(""" - SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, Tissue.Name + self.id, self.name, self.fullname, self.shortname, self.data_scale, self.tissue = g.db.execute(""" + SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, ProbeSetFreeze.DataScale, Tissue.Name FROM ProbeSetFreeze, ProbeFreeze, Tissue WHERE ProbeSetFreeze.public > %s AND ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id AND @@ -1035,75 +1036,6 @@ class MrnaAssayDataSet(DataSet): #print("After retrieve_sample_data") return trait_data - #def get_trait_data(self): - # self.samplelist = self.group.samplelist + self.group.parlist + self.group.f1list - # query = """ - # SELECT Strain.Name, Strain.Id FROM Strain, Species - # WHERE Strain.Name IN {} - # and Strain.SpeciesId=Species.Id - # and Species.name = '{}' - # """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) - # results = dict(g.db.execute(query).fetchall()) - # sample_ids = [results[item] for item in self.samplelist] - # - # # MySQL limits the number of tables that can be used in a join to 61, - # # so we break the sample ids into smaller chunks - # # Postgres doesn't have that limit, so we can get rid of this after we transition - # chunk_size = 50 - # number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - # trait_sample_data = [] - # for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - # - # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId - # #tempTable = None - # #if GeneId and db.type == "ProbeSet": - # # if method == "3": - # # tempTable = self.getTempLiteratureTable(species=species, - # # input_species_geneid=GeneId, - # # returnNumber=returnNumber) - # # - # # if method == "4" or method == "5": - # # tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol, - # # TissueProbeSetFreezeId=tissueProbeSetFreezeId, - # # method=method, - # # returnNumber=returnNumber) - # - # temp = ['T%s.value' % item for item in sample_ids_step] - # query = "SELECT {}.Name,".format(escape(self.type)) - # data_start_pos = 1 - # query += string.join(temp, ', ') - # query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(self.type, - # self.type, - # self.type)) - # - # for item in sample_ids_step: - # query += """ - # left join {}Data as T{} on T{}.Id = {}XRef.DataId - # and T{}.StrainId={}\n - # """.format(*mescape(self.type, item, item, self.type, item, item)) - # - # query += """ - # WHERE {}XRef.{}FreezeId = {}Freeze.Id - # and {}Freeze.Name = '{}' - # and {}.Id = {}XRef.{}Id - # order by {}.Id - # """.format(*mescape(self.type, self.type, self.type, self.type, - # self.name, self.type, self.type, self.type, self.type)) - # results = g.db.execute(query).fetchall() - # trait_sample_data.append(results) - # - # trait_count = len(trait_sample_data[0]) - # self.trait_data = collections.defaultdict(list) - # - # # put all of the separate data together into a dictionary where the keys are - # # trait names and values are lists of sample values - # for trait_counter in range(trait_count): - # trait_name = trait_sample_data[0][trait_counter][0] - # for chunk_counter in range(int(number_chunks)): - # self.trait_data[trait_name] += ( - # trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) - - def get_trait_info(self, trait_list=None, species=''): # Note: setting trait_list to [] is probably not a great idea. diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index fa439b55..c0bfd70b 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -79,6 +79,8 @@ class MarkerRegression(object): self.pair_scan = False # Initializing this since it is checked in views to determine which template to use self.score_type = "LRS" #ZS: LRS or LOD self.mapping_scale = "physic" + self.num_perm = 0 + self.perm_output = [] self.bootstrap_results = [] #ZS: This is passed to GN1 code for single chr mapping @@ -125,13 +127,14 @@ class MarkerRegression(object): try: if int(start_vars['num_perm']) > 0: self.num_perm = int(start_vars['num_perm']) - else: - self.num_perm = 0 except: self.num_perm = 0 self.LRSCheck = self.score_type - self.permCheck = "ON" + if self.num_perm > 0: + self.permCheck = "ON" + else: + self.permCheck = False self.showSNP = "ON" self.showGenes = "ON" self.viewLegend = "ON" @@ -278,6 +281,8 @@ class MarkerRegression(object): mapping_scale = self.mapping_scale, chromosomes = chromosome_mb_lengths, qtl_results = self.qtl_results, + num_perm = self.num_perm, + perm_results = self.perm_output, ) @@ -683,6 +688,7 @@ class MarkerRegression(object): self.perm_output = genotype.permutation(strains = trimmed_samples, trait = trimmed_values, nperm=self.num_perm) self.suggestive = self.perm_output[int(self.num_perm*0.37-1)] self.significant = self.perm_output[int(self.num_perm*0.95-1)] + self.highly_significant = self.perm_output[int(self.num_perm*0.99-1)] self.json_data['suggestive'] = self.suggestive self.json_data['significant'] = self.significant diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 2d4c952a..156510bb 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -153,6 +153,7 @@ class ShowTrait(object): self.trait_table_width = get_trait_table_width(self.sample_groups) js_data = dict(dataset_type = self.dataset.type, + data_scale = self.dataset.data_scale, sample_group_types = self.sample_group_types, sample_lists = sample_lists, attribute_names = self.sample_groups[0].attributes, diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 9d418e9b..9e249c28 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -190,9 +190,9 @@ } row_line = ""; if (row.url != null) { - row_line += "" + row.pretty + ""; + row_line += "" + row.pretty + ""; } else { - row_line += "" + row.pretty + ""; + row_line += "" + row.pretty + ""; } _ref1 = js_data.sample_group_types; for (key in _ref1) { diff --git a/wqflask/wqflask/templates/marker_regression_gn1.html b/wqflask/wqflask/templates/marker_regression_gn1.html index 7f269048..d86d981e 100644 --- a/wqflask/wqflask/templates/marker_regression_gn1.html +++ b/wqflask/wqflask/templates/marker_regression_gn1.html @@ -22,6 +22,7 @@ + @@ -148,9 +149,12 @@
A positive additive coefficient (green line) indicates that {{ dataset.group.parlist[1] }} alleles increase trait values. In contrast, a negative additive coefficient (orange line) indicates that {{ dataset.group.parlist[0] }} alleles increase trait values. {% endif %} - {% if nperm > 0 %} + {% if nperm > 0 and permChecked == "ON" %}

+

+ Total of {{ nperm }} permutations  Download Permutation Results +
{% endif %} @@ -292,7 +296,20 @@ "scrollCollapse": true, "paging": false } ); - console.timeEnd("Creating table"); + + $('#interval_analyst').dataTable( { + "columnDefs": [ { + "targets": 0, + "sortable": false + }], + "order": [[3, "asc"]], + "sDom": "RZtir", + "iDisplayLength": -1, + "autoWidth": true, + "bDeferRender": true, + "bSortClasses": false, + "paging": false + } ); $('#vector_map_tab').click(function(){ $('div#gn1_map_options').hide(); @@ -325,6 +342,16 @@ return $('#marker_regression_form').submit(); }; + export_perm_data = function() { + var num_perm, perm_data; + num_perm = js_data.num_perm + perm_data = js_data.perm_results + json_perm_data = JSON.stringify(perm_data); + $('input[name=perm_results]').val(json_perm_data); + $('#marker_regression_form').attr('action', '/export_perm_data'); + return $('#marker_regression_form').submit(); + }; + {% endblock %} diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index 7429cd5f..64638fc7 100755 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -254,7 +254,6 @@ } ); {% endif %} - console.log("SAMPLE GROUP TYPES:", js_data.sample_group_types) if (Object.keys(js_data.sample_group_types).length > 1) { $('#stats_table').DataTable( { "columnDefs": [ diff --git a/wqflask/wqflask/templates/show_trait_statistics.html b/wqflask/wqflask/templates/show_trait_statistics.html index 9a57b285..242f1c26 100755 --- a/wqflask/wqflask/templates/show_trait_statistics.html +++ b/wqflask/wqflask/templates/show_trait_statistics.html @@ -26,7 +26,7 @@
-
+
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3bf64a18..3cdb9339 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -265,6 +265,29 @@ def export_trait_csv(): return Response(csv_data, mimetype='text/csv', headers={"Content-Disposition":"attachment;filename=sample_data.csv"}) + +@app.route('/export_perm_data', methods=('POST',)) +def export_perm_data(): + """CSV file consisting of the permutation data for the mapping results""" + num_perm = float(request.form['num_perm']) + perm_data = json.loads(request.form['perm_results']) + + buff = StringIO.StringIO() + writer = csv.writer(buff) + writer.writerow(["Suggestive LRS (p=0.63) = " + str(perm_data[int(num_perm*0.37-1)])]) + writer.writerow(["Significant LRS (p=0.05) = " + str(perm_data[int(num_perm*0.95-1)])]) + writer.writerow(["Highly Significant LRS (p=0.01) = " + str(perm_data[int(num_perm*0.99-1)])]) + writer.writerow("") + writer.writerow([str(num_perm) + " Permutations"]) + writer.writerow("") + for item in perm_data: + writer.writerow([item]) + csv_data = buff.getvalue() + buff.close() + + return Response(csv_data, + mimetype='text/csv', + headers={"Content-Disposition":"attachment;filename=perm_data.csv"}) @app.route("/show_trait") def show_trait_page(): -- cgit v1.2.3 From b63e4b5111269dcaf4a9b9eec7d387d8aecb3ba5 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 03:07:41 -0500 Subject: Fix paths for images --- wqflask/base/webqtlConfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index cdce119a..f62207e0 100755 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -64,8 +64,8 @@ ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?sp TMPDIR = mk_dir(TEMPDIR+'/gn2/') CACHEDIR = mk_dir(TEMPDIR+'/cache/') # We can no longer write into the git tree: -GENERATED_IMAGE_DIR = mk_dir(TMPDIR+'/generate/') -GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generate_text/') +GENERATED_IMAGE_DIR = mk_dir(TMPDIR+'/generated/') +GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generated_text/') # Flat file directories GENODIR = flat_files('genotype')+'/' -- cgit v1.2.3 From 409848a4b3873fb0512ae17135b9794030eff024 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 03:21:20 -0500 Subject: WebqtlConfig.py: Constant handling --- wqflask/base/webqtlConfig.py | 7 +------ wqflask/wqflask/marker_regression/marker_regression_gn1.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index f62207e0..6890dacf 100755 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -71,14 +71,9 @@ GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generated_text/') GENODIR = flat_files('genotype')+'/' JSON_GENODIR = assert_dir(GENODIR+'json/') -# SITENAME = 'GN' -# PORTADDR = "http://50.16.251.170" -# BASEHREF = '' +PORTADDR = "http://50.16.251.170" INFOPAGEHREF = '/dbdoc/%s.html' CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR' SCRIPTFILE = 'main.py' -# GLOSSARYFILE = "/glossary.html" -# REFRESHSTR = '' -# REFRESHDIR = '%s' + SCRIPTFILE +'?sid=%s' diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 97e4b934..4460c06d 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -674,7 +674,7 @@ class MarkerRegression(object): fpText.write("Source: WebQTL, The GeneNetwork (%s)\n" % webqtlConfig.PORTADDR) # - fpText.write("Site: %s\n" % webqtlConfig.SITENAME) + fpText.write("Site: GN\n") fpText.write("Page: Map Viewer\n") fpText.write(time.strftime("Date and Time (US Center): %b %d, %Y at %I.%M %p\n", time.localtime())) fpText.write("Trait ID: %s\n" % self.this_trait.name) -- cgit v1.2.3 From 3706cf389a8ead64f02c2ed997fd3defbede9832 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 08:48:18 +0000 Subject: Removing ref. to HTMLPATH --- wqflask/base/webqtlConfig.py | 6 +++--- wqflask/wqflask/marker_regression/MarkerRegressionPage.py | 4 ++-- wqflask/wqflask/marker_regression/gemma_mapping.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 6890dacf..0358bcbf 100755 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -56,9 +56,9 @@ UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit" ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s" -# HTMLPATH = GNROOT + 'genotype_files/' -# PYLMM_PATH -# IMGDIR = GNROOT + '/wqflask/wqflask/static/output/' +# The following paths are no longer in use! +# HTMLPATH is replaced by GENODIR +# IMGDIR is replaced by GENERATED_IMAGE_DIR # Temporary storage: TMPDIR = mk_dir(TEMPDIR+'/gn2/') diff --git a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py index 455fcf95..4c3391e5 100755 --- a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py +++ b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py @@ -72,7 +72,7 @@ class MarkerRegressionPage(templatePage): #automatically generate pheno txt file for PLINK self.genPhenoTxtFileForPlink(phenoFileName=plinkOutputFileName,RISetName=fd.RISet,probesetName=probesetName, valueDict=allTraitValueDict) # os.system full path is required for input and output files; specify missing value is -9999 - plink_command = '%splink/plink --noweb --ped %splink/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %splink/%s.map --pheno %s/%s.txt --pheno-name %s --missing-phenotype -9999 --out %s%s --assoc ' % (webqtlConfig.HTMLPATH, webqtlConfig.HTMLPATH, fd.RISet, webqtlConfig.HTMLPATH, fd.RISet, webqtlConfig.TMPDIR, plinkOutputFileName, probesetName, webqtlConfig.TMPDIR, plinkOutputFileName) + plink_command = '%splink/plink --noweb --ped %splink/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %splink/%s.map --pheno %s/%s.txt --pheno-name %s --missing-phenotype -9999 --out %s%s --assoc ' % (webqtlConfig.GENODIR, webqtlConfig.GENODIR, fd.RISet, webqtlConfig.GENODIR, fd.RISet, webqtlConfig.TMPDIR, plinkOutputFileName, probesetName, webqtlConfig.TMPDIR, plinkOutputFileName) os.system(plink_command) @@ -1597,7 +1597,7 @@ class MarkerRegressionPage(templatePage): # get strain name from ped file in order def getStrainNameFromPedFile(self, RISetName=''): - pedFileopen= open("%splink/%s.ped"%(webqtlConfig.HTMLPATH, RISetName),"r") + pedFileopen= open("%splink/%s.ped"%(webqtlConfig.GENODIR, RISetName),"r") line =pedFileopen.readline() strainNameList=[] diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index 8fb086c1..caea5802 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -11,7 +11,7 @@ def run_gemma(this_dataset, samples, vals): gen_pheno_txt_file(this_dataset, samples, vals) # Don't do this! - # os.chdir("{}gemma".format(webqtlConfig.HTMLPATH)) + # os.chdir("{}gemma".format(webqtlConfig.GENODIR)) # use GEMMA_RUN in the next one, create a unique temp file -- cgit v1.2.3