From 82f731d11a870262b5d4ed693b86d7dcc40493a1 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sat, 13 Feb 2016 12:21:07 +0100 Subject: * ./bin/genenetwork2: New file --- bin/genenetwork2 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 bin/genenetwork2 (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 new file mode 100755 index 00000000..9cac31d5 --- /dev/null +++ b/bin/genenetwork2 @@ -0,0 +1,15 @@ +#! /bin/bash +# +# This will run the server with default settings + +# Absolute path to this script, e.g. /home/user/bin/foo.sh +SCRIPT=$(readlink -f "$0") +# Absolute path this script is in, thus /home/user/bin +SCRIPTPATH=$(dirname $(dirname "$SCRIPT")) +echo $SCRIPTPATH + +export PROFILE=$SCRIPTPATH +export PYTHONPATH=$PROFILE/wqflask:$PYTHONPATH +export WQFLASK_SETTINGS=$PROFILE/etc/default_settings.py + +/usr/bin/env python ./wqflask/runserver.py -- cgit v1.2.3 From 0ddefeab559746b6ab89a20e07edc1c04ee76e28 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sat, 13 Feb 2016 16:15:38 +0100 Subject: * bin/genenetwork2: path handling --- bin/genenetwork2 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 9cac31d5..d0177045 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -5,11 +5,11 @@ # Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") # Absolute path this script is in, thus /home/user/bin -SCRIPTPATH=$(dirname $(dirname "$SCRIPT")) -echo $SCRIPTPATH +GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) +echo $GN2_BASE_PATH -export PROFILE=$SCRIPTPATH -export PYTHONPATH=$PROFILE/wqflask:$PYTHONPATH -export WQFLASK_SETTINGS=$PROFILE/etc/default_settings.py +export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH +export WQFLASK_SETTINGS=$GN2_BASE_PATH/etc/default_settings.py -/usr/bin/env python ./wqflask/runserver.py +cd $GN2_BASE_PATH/wqflask +/usr/bin/env python runserver.py -- cgit v1.2.3 From c76299fa981add7b8cf56126c8d7ffeb9a6c4034 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 16 Feb 2016 10:30:27 +0000 Subject: Successful startup of GN2 with MySQL --- bin/genenetwork2 | 8 +++++++- doc/README.org | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index d0177045..a48e8b93 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -8,8 +8,14 @@ SCRIPT=$(readlink -f "$0") GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) echo $GN2_BASE_PATH +settings=$1 +if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH -export WQFLASK_SETTINGS=$GN2_BASE_PATH/etc/default_settings.py +export WQFLASK_SETTINGS=$settings + +# start the redis server +redis-server & cd $GN2_BASE_PATH/wqflask +echo "Starting with $settings" /usr/bin/env python runserver.py diff --git a/doc/README.org b/doc/README.org index 8e0823de..375a5d20 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,6 +3,8 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - [[#binary-deployment-wip][Binary deployment (WIP)]] + - [[#run-mysql-server][Run MySQL server]] + - [[#start-the-gn2-server][Start the GN2 server]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -16,6 +18,53 @@ explain the GeneNetwork deployment system which is based on GNU Guix GN can be deployed either as a binary tarball or as a GNU Guix package. +** Run MySQL server + +At this point we require the underlying distribution to install +and run mysqld. + +Download one of + +http://files.genenetwork.org/raw_database/ +https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip + +Check the md5sum. + +After installation inflate the database binary in the MySQL directory +(this is subject to change soon) + +: chown -R mysql:mysql db_webqtl_s/ +: chmod 700 db_webqtl_s/ +: chmod 660 db_webqtl_s/* + +restart MySQL service (mysqld). Login as root and + +: mysql> show databases; +: +--------------------+ +: | Database | +: +--------------------+ +: | information_schema | +: | db_webqtl_s | +: | mysql | +: | performance_schema | +: +--------------------+ + +Set permissions and match password in your settings file below: + +: mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; + +** Start the GN2 server + +Copy the default_settings.py file and modify + +: DB_URI = """mysql://gn2:mysql_password@localhost/db_webqtl""" +: SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password/db_webqtl' +: SECRET_KEY = 'secret_key' + +Start the server with your settings file + +: ./bin/genenetwork2 ~/my_gn2_settings.py + * Source deployment and other information on reproducibility See the document [[GUIX-Reproducible-from-source.org]]. -- cgit v1.2.3 From fe69b32993ba48623be49404ae07c817a568a01a Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 23 Feb 2016 07:41:22 +0000 Subject: Tell redis where to store its persistent db --- bin/genenetwork2 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index a48e8b93..54eee02e 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -13,8 +13,14 @@ if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH export WQFLASK_SETTINGS=$settings +if [ -z $TEMPDIR ]; then + TEMPDIR="/tmp" +fi + # start the redis server -redis-server & +echo -n "dir $TEMPDIR +dbfilename gn2.rdb +" | redis-server - & cd $GN2_BASE_PATH/wqflask echo "Starting with $settings" -- cgit v1.2.3 From 5fffa1aa4260af6663c1f9b4cf7494d3ebc6995d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 11:01:29 +0000 Subject: [PATCH 037/100] WebQtlConfig: sanitizing naming and used vars --- bin/genenetwork2 | 2 +- wqflask/base/data_set.py | 194 ++++++++++----------- wqflask/base/webqtlFormData.py | 2 +- .../basicStatistics/BasicStatisticsFunctions.py | 6 +- wqflask/maintenance/get_group_samplelists.py | 3 +- wqflask/utility/tools.py | 11 +- wqflask/wqflask/correlation/show_corr_results.py | 2 +- .../wqflask/correlation_matrix/show_corr_matrix.py | 1 - wqflask/wqflask/heatmap/heatmap.py | 2 - .../interval_analyst/IntervalAnalystPage.py | 138 +++++++-------- .../marker_regression/MarkerRegressionPage.py | 6 +- .../wqflask/marker_regression/marker_regression.py | 2 +- .../marker_regression/marker_regression_gn1.py | 116 ++++++------ wqflask/wqflask/search_results.py | 1 - wqflask/wqflask/show_trait/show_trait.py | 2 +- 15 files changed, 246 insertions(+), 242 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 54eee02e..00ee09bf 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,6 +1,6 @@ #! /bin/bash # -# This will run the server with default settings +# This will run the server with default settings. # Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index ce13dd77..6527657a 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -76,22 +76,22 @@ def create_dataset(dataset_name, dataset_type = None, get_samplelist = True): #def get_dataset_type_from_json(dataset_name): - + class Dataset_Types(object): - + def __init__(self): self.datasets = {} file_name = "wqflask/static/new/javascript/dataset_menu_structure.json" with open(file_name, 'r') as fh: data = json.load(fh) - + print("*" * 70) for species in data['datasets']: for group in data['datasets'][species]: for dataset_type in data['datasets'][species][group]: for dataset in data['datasets'][species][group][dataset_type]: #print("dataset is:", dataset) - + short_dataset_name = dataset[1] if dataset_type == "Phenotypes": new_type = "Publish" @@ -100,32 +100,32 @@ class Dataset_Types(object): else: new_type = "ProbeSet" self.datasets[short_dataset_name] = new_type - + def __call__(self, name): return self.datasets[name] - + # Do the intensive work at startup one time only Dataset_Getter = Dataset_Types() # #print("Running at startup:", get_dataset_type_from_json("HBTRC-MLPFC_0611")) - + def create_datasets_list(): key = "all_datasets" result = Redis.get(key) - + if result: print("Cache hit!!!") datasets = pickle.loads(result) - + else: datasets = list() with Bench("Creating DataSets object"): type_dict = {'Publish': 'PublishFreeze', 'ProbeSet': 'ProbeSetFreeze', 'Geno': 'GenoFreeze'} - + for dataset_type in type_dict: query = "SELECT Name FROM {}".format(type_dict[dataset_type]) for result in g.db.execute(query).fetchall(): @@ -134,10 +134,10 @@ def create_datasets_list(): #print("type: {}\tname: {}".format(dataset_type, result.Name)) dataset = create_dataset(result.Name, dataset_type) datasets.append(dataset) - + Redis.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL)) Redis.expire(key, 60*60) - + return datasets @@ -158,30 +158,30 @@ def mescape(*items): class Markers(object): """Todo: Build in cacheing so it saves us reading the same file more than once""" def __init__(self, name): - json_data_fh = open(os.path.join(webqtlConfig.NEWGENODIR + name + '.json')) + json_data_fh = open(locate(name + '.json','genotype/json')) try: markers = json.load(json_data_fh) except: markers = [] - + for marker in markers: if (marker['chr'] != "X") and (marker['chr'] != "Y"): marker['chr'] = int(marker['chr']) marker['Mb'] = float(marker['Mb']) - + self.markers = markers #print("self.markers:", self.markers) - - + + def add_pvalues(self, p_values): print("length of self.markers:", len(self.markers)) print("length of p_values:", len(p_values)) - + if type(p_values) is list: # THIS IS only needed for the case when we are limiting the number of p-values calculated #if len(self.markers) > len(p_values): # self.markers = self.markers[:len(p_values)] - + for marker, p_value in itertools.izip(self.markers, p_values): if not p_value: continue @@ -214,7 +214,7 @@ class Markers(object): #self.markers.remove(marker) #del self.markers[i] self.markers = filtered_markers - + #for i, marker in enumerate(self.markers): # if not 'p_value' in marker: @@ -223,9 +223,9 @@ class Markers(object): # #self.markers.remove(self.markers[i]) class HumanMarkers(Markers): - + def __init__(self, name, specified_markers = []): - marker_data_fh = open(os.path.join(webqtlConfig.PYLMM_PATH + name + '.bim')) + marker_data_fh = open(locate('genotype') + '/' + name + '.bim') self.markers = [] for line in marker_data_fh: splat = line.strip().split() @@ -244,7 +244,7 @@ class HumanMarkers(Markers): marker['name'] = splat[1] marker['Mb'] = float(splat[3]) / 1000000 self.markers.append(marker) - + #print("markers is: ", pf(self.markers)) @@ -257,26 +257,26 @@ class HumanMarkers(Markers): # marker['lod_score'] = -math.log10(marker['p_value']) # #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values # marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 - + #print("p_values2:", pf(p_values)) super(HumanMarkers, self).add_pvalues(p_values) - + #with Bench("deleting markers"): # markers = [] # for marker in self.markers: # if not marker['Mb'] <= 0 and not marker['chr'] == 0: # markers.append(marker) # self.markers = markers - - + + class DatasetGroup(object): """ Each group has multiple datasets; each species has multiple groups. - + For example, Mouse has multiple groups (BXD, BXA, etc), and each group has multiple datasets associated with it. - + """ def __init__(self, dataset): """This sets self.group and self.group_id""" @@ -284,14 +284,14 @@ class DatasetGroup(object): self.name, self.id = g.db.execute(dataset.query_for_group).fetchone() if self.name == 'BXD300': self.name = "BXD" - + self.f1list = None self.parlist = None self.get_f1_parent_strains() #print("parents/f1s: {}:{}".format(self.parlist, self.f1list)) - + self.species = webqtlDatabaseFunction.retrieve_species(self.name) - + self.incparentsf1 = False self.allsamples = None self._datasets = None @@ -302,7 +302,7 @@ class DatasetGroup(object): def get_markers(self): #print("self.species is:", self.species) if self.species == "human": - marker_class = HumanMarkers + marker_class = HumanMarkers else: marker_class = Markers @@ -356,7 +356,7 @@ class DatasetGroup(object): dataset_menu.append(dict(tissue=None, datasets=[(dataset, dataset_short)])) else: dataset_sub_menu = [item[1:] for item in dataset] - + tissue_already_exists = False tissue_position = None for i, tissue_dict in enumerate(dataset_menu): @@ -384,7 +384,7 @@ class DatasetGroup(object): f1, f12, maternal, paternal = webqtlUtil.ParInfo[self.name] except KeyError: f1 = f12 = maternal = paternal = None - + if f1 and f12: self.f1list = [f1, f12] if maternal and paternal: @@ -455,18 +455,18 @@ class DatasetGroup(object): #self.samplelist = list(self.genotype.prgy) self.samplelist = list(genotype.prgy) - + return genotype #class DataSets(object): # """Builds a list of DataSets""" -# +# # def __init__(self): # self.datasets = list() -# +# + - #query = """SELECT Name FROM ProbeSetFreeze # UNION # SELECT Name From PublishFreeze @@ -501,7 +501,7 @@ class DataSet(object): self.check_confidentiality() self.retrieve_other_names() - + self.group = DatasetGroup(self) # sets self.group and self.group_id and gets genotype if get_samplelist == True: self.group.get_samplelist() @@ -511,30 +511,30 @@ class DataSet(object): def get_desc(self): """Gets overridden later, at least for Temp...used by trait's get_given_name""" return None - + #@staticmethod #def get_by_trait_id(trait_id): # """Gets the dataset object given the trait id""" - # - # # - # name = g.db.execute(""" SELECT - # + # + # + # name = g.db.execute(""" SELECT + # # """) - # + # # return DataSet(name) # Delete this eventually @property def riset(): Weve_Renamed_This_As_Group - - + + #@property #def group(self): # if not self._group: # self.get_group() - # + # # return self._group @@ -546,7 +546,7 @@ class DataSet(object): This is not meant to retrieve the data set info if no name at all is passed. """ - + try: if self.type == "ProbeSet": query_args = tuple(escape(x) for x in ( @@ -582,17 +582,17 @@ class DataSet(object): except TypeError: print("Dataset {} is not yet available in GeneNetwork.".format(self.name)) pass - + def get_trait_data(self, sample_list=None): if sample_list: self.samplelist = sample_list else: self.samplelist = self.group.samplelist - + if self.group.parlist != None and self.group.f1list != None: if (self.group.parlist + self.group.f1list) in self.samplelist: self.samplelist += self.group.parlist + self.group.f1list - + query = """ SELECT Strain.Name, Strain.Id FROM Strain, Species WHERE Strain.Name IN {} @@ -610,9 +610,9 @@ class DataSet(object): trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId + #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId #tempTable = None - #if GeneId and db.type == "ProbeSet": + #if GeneId and db.type == "ProbeSet": # if method == "3": # tempTable = self.getTempLiteratureTable(species=species, # input_species_geneid=GeneId, @@ -623,7 +623,7 @@ class DataSet(object): # TissueProbeSetFreezeId=tissueProbeSetFreezeId, # method=method, # returnNumber=returnNumber) - + if self.type == "Publish": dataset_type = "Phenotype" else: @@ -644,7 +644,7 @@ class DataSet(object): left join {}Data as T{} on T{}.Id = {}XRef.DataId and T{}.StrainId={}\n """.format(*mescape(self.type, item, item, self.type, item, item)) - + if self.type == "Publish": query += """ WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId @@ -661,16 +661,16 @@ class DataSet(object): order by {}.Id """.format(*mescape(self.type, self.type, self.type, self.type, self.name, dataset_type, self.type, self.type, dataset_type)) - + #print("trait data query: ", query) - + results = g.db.execute(query).fetchall() #print("query results:", results) trait_sample_data.append(results) trait_count = len(trait_sample_data[0]) self.trait_data = collections.defaultdict(list) - + # put all of the separate data together into a dictionary where the keys are # trait names and values are lists of sample values for trait_counter in range(trait_count): @@ -683,9 +683,9 @@ class PhenotypeDataSet(DataSet): DS_NAME_MAP['Publish'] = 'PhenotypeDataSet' def setup(self): - + #print("IS A PHENOTYPEDATASET") - + # Fields in the database table self.search_fields = ['Phenotype.Post_publication_description', 'Phenotype.Pre_publication_description', @@ -756,26 +756,26 @@ class PhenotypeDataSet(DataSet): def get_trait_info(self, trait_list, species = ''): for this_trait in trait_list: - + if not this_trait.haveinfo: this_trait.retrieve_info(get_qtl_info=True) description = this_trait.post_publication_description - + #If the dataset is confidential and the user has access to confidential #phenotype traits, then display the pre-publication description instead #of the post-publication description if this_trait.confidential: this_trait.description_display = "" continue # for now - + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait( privilege=self.privilege, userName=self.userName, authorized_users=this_trait.authorized_users): - + description = this_trait.pre_publication_description - + if len(description) > 0: this_trait.description_display = description.strip() else: @@ -820,7 +820,7 @@ class PhenotypeDataSet(DataSet): this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs this_trait.LRS_score_value = LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb)) - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -878,7 +878,7 @@ class GenotypeDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - + def get_trait_list(self): query = """ select Geno.Name @@ -912,7 +912,7 @@ class GenotypeDataSet(DataSet): this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb) ) this_trait.location_value = trait_location_value - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1004,7 +1004,7 @@ class MrnaAssayDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - + def get_trait_list_1(self): query = """ select ProbeSet.Name @@ -1020,7 +1020,7 @@ class MrnaAssayDataSet(DataSet): trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) #print("After retrieve_sample_data") return trait_data - + #def get_trait_data(self): # self.samplelist = self.group.samplelist + self.group.parlist + self.group.f1list # query = """ @@ -1040,9 +1040,9 @@ class MrnaAssayDataSet(DataSet): # trait_sample_data = [] # for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): # - # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId + # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId # #tempTable = None - # #if GeneId and db.type == "ProbeSet": + # #if GeneId and db.type == "ProbeSet": # # if method == "3": # # tempTable = self.getTempLiteratureTable(species=species, # # input_species_geneid=GeneId, @@ -1053,7 +1053,7 @@ class MrnaAssayDataSet(DataSet): # # TissueProbeSetFreezeId=tissueProbeSetFreezeId, # # method=method, # # returnNumber=returnNumber) - # + # # temp = ['T%s.value' % item for item in sample_ids_step] # query = "SELECT {}.Name,".format(escape(self.type)) # data_start_pos = 1 @@ -1067,7 +1067,7 @@ class MrnaAssayDataSet(DataSet): # left join {}Data as T{} on T{}.Id = {}XRef.DataId # and T{}.StrainId={}\n # """.format(*mescape(self.type, item, item, self.type, item, item)) - # + # # query += """ # WHERE {}XRef.{}FreezeId = {}Freeze.Id # and {}Freeze.Name = '{}' @@ -1080,7 +1080,7 @@ class MrnaAssayDataSet(DataSet): # # trait_count = len(trait_sample_data[0]) # self.trait_data = collections.defaultdict(list) - # + # # # put all of the separate data together into a dictionary where the keys are # # trait names and values are lists of sample values # for trait_counter in range(trait_count): @@ -1088,11 +1088,11 @@ class MrnaAssayDataSet(DataSet): # for chunk_counter in range(int(number_chunks)): # self.trait_data[trait_name] += ( # trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) - + def get_trait_info(self, trait_list=None, species=''): - # Note: setting trait_list to [] is probably not a great idea. + # Note: setting trait_list to [] is probably not a great idea. if not trait_list: trait_list = [] @@ -1155,7 +1155,7 @@ class MrnaAssayDataSet(DataSet): #print("query is:", pf(query)) result = g.db.execute(query).fetchone() - + mean = result[0] if result else 0 if mean: @@ -1176,7 +1176,7 @@ class MrnaAssayDataSet(DataSet): Geno.SpeciesId = Species.Id """.format(species, this_trait.locus) result = g.db.execute(query).fetchone() - + if result: #if result[0] and result[1]: # lrs_chr = result[0] @@ -1184,7 +1184,7 @@ class MrnaAssayDataSet(DataSet): lrs_chr, lrs_mb = result #XZ: LRS_location_value is used for sorting lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb) - + #try: # lrs_location_value = int(lrs_chr)*1000 + float(lrs_mb) #except: @@ -1197,7 +1197,7 @@ class MrnaAssayDataSet(DataSet): this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs this_trait.LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb)) - + def convert_location_to_value(self, chromosome, mb): try: @@ -1208,7 +1208,7 @@ class MrnaAssayDataSet(DataSet): else: location_value = (ord(str(chromosome).upper()[0])*1000 + float(mb)) - + return location_value def get_sequence(self): @@ -1225,7 +1225,7 @@ class MrnaAssayDataSet(DataSet): """ % (escape(self.name), escape(self.dataset.name)) results = g.db.execute(query).fetchone() return results[0] - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1246,8 +1246,8 @@ class MrnaAssayDataSet(DataSet): results = g.db.execute(query).fetchall() #print("RETRIEVED RESULTS HERE:", results) return results - - + + def retrieve_genes(self, column_name): query = """ select ProbeSet.Name, ProbeSet.%s @@ -1256,7 +1256,7 @@ class MrnaAssayDataSet(DataSet): ProbeSetXRef.ProbeSetId=ProbeSet.Id; """ % (column_name, escape(str(self.id))) results = g.db.execute(query).fetchall() - + return dict(results) #def retrieve_gene_symbols(self): @@ -1285,8 +1285,8 @@ class MrnaAssayDataSet(DataSet): # for item in results: # symbol_dict[item[0]] = item[1] # return symbol_dict - - + + class TempDataSet(DataSet): @@ -1308,8 +1308,8 @@ class TempDataSet(DataSet): self.id = 1 self.fullname = 'Temporary Storage' self.shortname = 'Temp' - - + + @staticmethod def handle_pca(desc): if 'PCA' in desc: @@ -1318,13 +1318,13 @@ class TempDataSet(DataSet): else: desc = desc[:desc.index('entered')].strip() return desc - + def get_desc(self): g.db.execute('SELECT description FROM Temp WHERE Name=%s', self.name) desc = g.db.fetchone()[0] desc = self.handle_pca(desc) - return desc - + return desc + def get_group(self): self.cursor.execute(""" SELECT @@ -1337,7 +1337,7 @@ class TempDataSet(DataSet): """, self.name) self.group, self.group_id = self.cursor.fetchone() #return self.group - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1351,7 +1351,7 @@ class TempDataSet(DataSet): Order BY Strain.Name """ % escape(trait.name) - + results = g.db.execute(query).fetchall() diff --git a/wqflask/base/webqtlFormData.py b/wqflask/base/webqtlFormData.py index 44fdcc3f..10251756 100755 --- a/wqflask/base/webqtlFormData.py +++ b/wqflask/base/webqtlFormData.py @@ -157,7 +157,7 @@ class webqtlFormData(object): self.genotype_1 = reaper.Dataset() - full_filename = os.path.join(webqtlConfig.GENODIR, self.group + '.geno') + full_filename = locate(self.group + '.geno','genotype') # reaper barfs on unicode filenames, so here we ensure it's a string full_filename = str(full_filename) diff --git a/wqflask/basicStatistics/BasicStatisticsFunctions.py b/wqflask/basicStatistics/BasicStatisticsFunctions.py index 74784853..e748a822 100755 --- a/wqflask/basicStatistics/BasicStatisticsFunctions.py +++ b/wqflask/basicStatistics/BasicStatisticsFunctions.py @@ -118,7 +118,7 @@ def plotNormalProbability(vals=None, RISet='', title=None, showstrains=0, specia Plot.plotXY(c, dataZ, dataX, dataLabel = dataLabel, XLabel='Expected Z score', connectdot=0, YLabel='Trait value', title=title, specialCases=specialStrains, showLabel = showLabel) filename= webqtlUtil.genRandStr("nP_") - c.save(webqtlConfig.IMGDIR+filename, format='gif') + c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) @@ -145,7 +145,7 @@ def plotBoxPlot(vals): Plot.plotBoxPlot(canvas, XXX, offset=(xLeftOffset, xRightOffset, yTopOffset, yBottomOffset), XLabel= "Trait") filename= webqtlUtil.genRandStr("Box_") - canvas.save(webqtlConfig.IMGDIR+filename, format='gif') + canvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) plotLink = HT.Span("More about ", HT.Href(text="Box Plots", url="http://davidmlane.com/hyperstat/A37797.html", target="_blank", Class="fs13")) @@ -201,7 +201,7 @@ def plotBarGraph(identification='', RISet='', vals=None, type="name"): Plot.plotBarText(c, tvals, tnames, variance=tvars, YLabel='Value', title=title, sLabel = sLabel, barSpace = sw) filename= webqtlUtil.genRandStr("Bar_") - c.save(webqtlConfig.IMGDIR+filename, format='gif') + c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) return img diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py index b8397b47..a9059fad 100755 --- a/wqflask/maintenance/get_group_samplelists.py +++ b/wqflask/maintenance/get_group_samplelists.py @@ -6,7 +6,6 @@ import gzip from base import webqtlConfig - def process_genofiles(geno_dir=webqtlConfig.GENODIR): print("Yabba") #sys.exit("Dabba") @@ -54,4 +53,4 @@ def get_samplelist_from_plink(genofilename): line = line.split(" ") samplelist.append(line[0]) - return samplelist \ No newline at end of file + return samplelist diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 0f2e4d88..c0f6a49a 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -79,6 +79,15 @@ def flat_files(subdir=None): return valid_path(base+"/"+subdir) return valid_path(base) +def assert_dir(dir): + if not valid_path(dir): + raise Exception("ERROR: can not find directory "+dir) + return dir + +def mk_dir(dir): + os.makedirs(dir) + return assert_dir(dir) + def locate(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -98,7 +107,7 @@ def locate(name, subdir=None): if subdir: sys.stderr.write(subdir) raise IOError("Can not locate "+name+" in "+base) -def locate_without_error(name, subdir=None): +def locate_ignore_error(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 21a2c26c..06b4860e 100755 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -943,7 +943,7 @@ class CorrelationResults(object): use_tissue_corr = True DatabaseFileName = self.getFileName( target_db_name=self.target_db_name ) - datasetFile = open(webqtlConfig.TEXTDIR+DatabaseFileName,'r') + datasetFile = open(webqtlConfig.CACHEDIR+DatabaseFileName,'r') #XZ, 01/08/2009: read the first line line = datasetFile.readline() diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 6bc0ef77..f74e655d 100755 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -43,7 +43,6 @@ from pprint import pformat as pf from htmlgen import HTMLgen2 as HT import reaper -from base import webqtlConfig from utility.THCell import THCell from utility.TDCell import TDCell from base.trait import GeneralTrait diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 40f518f0..61847bc3 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -26,8 +26,6 @@ import reaper from base.trait import GeneralTrait from base import data_set from base import species -from base import webqtlConfig -from utility import webqtlUtil # from wqflask.my_pylmm.pyLMM import lmm # from wqflask.my_pylmm.pyLMM import input from utility import helper_functions diff --git a/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py b/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py index ec9aa29c..f45ec0c4 100755 --- a/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py +++ b/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py @@ -45,40 +45,40 @@ class IntervalAnalystPage(templatePage): #A dictionary that lets us map the html form names "txStart_mm6" -> "Mb Start (mm8)" #the first item is the short name (column headers) and the second item is the long name (dropdown list) # [short name, long name, category] - columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'], + columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'], "GeneDescription" : ["Description", "Gene Description", 'species'], - 'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'], - 'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'], - 'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'], + 'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'], + 'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'], + 'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'], "ProteinID" : ["Prot ID", "Protein ID", 'protein'], - "Chromosome" : ["Chr", "Chromosome", 'species'], - "TxStart" : ["Start", "Mb Start", 'species'], - "TxEnd" : ["End", "Mb End", 'species'], - "GeneLength" : ["Length", "Kb Length", 'species'], - "cdsStart" : ["CDS Start", "Mb CDS Start", 'species'], + "Chromosome" : ["Chr", "Chromosome", 'species'], + "TxStart" : ["Start", "Mb Start", 'species'], + "TxEnd" : ["End", "Mb End", 'species'], + "GeneLength" : ["Length", "Kb Length", 'species'], + "cdsStart" : ["CDS Start", "Mb CDS Start", 'species'], "cdsEnd" : ["CDS End", "Mb CDS End", 'species'], - "exonCount" : ["Num Exons", "Exon Count", 'species'], - "exonStarts" : ["Exon Starts", "Exon Starts", 'species'], - "exonEnds" : ["Exon Ends", "Exon Ends", 'species'], - "Strand" : ["Strand", "Strand", 'species'], + "exonCount" : ["Num Exons", "Exon Count", 'species'], + "exonStarts" : ["Exon Starts", "Exon Starts", 'species'], + "exonEnds" : ["Exon Ends", "Exon Ends", 'species'], + "Strand" : ["Strand", "Strand", 'species'], "GeneID" : ["Gene ID", "Gene ID", 'species'], - "GenBankID" : ["GenBank", "GenBank ID", 'species'], + "GenBankID" : ["GenBank", "GenBank ID", 'species'], "UnigenID" : ["Unigen", "Unigen ID", 'species'], - "NM_ID" : ["NM ID", "NM ID", 'species'], + "NM_ID" : ["NM ID", "NM ID", 'species'], "kgID" : ["kg ID", "kg ID", 'species'], - "snpCount" : ["SNPs", "SNP Count", 'species'], - "snpDensity" : ["SNP Density", "SNP Density", 'species'], - "lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'], - "lod" : ["LOD", "Likelihood Odds Ratio", 'misc'], - "pearson" : ["Pearson", "Pearson Product Moment", 'misc'], - "literature" : ["Lit Corr", "Literature Correlation", 'misc'], + "snpCount" : ["SNPs", "SNP Count", 'species'], + "snpDensity" : ["SNP Density", "SNP Density", 'species'], + "lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'], + "lod" : ["LOD", "Likelihood Odds Ratio", 'misc'], + "pearson" : ["Pearson", "Pearson Product Moment", 'misc'], + "literature" : ["Lit Corr", "Literature Correlation", 'misc'], } ###Species Freeze speciesFreeze = {'mouse':'mm9', 'rat':'rn3', 'human':'hg19'} for key in speciesFreeze.keys(): speciesFreeze[speciesFreeze[key]] = key - + def __init__(self, fd): templatePage.__init__(self, fd) @@ -86,7 +86,7 @@ class IntervalAnalystPage(templatePage): fd.formdata['remote_ip'] = fd.remote_ip if not self.openMysql(): return - + self.species = fd.formdata.getvalue("species", "mouse") try: self.startMb = float(fd.formdata.getvalue("startMb")) @@ -96,7 +96,7 @@ class IntervalAnalystPage(templatePage): self.endMb = float(fd.formdata.getvalue("endMb")) except: self.endMb = self.startMb + 10 - + self.Chr = fd.formdata.getvalue("chromosome", "1") self.xls = fd.formdata.getvalue("xls", "1") try: @@ -107,38 +107,38 @@ class IntervalAnalystPage(templatePage): self.diffColDefault = self.diffCol = [] if self.species != 'mouse': self.diffColDefault = [2, 3]#default is B6 and D2 for other species - + controlFrm, dispFields = self.genControlForm(fd) geneTable, filename = self.genGeneTable(fd, dispFields) - + infoTD = HT.TD(width=400, valign= "top") - infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"), - HT.Strong("Species : "), self.species.title(), HT.BR(), - HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(), - HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(), + infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"), + HT.Strong("Species : "), self.species.title(), HT.BR(), + HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(), + HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(), ) if filename: infoTD.append(HT.BR(), HT.BR(), HT.Href(text="Download", url = "/tmp/" + filename, Class="normalsize") , " output in MS excel format.") - + mainTable = HT.TableLite(HT.TR(infoTD, HT.TD(controlFrm, Class="doubleBorder", width=400), HT.TD(" ", width="")), cellpadding=10) mainTable.append(HT.TR(HT.TD(geneTable, colspan=3))) self.dict['body'] = HT.TD(mainTable) self.dict['title'] = "Interval Analyst" - + def genGeneTable(self, fd, dispFields): filename = "" if self.xls: #import pyXLWriter as xl filename = "IntAn_Chr%s_%2.6f-%2.6f" % (self.Chr, self.startMb, self.endMb) filename += ".xls" - + # Create a new Excel workbook workbook = xl.Writer(os.path.join(webqtlConfig.TMPDIR, filename)) worksheet = workbook.add_worksheet() titleStyle = workbook.add_format(align = 'left', bold = 0, size=18, border = 1, border_color="gray") headingStyle = workbook.add_format(align = 'center', bold = 1, size=13, fg_color = 0x1E, color="white", border = 1, border_color="gray") - + ##Write title Info worksheet.write([0, 0], "GeneNetwork Interval Analyst Table", titleStyle) worksheet.write([1, 0], "%s%s" % (webqtlConfig.PORTADDR, os.path.join(webqtlConfig.CGIDIR, self._scriptfile))) @@ -148,12 +148,12 @@ class IntervalAnalystPage(templatePage): worksheet.write([4, 0], "Search by : %s" % fd.formdata['remote_ip']) worksheet.write([5, 0], "view region : Chr %s %2.6f - %2.6f Mb" % (self.Chr, self.startMb, self.endMb)) nTitleRow = 7 - + geneTable = HT.TableLite(Class="collap", cellpadding=5) headerRow = HT.TR(HT.TD(" ", Class="fs13 fwb ffl b1 cw cbrb", width="1")) if self.xls: worksheet.write([nTitleRow, 0], "Index", headingStyle) - + for ncol, column in enumerate(dispFields): if len(column) == 1: headerRow.append(HT.TD(self.columnNames[column[0]][0], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1,align="Center")) @@ -162,24 +162,24 @@ class IntervalAnalystPage(templatePage): worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle) worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle)) else: - headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]], + headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1, align="Center")) if self.xls: colTitle = self.columnNames[column[0]][0] + " (%s)" % self.speciesFreeze[column[1]] worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle) worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle)) - #headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), - # "(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]), + #headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), + # "(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]), # Class="colorBlue", NOWRAP=1, align="Center")) geneTable.append(headerRow) - + geneCol = GeneUtil.loadGenes(self.cursor, self.Chr, self.diffColDefault, self.startMb, self.endMb, species=self.species) for gIndex, theGO in enumerate(geneCol): geneRow = HT.TR(HT.TD(gIndex+1, Class="fs12 fwn b1", align="right")) if self.xls: nTitleRow += 1 worksheet.write([nTitleRow, 0], gIndex + 1) - + for ncol, column in enumerate(dispFields): if len(column) == 1 or column[1]== self.species: keyValue = "" @@ -196,17 +196,17 @@ class IntervalAnalystPage(templatePage): curGO = theGO[subGO] if theGO[subGO].has_key(fieldName): keyValue = theGO[subGO][fieldName] - + if self.xls: worksheet.write([nTitleRow, ncol+1], keyValue) geneRow.append(self.formatTD(keyValue, fieldName, curSpecies, curGO)) - + geneTable.append(geneRow) - + if self.xls: workbook.close() return geneTable, filename - + def formatTD(self, keyValue, fieldName, Species, theGO): if keyValue is None: keyValue = "" @@ -219,7 +219,7 @@ class IntervalAnalystPage(templatePage): keyValue = "" return HT.TD(keyValue, Class="fs12 fwn b1", width=300) elif fieldName in ("GeneSymbol"): - webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), + webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), HT.Image("/images/webqtl_search.gif", border=0, valign="top"), target="_blank") if theGO['GeneID']: geneSymbolLink = HT.Href(webqtlConfig.NCBI_LOCUSID % theGO['GeneID'], keyValue, Class="normalsize", target="_blank") @@ -236,8 +236,8 @@ class IntervalAnalystPage(templatePage): return HT.TD(keyValue, Class="fs12 fwn b1",align="right") elif fieldName in ("snpCount"): if keyValue: - snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'), - theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]), + snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'), + theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]), text=theGO["snpCount"], target="_blank", Class="normalsize") else: snpString = keyValue @@ -252,13 +252,13 @@ class IntervalAnalystPage(templatePage): return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1) else: return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1,align="right") - + def genControlForm(self, fd): ##desc GeneList self.cursor.execute("Desc GeneList") GeneListFields = self.cursor.fetchall() GeneListFields = map(lambda X: X[0], GeneListFields) - + #group columns by category--used for creating the dropdown list of possible columns categories = {} for item in self.columnNames.keys(): @@ -267,7 +267,7 @@ class IntervalAnalystPage(templatePage): categories[category[-1]] = [item ] else: categories[category[-1]] = categories[category[-1]]+[item] - + ##List All Species in the Gene Table speciesDict = {} self.cursor.execute("select Species.Name, GeneList.SpeciesId from Species, GeneList where \ @@ -292,34 +292,34 @@ class IntervalAnalystPage(templatePage): pass AppliedField.append(item2) categories[specName] = AppliedField - + categoriesOrder += ['misc'] - + ############################################################ ## Create the list of possible columns for the dropdown list ############################################################ allColumnsList = HT.Select(name="allColumns", Class="snpBrowserDropBox") - + for category in categoriesOrder: allFields = categories[category] if allFields: geneOpt = HT.Optgroup(label=category.title()) for item in allFields: if category in self.speciesFreeze.keys(): - geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]), + geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]), "%s__%s" % (item, self.speciesFreeze[category]))) else: geneOpt.append((self.columnNames[item][1], item)) geneOpt.sort() allColumnsList.append(geneOpt) - + ###################################### ## Create the list of selected columns ###################################### - + #cols contains the value of all the selected columns submitCols = cols = fd.formdata.getvalue("columns", "default") - + if cols == "default": if self.species=="mouse": #these are the same columns that are shown on intervalPage.py cols = ['GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'Strand', 'GeneLength', 'GeneID', 'NM_ID', 'snpCount', 'snpDensity'] @@ -331,12 +331,12 @@ class IntervalAnalystPage(templatePage): else: if type(cols)==type(""): cols = [cols] - + colsLst = [] dispFields = [] for column in cols: if submitCols == "default" and column not in ('GeneSymbol') and (column in GeneListFields or column in speciesField): - colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]), + colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]), "%s__%s" % (column, self.speciesFreeze[self.species]))) dispFields.append([column, self.species]) else: @@ -346,17 +346,17 @@ class IntervalAnalystPage(templatePage): dispFields.append([column]) else: thisSpecies = self.speciesFreeze[column2[1]] - colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]), + colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]), column)) dispFields.append((column2[0], thisSpecies)) selectedColumnsList = HT.Select(name="columns", Class="snpBrowserSelectBox", multiple="true", data=colsLst, size=6) - + ########################## ## Create the columns form - ########################## + ########################## columnsForm = HT.Form(name="columnsForm", submit=HT.Input(type='hidden'), cgi=os.path.join(webqtlConfig.CGIDIR, self._scriptfile), enctype="multipart/form-data") columnsForm.append(HT.Input(type="hidden", name="fromdatabase", value= fd.formdata.getvalue("fromdatabase", "unknown"))) - columnsForm.append(HT.Input(type="hidden", name="species", value=self.species)) + columnsForm.append(HT.Input(type="hidden", name="species", value=self.species)) if self.diffCol: columnsForm.append(HT.Input(type="hidden", name="s1", value=self.diffCol[0])) columnsForm.append(HT.Input(type="hidden", name="s2", value=self.diffCol[1])) @@ -366,8 +366,8 @@ class IntervalAnalystPage(templatePage): removeButton = HT.Input(type="button", name="remove", value="Remove", Class="button", onClick="removeFromList(this.form.columns.selectedIndex, this.form.columns)") upButton = HT.Input(type="button", name="up", value="Up", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex-1, this.form.columns)") downButton = HT.Input(type="button", name="down", value="Down", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex+1, this.form.columns)") - clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)") - submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)") + clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)") + submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)") selectChrBox = HT.Select(name="chromosome") self.cursor.execute(""" @@ -375,11 +375,11 @@ class IntervalAnalystPage(templatePage): Chr_Length.Name, Length from Chr_Length, Species where Chr_Length.SpeciesId = Species.Id AND - Species.Name = '%s' + Species.Name = '%s' Order by Chr_Length.OrderId """ % self.species) - + results = self.cursor.fetchall() for chrInfo in results: selectChrBox.append((chrInfo[0], chrInfo[0])) @@ -401,5 +401,5 @@ class IntervalAnalystPage(templatePage): #columnsForm.append(HT.Input(type="hidden", name="sort", value=diffCol), # HT.Input(type="hidden", name="identification", value=identification), # HT.Input(type="hidden", name="traitInfo", value=traitInfo)) - + return columnsForm, dispFields diff --git a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py index d02d80b3..455fcf95 100755 --- a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py +++ b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py @@ -140,7 +140,7 @@ class MarkerRegressionPage(templatePage): intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight)) gifmap = self.plotIntMappingForPLINK(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, plinkResultDict=plinkResultDict) - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png') intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap') TD_LR = HT.TR(HT.TD(HT.Blockquote(gifmap,intImg, HT.P()), bgColor='#eeeeee', height = 200)) @@ -249,7 +249,7 @@ class MarkerRegressionPage(templatePage): intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight)) gifmap = self.plotIntMapping(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= "") filename= webqtlUtil.genRandStr("Itvl_") - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png') intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap') ################################################################ @@ -458,7 +458,7 @@ class MarkerRegressionPage(templatePage): #plotBar(myCanvas,10,10,390,290,LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test',identification=fd.identification) Plot.plotBar(myCanvas, LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test') filename= webqtlUtil.genRandStr("Reg_") - myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif') + myCanvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test') if fd.suggestive == None: diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 36334317..265f9473 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -30,7 +30,7 @@ from flask import Flask, g from base.trait import GeneralTrait from base import data_set from base import species -from base import webqtlConfig +# from base import webqtlConfig from utility import webqtlUtil from utility import helper_functions from utility import Plot, Bunch diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 01303b0f..decde579 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -178,8 +178,8 @@ class MarkerRegression(object): self.species = start_vars['species'] #Needing for form submission when doing single chr mapping or remapping after changing options - self.vals = start_vars['vals'] - self.mapping_method = start_vars['mapping_method'] + self.vals = start_vars['vals'] + self.mapping_method = start_vars['mapping_method'] if self.mapping_method == "rqtl_geno": self.mapmethod_rqtl_geno = start_vars['method'] self.mapmodel_rqtl_geno = start_vars['model'] @@ -232,7 +232,7 @@ class MarkerRegression(object): self.significant = start_vars['significant'] else: self.nperm = 0 - + if 'bootCheck' in start_vars.keys(): self.bootChecked = start_vars['bootCheck'] else: @@ -308,7 +308,7 @@ class MarkerRegression(object): if 'showSNP' in start_vars.keys(): self.SNPChecked = start_vars['showSNP'] else: - self.SNPChecked = False + self.SNPChecked = False if 'showGenes' in start_vars.keys(): self.geneChecked = start_vars['showGenes'] else: @@ -321,7 +321,7 @@ class MarkerRegression(object): self.endMb = float(start_vars['endMb']) except: self.endMb = -1 - try: + try: self.lrsMax = float(start_vars['lrsMax']) except: self.lrsMax = 0 @@ -363,7 +363,7 @@ class MarkerRegression(object): self.ChrList.append((indChr.name, i)) - + self.ChrLengthMbList = g.db.execute(""" Select Length from Chr_Length, InbredSet @@ -517,7 +517,7 @@ class MarkerRegression(object): chrName = self.selectedChr # Draw the genes for this chromosome / region of this chromosome webqtldatabase = self.dataset.name - + if self.dataset.group.species == "mouse": self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, webqtldatabase, "mouse") elif self.dataset.group.species == "rat": @@ -533,11 +533,11 @@ class MarkerRegression(object): # # #GENEID = fd.formdata.getvalue('GeneId') or None # GENEID = None - # + # # geneTableContainer = HT.Div(Id="sortable") #Div to hold table # geneTable = self.geneTable(self.geneCol,GENEID) # geneTableContainer.append(geneTable) - # + # # mainfmName = webqtlUtil.genRandStr("fm_") # tableForm = HT.Form(cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name=mainfmName, submit=HT.Input(type='hidden')) # tableForm.append(HT.Input(name='FormID', value='', type='hidden')) @@ -552,22 +552,22 @@ class MarkerRegression(object): #else: showLocusForm = "" intCanvas = pid.PILCanvas(size=(self.graphWidth, self.graphHeight)) - gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) + gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) self.gifmap = gifmap.__str__() #print("GIFMAP:", gifmap.__str__()) self.filename= webqtlUtil.genRandStr("Itvl_") - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, self.filename), format='jpeg') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename), format='jpeg') intImg=HT.Image('/image/'+self.filename+'.png', border=0, usemap='#WebQTLImageMap') #Scales plot differently for high resolution if self.draw2X: intCanvasX2 = pid.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2)) gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2) - intCanvasX2.save(os.path.join(webqtlConfig.IMGDIR, self.filename+"X2"), format='png') + intCanvasX2.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename+"X2"), format='png') #DLintImgX2=HT.Href(text='Download',url = '/image/'+self.filename+'X2.png', Class='smallsize', target='_blank') - + #textUrl = self.writeQTL2Text(fd, self.filename) ################################################################ @@ -597,7 +597,7 @@ class MarkerRegression(object): showLocusForm.append(intImg) else: showLocusForm = intImg - + if self.permChecked and self.nperm > 0 and not self.multipleInterval and 0 < self.nperm: self.perm_filename = self.drawPermutationHistogram() #perm_text_file = self.permutationTextFile() @@ -667,7 +667,7 @@ class MarkerRegression(object): TD_LR.append(HT.Blockquote(tableForm)) self.body = TD_LR - + #self.dict['body'] = TD_LR #self.dict['title'] = "Mapping" @@ -858,7 +858,7 @@ class MarkerRegression(object): BootCoord = [] i = 0 startX = xLeftOffset - + if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes for j, _chr in enumerate(self.genotype): BootCoord.append( []) @@ -881,8 +881,8 @@ class MarkerRegression(object): else: Xc = startX + (_locus.cM-_chr[0].cM)*plotXScale BootCoord[-1].append([Xc, self.bootResult[i]]) - i += 1 - + i += 1 + #reduce bootResult if self.selectedChr > -1: maxBootBar = 80.0 @@ -1411,7 +1411,7 @@ class MarkerRegression(object): if _strains[ii] in self.dataset.group.samplelist: temp = GeneralObject(name=_strains[ii], value=_val) smd.append(temp) - + smd.sort(lambda A, B: cmp(A.value, B.value)) smd.reverse() @@ -1566,14 +1566,14 @@ class MarkerRegression(object): firstGene = 0 else: lastGene = 0 - + for j, _geno in enumerate (self.genotype[0][1].genotype): plotbxd=0 for item in smd: if item.name == samplelist[j]: - plotbxd=1 - + plotbxd=1 + if (plotbxd == 1): ind = 0 @@ -1620,28 +1620,28 @@ class MarkerRegression(object): currentChromosome = self.genotype[0].name i = 0 - + paddingTop = yTopOffset ucscPaddingTop = paddingTop + self.WEBQTL_BAND_HEIGHT + self.BAND_SPACING ensemblPaddingTop = ucscPaddingTop + self.UCSC_BAND_HEIGHT + self.BAND_SPACING - + if zoom == 1: for pixel in range(xLeftOffset, xLeftOffset + plotWidth, pixelStep): - + calBase = self.kONE_MILLION*(startMb + (endMb-startMb)*(pixel-xLeftOffset-0.0)/plotWidth) - + xBrowse1 = pixel xBrowse2 = min(xLeftOffset + plotWidth, (pixel + pixelStep - 1)) - + WEBQTL_COORDS = "%d, %d, %d, %d" % (xBrowse1, paddingTop, xBrowse2, (paddingTop+self.WEBQTL_BAND_HEIGHT)) bandWidth = xBrowse2 - xBrowse1 WEBQTL_HREF = "javascript:rangeView('%s', %f, %f)" % (self.selectedChr - 1, max(0, (calBase-webqtlZoomWidth))/1000000.0, (calBase+webqtlZoomWidth)/1000000.0) - + WEBQTL_TITLE = "Click to view this section of the genome in WebQTL" gifmap.areas.append(HT.Area(shape='rect',coords=WEBQTL_COORDS,href=WEBQTL_HREF, title=WEBQTL_TITLE)) canvas.drawRect(xBrowse1, paddingTop, xBrowse2, (paddingTop + self.WEBQTL_BAND_HEIGHT), edgeColor=self.CLICKABLE_WEBQTL_REGION_COLOR, fillColor=self.CLICKABLE_WEBQTL_REGION_COLOR) canvas.drawLine(xBrowse1, paddingTop, xBrowse1, (paddingTop + self.WEBQTL_BAND_HEIGHT), color=self.CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR) - + UCSC_COORDS = "%d, %d, %d, %d" %(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT)) if self.species == "mouse": UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=%s/snp/chr%s" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases, webqtlConfig.PORTADDR, self.selectedChr) @@ -1651,7 +1651,7 @@ class MarkerRegression(object): gifmap.areas.append(HT.Area(shape='rect',coords=UCSC_COORDS,href=UCSC_HREF, title=UCSC_TITLE)) canvas.drawRect(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), edgeColor=self.CLICKABLE_UCSC_REGION_COLOR, fillColor=self.CLICKABLE_UCSC_REGION_COLOR) canvas.drawLine(xBrowse1, ucscPaddingTop, xBrowse1, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), color=self.CLICKABLE_UCSC_REGION_OUTLINE_COLOR) - + ENSEMBL_COORDS = "%d, %d, %d, %d" %(xBrowse1, ensemblPaddingTop, xBrowse2, (ensemblPaddingTop+self.ENSEMBL_BAND_HEIGHT)) if self.species == "mouse": ENSEMBL_HREF = "http://www.ensembl.org/Mus_musculus/contigview?highlight=&chr=%s&vc_start=%d&vc_end=%d&x=35&y=12" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) @@ -1766,7 +1766,7 @@ class MarkerRegression(object): ChrAInfo = [] preLpos = -1 distinctCount = 0.0 - + #if len(self.genotype) > 1: if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes for i, _chr in enumerate(self.genotype): @@ -1809,7 +1809,7 @@ class MarkerRegression(object): offsetA = -stepA lineColor = pid.lightblue startPosX = xLeftOffset - + for j, ChrInfo in enumerate(ChrAInfo): preLpos = -1 for i, item in enumerate(ChrInfo): @@ -1877,7 +1877,7 @@ class MarkerRegression(object): # lodm = self.LODFACTOR # else: # lodm = 1.0 - + #ZS: This is a mess, but I don't know a better way to account for different mapping methods returning results in different formats + the option to change between LRS and LOD if self.lrsMax <= 0: #sliding scale if "lrs_value" in self.qtlresults[0]: @@ -1890,7 +1890,7 @@ class MarkerRegression(object): else: if self.permChecked and self.nperm > 0 and not self.multipleInterval: self.significant = min(self.significant, webqtlConfig.MAXLRS) - self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) + self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) else: pass else: @@ -1903,10 +1903,10 @@ class MarkerRegression(object): else: if self.permChecked and self.nperm > 0 and not self.multipleInterval: self.significant = min(self.significant, webqtlConfig.MAXLRS) - self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) + self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) else: pass - + if self.permChecked and self.nperm > 0 and not self.multipleInterval: LRS_LOD_Max = max(self.significant, LRS_LOD_Max) @@ -1923,7 +1923,7 @@ class MarkerRegression(object): LRSScale = 2.5 else: LRSScale = 1.0 - + LRSAxisList = Plot.frange(LRSScale, LRS_LOD_Max, LRSScale) #make sure the user's value appears on the y-axis #update by NL 6-21-2011: round the LOD value to 100 when LRS_LOD_Max is equal to 460 @@ -1953,7 +1953,7 @@ class MarkerRegression(object): #"Significant" and "Suggestive" Drawing Routine # ======= Draw the thick lines for "Significant" and "Suggestive" ===== (crowell: I tried to make the SNPs draw over these lines, but piddle wouldn't have it...) - + #ZS: I don't know if what I did here with this inner function is clever or overly complicated, but it's the only way I could think of to avoid duplicating the code inside this function def add_suggestive_significant_lines_and_legend(start_pos_x, chr_length_dist): rightEdge = int(start_pos_x + chr_length_dist*plotXScale - self.SUGGESTIVE_WIDTH/1.5) @@ -1976,13 +1976,13 @@ class MarkerRegression(object): start_pos_x += (chr_length_dist+self.GraphInterval)*plotXScale return start_pos_x - + for i, _chr in enumerate(self.genotype): if self.selectedChr != -1: if _chr.name == self.ChrList[self.selectedChr][0]: startPosX = add_suggestive_significant_lines_and_legend(startPosX, self.ChrLengthDistList[0]) break - else: + else: continue else: startPosX = add_suggestive_significant_lines_and_legend(startPosX, self.ChrLengthDistList[i]) @@ -1997,7 +1997,7 @@ class MarkerRegression(object): #else: # dominanceMax = -1 lrsEdgeWidth = 2 - + if zoom == 2: lrsEdgeWidth = 2 * lrsEdgeWidth @@ -2018,7 +2018,7 @@ class MarkerRegression(object): if qtlresult['chr'] != previous_chr and self.selectedChr == -1: if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and self.additiveChecked: plusColor = self.ADDITIVE_COLOR_POSITIVE minusColor = self.ADDITIVE_COLOR_NEGATIVE @@ -2048,7 +2048,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + LRSCoordXY = [] AdditiveCoordXY = [] previous_chr = qtlresult['chr'] @@ -2061,7 +2061,7 @@ class MarkerRegression(object): #startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale - #for j, _chr in enumerate(self.genotype): + #for j, _chr in enumerate(self.genotype): #ZS: This is beause the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used if self.plotScale == "physic": this_chr = str(self.ChrList[self.selectedChr][0]) @@ -2126,7 +2126,7 @@ class MarkerRegression(object): if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and self.additiveChecked: plusColor = self.ADDITIVE_COLOR_POSITIVE minusColor = self.ADDITIVE_COLOR_NEGATIVE @@ -2156,7 +2156,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and INTERCROSS and self.dominanceChecked: plusColor = self.DOMINANCE_COLOR_POSITIVE minusColor = self.DOMINANCE_COLOR_NEGATIVE @@ -2186,7 +2186,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + ###draw additive scale if not self.multipleInterval and self.additiveChecked: @@ -2222,7 +2222,7 @@ class MarkerRegression(object): if zoom == 2: fontZoom = 1.5 yTopOffset += 30 - + #calculate plot scale if self.plotScale != 'physic': self.ChrLengthDistList = self.ChrLengthCMList @@ -2589,13 +2589,13 @@ class MarkerRegression(object): perm_output = [value/4.16 for value in self.perm_output] else: perm_output = self.perm_output - + Plot.plotBar(myCanvas, perm_output, XLabel=self.LRS_LOD, YLabel='Frequency', title=' Histogram of Permutation Test') filename= webqtlUtil.genRandStr("Reg_") myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif') - + return filename - + # img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test') # self.suggestive = self.perm_output[int(self.nperm*0.37-1)] @@ -2609,9 +2609,9 @@ class MarkerRegression(object): # permutation.append(HT.TR(HT.TD(img)), # HT.TR(HT.TD('')), # HT.TR(HT.TD('Total of %d permutations'%self.nperm))) - + # return permutation - + def permutationTextFile(self): filename= webqtlUtil.genRandStr("Reg_") fpText = open('%s.txt' % (webqtlConfig.TMPDIR+filename), 'wb') @@ -2624,12 +2624,12 @@ class MarkerRegression(object): '    Significant LRS =%3.2f\n'%self.significant, HT.BR(), '    Highly Significant LRS =%3.2f\n' % self.highlysignificant) - + for lrs_value in self.perm_output: fpText.write(str(lrs_value) + "\n") - + textUrl = HT.Href(text = 'Download Permutation Results', url= '/tmp/'+filename+'.txt', target = "_blank", Class='fs12 fwn') - + return textUrl def geneTable(self, geneCol, refGene=None): @@ -2654,7 +2654,7 @@ class MarkerRegression(object): else: gene_table = "" - return gene_table + return gene_table def getLiteratureCorrelation(cursor,geneId1=None,geneId2=None): if not geneId1 or not geneId2: @@ -2954,4 +2954,4 @@ And by voluntary, according to HFG when I talked to him, they have a choice betw sortby = ("", "") - return sortby \ No newline at end of file + return sortby diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 52fe2e34..9941a4d3 100755 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -26,7 +26,6 @@ from MySQLdb import escape_string as escape # Instead of importing HT we're going to build a class below until we can eliminate it # from htmlgen import HTMLgen2 as HT -from base import webqtlConfig from utility.benchmark import Bench from base.data_set import create_dataset from base.trait import GeneralTrait diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 35f7fe5f..e80cf191 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -166,7 +166,7 @@ class ShowTrait(object): return False def check_pylmm_rqtl(): - if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.NEWGENODIR+self.dataset.group.name+".json") > 0): + if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.JSON_GENODIR+self.dataset.group.name+".json") > 0): return True else: return False -- cgit v1.2.3 From 2fd8e09c3de9ee539e47bdb416a4c0df89ff15ab Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 1 Mar 2016 08:55:10 +0000 Subject: setup: prepare for Guix runtime --- MANIFEST.in | 5 +---- bin/genenetwork2 | 18 +++++++++++++++--- setup.py | 15 ++++++++++++--- 3 files changed, 28 insertions(+), 10 deletions(-) (limited to 'bin') diff --git a/MANIFEST.in b/MANIFEST.in index f4ea2316..bf23f9aa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,10 +17,6 @@ include wqflask/wqflask/templates/new_security/* include wqflask/wqflask/correlation/* include wqflask/wqflask/interval_mapping/* include wqflask/wqflask/interval_analyst/* -include wqflask/wqflask/my_pylmm/* -include wqflask/wqflask/my_pylmm/scripts/* -include wqflask/wqflask/my_pylmm/pyLMM/* -include wqflask/wqflask/my_pylmm/data/* include wqflask/wqflask/static/* include wqflask/wqflask/static/dbdoc/* include wqflask/wqflask/static/new/* @@ -260,3 +256,4 @@ include wqflask/flask_security/templates/security/email/* include wqflask/dbFunction/* include wqflask/basicStatistics/* include wqflask/base/* +include etc/*.py diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 00ee09bf..e5f04ee1 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,6 +1,9 @@ #! /bin/bash # -# This will run the server with default settings. +# This will run the GN2 server (with default settings if none supplied). +# +# Environment settings can be used to preconfigure as well as a +# settings.py file. # Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") @@ -8,20 +11,29 @@ SCRIPT=$(readlink -f "$0") GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) echo $GN2_BASE_PATH +# Handle setting parameter settings=$1 if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi -export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH +if [ ! -e $settings ]; then + echo "ERROR: can not load settings file $settings" + exit 1 +fi export WQFLASK_SETTINGS=$settings +# We may change this one: +export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH + +# TEMPDIR defaults to /tmp if nothing else if [ -z $TEMPDIR ]; then TEMPDIR="/tmp" fi -# start the redis server +# Start the redis server echo -n "dir $TEMPDIR dbfilename gn2.rdb " | redis-server - & +# Start the flask server running GN2 cd $GN2_BASE_PATH/wqflask echo "Starting with $settings" /usr/bin/env python runserver.py diff --git a/setup.py b/setup.py index 1b11acc0..a9b71fab 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,17 @@ +# Run setup from python - this script is used by the GNU Guix builder. + from setuptools import setup, find_packages setup(name='genenetwork2', version='2.0', + author = "The GeneNetwork Team", + author_email='rwilliams@uthsc.edu', + url = "https://github.com/genenetwork/genenetwork2/blob/master/README.md", + description = 'Website and tools for genetics.', include_package_data=True, - # packages=find_packages() - packages=['wqflask'] - ) + packages=['wqflask','etc'], + scripts=['bin/genenetwork2'], + # package_data = { + # 'etc': ['*.py'] + # } +) -- cgit v1.2.3 From 8b2598efcf276ef6a1edd06921a9cc28dbfcad14 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 1 Mar 2016 09:13:30 +0000 Subject: Handle settings in guix store --- bin/genenetwork2 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index e5f04ee1..682d8b3d 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -11,12 +11,15 @@ SCRIPT=$(readlink -f "$0") GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) echo $GN2_BASE_PATH -# Handle setting parameter +# Handle settings parameter settings=$1 if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi if [ ! -e $settings ]; then - echo "ERROR: can not load settings file $settings" - exit 1 + settings=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py + if [ ! -e $settings ]; then + echo "ERROR: can not locate settings file - pass it in the command line" + exit 1 + fi fi export WQFLASK_SETTINGS=$settings -- cgit v1.2.3 From c52bf79ba92d2a632637536a9ea607c22ef2d9bf Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 1 Mar 2016 09:29:24 +0000 Subject: GN2_BASE_PATH --- bin/genenetwork2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 682d8b3d..a0a013fc 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -15,7 +15,8 @@ echo $GN2_BASE_PATH settings=$1 if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi if [ ! -e $settings ]; then - settings=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py + GN2_BASE_PATH=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg + settings=$GN2_BASE_PATH/etc/default_settings.py if [ ! -e $settings ]; then echo "ERROR: can not locate settings file - pass it in the command line" exit 1 -- cgit v1.2.3 From a66858e99c2195d90a187899db6f4dd8966a0a2c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 3 Mar 2016 11:21:01 +0000 Subject: Show error when GENODIR is missing --- bin/genenetwork2 | 13 +++++++------ etc/default_settings.py | 2 +- wqflask/utility/tools.py | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index a0a013fc..bbb2a19f 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -9,18 +9,19 @@ SCRIPT=$(readlink -f "$0") # Absolute path this script is in, thus /home/user/bin GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) + +GN2_GUIX_PATH=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg +if [ -d $GN2_GUIX_PATH ]; then + GN2_BASE_PATH=$GN2_GUIX_PATH +fi echo $GN2_BASE_PATH # Handle settings parameter settings=$1 if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi if [ ! -e $settings ]; then - GN2_BASE_PATH=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg - settings=$GN2_BASE_PATH/etc/default_settings.py - if [ ! -e $settings ]; then - echo "ERROR: can not locate settings file - pass it in the command line" - exit 1 - fi + echo "ERROR: can not locate settings file - pass it in the command line" + exit 1 fi export WQFLASK_SETTINGS=$settings diff --git a/etc/default_settings.py b/etc/default_settings.py index 60a3a7b4..0cf40265 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -23,7 +23,7 @@ SERVER_PORT = 5003 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' # Path overrides for Genenetwork -GENENETWORK_FILES = "../../gn2_data" +GENENETWORK_FILES = os.environ['HOME']+"/gn2_data" PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) PLINK_COMMAND = str.strip(os.popen("which plink2").read()) GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index b8eff12a..9405a9c6 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -76,8 +76,8 @@ def plink_command(guess=None): def flat_files(subdir=None): base = get_setting("GENENETWORK_FILES") if subdir: - return valid_path(base+"/"+subdir) - return valid_path(base) + return assert_dir(base+"/"+subdir) + return assert_dir(base) def assert_dir(dir): if not valid_path(dir): -- cgit v1.2.3 From ebf1db54525a6f344725717f83dfeb1f7b1ada18 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 26 Apr 2016 16:23:38 +0000 Subject: Test: main page and first search --- bin/test-website | 37 +++++++++++++++++++++++++++++++++++++ test/lib/main_web_functionality.rb | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100755 bin/test-website create mode 100644 test/lib/main_web_functionality.rb (limited to 'bin') diff --git a/bin/test-website b/bin/test-website new file mode 100755 index 00000000..b766edd8 --- /dev/null +++ b/bin/test-website @@ -0,0 +1,37 @@ +#!/usr/bin/env ruby + + +USAGE = <0 + ARGV.shift + else + "http://localhost:5003" + end + +$stderr.print "Testing <",$host,">\n" + +require 'mechanize' +require 'minitest/spec' +require 'minitest/autorun' +require 'main_web_functionality' + + diff --git a/test/lib/main_web_functionality.rb b/test/lib/main_web_functionality.rb new file mode 100644 index 00000000..03c76d42 --- /dev/null +++ b/test/lib/main_web_functionality.rb @@ -0,0 +1,32 @@ +class MainWebFunctionality +end + +describe MainWebFunctionality do + before do + @agent = Mechanize.new + page = @agent.get($host) + # p page + @form = page.forms[1] + + end + + describe MainWebFunctionality do + it "The main page contains a search button" do + @form.buttons[0].value.must_equal "Search" + end + + # http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult + it "Search for MEAN=(15 16) LRS=(23 46)" do + @form.fields[2].value = "MEAN=(15 16) LRS=(23 46)" + @form.fields[3].value = "mouse" + @form.fields[4].value = "BXD" + @form.fields[5].value = "Hippocampus mRNA" + @form.fields[6].value = "HC_M2_0606_P" + @searchpage = @agent.submit(@form, @form.buttons.first) + probe_link = @searchpage.links.find { |l| l.text =~ /1435395_s_at/ } + probe_link.uri.to_s.must_equal "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" + end + end + +end + -- cgit v1.2.3 From c9c0b876318ae0c192a2895c59baeb394a77295e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 27 Apr 2016 09:40:24 +0000 Subject: Test: get to trait page --- bin/test-website | 3 +++ test/lib/main_web_functionality.rb | 36 +++++++++++++++++++----------------- 2 files changed, 22 insertions(+), 17 deletions(-) (limited to 'bin') diff --git a/bin/test-website b/bin/test-website index b766edd8..8efa37b1 100755 --- a/bin/test-website +++ b/bin/test-website @@ -32,6 +32,9 @@ $stderr.print "Testing <",$host,">\n" require 'mechanize' require 'minitest/spec' require 'minitest/autorun' + +# These are the actual testing modules + require 'main_web_functionality' diff --git a/test/lib/main_web_functionality.rb b/test/lib/main_web_functionality.rb index 03c76d42..4103d653 100644 --- a/test/lib/main_web_functionality.rb +++ b/test/lib/main_web_functionality.rb @@ -1,30 +1,32 @@ +# In these tests we navigate from the main page to search + class MainWebFunctionality end describe MainWebFunctionality do before do @agent = Mechanize.new - page = @agent.get($host) - # p page - @form = page.forms[1] - end describe MainWebFunctionality do - it "The main page contains a search button" do - @form.buttons[0].value.must_equal "Search" - end - - # http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult - it "Search for MEAN=(15 16) LRS=(23 46)" do - @form.fields[2].value = "MEAN=(15 16) LRS=(23 46)" - @form.fields[3].value = "mouse" - @form.fields[4].value = "BXD" - @form.fields[5].value = "Hippocampus mRNA" - @form.fields[6].value = "HC_M2_0606_P" - @searchpage = @agent.submit(@form, @form.buttons.first) - probe_link = @searchpage.links.find { |l| l.text =~ /1435395_s_at/ } + it "Get to trait page" do + page = @agent.get($host) + form = page.forms[1] + form.buttons[0].value.must_equal "Search" + # http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult + form.fields[2].value = "MEAN=(15 16) LRS=(23 46)" + form.fields[3].value = "mouse" + form.fields[4].value = "BXD" + form.fields[5].value = "Hippocampus mRNA" + form.fields[6].value = "HC_M2_0606_P" + search_page = @agent.submit(form, form.buttons.first) + probe_link = search_page.links.find { |l| l.text =~ /1435395_s_at/ } probe_link.uri.to_s.must_equal "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" + show_trait_page = probe_link.click + # Get to trait page for 1435395_s_at + form2 = show_trait_page.forms_with("trait_page")[0] + form2.fields[30].name.must_equal "value:DBA/2J" + form2.fields[30].value.must_equal "15.287" end end -- cgit v1.2.3 From a46be3cb631b3e1c7f76593a65a0ae1692c72fda Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 28 Apr 2016 08:55:45 +0000 Subject: Test: splitting out static URL tests --- bin/test-website | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) (limited to 'bin') diff --git a/bin/test-website b/bin/test-website index 8efa37b1..24274713 100755 --- a/bin/test-website +++ b/bin/test-website @@ -14,11 +14,30 @@ the root of the genenetwork2 source tree as, for example, For more information see http://genenetwork.org/ EOT +$stderr.print USAGE -libpath = File.dirname(File.dirname(__FILE__)) -$: << File.join(libpath,'test/lib') +require 'optparse' + +options = {} +opts = OptionParser.new do |o| + o.banner = "Usage: #{File.basename($0)} [options] URL" -$stderr.print USAGE + o.on('-l','--link-checker', 'Check for dead links') do + options[:link_checker] = true + end + + o.separator "" + o.on_tail('-h', '--help', 'display this help and exit') do + options[:show_help] = true + end +end + +opts.parse!(ARGV) + +if options[:show_help] + print opts + exit 1 +end $host = if ARGV.size>0 @@ -35,6 +54,13 @@ require 'minitest/autorun' # These are the actual testing modules -require 'main_web_functionality' +libpath = File.dirname(File.dirname(__FILE__)) +$: << File.join(libpath,'test/lib') +if options[:link_checker] + require 'link_checker' +else + require 'main_web_functionality' +end + -- cgit v1.2.3 From 25cea57dbbf8bf37108481dc83c2d536890b1a47 Mon Sep 17 00:00:00 2001 From: Dennis E. Mungai Date: Wed, 18 May 2016 02:24:10 +0300 Subject: Added a new Navigation Test suite A new option for running the script is now available. See the output of ./test-website -n for usage.--- bin/test-website | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'bin') diff --git a/bin/test-website b/bin/test-website index 24274713..aeab459b 100755 --- a/bin/test-website +++ b/bin/test-website @@ -25,6 +25,10 @@ opts = OptionParser.new do |o| o.on('-l','--link-checker', 'Check for dead links') do options[:link_checker] = true end + + o.on('-n','--navigation-test', 'Check for navigation test results') do + options[:navigation_test] = true + end o.separator "" o.on_tail('-h', '--help', 'display this help and exit') do @@ -63,4 +67,7 @@ else require 'main_web_functionality' end +if options[:navigation_test] + require 'NavigationTest' +end -- cgit v1.2.3