From c7e661b8ff9f70955418fbc4527378904beb0cf4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 30 Apr 2021 12:16:51 +0300 Subject: autopep8: Fix E20-E27 Run: python -m autopep8 --in-place --recrusive ./ --select\ E20,E211,E22,E224,E224,E225,E226,E227,E228,E231,E241,\ E242,E251,E252,E26,E265,E266,E27 -p 3 --- wqflask/maintenance/convert_dryad_to_bimbam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'wqflask/maintenance/convert_dryad_to_bimbam.py') diff --git a/wqflask/maintenance/convert_dryad_to_bimbam.py b/wqflask/maintenance/convert_dryad_to_bimbam.py index 12ce35e9..8eab66e8 100644 --- a/wqflask/maintenance/convert_dryad_to_bimbam.py +++ b/wqflask/maintenance/convert_dryad_to_bimbam.py @@ -41,7 +41,7 @@ def read_dryad_file(filename): return geno_rows - #for i, marker in enumerate(marker_list): + # for i, marker in enumerate(marker_list): # this_row = [] # this_row.append(marker) # this_row.append("X") @@ -53,7 +53,7 @@ def read_dryad_file(filename): # print("row: " + str(i)) # geno_rows.append(this_row) # - #return geno_rows + # return geno_rows def write_bimbam_files(geno_rows): with open('/home/zas1024/cfw_data/CFW_geno.txt', 'w') as geno_fh: @@ -64,6 +64,6 @@ def convert_dryad_to_bimbam(filename): geno_file_rows = read_dryad_file(filename) write_bimbam_files(geno_file_rows) -if __name__=="__main__": +if __name__ == "__main__": input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" convert_dryad_to_bimbam(input_filename) -- cgit v1.2.3 From 406eb27859cca232a562c722cbbd37aca2e3be84 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 30 Apr 2021 12:26:19 +0300 Subject: autopep8: Fix E301,E302,E303,E304,E305,E306 --- etc/default_settings.py | 21 +++---- wqflask/base/data_set.py | 1 - wqflask/base/mrna_assay_tissue_data.py | 1 + wqflask/base/species.py | 3 + wqflask/base/webqtlCaseData.py | 3 +- wqflask/db/call.py | 4 ++ wqflask/db/gn_server.py | 1 + wqflask/db/webqtlDatabaseFunction.py | 1 + wqflask/maintenance/convert_dryad_to_bimbam.py | 3 + wqflask/maintenance/convert_geno_to_bimbam.py | 4 ++ wqflask/maintenance/gen_select_dataset.py | 5 ++ .../maintenance/generate_kinship_from_bimbam.py | 1 + .../maintenance/generate_probesetfreeze_file.py | 7 +++ wqflask/maintenance/geno_to_json.py | 6 +- wqflask/maintenance/get_group_samplelists.py | 3 + wqflask/maintenance/print_benchmark.py | 4 ++ wqflask/maintenance/quantile_normalize.py | 6 ++ wqflask/maintenance/set_resource_defaults.py | 7 +++ wqflask/run_gunicorn.py | 2 + wqflask/tests/unit/base/test_webqtl_case_data.py | 1 + .../unit/utility/test_authentication_tools.py | 1 + wqflask/tests/unit/utility/test_chunks.py | 1 + .../test_display_mapping_results.py | 2 + .../marker_regression/test_plink_mapping.py | 3 +- .../marker_regression/test_qtlreaper_mapping.py | 2 + .../wqflask/marker_regression/test_rqtl_mapping.py | 2 +- wqflask/tests/unit/wqflask/test_collect.py | 2 + .../tests/wqflask/show_trait/test_show_trait.py | 1 - wqflask/utility/Plot.py | 13 +++++ wqflask/utility/__init__.py | 2 + wqflask/utility/after.py | 1 + wqflask/utility/authentication_tools.py | 1 + wqflask/utility/benchmark.py | 1 + wqflask/utility/corestats.py | 2 + wqflask/utility/elasticsearch_tools.py | 7 +++ wqflask/utility/external.py | 1 + wqflask/utility/gen_geno_ob.py | 3 + wqflask/utility/genofile_parser.py | 1 + wqflask/utility/logger.py | 3 + wqflask/utility/pillow_utils.py | 4 ++ wqflask/utility/startup_config.py | 1 + wqflask/utility/svg.py | 67 +++++++++++++++++++++- wqflask/utility/temp_data.py | 1 + wqflask/utility/tools.py | 25 ++++++++ wqflask/utility/type_checking.py | 5 ++ wqflask/utility/webqtlUtil.py | 5 ++ wqflask/wqflask/__init__.py | 1 + wqflask/wqflask/api/correlation.py | 7 +++ wqflask/wqflask/api/mapping.py | 2 +- wqflask/wqflask/api/router.py | 20 +++++++ wqflask/wqflask/collect.py | 8 +++ wqflask/wqflask/correlation/corr_scatter_plot.py | 1 + wqflask/wqflask/correlation/show_corr_results.py | 4 +- .../wqflask/correlation_matrix/show_corr_matrix.py | 3 + wqflask/wqflask/ctl/ctl_analysis.py | 1 + wqflask/wqflask/database.py | 2 + wqflask/wqflask/db_info.py | 1 + wqflask/wqflask/do_search.py | 21 ++++++- wqflask/wqflask/docs.py | 2 +- wqflask/wqflask/export_traits.py | 2 + wqflask/wqflask/external_tools/send_to_bnw.py | 1 + .../wqflask/external_tools/send_to_geneweaver.py | 3 + .../wqflask/external_tools/send_to_webgestalt.py | 3 + wqflask/wqflask/group_manager.py | 9 +++ wqflask/wqflask/gsearch.py | 1 + wqflask/wqflask/heatmap/heatmap.py | 3 + wqflask/wqflask/interval_analyst/GeneUtil.py | 2 + .../marker_regression/display_mapping_results.py | 9 --- wqflask/wqflask/marker_regression/plink_mapping.py | 9 ++- .../wqflask/marker_regression/qtlreaper_mapping.py | 5 ++ wqflask/wqflask/marker_regression/rqtl_mapping.py | 15 +++++ wqflask/wqflask/marker_regression/run_mapping.py | 9 +++ wqflask/wqflask/model.py | 7 ++- wqflask/wqflask/network_graph/network_graph.py | 1 + wqflask/wqflask/news.py | 1 + wqflask/wqflask/parser.py | 1 + wqflask/wqflask/pbkdf2.py | 2 + wqflask/wqflask/resource_manager.py | 7 +++ wqflask/wqflask/search_results.py | 4 ++ wqflask/wqflask/send_mail.py | 3 +- wqflask/wqflask/server_side.py | 1 - wqflask/wqflask/show_trait/SampleList.py | 1 + wqflask/wqflask/show_trait/export_trait_data.py | 3 + wqflask/wqflask/snp_browser/snp_browser.py | 9 ++- wqflask/wqflask/submit_bnw.py | 1 + wqflask/wqflask/update_search_results.py | 1 + wqflask/wqflask/user_login.py | 21 +++++++ wqflask/wqflask/user_manager.py | 38 +++++++++++- wqflask/wqflask/user_session.py | 6 ++ 89 files changed, 446 insertions(+), 46 deletions(-) (limited to 'wqflask/maintenance/convert_dryad_to_bimbam.py') diff --git a/etc/default_settings.py b/etc/default_settings.py index 27522187..a194b10e 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -73,23 +73,24 @@ SMTP_PASSWORD = "UNKNOWN" # ---- Behavioural settings (defaults) note that logger and log levels can # be overridden at the module level and with enviroment settings -WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) +WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) WEBSERVER_BRANDING = None # Set the branding (nyi) WEBSERVER_DEPLOY = None # Deployment specifics (nyi) -WEBSERVER_URL = "http://localhost:"+str(SERVER_PORT)+"/" # external URL +WEBSERVER_URL = "http://localhost:" + str(SERVER_PORT) + "/" # external URL -LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) +LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) LOG_LEVEL_DEBUG = '0' # logger.debugf log level (0-5, 5 = show all) -LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls +LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls LOG_SQL_ALCHEMY = 'False' -LOG_BENCH = True # Log bench marks +LOG_BENCH = True # Log bench marks -USE_REDIS = True # REDIS caching (note that redis will be phased out) -USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls -HOME = os.environ['HOME'] +USE_REDIS = True # REDIS caching (note that redis will be phased out) +USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls +HOME = os.environ['HOME'] # ---- Default locations -GENENETWORK_FILES = HOME+"/genotype_files" # base dir for all static data files +# base dir for all static data files +GENENETWORK_FILES = HOME + "/genotype_files" # ---- Path overrides for Genenetwork - the defaults are normally # picked up from Guix or in the HOME directory @@ -98,7 +99,7 @@ GENENETWORK_FILES = HOME+"/genotype_files" # base dir for all static data fil # PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) # ---- Local path to JS libraries - for development modules (only) -JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript" +JS_GN_PATH = os.environ['HOME'] + "/genenetwork/javascript" # ---- GN2 Executables (overwrite for testing only) # PLINK_COMMAND = str.strip(os.popen("which plink2").read()) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 55ab45f5..c14808d6 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -311,7 +311,6 @@ class HumanMarkers(Markers): marker['Mb'] = float(splat[3]) / 1000000 self.markers.append(marker) - def add_pvalues(self, p_values): super(HumanMarkers, self).add_pvalues(p_values) diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index f3264b3d..cbc05738 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -11,6 +11,7 @@ from utility.db_tools import escape from utility.logger import getLogger logger = getLogger(__name__) + class MrnaAssayTissueData: def __init__(self, gene_symbols=None): diff --git a/wqflask/base/species.py b/wqflask/base/species.py index cf764d72..44f133b5 100644 --- a/wqflask/base/species.py +++ b/wqflask/base/species.py @@ -6,6 +6,7 @@ from flask import Flask, g from utility.logger import getLogger logger = getLogger(__name__) + class TheSpecies: def __init__(self, dataset=None, species_name=None): if species_name != None: @@ -15,6 +16,7 @@ class TheSpecies: self.dataset = dataset self.chromosomes = Chromosomes(dataset=self.dataset) + class IndChromosome: def __init__(self, name, length): self.name = name @@ -25,6 +27,7 @@ class IndChromosome: """Chromosome length in megabases""" return self.length / 1000000 + class Chromosomes: def __init__(self, dataset=None, species=None): self.chromosomes = collections.OrderedDict() diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py index aa55470f..2d07ab9d 100644 --- a/wqflask/base/webqtlCaseData.py +++ b/wqflask/base/webqtlCaseData.py @@ -28,6 +28,7 @@ import utility.tools utility.tools.show_settings() + class webqtlCaseData: """one case data in one trait""" @@ -78,4 +79,4 @@ class webqtlCaseData: def display_num_cases(self): if self.num_cases is not None: return "%s" % self.num_cases - return "x" \ No newline at end of file + return "x" diff --git a/wqflask/db/call.py b/wqflask/db/call.py index 555878ad..9412b376 100644 --- a/wqflask/db/call.py +++ b/wqflask/db/call.py @@ -16,6 +16,7 @@ logger = getLogger(__name__) # from inspect import stack + def fetch1(query, path=None, func=None): """Fetch one result as a Tuple using either a SQL query or the URI path to GN_SERVER (when USE_GN_SERVER is True). Apply func to @@ -35,6 +36,7 @@ GN_SERVER result when set (which should return a Tuple) else: return fetchone(query) + def fetchone(query): """Return tuple containing one row by calling SQL directly (the original fetchone, but with logging) @@ -46,6 +48,7 @@ original fetchone, but with logging) return res.fetchone() return logger.sql(query, helper) + def fetchall(query): """Return row iterator by calling SQL directly (the original fetchall, but with logging) @@ -57,6 +60,7 @@ original fetchall, but with logging) return res.fetchall() return logger.sql(query, helper) + def gn_server(path): """Return JSON record by calling GN_SERVER diff --git a/wqflask/db/gn_server.py b/wqflask/db/gn_server.py index 6c7383d0..f9b01658 100644 --- a/wqflask/db/gn_server.py +++ b/wqflask/db/gn_server.py @@ -5,5 +5,6 @@ from db.call import gn_server from utility.logger import getLogger logger = getLogger(__name__) + def menu_main(): return gn_server("/int/menu/main.json") diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index 18ade405..50ac06fd 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -31,6 +31,7 @@ logger = getLogger(__name__) # function: connect to database and return cursor instance ########################################################################### + def retrieve_species(group): """Get the species of a group (e.g. returns string "mouse" on "BXD" diff --git a/wqflask/maintenance/convert_dryad_to_bimbam.py b/wqflask/maintenance/convert_dryad_to_bimbam.py index 8eab66e8..e417c280 100644 --- a/wqflask/maintenance/convert_dryad_to_bimbam.py +++ b/wqflask/maintenance/convert_dryad_to_bimbam.py @@ -55,15 +55,18 @@ def read_dryad_file(filename): # # return geno_rows + def write_bimbam_files(geno_rows): with open('/home/zas1024/cfw_data/CFW_geno.txt', 'w') as geno_fh: for row in geno_rows: geno_fh.write(", ".join(row) + "\n") + def convert_dryad_to_bimbam(filename): geno_file_rows = read_dryad_file(filename) write_bimbam_files(geno_file_rows) + if __name__ == "__main__": input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" convert_dryad_to_bimbam(input_filename) diff --git a/wqflask/maintenance/convert_geno_to_bimbam.py b/wqflask/maintenance/convert_geno_to_bimbam.py index dc01cbb3..5b2369c9 100644 --- a/wqflask/maintenance/convert_geno_to_bimbam.py +++ b/wqflask/maintenance/convert_geno_to_bimbam.py @@ -20,8 +20,10 @@ import simplejson as json from pprint import pformat as pf + class EmptyConfigurations(Exception): pass + class Marker: def __init__(self): self.name = None @@ -30,6 +32,7 @@ class Marker: self.Mb = None self.genotypes = [] + class ConvertGenoFile: def __init__(self, input_file, output_files): @@ -178,6 +181,7 @@ class ConvertGenoFile: print(" Row is:", convertob.latest_row_value) break + if __name__ == "__main__": Old_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype""" New_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype/bimbam""" diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index f480d63f..583a06e1 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -55,6 +55,7 @@ from pprint import pformat as pf #conn = Engine.connect() + def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" @@ -143,6 +144,7 @@ def phenotypes_exist(group_name): else: return False + def genotypes_exist(group_name): #print("group_name:", group_name) Cursor.execute("""select Name from GenoFreeze @@ -156,6 +158,7 @@ def genotypes_exist(group_name): else: return False + def build_types(species, group): """Fetches tissues @@ -184,6 +187,7 @@ def build_types(species, group): return results + def get_datasets(types): """Build datasets list""" datasets = {} @@ -308,6 +312,7 @@ def _test_it(): datasets = build_datasets("Mouse", "BXD", "Hippocampus") #print("build_datasets:", pf(datasets)) + if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() diff --git a/wqflask/maintenance/generate_kinship_from_bimbam.py b/wqflask/maintenance/generate_kinship_from_bimbam.py index 664e9e48..7cc60c9e 100644 --- a/wqflask/maintenance/generate_kinship_from_bimbam.py +++ b/wqflask/maintenance/generate_kinship_from_bimbam.py @@ -13,6 +13,7 @@ sys.path.append("..") import os import glob + class GenerateKinshipMatrices: def __init__(self, group_name, geno_file, pheno_file): self.group_name = group_name diff --git a/wqflask/maintenance/generate_probesetfreeze_file.py b/wqflask/maintenance/generate_probesetfreeze_file.py index b1e41e9a..bd9c2ab4 100644 --- a/wqflask/maintenance/generate_probesetfreeze_file.py +++ b/wqflask/maintenance/generate_probesetfreeze_file.py @@ -23,10 +23,12 @@ def get_cursor(): cursor = con.cursor() return cursor + def show_progress(process, counter): if counter % 1000 == 0: print("{}: {}".format(process, counter)) + def get_strains(cursor): cursor.execute("""select Strain.Name from Strain, StrainXRef, InbredSet @@ -42,6 +44,7 @@ def get_strains(cursor): return strains + def get_probeset_vals(cursor, dataset_name): cursor.execute(""" select ProbeSet.Id, ProbeSet.Name from ProbeSetXRef, @@ -77,6 +80,7 @@ def get_probeset_vals(cursor, dataset_name): return probeset_vals + def trim_strains(strains, probeset_vals): trimmed_strains = [] #print("probeset_vals is:", pf(probeset_vals)) @@ -89,6 +93,7 @@ def trim_strains(strains, probeset_vals): print("trimmed_strains:", pf(trimmed_strains)) return trimmed_strains + def write_data_matrix_file(strains, probeset_vals, filename): with open(filename, "wb") as fh: csv_writer = csv.writer(fh, delimiter=",", quoting=csv.QUOTE_ALL) @@ -103,6 +108,7 @@ def write_data_matrix_file(strains, probeset_vals, filename): csv_writer.writerow(row_data) show_progress("Writing", counter) + def main(): filename = os.path.expanduser("~/gene/wqflask/maintenance/" + "ProbeSetFreezeId_210_FullName_Eye_AXBXA_Illumina_V6.2" + @@ -117,5 +123,6 @@ def main(): trimmed_strains = trim_strains(strains, probeset_vals) write_data_matrix_file(trimmed_strains, probeset_vals, filename) + if __name__ == '__main__': main() diff --git a/wqflask/maintenance/geno_to_json.py b/wqflask/maintenance/geno_to_json.py index fa0dcebd..ad3f2b72 100644 --- a/wqflask/maintenance/geno_to_json.py +++ b/wqflask/maintenance/geno_to_json.py @@ -25,10 +25,10 @@ from pprint import pformat as pf #from utility.tools import flat_files + class EmptyConfigurations(Exception): pass - class Marker: def __init__(self): self.name = None @@ -37,6 +37,7 @@ class Marker: self.Mb = None self.genotypes = [] + class ConvertGenoFile: def __init__(self, input_file, output_file): @@ -78,7 +79,6 @@ class ConvertGenoFile: # elif self.file_type == "snps": # self.process_snps_file() - def process_csv(self): for row_count, row in enumerate(self.process_rows()): row_items = row.split("\t") @@ -121,7 +121,6 @@ class ConvertGenoFile: # self.output_fh.write("\n") - def process_rows(self): for self.latest_row_pos, row in enumerate(self.input_fh): # if self.input_file.endswith(".geno.gz"): @@ -182,7 +181,6 @@ class ConvertGenoFile: # convertob = ConvertGenoFile(input_file, output_file) - if __name__ == "__main__": Old_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype""" New_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype/json""" diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py index 3f9d0278..0a450d3f 100644 --- a/wqflask/maintenance/get_group_samplelists.py +++ b/wqflask/maintenance/get_group_samplelists.py @@ -4,12 +4,14 @@ import gzip from base import webqtlConfig + def get_samplelist(file_type, geno_file): if file_type == "geno": return get_samplelist_from_geno(geno_file) elif file_type == "plink": return get_samplelist_from_plink(geno_file) + def get_samplelist_from_geno(genofilename): if os.path.isfile(genofilename + '.gz'): genofilename += '.gz' @@ -33,6 +35,7 @@ def get_samplelist_from_geno(genofilename): samplelist = headers[3:] return samplelist + def get_samplelist_from_plink(genofilename): genofile = open(genofilename) diff --git a/wqflask/maintenance/print_benchmark.py b/wqflask/maintenance/print_benchmark.py index a1046c86..9d12da8a 100644 --- a/wqflask/maintenance/print_benchmark.py +++ b/wqflask/maintenance/print_benchmark.py @@ -15,15 +15,18 @@ class TheCounter: self.time_took = time.time() - start_time TheCounter.Counters[self.__class__.__name__] = self.time_took + class PrintAll(TheCounter): def print_it(self, counter): print(counter) + class PrintSome(TheCounter): def print_it(self, counter): if counter % 1000 == 0: print(counter) + class PrintNone(TheCounter): def print_it(self, counter): pass @@ -37,5 +40,6 @@ def new_main(): print(pf(TheCounter.Counters)) + if __name__ == '__main__': new_main() diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 6751a8e5..1896bc52 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -14,6 +14,7 @@ from wqflask import app from utility.elasticsearch_tools import get_elasticsearch_connection from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI + def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" @@ -28,6 +29,7 @@ def parse_db_uri(): print(db_conn_info) return db_conn_info + def create_dataframe(input_file): with open(input_file) as f: ncols = len(f.readline().split("\t")) @@ -36,6 +38,8 @@ def create_dataframe(input_file): return pd.DataFrame(input_array) # This function taken from https://github.com/ShawnLYU/Quantile_Normalize + + def quantileNormalize(df_input): df = df_input.copy() # compute rank @@ -50,6 +54,7 @@ def quantileNormalize(df_input): df[col] = [rank[i] for i in t] return df + def set_data(dataset_name): orig_file = "/home/zas1024/cfw_data/" + dataset_name + ".txt" @@ -95,6 +100,7 @@ def set_data(dataset_name): } } + if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index 286094dd..c6c4f44c 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -37,6 +37,7 @@ import urllib.parse from utility.logger import getLogger logger = getLogger(__name__) + def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" @@ -51,6 +52,7 @@ def parse_db_uri(): print(db_conn_info) return db_conn_info + def insert_probeset_resources(default_owner_id): current_resources = Redis.hgetall("resources") Cursor.execute(""" SELECT @@ -77,6 +79,7 @@ def insert_probeset_resources(default_owner_id): add_resource(resource_ob, update=False) + def insert_publish_resources(default_owner_id): current_resources = Redis.hgetall("resources") Cursor.execute(""" SELECT @@ -110,6 +113,7 @@ def insert_publish_resources(default_owner_id): else: continue + def insert_geno_resources(default_owner_id): current_resources = Redis.hgetall("resources") Cursor.execute(""" SELECT @@ -139,6 +143,7 @@ def insert_geno_resources(default_owner_id): add_resource(resource_ob, update=False) + def insert_resources(default_owner_id): current_resources = get_resources() print("START") @@ -149,6 +154,7 @@ def insert_resources(default_owner_id): insert_probeset_resources(default_owner_id) print("AFTER PROBESET") + def main(): """Generates and outputs (as json file) the data for the main dropdown menus on the home page""" @@ -158,6 +164,7 @@ def main(): insert_resources(owner_id) + if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index 58108e03..03f310eb 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -14,9 +14,11 @@ from utility.startup_config import app_config app_config() + @app.route("/gunicorn") def hello(): return "

Hello There!

" + if __name__ == "__main__": app.run(host='0.0.0.0') diff --git a/wqflask/tests/unit/base/test_webqtl_case_data.py b/wqflask/tests/unit/base/test_webqtl_case_data.py index 8e8ba482..cebd41ce 100644 --- a/wqflask/tests/unit/base/test_webqtl_case_data.py +++ b/wqflask/tests/unit/base/test_webqtl_case_data.py @@ -4,6 +4,7 @@ import unittest from wqflask import app # Required because of utility.tools in webqtlCaseData.py from base.webqtlCaseData import webqtlCaseData + class TestWebqtlCaseData(unittest.TestCase): """Tests for WebqtlCaseData class""" diff --git a/wqflask/tests/unit/utility/test_authentication_tools.py b/wqflask/tests/unit/utility/test_authentication_tools.py index 42dcae88..024ab43f 100644 --- a/wqflask/tests/unit/utility/test_authentication_tools.py +++ b/wqflask/tests/unit/utility/test_authentication_tools.py @@ -5,6 +5,7 @@ from unittest import mock from utility.authentication_tools import check_resource_availability from utility.authentication_tools import add_new_resource + class TestResponse: """Mock Test Response after a request""" @property diff --git a/wqflask/tests/unit/utility/test_chunks.py b/wqflask/tests/unit/utility/test_chunks.py index 8d90a1ec..1d349193 100644 --- a/wqflask/tests/unit/utility/test_chunks.py +++ b/wqflask/tests/unit/utility/test_chunks.py @@ -7,6 +7,7 @@ from utility.chunks import divide_into_chunks class TestChunks(unittest.TestCase): "Test Utility method for chunking" + def test_divide_into_chunks(self): "Check that a list is chunked correctly" self.assertEqual(divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 3), diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_display_mapping_results.py b/wqflask/tests/unit/wqflask/marker_regression/test_display_mapping_results.py index 8ae0f09f..219a6a29 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_display_mapping_results.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_display_mapping_results.py @@ -9,6 +9,7 @@ from wqflask.marker_regression.display_mapping_results import ( class TestDisplayMappingResults(unittest.TestCase): """Basic Methods to test Mapping Results""" + def test_pil_colors(self): """Test that colors use PILLOW color format""" self.assertEqual(DisplayMappingResults.CLICKABLE_WEBQTL_REGION_COLOR, @@ -17,6 +18,7 @@ class TestDisplayMappingResults(unittest.TestCase): class TestHtmlGenWrapper(unittest.TestCase): """Test Wrapper around HTMLGen""" + def test_create_image(self): """Test HT.Image method""" self.assertEqual( diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_plink_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_plink_mapping.py index 5eec93f1..fd21a825 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_plink_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_plink_mapping.py @@ -12,9 +12,10 @@ class AttributeSetter: def __init__(self, obj): for key, val in obj.items(): setattr(self, key, val) -class TestPlinkMapping(unittest.TestCase): +class TestPlinkMapping(unittest.TestCase): + def test_build_line_list(self): """test for building line list""" line_1 = "this is line one test" diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py index bbb79f98..5cc8fd0f 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py @@ -4,6 +4,8 @@ from wqflask.marker_regression.qtlreaper_mapping import gen_pheno_txt_file # issues some methods in genofile object are not defined # modify samples should equal to vals + + class TestQtlReaperMapping(unittest.TestCase): @mock.patch("wqflask.marker_regression.qtlreaper_mapping.TEMPDIR", "/home/user/data") def test_gen_pheno_txt_file(self): diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 0e617e93..6267ce9a 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -5,6 +5,7 @@ from wqflask.marker_regression.rqtl_mapping import get_trait_data_type from wqflask.marker_regression.rqtl_mapping import sanitize_rqtl_phenotype from wqflask.marker_regression.rqtl_mapping import sanitize_rqtl_names + class TestRqtlMapping(unittest.TestCase): def setUp(self): @@ -14,7 +15,6 @@ class TestRqtlMapping(unittest.TestCase): def tearDown(self): self.app_context.pop() - @mock.patch("wqflask.marker_regression.rqtl_mapping.g") @mock.patch("wqflask.marker_regression.rqtl_mapping.logger") def test_get_trait_data(self, mock_logger, mock_db): diff --git a/wqflask/tests/unit/wqflask/test_collect.py b/wqflask/tests/unit/wqflask/test_collect.py index 9a36132d..2a914fb2 100644 --- a/wqflask/tests/unit/wqflask/test_collect.py +++ b/wqflask/tests/unit/wqflask/test_collect.py @@ -11,6 +11,7 @@ app = Flask(__name__) class MockSession: """Helper class for mocking wqflask.collect.g.user_session.logged_in""" + def __init__(self, is_logged_in=False): self.is_logged_in = is_logged_in @@ -21,6 +22,7 @@ class MockSession: class MockFlaskG: """Helper class for mocking wqflask.collect.g.user_session""" + def __init__(self, is_logged_in=False): self.is_logged_in = is_logged_in diff --git a/wqflask/tests/wqflask/show_trait/test_show_trait.py b/wqflask/tests/wqflask/show_trait/test_show_trait.py index 8c866874..24c3923e 100644 --- a/wqflask/tests/wqflask/show_trait/test_show_trait.py +++ b/wqflask/tests/wqflask/show_trait/test_show_trait.py @@ -242,7 +242,6 @@ class TestTraits(unittest.TestCase): self.assertEqual(get_genotype_scales(file_location), expected_results) mock_get_scales.assert_called_once_with(file_location) - @mock.patch("wqflask.show_trait.show_trait.locate_ignore_error") def test_get_scales_from_genofile_found(self, mock_ignore_location): """"add test for get scales from genofile where file is found""" diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 68c2cb72..d35b2089 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -47,6 +47,7 @@ COUR_FILE = "./wqflask/static/fonts/courbd.ttf" TAHOMA_FILE = "./wqflask/static/fonts/tahoma.ttf" # ---- END: FONT FILES ---- # + def cformat(d, rank=0): 'custom string format' strD = "%2.6f" % d @@ -68,6 +69,7 @@ def cformat(d, rank=0): strD = '0.0' return strD + def frange(start, end=None, inc=1.0): "A faster range-like function that does accept float increments..." if end == None: @@ -84,6 +86,7 @@ def frange(start, end=None, inc=1.0): L[i] = start + i * inc return L + def find_outliers(vals): """Calculates the upper and lower bounds of a set of sample/case values @@ -119,6 +122,8 @@ def find_outliers(vals): # parameter: data is either object returned by reaper permutation function (called by MarkerRegressionPage.py) # or the first object returned by direct (pair-scan) permu function (called by DirectPlotPage.py) + + def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLabel=None, YLabel=None, title=None, offset=(60, 20, 40, 40), zoom=1): im_drawer = ImageDraw.Draw(canvas) xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset @@ -230,6 +235,8 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab font=labelFont, fill=labelColor) # This function determines the scale of the plot + + def detScaleOld(min, max): if min >= max: return None @@ -246,6 +253,7 @@ def detScaleOld(min, max): high = c * ceil(max / c) return [low, high, round((high - low) / c)] + def detScale(min=0, max=0): if min >= max: @@ -283,15 +291,19 @@ def detScale(min=0, max=0): return [low, high, n] + def bluefunc(x): return 1.0 / (1.0 + exp(-10 * (x - 0.6))) + def redfunc(x): return 1.0 / (1.0 + exp(10 * (x - 0.5))) + def greenfunc(x): return 1 - pow(redfunc(x + 0.2), 2) - bluefunc(x - 0.3) + def colorSpectrum(n=100): multiple = 10 if n == 1: @@ -319,6 +331,7 @@ def colorSpectrum(n=100): out2.append(out[-1]) return out2 + def _test(): import doctest doctest.testmod() diff --git a/wqflask/utility/__init__.py b/wqflask/utility/__init__.py index ec7e72d0..d540c96e 100644 --- a/wqflask/utility/__init__.py +++ b/wqflask/utility/__init__.py @@ -2,8 +2,10 @@ from pprint import pformat as pf # Todo: Move these out of __init__ + class Bunch: """Like a dictionary but using object notation""" + def __init__(self, **kw): self.__dict__ = kw diff --git a/wqflask/utility/after.py b/wqflask/utility/after.py index 06091ecb..2b560e48 100644 --- a/wqflask/utility/after.py +++ b/wqflask/utility/after.py @@ -7,6 +7,7 @@ from flask import g from wqflask import app + def after_this_request(f): if not hasattr(g, 'after_request_callbacks'): g.after_request_callbacks = [] diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 672b36d5..57dbf8ba 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -11,6 +11,7 @@ from utility.redis_tools import (get_redis_conn, add_resource) Redis = get_redis_conn() + def check_resource_availability(dataset, trait_id=None): # At least for now assume temporary entered traits are accessible if type(dataset) == str or dataset.type == "Temp": diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py index 3d40a3b8..48ab1dc0 100644 --- a/wqflask/utility/benchmark.py +++ b/wqflask/utility/benchmark.py @@ -6,6 +6,7 @@ from utility.tools import LOG_BENCH from utility.logger import getLogger logger = getLogger(__name__) + class Bench: entries = collections.OrderedDict() diff --git a/wqflask/utility/corestats.py b/wqflask/utility/corestats.py index 15d1cb8d..523280a1 100644 --- a/wqflask/utility/corestats.py +++ b/wqflask/utility/corestats.py @@ -16,6 +16,8 @@ import sys # ZS: Should switch to using some third party library for this; maybe scipy has an equivalent + + class Stats: def __init__(self, sequence): diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index ae1181e4..9415cef0 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -47,11 +47,13 @@ logger = getLogger(__name__) from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + def test_elasticsearch_connection(): es = Elasticsearch(['http://' + ELASTICSEARCH_HOST + ":" + str(ELASTICSEARCH_PORT) + '/'], verify_certs=True) if not es.ping(): logger.warning("Elasticsearch is DOWN") + def get_elasticsearch_connection(for_user=True): """Return a connection to ES. Returns None on failure""" logger.info("get_elasticsearch_connection") @@ -77,6 +79,7 @@ def get_elasticsearch_connection(for_user=True): return es + def setup_users_index(es_connection): if es_connection: index_settings = { @@ -87,12 +90,15 @@ def setup_users_index(es_connection): es_connection.indices.create(index='users', ignore=400) es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local") + def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) + def save_user(es, user, user_id): es_save_data(es, "users", "local", user, user_id) + def get_item_by_unique_column(es, column_name, column_value, index, doc_type): item_details = None try: @@ -106,6 +112,7 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type): pass return item_details + def es_save_data(es, index, doc_type, data_item, data_id,): from time import sleep es.create(index, doc_type, body=data_item, id=data_id) diff --git a/wqflask/utility/external.py b/wqflask/utility/external.py index c1bf4043..805d2ffe 100644 --- a/wqflask/utility/external.py +++ b/wqflask/utility/external.py @@ -4,6 +4,7 @@ import os import sys import subprocess + def shell(command): if subprocess.call(command, shell=True) != 0: raise Exception("ERROR: failed on " + command) diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index 9cfa39f9..24604e58 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -1,6 +1,7 @@ import utility.logger logger = utility.logger.getLogger(__name__) + class genotype: """ Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure @@ -119,6 +120,7 @@ class genotype: self.chromosomes.append(chr_ob) + class Chr: def __init__(self, name, geno_ob): self.name = name @@ -140,6 +142,7 @@ class Chr: def add_marker(self, marker_row): self.loci.append(Locus(self.geno_ob, marker_row)) + class Locus: def __init__(self, geno_ob, marker_row=None): self.chr = None diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py index f8e96d19..94a08c17 100644 --- a/wqflask/utility/genofile_parser.py +++ b/wqflask/utility/genofile_parser.py @@ -12,6 +12,7 @@ import simplejson as json from pprint import pformat as pf + class Marker: def __init__(self): self.name = None diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py index 16912e58..47079818 100644 --- a/wqflask/utility/logger.py +++ b/wqflask/utility/logger.py @@ -35,6 +35,7 @@ import datetime from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL + class GNLogger: """A logger class with some additional functionality, such as multiple parameter logging, SQL logging, timing, colors, and lazy @@ -139,6 +140,8 @@ LOG_LEVEL_DEBUG (NYI). # Get the module logger. You can override log levels at the # module level + + def getLogger(name, level=None): gnlogger = GNLogger(name) logger = gnlogger.logger diff --git a/wqflask/utility/pillow_utils.py b/wqflask/utility/pillow_utils.py index 6e95beb0..5713e155 100644 --- a/wqflask/utility/pillow_utils.py +++ b/wqflask/utility/pillow_utils.py @@ -9,6 +9,8 @@ BLACK = ImageColor.getrgb("black") WHITE = ImageColor.getrgb("white") # def draw_rotated_text(canvas: Image, text: str, font: ImageFont, xy: tuple, fill: ImageColor=BLACK, angle: int=-90): + + def draw_rotated_text(canvas, text, font, xy, fill=BLACK, angle=-90): # type: (Image, str, ImageFont, tuple, ImageColor, int) """Utility function draw rotated text""" @@ -20,6 +22,8 @@ def draw_rotated_text(canvas, text, font, xy, fill=BLACK, angle=-90): canvas.paste(im=tmp_img2, box=tuple([int(i) for i in xy])) # def draw_open_polygon(canvas: Image, xy: tuple, fill: ImageColor=WHITE, outline: ImageColor=BLACK): + + def draw_open_polygon(canvas, xy, fill=None, outline=BLACK, width=0): # type: (Image, tuple, ImageColor, ImageColor) draw_ctx = ImageDraw.Draw(canvas) diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index f22f4b14..92f944bc 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -10,6 +10,7 @@ GREEN = '\033[92m' BOLD = '\033[1m' ENDC = '\033[0m' + def app_config(): app.config['SESSION_TYPE'] = 'filesystem' if not app.config.get('SECRET_KEY'): diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 4c478c36..f5ef81e1 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -459,11 +459,13 @@ class rect(SVGelement): if stroke_width != None: self.attributes['stroke-width'] = stroke_width + class ellipse(SVGelement): """e=ellipse(rx,ry,x,y,fill,stroke,stroke_width,**args) an ellipse is defined as a center and a x and y radius. """ + def __init__(self, cx=None, cy=None, rx=None, ry=None,fill=None,stroke=None,stroke_width=None,**args): if rx == None or ry == None: raise ValueError('both rx and ry are required') @@ -486,6 +488,7 @@ class circle(SVGelement): The circle creates an element using a x, y and radius values eg """ + def __init__(self, cx=None, cy=None, r=None, fill=None,stroke=None,stroke_width=None,**args): if r == None: raise ValueError('r is required') @@ -501,20 +504,24 @@ class circle(SVGelement): if stroke_width != None: self.attributes['stroke-width'] = stroke_width + class point(circle): """p=point(x,y,color) A point is defined as a circle with a size 1 radius. It may be more efficient to use a very small rectangle if you use many points because a circle is difficult to render. """ + def __init__(self, x, y, fill='black', **args): circle.__init__(self, x, y, 1, fill, **args) + class line(SVGelement): """l=line(x1,y1,x2,y2,stroke,stroke_width,**args) A line is defined by a begin x,y pair and an end x,y pair """ + def __init__(self, x1=None, y1=None, x2=None, y2=None,stroke=None,stroke_width=None,**args): SVGelement.__init__(self, 'line', **args) if x1 != None: @@ -530,11 +537,13 @@ class line(SVGelement): if stroke != None: self.attributes['stroke'] = stroke + class polyline(SVGelement): """pl=polyline([[x1,y1],[x2,y2],...],fill,stroke,stroke_width,**args) a polyline is defined by a list of xy pairs """ + def __init__(self, points, fill=None, stroke=None, stroke_width=None,**args): SVGelement.__init__(self, 'polyline', {'points': _xypointlist(points)}, **args) if fill != None: @@ -544,11 +553,13 @@ class polyline(SVGelement): if stroke != None: self.attributes['stroke'] = stroke + class polygon(SVGelement): """pl=polyline([[x1,y1],[x2,y2],...],fill,stroke,stroke_width,**args) a polygon is defined by a list of xy pairs """ + def __init__(self, points, fill=None, stroke=None, stroke_width=None,**args): SVGelement.__init__(self, 'polygon', {'points': _xypointlist(points)}, **args) if fill != None: @@ -558,11 +569,13 @@ class polygon(SVGelement): if stroke != None: self.attributes['stroke'] = stroke + class path(SVGelement): """p=path(path,fill,stroke,stroke_width,**args) a path is defined by a path object and optional width, stroke and fillcolor """ + def __init__(self, pathdata, fill=None, stroke=None, stroke_width=None,id=None,**args): SVGelement.__init__(self, 'path', {'d': str(pathdata)}, **args) if stroke != None: @@ -580,6 +593,7 @@ class text(SVGelement): a text element can bge used for displaying text on the screen """ + def __init__(self, x=None, y=None, text=None, font_size=None,font_family=None,text_anchor=None,**args): SVGelement.__init__(self, 'text', **args) if x != None: @@ -601,11 +615,13 @@ class textpath(SVGelement): a textpath places a text on a path which is referenced by a link. """ + def __init__(self, link, text=None, **args): SVGelement.__init__(self, 'textPath', {'xlink:href': link}, **args) if text != None: self.text = text + class pattern(SVGelement): """p=pattern(x,y,width,height,patternUnits,**args) @@ -613,6 +629,7 @@ class pattern(SVGelement): graphic object which can be replicated ("tiled") at fixed intervals in x and y to cover the areas to be painted. """ + def __init__(self, x=None, y=None, width=None, height=None,patternUnits=None,**args): SVGelement.__init__(self, 'pattern', **args) if x != None: @@ -626,34 +643,40 @@ class pattern(SVGelement): if patternUnits != None: self.attributes['patternUnits'] = patternUnits + class title(SVGelement): """t=title(text,**args) a title is a text element. The text is displayed in the title bar add at least one to the root svg element """ + def __init__(self, text=None, **args): SVGelement.__init__(self, 'title', **args) if text != None: self.text = text + class description(SVGelement): """d=description(text,**args) a description can be added to any element and is used for a tooltip Add this element before adding other elements. """ + def __init__(self, text=None, **args): SVGelement.__init__(self, 'desc', **args) if text != None: self.text = text + class lineargradient(SVGelement): """lg=lineargradient(x1,y1,x2,y2,id,**args) defines a lineargradient using two xy pairs. stop elements van be added to define the gradient colors. """ + def __init__(self, x1=None, y1=None, x2=None, y2=None,id=None,**args): SVGelement.__init__(self, 'linearGradient', **args) if x1 != None: @@ -667,12 +690,14 @@ class lineargradient(SVGelement): if id != None: self.attributes['id'] = id + class radialgradient(SVGelement): """rg=radialgradient(cx,cy,r,fx,fy,id,**args) defines a radial gradient using a outer circle which are defined by a cx,cy and r and by using a focalpoint. stop elements van be added to define the gradient colors. """ + def __init__(self, cx=None, cy=None, r=None, fx=None,fy=None,id=None,**args): SVGelement.__init__(self, 'radialGradient', **args) if cx != None: @@ -688,21 +713,25 @@ class radialgradient(SVGelement): if id != None: self.attributes['id'] = id + class stop(SVGelement): """st=stop(offset,stop_color,**args) Puts a stop color at the specified radius """ + def __init__(self, offset, stop_color=None, **args): SVGelement.__init__(self, 'stop', {'offset': offset}, **args) if stop_color != None: self.attributes['stop-color'] = stop_color + class style(SVGelement): """st=style(type,cdata=None,**args) Add a CDATA element to this element for defing in line stylesheets etc.. """ + def __init__(self, type, cdata=None, **args): SVGelement.__init__(self, 'style', {'type': type}, cdata=cdata, **args) @@ -712,6 +741,7 @@ class image(SVGelement): adds an image to the drawing. Supported formats are .png, .jpg and .svg. """ + def __init__(self, url, x=None, y=None, width=None,height=None,**args): if width == None or height == None: raise ValueError('both height and width are required') @@ -721,11 +751,13 @@ class image(SVGelement): if y != None: self.attributes['y'] = y + class cursor(SVGelement): """c=cursor(url,**args) defines a custom cursor for a element or a drawing """ + def __init__(self, url, **args): SVGelement.__init__(self, 'cursor', {'xlink:href': url}, **args) @@ -736,6 +768,7 @@ class marker(SVGelement): defines a marker which can be used as an endpoint for a line or other pathtypes add an element to it which should be used as a marker. """ + def __init__(self, id=None, viewBox=None, refx=None, refy=None,markerWidth=None,markerHeight=None,**args): SVGelement.__init__(self, 'marker', **args) if id != None: @@ -751,17 +784,20 @@ class marker(SVGelement): if markerHeight != None: self.attributes['markerHeight'] = markerHeight + class group(SVGelement): """g=group(id,**args) a group is defined by an id and is used to contain elements g.addElement(SVGelement) """ + def __init__(self, id=None, **args): SVGelement.__init__(self, 'g', **args) if id != None: self.attributes['id'] = id + class symbol(SVGelement): """sy=symbol(id,viewbox,**args) @@ -778,14 +814,17 @@ class symbol(SVGelement): if viewBox != None: self.attributes['viewBox'] = _viewboxlist(viewBox) + class defs(SVGelement): """d=defs(**args) container for defining elements """ + def __init__(self, **args): SVGelement.__init__(self, 'defs', **args) + class switch(SVGelement): """sw=switch(**args) @@ -793,6 +832,7 @@ class switch(SVGelement): requiredFeatures, requiredExtensions and systemLanguage. Refer to the SVG specification for details. """ + def __init__(self, **args): SVGelement.__init__(self, 'switch', **args) @@ -802,6 +842,7 @@ class use(SVGelement): references a symbol by linking to its id and its position, height and width """ + def __init__(self, link, x=None, y=None, width=None,height=None,**args): SVGelement.__init__(self, 'use', {'xlink:href': link}, **args) if x != None: @@ -821,32 +862,39 @@ class link(SVGelement): a link is defined by a hyperlink. add elements which have to be linked a.addElement(SVGelement) """ + def __init__(self, link='', **args): SVGelement.__init__(self, 'a', {'xlink:href': link}, **args) + class view(SVGelement): """v=view(id,**args) a view can be used to create a view with different attributes""" + def __init__(self, id=None, **args): SVGelement.__init__(self, 'view', **args) if id != None: self.attributes['id'] = id + class script(SVGelement): """sc=script(type,type,cdata,**args) adds a script element which contains CDATA to the SVG drawing """ + def __init__(self, type, cdata=None, **args): SVGelement.__init__(self, 'script', {'type': type}, cdata=cdata, **args) + class animate(SVGelement): """an=animate(attribute,from,to,during,**args) animates an attribute. """ + def __init__(self, attribute, fr=None, to=None, dur=None,**args): SVGelement.__init__(self, 'animate', {'attributeName': attribute}, **args) if fr != None: @@ -856,11 +904,13 @@ class animate(SVGelement): if dur != None: self.attributes['dur'] = dur + class animateMotion(SVGelement): """an=animateMotion(pathdata,dur,**args) animates a SVGelement over the given path in dur seconds """ + def __init__(self, pathdata, dur, **args): SVGelement.__init__(self, 'animateMotion', **args) if pathdata != None: @@ -868,11 +918,13 @@ class animateMotion(SVGelement): if dur != None: self.attributes['dur'] = dur + class animateTransform(SVGelement): """antr=animateTransform(type,from,to,dur,**args) transform an element from and to a value. """ + def __init__(self, type=None, fr=None, to=None, dur=None,**args): SVGelement.__init__(self, 'animateTransform', {'attributeName': 'transform'}, **args) # As far as I know the attributeName is always transform @@ -884,11 +936,14 @@ class animateTransform(SVGelement): self.attributes['to'] = to if dur != None: self.attributes['dur'] = dur + + class animateColor(SVGelement): """ac=animateColor(attribute,type,from,to,dur,**args) Animates the color of a element """ + def __init__(self, attribute, type=None, fr=None, to=None,dur=None,**args): SVGelement.__init__(self, 'animateColor', {'attributeName': attribute}, **args) if type != None: @@ -899,11 +954,14 @@ class animateColor(SVGelement): self.attributes['to'] = to if dur != None: self.attributes['dur'] = dur + + class set(SVGelement): """st=set(attribute,to,during,**args) sets an attribute to a value for a """ + def __init__(self, attribute, to=None, dur=None, **args): SVGelement.__init__(self, 'set', {'attributeName': attribute}, **args) if to != None: @@ -912,7 +970,6 @@ class set(SVGelement): self.attributes['dur'] = dur - class svg(SVGelement): """s=svg(viewbox,width,height,**args) @@ -928,6 +985,7 @@ class svg(SVGelement): d.setSVG(s) d.toXml() """ + def __init__(self, viewBox=None, width=None, height=None, **args): SVGelement.__init__(self, 'svg', **args) if viewBox != None: @@ -938,6 +996,7 @@ class svg(SVGelement): self.attributes['height'] = height self.namespace = "http://www.w3.org/2000/svg" + class drawing: """d=drawing() @@ -952,6 +1011,7 @@ class drawing: def __init__(self, entity={}): self.svg = None self.entity = entity + def setSVG(self, svg): self.svg = svg # Voeg een element toe aan de grafiek toe. @@ -1005,6 +1065,7 @@ class drawing: root = implementation.createDocument(None, None, doctype) # Create the xml document. global appender + def appender(element, elementroot): """This recursive function appends elements to an element and sets the attributes and type. It stops when alle elements have been appended""" @@ -1053,6 +1114,7 @@ class drawing: f.close() except: print(("Cannot write SVG file: " + filename)) + def validate(self): try: import xml.parsers.xmlproc.xmlval @@ -1066,9 +1128,10 @@ class drawing: raise Exception("SVG is not well formed, see messages above") else: print("SVG well formed") -if __name__ == '__main__': +if __name__ == '__main__': + d = drawing() s = svg((0, 0, 100, 100)) r = rect(-100, -100, 300, 300, 'cyan') diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 2088ba9a..07c5a318 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -2,6 +2,7 @@ from redis import Redis import simplejson as json + class TempData: def __init__(self, temp_uuid): diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 9b751344..4fe4db08 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -13,11 +13,13 @@ logger = logging.getLogger(__name__) OVERRIDES = {} + def app_set(command_id, value): """Set application wide value""" app.config.setdefault(command_id, value) return value + def get_setting(command_id, guess=None): """Resolve a setting from the environment or the global settings in app.config, with valid_path is a function checking whether the @@ -66,12 +68,14 @@ def get_setting(command_id, guess=None): # print("Set "+command_id+"="+str(command)) return command + def get_setting_bool(id): v = get_setting(id) if v not in [0, False, 'False', 'FALSE', None]: return True return False + def get_setting_int(id): v = get_setting(id) if isinstance(v, str): @@ -80,21 +84,25 @@ def get_setting_int(id): return 0 return v + def valid_bin(bin): if os.path.islink(bin) or valid_file(bin): return bin return None + def valid_file(fn): if os.path.isfile(fn): return fn return None + def valid_path(dir): if os.path.isdir(dir): return dir return None + def js_path(module=None): """ Find the JS module in the two paths @@ -107,38 +115,47 @@ def js_path(module=None): return try_guix raise "No JS path found for " + module + " (if not in Guix check JS_GN_PATH)" + def reaper_command(guess=None): return get_setting("REAPER_COMMAND", guess) + def gemma_command(guess=None): return assert_bin(get_setting("GEMMA_COMMAND", guess)) + def gemma_wrapper_command(guess=None): return assert_bin(get_setting("GEMMA_WRAPPER_COMMAND", guess)) + def plink_command(guess=None): return assert_bin(get_setting("PLINK_COMMAND", guess)) + def flat_file_exists(subdir): base = get_setting("GENENETWORK_FILES") return valid_path(base + "/" + subdir) + def flat_files(subdir=None): base = get_setting("GENENETWORK_FILES") if subdir: return assert_dir(base + "/" + subdir) return assert_dir(base) + def assert_bin(fn): if not valid_bin(fn): raise Exception("ERROR: can not find binary " + fn) return fn + def assert_dir(dir): if not valid_path(dir): raise Exception("ERROR: can not find directory " + dir) return dir + def assert_writable_dir(dir): try: fn = dir + "/test.txt" @@ -150,16 +167,19 @@ def assert_writable_dir(dir): raise Exception('Unable to write test.txt to directory ' + dir) return dir + def assert_file(fn): if not valid_file(fn): raise Exception('Unable to find file ' + fn) return fn + def mk_dir(dir): if not valid_path(dir): os.makedirs(dir) return assert_dir(dir) + def locate(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -179,9 +199,11 @@ def locate(name, subdir=None): if subdir: sys.stderr.write(subdir) raise Exception("Can not locate " + name + " in " + base) + def locate_phewas(name, subdir=None): return locate(name, '/phewas/' + subdir) + def locate_ignore_error(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -200,17 +222,20 @@ def locate_ignore_error(name, subdir=None): logger.info("WARNING: file " + name + " not found\n") return None + def tempdir(): """ Get UNIX TMPDIR by default """ return valid_path(get_setting("TMPDIR", "/tmp")) + BLUE = '\033[94m' GREEN = '\033[92m' BOLD = '\033[1m' ENDC = '\033[0m' + def show_settings(): from utility.tools import LOG_LEVEL diff --git a/wqflask/utility/type_checking.py b/wqflask/utility/type_checking.py index 662bf794..00f14ba9 100644 --- a/wqflask/utility/type_checking.py +++ b/wqflask/utility/type_checking.py @@ -7,6 +7,7 @@ def is_float(value): except: return False + def is_int(value): try: int(value) @@ -14,6 +15,7 @@ def is_int(value): except: return False + def is_str(value): if value is None: return False @@ -23,18 +25,21 @@ def is_str(value): except: return False + def get_float(vars_obj, name, default=None): if name in vars_obj: if is_float(vars_obj[name]): return float(vars_obj[name]) return default + def get_int(vars_obj, name, default=None): if name in vars_obj: if is_int(vars_obj[name]): return float(vars_obj[name]) return default + def get_string(vars_obj, name, default=None): if name in vars_obj: if not vars_obj[name] is None: diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index ed59b0eb..f355a865 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -64,6 +64,7 @@ ParInfo = { # Accessory Functions ######################################### + def genRandStr(prefix="", length=8, chars=string.ascii_letters + string.digits): from random import choice _str = prefix[:] @@ -71,6 +72,7 @@ def genRandStr(prefix="", length=8, chars=string.ascii_letters + string.digits): _str += choice(chars) return _str + def ListNotNull(lst): '''Obsolete - Use built in function any (or all or whatever) @@ -83,6 +85,7 @@ def ListNotNull(lst): return 1 return None + def readLineCSV(line): # dcrowell July 2008 """Parses a CSV string of text and returns a list containing each element as a string. Used by correlationPage""" @@ -91,6 +94,7 @@ def readLineCSV(line): # dcrowell July 2008 returnList[0] = returnList[0][1:] return returnList + def cmpEigenValue(A, B): try: if A[0] > B[0]: @@ -102,6 +106,7 @@ def cmpEigenValue(A, B): except: return 0 + def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users): access_to_confidential_phenotype_trait = 0 if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']: diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index bab87115..712517a3 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -30,6 +30,7 @@ app.register_blueprint(policies_blueprint, url_prefix="/policies") app.register_blueprint(environments_blueprint, url_prefix="/environments") app.register_blueprint(facilities_blueprint, url_prefix="/facilities") + @app.before_request def before_request(): g.request_start_time = time.time() diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index e0b7fea0..f1dd148f 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -18,6 +18,7 @@ from utility.benchmark import Bench import utility.logger logger = utility.logger.getLogger(__name__) + def do_correlation(start_vars): assert('db' in start_vars) assert('target_db' in start_vars) @@ -66,6 +67,7 @@ def do_correlation(start_vars): return final_results + def calculate_results(this_trait, this_dataset, target_dataset, corr_params): corr_results = {} @@ -91,6 +93,7 @@ def calculate_results(this_trait, this_dataset, target_dataset, corr_params): return sorted_results + def do_tissue_correlation_for_all_traits(this_trait, trait_symbol_dict, corr_params, tissue_dataset_id=1): # Gets tissue expression values for the primary trait primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(symbol_list=[this_trait.symbol]) @@ -113,6 +116,7 @@ def do_tissue_correlation_for_all_traits(this_trait, trait_symbol_dict, corr_par return tissue_corr_data + def do_literature_correlation_for_all_traits(this_trait, target_dataset, trait_geneid_dict, corr_params): input_trait_mouse_gene_id = convert_to_mouse_gene_id(target_dataset.group.species.lower(), this_trait.geneid) @@ -145,6 +149,7 @@ def do_literature_correlation_for_all_traits(this_trait, target_dataset, trait_g return lit_corr_data + def get_sample_r_and_p_values(this_trait, this_dataset, target_vals, target_dataset, type): """ Calculates the sample r (or rho) and p-value @@ -176,6 +181,7 @@ def get_sample_r_and_p_values(this_trait, this_dataset, target_vals, target_data else: return [sample_r, sample_p, num_overlap] + def convert_to_mouse_gene_id(species=None, gene_id=None): """If the species is rat or human, translate the gene_id to the mouse geneid @@ -212,6 +218,7 @@ def convert_to_mouse_gene_id(species=None, gene_id=None): return mouse_gene_id + def init_corr_params(start_vars): method = "pearson" if 'method' in start_vars: diff --git a/wqflask/wqflask/api/mapping.py b/wqflask/wqflask/api/mapping.py index fbfbc879..662090d5 100644 --- a/wqflask/wqflask/api/mapping.py +++ b/wqflask/wqflask/api/mapping.py @@ -10,6 +10,7 @@ from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_map import utility.logger logger = utility.logger.getLogger(__name__) + def do_mapping_for_api(start_vars): assert('db' in start_vars) assert('trait_id' in start_vars) @@ -74,7 +75,6 @@ def do_mapping_for_api(start_vars): return result_markers, None - def initialize_parameters(start_vars, dataset, this_trait): mapping_params = {} diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index b0559a07..4f9cc6e5 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -27,10 +27,12 @@ logger = utility.logger.getLogger(__name__) version = "pre1" + @app.route("/api/v_{}/".format(version)) def hello_world(): return flask.jsonify({"hello": "world"}) + @app.route("/api/v_{}/species".format(version)) def get_species_list(): results = g.db.execute("SELECT SpeciesId, Name, FullName, TaxonomyId FROM Species;") @@ -47,6 +49,7 @@ def get_species_list(): return flask.jsonify(species_list) + @app.route("/api/v_{}/species/".format(version)) @app.route("/api/v_{}/species/.".format(version)) def get_species_info(species_name, file_format="json"): @@ -64,6 +67,7 @@ def get_species_info(species_name, file_format="json"): return flask.jsonify(species_dict) + @app.route("/api/v_{}/groups".format(version)) @app.route("/api/v_{}/groups/".format(version)) def get_groups_list(species_name=None): @@ -102,6 +106,7 @@ def get_groups_list(species_name=None): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/group/".format(version)) @app.route("/api/v_{}/group/.".format(version)) @app.route("/api/v_{}/group//".format(version)) @@ -145,6 +150,7 @@ def get_group_info(group_name, species_name=None, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/datasets/".format(version)) @app.route("/api/v_{}/datasets//".format(version)) def get_datasets_for_group(group_name, species_name=None): @@ -197,6 +203,7 @@ def get_datasets_for_group(group_name, species_name=None): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/dataset/".format(version)) @app.route("/api/v_{}/dataset/.".format(version)) @app.route("/api/v_{}/dataset//".format(version)) @@ -302,6 +309,7 @@ def get_dataset_info(dataset_name, group_name=None, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/traits/".format(version), methods=("GET",)) @app.route("/api/v_{}/traits/.".format(version), methods=("GET",)) def fetch_traits(dataset_name, file_format="json"): @@ -430,6 +438,7 @@ def fetch_traits(dataset_name, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/sample_data/".format(version)) @app.route("/api/v_{}/sample_data/.".format(version)) def all_sample_data(dataset_name, file_format="csv"): @@ -536,6 +545,7 @@ def all_sample_data(dataset_name, file_format="csv"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/sample_data//".format(version)) @app.route("/api/v_{}/sample_data//.".format(version)) def trait_sample_data(dataset_name, trait_name, file_format="json"): @@ -625,6 +635,7 @@ def trait_sample_data(dataset_name, trait_name, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/trait//".format(version)) @app.route("/api/v_{}/trait//.".format(version)) @app.route("/api/v_{}/trait_info//".format(version)) @@ -694,6 +705,7 @@ def get_trait_info(dataset_name, trait_name, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/correlation".format(version), methods=("GET",)) def get_corr_results(): results = correlation.do_correlation(request.args) @@ -703,6 +715,7 @@ def get_corr_results(): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/mapping".format(version), methods=("GET",)) def get_mapping_results(): results, format = mapping.do_mapping_for_api(request.args) @@ -726,6 +739,7 @@ def get_mapping_results(): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/genotypes///.zip".format(version)) @app.route("/api/v_{}/genotypes///".format(version)) @app.route("/api/v_{}/genotypes//.zip".format(version)) @@ -813,6 +827,7 @@ def get_genotypes(group_name, file_format="csv", dataset_name=None): return output + @app.route("/api/v_{}/gen_dropdown".format(version), methods=("GET",)) def gen_dropdown_menu(): results = gen_menu.gen_dropdown_json() @@ -822,6 +837,7 @@ def gen_dropdown_menu(): else: return return_error(code=500, source=request.url_rule.rule, title="Some error occurred", details="") + def return_error(code, source, title, details): json_ob = {"errors": [ { @@ -834,6 +850,7 @@ def return_error(code, source, title, details): return flask.jsonify(json_ob) + def get_dataset_trait_ids(dataset_name, start_vars): if 'limit_to' in start_vars: @@ -906,6 +923,7 @@ def get_dataset_trait_ids(dataset_name, start_vars): dataset_id = results[0][2] return trait_ids, trait_names, data_type, dataset_id + def get_samplelist(dataset_name): group_id = get_group_id_from_dataset(dataset_name) @@ -922,6 +940,7 @@ def get_samplelist(dataset_name): return samplelist + def get_group_id_from_dataset(dataset_name): if "Publish" in dataset_name: query = """ @@ -962,6 +981,7 @@ def get_group_id_from_dataset(dataset_name): else: return None + def get_group_id(group_name): query = """ SELECT InbredSet.Id diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 6a1b88ca..61f73106 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -40,6 +40,7 @@ def process_traits(unprocessed_traits): return traits + def report_change(len_before, len_now): new_length = len_now - len_before if new_length: @@ -48,6 +49,7 @@ def report_change(len_before, len_now): else: logger.debug("No new traits were added.") + @app.route("/collections/store_trait_list", methods=('POST',)) def store_traits_list(): params = request.form @@ -59,6 +61,7 @@ def store_traits_list(): return hash + @app.route("/collections/add") def collections_add(): @@ -82,6 +85,7 @@ def collections_add(): collections=collections, ) + @app.route("/collections/new") def collections_new(): params = request.args @@ -118,6 +122,7 @@ def collections_new(): # CauseAnError pass + def create_new(collection_name): params = request.args @@ -133,6 +138,7 @@ def create_new(collection_name): return redirect(url_for('view_collection', uc_id=uc_id)) + @app.route("/collections/list") def list_collections(): params = request.args @@ -143,6 +149,7 @@ def list_collections(): collections=user_collections, ) + @app.route("/collections/remove", methods=('POST',)) def remove_traits(): params = request.form @@ -216,6 +223,7 @@ def view_collection(): **collection_info ) + @app.route("/collections/change_name", methods=('POST',)) def change_collection_name(): params = request.form diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index 22941ad5..6afea715 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -11,6 +11,7 @@ import numpy as np import utility.logger logger = utility.logger.getLogger(__name__) + class CorrScatterPlot: """Page that displays a correlation scatterplot with a line fitted to it""" diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 051ac1cb..e2fe1ff4 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -172,7 +172,6 @@ class CorrelationResults: self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), key=lambda t: -abs(t[1][0]))) - # ZS: Convert min/max chromosome to an int for the location range option range_chr_as_int = None for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): @@ -348,7 +347,6 @@ class CorrelationResults: else: trait.lit_corr = 0 - def do_lit_correlation_for_all_traits(self): input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), self.this_trait.geneid) @@ -484,6 +482,7 @@ def do_bicor(this_trait_vals, target_trait_vals): return the_r, the_p + def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_api=False): results_list = [] for i, trait in enumerate(corr_results): @@ -574,6 +573,7 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap return json.dumps(results_list) + def get_header_fields(data_type, corr_method): if data_type == "ProbeSet": if corr_method == "spearman": diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 94c8931f..331cb1dc 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -233,6 +233,7 @@ class CorrelationMatrix: loadings_array.append(loadings_row) return loadings_array + def export_corr_matrix(corr_results): corr_matrix_filename = "corr_matrix_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) matrix_export_path = "{}{}.csv".format(GENERATED_TEXT_DIR, corr_matrix_filename) @@ -263,6 +264,7 @@ def export_corr_matrix(corr_results): return corr_matrix_filename, matrix_export_path + def zScore(trait_data_array): NN = len(trait_data_array[0]) if NN < 10: @@ -283,6 +285,7 @@ def zScore(trait_data_array): i += 1 return trait_data_array + def sortEigenVectors(vector): try: eigenValues = vector[0].tolist() diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 2fa90a93..ec66e59f 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -39,6 +39,7 @@ r_write_table = ro.r["write.table"] # Map the write.table function r_data_frame = ro.r["data.frame"] # Map the write.table function r_as_numeric = ro.r["as.numeric"] # Map the write.table function + class CTL: def __init__(self): logger.info("Initialization of CTL") diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index b6e85494..e743c4b3 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -17,6 +17,7 @@ db_session = scoped_session(sessionmaker(autocommit=False, Base = declarative_base() Base.query = db_session.query_property() + def init_db(): # import all modules here that might define models so that # they will be registered properly on the metadata. Otherwise @@ -27,4 +28,5 @@ def init_db(): Base.metadata.create_all(bind=engine) logger.info("Done creating all model metadata") + init_db() diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index 25e624ef..c7558ed8 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -90,6 +90,7 @@ class InfoPage: except Exception as e: pass + def process_query_results(results): info_ob = { 'info_page_name': results[0], diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 115182e4..7442dc72 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -73,6 +73,7 @@ class DoSearch: else: return None + class MrnaAssaySearch(DoSearch): """A search within an expression dataset, including mRNA, protein, SNP, but not phenotype or metabolites""" @@ -296,6 +297,7 @@ class PhenotypeSearch(DoSearch): return self.execute(query) + class GenotypeSearch(DoSearch): """A search within a genotype dataset""" @@ -339,7 +341,6 @@ class GenotypeSearch(DoSearch): from_clause = self.normalize_spaces(from_clause) - if self.search_term[0] == "*": query = (self.base_query + """WHERE Geno.Id = GenoXRef.GenoId @@ -367,6 +368,7 @@ class GenotypeSearch(DoSearch): return self.execute(self.query) + class RifSearch(MrnaAssaySearch): """Searches for traits with a Gene RIF entry including the search term.""" @@ -390,6 +392,7 @@ class RifSearch(MrnaAssaySearch): return self.execute(query) + class WikiSearch(MrnaAssaySearch): """Searches GeneWiki for traits other people have annotated""" @@ -415,6 +418,7 @@ class WikiSearch(MrnaAssaySearch): return self.execute(query) + class GoSearch(MrnaAssaySearch): """Searches for synapse-associated genes listed in the Gene Ontology.""" @@ -449,6 +453,8 @@ class GoSearch(MrnaAssaySearch): return self.execute(query) # ZS: Not sure what the best way to deal with LRS searches is + + class LrsSearch(DoSearch): """Searches for genes with a QTL within the given LRS values @@ -526,7 +532,6 @@ class LrsSearch(DoSearch): return where_clause - def run(self): self.from_clause = self.get_from_clause() @@ -550,6 +555,7 @@ class MrnaLrsSearch(LrsSearch, MrnaAssaySearch): return self.execute(self.query) + class PhenotypeLrsSearch(LrsSearch, PhenotypeSearch): for search_key in ('LRS', 'LOD'): @@ -649,6 +655,7 @@ class CisTransLrsSearch(DoSearch): return where_clause + class CisLrsSearch(CisTransLrsSearch, MrnaAssaySearch): """ Searches for genes on a particular chromosome with a cis-eQTL within the given LRS values @@ -680,6 +687,7 @@ class CisLrsSearch(CisTransLrsSearch, MrnaAssaySearch): return self.execute(self.query) + class TransLrsSearch(CisTransLrsSearch, MrnaAssaySearch): """Searches for genes on a particular chromosome with a cis-eQTL within the given LRS values @@ -744,6 +752,7 @@ class MeanSearch(MrnaAssaySearch): return self.execute(self.query) + class RangeSearch(MrnaAssaySearch): """Searches for genes with a range of expression varying between two values""" @@ -779,6 +788,7 @@ class RangeSearch(MrnaAssaySearch): return self.execute(self.query) + class PositionSearch(DoSearch): """Searches for genes/markers located within a specified range on a specified chromosome""" @@ -800,7 +810,6 @@ class PositionSearch(DoSearch): self.dataset.type, max(self.mb_min, self.mb_max)) - return where_clause def get_chr(self): @@ -819,6 +828,7 @@ class PositionSearch(DoSearch): return self.execute(self.query) + class MrnaPositionSearch(PositionSearch, MrnaAssaySearch): """Searches for genes located within a specified range on a specified chromosome""" @@ -832,6 +842,7 @@ class MrnaPositionSearch(PositionSearch, MrnaAssaySearch): return self.execute(self.query) + class GenotypePositionSearch(PositionSearch, GenotypeSearch): """Searches for genes located within a specified range on a specified chromosome""" @@ -845,6 +856,7 @@ class GenotypePositionSearch(PositionSearch, GenotypeSearch): return self.execute(self.query) + class PvalueSearch(MrnaAssaySearch): """Searches for traits with a permutationed p-value between low and high""" @@ -878,6 +890,7 @@ class PvalueSearch(MrnaAssaySearch): logger.sql(self.query) return self.execute(self.query) + class AuthorSearch(PhenotypeSearch): """Searches for phenotype traits with specified author(s)""" @@ -900,6 +913,7 @@ def is_number(s): except ValueError: return False + def get_aliases(symbol, species): if species == "mouse": symbol_string = symbol.capitalize() @@ -923,6 +937,7 @@ def get_aliases(symbol, species): return filtered_aliases + if __name__ == "__main__": # Usually this will be used as a library, but call it from the command line for testing # And it runs the code below diff --git a/wqflask/wqflask/docs.py b/wqflask/wqflask/docs.py index 207767c4..81424b9c 100644 --- a/wqflask/wqflask/docs.py +++ b/wqflask/wqflask/docs.py @@ -5,6 +5,7 @@ from flask import g from utility.logger import getLogger logger = getLogger(__name__) + class Docs: def __init__(self, entry, start_vars={}): @@ -23,7 +24,6 @@ class Docs: self.title = result[0] self.content = result[1].decode("utf-8") - self.editable = "false" # ZS: Removing option to edit to see if text still gets vandalized try: diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 95c20673..d0745ef7 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -15,6 +15,7 @@ from pprint import pformat as pf from utility.logger import getLogger logger = getLogger(__name__) + def export_search_results_csv(targs): table_data = json.loads(targs['export_data']) @@ -132,6 +133,7 @@ def export_search_results_csv(targs): return file_list + def sort_traits_by_group(trait_list=[]): traits_by_group = {} for trait in trait_list: diff --git a/wqflask/wqflask/external_tools/send_to_bnw.py b/wqflask/wqflask/external_tools/send_to_bnw.py index 9836eb9c..1556c6a0 100644 --- a/wqflask/wqflask/external_tools/send_to_bnw.py +++ b/wqflask/wqflask/external_tools/send_to_bnw.py @@ -24,6 +24,7 @@ from utility import helper_functions, corr_result_helpers import utility.logger logger = utility.logger.getLogger(__name__) + class SendToBNW: def __init__(self, start_vars): trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] diff --git a/wqflask/wqflask/external_tools/send_to_geneweaver.py b/wqflask/wqflask/external_tools/send_to_geneweaver.py index 36f1b8e5..c55c43e6 100644 --- a/wqflask/wqflask/external_tools/send_to_geneweaver.py +++ b/wqflask/wqflask/external_tools/send_to_geneweaver.py @@ -29,6 +29,7 @@ from utility import helper_functions, corr_result_helpers import utility.logger logger = utility.logger.getLogger(__name__) + class SendToGeneWeaver: def __init__(self, start_vars): trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -58,6 +59,7 @@ class SendToGeneWeaver: 'list': ",".join(trait_name_list), } + def get_trait_name_list(trait_list): name_list = [] for trait_db in trait_list: @@ -65,6 +67,7 @@ def get_trait_name_list(trait_list): return name_list + def test_chip(trait_list): final_chip_name = "" diff --git a/wqflask/wqflask/external_tools/send_to_webgestalt.py b/wqflask/wqflask/external_tools/send_to_webgestalt.py index f50eeb8b..6b78725c 100644 --- a/wqflask/wqflask/external_tools/send_to_webgestalt.py +++ b/wqflask/wqflask/external_tools/send_to_webgestalt.py @@ -29,6 +29,7 @@ from utility import helper_functions, corr_result_helpers import utility.logger logger = utility.logger.getLogger(__name__) + class SendToWebGestalt: def __init__(self, start_vars): trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -69,6 +70,7 @@ class SendToWebGestalt: else: self.hidden_vars['organism'] = "others" + def test_chip(trait_list): final_chip_name = "" @@ -113,6 +115,7 @@ def test_chip(trait_list): return chip_name + def gen_gene_id_list(trait_list): trait_name_list = [] gene_id_list = [] diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index f29c0e4d..92a65112 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -12,6 +12,7 @@ from utility.redis_tools import get_user_groups, get_group_info, save_user, crea from utility.logger import getLogger logger = getLogger(__name__) + @app.route("/groups/manage", methods=('GET', 'POST')) def manage_groups(): params = request.form if request.form else request.args @@ -21,6 +22,7 @@ def manage_groups(): admin_groups, member_groups = get_user_groups(g.user_session.user_id) return render_template("admin/group_manager.html", admin_groups=admin_groups, member_groups=member_groups) + @app.route("/groups/view", methods=('GET', 'POST')) def view_group(): params = request.form if request.form else request.args @@ -58,6 +60,7 @@ def view_group(): return render_template("admin/view_group.html", group_info=group_info, admins=admins_info, members=members_info, user_is_admin=user_is_admin, resources=resources_info) + @app.route("/groups/remove", methods=('POST',)) def remove_groups(): group_ids_to_remove = request.form['selected_group_ids'] @@ -66,6 +69,7 @@ def remove_groups(): return redirect(url_for('manage_groups')) + @app.route("/groups/remove_users", methods=('POST',)) def remove_users(): group_id = request.form['group_id'] @@ -77,6 +81,7 @@ def remove_users(): return redirect(url_for('view_group', id=group_id)) + @app.route("/groups/add_", methods=('POST',)) def add_users(user_type='members'): group_id = request.form['group_id'] @@ -89,6 +94,7 @@ def add_users(user_type='members'): return redirect(url_for('view_group', id=group_id)) + @app.route("/groups/change_name", methods=('POST',)) def change_name(): group_id = request.form['group_id'] @@ -97,6 +103,7 @@ def change_name(): return new_name + @app.route("/groups/create", methods=('GET', 'POST')) def add_or_edit_group(): params = request.form if request.form else request.args @@ -125,6 +132,8 @@ def add_or_edit_group(): return render_template("admin/create_group.html") # ZS: Will integrate this later, for now just letting users be added directly + + def send_group_invites(group_id, user_email_list=[], user_type="members"): for user_email in user_email_list: user_details = get_user_by_unique_column("email_address", user_email) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 8cb81dcc..a21dae84 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -18,6 +18,7 @@ from utility.type_checking import is_float, is_int, is_str, get_float, get_int, from utility.logger import getLogger logger = getLogger(__name__) + class GSearch: def __init__(self, kw): diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index f8ef7028..0b477446 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -14,6 +14,7 @@ Redis = Redis() logger = getLogger(__name__) + class Heatmap: def __init__(self, start_vars, temp_uuid): @@ -132,6 +133,7 @@ class Heatmap: else: self.trait_results[this_trait.name].append(float(qtl['lrs_value'])) + def gen_pheno_txt_file(samples, vals, filename): """Generates phenotype file for GEMMA""" @@ -151,6 +153,7 @@ def gen_pheno_txt_file(samples, vals, filename): values_string = "\t".join(filtered_vals_list) outfile.write(values_string) + def parse_reaper_output(gwa_filename): included_markers = [] p_values = [] diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index 9779878e..8dd1c7c0 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -4,6 +4,8 @@ from flask import Flask, g # Just return a list of dictionaries # each dictionary contains sub-dictionary + + def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): fetchFields = ['SpeciesId', 'Id', 'GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'TxEnd', 'Strand', 'GeneID', 'NM_ID', 'kgID', 'GenBankID', 'UnigenID', 'ProteinID', 'AlignID', diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 3753d1ce..5c7b81dd 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -509,9 +509,6 @@ class DisplayMappingResults: self.graphHeight = self.graphHeight + 2 * (self.NR_INDIVIDUALS + 10) * self.EACH_GENE_HEIGHT # END HaplotypeAnalyst - - - ######################### # Get the sorting column ######################### @@ -1640,7 +1637,6 @@ class DisplayMappingResults: geneYLocation + 2 *ind*self.EACH_GENE_HEIGHT + 2*self.EACH_GENE_HEIGHT*zoom)), outline=outlineColor, fill=fillColor) - COORDS = "%d, %d, %d, %d" % (geneStartPix, geneYLocation + ind * self.EACH_GENE_HEIGHT, geneEndPix + 1, (geneYLocation + ind * self.EACH_GENE_HEIGHT)) TITLE = "Strain: %s, marker (%s) \n Position %2.3f Mb." % (samplelist[k], _chr[j].name, float(txStart)) HREF = '' @@ -1663,7 +1659,6 @@ class DisplayMappingResults: geneYLocation + 7 + 2*ind*self.EACH_GENE_HEIGHT*zoom)), fill= mylineColor, width=zoom * (self.EACH_GENE_HEIGHT + 2)) - if lastGene == 0: draw_rotated_text( canvas, text="%s" % (_chr[j].name), @@ -2085,7 +2080,6 @@ class DisplayMappingResults: im_drawer.line(xy=((xLeftOffset, yZero), (xLeftOffset + plotWidth, yZero)), fill=BLACK, width=X_AXIS_THICKNESS) # Draw the X axis itself - def drawQTL(self, canvas, drawAreaHeight, gifmap, plotXScale, offset=(40, 120, 80, 10), zoom=1, startMb= None, endMb = None): im_drawer = ImageDraw.Draw(canvas) xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset @@ -2427,7 +2421,6 @@ class DisplayMappingResults: yLRS = yZero - (item / LRS_LOD_Max) * LRSHeightThresh - if 'lrs_value' in qtlresult: if self.LRS_LOD == "LOD" or self.LRS_LOD == "-logP": if qtlresult['lrs_value'] > 460 or qtlresult['lrs_value'] == 'inf': @@ -2621,7 +2614,6 @@ class DisplayMappingResults: # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) - # draw additive scale if not self.multipleInterval and self.additiveChecked: additiveScaleFont = ImageFont.truetype(font=VERDANA_FILE, size=16 * zoom) @@ -2653,7 +2645,6 @@ class DisplayMappingResults: xy=((xLeftOffset, yZero), (xLeftOffset, yTopOffset + 30 * (zoom - 1))), fill=self.LRS_COLOR, width=1 * zoom) # the blue line running up the y axis - def drawGraphBackground(self, canvas, gifmap, offset=(80, 120, 80, 50), zoom=1, startMb= None, endMb = None): # conditions # multiple Chromosome view diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index e6c78536..22a50bb8 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -8,11 +8,11 @@ from utility.tools import flat_files, PLINK_COMMAND import utility.logger logger = utility.logger.getLogger(__name__) + def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr(f"{dataset.group.name}_{this_trait.name}_") gen_pheno_txt_file(dataset, vals) - plink_command = f"{PLINK_COMMAND} --noweb --bfile {flat_files('mapping')}/{dataset.group.name} --no-pheno --no-fid --no-parents --no-sex --maf {maf} --out { TMPDIR}{plink_output_filename} --assoc " logger.debug("plink_command:", plink_command) @@ -25,6 +25,7 @@ def run_plink(this_trait, dataset, species, vals, maf): return dataset.group.markers.markers + def gen_pheno_txt_file(this_dataset, vals): """Generates phenotype file for GEMMA/PLINK""" @@ -42,6 +43,7 @@ def gen_pheno_txt_file(this_dataset, vals): this_val = vals[i] outfile.write("0 " + line[1] + " " + line[2] + " " + line[3] + " " + line[4] + " " + str(this_val) + "\n") + def gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename=''): ped_sample_list = get_samples_from_ped_file(dataset) output_file = open(f"{TMPDIR}{pheno_filename}.txt", "wb") @@ -77,6 +79,8 @@ def gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename=''): output_file.close() # get strain name from ped file in order + + def get_samples_from_ped_file(dataset): ped_file = open(f"{flat_files('mapping')}{dataset.group.name}.ped", "r") line = ped_file.readline() @@ -93,6 +97,7 @@ def get_samples_from_ped_file(dataset): return sample_list + def parse_plink_output(output_filename, species): plink_results = {} @@ -154,6 +159,8 @@ def parse_plink_output(output_filename, species): # function: convert line from str to list; # output: lineList list ####################################################### + + def build_line_list(line=""): line_list = line.strip().split(' ') # irregular number of whitespaces between columns line_list = [item for item in line_list if item != ''] diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index dd044cb0..313c40ca 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -8,6 +8,7 @@ from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR import utility.logger logger = utility.logger.getLogger(__name__) + def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boot_check, num_bootstrap, do_control, control_marker, manhattan_plot, first_run=True, output_files=None): """Generates p-values for each marker using qtlreaper""" @@ -73,6 +74,7 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo return (marker_obs, permu_vals, suggestive, significant, bootstrap_vals, [output_filename, permu_filename, bootstrap_filename]) + def gen_pheno_txt_file(samples, vals, trait_filename): """Generates phenotype file for GEMMA""" @@ -92,6 +94,7 @@ def gen_pheno_txt_file(samples, vals, trait_filename): values_string = "\t".join(filtered_vals_list) outfile.write(values_string) + def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename): included_markers = [] p_values = [] @@ -163,6 +166,7 @@ def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename): return marker_obs, permu_vals, bootstrap_vals + def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot): genotype = dataset.group.read_genotype_file(use_reaper=True) @@ -255,6 +259,7 @@ def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_da qtl_results.append(qtl) return qtl_results, json_data, perm_output, suggestive, significant, bootstrap_results + def natural_sort(marker_list): """ Function to naturally sort numbers + strings, adopted from user Mark Byers here: https://stackoverflow.com/questions/4836710/does-python-have-a-built-in-function-for-string-natural-sort diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 2bd94512..588600f5 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -16,6 +16,8 @@ import utility.logger logger = utility.logger.getLogger(__name__) # Get a trait's type (numeric, categorical, etc) from the DB + + def get_trait_data_type(trait_db_string): logger.info("get_trait_data_type"); the_query = "SELECT value FROM TraitMetadata WHERE type='trait_data_type'" @@ -133,6 +135,7 @@ def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permChec else: return process_rqtl_results(result_data_frame, dataset.group.species) + def generate_cross_from_rdata(dataset): rdata_location = locate(dataset.group.name + ".RData", "genotype/rdata") ro.r(""" @@ -143,6 +146,7 @@ def generate_cross_from_rdata(dataset): } """ % (rdata_location)) + def generate_cross_from_geno(dataset, scale_units): # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly ro.r(""" @@ -187,6 +191,7 @@ def generate_cross_from_geno(dataset, scale_units): # TODO: Need to figur } """ % (dataset.group.genofile, scale_units)) + def add_perm_strata(cross, perm_strata): col_string = 'c("the_strata")' perm_strata_string = "c(" @@ -201,6 +206,7 @@ def add_perm_strata(cross, perm_strata): return cross, strata_ob + def sanitize_rqtl_phenotype(vals): pheno_as_string = "c(" for i, val in enumerate(vals): @@ -218,6 +224,7 @@ def sanitize_rqtl_phenotype(vals): return pheno_as_string + def sanitize_rqtl_names(vals): pheno_as_string = "c(" for i, val in enumerate(vals): @@ -235,12 +242,14 @@ def sanitize_rqtl_names(vals): return pheno_as_string + def add_phenotype(cross, pheno_as_string, col_name): ro.globalenv["the_cross"] = cross ro.r('pheno <- data.frame(pull.pheno(the_cross))') ro.r('the_cross$pheno <- cbind(pheno, ' + col_name + ' = as.numeric(' + pheno_as_string + '))') return ro.r["the_cross"] + def add_categorical_covar(cross, covar_as_string, i): ro.globalenv["the_cross"] = cross logger.info("cross set"); @@ -275,12 +284,14 @@ def add_names(cross, names_as_string, col_name): ro.r('the_cross$pheno <- cbind(pheno, ' + col_name + ' = ' + names_as_string + ')') return ro.r["the_cross"] + def pull_var(var_name, cross, var_string): ro.globalenv["the_cross"] = cross ro.r(var_name + ' <- pull.pheno(the_cross, ' + var_string + ')') return ro.r[var_name] + def add_cofactors(cross, this_dataset, covariates, samples): ro.numpy2ri.activate() @@ -341,6 +352,7 @@ def add_cofactors(cross, this_dataset, covariates, samples): covars_ob = pull_var("trait_covars", cross, covar_name_string) return cross, covars_ob + def create_marker_covariates(control_marker, cross): ro.globalenv["the_cross"] = cross ro.r('genotypes <- pull.geno(the_cross)') # Get the genotype matrix @@ -358,6 +370,7 @@ def create_marker_covariates(control_marker, cross): # TODO: Create a design matrix from the marker covars for the markers in case of an F2, 4way, etc return ro.r["marker_covars"] + def process_pair_scan_results(result): pair_scan_results = [] @@ -374,6 +387,7 @@ def process_pair_scan_results(result): return pair_scan_results + def process_rqtl_perm_results(num_perm, results): perm_vals = [] for line in str(results).split("\n")[1:(num_perm + 1)]: @@ -386,6 +400,7 @@ def process_rqtl_perm_results(num_perm, results): return perm_output, suggestive, significant + def process_rqtl_results(result, species_name): # TODO: how to make this a one liner and not copy the stuff in a loop qtl_results = [] output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)] diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index f1665570..31c58083 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -45,6 +45,7 @@ from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR import utility.logger logger = utility.logger.getLogger(__name__) + class RunMapping: def __init__(self, start_vars, temp_uuid): @@ -504,6 +505,7 @@ class RunMapping: trimmed_genotype_data.append(new_genotypes) return trimmed_genotype_data + def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, score_type, transform, covariates, n_samples): with open(results_path, "w+") as output_file: output_file.write("Time/Date: " + datetime.datetime.now().strftime("%x / %X") + "\n") @@ -564,6 +566,7 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, if i < (len(markers) - 1): output_file.write("\n") + def trim_markers_for_figure(markers): if 'p_wald' in list(markers[0].keys()): score_type = 'p_wald' @@ -624,6 +627,7 @@ def trim_markers_for_figure(markers): filtered_markers.append(marker) return filtered_markers + def trim_markers_for_table(markers): if 'lod_score' in list(markers[0].keys()): sorted_markers = sorted(markers, key=lambda k: k['lod_score'], reverse=True) @@ -637,6 +641,7 @@ def trim_markers_for_table(markers): else: return sorted_markers + def write_input_for_browser(this_dataset, gwas_results, annotations): file_base = this_dataset.group.name + "_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) gwas_filename = file_base + "_GWAS" @@ -650,6 +655,7 @@ def write_input_for_browser(this_dataset, gwas_results, annotations): return [gwas_filename, annot_filename] + def geno_db_exists(this_dataset): geno_db_name = this_dataset.group.name + "Geno" try: @@ -658,6 +664,7 @@ def geno_db_exists(this_dataset): except: return "False" + def get_chr_lengths(mapping_scale, mapping_method, dataset, qtl_results): chr_lengths = [] if mapping_scale == "physic": @@ -696,6 +703,7 @@ def get_chr_lengths(mapping_scale, mapping_method, dataset, qtl_results): return chr_lengths + def get_genofile_samplelist(dataset): genofile_samplelist = [] @@ -706,6 +714,7 @@ def get_genofile_samplelist(dataset): return genofile_samplelist + def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): perm_strata_strings = [] for sample in used_samples: diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py index 8abd6516..d7c9ef95 100644 --- a/wqflask/wqflask/model.py +++ b/wqflask/wqflask/model.py @@ -14,6 +14,7 @@ from sqlalchemy.orm import relationship from wqflask.database import Base, init_db + class User(Base): __tablename__ = "user" id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) @@ -63,7 +64,6 @@ class User(Base): print("Couldn't display_num_collections:", why) return "" - def get_collection_by_name(self, collection_name): try: collect = self.user_collections.filter_by(name=collection_name).first() @@ -83,7 +83,6 @@ class User(Base): def login_count(self): return self.logins.filter_by(successful=True).count() - @property def confirmed_at(self): if self.confirmed: @@ -116,6 +115,7 @@ class User(Base): except IndexError: return None + class Login(Base): __tablename__ = "login" id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) @@ -134,6 +134,7 @@ class Login(Base): ################################################################################################## + class UserCollection(Base): __tablename__ = "user_collection" id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) @@ -158,12 +159,14 @@ class UserCollection(Base): def members_as_set(self): return set(json.loads(self.members)) + def display_collapsible(number): if number: return number else: return "" + def user_uuid(): """Unique cookie for a user""" user_uuid = request.cookies.get('user_uuid') diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index 93785a3a..e089643e 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -27,6 +27,7 @@ from utility import helper_functions from utility import corr_result_helpers from utility.tools import GN2_BRANCH_URL + class NetworkGraph: def __init__(self, start_vars): diff --git a/wqflask/wqflask/news.py b/wqflask/wqflask/news.py index 861a93f2..e262dd51 100644 --- a/wqflask/wqflask/news.py +++ b/wqflask/wqflask/news.py @@ -1,5 +1,6 @@ from flask import g + class News: def __init__(self): diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index dfd374e2..6b836e20 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -24,6 +24,7 @@ from pprint import pformat as pf from utility.logger import getLogger logger = getLogger(__name__) + def parse(pstring): """ diff --git a/wqflask/wqflask/pbkdf2.py b/wqflask/wqflask/pbkdf2.py index 6346df03..1a965fc5 100644 --- a/wqflask/wqflask/pbkdf2.py +++ b/wqflask/wqflask/pbkdf2.py @@ -4,6 +4,8 @@ from werkzeug.security import safe_str_cmp as ssc # Replace this because it just wraps around Python3's internal # functions. Added this during migration. + + def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc="sha1"): """Wrapper function of python's hashlib.pbkdf2_hmac. """ diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index a1fe0f8f..4591710c 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -11,6 +11,7 @@ from utility.redis_tools import get_resource_info, get_group_info, get_groups_li from utility.logger import getLogger logger = getLogger(__name__) + @app.route("/resources/manage", methods=('GET', 'POST')) def manage_resource(): params = request.form if request.form else request.args @@ -40,6 +41,7 @@ def manage_resource(): return render_template("admin/manage_resource.html", owner_name=owner_display_name, resource_id=resource_id, resource_info=resource_info, default_mask=default_mask, group_masks=group_masks_with_names, admin_status=admin_status) + @app.route("/search_for_users", methods=('POST',)) def search_for_user(): params = request.form @@ -49,6 +51,7 @@ def search_for_user(): return json.dumps(user_list) + @app.route("/search_for_groups", methods=('POST',)) def search_for_groups(): params = request.form @@ -65,6 +68,7 @@ def search_for_groups(): return json.dumps(group_list) + @app.route("/resources/change_owner", methods=('POST',)) def change_owner(): resource_id = request.form['resource_id'] @@ -81,6 +85,7 @@ def change_owner(): else: return render_template("admin/change_resource_owner.html", resource_id=resource_id) + @app.route("/resources/change_default_privileges", methods=('POST',)) def change_default_privileges(): resource_id = request.form['resource_id'] @@ -99,6 +104,7 @@ def change_default_privileges(): else: return redirect(url_for("no_access_page")) + @app.route("/resources/add_group", methods=('POST',)) def add_group_to_resource(): resource_id = request.form['resource_id'] @@ -125,6 +131,7 @@ def add_group_to_resource(): else: return redirect(url_for("no_access_page")) + def get_group_names(group_masks): group_masks_with_names = {} for group_id, group_mask in list(group_masks.items()): diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 16eb1864..ed5f9bad 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -24,6 +24,7 @@ from utility.type_checking import is_str from utility.logger import getLogger logger = getLogger(__name__) + class SearchResultPage: #maxReturn = 3000 @@ -270,6 +271,7 @@ class SearchResultPage: else: return None + def get_GO_symbols(a_search): query = """SELECT genes FROM GORef @@ -287,6 +289,7 @@ def get_GO_symbols(a_search): return new_terms + def insert_newlines(string, every=64): """ This is because it is seemingly impossible to change the width of the description column, so I'm just manually adding line breaks """ lines = [] @@ -294,6 +297,7 @@ def insert_newlines(string, every=64): lines.append(string[i:i + every]) return '\n'.join(lines) + def get_aliases(symbol_list, species): updated_symbols = [] diff --git a/wqflask/wqflask/send_mail.py b/wqflask/wqflask/send_mail.py index 86e8a558..299c866a 100644 --- a/wqflask/wqflask/send_mail.py +++ b/wqflask/wqflask/send_mail.py @@ -8,10 +8,12 @@ Redis = StrictRedis() import mailer + def timestamp(): ts = datetime.datetime.utcnow() return ts.isoformat() + def main(): while True: print("I'm alive!") @@ -31,7 +33,6 @@ def main(): process_message(msg) - def process_message(msg): msg = json.loads(msg) diff --git a/wqflask/wqflask/server_side.py b/wqflask/wqflask/server_side.py index 48761fa0..8b3a4faa 100644 --- a/wqflask/wqflask/server_side.py +++ b/wqflask/wqflask/server_side.py @@ -1,7 +1,6 @@ # handles server side table processing - class ServerSideTable: """ This class is used to do server-side processing diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 388f831f..496dee57 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -8,6 +8,7 @@ from pprint import pformat as pf from utility import Plot from utility import Bunch + class SampleList: def __init__(self, dataset, diff --git a/wqflask/wqflask/show_trait/export_trait_data.py b/wqflask/wqflask/show_trait/export_trait_data.py index 379b746c..f0fcd27d 100644 --- a/wqflask/wqflask/show_trait/export_trait_data.py +++ b/wqflask/wqflask/show_trait/export_trait_data.py @@ -5,6 +5,7 @@ from functools import cmp_to_key from base.trait import create_trait from base import data_set + def export_sample_table(targs): sample_data = json.loads(targs['export_data']) @@ -28,6 +29,7 @@ def export_sample_table(targs): return trait_name, final_sample_data + def get_export_metadata(trait_id, dataset_name): dataset = data_set.create_dataset(dataset_name) this_trait = create_trait(dataset=dataset, @@ -64,6 +66,7 @@ def dict_to_sorted_list(dictionary): sorted_values = [item[1] for item in sorted_list] return sorted_values + def cmp_samples(a, b): if b[0] == 'name': return 1 diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index 7a0df94b..e5c67165 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -9,6 +9,7 @@ logger = getLogger(__name__) from base import species from base import webqtlConfig + class SnpBrowser: def __init__(self, start_vars): @@ -472,7 +473,6 @@ class SnpBrowser: base_color_dict = {"A": "#C33232", "C": "#1569C7", "T": "#CFCF32", "G": "#32C332", "t": "#FF6", "c": "#5CB3FF", "a": "#F66", "g": "#CF9", ":": "#FFFFFF", "-": "#FFFFFF", "?": "#FFFFFF"} - the_bases = [] for j, item in enumerate(allele_value_list): if item and isinstance(item, str): @@ -641,6 +641,7 @@ class SnpBrowser: # for i in range(n_click): # href = url_for('snp_browser', first_run="false", chosen_strains_mouse=self.chosen_strains_mouse, chosen_strains_rat=self.chosen_strains_rat, variant=self.variant_type, species=self.species_name, gene_name=self.gene_name, chr=self.chr, start_mb=self.start_mb, end_mb=self.end_mb, limit_strains=self.limit_strains, domain=self.domain, function=self.function, criteria=self.criteria, score=self.score, diff_alleles=self.diff_alleles) + def get_browser_sample_lists(species_id=1): strain_lists = {} mouse_strain_list = [] @@ -660,6 +661,7 @@ def get_browser_sample_lists(species_id=1): return strain_lists + def get_header_list(variant_type, strains, species=None, empty_columns=None): if species == "Mouse": strain_list = strains['mouse'] @@ -715,6 +717,7 @@ def get_header_list(variant_type, strains, species=None, empty_columns=None): return header_fields, empty_field_count, header_data_names + def get_effect_details_by_category(effect_name=None, effect_value=None): gene_list = [] transcript_list = [] @@ -764,6 +767,7 @@ def get_effect_details_by_category(effect_name=None, effect_value=None): return [gene_list, transcript_list, exon_list, function_list, function_detail_list] + def get_effect_info(effect_list): domain = "" effect_detail_list = [] @@ -836,6 +840,7 @@ def get_effect_info(effect_list): return effect_info_dict + def get_gene_id(species_id, gene_name): query = """ SELECT @@ -853,6 +858,7 @@ def get_gene_id(species_id, gene_name): else: return "" + def get_gene_id_name_dict(species_id, gene_name_list): gene_id_name_dict = {} if len(gene_name_list) == 0: @@ -877,6 +883,7 @@ def get_gene_id_name_dict(species_id, gene_name_list): return gene_id_name_dict + def check_if_in_gene(species_id, chr, mb): if species_id != 0: # ZS: Check if this is necessary query = """SELECT geneId, geneSymbol diff --git a/wqflask/wqflask/submit_bnw.py b/wqflask/wqflask/submit_bnw.py index 4ad6f9e3..b21a88cc 100644 --- a/wqflask/wqflask/submit_bnw.py +++ b/wqflask/wqflask/submit_bnw.py @@ -5,5 +5,6 @@ from utility import helper_functions import utility.logger logger = utility.logger.getLogger(__name__) + def get_bnw_input(start_vars): logger.debug("BNW VARS:", start_vars) diff --git a/wqflask/wqflask/update_search_results.py b/wqflask/wqflask/update_search_results.py index 08b4f9f5..07073d6a 100644 --- a/wqflask/wqflask/update_search_results.py +++ b/wqflask/wqflask/update_search_results.py @@ -10,6 +10,7 @@ from utility.benchmark import Bench from utility.logger import getLogger logger = getLogger(__name__) + class GSearch: def __init__(self, kw): diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index 3f5b43ee..2a2f8484 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -29,9 +29,11 @@ from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_AL THREE_DAYS = 60 * 60 * 24 * 3 + def timestamp(): return datetime.datetime.utcnow().isoformat() + def basic_info(): return dict(timestamp=timestamp(), ip_address=request.remote_addr, @@ -54,6 +56,7 @@ def encode_password(pass_gen_fields, unencrypted_password): return pass_gen_fields + def set_password(password): pass_gen_fields = { "unencrypted_password": password, @@ -71,6 +74,7 @@ def set_password(password): return encoded_password + def get_signed_session_id(user): session_id = str(uuid.uuid4()) @@ -109,6 +113,7 @@ def get_signed_session_id(user): return session_id_signed + def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): """Send an E-mail through SMTP_CONNECT host. If SMTP_USERNAME is not 'UNKNOWN' TLS is used @@ -125,6 +130,7 @@ def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): server.quit() logger.info("Successfully sent email to " + toaddr) + def send_verification_email(user_details, template_name="email/user_verification.txt", key_prefix="verification_code", subject = "GeneNetwork e-mail verification"): verification_code = str(uuid.uuid4()) key = key_prefix + ":" + verification_code @@ -139,12 +145,14 @@ def send_verification_email(user_details, template_name="email/user_verification send_email(recipient, subject, body) return {"recipient": recipient, "subject": subject, "body": body} + def send_invitation_email(user_email, temp_password, template_name="email/user_invitation.txt", subject= "You've been added to a GeneNetwork user group"): recipient = user_email body = render_template(template_name, temp_password) send_email(recipient, subject, body) return {"recipient": recipient, "subject": subject, "body": body} + @app.route("/manage/verify_email") def verify_email(): if 'code' in request.args: @@ -160,6 +168,7 @@ def verify_email(): else: flash("Invalid code: Password reset code does not exist or might have expired!", "error") + @app.route("/n/login", methods=('GET', 'POST')) def login(): params = request.form if request.form else request.args @@ -232,6 +241,7 @@ def login(): return response + @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) def github_oauth2(): from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET, GITHUB_AUTH_URL @@ -264,12 +274,14 @@ def github_oauth2(): url = "/n/login?type=github&uid=" + user_details["user_id"] return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, headers={'Authorization': 'token ' + access_token}).content return json.loads(result) + @app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) def orcid_oauth2(): from uuid import uuid4 @@ -308,6 +320,7 @@ def orcid_oauth2(): flash("There was an error getting code from ORCID") return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, headers={'Authorization': 'token ' + access_token}).content @@ -325,6 +338,7 @@ def logout(): response.set_cookie(UserSession.user_cookie_name, '', expires=0) return response + @app.route("/n/forgot_password", methods=['GET']) def forgot_password(): """Entry point for forgotten password""" @@ -333,6 +347,7 @@ def forgot_password(): print("ERRORS: ", errors) return render_template("new_security/forgot_password.html", errors=errors) + def send_forgot_password_email(verification_email): from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -365,6 +380,7 @@ def send_forgot_password_email(verification_email): return subject + @app.route("/n/forgot_password_submit", methods=('POST',)) def forgot_password_submit(): """When a forgotten password form is submitted we get here""" @@ -386,6 +402,7 @@ def forgot_password_submit(): flash("You MUST provide an email", "alert-danger") return redirect(url_for("forgot_password")) + @app.route("/n/password_reset", methods=['GET']) def password_reset(): """Entry point after user clicks link in E-mail""" @@ -405,6 +422,7 @@ def password_reset(): else: return redirect(url_for("login")) + @app.route("/n/password_reset_step2", methods=('POST',)) def password_reset_step2(): """Handle confirmation E-mail for password reset""" @@ -422,6 +440,7 @@ def password_reset_step2(): flash("Password changed successfully. You can now sign in.", "alert-info") return redirect(url_for('login')) + def register_user(params): thank_you_mode = False errors = [] @@ -461,6 +480,7 @@ def register_user(params): return errors + @app.route("/n/register", methods=('GET', 'POST')) def register(): errors = [] @@ -478,6 +498,7 @@ def register(): return render_template("new_security/register_user.html", values=params, errors=errors) + @app.errorhandler(401) def unauthorized(error): return redirect(url_for('login')) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 013920f9..a9bd65e6 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -151,6 +151,7 @@ def verify_cookie(cookie): assert the_signature == actual_hmac_creation(the_uuid), "Uh-oh, someone tampering with the cookie?" return the_uuid + def create_signed_cookie(): the_uuid = str(uuid.uuid4()) signature = actual_hmac_creation(the_uuid) @@ -158,6 +159,7 @@ def create_signed_cookie(): logger.debug("uuid_signed:", uuid_signed) return the_uuid, uuid_signed + class UserSession: """Logged in user handling""" @@ -341,6 +343,7 @@ class UserSession: Redis.delete(self.cookie_name) logger.debug("At end of delete_session") + @app.before_request def get_cookie(): logger.info("@app.before_request get cookie") @@ -348,16 +351,20 @@ def get_cookie(): g.cookie_session = AnonUser() # @app.after_request + + def set_cookie(response): if not request.cookies.get(g.cookie_session.cookie_name): response.set_cookie(g.cookie_session.cookie_name, g.cookie_session.cookie) return response + class UsersManager: def __init__(self): self.users = model.User.query.all() logger.debug("Users are:", self.users) + class UserManager: def __init__(self, kw): self.user_id = kw['user_id'] @@ -419,6 +426,7 @@ class RegisterUser: self.user.registration_info = json.dumps(basic_info(), sort_keys=True) save_user(self.user.__dict__, self.user.user_id) + def set_password(password, user): pwfields = Bunch() @@ -476,6 +484,7 @@ class VerificationEmail: verification_code=verification_code) send_email(to, subject, body) + class ForgotPasswordEmail(VerificationEmail): template_name = "email/forgot_password.txt" key_prefix = "forgot_password_code" @@ -496,7 +505,6 @@ class ForgotPasswordEmail(VerificationEmail): save_verification_code(toaddr, verification_code) - subject = self.subject body = render_template( self.template_name, @@ -530,6 +538,8 @@ def basic_info(): user_agent=request.headers.get('User-Agent')) # @app.route("/manage/verify_email") + + def verify_email(): user = DecodeUser(VerificationEmail.key_prefix).user user.confirmed = json.dumps(basic_info(), sort_keys=True) @@ -544,6 +554,8 @@ def verify_email(): return response # @app.route("/n/password_reset", methods=['GET']) + + def password_reset(): """Entry point after user clicks link in E-mail""" logger.debug("in password_reset request.url is:", request.url) @@ -568,6 +580,8 @@ def password_reset(): return redirect(url_for("login")) # @app.route("/n/password_reset_step2", methods=('POST',)) + + def password_reset_step2(): """Handle confirmation E-mail for password reset""" logger.debug("in password_reset request.url is:", request.url) @@ -577,7 +591,6 @@ def password_reset_step2(): logger.debug("locals are:", locals()) - user = Bunch() password = request.form['password'] set_password(password, user) @@ -589,6 +602,7 @@ def password_reset_step2(): return response + class DecodeUser: def __init__(self, code_prefix): @@ -612,6 +626,8 @@ class DecodeUser: return model.User.query.get(data['id']) # @app.route("/n/login", methods=('GET', 'POST')) + + def login(): lu = LoginUser() login_type = request.args.get("type") @@ -622,6 +638,8 @@ def login(): return lu.standard_login() # @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) + + def github_oauth2(): from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET code = request.args.get("code") @@ -646,6 +664,8 @@ def github_oauth2(): return redirect(url) # @app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) + + def orcid_oauth2(): from uuid import uuid4 from utility.tools import ORCID_CLIENT_ID, ORCID_CLIENT_SECRET, ORCID_TOKEN_URL, ORCID_AUTH_URL @@ -673,11 +693,13 @@ def orcid_oauth2(): flash("There was an error getting code from ORCID") return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, params={"access_token": access_token}) return result.json() + class LoginUser: remember_time = 60 * 60 * 24 * 30 # One month in seconds @@ -814,6 +836,8 @@ class LoginUser: db_session.commit() # @app.route("/n/logout") + + def logout(): logger.debug("Logging out...") UserSession().delete_session() @@ -833,6 +857,8 @@ def forgot_password(): return render_template("new_security/forgot_password.html", errors=errors) # @app.route("/n/forgot_password_submit", methods=('POST',)) + + def forgot_password_submit(): """When a forgotten password form is submitted we get here""" params = request.form @@ -853,10 +879,12 @@ def forgot_password_submit(): flash("You MUST provide an email", "alert-danger") return redirect(url_for("forgot_password")) + @app.errorhandler(401) def unauthorized(error): return redirect(url_for('login')) + def is_redis_available(): try: Redis.ping() @@ -922,7 +950,6 @@ def register(): params = None errors = None - params = request.form if request.form else request.args params = params.to_dict(flat=True) @@ -952,6 +979,7 @@ def url_for_hmac(endpoint, **values): combiner = "?" return url + combiner + "hm=" + hm + def data_hmac(stringy): """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with""" return stringy + ":" + actual_hmac_creation(stringy) @@ -974,6 +1002,7 @@ def verify_url_hmac(url): assert hm == hmac, "Unexpected url (stage 3)" + def actual_hmac_creation(stringy): """Helper function to create the actual hmac""" @@ -986,6 +1015,7 @@ def actual_hmac_creation(stringy): hm = hm[:20] return hm + app.jinja_env.globals.update(url_for_hmac=url_for_hmac, data_hmac=data_hmac) @@ -998,6 +1028,7 @@ app.jinja_env.globals.update(url_for_hmac=url_for_hmac, # Body=body)) # Redis.rpush("mail_queue", msg) + def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): """Send an E-mail through SMTP_CONNECT host. If SMTP_USERNAME is not 'UNKNOWN' TLS is used @@ -1020,6 +1051,7 @@ def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): server.quit() logger.info("Successfully sent email to " + toaddr) + class GroupsManager: def __init__(self, kw): self.datasets = create_datasets_list() diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 78db7bd2..f0f0d60c 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -20,6 +20,7 @@ logger = getLogger(__name__) THREE_DAYS = 60 * 60 * 24 * 3 THIRTY_DAYS = 60 * 60 * 24 * 30 + @app.before_request def get_user_session(): logger.info("@app.before_request get_session") @@ -30,6 +31,7 @@ def get_user_session(): response.set_cookie('session_id_v2', '', expires=0) return response + @app.after_request def set_user_session(response): if hasattr(g, 'user_session'): @@ -37,6 +39,7 @@ def set_user_session(response): response.set_cookie(g.user_session.cookie_name, g.user_session.cookie) return response + def verify_cookie(cookie): the_uuid, separator, the_signature = cookie.partition(':') assert len(the_uuid) == 36, "Is session_id a uuid?" @@ -44,6 +47,7 @@ def verify_cookie(cookie): assert the_signature == hmac.hmac_creation(the_uuid), "Uh-oh, someone tampering with the cookie?" return the_uuid + def create_signed_cookie(): the_uuid = str(uuid.uuid4()) signature = hmac.hmac_creation(the_uuid) @@ -51,6 +55,7 @@ def create_signed_cookie(): logger.debug("uuid_signed:", uuid_signed) return the_uuid, uuid_signed + @app.route("/user/manage", methods=('GET', 'POST')) def manage_user(): params = request.form if request.form else request.args @@ -63,6 +68,7 @@ def manage_user(): return render_template("admin/manage_user.html", user_details=user_details) + class UserSession: """Logged in user handling""" -- cgit v1.2.3 From bd702e59d7a426fe351d34367bf824683c655696 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 30 Apr 2021 12:52:55 +0300 Subject: autopep8: Fix W291, W292, W293, W391 --- wqflask/base/mrna_assay_tissue_data.py | 2 +- wqflask/maintenance/convert_dryad_to_bimbam.py | 2 +- wqflask/maintenance/convert_geno_to_bimbam.py | 4 +-- .../maintenance/generate_kinship_from_bimbam.py | 8 ++--- wqflask/maintenance/geno_to_json.py | 34 +++++++++--------- wqflask/tests/unit/wqflask/api/test_correlation.py | 4 +-- .../correlation/test_correlation_functions.py | 2 +- .../marker_regression/test_qtlreaper_mapping.py | 4 +-- .../wqflask/marker_regression/test_rqtl_mapping.py | 7 ---- .../unit/wqflask/snp_browser/test_snp_browser.py | 8 ++--- wqflask/tests/unit/wqflask/test_server_side.py | 4 +-- wqflask/utility/__init__.py | 2 -- wqflask/utility/genofile_parser.py | 1 - wqflask/wqflask/__init__.py | 2 +- wqflask/wqflask/api/mapping.py | 2 -- wqflask/wqflask/api/router.py | 14 ++++---- wqflask/wqflask/collect.py | 1 - .../comparison_bar_chart/comparison_bar_chart.py | 7 ++-- wqflask/wqflask/correlation/corr_scatter_plot.py | 4 +-- wqflask/wqflask/correlation/show_corr_results.py | 1 - .../wqflask/correlation_matrix/show_corr_matrix.py | 2 +- wqflask/wqflask/ctl/ctl_analysis.py | 3 +- wqflask/wqflask/db_info.py | 2 -- wqflask/wqflask/docs.py | 2 +- wqflask/wqflask/export_traits.py | 4 +-- .../wqflask/external_tools/send_to_webgestalt.py | 2 +- wqflask/wqflask/heatmap/heatmap.py | 2 +- wqflask/wqflask/interval_analyst/GeneUtil.py | 24 ++++++------- .../wqflask/marker_regression/qtlreaper_mapping.py | 10 +++--- wqflask/wqflask/marker_regression/rqtl_mapping.py | 2 +- wqflask/wqflask/resource_manager.py | 2 +- wqflask/wqflask/search_results.py | 1 - wqflask/wqflask/server_side.py | 2 +- wqflask/wqflask/snp_browser/snp_browser.py | 7 ++-- wqflask/wqflask/user_login.py | 42 +++++++++++----------- wqflask/wqflask/user_session.py | 2 -- 36 files changed, 98 insertions(+), 124 deletions(-) (limited to 'wqflask/maintenance/convert_dryad_to_bimbam.py') diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index 9bb29664..882ae911 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -74,7 +74,7 @@ class MrnaAssayTissueData: def get_symbol_values_pairs(self): id_list = [self.data[symbol].data_id for symbol in self.data] - + symbol_values_dict = {} if len(id_list) > 0: diff --git a/wqflask/maintenance/convert_dryad_to_bimbam.py b/wqflask/maintenance/convert_dryad_to_bimbam.py index e417c280..18fbb8a1 100644 --- a/wqflask/maintenance/convert_dryad_to_bimbam.py +++ b/wqflask/maintenance/convert_dryad_to_bimbam.py @@ -52,7 +52,7 @@ def read_dryad_file(filename): # this_row.append(line.split(" ")[i+2]) # print("row: " + str(i)) # geno_rows.append(this_row) - # + # # return geno_rows diff --git a/wqflask/maintenance/convert_geno_to_bimbam.py b/wqflask/maintenance/convert_geno_to_bimbam.py index a2ede1f9..c5af1ca6 100644 --- a/wqflask/maintenance/convert_geno_to_bimbam.py +++ b/wqflask/maintenance/convert_geno_to_bimbam.py @@ -91,7 +91,7 @@ class ConvertGenoFile: self.markers.append(this_marker.__dict__) - self.write_to_bimbam() + self.write_to_bimbam() def write_to_bimbam(self): with open(self.output_files[0], "w") as geno_fh: @@ -126,7 +126,7 @@ class ConvertGenoFile: self.sample_list = row_contents[3:] else: self.sample_list = row_contents[2:] - + def process_rows(self): for self.latest_row_pos, row in enumerate(self.input_fh): self.latest_row_value = row diff --git a/wqflask/maintenance/generate_kinship_from_bimbam.py b/wqflask/maintenance/generate_kinship_from_bimbam.py index bed634fa..cd39fceb 100644 --- a/wqflask/maintenance/generate_kinship_from_bimbam.py +++ b/wqflask/maintenance/generate_kinship_from_bimbam.py @@ -19,7 +19,7 @@ class GenerateKinshipMatrices: self.group_name = group_name self.geno_file = geno_file self.pheno_file = pheno_file - + def generate_kinship(self): gemma_command = "/gnu/store/xhzgjr0jvakxv6h3blj8z496xjig69b0-profile/bin/gemma -g " + self.geno_file + \ " -p " + self.pheno_file + \ @@ -56,11 +56,11 @@ class GenerateKinshipMatrices: print(" Column is:", convertob.latest_col_value) print(" Row is:", convertob.latest_row_value) break - - + + if __name__ == "__main__": Geno_Directory = """/export/local/home/zas1024/genotype_files/genotype/""" Bimbam_Directory = """/export/local/home/zas1024/genotype_files/genotype/bimbam/""" GenerateKinshipMatrices.process_all(Geno_Directory, Bimbam_Directory) - + # ./gemma -g /home/zas1024/genotype_files/genotype/bimbam/BXD_geno.txt -p /home/zas1024/genotype_files/genotype/bimbam/BXD_pheno.txt -gk 1 -o BXD diff --git a/wqflask/maintenance/geno_to_json.py b/wqflask/maintenance/geno_to_json.py index 76a0fc98..27eb6553 100644 --- a/wqflask/maintenance/geno_to_json.py +++ b/wqflask/maintenance/geno_to_json.py @@ -29,7 +29,7 @@ from pprint import pformat as pf class EmptyConfigurations(Exception): pass - + class Marker: def __init__(self): self.name = None @@ -42,20 +42,20 @@ class Marker: class ConvertGenoFile: def __init__(self, input_file, output_file): - + self.input_file = input_file self.output_file = output_file - + self.mb_exists = False self.cm_exists = False self.markers = [] - + self.latest_row_pos = None self.latest_col_pos = None - + self.latest_row_value = None self.latest_col_value = None - + def convert(self): self.haplotype_notation = { @@ -64,16 +64,16 @@ class ConvertGenoFile: '@het': "0.5", '@unk': "NA" } - + self.configurations = {} #self.skipped_cols = 3 - + # if self.input_file.endswith(".geno.gz"): # print("self.input_file: ", self.input_file) # self.input_fh = gzip.open(self.input_file) # else: self.input_fh = open(self.input_file) - + with open(self.output_file, "w") as self.output_fh: # if self.file_type == "geno": self.process_csv() @@ -105,22 +105,22 @@ class ConvertGenoFile: self.configurations[genotype.upper()]) else: this_marker.genotypes.append("NA") - - #print("this_marker is:", pf(this_marker.__dict__)) + + #print("this_marker is:", pf(this_marker.__dict__)) # if this_marker.chr == "14": self.markers.append(this_marker.__dict__) with open(self.output_file, 'w') as fh: json.dump(self.markers, fh, indent=" ", sort_keys=True) - + # print('configurations:', str(configurations)) #self.latest_col_pos = item_count + self.skipped_cols #self.latest_col_value = item - + # if item_count != 0: # self.output_fh.write(" ") # self.output_fh.write(self.configurations[item.upper()]) - + # self.output_fh.write("\n") def process_rows(self): @@ -176,12 +176,12 @@ class ConvertGenoFile: print(" Column is:", convertob.latest_col_value) print(" Row is:", convertob.latest_row_value) break - + # def process_snps_file(cls, snps_file, new_directory): # output_file = os.path.join(new_directory, "mouse_families.json") # print("%s -> %s" % (snps_file, output_file)) # convertob = ConvertGenoFile(input_file, output_file) - + if __name__ == "__main__": Old_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype""" @@ -192,5 +192,5 @@ if __name__ == "__main__": # convertob.convert() ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory) # ConvertGenoFiles(Geno_Directory) - + #process_csv(Input_File, Output_File) diff --git a/wqflask/tests/unit/wqflask/api/test_correlation.py b/wqflask/tests/unit/wqflask/api/test_correlation.py index bd99838d..34ffa9ef 100644 --- a/wqflask/tests/unit/wqflask/api/test_correlation.py +++ b/wqflask/tests/unit/wqflask/api/test_correlation.py @@ -105,9 +105,9 @@ class TestCorrelations(unittest.TestCase): target_dataset = AttributeSetter({"group": group}) target_vals = [3.4, 6.2, 4.1, 3.4, 1.2, 5.6] - trait_data = {"S1": AttributeSetter({"value": 2.3}), "S2": AttributeSetter({"value": 1.1}), + trait_data = {"S1": AttributeSetter({"value": 2.3}), "S2": AttributeSetter({"value": 1.1}), "S3": AttributeSetter( - {"value": 6.3}), "S4": AttributeSetter({"value": 3.6}), "S5": AttributeSetter({"value": 4.1}), + {"value": 6.3}), "S4": AttributeSetter({"value": 3.6}), "S5": AttributeSetter({"value": 4.1}), "S6": AttributeSetter({"value": 5.0})} this_trait = AttributeSetter({"data": trait_data}) mock_normalize.return_value = ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0], diff --git a/wqflask/tests/unit/wqflask/correlation/test_correlation_functions.py b/wqflask/tests/unit/wqflask/correlation/test_correlation_functions.py index 44d2e0fc..2bbeab1f 100644 --- a/wqflask/tests/unit/wqflask/correlation/test_correlation_functions.py +++ b/wqflask/tests/unit/wqflask/correlation/test_correlation_functions.py @@ -5,7 +5,7 @@ from wqflask.correlation.correlation_functions import cal_zero_order_corr_for_ti class TestCorrelationFunctions(unittest.TestCase): - + @mock.patch("wqflask.correlation.correlation_functions.MrnaAssayTissueData") def test_get_trait_symbol_and_tissue_values(self, mock_class): """test for getting trait symbol and tissue_values""" diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py index c762982b..1198740d 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py @@ -8,7 +8,7 @@ from wqflask.marker_regression.qtlreaper_mapping import gen_pheno_txt_file class TestQtlReaperMapping(unittest.TestCase): @mock.patch("wqflask.marker_regression.qtlreaper_mapping.TEMPDIR", "/home/user/data") - def test_gen_pheno_txt_file(self): + def test_gen_pheno_txt_file(self): vals = ["V1", "x", "V4", "V3","x"] samples = ["S1", "S2", "S3", "S4","S5"] trait_filename = "trait_file" @@ -21,5 +21,3 @@ class TestQtlReaperMapping(unittest.TestCase): 'S1\tS3\tS4\n'), mock.call('T1\t'), mock.call('V1\tV4\tV3')] filehandler.write.assert_has_calls(write_calls) - - diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 6996c275..d69a20d3 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -40,10 +40,3 @@ class TestRqtlMapping(unittest.TestCase): expected_sanitized_name = "c('f',NA,'r',NA,NA)" results = sanitize_rqtl_names(vals) self.assertEqual(expected_sanitized_name, results) - - - - - - - diff --git a/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py b/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py index ce3e7b83..8823e1fc 100644 --- a/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py +++ b/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py @@ -21,9 +21,9 @@ class TestSnpBrowser(unittest.TestCase): "transcript": "false", "exon": "false", "domain_2": "true", "function": "false", "function_details": "true"} strains = {"mouse": ["S1", "S2", "S3", "S4", "S5"], "rat": []} expected_results = ([['Index', 'SNP ID', 'Chr', 'Mb', 'Alleles', 'ConScore', - 'Domain 1', 'Domain 2', 'Details'], - ['S1', 'S2', 'S3', 'S4', 'S5']], 5, - ['index', 'snp_name', 'chr', 'mb_formatted', 'alleles', + 'Domain 1', 'Domain 2', 'Details'], + ['S1', 'S2', 'S3', 'S4', 'S5']], 5, + ['index', 'snp_name', 'chr', 'mb_formatted', 'alleles', 'conservation_score', 'domain_1', 'domain_2', 'function_details', 'S1', 'S2', 'S3', 'S4', 'S5']) @@ -33,7 +33,7 @@ class TestSnpBrowser(unittest.TestCase): variant_type="InDel", strains=strains, species="rat", empty_columns=[]) expected_results_with_indel = ( ['Index', 'ID', 'Type', 'InDel Chr', 'Mb Start', - 'Mb End', 'Strand', 'Size', 'Sequence', 'Source'], 0, + 'Mb End', 'Strand', 'Size', 'Sequence', 'Source'], 0, ['index', 'indel_name', 'indel_type', 'indel_chr', 'indel_mb_s', 'indel_mb_e', 'indel_strand', 'indel_size', 'indel_sequence', 'source_name']) diff --git a/wqflask/tests/unit/wqflask/test_server_side.py b/wqflask/tests/unit/wqflask/test_server_side.py index 69977146..9d988aea 100644 --- a/wqflask/tests/unit/wqflask/test_server_side.py +++ b/wqflask/tests/unit/wqflask/test_server_side.py @@ -17,8 +17,8 @@ class TestServerSideTableTests(unittest.TestCase): def test_get_page(self): rows_count = 3 table_rows = [ - {'first': 'd', 'second': 4, 'third': 'zz'}, - {'first': 'b', 'second': 2, 'third': 'aa'}, + {'first': 'd', 'second': 4, 'third': 'zz'}, + {'first': 'b', 'second': 2, 'third': 'aa'}, {'first': 'c', 'second': 1, 'third': 'ss'}, ] headers = ['first', 'second', 'third'] diff --git a/wqflask/utility/__init__.py b/wqflask/utility/__init__.py index d540c96e..816bc4df 100644 --- a/wqflask/utility/__init__.py +++ b/wqflask/utility/__init__.py @@ -33,5 +33,3 @@ class Struct: def __repr__(self): return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v) in list(self.__dict__.items()))) - - diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py index 09100bd9..c0629b5d 100644 --- a/wqflask/utility/genofile_parser.py +++ b/wqflask/utility/genofile_parser.py @@ -98,4 +98,3 @@ class ConvertGenoFile: print("WARNING:", genotype.upper()) this_marker.genotypes.append("NA") self.markers.append(this_marker.__dict__) - diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 712517a3..a2bf3085 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -50,4 +50,4 @@ from wqflask import db_info from wqflask import user_login from wqflask import user_session -import wqflask.views +import wqflask.views diff --git a/wqflask/wqflask/api/mapping.py b/wqflask/wqflask/api/mapping.py index c22b44a9..e4a3fb77 100644 --- a/wqflask/wqflask/api/mapping.py +++ b/wqflask/wqflask/api/mapping.py @@ -140,5 +140,3 @@ def initialize_parameters(start_vars, dataset, this_trait): mapping_params['perm_check'] = False return mapping_params - - diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index e7dfa4e0..f7d52ca3 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -59,13 +59,13 @@ def get_species_info(species_name, file_format="json"): WHERE (Name="{0}" OR FullName="{0}" OR SpeciesName="{0}");""".format(species_name)) the_species = results.fetchone() - species_dict = { + species_dict = { "Id": the_species[0], "Name": the_species[1], "FullName": the_species[2], "TaxonomyId": the_species[3] } - + return flask.jsonify(species_dict) @@ -639,7 +639,7 @@ def trait_sample_data(dataset_name, trait_name, file_format="json"): return flask.jsonify(sample_list) else: - return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") @app.route("/api/v_{}/trait//".format(version)) @@ -685,7 +685,7 @@ def get_trait_info(dataset_name, trait_name, file_format="json"): # ZS: Check if the user input the dataset_name as BXDPublish, etc (which is always going to be the group name + "Publish" if "Publish" in dataset_name: dataset_name = dataset_name.replace("Publish", "") - + group_id = get_group_id(dataset_name) pheno_query = """ SELECT @@ -898,7 +898,7 @@ def get_dataset_trait_ids(dataset_name, start_vars): data_type = "Publish" dataset_name = dataset_name.replace("Publish", "") dataset_id = get_group_id(dataset_name) - + query = """ SELECT PublishXRef.PhenotypeId, PublishXRef.Id, InbredSet.InbredSetCode @@ -949,9 +949,9 @@ def get_samplelist(dataset_name): WHERE StrainXRef.StrainId = Strain.Id AND StrainXRef.InbredSetId = {} """.format(group_id) - + results = g.db.execute(query).fetchall() - + samplelist = [result[0] for result in results] return samplelist diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 1fcf15f0..9fd89524 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -239,4 +239,3 @@ def change_collection_name(): g.user_session.change_collection_name(collection_id, new_name) return new_name - diff --git a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py index 0fabb833..d86c8e16 100644 --- a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py +++ b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py @@ -45,7 +45,7 @@ class ComparisonBarChart: # ZS: Getting initial group name before verifying all traits are in the same group in the following loop this_group = self.trait_list[0][1].group.name for trait_db in self.trait_list: - + if trait_db[1].group.name != this_group: self.insufficient_shared_samples = True break @@ -53,7 +53,7 @@ class ComparisonBarChart: this_group = trait_db[1].group.name this_trait = trait_db[0] self.traits.append(this_trait) - + this_sample_data = this_trait.data for sample in this_sample_data: @@ -79,7 +79,7 @@ class ComparisonBarChart: self.js_data = dict(traits=[trait.name for trait in self.traits], samples=self.all_sample_list, sample_data=self.sample_data,) - + def get_trait_db_obs(self, trait_db_list): self.trait_list = [] @@ -95,4 +95,3 @@ class ComparisonBarChart: self.trait_list.append((trait_ob, dataset_ob)) #print("trait_list:", self.trait_list) - diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index f9a0ea11..4f756f58 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -68,7 +68,7 @@ class CorrScatterPlot: slope_string = '%.3E' % slope else: slope_string = '%.3f' % slope - + x_buffer = (max(vals_1) - min(vals_1)) * 0.1 y_buffer = (max(vals_2) - min(vals_2)) * 0.1 @@ -82,7 +82,7 @@ class CorrScatterPlot: ry = stats.rankdata(vals_2) self.rdata = [] self.rdata.append(rx.tolist()) - self.rdata.append(ry.tolist()) + self.rdata.append(ry.tolist()) srslope, srintercept, srr_value, srp_value, srstd_err = stats.linregress( rx, ry) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index e75c4a85..8ee24246 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -680,4 +680,3 @@ def get_header_fields(data_type, corr_method): 'Sample p(r)'] return header_fields - diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index aefb4453..59469428 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -324,7 +324,7 @@ def sortEigenVectors(vector): A.append(item[0]) B.append(item[1]) sum = reduce(lambda x, y: x + y, A, 0.0) - A = [x * 100.0 / sum for x in A] + A = [x * 100.0 / sum for x in A] return [A, B] except: return [] diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 48a82435..f4eafbe7 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -66,7 +66,7 @@ class CTL: def addNode(self, gt): node_dict = {'data': {'id': str(gt.name) + ":" + str(gt.dataset.name), - 'sid': str(gt.name), + 'sid': str(gt.name), 'dataset': str(gt.dataset.name), 'label': gt.name, 'symbol': gt.symbol, @@ -238,4 +238,3 @@ class CTL: self.render_image(self.results) sys.stdout.flush() return(dict(template_vars)) - diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index c7558ed8..8d28fef0 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -135,5 +135,3 @@ def process_query_results(results): } return info_ob - - diff --git a/wqflask/wqflask/docs.py b/wqflask/wqflask/docs.py index fc93248a..0a1a597d 100644 --- a/wqflask/wqflask/docs.py +++ b/wqflask/wqflask/docs.py @@ -20,7 +20,7 @@ class Docs: self.title = self.entry.capitalize() self.content = "" else: - + self.title = result[0] self.content = result[1].decode("utf-8") diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 5bd54f9d..2c180d49 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -1,6 +1,6 @@ import csv import xlsxwriter -import io +import io import datetime import itertools @@ -20,7 +20,7 @@ def export_search_results_csv(targs): table_data = json.loads(targs['export_data']) table_rows = table_data['rows'] - + now = datetime.datetime.now() time_str = now.strftime('%H:%M_%d%B%Y') if 'file_name' in targs: diff --git a/wqflask/wqflask/external_tools/send_to_webgestalt.py b/wqflask/wqflask/external_tools/send_to_webgestalt.py index fd12562f..fcd943ba 100644 --- a/wqflask/wqflask/external_tools/send_to_webgestalt.py +++ b/wqflask/wqflask/external_tools/send_to_webgestalt.py @@ -48,7 +48,7 @@ class SendToWebGestalt: id_type = "entrezgene" - self.hidden_vars = { + self.hidden_vars = { 'gene_list': "\n".join(gene_id_list), 'id_type': "entrezgene", 'ref_set': "genome", diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index aa11caa8..02eb66e5 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -125,7 +125,7 @@ class Heatmap: webqtlConfig.GENERATED_IMAGE_DIR, output_filename) - os.system(reaper_command) + os.system(reaper_command) reaper_results = parse_reaper_output(output_filename) diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index cadff080..e624a146 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -10,7 +10,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): fetchFields = ['SpeciesId', 'Id', 'GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'TxEnd', 'Strand', 'GeneID', 'NM_ID', 'kgID', 'GenBankID', 'UnigenID', 'ProteinID', 'AlignID', 'exonCount', 'exonStarts', 'exonEnds', 'cdsStart', 'cdsEnd'] - + # List All Species in the Gene Table speciesDict = {} results = g.db.execute(""" @@ -21,7 +21,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): for item in results: speciesDict[item[0]] = item[1] - + # List current Species and other Species speciesId = speciesDict[species] otherSpecies = [[X, speciesDict[X]] for X in list(speciesDict.keys())] @@ -45,7 +45,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): newdict = {} for j, item in enumerate(fetchFields): newdict[item] = result[j] - # count SNPs if possible + # count SNPs if possible if diffCol and species == 'mouse': newdict["snpCount"] = g.db.execute(""" SELECT count(*) @@ -58,17 +58,17 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): (newdict["TxEnd"] - newdict["TxStart"]) / 1000.0 else: newdict["snpDensity"] = newdict["snpCount"] = 0 - + try: newdict['GeneLength'] = 1000.0 * (newdict['TxEnd'] - newdict['TxStart']) except: pass - + # load gene from other Species by the same name for item in otherSpecies: othSpec, othSpecId = item newdict2 = {} - + resultsOther = g.db.execute("SELECT %s FROM GeneList WHERE SpeciesId = %d AND geneSymbol= '%s' LIMIT 1" % (", ".join(fetchFields), othSpecId, newdict["GeneSymbol"])).fetchone() @@ -76,8 +76,8 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): if resultsOther: for j, item in enumerate(fetchFields): newdict2[item] = resultsOther[j] - - # count SNPs if possible, could be a separate function + + # count SNPs if possible, could be a separate function if diffCol and othSpec == 'mouse': newdict2["snpCount"] = g.db.execute(""" SELECT count(*) @@ -91,17 +91,15 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): (newdict2["TxEnd"] - newdict2["TxStart"]) / 1000.0 else: newdict2["snpDensity"] = newdict2["snpCount"] = 0 - + try: newdict2['GeneLength'] = 1000.0 * \ (newdict2['TxEnd'] - newdict2['TxStart']) except: pass - + newdict['%sGene' % othSpec] = newdict2 - + GeneList.append(newdict) return GeneList - - diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index f932498f..c51b7a9a 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -34,7 +34,7 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo opt_list = [] if boot_check and num_bootstrap > 0: - bootstrap_filename = (f"{this_dataset.group.name}_BOOTSTRAP_" + + bootstrap_filename = (f"{this_dataset.group.name}_BOOTSTRAP_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) ) @@ -44,8 +44,8 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo opt_list.append( f"--bootstrap_output {webqtlConfig.GENERATED_IMAGE_DIR}{bootstrap_filename}.txt") if num_perm > 0: - permu_filename = ("{this_dataset.group.name}_PERM_" + - ''.join(random.choice(string.ascii_uppercase + + permu_filename = ("{this_dataset.group.name}_PERM_" + + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) ) opt_list.append("-n " + str(num_perm)) @@ -56,7 +56,7 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo if manhattan_plot != True: opt_list.append("--interval 1") - reaper_command = (REAPER_COMMAND + + reaper_command = (REAPER_COMMAND + ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt {4} -o {5}{6}.txt'.format(flat_files('genotype'), genofile_name, @@ -81,7 +81,7 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo suggestive = permu_vals[int(num_perm * 0.37 - 1)] significant = permu_vals[int(num_perm * 0.95 - 1)] - return (marker_obs, permu_vals, suggestive, significant, bootstrap_vals, + return (marker_obs, permu_vals, suggestive, significant, bootstrap_vals, [output_filename, permu_filename, bootstrap_filename]) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 741d6c23..b3c9fddf 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -300,7 +300,7 @@ def add_categorical_covar(cross, covar_as_string, i): ro.r('the_cross$pheno <- cbind(pheno, ' + \ col_name + ' = newcovar[,' + str(x) + '])') col_names.append(col_name) - #logger.info("loop" + str(x) + "done"); + #logger.info("loop" + str(x) + "done"); logger.info("returning from add_categorical_covar") return ro.r["the_cross"], col_names diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 36d4cd61..61f3b202 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -142,5 +142,5 @@ def get_group_names(group_masks): group_name = get_group_info(group_id)['name'] this_mask['name'] = group_name group_masks_with_names[group_id] = this_mask - + return group_masks_with_names diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 273a97a4..0d2fb2f8 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -343,4 +343,3 @@ def get_aliases(symbol_list, species): search_terms.append(the_search_term) return search_terms - diff --git a/wqflask/wqflask/server_side.py b/wqflask/wqflask/server_side.py index 8b3a4faa..8ca3a9eb 100644 --- a/wqflask/wqflask/server_side.py +++ b/wqflask/wqflask/server_side.py @@ -30,7 +30,7 @@ class ServerSideTable: self.rows_count = rows_count self.table_rows = table_rows self.header_data_names = header_data_names - + self.sort_rows() self.paginate_rows() diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index e98cfb71..5b7a663c 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -494,10 +494,10 @@ class SnpBrowser: function_details = function_details + ", Coding Region Unknown" self.empty_columns['function_details'] = "true" - + #[snp_href, chr, mb_formatted, alleles, snp_source_cell, conservation_score, gene_name_cell, transcript_href, exon, domain_1, domain_2, function, function_details] - base_color_dict = {"A": "#C33232", "C": "#1569C7", "T": "#CFCF32", "G": "#32C332", + base_color_dict = {"A": "#C33232", "C": "#1569C7", "T": "#CFCF32", "G": "#32C332", "t": "#FF6", "c": "#5CB3FF", "a": "#F66", "g": "#CF9", ":": "#FFFFFF", "-": "#FFFFFF", "?": "#FFFFFF"} the_bases = [] @@ -735,7 +735,7 @@ def get_header_list(variant_type, strains, species=None, empty_columns=None): if empty_columns['function_details'] == "false": empty_field_count += 1 header_fields[0].remove('Details') - + for col in empty_columns.keys(): if empty_columns[col] == "false": header_data_names.remove(col) @@ -952,4 +952,3 @@ def check_if_in_gene(species_id, chr, mb): return [result[0], result[1]] else: return "" - diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index 708d43d2..bfaed9c2 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -45,10 +45,10 @@ def encode_password(pass_gen_fields, unencrypted_password): salt = pass_gen_fields['salt'] else: salt = bytes(pass_gen_fields['salt'], "utf-8") - encrypted_password = pbkdf2.pbkdf2_hex(str(unencrypted_password), + encrypted_password = pbkdf2.pbkdf2_hex(str(unencrypted_password), salt, - pass_gen_fields['iterations'], - pass_gen_fields['keylength'], + pass_gen_fields['iterations'], + pass_gen_fields['keylength'], pass_gen_fields['hashfunc']) pass_gen_fields.pop("unencrypted_password", None) @@ -111,7 +111,7 @@ def get_signed_session_id(user): key = UserSession.user_cookie_name + ":" + session_id Redis.hmset(key, session) Redis.expire(key, THREE_DAYS) - + return session_id_signed @@ -207,7 +207,7 @@ def login(): UserSession.user_cookie_name, session_id_signed, max_age=None) else: flash("Something went unexpectedly wrong.", "alert-danger") - response = make_response(redirect(url_for('index_page'))) + response = make_response(redirect(url_for('index_page'))) return response else: user_details = get_user_by_unique_column( @@ -276,13 +276,13 @@ def github_oauth2(): user_details = get_user_by_unique_column("github_id", github_user["id"]) if user_details == None: user_details = { - "user_id": str(uuid.uuid4()), - "name": github_user["name"].encode("utf-8") if github_user["name"] else "None", + "user_id": str(uuid.uuid4()), + "name": github_user["name"].encode("utf-8") if github_user["name"] else "None", "github_id": github_user["id"], - "user_url": github_user["html_url"].encode("utf-8"), - "login_type": "github", - "organization": "", - "active": 1, + "user_url": github_user["html_url"].encode("utf-8"), + "login_type": "github", + "organization": "", + "active": 1, "confirmed": 1 } save_user(user_details, user_details["user_id"]) @@ -308,8 +308,8 @@ def orcid_oauth2(): url = "/n/login" if code: data = { - "client_id": ORCID_CLIENT_ID, - "client_secret": ORCID_CLIENT_SECRET, + "client_id": ORCID_CLIENT_ID, + "client_secret": ORCID_CLIENT_SECRET, "grant_type": "authorization_code", "redirect_uri": GN2_BRANCH_URL + "n/login/orcid_oauth2", "code": code @@ -321,13 +321,13 @@ def orcid_oauth2(): user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) if user_details == None: user_details = { - "user_id": str(uuid4()), - "name": result_dict["name"], - "orcid": result_dict["orcid"], - "user_url": "%s/%s" % ("/".join(ORCID_AUTH_URL.split("/")[:-2]), result_dict["orcid"]), - "login_type": "orcid", - "organization": "", - "active": 1, + "user_id": str(uuid4()), + "name": result_dict["name"], + "orcid": result_dict["orcid"], + "user_url": "%s/%s" % ("/".join(ORCID_AUTH_URL.split("/")[:-2]), result_dict["orcid"]), + "login_type": "orcid", + "organization": "", + "active": 1, "confirmed": 1 } save_user(user_details, user_details["user_id"]) @@ -374,7 +374,7 @@ def send_forgot_password_email(verification_email): key_prefix = "forgot_password_code" subject = "GeneNetwork password reset" fromaddr = "no-reply@genenetwork.org" - + verification_code = str(uuid.uuid4()) key = key_prefix + ":" + verification_code diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 6ccb2e80..963288b3 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -318,5 +318,3 @@ class UserSession: # And more importantly delete the redis record Redis.delete(self.redis_key) self.logged_in = False - - -- cgit v1.2.3