From 66c6bbfcbbd5fb23145ec09956f4809e5f701bec Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 Jun 2019 13:01:36 -0500 Subject: Fixed issue that caused interval mapping to not work because the python implementation of the reaper Dataset object doesn't include the addinterval method (so for those situations I still use reaper) Fixed issue where the last chromosome wasn't displayed for mapping results (though still need to fix issue where points are drawn too far to the right when a specific range is viewed) --- wqflask/utility/gen_geno_ob.py | 270 +++++++++++++++++++++-------------------- 1 file changed, 136 insertions(+), 134 deletions(-) (limited to 'wqflask/utility') diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index 5824b0b3..5172369f 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -1,135 +1,137 @@ -from __future__ import absolute_import, division, print_function - -class genotype(object): - """ - Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure - """ - - def __init__(self, filename): - self.group = None - self.type = "riset" - self.prgy = [] - self.nprgy = 0 - self.mat = -1 - self.pat = 1 - self.het = 0 - self.unk = "U" - self.filler = False - self.mb_exists = False - - #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary - self.cm_column = 2 - self.mb_column = 3 - - self.chromosomes = [] - - self.read_file(filename) - - def __iter__(self): - return iter(self.chromosomes) - - def __getitem__(self, index): - return self.chromosomes[index] - - def __len__(self): - return len(self.chromosomes) - - def read_file(self, filename): - - with open(filename, 'r') as geno_file: - lines = geno_file.readlines() - - this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers - chr_ob = None - for line in lines: - if line[0] == "#": - continue - elif line[0] == "@": - label = line.split(":")[0][1:] - if label == "name": - self.group = line.split(":")[1] - elif label == "filler": - if line.split(":")[1] == "yes": - self.filler = True - elif label == "type": - self.type = line.split(":")[1] - elif label == "mat": - self.mat = line.split(":")[1] - elif label == "pat": - self.pat = line.split(":")[1] - elif label == "het": - self.het = line.split(":")[1] - elif label == "unk": - self.unk = line.split(":")[1] - else: - continue - elif line[:3] == "Chr": - header_row = line.split("\t") - if header_row[2] == "Mb": - self.mb_exists = True - self.mb_column = 2 - self.cm_column = 3 - elif header_row[3] == "Mb": - self.mb_exists = True - self.mb_column = 3 - elif header_row[2] == "cM": - self.cm_column = 2 - - if self.mb_exists: - self.prgy = header_row[4:] - else: - self.prgy = header_row[3:] - self.nprgy = len(self.prgy) - else: - if line.split("\t")[0] != this_chr: - if this_chr != "": - self.chromosomes.append(chr_ob) - this_chr = line.split("\t")[0] - chr_ob = Chr(line.split("\t")[0], self) - chr_ob.add_marker(line.split("\t")) - -class Chr(object): - def __init__(self, name, geno_ob): - self.name = name - self.loci = [] - self.mb_exists = geno_ob.mb_exists - self.cm_column = geno_ob.cm_column - self.mb_column = geno_ob.mb_column - self.geno_ob = geno_ob - - def __iter__(self): - return iter(self.loci) - - def __getitem__(self, index): - return self.loci[index] - - def __len__(self): - return len(self.loci) - - def add_marker(self, marker_row): - self.loci.append(Locus(marker_row, self.geno_ob)) - -class Locus(object): - def __init__(self, marker_row, geno_ob): - self.chr = marker_row[0] - self.name = marker_row[1] - self.cM = float(marker_row[geno_ob.cm_column]) - self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None - - geno_table = { - geno_ob.mat: -1, - geno_ob.pat: 1, - geno_ob.het: 0, - geno_ob.unk: "U" - } - - self.genotype = [] - if geno_ob.mb_exists: - start_pos = 4 - else: - start_pos = 3 - for allele in marker_row[start_pos:]: - if allele in geno_table.keys(): - self.genotype.append(geno_table[allele]) - else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown +from __future__ import absolute_import, division, print_function + +class genotype(object): + """ + Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure + """ + + def __init__(self, filename): + self.group = None + self.type = "riset" + self.prgy = [] + self.nprgy = 0 + self.mat = -1 + self.pat = 1 + self.het = 0 + self.unk = "U" + self.filler = False + self.mb_exists = False + + #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary + self.cm_column = 2 + self.mb_column = 3 + + self.chromosomes = [] + + self.read_file(filename) + + def __iter__(self): + return iter(self.chromosomes) + + def __getitem__(self, index): + return self.chromosomes[index] + + def __len__(self): + return len(self.chromosomes) + + def read_file(self, filename): + + with open(filename, 'r') as geno_file: + lines = geno_file.readlines() + + this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers + chr_ob = None + for line in lines: + if line[0] == "#": + continue + elif line[0] == "@": + label = line.split(":")[0][1:] + if label == "name": + self.group = line.split(":")[1] + elif label == "filler": + if line.split(":")[1] == "yes": + self.filler = True + elif label == "type": + self.type = line.split(":")[1] + elif label == "mat": + self.mat = line.split(":")[1] + elif label == "pat": + self.pat = line.split(":")[1] + elif label == "het": + self.het = line.split(":")[1] + elif label == "unk": + self.unk = line.split(":")[1] + else: + continue + elif line[:3] == "Chr": + header_row = line.split("\t") + if header_row[2] == "Mb": + self.mb_exists = True + self.mb_column = 2 + self.cm_column = 3 + elif header_row[3] == "Mb": + self.mb_exists = True + self.mb_column = 3 + elif header_row[2] == "cM": + self.cm_column = 2 + + if self.mb_exists: + self.prgy = header_row[4:] + else: + self.prgy = header_row[3:] + self.nprgy = len(self.prgy) + else: + if line.split("\t")[0] != this_chr: + if this_chr != "": + self.chromosomes.append(chr_ob) + this_chr = line.split("\t")[0] + chr_ob = Chr(line.split("\t")[0], self) + chr_ob.add_marker(line.split("\t")) + + self.chromosomes.append(chr_ob) + +class Chr(object): + def __init__(self, name, geno_ob): + self.name = name + self.loci = [] + self.mb_exists = geno_ob.mb_exists + self.cm_column = geno_ob.cm_column + self.mb_column = geno_ob.mb_column + self.geno_ob = geno_ob + + def __iter__(self): + return iter(self.loci) + + def __getitem__(self, index): + return self.loci[index] + + def __len__(self): + return len(self.loci) + + def add_marker(self, marker_row): + self.loci.append(Locus(marker_row, self.geno_ob)) + +class Locus(object): + def __init__(self, marker_row, geno_ob): + self.chr = marker_row[0] + self.name = marker_row[1] + self.cM = float(marker_row[geno_ob.cm_column]) + self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None + + geno_table = { + geno_ob.mat: -1, + geno_ob.pat: 1, + geno_ob.het: 0, + geno_ob.unk: "U" + } + + self.genotype = [] + if geno_ob.mb_exists: + start_pos = 4 + else: + start_pos = 3 + for allele in marker_row[start_pos:]: + if allele in geno_table.keys(): + self.genotype.append(geno_table[allele]) + else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown self.genotype.append("U") \ No newline at end of file -- cgit v1.2.3 From 3659ceb0ee7058e41cd0b789558b07d7e8bf415e Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 7 Jun 2019 13:08:47 -0500 Subject: Global search and correlation results can handle null expression now --- wqflask/utility/type_checking.py | 2 +- wqflask/wqflask/correlation/show_corr_results.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'wqflask/utility') diff --git a/wqflask/utility/type_checking.py b/wqflask/utility/type_checking.py index 220e5f62..f15b17e2 100644 --- a/wqflask/utility/type_checking.py +++ b/wqflask/utility/type_checking.py @@ -27,7 +27,7 @@ def get_float(vars,name,default=None): if name in vars: if is_float(vars[name]): return float(vars[name]) - return None + return default def get_int(vars,name,default=None): if name in vars: diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 1aa7945d..6e9abcd6 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -83,7 +83,7 @@ class CorrelationResults(object): assert('corr_sample_method' in start_vars) assert('corr_samples_group' in start_vars) assert('corr_dataset' in start_vars) - assert('min_expr' in start_vars) + #assert('min_expr' in start_vars) assert('corr_return_results' in start_vars) if 'loc_chr' in start_vars: assert('min_loc_mb' in start_vars) -- cgit v1.2.3 From 5f5adb62bf13ace887c03522b0b8e83181cd6503 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 23 Jul 2019 11:24:56 -0500 Subject: Added change to automatically update datasets list using rest api instead of json file Removed option to edit certain html pages, like news, policies, etc --- etc/default_settings.py | 1 + wqflask/base/data_set.py | 4 +- wqflask/utility/tools.py | 4 ++ .../marker_regression/display_mapping_results.py | 1 - wqflask/wqflask/marker_regression/run_mapping.py | 17 +++++++- .../new/javascript/dataset_menu_structure.json | 51 ++++++++++++---------- wqflask/wqflask/templates/docs.html | 4 +- 7 files changed, 53 insertions(+), 29 deletions(-) (limited to 'wqflask/utility') diff --git a/etc/default_settings.py b/etc/default_settings.py index a1fe81e5..a16ac3ad 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -91,4 +91,5 @@ JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript" # ---- GN2 Executables (overwrite for testing only) # PLINK_COMMAND = str.strip(os.popen("which plink2").read()) # GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) +REAPER_COMMAND = HOME + "/gn2-zach/rust-qtlreaper/target/release/qtlreaper" # GEMMA_WRAPPER_COMMAND = str.strip(os.popen("which gemma-wrapper").read()) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index d766e284..41de8492 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -46,6 +46,8 @@ from utility import chunks from utility import gen_geno_ob from utility.tools import locate, locate_ignore_error, flat_files +from wqflask.api import gen_menu + from maintenance import get_group_samplelists from MySQLdb import escape_string as escape @@ -92,7 +94,7 @@ Publish or ProbeSet. E.g. """ self.datasets = {} if USE_GN_SERVER: - data = menu_main() + data = gen_menu.gen_dropdown_json() else: file_name = "wqflask/static/new/javascript/dataset_menu_structure.json" with open(file_name, 'r') as fh: diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 8b2260f5..31ab2046 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -107,6 +107,9 @@ def js_path(module=None): return try_guix raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)" +def reaper_command(guess=None): + return get_setting("REAPER_COMMAND",guess) + def gemma_command(guess=None): return assert_bin(get_setting("GEMMA_COMMAND",guess)) @@ -274,6 +277,7 @@ SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') SMTP_PASSWORD = get_setting('SMTP_PASSWORD') +REAPER_COMMAND = app_set("REAPER_COMMAND",reaper_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) assert(GEMMA_COMMAND is not None) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 3bcd613f..d9601405 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -1790,7 +1790,6 @@ class DisplayMappingResults(object): m = 0 thisLRSColor = self.colorCollection[0] if qtlresult['chr'] != previous_chr and self.selectedChr == -1: - if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 2bde2b53..6e9fe85c 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -36,7 +36,7 @@ from utility import helper_functions from utility import Plot, Bunch from utility import temp_data from utility.benchmark import Bench -from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping +from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping, rust_reaper_mapping from utility.tools import locate, locate_ignore_error, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR from utility.external import shell @@ -242,7 +242,8 @@ class RunMapping(object): self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] logger.info("Running qtlreaper") - results, self.json_data, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = qtlreaper_mapping.gen_reaper_results(self.this_trait, + + results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = rust_reaper_mapping.run_reaper(self.this_trait, self.dataset, self.samples, self.vals, @@ -253,6 +254,18 @@ class RunMapping(object): self.do_control, self.control_marker, self.manhattan_plot) + + # results, self.json_data, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = qtlreaper_mapping.gen_reaper_results(self.this_trait, + # self.dataset, + # self.samples, + # self.vals, + # self.json_data, + # self.num_perm, + # self.bootCheck, + # self.num_bootstrap, + # self.do_control, + # self.control_marker, + # self.manhattan_plot) elif self.mapping_method == "plink": self.score_type = "-log(p)" self.manhattan_plot = True diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json index 0d73213d..6531f5a0 100644 --- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json +++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json @@ -1440,7 +1440,7 @@ [ "None", "HSBPublish", - "HSB Phenotypes" + "HSB Published Phenotypes" ] ], "Posterior Inferior Parietal Cortex mRNA": [ @@ -1721,7 +1721,7 @@ [ "None", "B6D2Publish", - "B6D2 Phenotypes" + "UTHSC-Glaucoma-Aged-Retina Phenotypes" ] ] }, @@ -2247,6 +2247,11 @@ "Eye_M2_0908_R_ND", "Eye M430v2 WT Gpnmb (Sep08) RMA" ], + [ + "382", + "Eye_M2_0908_WTWT", + "Eye M430v2 WT WT (Sep08) RMA" + ], [ "279", "Eye_M2_0908_R_WT", @@ -2257,11 +2262,6 @@ "Eye_M2_0908_R_MT", "Eye M430v2 Mutant Tyrp1 (Sep08) RMA" ], - [ - "382", - "Eye_M2_0908_WTWT", - "Eye M430v2 WT WT (Sep08) RMA" - ], [ "400", "DBA2J-ONH-1212", @@ -2500,6 +2500,11 @@ "245", "UT_ILM_BXD_hipp_RSE_0909", "UTHSC Hippocampus Illumina v6.1 RSE (Sep09) RankInv" + ], + [ + "711", + "UTHSC_BXD_AgeHipp0515", + "UTHSC BXD Aged Hippocampus Affy MoGene1.0 ST (May15) RMA Gene Level" ] ], "Hypothalamus mRNA": [ @@ -3028,16 +3033,6 @@ ] ], "Striatum mRNA": [ - [ - "376", - "DevStriatum_ILM6.2P3RInv_1111", - "BIDMC/UTHSC Dev Striatum P3 ILMv6.2 (Nov11) RankInv" - ], - [ - "377", - "DevStriatum_ILM6.2P14RInv_1111", - "BIDMC/UTHSC Dev Striatum P14 ILMv6.2 (Nov11) RankInv" - ], [ "399", "Striatum_Exon_1212", @@ -3048,6 +3043,16 @@ "Striatum_Exon_0209", "HQF Striatum Affy Mouse Exon 1.0ST Exon Level (Dec09) RMA" ], + [ + "376", + "DevStriatum_ILM6.2P3RInv_1111", + "BIDMC/UTHSC Dev Striatum P3 ILMv6.2 (Nov11) RankInv" + ], + [ + "377", + "DevStriatum_ILM6.2P14RInv_1111", + "BIDMC/UTHSC Dev Striatum P14 ILMv6.2 (Nov11) RankInv" + ], [ "285", "UTHSC_Striatum_RankInv_1210", @@ -3554,6 +3559,11 @@ ] ], "Hippocampus mRNA": [ + [ + "211", + "Illum_LXS_Hipp_RSS_1008", + "Hippocampus Illumina RSS (Oct08) RankInv beta" + ], [ "213", "Illum_LXS_Hipp_NOS_1008", @@ -3574,11 +3584,6 @@ "Illum_LXS_Hipp_NOE_1008", "Hippocampus Illumina NOE (Oct08) RankInv beta" ], - [ - "211", - "Illum_LXS_Hipp_RSS_1008", - "Hippocampus Illumina RSS (Oct08) RankInv beta" - ], [ "143", "Illum_LXS_Hipp_loess0807", @@ -3835,7 +3840,7 @@ [ "None", "HSNIH-RGSMCPublish", - "HSNIH-RGSMC Phenotypes" + "HSNIH Published Phenotypes" ] ] }, diff --git a/wqflask/wqflask/templates/docs.html b/wqflask/wqflask/templates/docs.html index c485f757..1a241ce7 100644 --- a/wqflask/wqflask/templates/docs.html +++ b/wqflask/wqflask/templates/docs.html @@ -6,9 +6,9 @@

{{title}}

- + - +
{{content|safe}}
-- cgit v1.2.3 From 4bd1db34b109bff3b9abdcc161472885e9ae700d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 6 Sep 2019 03:29:12 -0500 Subject: Fix name of javascript-twitter-post-fetcher --- wqflask/utility/tools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'wqflask/utility') diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 31ab2046..ec9f01bd 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -289,11 +289,14 @@ assert_dir(TEMPDIR) JS_GUIX_PATH = get_setting("JS_GUIX_PATH") assert_dir(JS_GUIX_PATH) assert_dir(JS_GUIX_PATH+'/cytoscape-panzoom') + CSS_PATH = "UNKNOWN" # assert_dir(JS_PATH) -JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher")) + +JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("javascript-twitter-post-fetcher")) assert_dir(JS_TWITTER_POST_FETCHER_PATH) assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") + JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH",js_path("cytoscape")) assert_dir(JS_CYTOSCAPE_PATH) assert_file(JS_CYTOSCAPE_PATH+'/cytoscape.min.js') -- cgit v1.2.3