From ee1e49b90663be72032c8dee32748e97bc28e1e9 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 21 Dec 2018 10:48:33 -0600 Subject: Fixed collections so they can be very large (1000+ traits) Added option to submit traits in collection to BNW Fixed issue with "x" values for user-submitted traits Fixed issue where post-publications descriptions were wrongly appearing in global search results --- wqflask/base/data_set.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 4a422ee4..55782576 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -371,23 +371,16 @@ class DatasetGroup(object): result = Redis.get(key) if result is not None: - #logger.debug("Sample List Cache hit!!!") - #logger.debug("Before unjsonifying {}: {}".format(type(result), result)) self.samplelist = json.loads(result) - #logger.debug(" type: ", type(self.samplelist)) - #logger.debug(" self.samplelist: ", self.samplelist) else: logger.debug("Cache not hit") genotype_fn = locate_ignore_error(self.name+".geno",'genotype') - mapping_fn = locate_ignore_error(self.name+".fam",'mapping') - if mapping_fn: - self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn) - elif genotype_fn: + if genotype_fn: self.samplelist = get_group_samplelists.get_samplelist("geno", genotype_fn) else: self.samplelist = None - logger.debug("Sample list: ",self.samplelist) + if USE_REDIS: Redis.set(key, json.dumps(self.samplelist)) Redis.expire(key, 60*5) -- cgit v1.2.3 From f1ea496ae71388810f0e6dffe36b005ceabaed6c Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 11 Jan 2019 10:46:38 -0600 Subject: Got InDel working for Variant Browser Changed Variant Browser to display gene name under "Domain" when the user searches a gene term and rows are mislabeled "Intergenic" Fixed Chr option to display correct chromosomes for different species Fixed order for correlation drop-down on trait page Fixed some appearance issues with the mapping results table Improved appearance of correlation scatterplot page --- wqflask/base/data_set.py | 7 ++- .../marker_regression/display_mapping_results.py | 2 +- wqflask/wqflask/snp_browser/snp_browser.py | 63 ++++++++++++++++------ .../wqflask/static/new/css/marker_regression.css | 2 +- .../new/javascript/dataset_select_menu_orig.js | 7 ++- wqflask/wqflask/templates/collections/list.html | 4 +- wqflask/wqflask/templates/corr_scatterplot.html | 10 ++-- wqflask/wqflask/templates/mapping_results.html | 19 ++++++- wqflask/wqflask/templates/snp_browser.html | 32 ++++++++++- 9 files changed, 117 insertions(+), 29 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 55782576..2acb3b61 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -109,6 +109,7 @@ Publish or ProbeSet. E.g. else: new_type = "ProbeSet" self.datasets[short_dataset_name] = new_type + # Set LOG_LEVEL_DEBUG=5 to see the following: logger.debugf(5, "datasets",self.datasets) @@ -450,12 +451,14 @@ def datasets(group_name, this_group = None): and InbredSet.Name like %s and ProbeSetFreeze.public > %s and ProbeSetFreeze.confidentiality < 1 - ORDER BY Tissue.Name, ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId) + ORDER BY Tissue.Name) ''' % (group_name, webqtlConfig.PUBLICTHRESH, group_name, webqtlConfig.PUBLICTHRESH, "'" + group_name + "'", webqtlConfig.PUBLICTHRESH)) - for dataset_item in the_results: + sorted_results = sorted(the_results, key=lambda kv: kv[0]) + + for dataset_item in sorted_results: tissue_name = dataset_item[0] dataset = dataset_item[1] dataset_short = dataset_item[2] diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index e63e4af2..28831937 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2083,7 +2083,7 @@ class DisplayMappingResults(object): tableIterationsCnt = tableIterationsCnt + 1 this_row = [] #container for the cells of each row - selectCheck = HT.Input(type="checkbox", name="searchResult", value=theGO["GeneSymbol"], Class="checkbox", onClick="highlight(this)") #checkbox for each row + selectCheck = HT.Input(type="checkbox", name="searchResult", value=theGO["GeneSymbol"], Class="checkbox trait_checkbox") #checkbox for each row geneLength = (theGO["TxEnd"] - theGO["TxStart"])*1000.0 tenPercentLength = geneLength*0.0001 diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index 0e999ba2..0058070c 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -51,11 +51,19 @@ class SnpBrowser(object): if self.species_name.capitalize() == "Rat": self.species_id = 2 - species_ob = species.TheSpecies(species_name=self.species_name) + self.mouse_chr_list = [] + self.rat_chr_list = [] + mouse_species_ob = species.TheSpecies(species_name="Mouse") + for key in mouse_species_ob.chromosomes.chromosomes: + self.mouse_chr_list.append(mouse_species_ob.chromosomes.chromosomes[key].name) + rat_species_ob = species.TheSpecies(species_name="Rat") + for key in rat_species_ob.chromosomes.chromosomes: + self.rat_chr_list.append(rat_species_ob.chromosomes.chromosomes[key].name) - self.chr_list = [] - for key in species_ob.chromosomes.chromosomes: - self.chr_list.append(species_ob.chromosomes.chromosomes[key].name) + if self.species_id == 1: + self.this_chr_list = self.mouse_chr_list + else: + self.this_chr_list = self.rat_chr_list if self.first_run == "true": self.chr = "19" @@ -213,7 +221,7 @@ class SnpBrowser(object): query = rat_query elif self.variant_type == "InDel": - if species_id != 0: + if self.species_id != 0: query = """ SELECT DISTINCT a.Name, a.Chromosome, a.SourceId, a.Mb_start, a.Mb_end, a.Strand, a.Type, a.Size, a.InDelSequence, b.Name @@ -289,7 +297,12 @@ class SnpBrowser(object): domain = [key, ''] if 'Intergenic' in domain: - gene = transcript = exon = function = function_details = '' + if self.gene_name != "": + gene_id = get_gene_id(self.species_id, self.gene_name) + gene = [gene_id, self.gene_name] + else: + gene = "" + transcript = exon = function = function_details = '' if self.redundant == "false" or last_mb != mb: # filter redundant if self.include_record(domain, function, snp_source, conservation_score): info_list = [snp_name, rs, chr, mb, alleles, gene, transcript, exon, domain, function, function_details, snp_source, conservation_score, snp_id] @@ -325,7 +338,7 @@ class SnpBrowser(object): filtered_results.append(info_list) last_mb = mb - elif self.variant == "InDel": + elif self.variant_type == "InDel": # The order of variables is important; this applies to anything from the variant table as indel indel_name, indel_chr, source_id, indel_mb_start, indel_mb_end, indel_strand, indel_type, indel_size, indel_sequence, source_name = result @@ -354,7 +367,7 @@ class SnpBrowser(object): if gene_name and (gene_name not in gene_name_list): gene_name_list.append(gene_name) if len(gene_name_list) > 0: - gene_id_name_dict = get_gene_id_name_dict(gene_name_list) + gene_id_name_dict = get_gene_id_name_dict(self.species_id, gene_name_list) the_rows = [] for result in self.filtered_results: @@ -395,7 +408,7 @@ class SnpBrowser(object): gene_id = gene_id_name_dict[gene[1]] gene_link = webqtlConfig.NCBI_LOCUSID % gene_id else: - gene_link = "http://www.ncbi.nln.nih.gov/entrez/query.fcgi?CMD=search&DB=gene&term=%s" % gene_name + gene_link = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?CMD=search&DB=gene&term=%s" % gene_name else: gene_name = "" gene_link = "" @@ -408,13 +421,16 @@ class SnpBrowser(object): if exon: exon = exon[1] # exon[0] is exon_id, exon[1] is exon_rank else: - exon = "1" + exon = "" if domain: domain_1 = domain[0] domain_2 = domain[1] - if domain_1 == "Exon": - domain_1 = domain_1 + " " + exon + if domain_1 == "Intergenic" and self.gene_name != "": + domain_1 = self.gene_name + else: + if domain_1 == "Exon": + domain_1 = domain_1 + " " + exon function_list = [] if function_details: @@ -759,7 +775,24 @@ def get_effect_info(effect_list): return effect_info_dict -def get_gene_id_name_dict(gene_name_list): +def get_gene_id(species_id, gene_name): + query = """ + SELECT + geneId + FROM + GeneList + WHERE + SpeciesId = %s AND geneSymbol = '%s' + """ % (species_id, gene_name) + + result = g.db.execute(query).fetchone() + + if len(result) > 0: + return result + else: + return "" + +def get_gene_id_name_dict(species_id, gene_name_list): gene_id_name_dict = {} if len(gene_name_list) == 0: return "" @@ -772,8 +805,8 @@ def get_gene_id_name_dict(gene_name_list): FROM GeneList WHERE - SpeciesId = 1 AND geneSymbol in (%s) - """ % gene_name_str + SpeciesId = %s AND geneSymbol in (%s) + """ % (species_id, gene_name_str) results = g.db.execute(query).fetchall() diff --git a/wqflask/wqflask/static/new/css/marker_regression.css b/wqflask/wqflask/static/new/css/marker_regression.css index 8d205143..f1a26a83 100644 --- a/wqflask/wqflask/static/new/css/marker_regression.css +++ b/wqflask/wqflask/static/new/css/marker_regression.css @@ -59,7 +59,7 @@ table.dataTable thead th { } table.dataTable tbody td { - padding: 3px 20px 1px 10px; + padding: 4px 20px 2px 10px; } table.dataTable.cell-border tbody th, table.dataTable.cell-border tbody td { diff --git a/wqflask/wqflask/static/new/javascript/dataset_select_menu_orig.js b/wqflask/wqflask/static/new/javascript/dataset_select_menu_orig.js index 3f123d6f..fc1f1beb 100644 --- a/wqflask/wqflask/static/new/javascript/dataset_select_menu_orig.js +++ b/wqflask/wqflask/static/new/javascript/dataset_select_menu_orig.js @@ -108,7 +108,12 @@ $(function() { dataset_info = function() { var dataset, url; accession_id = $('#dataset option:selected').data("id"); - url = "http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&GN_AccessionId=" + accession_id; + if (accession_id != "None") { + url = "http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&GN_AccessionId=" + accession_id; + } else { + name = $('#dataset option:selected').val(); + url = "http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=" + name; + } return open_window(url, "Dataset Info"); }; $('#dataset_info').click(dataset_info); diff --git a/wqflask/wqflask/templates/collections/list.html b/wqflask/wqflask/templates/collections/list.html index 5a2df5e3..83e74613 100644 --- a/wqflask/wqflask/templates/collections/list.html +++ b/wqflask/wqflask/templates/collections/list.html @@ -9,7 +9,7 @@ {% block content %} {% if g.user_session.logged_in %} - {{ header("Your Collections", + {{ header("Collections owned by {{ g.user_session.user_name }}", 'You have {}.'.format(numify(collections|count, "collection", "collections"))) }} {% else %} {{ header("Your Collections", @@ -17,6 +17,7 @@ {% endif %}
+
diff --git a/wqflask/wqflask/templates/corr_scatterplot.html b/wqflask/wqflask/templates/corr_scatterplot.html index e520b4b9..e3d90e30 100644 --- a/wqflask/wqflask/templates/corr_scatterplot.html +++ b/wqflask/wqflask/templates/corr_scatterplot.html @@ -131,8 +131,8 @@

-
-
+
+
@@ -168,7 +168,7 @@
StatisticValue
-
+
{% if trait_1.dataset.type == "ProbeSet" %}
X axis: @@ -227,7 +227,7 @@

-
+
@@ -256,7 +256,7 @@
StatisticValue
-
+
{% if trait_1.dataset.type == "ProbeSet" %}
X axis: diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index 3138d855..72d9d0dd 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -215,7 +215,7 @@ {% for marker in trimmed_markers %} - + @@ -269,7 +269,11 @@ {% for row in gene_table_body %} {% for n in range(row|length) %} + {% if n == 0 %} + {{ row[n]|safe }} + {% else %} {{ row[n]|safe }} + {% endif %} {% endfor %} {% endfor %} @@ -339,6 +343,19 @@ } ); $('#interval_analyst').dataTable( { + "columns": [ + { "bSortable": false}, + { "type": "natural" }, + { "type": "natural" }, + { "type": "natural" }, + { "type": "natural" }, + { "type": "natural" }, + { "type": "natural" }, + { "type": "natural" }, + { "type": "natural" }, + { "type": "natural" }, + { "type": "natural" } + ], "columnDefs": [ { "targets": 0, "sortable": false diff --git a/wqflask/wqflask/templates/snp_browser.html b/wqflask/wqflask/templates/snp_browser.html index 3608d0fe..cbca9d22 100644 --- a/wqflask/wqflask/templates/snp_browser.html +++ b/wqflask/wqflask/templates/snp_browser.html @@ -30,6 +30,7 @@
@@ -45,8 +46,8 @@
- + {% for item in this_chr_list %} {% endfor %} @@ -204,6 +205,7 @@ {% for row in table_rows %} + {% if variant_type == "SNP" %} {{ loop.index }} @@ -224,6 +226,21 @@ {{ item[0] }} {% endfor %} + {% else %} + + + {{ loop.index }} + {{ row.indel_name }} + {{ row.indel_type }} + {{ row.indel_chr }} + {{ row.indel_mb_s }} + {{ row.indel_mb_e }} + {{ row.indel_strand }} + {{ row.indel_size }} + {{ row.indel_sequence }} + {{ row.source_name }} + + {% endif %} {% endfor %} @@ -263,6 +280,7 @@ this_species = $(this).val(); $("#strain_select").empty() $("#chosen_strains_select").empty() + $("#chr_select").empty() if (this_species == "Mouse") { {% for strain in strain_lists["mouse"] %} @@ -270,6 +288,11 @@ $("select[name=strains]").append(option); {% endfor %} + {% for chr in mouse_chr_list %} + var option = $('').attr("value", "{{ chr }}").text("{{ chr }}"); + $("select[name=chr]").append(option); + {% endfor %} + chosen_strains = $("input[name=chosen_strains_mouse]").val().split(",") } else if (this_species == "Rat") { {% for strain in strain_lists["rat"] %} @@ -277,6 +300,11 @@ $("select[name=strains]").append(option); {% endfor %} + {% for chr in rat_chr_list %} + var option = $('').attr("value", "{{ chr }}").text("{{ chr }}"); + $("select[name=chr]").append(option); + {% endfor %} + chosen_strains = $("input[name=chosen_strains_rat]").val().split(",") } -- cgit v1.2.3 From 85f014695b7873941ad200051606635ff37b2c20 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 5 Feb 2019 10:42:46 -0600 Subject: Fixed issue with getting marker info that affected heatmap and possibly other functions Added the collection page features (correlation matrix, network graph, third party link-outs, etc) to the search result and correlation result pages --- wqflask/base/data_set.py | 38 +++++------ wqflask/wqflask/heatmap/heatmap.py | 4 ++ .../static/new/javascript/search_results.js | 59 +++++++++++++++++ wqflask/wqflask/templates/collections/view.html | 58 ----------------- wqflask/wqflask/templates/correlation_page.html | 75 ++++++++-------------- wqflask/wqflask/templates/search_result_page.html | 57 +++++++++++++++- 6 files changed, 161 insertions(+), 130 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 2acb3b61..79f72390 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -171,28 +171,22 @@ class Markers(object): def __init__(self, name): json_data_fh = open(locate(name + ".json",'genotype/json')) - try: - markers = [] - with open(locate(name + "_snps.txt", 'r')) as bimbam_fh: + markers = [] + with open("%s/%s_snps.txt" % (flat_files('genotype/bimbam'), name), 'r') as bimbam_fh: + if len(bimbam_fh.readline().split(", ")) > 2: + delimiter = ", " + elif len(bimbam_fh.readline().split(",")) > 2: + delimiter = "," + elif len(bimbam_fh.readline().split("\t")) > 2: + delimiter = "\t" + else: + delimiter = " " + for line in bimbam_fh: marker = {} - if len(bimbam_fh[0].split(", ")) > 2: - delimiter = ", " - elif len(bimbam_fh[0].split(",")) > 2: - delimiter = "," - elif len(bimbam_fh[0].split("\t")) > 2: - delimiter = "\t" - else: - delimiter = " " - for line in bimbam_fh: - marker['name'] = line.split(delimiter)[0] - marker['Mb'] - marker['chr'] = line.split(delimiter)[2] - marker['cM'] - markers.append(marker) - #try: - # markers = json.load(json_data_fh) - except: - markers = [] + marker['name'] = line.split(delimiter)[0].rstrip() + marker['Mb'] = float(line.split(delimiter)[1].rstrip())/1000000 + marker['chr'] = line.split(delimiter)[2].rstrip() + markers.append(marker) for marker in markers: if (marker['chr'] != "X") and (marker['chr'] != "Y"): @@ -334,8 +328,6 @@ class DatasetGroup(object): return mapping_id, mapping_names def get_markers(self): - logger.debug("self.species is:", self.species) - def check_plink_gemma(): if flat_file_exists("mapping"): MAPPING_PATH = flat_files("mapping")+"/" diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 7b328cb5..ff589693 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -43,6 +43,9 @@ Redis = Redis() from flask import Flask, g +from utility.logger import getLogger +logger = getLogger(__name__ ) + class Heatmap(object): def __init__(self, start_vars, temp_uuid): @@ -96,6 +99,7 @@ class Heatmap(object): for trait in self.trait_results.keys(): lodnames.append(trait) + self.dataset.group.get_markers() for marker in self.dataset.group.markers.markers: chr_pos.append(marker['chr']) pos.append(marker['Mb']) diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index abcdf7e6..2277a157 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -223,6 +223,65 @@ $(function() { $('#export_form').submit(); }; + $("#corr_matrix").on("click", function() { + traits = $("#trait_table input:checked").map(function() { + return $(this).val(); + }).get(); + $("#trait_list").val(traits) + url = $(this).data("url") + return submit_special(url) + }); + $("#network_graph").on("click", function() { + traits = $("#trait_table input:checked").map(function() { + return $(this).val(); + }).get(); + $("#trait_list").val(traits) + url = $(this).data("url") + return submit_special(url) + }); + $("#wgcna_setup").on("click", function() { + traits = $("#trait_table input:checked").map(function() { + return $(this).val(); + }).get(); + $("#trait_list").val(traits) + url = $(this).data("url") + return submit_special(url) + }); + $("#ctl_setup").on("click", function() { + traits = $("#trait_table input:checked").map(function() { + return $(this).val(); + }).get(); + $("#trait_list").val(traits) + url = $(this).data("url") + return submit_special(url) + }); + $("#heatmap").on("click", function() { + traits = $("#trait_table input:checked").map(function() { + return $(this).val(); + }).get(); + $("#trait_list").val(traits) + url = $(this).data("url") + return submit_special(url) + }); + $("#comp_bar_chart").on("click", function() { + traits = $("#trait_table input:checked").map(function() { + return $(this).val(); + }).get(); + $("#trait_list").val(traits) + url = $(this).data("url") + return submit_special(url) + }); + + $("#send_to_webgestalt, #send_to_bnw").on("click", function() { + traits = $("#trait_table input:checked").map(function() { + return $(this).val(); + }).get(); + $("#trait_list").val(traits) + url = $(this).data("url") + return submit_special(url) + }); + + $("#select_all").click(select_all); $("#deselect_all").click(deselect_all); $("#invert").click(invert); diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index a2b63374..a219b868 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -220,64 +220,6 @@ url = $(this).data("url") return submit_special(url) }); - - $("#corr_matrix").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - $("#network_graph").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - $("#wgcna_setup").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - $("#ctl_setup").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - $("#heatmap").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - $("#comp_bar_chart").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - - $("#send_to_webgestalt, #send_to_bnw").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); }); diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 2d4ed2a4..4786e379 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -20,31 +20,53 @@

- - {% if uc %} - - {% endif %} - + + + + + + + + + + + + + + + +
@@ -465,49 +487,6 @@ url = $(this).data("url") return submit_special(url) }); - - $("#corr_matrix").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - console.log($("#trait_list").val(traits)) - url = $(this).data("url") - //return submit_special(url) - }); - $("#network_graph").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - $("#wgcna_setup").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - $("#ctl_setup").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - $("#heatmap").on("click", function() { - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); - $("#trait_list").val(traits) - url = $(this).data("url") - return submit_special(url) - }); - }); {% endblock %} diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 79a279f8..6fcbcb01 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -46,13 +46,63 @@

To study a record click on its ID below, and to view the whole description {% if dataset.type == "Publish" %}or list of authors {% endif %} hover over the table cell. Check records below and click Add button to add to selection.

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+
+

-
@@ -355,6 +405,11 @@ table.colReorder.reset() }); + submit_special = function(url) { + $("#trait_submission_form").attr("action", url); + return $("#trait_submission_form").submit(); + }; + }); {% endblock %} -- cgit v1.2.3 From 61c13a09dba95958f183dc55f3d59c8856b5f753 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 13 Feb 2019 12:41:45 +0000 Subject: Removed pylmm references and related functions --- bin/genenetwork2 | 1 - etc/default_settings.py | 1 - test/requests/mapping_tests.py | 10 -- wqflask/base/data_set.py | 2 +- wqflask/utility/tools.py | 4 - wqflask/wqflask/heatmap/heatmap.py | 5 +- wqflask/wqflask/marker_regression/run_mapping.py | 204 +---------------------- 7 files changed, 4 insertions(+), 223 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 21f0db13..7c875274 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -131,7 +131,6 @@ else export LC_ALL=C # FIXME export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" - export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" if [ -z $GEMMA_WRAPPER_COMMAND ]; then export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" diff --git a/etc/default_settings.py b/etc/default_settings.py index 3e54ad1f..a1fe81e5 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -89,7 +89,6 @@ GENENETWORK_FILES = HOME+"/genotype_files" # base dir for all static data fil JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript" # ---- GN2 Executables (overwrite for testing only) -# PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) # PLINK_COMMAND = str.strip(os.popen("which plink2").read()) # GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) # GEMMA_WRAPPER_COMMAND = str.strip(os.popen("which gemma-wrapper").read()) diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py index 8eb19de7..6de81bfe 100644 --- a/test/requests/mapping_tests.py +++ b/test/requests/mapping_tests.py @@ -11,16 +11,6 @@ def load_data_from_file(): file_data = json.loads(file_handle.read().encode("utf-8")) return file_data -def check_pylmm_tool_selection(host, data): - print("") - print("pylmm mapping tool selection") - data["method"] = "pylmm" - page = requests.post(host+"/marker_regression", data=data) - doc = fromstring(page.text) - form = doc.forms[1] - assert form.fields["dataset"] == "HC_M2_0606_P" - assert form.fields["value:BXD1"] == "15.034" # Check value in the file - def check_R_qtl_tool_selection(host, data): print("") print("R/qtl mapping tool selection") diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 79f72390..ca6621e9 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -317,7 +317,7 @@ class DatasetGroup(object): mapping_id = g.db.execute("select MappingMethodId from InbredSet where Name= '%s'" % self.name).fetchone()[0] if mapping_id == "1": - mapping_names = ["QTLReaper", "PYLMM", "R/qtl"] + mapping_names = ["QTLReaper", "R/qtl"] elif mapping_id == "2": mapping_names = ["GEMMA"] elif mapping_id == "4": diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index ea216a35..86ef2e1e 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -107,9 +107,6 @@ def js_path(module=None): return try_guix raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)" -def pylmm_command(guess=None): - return assert_bin(get_setting("PYLMM_COMMAND",guess)) - def gemma_command(guess=None): return assert_bin(get_setting("GEMMA_COMMAND",guess)) @@ -276,7 +273,6 @@ SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') SMTP_PASSWORD = get_setting('SMTP_PASSWORD') -PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) assert(GEMMA_COMMAND is not None) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index ff589693..1bdf252b 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -24,12 +24,9 @@ import reaper from base.trait import GeneralTrait from base import data_set from base import species -# from wqflask.my_pylmm.pyLMM import lmm -# from wqflask.my_pylmm.pyLMM import input from utility import helper_functions from utility import Plot, Bunch from utility import temp_data -from utility.tools import PYLMM_COMMAND from MySQLdb import escape_string as escape @@ -144,4 +141,4 @@ class Heatmap(object): if qtl.additive > 0: self.trait_results[this_trait.name].append(-float(qtl.lrs)) else: - self.trait_results[this_trait.name].append(float(qtl.lrs)) \ No newline at end of file + self.trait_results[this_trait.name].append(float(qtl.lrs)) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 3057e340..73d985b8 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -38,7 +38,7 @@ from utility import temp_data from utility.benchmark import Bench from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping -from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR +from utility.tools import locate, locate_ignore_error, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR from utility.external import shell from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR @@ -239,11 +239,6 @@ class RunMapping(object): self.manhattan_plot = True results = plink_mapping.run_plink(self.this_trait, self.dataset, self.species, self.vals, self.maf) #results = self.run_plink() - elif self.mapping_method == "pylmm": - logger.debug("RUNNING PYLMM") - if self.num_perm > 0: - self.run_permutations(str(temp_uuid)) - results = self.gen_data(str(temp_uuid)) else: logger.debug("RUNNING NOTHING") @@ -354,201 +349,6 @@ class RunMapping(object): count, p_values = self.parse_rqtl_output(plink_output_filename) - def run_permutations(self, temp_uuid): - """Runs permutations and gets significant and suggestive LOD scores""" - - top_lod_scores = [] - - #logger.debug("self.num_perm:", self.num_perm) - - for permutation in range(self.num_perm): - - pheno_vector = np.array([val == "x" and np.nan or float(val) for val in self.vals]) - np.random.shuffle(pheno_vector) - - key = "pylmm:input:" + temp_uuid - - if self.dataset.group.species == "human": - p_values, t_stats = self.gen_human_results(pheno_vector, key, temp_uuid) - else: - genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers] - - no_val_samples = self.identify_empty_samples() - trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples) - - genotype_matrix = np.array(trimmed_genotype_data).T - - params = dict(pheno_vector = pheno_vector.tolist(), - genotype_matrix = genotype_matrix.tolist(), - restricted_max_likelihood = True, - refit = False, - temp_uuid = temp_uuid, - - # meta data - timestamp = datetime.datetime.now().isoformat(), - ) - - json_params = json.dumps(params) - Redis.set(key, json_params) - Redis.expire(key, 60*60) - - command = PYLMM_COMMAND+' --key {} --species {}'.format(key,"other") - shell(command) - - json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60) - results = json.loads(json_results[1]) - p_values = [float(result) for result in results['p_values']] - - lowest_p_value = 1 - for p_value in p_values: - if p_value < lowest_p_value: - lowest_p_value = p_value - - #logger.debug("lowest_p_value:", lowest_p_value) - top_lod_scores.append(-math.log10(lowest_p_value)) - - #logger.debug("top_lod_scores:", top_lod_scores) - - self.suggestive = np.percentile(top_lod_scores, 67) - self.significant = np.percentile(top_lod_scores, 95) - - def gen_data(self, temp_uuid): - """Generates p-values for each marker""" - - logger.debug("self.vals is:", self.vals) - pheno_vector = np.array([(val == "x" or val == "") and np.nan or float(val) for val in self.vals]) - - #lmm_uuid = str(uuid.uuid4()) - - key = "pylmm:input:" + temp_uuid - logger.debug("key is:", pf(key)) - #with Bench("Loading cache"): - # result = Redis.get(key) - - if self.dataset.group.species == "human": - p_values, t_stats = self.gen_human_results(pheno_vector, key, temp_uuid) - - else: - logger.debug("NOW CWD IS:", os.getcwd()) - genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers] - - no_val_samples = self.identify_empty_samples() - trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples) - - genotype_matrix = np.array(genotype_data).T - - #logger.debug("pheno_vector: ", pf(pheno_vector)) - #logger.debug("genotype_matrix: ", pf(genotype_matrix)) - #logger.debug("genotype_matrix.shape: ", pf(genotype_matrix.shape)) - - #params = {"pheno_vector": pheno_vector, - # "genotype_matrix": genotype_matrix, - # "restricted_max_likelihood": True, - # "refit": False, - # "temp_data": tempdata} - - # logger.debug("genotype_matrix:", str(genotype_matrix.tolist())) - # logger.debug("pheno_vector:", str(pheno_vector.tolist())) - - params = dict(pheno_vector = pheno_vector.tolist(), - genotype_matrix = genotype_matrix.tolist(), - restricted_max_likelihood = True, - refit = False, - temp_uuid = temp_uuid, - - # meta data - timestamp = datetime.datetime.now().isoformat(), - ) - - json_params = json.dumps(params) - #logger.debug("json_params:", json_params) - Redis.set(key, json_params) - Redis.expire(key, 60*60) - logger.debug("before printing command") - - command = PYLMM_COMMAND + ' --key {} --species {}'.format(key, "other") - logger.debug("command is:", command) - logger.debug("after printing command") - - shell(command) - - #t_stats, p_values = lmm.run(key) - #lmm.run(key) - - json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60) - results = json.loads(json_results[1]) - p_values = [float(result) for result in results['p_values']] - t_stats = results['t_stats'] - - #t_stats, p_values = lmm.run( - # pheno_vector, - # genotype_matrix, - # restricted_max_likelihood=True, - # refit=False, - # temp_data=tempdata - #) - #logger.debug("p_values:", p_values) - - self.dataset.group.markers.add_pvalues(p_values) - - return self.dataset.group.markers.markers - - def gen_human_results(self, pheno_vector, key, temp_uuid): - file_base = locate(self.dataset.group.name,"mapping") - - plink_input = input.plink(file_base, type='b') - input_file_name = os.path.join(webqtlConfig.SNP_PATH, self.dataset.group.name + ".snps.gz") - - pheno_vector = pheno_vector.reshape((len(pheno_vector), 1)) - covariate_matrix = np.ones((pheno_vector.shape[0],1)) - kinship_matrix = np.fromfile(open(file_base + '.kin','r'),sep=" ") - kinship_matrix.resize((len(plink_input.indivs),len(plink_input.indivs))) - - logger.debug("Before creating params") - - params = dict(pheno_vector = pheno_vector.tolist(), - covariate_matrix = covariate_matrix.tolist(), - input_file_name = input_file_name, - kinship_matrix = kinship_matrix.tolist(), - refit = False, - temp_uuid = temp_uuid, - - # meta data - timestamp = datetime.datetime.now().isoformat(), - ) - - logger.debug("After creating params") - - json_params = json.dumps(params) - Redis.set(key, json_params) - Redis.expire(key, 60*60) - - logger.debug("Before creating the command") - - command = PYLMM_COMMAND+' --key {} --species {}'.format(key, "human") - - logger.debug("command is:", command) - - os.system(command) - - json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60) - results = json.loads(json_results[1]) - t_stats = results['t_stats'] - p_values = results['p_values'] - - - #p_values, t_stats = lmm.run_human(key) - - #p_values, t_stats = lmm.run_human( - # pheno_vector, - # covariate_matrix, - # input_file_name, - # kinship_matrix, - # loading_progress=tempdata - # ) - - return p_values, t_stats - def identify_empty_samples(self): no_val_samples = [] for sample_count, val in enumerate(self.vals): @@ -657,4 +457,4 @@ def trim_markers_for_table(markers): trimmed_sorted_markers = sorted_markers[:2000] return trimmed_sorted_markers else: - return sorted_markers \ No newline at end of file + return sorted_markers -- cgit v1.2.3 From d515061c4878b448f5b866e32eee7e37301ecdb7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 18 Mar 2019 17:03:30 -0500 Subject: - Added PCA traits to correlation matrix. You can't change their names yet or add them to a collection from the corr matrix page, but you can click them and access them from the trait page. I'll add the option to access them from the corr matrix page as well, but adding the option to change their names might be trickier since they're currently used as their Redis keys. I need some better way of passing the Redis key around so it can be stored in collections, but this is tricky without changing the structure in ElasticSearch - Fixed the way temp traits work so you can use them with various functions, like mapping, correlations, network graph, etc - Fixed some appearance issue where the network graph options were too wide if a trait name was too long --- wqflask/base/data_set.py | 3 +- wqflask/base/trait.py | 40 +++++++++++++++------- wqflask/utility/helper_functions.py | 5 ++- wqflask/wqflask/collect.py | 5 ++- wqflask/wqflask/correlation/show_corr_results.py | 2 +- .../wqflask/correlation_matrix/show_corr_matrix.py | 37 ++++++++++++++++---- .../marker_regression/display_mapping_results.py | 11 +++--- wqflask/wqflask/templates/correlation_matrix.html | 7 ++++ wqflask/wqflask/templates/network_graph.html | 15 ++++++-- 9 files changed, 95 insertions(+), 30 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index ca6621e9..beb2a8a2 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -277,7 +277,6 @@ class DatasetGroup(object): """ def __init__(self, dataset, name=None): """This sets self.group and self.group_id""" - #logger.debug("DATASET NAME2:", dataset.name) if name == None: self.name, self.id, self.genetic_type = fetchone(dataset.query_for_group) else: @@ -500,7 +499,7 @@ class DataSet(object): self.setup() if self.type == "Temp": #Need to supply group name as input if temp trait - self.group = DatasetGroup(self, group_name) # sets self.group and self.group_id and gets genotype + self.group = DatasetGroup(self, name=group_name) # sets self.group and self.group_id and gets genotype else: self.check_confidentiality() self.retrieve_other_names() diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 79aa196f..0689e950 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -6,6 +6,9 @@ import codecs from htmlgen import HTMLgen2 as HT +import redis +Redis = redis.StrictRedis() + from base import webqtlConfig from base.webqtlCaseData import webqtlCaseData from base.data_set import create_dataset @@ -35,13 +38,15 @@ class GeneralTrait(object): def __init__(self, get_qtl_info=False, get_sample_info=True, **kw): # xor assertion assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name"; + self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. if kw.get('dataset_name'): - self.dataset = create_dataset(kw.get('dataset_name')) - #print(" in GeneralTrait created dataset:", self.dataset) + if kw.get('dataset_name') == "Temp": + temp_group = self.name.split("_")[2] + self.dataset = create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = temp_group) + else: + self.dataset = create_dataset(kw.get('dataset_name')) else: self.dataset = kw.get('dataset') - self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. - #print("THE NAME IS:", self.name) self.cellid = kw.get('cellid') self.identification = kw.get('identification', 'un-named trait') self.haveinfo = kw.get('haveinfo', False) @@ -73,8 +78,8 @@ class GeneralTrait(object): # So we could add a simple if statement to short-circuit this if necessary if self.dataset.type != "Temp": self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info) - if get_sample_info != False: - self = retrieve_sample_data(self, self.dataset) + if get_sample_info != False: + self = retrieve_sample_data(self, self.dataset) def export_informative(self, include_variance=0): """ @@ -154,18 +159,27 @@ def retrieve_sample_data(trait, dataset, samplelist=None): if samplelist == None: samplelist = [] - results = dataset.retrieve_sample_data(trait.name) + if dataset.type == "Temp": + results = Redis.get(trait.name).split() + else: + results = dataset.retrieve_sample_data(trait.name) # Todo: is this necessary? If not remove trait.data.clear() - all_samples_ordered = dataset.group.all_samples_ordered() - if results: - for item in results: - name, value, variance, num_cases, name2 = item - if not samplelist or (samplelist and name in samplelist): - trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) + if dataset.type == "Temp": + all_samples_ordered = dataset.group.all_samples_ordered() + for i, item in enumerate(results): + try: + trait.data[all_samples_ordered[i]] = webqtlCaseData(all_samples_ordered[i], float(item)) + except: + pass + else: + for item in results: + name, value, variance, num_cases, name2 = item + if not samplelist or (samplelist and name in samplelist): + trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) return trait @app.route("/trait/get_sample_data") diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 1c8dad10..6980af4e 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -45,7 +45,10 @@ def get_trait_db_obs(self, trait_db_list): data = data.strip() assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?" trait_name, dataset_name = data.split(":") - dataset_ob = data_set.create_dataset(dataset_name) + if dataset_name == "Temp": + dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2]) + else: + dataset_ob = data_set.create_dataset(dataset_name) trait_ob = GeneralTrait(dataset=dataset_ob, name=trait_name, cellid=None) diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index eb0e2726..6e1ac592 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -233,7 +233,10 @@ def collections_new(): collection_id = params['existing_collection'].split(":")[0] collection_name = params['existing_collection'].split(":")[1] if g.user_session.logged_in: - unprocessed_traits = Redis.get(params['hash']) + if "hash" in params: + unprocessed_traits = Redis.get(params['hash']) + else: + unprocessed_traits = params['traits'] traits = list(process_traits(unprocessed_traits)) g.user_session.add_traits_to_collection(collection_id, traits) return redirect(url_for('view_collection', uc_id=collection_id)) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index c15c3579..0c6b8a2b 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -93,7 +93,7 @@ class CorrelationResults(object): with Bench("Doing correlations"): if start_vars['dataset'] == "Temp": self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) - self.trait_id = "Temp" + self.trait_id = start_vars['trait_id'] self.this_trait = GeneralTrait(dataset=self.dataset, name=self.trait_id, cellid=None) diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 4bb4d65d..007e8e47 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -26,6 +26,7 @@ import sys import string import cPickle import os +import datetime import time import pp import math @@ -45,6 +46,9 @@ from pprint import pformat as pf from htmlgen import HTMLgen2 as HT import reaper +import redis +Redis = redis.StrictRedis() + from utility.THCell import THCell from utility.TDCell import TDCell from base.trait import GeneralTrait @@ -59,8 +63,10 @@ from MySQLdb import escape_string as escape from pprint import pformat as pf -from flask import Flask, g +from flask import Flask, g, url_for +import utility.logger +logger = utility.logger.getLogger(__name__ ) class CorrelationMatrix(object): @@ -111,6 +117,7 @@ class CorrelationMatrix(object): self.corr_results = [] self.pca_corr_results = [] self.trait_data_array = [] + self.shared_samples_list = self.all_sample_list for trait_db in self.trait_list: this_trait = trait_db[0] this_db = trait_db[1] @@ -138,6 +145,8 @@ class CorrelationMatrix(object): target_vals = [] for index, sample in enumerate(target_samples): if (sample in this_sample_data) and (sample in target_sample_data): + if sample not in self.shared_samples_list: + self.shared_samples_list.remove(sample) sample_value = this_sample_data[sample].value target_sample_value = target_sample_data[sample].value this_trait_vals.append(sample_value) @@ -174,6 +183,7 @@ class CorrelationMatrix(object): try: self.pca_works = "True" + self.pca_trait_ids = [] pca = self.calculate_pca(range(len(self.traits)), corr_eigen_value, corr_eigen_vectors) self.loadings_array = self.process_loadings() except: @@ -201,9 +211,6 @@ class CorrelationMatrix(object): cellid=None) self.trait_list.append((trait_ob, dataset_ob)) - #print("trait_list:", self.trait_list) - - def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors): base = importr('base') stats = importr('stats') @@ -224,9 +231,27 @@ class CorrelationMatrix(object): pca_traits = [] for i, vector in enumerate(trait_array_vectors): if corr_eigen_value[i-1] < 100.0/len(self.trait_list): - pca_traits.append(vector*-1.0) + pca_traits.append((vector*-1.0).tolist()) + + this_group_name = self.trait_list[0][1].group.name + temp_dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = this_group_name) + temp_dataset.group.get_samplelist() + for i, pca_trait in enumerate(pca_traits): + trait_id = "PCA" + str(i+1) + "_" + temp_dataset.group.species + "_" + this_group_name + "_" + datetime.datetime.now().strftime("%m%d%H%M%S") + this_vals_string = "" + position = 0 + for sample in temp_dataset.group.all_samples_ordered(): + if sample in self.shared_samples_list: + this_vals_string += str(pca_trait[position]) + this_vals_string += " " + position += 1 + else: + this_vals_string += "x " + this_vals_string = this_vals_string[:-1] + + Redis.set(trait_id, this_vals_string) + self.pca_trait_ids.append(trait_id) - print("pca_traits:", pca_traits) return pca def process_loadings(self): diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 41d5c9d0..39e0d712 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -932,10 +932,13 @@ class DisplayMappingResults(object): if self.this_trait.symbol: identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.symbol) - elif self.this_trait.post_publication_abbreviation: - identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.post_publication_abbreviation) - elif self.this_trait.pre_publication_abbreviation: - identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.pre_publication_abbreviation) + elif self.dataset.type == "Publish": + if self.this_trait.post_publication_abbreviation: + identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.post_publication_abbreviation) + elif self.this_trait.pre_publication_abbreviation: + identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.pre_publication_abbreviation) + else: + identification += "Trait: %s" % (self.this_trait.name) else: identification += "Trait: %s" % (self.this_trait.name) identification += " with %s samples" % (self.n_samples) diff --git a/wqflask/wqflask/templates/correlation_matrix.html b/wqflask/wqflask/templates/correlation_matrix.html index 8698b710..9c790780 100644 --- a/wqflask/wqflask/templates/correlation_matrix.html +++ b/wqflask/wqflask/templates/correlation_matrix.html @@ -60,6 +60,13 @@ {% if pca_works == "True" %}

+

PCA Traits

+ +

Factor Loadings Plot

Factor Loadings Table

diff --git a/wqflask/wqflask/templates/network_graph.html b/wqflask/wqflask/templates/network_graph.html index 6bee11e7..4492dd3f 100644 --- a/wqflask/wqflask/templates/network_graph.html +++ b/wqflask/wqflask/templates/network_graph.html @@ -34,7 +34,11 @@ @@ -46,11 +50,18 @@ +
+
-1
+ 0 +
1
+
+ + + -- cgit v1.2.3 From e304c2cfdf1b38da483efa607a2acd72b66ad0de Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 24 May 2019 11:35:12 -0500 Subject: Replaced qtlreaper Dataset object with python object --- wqflask/base/data_set.py | 7 ++- wqflask/utility/gen_geno_ob.py | 135 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 wqflask/utility/gen_geno_ob.py (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index beb2a8a2..b324ac74 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -43,6 +43,7 @@ from db import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks +from utility import gen_geno_ob from utility.tools import locate, locate_ignore_error, flat_files from maintenance import get_group_samplelists @@ -388,14 +389,16 @@ class DatasetGroup(object): #genotype_1 is Dataset Object without parents and f1 #genotype_2 is Dataset Object with parents and f1 (not for intercross) - genotype_1 = reaper.Dataset() + #genotype_1 = reaper.Dataset() # reaper barfs on unicode filenames, so here we ensure it's a string if self.genofile: full_filename = str(locate(self.genofile, 'genotype')) else: full_filename = str(locate(self.name + '.geno', 'genotype')) - genotype_1.read(full_filename) + #genotype_1.read(full_filename) + + genotype_1 = gen_geno_ob.genotype(full_filename) if genotype_1.type == "group" and self.parlist: genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1]) #, F1=_f1) diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py new file mode 100644 index 00000000..5824b0b3 --- /dev/null +++ b/wqflask/utility/gen_geno_ob.py @@ -0,0 +1,135 @@ +from __future__ import absolute_import, division, print_function + +class genotype(object): + """ + Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure + """ + + def __init__(self, filename): + self.group = None + self.type = "riset" + self.prgy = [] + self.nprgy = 0 + self.mat = -1 + self.pat = 1 + self.het = 0 + self.unk = "U" + self.filler = False + self.mb_exists = False + + #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary + self.cm_column = 2 + self.mb_column = 3 + + self.chromosomes = [] + + self.read_file(filename) + + def __iter__(self): + return iter(self.chromosomes) + + def __getitem__(self, index): + return self.chromosomes[index] + + def __len__(self): + return len(self.chromosomes) + + def read_file(self, filename): + + with open(filename, 'r') as geno_file: + lines = geno_file.readlines() + + this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers + chr_ob = None + for line in lines: + if line[0] == "#": + continue + elif line[0] == "@": + label = line.split(":")[0][1:] + if label == "name": + self.group = line.split(":")[1] + elif label == "filler": + if line.split(":")[1] == "yes": + self.filler = True + elif label == "type": + self.type = line.split(":")[1] + elif label == "mat": + self.mat = line.split(":")[1] + elif label == "pat": + self.pat = line.split(":")[1] + elif label == "het": + self.het = line.split(":")[1] + elif label == "unk": + self.unk = line.split(":")[1] + else: + continue + elif line[:3] == "Chr": + header_row = line.split("\t") + if header_row[2] == "Mb": + self.mb_exists = True + self.mb_column = 2 + self.cm_column = 3 + elif header_row[3] == "Mb": + self.mb_exists = True + self.mb_column = 3 + elif header_row[2] == "cM": + self.cm_column = 2 + + if self.mb_exists: + self.prgy = header_row[4:] + else: + self.prgy = header_row[3:] + self.nprgy = len(self.prgy) + else: + if line.split("\t")[0] != this_chr: + if this_chr != "": + self.chromosomes.append(chr_ob) + this_chr = line.split("\t")[0] + chr_ob = Chr(line.split("\t")[0], self) + chr_ob.add_marker(line.split("\t")) + +class Chr(object): + def __init__(self, name, geno_ob): + self.name = name + self.loci = [] + self.mb_exists = geno_ob.mb_exists + self.cm_column = geno_ob.cm_column + self.mb_column = geno_ob.mb_column + self.geno_ob = geno_ob + + def __iter__(self): + return iter(self.loci) + + def __getitem__(self, index): + return self.loci[index] + + def __len__(self): + return len(self.loci) + + def add_marker(self, marker_row): + self.loci.append(Locus(marker_row, self.geno_ob)) + +class Locus(object): + def __init__(self, marker_row, geno_ob): + self.chr = marker_row[0] + self.name = marker_row[1] + self.cM = float(marker_row[geno_ob.cm_column]) + self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None + + geno_table = { + geno_ob.mat: -1, + geno_ob.pat: 1, + geno_ob.het: 0, + geno_ob.unk: "U" + } + + self.genotype = [] + if geno_ob.mb_exists: + start_pos = 4 + else: + start_pos = 3 + for allele in marker_row[start_pos:]: + if allele in geno_table.keys(): + self.genotype.append(geno_table[allele]) + else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown + self.genotype.append("U") \ No newline at end of file -- cgit v1.2.3 From 66c6bbfcbbd5fb23145ec09956f4809e5f701bec Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 Jun 2019 13:01:36 -0500 Subject: Fixed issue that caused interval mapping to not work because the python implementation of the reaper Dataset object doesn't include the addinterval method (so for those situations I still use reaper) Fixed issue where the last chromosome wasn't displayed for mapping results (though still need to fix issue where points are drawn too far to the right when a specific range is viewed) --- wqflask/base/data_set.py | 9 +- wqflask/utility/gen_geno_ob.py | 270 +++++++++++---------- .../marker_regression/display_mapping_results.py | 6 +- .../wqflask/marker_regression/qtlreaper_mapping.py | 2 +- 4 files changed, 147 insertions(+), 140 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index b324ac74..1fd1792e 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -384,7 +384,7 @@ class DatasetGroup(object): [result.extend(l) for l in lists if l] return result - def read_genotype_file(self): + def read_genotype_file(self, use_reaper=False): '''Read genotype from .geno file instead of database''' #genotype_1 is Dataset Object without parents and f1 #genotype_2 is Dataset Object with parents and f1 (not for intercross) @@ -396,9 +396,12 @@ class DatasetGroup(object): full_filename = str(locate(self.genofile, 'genotype')) else: full_filename = str(locate(self.name + '.geno', 'genotype')) - #genotype_1.read(full_filename) - genotype_1 = gen_geno_ob.genotype(full_filename) + if use_reaper: + genotype_1 = reaper.Dataset() + genotype_1.read(full_filename) + else: + genotype_1 = gen_geno_ob.genotype(full_filename) if genotype_1.type == "group" and self.parlist: genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1]) #, F1=_f1) diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index 5824b0b3..5172369f 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -1,135 +1,137 @@ -from __future__ import absolute_import, division, print_function - -class genotype(object): - """ - Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure - """ - - def __init__(self, filename): - self.group = None - self.type = "riset" - self.prgy = [] - self.nprgy = 0 - self.mat = -1 - self.pat = 1 - self.het = 0 - self.unk = "U" - self.filler = False - self.mb_exists = False - - #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary - self.cm_column = 2 - self.mb_column = 3 - - self.chromosomes = [] - - self.read_file(filename) - - def __iter__(self): - return iter(self.chromosomes) - - def __getitem__(self, index): - return self.chromosomes[index] - - def __len__(self): - return len(self.chromosomes) - - def read_file(self, filename): - - with open(filename, 'r') as geno_file: - lines = geno_file.readlines() - - this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers - chr_ob = None - for line in lines: - if line[0] == "#": - continue - elif line[0] == "@": - label = line.split(":")[0][1:] - if label == "name": - self.group = line.split(":")[1] - elif label == "filler": - if line.split(":")[1] == "yes": - self.filler = True - elif label == "type": - self.type = line.split(":")[1] - elif label == "mat": - self.mat = line.split(":")[1] - elif label == "pat": - self.pat = line.split(":")[1] - elif label == "het": - self.het = line.split(":")[1] - elif label == "unk": - self.unk = line.split(":")[1] - else: - continue - elif line[:3] == "Chr": - header_row = line.split("\t") - if header_row[2] == "Mb": - self.mb_exists = True - self.mb_column = 2 - self.cm_column = 3 - elif header_row[3] == "Mb": - self.mb_exists = True - self.mb_column = 3 - elif header_row[2] == "cM": - self.cm_column = 2 - - if self.mb_exists: - self.prgy = header_row[4:] - else: - self.prgy = header_row[3:] - self.nprgy = len(self.prgy) - else: - if line.split("\t")[0] != this_chr: - if this_chr != "": - self.chromosomes.append(chr_ob) - this_chr = line.split("\t")[0] - chr_ob = Chr(line.split("\t")[0], self) - chr_ob.add_marker(line.split("\t")) - -class Chr(object): - def __init__(self, name, geno_ob): - self.name = name - self.loci = [] - self.mb_exists = geno_ob.mb_exists - self.cm_column = geno_ob.cm_column - self.mb_column = geno_ob.mb_column - self.geno_ob = geno_ob - - def __iter__(self): - return iter(self.loci) - - def __getitem__(self, index): - return self.loci[index] - - def __len__(self): - return len(self.loci) - - def add_marker(self, marker_row): - self.loci.append(Locus(marker_row, self.geno_ob)) - -class Locus(object): - def __init__(self, marker_row, geno_ob): - self.chr = marker_row[0] - self.name = marker_row[1] - self.cM = float(marker_row[geno_ob.cm_column]) - self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None - - geno_table = { - geno_ob.mat: -1, - geno_ob.pat: 1, - geno_ob.het: 0, - geno_ob.unk: "U" - } - - self.genotype = [] - if geno_ob.mb_exists: - start_pos = 4 - else: - start_pos = 3 - for allele in marker_row[start_pos:]: - if allele in geno_table.keys(): - self.genotype.append(geno_table[allele]) - else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown +from __future__ import absolute_import, division, print_function + +class genotype(object): + """ + Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure + """ + + def __init__(self, filename): + self.group = None + self.type = "riset" + self.prgy = [] + self.nprgy = 0 + self.mat = -1 + self.pat = 1 + self.het = 0 + self.unk = "U" + self.filler = False + self.mb_exists = False + + #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary + self.cm_column = 2 + self.mb_column = 3 + + self.chromosomes = [] + + self.read_file(filename) + + def __iter__(self): + return iter(self.chromosomes) + + def __getitem__(self, index): + return self.chromosomes[index] + + def __len__(self): + return len(self.chromosomes) + + def read_file(self, filename): + + with open(filename, 'r') as geno_file: + lines = geno_file.readlines() + + this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers + chr_ob = None + for line in lines: + if line[0] == "#": + continue + elif line[0] == "@": + label = line.split(":")[0][1:] + if label == "name": + self.group = line.split(":")[1] + elif label == "filler": + if line.split(":")[1] == "yes": + self.filler = True + elif label == "type": + self.type = line.split(":")[1] + elif label == "mat": + self.mat = line.split(":")[1] + elif label == "pat": + self.pat = line.split(":")[1] + elif label == "het": + self.het = line.split(":")[1] + elif label == "unk": + self.unk = line.split(":")[1] + else: + continue + elif line[:3] == "Chr": + header_row = line.split("\t") + if header_row[2] == "Mb": + self.mb_exists = True + self.mb_column = 2 + self.cm_column = 3 + elif header_row[3] == "Mb": + self.mb_exists = True + self.mb_column = 3 + elif header_row[2] == "cM": + self.cm_column = 2 + + if self.mb_exists: + self.prgy = header_row[4:] + else: + self.prgy = header_row[3:] + self.nprgy = len(self.prgy) + else: + if line.split("\t")[0] != this_chr: + if this_chr != "": + self.chromosomes.append(chr_ob) + this_chr = line.split("\t")[0] + chr_ob = Chr(line.split("\t")[0], self) + chr_ob.add_marker(line.split("\t")) + + self.chromosomes.append(chr_ob) + +class Chr(object): + def __init__(self, name, geno_ob): + self.name = name + self.loci = [] + self.mb_exists = geno_ob.mb_exists + self.cm_column = geno_ob.cm_column + self.mb_column = geno_ob.mb_column + self.geno_ob = geno_ob + + def __iter__(self): + return iter(self.loci) + + def __getitem__(self, index): + return self.loci[index] + + def __len__(self): + return len(self.loci) + + def add_marker(self, marker_row): + self.loci.append(Locus(marker_row, self.geno_ob)) + +class Locus(object): + def __init__(self, marker_row, geno_ob): + self.chr = marker_row[0] + self.name = marker_row[1] + self.cM = float(marker_row[geno_ob.cm_column]) + self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None + + geno_table = { + geno_ob.mat: -1, + geno_ob.pat: 1, + geno_ob.het: 0, + geno_ob.unk: "U" + } + + self.genotype = [] + if geno_ob.mb_exists: + start_pos = 4 + else: + start_pos = 3 + for allele in marker_row[start_pos:]: + if allele in geno_table.keys(): + self.genotype.append(geno_table[allele]) + else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown self.genotype.append("U") \ No newline at end of file diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 993fc2d9..e53e5279 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -236,9 +236,11 @@ class DisplayMappingResults(object): self.selectedChr = int(start_vars['selected_chr']) self.strainlist = start_vars['samples'] - self.genotype = self.dataset.group.read_genotype_file() + if self.mapping_method == "reaper" and self.manhattan_plot != True: - self.genotype = self.genotype.addinterval() + self.genotype = self.dataset.group.read_genotype_file(use_reaper=True) + else: + self.genotype = self.dataset.group.read_genotype_file() #Darwing Options try: diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index 35bed8d8..d58c59c8 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -2,7 +2,7 @@ import utility.logger logger = utility.logger.getLogger(__name__ ) def gen_reaper_results(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot): - genotype = dataset.group.read_genotype_file() + genotype = dataset.group.read_genotype_file(use_reaper=True) if manhattan_plot != True: genotype = genotype.addinterval() -- cgit v1.2.3 From 172bf33d20d6f42650b415571c1185af4cbd22c5 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 Jun 2019 15:39:36 -0500 Subject: Fixed issue with sorting involving 0's on trait page Fixed links to dataset info pages in search results and correlation results pages --- wqflask/base/data_set.py | 41 ++++++++++++++--------- wqflask/wqflask/templates/correlation_page.html | 4 +-- wqflask/wqflask/templates/search_result_page.html | 2 +- wqflask/wqflask/templates/show_trait.html | 4 +-- 4 files changed, 31 insertions(+), 20 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 1fd1792e..4fee5c7a 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -289,7 +289,6 @@ class DatasetGroup(object): self.parlist = None self.get_f1_parent_strains() - self.accession_id = self.get_accession_id() self.mapping_id, self.mapping_names = self.get_mapping_methods() self.species = webqtlDatabaseFunction.retrieve_species(self.name) @@ -299,20 +298,6 @@ class DatasetGroup(object): self._datasets = None self.genofile = None - def get_accession_id(self): - results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where - InbredSet.Name = %s and - PublishFreeze.InbredSetId = InbredSet.Id and - InfoFiles.InfoPageName = PublishFreeze.Name and - PublishFreeze.public > 0 and - PublishFreeze.confidentiality < 1 order by - PublishFreeze.CreateTime desc""", (self.name)).fetchone() - - if results != None: - return str(results[0]) - else: - return "None" - def get_mapping_methods(self): mapping_id = g.db.execute("select MappingMethodId from InbredSet where Name= '%s'" % self.name).fetchone()[0] @@ -510,6 +495,7 @@ class DataSet(object): self.check_confidentiality() self.retrieve_other_names() self.group = DatasetGroup(self) # sets self.group and self.group_id and gets genotype + self.accession_id = self.get_accession_id() if get_samplelist == True: self.group.get_samplelist() self.species = species.TheSpecies(self) @@ -524,6 +510,31 @@ class DataSet(object): def riset(): Weve_Renamed_This_As_Group + def get_accession_id(self): + if self.type == "Publish": + results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where + InbredSet.Name = %s and + PublishFreeze.InbredSetId = InbredSet.Id and + InfoFiles.InfoPageName = PublishFreeze.Name and + PublishFreeze.public > 0 and + PublishFreeze.confidentiality < 1 order by + PublishFreeze.CreateTime desc""", (self.group.name)).fetchone() + elif self.type == "Geno": + results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, GenoFreeze, InbredSet where + InbredSet.Name = %s and + GenoFreeze.InbredSetId = InbredSet.Id and + InfoFiles.InfoPageName = GenoFreeze.ShortName and + GenoFreeze.public > 0 and + GenoFreeze.confidentiality < 1 order by + GenoFreeze.CreateTime desc""", (self.group.name)).fetchone() + else: + results = None + + if results != None: + return str(results[0]) + else: + return "None" + def retrieve_other_names(self): """This method fetches the the dataset names in search_result. diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 7cc998bb..32f5e774 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -12,8 +12,8 @@
-

Values of record {{ this_trait.name }} in the {{ dataset.fullname }} - dataset were compared to all records in the {{ target_dataset.fullname }} +

Values of record {{ this_trait.name }} in the {{ dataset.fullname }} + dataset were compared to all records in the {{ target_dataset.fullname }} dataset. The top {{ return_number }} correlations ranked by the {{ formatted_corr_type }} are displayed. You can resort this list by clicking the headers. Select the Record ID to open the trait data and analysis page. diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 61a533f7..374347bd 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -13,7 +13,7 @@

-

We searched {{ dataset.fullname }} +

We searched {{ dataset.fullname }} to find all records {% for word in search_terms %} {% if word.key|lower == "rif" %} diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index 378f91b1..d5473bca 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -183,10 +183,10 @@ var x = getValue(a); var y = getValue(b); - if (x == 'x' || x == '') { + if (x == 'x' || x === '') { return 1; } - else if (y == 'x' || y == '') { + else if (y == 'x' || y === '') { return -1; } else { -- cgit v1.2.3 From 4a7c35204863066dc387637bd0f8af7d274cde55 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 24 Jun 2019 12:09:35 -0500 Subject: Got non-LOCO GEMMA mapping working with gemma-wrapper (so caching should work for that now) Fixed position digits and row highlighting on interval analyst table Updated default MAF to 0.05 Updated footer text --- wqflask/base/data_set.py | 2 +- .../marker_regression/display_mapping_results.py | 10 +-- wqflask/wqflask/marker_regression/gemma_mapping.py | 73 +++++++++++----------- wqflask/wqflask/show_trait/show_trait.py | 2 +- wqflask/wqflask/templates/base.html | 42 ++++++++----- wqflask/wqflask/templates/mapping_results.html | 8 ++- 6 files changed, 74 insertions(+), 63 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 4fee5c7a..d766e284 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -433,7 +433,7 @@ def datasets(group_name, this_group = None): and InbredSet.Name like %s and ProbeSetFreeze.public > %s and ProbeSetFreeze.confidentiality < 1 - ORDER BY Tissue.Name) + ORDER BY Tissue.Name, ProbeSetFreeze.OrderList DESC) ''' % (group_name, webqtlConfig.PUBLICTHRESH, group_name, webqtlConfig.PUBLICTHRESH, "'" + group_name + "'", webqtlConfig.PUBLICTHRESH)) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index cafbf38c..41cdf819 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2127,7 +2127,7 @@ class DisplayMappingResults(object): tableIterationsCnt = tableIterationsCnt + 1 this_row = [] #container for the cells of each row - selectCheck = HT.Input(type="checkbox", name="searchResult", value=theGO["GeneSymbol"], Class="checkbox trait_checkbox") #checkbox for each row + selectCheck = HT.Input(type="checkbox", name="selectCheck", value=theGO["GeneSymbol"], Class="checkbox trait_checkbox") #checkbox for each row geneLength = (theGO["TxEnd"] - theGO["TxStart"])*1000.0 tenPercentLength = geneLength*0.0001 @@ -2213,7 +2213,7 @@ class DisplayMappingResults(object): elif self.dataset.group.species == 'rat': for gIndex, theGO in enumerate(geneCol): this_row = [] #container for the cells of each row - selectCheck = HT.Input(type="checkbox", name="searchResult", Class="checkbox", onClick="highlight(this)").__str__() #checkbox for each row + selectCheck = HT.Input(type="checkbox", name="selectCheck", Class="checkbox trait_checkbox").__str__() #checkbox for each row #ZS: May want to get this working again later #webqtlSearch = HT.Href(os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE)+"?cmd=sch&gene=%s&alias=1&species=rat" % theGO["GeneSymbol"], ">>", target="_blank").__str__() @@ -2240,14 +2240,14 @@ class DisplayMappingResults(object): #Mouse Gene if theGO['mouseGene']: mouseChr = theGO['mouseGene']["Chromosome"] - mouseTxStart = theGO['mouseGene']["TxStart"] + mouseTxStart = "%0.6f" % theGO['mouseGene']["TxStart"] else: mouseChr = mouseTxStart = "" #the chromosomes for human 1 are 1qXX.XX if theGO['humanGene']: humanChr = theGO['humanGene']["Chromosome"] - humanTxStart = theGO['humanGene']["TxStart"] + humanTxStart = "%0.6f" % theGO['humanGene']["TxStart"] else: humanChr = humanTxStart = "" @@ -2258,7 +2258,7 @@ class DisplayMappingResults(object): this_row = [selectCheck.__str__(), str(gIndex+1), geneSymbolNCBI, - theGO["TxStart"], + "%0.6f" % theGO["TxStart"], HT.Href(geneLengthURL, "%0.3f" % (geneLength*1000.0)).__str__(), avgExprVal, mouseChr, diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index 0f37e711..4e3c203d 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -73,50 +73,51 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, maf gwa_output_filename) else: - generate_k_command = GEMMA_COMMAND + ' ' + GEMMAOPTS + ' -g %s/%s_geno.txt -p %s/gn2/%s.txt -a %s/%s_snps.txt -gk -outdir %s/gn2/ -o %s' % (flat_files('genotype/bimbam'), - genofile_name, - TEMPDIR, - trait_filename, - flat_files('genotype/bimbam'), - genofile_name, - TEMPDIR, - k_output_filename) - #generate_k_command = GEMMA_WRAPPER_COMMAND + ' --json -- ' + GEMMAOPTS + ' -g %s/%s_geno.txt -p %s/%s.txt -a %s/%s_snps.txt -gk > %s/gn2/%s.json' % (flat_files('genotype/bimbam'), - # genofile_name, - # flat_files('genotype/bimbam'), - # trait_filename, - # flat_files('genotype/bimbam'), - # genofile_name, - # TEMPDIR, - # k_output_filename) + # generate_k_command = GEMMA_COMMAND + ' ' + GEMMAOPTS + ' -g %s/%s_geno.txt -p %s/gn2/%s.txt -a %s/%s_snps.txt -gk -outdir %s/gn2/ -o %s' % (flat_files('genotype/bimbam'), + # genofile_name, + # TEMPDIR, + # trait_filename, + # flat_files('genotype/bimbam'), + # genofile_name, + # TEMPDIR, + # k_output_filename) + generate_k_command = GEMMA_WRAPPER_COMMAND + ' --json -- ' + GEMMAOPTS + ' -g %s/%s_geno.txt -p %s/gn2/%s.txt -a %s/%s_snps.txt -gk > %s/gn2/%s.json' % (flat_files('genotype/bimbam'), + genofile_name, + TEMPDIR, + trait_filename, + flat_files('genotype/bimbam'), + genofile_name, + TEMPDIR, + k_output_filename) logger.debug("k_command:" + generate_k_command) os.system(generate_k_command) - gemma_command = GEMMA_COMMAND + ' ' + GEMMAOPTS + ' -g %s/%s_geno.txt -p %s/gn2/%s.txt -a %s/%s_snps.txt -k %s/gn2/%s.cXX.txt -lmm 2 -maf %s' % (flat_files('genotype/bimbam'), - genofile_name, - TEMPDIR, - trait_filename, - flat_files('genotype/bimbam'), - genofile_name, - TEMPDIR, - k_output_filename, - maf) + # gemma_command = GEMMA_COMMAND + ' ' + GEMMAOPTS + ' -g %s/%s_geno.txt -p %s/gn2/%s.txt -a %s/%s_snps.txt -k %s/gn2/%s.cXX.txt -lmm 2 -maf %s' % (flat_files('genotype/bimbam'), + # genofile_name, + # TEMPDIR, + # trait_filename, + # flat_files('genotype/bimbam'), + # genofile_name, + # TEMPDIR, + # k_output_filename, + # maf) - #gemma_command = GEMMA_WRAPPER_COMMAND + ' --json --input %s/gn2/%s.json -- ' % (TEMPDIR, k_output_filename) + GEMMAOPTS + ' -g %s/%s_geno.txt -p %s/%s_pheno.txt' % (flat_files('genotype/bimbam'), - # genofile_name, - # flat_files('genotype/bimbam'), - # genofile_name) + gemma_command = GEMMA_WRAPPER_COMMAND + ' --json --input %s/gn2/%s.json -- ' % (TEMPDIR, k_output_filename) + GEMMAOPTS + ' -lmm 2 -g %s/%s_geno.txt -p %s/gn2/%s.txt' % (flat_files('genotype/bimbam'), + genofile_name, + TEMPDIR, + trait_filename) if covariates != "": - gemma_command += ' -c %s/%s_covariates.txt -outdir %s -o %s_output' % (flat_files('mapping'), - this_dataset.group.name, - webqtlConfig.GENERATED_IMAGE_DIR, - genofile_name) - else: - gemma_command += ' -outdir %s -o %s_output' % (webqtlConfig.GENERATED_IMAGE_DIR, - genofile_name) + gemma_command += ' -c %s/%s_covariates.txt' % (flat_files('mapping'), this_dataset.group.name) + # gemma_command += ' -c %s/%s_covariates.txt -outdir %s -o %s_output' % (flat_files('mapping'), + # this_dataset.group.name, + # webqtlConfig.GENERATED_IMAGE_DIR, + # genofile_name) + # else: + # gemma_command += ' -outdir %s -o %s_output' % (webqtlConfig.GENERATED_IMAGE_DIR, + # genofile_name) logger.debug("gemma_command:" + gemma_command) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index e10b31c0..5178ece8 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -168,7 +168,7 @@ class ShowTrait(object): hddn['control_marker'] = self.nearest_marker #hddn['control_marker'] = self.nearest_marker1+","+self.nearest_marker2 hddn['do_control'] = False - hddn['maf'] = 0.01 + hddn['maf'] = 0.05 hddn['compare_traits'] = [] hddn['export_data'] = "" hddn['export_format'] = "excel" diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 21fc99d3..3fd9faf5 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -127,27 +127,35 @@ JOSS


-

GeneNetwork is supported by:

-
    -
  • - +

    GeneNetwork support from:

    +
      +
    • + The UT Center for Integrative and Translational Genomics - +
      • -
    • NIAAA - Integrative Neuroscience Initiative on Alcoholism - (U01 AA016662, U01 AA013499, U24 AA013513, U01 AA014425) +
    • + NIGMS + Systems Genetics and Precision Medicine Project (R01 GM123489, 2017-2021)
      • -
    • - NIDA, NIMH - , and - NIAAA (P20-DA 21131) +
    • + NIDA + NIDA Core Center of Excellence in Transcriptomics, Systems Genetics, and the Addictome (P30 DA044223, 2017-2022)
      • -
    • NCI MMHCC (U01CA105417) and - NCRR - BIRN - (U24 RR021760) +
    • + NIA + Translational Systems Genetics of Mitochondria, Metabolism, and Aging (R01AG043930, 2013-2018) +
      • +
    • + NIAAA + Integrative Neuroscience Initiative on Alcoholism (U01 AA016662, U01 AA013499, U24 AA013513, U01 AA014425, 2006-2017) +
      • +
    • + NIDA, NIMH, and NIAAA + (P20-DA 21131, 2001-2012) +
      • +
    • + NCI MMHCC (U01CA105417), NCRR, BIRN, (U24 RR021760)
    diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index ba4d4d7c..00a7b811 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -262,7 +262,7 @@

    Interval Analyst

    - +
    {% for header in gene_table_header %} @@ -322,6 +322,7 @@ + - - diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index a806a8b3..ad7412b2 100644 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -4,25 +4,18 @@
      - {% if dataset.group.mapping_id == "1" %} -
    • - GEMMA -
      • -
    • - Haley-Knott Regression -
      • -
    • - R/qtl -
      • - {% endif %} {% for mapping_method in dataset.group.mapping_names %} {% if mapping_method == "GEMMA" %} -
    • +
    • GEMMA
      • - {% elif mapping_method == "PLINK" %} -
    • - PLINK + {% elif mapping_method == "R/qtl" %} +
    • + R/qtl +
      • + {% elif mapping_method == "QTLReaper" %} +
    • + Haley-Knott Regression
      • {% endif %} {% endfor %} @@ -32,7 +25,9 @@
    -
    + {% for mapping_method in dataset.group.mapping_names %} + {% if mapping_method == "GEMMA" %} +
    @@ -102,7 +97,7 @@
    - {% if dataset.group.mapping_id == "1" %} + {% elif mapping_method == "QTLReaper" %}
    @@ -217,7 +212,8 @@
    -
    + {% elif mapping_method == "R/qtl" %} +
    @@ -341,19 +337,24 @@
    {% endif %} + {% endfor %}
    + {% for mapping_method in dataset.group.mapping_names %} + {% if mapping_method == "GEMMA" %}
    GEMMA
    Maps traits with correction for kinship among samples using a linear mixed model method, and also allows users to fit multiple covariates such as sex, age, treatment, and genetic markers (PMID: 2453419, and GitHub code). GEMMA incorporates the Leave One Chromosome Out (LOCO) method to ensure that the correction for kinship does not remove useful genetic variance near each marker. Markers can be filtered to include only those with minor allele frequencies (MAF) above a threshold. The default MAF is 0.05.
    - {% if dataset.group.mapping_id == "1" %} + {% elif mapping_method == "R/qtl" %}
    R/qtl
    Major upgrade of R/qtl that supports most experimental populations including those with complex admixture and two or more parental lines as well as large omic data sets (PMID: 30591514). Both R/qtl and R/qtl2 are available as stand-alone R packages (R suite).
    + {% elif mapping_method == "QTLReaper" %}
    Haley-Knott Regression
    Fast linear mapping method (PMID 16718932) works well with F2 intercrosses and backcrosses, but that is not recommended for complex or admixed populations (e.g., GWAS or heterogeneous stock studies) or for advanced intercrosses, recombinant inbred families, or diallel crosses. Interactive plots in GeneNetwork have relied on the fast HK mapping for two decades and we still use this method for mapping omics data sets and computing genome-wide permutation threshold (QTL Reaper code).
    {% endif %} + {% endfor %}

    GeneNetwork v2:

    GeneNetwork v1:

      diff --git a/wqflask/wqflask/templates/network_graph.html b/wqflask/wqflask/templates/network_graph.html index 4492dd3f..25af7bb1 100644 --- a/wqflask/wqflask/templates/network_graph.html +++ b/wqflask/wqflask/templates/network_graph.html @@ -137,6 +137,7 @@ -- cgit v1.2.3 From f0139ef93c7e8329c1a84fe9f75273c36626df0e Mon Sep 17 00:00:00 2001 From: zsloan Date: Sun, 26 Apr 2020 10:57:09 -0500 Subject: Removed the code that was still writing the database menu JSON --- wqflask/base/data_set.py | 14 +++++--------- wqflask/wqflask/api/gen_menu.py | 5 ----- 2 files changed, 5 insertions(+), 14 deletions(-) (limited to 'wqflask/base/data_set.py') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8652e6b7..1f99df49 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -64,7 +64,7 @@ logger = getLogger(__name__ ) # Each subclass will add to this DS_NAME_MAP = {} -def create_dataset(dataset_name, rebuild=True, dataset_type = None, get_samplelist = True, group_name = None): +def create_dataset(dataset_name, dataset_type = None, get_samplelist = True, group_name = None): if not dataset_type: dataset_type = Dataset_Getter(dataset_name) @@ -77,7 +77,7 @@ def create_dataset(dataset_name, rebuild=True, dataset_type = None, get_sampleli class Dataset_Types(object): - def __init__(self, rebuild=False): + def __init__(self): """Create a dictionary of samples where the value is set to Geno, Publish or ProbeSet. E.g. @@ -93,13 +93,9 @@ Publish or ProbeSet. E.g. """ self.datasets = {} - if rebuild: #ZS: May make this the only option - data = json.loads(requests.get(GN2_BASE_URL + "/api/v_pre1/gen_dropdown").content) - #data = gen_menu.gen_dropdown_json() - else: - file_name = "wqflask/static/new/javascript/dataset_menu_structure.json" - with open(file_name, 'r') as fh: - data = json.load(fh) + data = json.loads(requests.get(GN2_BASE_URL + "/api/v_pre1/gen_dropdown").content) + #data = gen_menu.gen_dropdown_json() + for species in data['datasets']: for group in data['datasets'][species]: diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index 64a413c2..adf66fb5 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -36,11 +36,6 @@ def gen_dropdown_json(): types=types, datasets=datasets) - output_file = """./wqflask/static/new/javascript/dataset_menu_structure.json""" - - with open(output_file, 'w') as fh: - json.dump(data, fh, indent=3, sort_keys=True) - return data def get_species(): -- cgit v1.2.3