diff options
Diffstat (limited to 'wqflask')
-rw-r--r-- | wqflask/wqflask/heatmap/heatmap.py | 156 | ||||
-rwxr-xr-x | wqflask/wqflask/marker_regression/marker_regression.py | 3 | ||||
-rw-r--r-- | wqflask/wqflask/static/new/javascript/create_heatmap.coffee | 18 | ||||
-rw-r--r-- | wqflask/wqflask/static/new/javascript/create_heatmap.js | 14 | ||||
-rw-r--r-- | wqflask/wqflask/static/new/javascript/lodheatmap.coffee | 2 | ||||
-rw-r--r-- | wqflask/wqflask/static/new/javascript/lodheatmap.js | 274 | ||||
-rw-r--r-- | wqflask/wqflask/templates/heatmap.html | 41 | ||||
-rwxr-xr-x | wqflask/wqflask/views.py | 54 |
8 files changed, 524 insertions, 38 deletions
diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index d9c02381..8f910fa9 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -7,6 +7,7 @@ import gc import string
import cPickle
import os
+import datetime
import time
import pp
import math
@@ -14,38 +15,50 @@ import collections import resource
import scipy
+import numpy as np
+from scipy import linalg
from pprint import pformat as pf
from htmlgen import HTMLgen2 as HT
import reaper
-from base import webqtlConfig
-from utility.THCell import THCell
-from utility.TDCell import TDCell
from base.trait import GeneralTrait
from base import data_set
-from base.templatePage import templatePage
-from utility import webqtlUtil, helper_functions, corr_result_helpers
-from dbFunction import webqtlDatabaseFunction
-import utility.webqtlUtil #this is for parallel computing only.
-from wqflask.correlation import correlation_functions
-from utility.benchmark import Bench
+from base import species
+from base import webqtlConfig
+from utility import webqtlUtil
+from wqflask.my_pylmm.data import prep_data
+from wqflask.my_pylmm.pyLMM import lmm
+from wqflask.my_pylmm.pyLMM import input
+from utility import helper_functions
+from utility import Plot, Bunch
+from utility import temp_data
from MySQLdb import escape_string as escape
+import cPickle as pickle
+import simplejson as json
+
from pprint import pformat as pf
+from redis import Redis
+Redis = Redis()
+
from flask import Flask, g
class Heatmap(object):
- def __init__(self, start_vars):
+ def __init__(self, start_vars, temp_uuid):
trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')]
helper_functions.get_trait_db_obs(self, trait_db_list)
+ self.dataset = self.trait_list[0][1]
+
+ self.json_data = {} #The dictionary that will be used to create the json object that contains all the data needed to create the figure
+
self.all_sample_list = []
self.traits = []
for trait_db in self.trait_list:
@@ -76,19 +89,122 @@ class Heatmap(object): self.trait_results = {}
for trait_db in self.trait_list:
this_trait = trait_db[0]
- this_db = trait_db[1]
+ #this_db = trait_db[1]
+ self.dataset.group.get_markers()
- this_db_samples = this_db.group.samplelist
+ this_db_samples = self.dataset.group.samplelist
this_sample_data = this_trait.data
- print("this_sample_data", this_sample_data)
- this_trait_vals = []
- for index, sample in enumerate(target_samples):
+ #print("this_sample_data", this_sample_data)
+ this_trait_vals = []
+ for index, sample in enumerate(this_db_samples):
+ if sample in this_sample_data:
+ sample_value = this_sample_data[sample].value
+ this_trait_vals.append(sample_value)
+ else:
+ this_trait_vals.append("x")
- if (sample in this_sample_data) and (sample in target_sample_data):
- sample_value = this_sample_data[sample].value
- target_sample_value = target_sample_data[sample].value
- this_trait_vals.append(sample_value)
- target_vals.append(target_sample_value)
+ pheno_vector = np.array([val == "x" and np.nan or float(val) for val in this_trait_vals])
+
+ key = "pylmm:input:" + str(temp_uuid)
+ #print("key is:", pf(key))
+
+ genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers]
+
+ no_val_samples = self.identify_empty_samples(this_trait_vals)
+ trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples)
+
+ genotype_matrix = np.array(trimmed_genotype_data).T
+
+ #print("genotype_matrix:", str(genotype_matrix.tolist()))
+ #print("pheno_vector:", str(pheno_vector.tolist()))
+
+ params = dict(pheno_vector = pheno_vector.tolist(),
+ genotype_matrix = genotype_matrix.tolist(),
+ restricted_max_likelihood = True,
+ refit = False,
+ temp_uuid = str(temp_uuid),
+
+ # meta data
+ timestamp = datetime.datetime.now().isoformat(),
+ )
+ json_params = json.dumps(params)
+ #print("json_params:", json_params)
+ Redis.set(key, json_params)
+ Redis.expire(key, 60*60)
+ print("before printing command")
+
+ command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key,
+ "other")
+ print("command is:", command)
+ print("after printing command")
+
+ os.system(command)
+
+ json_results = Redis.blpop("pylmm:results:" + str(temp_uuid), 45*60)
+ results = json.loads(json_results[1])
+ p_values = [float(result) for result in results['p_values']]
+ #print("p_values:", p_values)
+ self.dataset.group.markers.add_pvalues(p_values)
+
+ self.trait_results[this_trait.name] = []
+ for marker in self.dataset.group.markers.markers:
+ self.trait_results[this_trait.name].append(marker['lod_score'])
+
+ #print("self.trait_results:", self.trait_results)
+
+ chrnames = []
+ lodnames = []
+ chr_pos = []
+ pos = []
+ markernames = []
+
+ for trait in self.trait_results.keys():
+ lodnames.append(trait)
+
+ for marker in self.dataset.group.markers.markers:
+ if marker['chr'] not in chrnames:
+ chrnames.append(marker['chr'])
+ chr_pos.append(marker['chr'])
+ pos.append(marker['Mb'])
+ markernames.append(marker['name'])
+
+ self.json_data['chrnames'] = chrnames
+ self.json_data['lodnames'] = lodnames
+ self.json_data['chr'] = chr_pos
+ self.json_data['pos'] = pos
+ self.json_data['markernames'] = markernames
+
+ for trait in self.trait_results:
+ self.json_data[trait] = self.trait_results[trait]
+
+ self.js_data = dict(
+ json_data = self.json_data
+ )
+
+ print("self.js_data:", self.js_data)
+
+ def identify_empty_samples(self, values):
+ no_val_samples = []
+ for sample_count, val in enumerate(values):
+ if val == "x":
+ no_val_samples.append(sample_count)
+ return no_val_samples
+
+ def trim_genotypes(self, genotype_data, no_value_samples):
+ trimmed_genotype_data = []
+ for marker in genotype_data:
+ new_genotypes = []
+ for item_count, genotype in enumerate(marker):
+ if item_count in no_value_samples:
+ continue
+ try:
+ genotype = float(genotype)
+ except ValueError:
+ genotype = np.nan
+ pass
+ new_genotypes.append(genotype)
+ trimmed_genotype_data.append(new_genotypes)
+ return trimmed_genotype_data
\ No newline at end of file diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 0bee5994..0a83f9f8 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -428,6 +428,9 @@ class MarkerRegression(object): # "refit": False,
# "temp_data": tempdata}
+ print("genotype_matrix:", str(genotype_matrix.tolist()))
+ print("pheno_vector:", str(pheno_vector.tolist()))
+
params = dict(pheno_vector = pheno_vector.tolist(),
genotype_matrix = genotype_matrix.tolist(),
restricted_max_likelihood = True,
diff --git a/wqflask/wqflask/static/new/javascript/create_heatmap.coffee b/wqflask/wqflask/static/new/javascript/create_heatmap.coffee index 2bcb5c0e..51ca5e8f 100644 --- a/wqflask/wqflask/static/new/javascript/create_heatmap.coffee +++ b/wqflask/wqflask/static/new/javascript/create_heatmap.coffee @@ -1,12 +1,18 @@ -h = 700
-w = 1000
+create_heatmap = () ->
+
+ h = 700
+ w = 1000
-# Example: simplest use
-d3.json "data.json", (data) ->
mychart = lodheatmap().height(h)
.width(w)
- .zthresh(1.0)
+ .zthresh(0.5)
+
+ data = js_data.json_data
+
+ console.log("data:", data)
d3.select("div#chart")
.datum(data)
- .call(mychart)
\ No newline at end of file + .call(mychart)
+
+create_heatmap()
\ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/create_heatmap.js b/wqflask/wqflask/static/new/javascript/create_heatmap.js new file mode 100644 index 00000000..7d6f7a9d --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/create_heatmap.js @@ -0,0 +1,14 @@ +// Generated by CoffeeScript 1.7.1 +var create_heatmap; + +create_heatmap = function() { + var data, h, mychart, w; + h = 700; + w = 1000; + mychart = lodheatmap().height(h).width(w).zthresh(0.5); + data = js_data.json_data; + console.log("data:", data); + return d3.select("div#chart").datum(data).call(mychart); +}; + +create_heatmap(); diff --git a/wqflask/wqflask/static/new/javascript/lodheatmap.coffee b/wqflask/wqflask/static/new/javascript/lodheatmap.coffee index 9d0687d0..6cb8bfc7 100644 --- a/wqflask/wqflask/static/new/javascript/lodheatmap.coffee +++ b/wqflask/wqflask/static/new/javascript/lodheatmap.coffee @@ -3,7 +3,7 @@ lodheatmap = () ->
width = 1200
height = 600
- margin = {left:60, top:40, right:40, bottom: 40}
+ margin = {left:100, top:40, right:40, bottom: 40}
axispos = {xtitle:25, ytitle:30, xlabel:5, ylabel:5}
chrGap = 8
titlepos = 20
diff --git a/wqflask/wqflask/static/new/javascript/lodheatmap.js b/wqflask/wqflask/static/new/javascript/lodheatmap.js new file mode 100644 index 00000000..447287da --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/lodheatmap.js @@ -0,0 +1,274 @@ +// Generated by CoffeeScript 1.7.1 +var lodheatmap; + +lodheatmap = function() { + var axispos, cellSelect, chart, chrGap, colors, height, margin, rectcolor, rotate_ylab, title, titlepos, width, xlab, xscale, ylab, yscale, zlim, zscale, zthresh; + width = 1200; + height = 600; + margin = { + left: 100, + top: 40, + right: 40, + bottom: 40 + }; + axispos = { + xtitle: 25, + ytitle: 30, + xlabel: 5, + ylabel: 5 + }; + chrGap = 8; + titlepos = 20; + rectcolor = d3.rgb(230, 230, 230); + colors = ["slateblue", "white", "crimson"]; + title = ""; + xlab = "Chromosome"; + ylab = ""; + rotate_ylab = null; + zlim = null; + zthresh = null; + xscale = d3.scale.linear(); + yscale = d3.scale.linear(); + zscale = d3.scale.linear(); + cellSelect = null; + chart = function(selection) { + return selection.each(function(data) { + var cells, celltip, chr, extent, g, gEnter, i, j, lod, lodcol, nlod, pos, rectHeight, svg, titlegrp, xLR, xaxis, yaxis, zmax, zmin, _i, _j, _k, _l, _len, _len1, _len2, _len3, _len4, _m, _ref, _ref1, _ref2, _ref3, _ref4; + data = reorgLodData(data); + data = chrscales(data, width, chrGap, margin.left, true); + xscale = data.xscale; + nlod = data.lodnames.length; + yscale.domain([-0.5, nlod - 0.5]).range([margin.top + height, margin.top]); + rectHeight = yscale(0) - yscale(1); + xLR = {}; + _ref = data.chrnames; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + chr = _ref[_i]; + xLR[chr] = getLeftRight(data.posByChr[chr]); + } + zmin = 0; + zmax = 0; + _ref1 = data.lodnames; + for (_j = 0, _len1 = _ref1.length; _j < _len1; _j++) { + lodcol = _ref1[_j]; + extent = d3.extent(data[lodcol]); + if (extent[0] < zmin) { + zmin = extent[0]; + } + if (extent[1] > zmax) { + zmax = extent[1]; + } + } + if (-zmin > zmax) { + zmax = -zmin; + } + zlim = zlim != null ? zlim : [-zmax, 0, zmax]; + if (zlim.length !== colors.length) { + console.log("zlim.length (" + zlim.length + ") != colors.length (" + colors.length + ")"); + } + zscale.domain(zlim).range(colors); + zthresh = zthresh != null ? zthresh : zmin - 1; + data.cells = []; + _ref2 = data.chrnames; + for (_k = 0, _len2 = _ref2.length; _k < _len2; _k++) { + chr = _ref2[_k]; + _ref3 = data.posByChr[chr]; + for (i = _l = 0, _len3 = _ref3.length; _l < _len3; i = ++_l) { + pos = _ref3[i]; + _ref4 = data.lodByChr[chr][i]; + for (j = _m = 0, _len4 = _ref4.length; _m < _len4; j = ++_m) { + lod = _ref4[j]; + if (lod >= zthresh || lod <= -zthresh) { + data.cells.push({ + z: lod, + left: (xscale[chr](pos) + xscale[chr](xLR[chr][pos].left)) / 2, + right: (xscale[chr](pos) + xscale[chr](xLR[chr][pos].right)) / 2, + lodindex: j, + chr: chr, + pos: pos + }); + } + } + } + } + svg = d3.select(this).selectAll("svg").data([data]); + gEnter = svg.enter().append("svg").append("g"); + svg.attr("width", width + margin.left + margin.right).attr("height", height + margin.top + margin.bottom); + g = svg.select("g"); + g.append("g").attr("id", "boxes").selectAll("empty").data(data.chrnames).enter().append("rect").attr("id", function(d) { + return "box" + d; + }).attr("x", function(d, i) { + return data.chrStart[i]; + }).attr("y", function(d) { + return margin.top; + }).attr("height", height).attr("width", function(d, i) { + return data.chrEnd[i] - data.chrStart[i]; + }).attr("fill", rectcolor).attr("stroke", "none"); + titlegrp = g.append("g").attr("class", "title").append("text").attr("x", margin.left + width / 2).attr("y", margin.top - titlepos).text(title); + xaxis = g.append("g").attr("class", "x axis"); + xaxis.selectAll("empty").data(data.chrnames).enter().append("text").attr("x", function(d, i) { + return (data.chrStart[i] + data.chrEnd[i]) / 2; + }).attr("y", margin.top + height + axispos.xlabel).text(function(d) { + return d; + }); + xaxis.append("text").attr("class", "title").attr("x", margin.left + width / 2).attr("y", margin.top + height + axispos.xtitle).text(xlab); + rotate_ylab = rotate_ylab != null ? rotate_ylab : ylab.length > 1; + yaxis = g.append("g").attr("class", "y axis"); + yaxis.append("text").attr("class", "title").attr("y", margin.top + height / 2).attr("x", margin.left - axispos.ytitle).text(ylab).attr("transform", rotate_ylab ? "rotate(270," + (margin.left - axispos.ytitle) + "," + (margin.top + height / 2) + ")" : ""); + yaxis.selectAll("empty").data(data.lodnames).enter().append("text").attr("id", function(d, i) { + return "yaxis" + i; + }).attr("y", function(d, i) { + return yscale(i); + }).attr("x", margin.left - axispos.ylabel).text(function(d) { + return d; + }).attr("opacity", 0); + celltip = d3.tip().attr('class', 'd3-tip').html(function(d) { + var p, z; + z = d3.format(".2f")(Math.abs(d.z)); + p = d3.format(".1f")(d.pos); + return "" + d.chr + "@" + p + " → " + z; + }).direction('e').offset([0, 10]); + svg.call(celltip); + cells = g.append("g").attr("id", "cells"); + cellSelect = cells.selectAll("empty").data(data.cells).enter().append("rect").attr("x", function(d) { + return d.left; + }).attr("y", function(d) { + return yscale(d.lodindex) - rectHeight / 2; + }).attr("width", function(d) { + return d.right - d.left; + }).attr("height", rectHeight).attr("class", function(d, i) { + return "cell" + i; + }).attr("fill", function(d) { + return zscale(d.z); + }).attr("stroke", "none").attr("stroke-width", "1").on("mouseover.paneltip", function(d) { + yaxis.select("text#yaxis" + d.lodindex).attr("opacity", 1); + d3.select(this).attr("stroke", "black"); + return celltip.show(d); + }).on("mouseout.paneltip", function(d) { + yaxis.select("text#yaxis" + d.lodindex).attr("opacity", 0); + d3.select(this).attr("stroke", "none"); + return celltip.hide(); + }); + return g.append("g").attr("id", "boxes").selectAll("empty").data(data.chrnames).enter().append("rect").attr("id", function(d) { + return "box" + d; + }).attr("x", function(d, i) { + return data.chrStart[i]; + }).attr("y", function(d) { + return margin.top; + }).attr("height", height).attr("width", function(d, i) { + return data.chrEnd[i] - data.chrStart[i]; + }).attr("fill", "none").attr("stroke", "black").attr("stroke-width", "none"); + }); + }; + chart.width = function(value) { + if (!arguments.length) { + return width; + } + width = value; + return chart; + }; + chart.height = function(value) { + if (!arguments.length) { + return height; + } + height = value; + return chart; + }; + chart.margin = function(value) { + if (!arguments.length) { + return margin; + } + margin = value; + return chart; + }; + chart.axispos = function(value) { + if (!arguments.length) { + return axispos; + } + axispos = value; + return chart; + }; + chart.titlepos = function(value) { + if (!arguments.length) { + return titlepos; + } + titlepos; + return chart; + }; + chart.rectcolor = function(value) { + if (!arguments.length) { + return rectcolor; + } + rectcolor = value; + return chart; + }; + chart.colors = function(value) { + if (!arguments.length) { + return colors; + } + colors = value; + return chart; + }; + chart.title = function(value) { + if (!arguments.length) { + return title; + } + title = value; + return chart; + }; + chart.xlab = function(value) { + if (!arguments.length) { + return xlab; + } + xlab = value; + return chart; + }; + chart.ylab = function(value) { + if (!arguments.length) { + return ylab; + } + ylab = value; + return chart; + }; + chart.rotate_ylab = function(value) { + if (!arguments.length) { + return rotate_ylab; + } + rotate_ylab = value; + return chart; + }; + chart.zthresh = function(value) { + if (!arguments.length) { + return zthresh; + } + zthresh = value; + return chart; + }; + chart.zlim = function(value) { + if (!arguments.length) { + return zlim; + } + zlim = value; + return chart; + }; + chart.chrGap = function(value) { + if (!arguments.length) { + return chrGap; + } + chrGap = value; + return chart; + }; + chart.xscale = function() { + return xscale; + }; + chart.yscale = function() { + return yscale; + }; + chart.zscale = function() { + return zscale; + }; + chart.cellSelect = function() { + return cellSelect; + }; + return chart; +}; diff --git a/wqflask/wqflask/templates/heatmap.html b/wqflask/wqflask/templates/heatmap.html new file mode 100644 index 00000000..1ffbd5d8 --- /dev/null +++ b/wqflask/wqflask/templates/heatmap.html @@ -0,0 +1,41 @@ +{% extends "base.html" %}
+{% block title %}Interval Mapping{% endblock %}
+{% block css %}
+ <link rel="stylesheet" type="text/css" href="/static/new/css/d3-tip.min.css" />
+ <link rel="stylesheet" type="text/css" href="/static/new/css/panelutil.css" />
+{% endblock %}
+{% block content %} <!-- Start of body -->
+
+ {{ header("Heatmap") }}
+
+ <div class="container">
+ <div>
+ <h2>
+ Heatmap
+ </h2>
+ </div>
+ <div id="chart_container">
+ <div class="qtlcharts" id="chart">
+
+ </div>
+ </div>
+
+ </div>
+
+ <!-- End of body -->
+
+{% endblock %}
+
+{% block js %}
+ <script>
+ js_data = {{ js_data | safe }}
+ </script>
+
+ <script language="javascript" type="text/javascript" src="http://d3js.org/d3.v3.min.js"></script>
+ <script language="javascript" type="text/javascript" src="/static/new/js_external/d3-tip.min.js"></script>
+ <script language="javascript" type="text/javascript" src="/static/new/javascript/panelutil.js"></script>
+ <script language="javascript" type="text/javascript" src="/static/new/javascript/lodheatmap.js"></script>
+ <script language="javascript" type="text/javascript" src="/static/new/javascript/create_heatmap.js"></script>
+ <script language="javascript" type="text/javascript" src="/static/packages/underscore/underscore-min.js"></script>
+
+{% endblock %}
\ No newline at end of file diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index e65da914..b55189eb 100755 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -9,6 +9,7 @@ import StringIO # Todo: Use cStringIO? import gc import cPickle as pickle +import uuid import simplejson as json #import json @@ -201,6 +202,47 @@ def show_trait_page(): #print("show_trait template_vars:", pf(template_vars.__dict__)) return render_template("show_trait.html", **template_vars.__dict__) +@app.route("/heatmap", methods=('POST',)) +def heatmap_page(): + print("In heatmap, request.form is:", pf(request.form)) + + start_vars = request.form + temp_uuid = uuid.uuid4() + + version = "v1" + key = "heatmap:{}:".format(version) + json.dumps(start_vars, sort_keys=True) + print("key is:", pf(key)) + with Bench("Loading cache"): + result = Redis.get(key) + + if result: + print("Cache hit!!!") + with Bench("Loading results"): + result = pickle.loads(result) + + else: + print("Cache miss!!!") + + template_vars = heatmap.Heatmap(request.form, temp_uuid) + template_vars.js_data = json.dumps(template_vars.js_data, + default=json_default_handler, + indent=" ") + + result = template_vars.__dict__ + + for item in template_vars.__dict__.keys(): + print(" ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) + + pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) + print("pickled result length:", len(pickled_result)) + Redis.set(key, pickled_result) + Redis.expire(key, 60*60) + + with Bench("Rendering template"): + rendered_template = render_template("heatmap.html", **result) + + return rendered_template + @app.route("/marker_regression", methods=('POST',)) def marker_regression_page(): initial_start_vars = request.form @@ -218,7 +260,7 @@ def marker_regression_page(): if key in wanted or key.startswith(('value:')): start_vars[key] = value - version = "v1" + version = "v3" key = "marker_regression:{}:".format(version) + json.dumps(start_vars, sort_keys=True) print("key is:", pf(key)) with Bench("Loading cache"): @@ -353,16 +395,6 @@ def corr_matrix_page(): return render_template("correlation_matrix.html", **template_vars.__dict__) -@app.route("/heatmap", methods=('POST',)) -def heatmap_page(): - print("In heatmap, request.form is:", pf(request.form)) - template_vars = heatmap.Heatmap(request.form) - template_vars.js_data = json.dumps(template_vars.js_data, - default=json_default_handler, - indent=" ") - - return render_template("heatmap.html", **template_vars.__dict__) - @app.route("/corr_scatter_plot") def corr_scatter_plot_page(): template_vars = corr_scatter_plot.CorrScatterPlot(request.args) |