From 196979a2a0690ddf0d085f6e593e1c79dc3c1d39 Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Mon, 21 Jul 2014 16:10:02 +0000 Subject: Heatmap is now working (using Karl's qtlcharts code), but still need to figure out the issue with the color range --- wqflask/wqflask/heatmap/heatmap.py | 156 ++++++++++-- .../wqflask/marker_regression/marker_regression.py | 3 + .../static/new/javascript/create_heatmap.coffee | 18 +- .../static/new/javascript/create_heatmap.js | 14 ++ .../static/new/javascript/lodheatmap.coffee | 2 +- .../wqflask/static/new/javascript/lodheatmap.js | 274 +++++++++++++++++++++ wqflask/wqflask/templates/heatmap.html | 41 +++ wqflask/wqflask/views.py | 54 +++- 8 files changed, 524 insertions(+), 38 deletions(-) create mode 100644 wqflask/wqflask/static/new/javascript/create_heatmap.js create mode 100644 wqflask/wqflask/static/new/javascript/lodheatmap.js create mode 100644 wqflask/wqflask/templates/heatmap.html diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index d9c02381..8f910fa9 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -7,6 +7,7 @@ import gc import string import cPickle import os +import datetime import time import pp import math @@ -14,38 +15,50 @@ import collections import resource import scipy +import numpy as np +from scipy import linalg from pprint import pformat as pf from htmlgen import HTMLgen2 as HT import reaper -from base import webqtlConfig -from utility.THCell import THCell -from utility.TDCell import TDCell from base.trait import GeneralTrait from base import data_set -from base.templatePage import templatePage -from utility import webqtlUtil, helper_functions, corr_result_helpers -from dbFunction import webqtlDatabaseFunction -import utility.webqtlUtil #this is for parallel computing only. -from wqflask.correlation import correlation_functions -from utility.benchmark import Bench +from base import species +from base import webqtlConfig +from utility import webqtlUtil +from wqflask.my_pylmm.data import prep_data +from wqflask.my_pylmm.pyLMM import lmm +from wqflask.my_pylmm.pyLMM import input +from utility import helper_functions +from utility import Plot, Bunch +from utility import temp_data from MySQLdb import escape_string as escape +import cPickle as pickle +import simplejson as json + from pprint import pformat as pf +from redis import Redis +Redis = Redis() + from flask import Flask, g class Heatmap(object): - def __init__(self, start_vars): + def __init__(self, start_vars, temp_uuid): trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] helper_functions.get_trait_db_obs(self, trait_db_list) + self.dataset = self.trait_list[0][1] + + self.json_data = {} #The dictionary that will be used to create the json object that contains all the data needed to create the figure + self.all_sample_list = [] self.traits = [] for trait_db in self.trait_list: @@ -76,19 +89,122 @@ class Heatmap(object): self.trait_results = {} for trait_db in self.trait_list: this_trait = trait_db[0] - this_db = trait_db[1] + #this_db = trait_db[1] + self.dataset.group.get_markers() - this_db_samples = this_db.group.samplelist + this_db_samples = self.dataset.group.samplelist this_sample_data = this_trait.data - print("this_sample_data", this_sample_data) - this_trait_vals = [] - for index, sample in enumerate(target_samples): + #print("this_sample_data", this_sample_data) + this_trait_vals = [] + for index, sample in enumerate(this_db_samples): + if sample in this_sample_data: + sample_value = this_sample_data[sample].value + this_trait_vals.append(sample_value) + else: + this_trait_vals.append("x") - if (sample in this_sample_data) and (sample in target_sample_data): - sample_value = this_sample_data[sample].value - target_sample_value = target_sample_data[sample].value - this_trait_vals.append(sample_value) - target_vals.append(target_sample_value) + pheno_vector = np.array([val == "x" and np.nan or float(val) for val in this_trait_vals]) + + key = "pylmm:input:" + str(temp_uuid) + #print("key is:", pf(key)) + + genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers] + + no_val_samples = self.identify_empty_samples(this_trait_vals) + trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples) + + genotype_matrix = np.array(trimmed_genotype_data).T + + #print("genotype_matrix:", str(genotype_matrix.tolist())) + #print("pheno_vector:", str(pheno_vector.tolist())) + + params = dict(pheno_vector = pheno_vector.tolist(), + genotype_matrix = genotype_matrix.tolist(), + restricted_max_likelihood = True, + refit = False, + temp_uuid = str(temp_uuid), + + # meta data + timestamp = datetime.datetime.now().isoformat(), + ) + json_params = json.dumps(params) + #print("json_params:", json_params) + Redis.set(key, json_params) + Redis.expire(key, 60*60) + print("before printing command") + + command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key, + "other") + print("command is:", command) + print("after printing command") + + os.system(command) + + json_results = Redis.blpop("pylmm:results:" + str(temp_uuid), 45*60) + results = json.loads(json_results[1]) + p_values = [float(result) for result in results['p_values']] + #print("p_values:", p_values) + self.dataset.group.markers.add_pvalues(p_values) + + self.trait_results[this_trait.name] = [] + for marker in self.dataset.group.markers.markers: + self.trait_results[this_trait.name].append(marker['lod_score']) + + #print("self.trait_results:", self.trait_results) + + chrnames = [] + lodnames = [] + chr_pos = [] + pos = [] + markernames = [] + + for trait in self.trait_results.keys(): + lodnames.append(trait) + + for marker in self.dataset.group.markers.markers: + if marker['chr'] not in chrnames: + chrnames.append(marker['chr']) + chr_pos.append(marker['chr']) + pos.append(marker['Mb']) + markernames.append(marker['name']) + + self.json_data['chrnames'] = chrnames + self.json_data['lodnames'] = lodnames + self.json_data['chr'] = chr_pos + self.json_data['pos'] = pos + self.json_data['markernames'] = markernames + + for trait in self.trait_results: + self.json_data[trait] = self.trait_results[trait] + + self.js_data = dict( + json_data = self.json_data + ) + + print("self.js_data:", self.js_data) + + def identify_empty_samples(self, values): + no_val_samples = [] + for sample_count, val in enumerate(values): + if val == "x": + no_val_samples.append(sample_count) + return no_val_samples + + def trim_genotypes(self, genotype_data, no_value_samples): + trimmed_genotype_data = [] + for marker in genotype_data: + new_genotypes = [] + for item_count, genotype in enumerate(marker): + if item_count in no_value_samples: + continue + try: + genotype = float(genotype) + except ValueError: + genotype = np.nan + pass + new_genotypes.append(genotype) + trimmed_genotype_data.append(new_genotypes) + return trimmed_genotype_data \ No newline at end of file diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 0bee5994..0a83f9f8 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -428,6 +428,9 @@ class MarkerRegression(object): # "refit": False, # "temp_data": tempdata} + print("genotype_matrix:", str(genotype_matrix.tolist())) + print("pheno_vector:", str(pheno_vector.tolist())) + params = dict(pheno_vector = pheno_vector.tolist(), genotype_matrix = genotype_matrix.tolist(), restricted_max_likelihood = True, diff --git a/wqflask/wqflask/static/new/javascript/create_heatmap.coffee b/wqflask/wqflask/static/new/javascript/create_heatmap.coffee index 2bcb5c0e..51ca5e8f 100644 --- a/wqflask/wqflask/static/new/javascript/create_heatmap.coffee +++ b/wqflask/wqflask/static/new/javascript/create_heatmap.coffee @@ -1,12 +1,18 @@ -h = 700 -w = 1000 +create_heatmap = () -> + + h = 700 + w = 1000 -# Example: simplest use -d3.json "data.json", (data) -> mychart = lodheatmap().height(h) .width(w) - .zthresh(1.0) + .zthresh(0.5) + + data = js_data.json_data + + console.log("data:", data) d3.select("div#chart") .datum(data) - .call(mychart) \ No newline at end of file + .call(mychart) + +create_heatmap() \ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/create_heatmap.js b/wqflask/wqflask/static/new/javascript/create_heatmap.js new file mode 100644 index 00000000..7d6f7a9d --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/create_heatmap.js @@ -0,0 +1,14 @@ +// Generated by CoffeeScript 1.7.1 +var create_heatmap; + +create_heatmap = function() { + var data, h, mychart, w; + h = 700; + w = 1000; + mychart = lodheatmap().height(h).width(w).zthresh(0.5); + data = js_data.json_data; + console.log("data:", data); + return d3.select("div#chart").datum(data).call(mychart); +}; + +create_heatmap(); diff --git a/wqflask/wqflask/static/new/javascript/lodheatmap.coffee b/wqflask/wqflask/static/new/javascript/lodheatmap.coffee index 9d0687d0..6cb8bfc7 100644 --- a/wqflask/wqflask/static/new/javascript/lodheatmap.coffee +++ b/wqflask/wqflask/static/new/javascript/lodheatmap.coffee @@ -3,7 +3,7 @@ lodheatmap = () -> width = 1200 height = 600 - margin = {left:60, top:40, right:40, bottom: 40} + margin = {left:100, top:40, right:40, bottom: 40} axispos = {xtitle:25, ytitle:30, xlabel:5, ylabel:5} chrGap = 8 titlepos = 20 diff --git a/wqflask/wqflask/static/new/javascript/lodheatmap.js b/wqflask/wqflask/static/new/javascript/lodheatmap.js new file mode 100644 index 00000000..447287da --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/lodheatmap.js @@ -0,0 +1,274 @@ +// Generated by CoffeeScript 1.7.1 +var lodheatmap; + +lodheatmap = function() { + var axispos, cellSelect, chart, chrGap, colors, height, margin, rectcolor, rotate_ylab, title, titlepos, width, xlab, xscale, ylab, yscale, zlim, zscale, zthresh; + width = 1200; + height = 600; + margin = { + left: 100, + top: 40, + right: 40, + bottom: 40 + }; + axispos = { + xtitle: 25, + ytitle: 30, + xlabel: 5, + ylabel: 5 + }; + chrGap = 8; + titlepos = 20; + rectcolor = d3.rgb(230, 230, 230); + colors = ["slateblue", "white", "crimson"]; + title = ""; + xlab = "Chromosome"; + ylab = ""; + rotate_ylab = null; + zlim = null; + zthresh = null; + xscale = d3.scale.linear(); + yscale = d3.scale.linear(); + zscale = d3.scale.linear(); + cellSelect = null; + chart = function(selection) { + return selection.each(function(data) { + var cells, celltip, chr, extent, g, gEnter, i, j, lod, lodcol, nlod, pos, rectHeight, svg, titlegrp, xLR, xaxis, yaxis, zmax, zmin, _i, _j, _k, _l, _len, _len1, _len2, _len3, _len4, _m, _ref, _ref1, _ref2, _ref3, _ref4; + data = reorgLodData(data); + data = chrscales(data, width, chrGap, margin.left, true); + xscale = data.xscale; + nlod = data.lodnames.length; + yscale.domain([-0.5, nlod - 0.5]).range([margin.top + height, margin.top]); + rectHeight = yscale(0) - yscale(1); + xLR = {}; + _ref = data.chrnames; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + chr = _ref[_i]; + xLR[chr] = getLeftRight(data.posByChr[chr]); + } + zmin = 0; + zmax = 0; + _ref1 = data.lodnames; + for (_j = 0, _len1 = _ref1.length; _j < _len1; _j++) { + lodcol = _ref1[_j]; + extent = d3.extent(data[lodcol]); + if (extent[0] < zmin) { + zmin = extent[0]; + } + if (extent[1] > zmax) { + zmax = extent[1]; + } + } + if (-zmin > zmax) { + zmax = -zmin; + } + zlim = zlim != null ? zlim : [-zmax, 0, zmax]; + if (zlim.length !== colors.length) { + console.log("zlim.length (" + zlim.length + ") != colors.length (" + colors.length + ")"); + } + zscale.domain(zlim).range(colors); + zthresh = zthresh != null ? zthresh : zmin - 1; + data.cells = []; + _ref2 = data.chrnames; + for (_k = 0, _len2 = _ref2.length; _k < _len2; _k++) { + chr = _ref2[_k]; + _ref3 = data.posByChr[chr]; + for (i = _l = 0, _len3 = _ref3.length; _l < _len3; i = ++_l) { + pos = _ref3[i]; + _ref4 = data.lodByChr[chr][i]; + for (j = _m = 0, _len4 = _ref4.length; _m < _len4; j = ++_m) { + lod = _ref4[j]; + if (lod >= zthresh || lod <= -zthresh) { + data.cells.push({ + z: lod, + left: (xscale[chr](pos) + xscale[chr](xLR[chr][pos].left)) / 2, + right: (xscale[chr](pos) + xscale[chr](xLR[chr][pos].right)) / 2, + lodindex: j, + chr: chr, + pos: pos + }); + } + } + } + } + svg = d3.select(this).selectAll("svg").data([data]); + gEnter = svg.enter().append("svg").append("g"); + svg.attr("width", width + margin.left + margin.right).attr("height", height + margin.top + margin.bottom); + g = svg.select("g"); + g.append("g").attr("id", "boxes").selectAll("empty").data(data.chrnames).enter().append("rect").attr("id", function(d) { + return "box" + d; + }).attr("x", function(d, i) { + return data.chrStart[i]; + }).attr("y", function(d) { + return margin.top; + }).attr("height", height).attr("width", function(d, i) { + return data.chrEnd[i] - data.chrStart[i]; + }).attr("fill", rectcolor).attr("stroke", "none"); + titlegrp = g.append("g").attr("class", "title").append("text").attr("x", margin.left + width / 2).attr("y", margin.top - titlepos).text(title); + xaxis = g.append("g").attr("class", "x axis"); + xaxis.selectAll("empty").data(data.chrnames).enter().append("text").attr("x", function(d, i) { + return (data.chrStart[i] + data.chrEnd[i]) / 2; + }).attr("y", margin.top + height + axispos.xlabel).text(function(d) { + return d; + }); + xaxis.append("text").attr("class", "title").attr("x", margin.left + width / 2).attr("y", margin.top + height + axispos.xtitle).text(xlab); + rotate_ylab = rotate_ylab != null ? rotate_ylab : ylab.length > 1; + yaxis = g.append("g").attr("class", "y axis"); + yaxis.append("text").attr("class", "title").attr("y", margin.top + height / 2).attr("x", margin.left - axispos.ytitle).text(ylab).attr("transform", rotate_ylab ? "rotate(270," + (margin.left - axispos.ytitle) + "," + (margin.top + height / 2) + ")" : ""); + yaxis.selectAll("empty").data(data.lodnames).enter().append("text").attr("id", function(d, i) { + return "yaxis" + i; + }).attr("y", function(d, i) { + return yscale(i); + }).attr("x", margin.left - axispos.ylabel).text(function(d) { + return d; + }).attr("opacity", 0); + celltip = d3.tip().attr('class', 'd3-tip').html(function(d) { + var p, z; + z = d3.format(".2f")(Math.abs(d.z)); + p = d3.format(".1f")(d.pos); + return "" + d.chr + "@" + p + " → " + z; + }).direction('e').offset([0, 10]); + svg.call(celltip); + cells = g.append("g").attr("id", "cells"); + cellSelect = cells.selectAll("empty").data(data.cells).enter().append("rect").attr("x", function(d) { + return d.left; + }).attr("y", function(d) { + return yscale(d.lodindex) - rectHeight / 2; + }).attr("width", function(d) { + return d.right - d.left; + }).attr("height", rectHeight).attr("class", function(d, i) { + return "cell" + i; + }).attr("fill", function(d) { + return zscale(d.z); + }).attr("stroke", "none").attr("stroke-width", "1").on("mouseover.paneltip", function(d) { + yaxis.select("text#yaxis" + d.lodindex).attr("opacity", 1); + d3.select(this).attr("stroke", "black"); + return celltip.show(d); + }).on("mouseout.paneltip", function(d) { + yaxis.select("text#yaxis" + d.lodindex).attr("opacity", 0); + d3.select(this).attr("stroke", "none"); + return celltip.hide(); + }); + return g.append("g").attr("id", "boxes").selectAll("empty").data(data.chrnames).enter().append("rect").attr("id", function(d) { + return "box" + d; + }).attr("x", function(d, i) { + return data.chrStart[i]; + }).attr("y", function(d) { + return margin.top; + }).attr("height", height).attr("width", function(d, i) { + return data.chrEnd[i] - data.chrStart[i]; + }).attr("fill", "none").attr("stroke", "black").attr("stroke-width", "none"); + }); + }; + chart.width = function(value) { + if (!arguments.length) { + return width; + } + width = value; + return chart; + }; + chart.height = function(value) { + if (!arguments.length) { + return height; + } + height = value; + return chart; + }; + chart.margin = function(value) { + if (!arguments.length) { + return margin; + } + margin = value; + return chart; + }; + chart.axispos = function(value) { + if (!arguments.length) { + return axispos; + } + axispos = value; + return chart; + }; + chart.titlepos = function(value) { + if (!arguments.length) { + return titlepos; + } + titlepos; + return chart; + }; + chart.rectcolor = function(value) { + if (!arguments.length) { + return rectcolor; + } + rectcolor = value; + return chart; + }; + chart.colors = function(value) { + if (!arguments.length) { + return colors; + } + colors = value; + return chart; + }; + chart.title = function(value) { + if (!arguments.length) { + return title; + } + title = value; + return chart; + }; + chart.xlab = function(value) { + if (!arguments.length) { + return xlab; + } + xlab = value; + return chart; + }; + chart.ylab = function(value) { + if (!arguments.length) { + return ylab; + } + ylab = value; + return chart; + }; + chart.rotate_ylab = function(value) { + if (!arguments.length) { + return rotate_ylab; + } + rotate_ylab = value; + return chart; + }; + chart.zthresh = function(value) { + if (!arguments.length) { + return zthresh; + } + zthresh = value; + return chart; + }; + chart.zlim = function(value) { + if (!arguments.length) { + return zlim; + } + zlim = value; + return chart; + }; + chart.chrGap = function(value) { + if (!arguments.length) { + return chrGap; + } + chrGap = value; + return chart; + }; + chart.xscale = function() { + return xscale; + }; + chart.yscale = function() { + return yscale; + }; + chart.zscale = function() { + return zscale; + }; + chart.cellSelect = function() { + return cellSelect; + }; + return chart; +}; diff --git a/wqflask/wqflask/templates/heatmap.html b/wqflask/wqflask/templates/heatmap.html new file mode 100644 index 00000000..1ffbd5d8 --- /dev/null +++ b/wqflask/wqflask/templates/heatmap.html @@ -0,0 +1,41 @@ +{% extends "base.html" %} +{% block title %}Interval Mapping{% endblock %} +{% block css %} + + +{% endblock %} +{% block content %} + + {{ header("Heatmap") }} + +