Merge /home/zas1024/gene

13 files changed, 554 insertions, 497 deletions

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 685cd648..7e187802 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -79,7 +79,7 @@ class Dataset_Types(object):
     
     def __init__(self):
         self.datasets = {}
-        file_name = "wqflask/static/new/javascript/dataset_menu_structure.json"
+        file_name = "/home/zas1024/gene/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json"
         with open(file_name, 'r') as fh:
             data = json.load(fh)
         
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index 7f215d9b..60221c05 100755
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -513,7 +513,7 @@ class MarkerRegression(object):
                 p_value = float(line_list[-1])
                 if threshold_p_value >= 0 and threshold_p_value <= 1:
                     if p_value < threshold_p_value:
-                        p_value_dict[snp] = p_value
+                        p_value_dict[snp] = float(p_value)
                 
                 if plink_results.has_key(chr_name):
                     value_list = plink_results[chr_name]
diff --git a/wqflask/wqflask/static/new/javascript/create_lodchart.coffee b/wqflask/wqflask/static/new/javascript/create_lodchart.coffee
index 2d5e3acb..76be5490 100644
--- a/wqflask/wqflask/static/new/javascript/create_lodchart.coffee
+++ b/wqflask/wqflask/static/new/javascript/create_lodchart.coffee
@@ -1,87 +1,87 @@
-create_manhattan_plot = ->

-    h = 500

-    w = 1200

-    margin = {left:60, top:40, right:40, bottom: 40, inner:5}

-    halfh = (h+margin.top+margin.bottom)

-    totalh = halfh*2

-    totalw = (w+margin.left+margin.right)

-    if 'additive' of js_data

-        additive = js_data.additive

-    else

-        additive = false

-    

-    console.log("js_data:", js_data)

-    

-    # simplest use

-    #d3.json "data.json", (data) ->

-    mychart = lodchart().lodvarname("lod.hk")

-                        .height(h)

-                        .width(w)

-                        .margin(margin)

-                        .ylab("LOD score")

-                        .manhattanPlot(js_data.manhattan_plot)

-                        #.additive(additive)

-                        

-    data = js_data.json_data

-    

-    d3.select("div#topchart")

-      .datum(data)

-      .call(mychart)

-    

-    # grab chromosome rectangles; color pink on hover

-    chrrect = mychart.chrSelect()

-    chrrect.on "mouseover", ->

-                d3.select(this).attr("fill", "#E9CFEC")

-           .on "mouseout", (d,i) ->

-                d3.select(this).attr("fill", ->

-                      return "#F1F1F9"  if i % 2

-                      "#FBFBFF")

-    

-    # animate points at markers on click

-    mychart.markerSelect()

-              .on "click", (d) ->

-                    r = d3.select(this).attr("r")

-                    d3.select(this)

-                      .transition().duration(500).attr("r", r*3)

-                      .transition().duration(500).attr("r", r)

-                  

-create_manhattan_plot()

-

-$("#export").click =>

-    #Get d3 SVG element

-    svg = $("#topchart").find("svg")[0]

-    

-    #Extract SVG text string

-    svg_xml = (new XMLSerializer).serializeToString(svg)

-    console.log("svg_xml:", svg_xml)

-        

-    #Set filename

-    filename = "manhattan_plot_" + js_data.this_trait

-

-    #Make a form with the SVG data

-    form = $("#exportform")

-    form.find("#data").val(svg_xml)

-    form.find("#filename").val(filename)

-    form.submit()

-

-$("#export_pdf").click =>

-    

-    #$('#topchart').remove()

-    #$('#chart_container').append('<div class="qtlcharts" id="topchart"></div>')

-    #create_interval_map()

-    

-    #Get d3 SVG element

-    svg = $("#topchart").find("svg")[0]

-    

-    #Extract SVG text string

-    svg_xml = (new XMLSerializer).serializeToString(svg)

-    console.log("svg_xml:", svg_xml)

-        

-    #Set filename

-    filename = "manhattan_plot_" + js_data.this_trait

-

-    #Make a form with the SVG data

-    form = $("#exportpdfform")

-    form.find("#data").val(svg_xml)

-    form.find("#filename").val(filename)

-    form.submit()

+create_manhattan_plot = ->
+    h = 500
+    w = 1200
+    margin = {left:60, top:40, right:40, bottom: 40, inner:5}
+    halfh = (h+margin.top+margin.bottom)
+    totalh = halfh*2
+    totalw = (w+margin.left+margin.right)
+    if 'additive' of js_data
+        additive = js_data.additive
+    else
+        additive = false
+    
+    console.log("js_data:", js_data)
+    
+    # simplest use
+    #d3.json "data.json", (data) ->
+    mychart = lodchart().lodvarname("lod.hk")
+                        .height(h)
+                        .width(w)
+                        .margin(margin)
+                        .ylab("LOD score")
+                        .manhattanPlot(js_data.manhattan_plot)
+                        #.additive(additive)
+                        
+    data = js_data.json_data
+    
+    d3.select("div#topchart")
+      .datum(data)
+      .call(mychart)
+    
+    # grab chromosome rectangles; color pink on hover
+    chrrect = mychart.chrSelect()
+    chrrect.on "mouseover", ->
+                d3.select(this).attr("fill", "#E9CFEC")
+           .on "mouseout", (d,i) ->
+                d3.select(this).attr("fill", ->
+                      return "#F1F1F9"  if i % 2
+                      "#FBFBFF")
+    
+    # animate points at markers on click
+    mychart.markerSelect()
+              .on "click", (d) ->
+                    r = d3.select(this).attr("r")
+                    d3.select(this)
+                      .transition().duration(500).attr("r", r*3)
+                      .transition().duration(500).attr("r", r)
+                  
+create_manhattan_plot()
+
+$("#export").click =>
+    #Get d3 SVG element
+    svg = $("#topchart").find("svg")[0]
+    
+    #Extract SVG text string
+    svg_xml = (new XMLSerializer).serializeToString(svg)
+    console.log("svg_xml:", svg_xml)
+        
+    #Set filename
+    filename = "manhattan_plot_" + js_data.this_trait
+
+    #Make a form with the SVG data
+    form = $("#exportform")
+    form.find("#data").val(svg_xml)
+    form.find("#filename").val(filename)
+    form.submit()
+
+$("#export_pdf").click =>
+    
+    #$('#topchart').remove()
+    #$('#chart_container').append('<div class="qtlcharts" id="topchart"></div>')
+    #create_interval_map()
+    
+    #Get d3 SVG element
+    svg = $("#topchart").find("svg")[0]
+    
+    #Extract SVG text string
+    svg_xml = (new XMLSerializer).serializeToString(svg)
+    console.log("svg_xml:", svg_xml)
+        
+    #Set filename
+    filename = "manhattan_plot_" + js_data.this_trait
+
+    #Make a form with the SVG data
+    form = $("#exportpdfform")
+    form.find("#data").val(svg_xml)
+    form.find("#filename").val(filename)
+    form.submit()
diff --git a/wqflask/wqflask/static/new/javascript/panelutil.coffee b/wqflask/wqflask/static/new/javascript/panelutil.coffee
index d1eb83e5..a3bc0b44 100644
--- a/wqflask/wqflask/static/new/javascript/panelutil.coffee
+++ b/wqflask/wqflask/static/new/javascript/panelutil.coffee
@@ -1,357 +1,357 @@
-# A variety of utility functions used by the different panel functions

-

-# determine rounding of axis labels

-formatAxis = (d, extra_digits=0) ->

-    d = d[1] - d[0]

-    ndig = Math.floor( Math.log(d % 10) / Math.log(10) )

-    ndig = 0 if ndig > 0

-    ndig = Math.abs(ndig) + extra_digits

-    d3.format(".#{ndig}f")

-

-# unique values of array (ignore nulls)

-unique = (x) ->

-    output = {}

-    output[v] = v for v in x when v

-    output[v] for v of output

-

-# Pull out a variable (column) from a two-dimensional array

-pullVarAsArray = (data, variable) ->

-    v = []

-    for i of data

-        v = v.concat data[i][variable]

-    v

-  

-  

-# reorganize lod/pos by chromosome

-# lodvarname==null    -> case for multiple LOD columns (lodheatmap)

-# lodvarname provided -> case for one LOD column (lodchart)

-reorgLodData = (data, lodvarname=null) ->

-    data.posByChr = {}

-    data.lodByChr = {}

-    

-    for chr,i in data.chrnames

-      #console.log("chr:", chr)

-      data.posByChr[chr[0]] = []

-      data.lodByChr[chr[0]] = []

-      for pos, j in data.pos

-        if data.chr[j].toString() == chr[0]

-          #console.log(data.chr[j] + " AND " + chr[0])

-          data.posByChr[chr[0]].push(pos)

-          data.lodnames = [data.lodnames] unless Array.isArray(data.lodnames)

-          lodval = (data[lodcolumn][j] for lodcolumn in data.lodnames)

-          data.lodByChr[chr[0]].push(lodval)

-

-    #console.log("data.posByChr:", data.posByChr)

-

-    if lodvarname?

-      data.markers = []

-      for marker,i in data.markernames

-        if marker != ""

-          data.markers.push({name:marker, chr:data.chr[i], pos:data.pos[i], lod:data[lodvarname][i]})

-    

-    data

-  

-# calculate chromosome start/end + scales, for heat map

-chrscales = (data, width, chrGap, leftMargin, pad4heatmap) ->

-    # start and end of chromosome positions

-    chrStart = []

-    chrEnd = []

-    chrLength = []

-    totalChrLength = 0

-    maxd = 0

-    for chr in data.chrnames

-      d = maxdiff(data.posByChr[chr[0]])

-      maxd = d if d > maxd

-  

-      rng = d3.extent(data.posByChr[chr[0]])

-      chrStart.push(rng[0])

-      chrEnd.push(rng[1])

-      L = rng[1] - rng[0]

-      chrLength.push(L)

-      totalChrLength += L

-  

-    # adjust lengths for heatmap

-    if pad4heatmap

-      data.recwidth = maxd

-      chrStart = chrStart.map (x) -> x-maxd/2

-      chrEnd = chrEnd.map (x) -> x+maxd/2

-      chrLength = chrLength.map (x) -> x+maxd

-      totalChrLength += (chrLength.length*maxd)

-  

-    # break up x axis into chromosomes by length, with gaps

-    data.chrStart = []

-    data.chrEnd = []

-    cur = leftMargin

-    cur += chrGap/2 unless pad4heatmap

-    data.xscale = {}

-    for chr,i in data.chrnames

-      data.chrStart.push(cur)

-      w = Math.round((width-chrGap*(data.chrnames.length-pad4heatmap))/totalChrLength*chrLength[i])

-      data.chrEnd.push(cur + w)

-      cur = data.chrEnd[i] + chrGap

-      # x-axis scales, by chromosome

-      data.xscale[chr[0]] = d3.scale.linear()

-                           .domain([chrStart[i], chrEnd[i]])

-                           .range([data.chrStart[i], data.chrEnd[i]])

-  

-    # return data with new stuff added

-    data

-    

-    

-# reorganize lod/pos by chromosome

-# lodvarname==null    -> case for multiple LOD columns (lodheatmap)

-# lodvarname provided -> case for one LOD column (lodchart)

-#reorgLodData = (data, lodvarname=null) ->

-#    data.posByChr = {}

-#    data.lodByChr = {}

-#

-#    #console.log("data.chr", data.chr)

-#    #console.log("data.chrnames:", data.chrnames)

-#    the_chr = "0"

-#    for chr,i in data.chrnames

-#        data.posByChr[chr] = []

-#        data.lodByChr[chr] = []

-#        for pos,j in data.pos

-#            console.log("data.chr[j][0]:", data.chr[j][0])

-#            if data.chr[j][0] == chr

-#                console.log("IS EQUAL")

-#                data.posByChr[chr].push(pos)

-#                data.lodnames = [data.lodnames] unless Array.isArray(data.lodnames)

-#                lodval = (data[lodcolumn][j] for lodcolumn in data.lodnames)

-#                data.lodByChr[chr].push(lodval)

-#

-#    if lodvarname?

-#        data.markers = []

-#        for marker,i in data.markernames

-#            if marker != ""

-#                data.markers.push({name:marker, chr:data.chr[i][0], pos:data.pos[i], lod:data[lodvarname][i]})

-#

-#    data

-

-# calculate chromosome start/end + scales, for heat map

-#chrscales = (data, width, chrGap, leftMargin, pad4heatmap) ->

-#    # start and end of chromosome positions

-#    chrStart = []

-#    chrEnd = []

-#    chrLength = []

-#    totalChrLength = 0

-#    maxd = 0

-#    for chr in data.chrnames

-#        d = maxdiff(data.posByChr[chr])

-#        maxd = d if d > maxd

-#

-#        rng = d3.extent(data.posByChr[chr])

-#        chrStart.push(rng[0])

-#        chrEnd.push(rng[1])

-#        L = rng[1] - rng[0]

-#        chrLength.push(L)

-#        totalChrLength += L

-#

-#    # adjust lengths for heatmap

-#    if pad4heatmap

-#        data.recwidth = maxd

-#        chrStart = chrStart.map (x) -> x-maxd/2

-#        chrEnd = chrEnd.map (x) -> x+maxd/2

-#        chrLength = chrLength.map (x) -> x+maxd

-#        totalChrLength += (chrLength.length*maxd)

-#

-#    # break up x axis into chromosomes by length, with gaps

-#    data.chrStart = []

-#    data.chrEnd = []

-#    cur = leftMargin

-#    cur += chrGap/2 unless pad4heatmap

-#    data.xscale = {}

-#    for chr,i in data.chrnames

-#        data.chrStart.push(cur)

-#        w = Math.round((width-chrGap*(data.chrnames.length-pad4heatmap))/totalChrLength*chrLength[i])

-#        data.chrEnd.push(cur + w)

-#        cur = data.chrEnd[i] + chrGap

-#        # x-axis scales, by chromosome

-#        data.xscale[chr] = d3.scale.linear()

-#                             .domain([chrStart[i], chrEnd[i]])

-#                             .range([data.chrStart[i], data.chrEnd[i]])

-#

-#    # return data with new stuff added

-#    data

-

-# Select a set of categorical colors

-# ngroup is positive integer

-# palette = "dark" or "pastel"

-selectGroupColors = (ngroup, palette) ->

-    return [] if ngroup == 0

-

-    if palette == "dark"

-        return ["slateblue"] if ngroup == 1

-        return ["MediumVioletRed", "slateblue"] if ngroup == 2

-        return colorbrewer.Set1[ngroup] if ngroup <= 9

-        return d3.scale.category20().range()[0...ngroup]

-    else

-        return ["#bebebe"] if ngroup == 1

-        return ["lightpink", "lightblue"] if ngroup == 2

-        return colorbrewer.Pastel1[ngroup] if ngroup <= 9

-        # below is rough attempt to make _big_ pastel palette

-        return ["#8fc7f4", "#fed7f8", "#ffbf8e", "#fffbb8",

-                "#8ce08c", "#d8ffca", "#f68788", "#ffd8d6",

-                "#d4a7fd", "#f5f0f5", "#cc968b", "#f4dcd4",

-                "#f3b7f2", "#f7f6f2", "#bfbfbf", "#f7f7f7",

-                "#fcfd82", "#fbfbcd", "#87feff", "#defaf5"][0...ngroup]

-

-# expand element/array (e.g., of colors) to a given length

-#     single elment -> array, then repeated to length n

-expand2vector = (input, n) ->

-    return input unless input? # return null if null

-    return input if Array.isArray(input) and input.length >= n

-    input = [input] unless Array.isArray(input)

-    input = (input[0] for i of d3.range(n)) if input.length == 1 and n > 1

-    input

-

-# median of a vector

-median = (x) ->

-    return null if !x? 

-    n = x.length

-    x.sort((a,b) -> a-b)

-    if n % 2 == 1

-        return x[(n-1)/2]

-    (x[n/2] + x[(n/2)-1])/2

-

-# given a vector of x's, return hash with values to left and right, and the differences

-getLeftRight = (x) ->

-    n = x.length

-    x.sort( (a,b) -> a-b )

-

-    xdif = []

-    result = {}

-    for v in x

-        result[v] = {}

-

-    for i in [1...n]

-        #console.log("result:", result)

-        xdif.push(x[i]-x[i-1])

-        result[x[i]].left = x[i-1]

-    for i in [0...(n-1)]

-        result[x[i]].right = x[i+1]

-

-    xdif = median(xdif)

-    result.mediandiff = xdif

-

-    result[x[0]].left = x[0]-xdif

-    result[x[n-1]].right = x[n-1]+xdif

-    result.extent = [x[0]-xdif/2, x[n-1]+xdif/2]

-

-    result

-

-# maximum difference between adjacent values in a vector

-maxdiff = (x) ->

-    return null if x.length < 2

-    result = x[1] - x[0]

-    return result if x.length < 3

-    for i in [2...x.length]

-        d = x[i] - x[i-1]

-        result = d if d > result

-    result

-

-# matrix extent, min max

-matrixMin = (mat) ->

-    result = mat[0][0]

-    for i of mat

-        for j of mat[i]

-            result = mat[i][j] if result > mat[i][j]

-    result      

-

-matrixMax = (mat) ->

-    result = mat[0][0]

-    for i of mat

-        for j of mat[i]

-            result = mat[i][j] if result < mat[i][j]

-    result      

-

-matrixMaxAbs = (mat) ->

-    result = Math.abs(mat[0][0])

-    for i of mat

-        for j of mat[i]

-            result = Math.abs(mat[i][j]) if result < mat[i][j]

-    result      

-

-matrixExtent = (mat) -> [matrixMin(mat), matrixMax(mat)]

-

-d3.selection.prototype.moveToFront = () ->

-    this.each () -> this.parentNode.appendChild(this)

-  

-d3.selection.prototype.moveToBack = () ->

-    this.each () ->

-        firstChild = this.parentNode.firstchild

-        this.parentNode.insertBefore(this, firstChild) if firstChild

-

-forceAsArray = (x) ->

-    return x unless x? # if null, return null

-    return x if Array.isArray(x)

-    [x]

-

-# any values in vec that appear in missing are made null

-missing2null = (vec, missingvalues=['NA', '']) ->

-    vec.map (value) -> if missingvalues.indexOf(value) > -1 then null else value

-

-# display error at top of page

-displayError = (message) ->

-    if d3.select("div.error").empty() # no errors yet

-        d3.select("body")

-          .insert("div", ":first-child")

-          .attr("class", "error")

-    d3.select("div.error")

-      .append("p")

-      .text(message)

-

-# sum values in an array

-sumArray = (vec) -> (vec.reduce (a,b) -> a+b)

-

-# calculate cross-tabulation

-calc_crosstab = (data) ->

-    nrow = data.ycat.length

-    ncol = data.xcat.length

-

-    result = ((0 for col in [0..ncol]) for row in [0..nrow]) # matrix of 0's

-

-    # count things up

-    for i of data.x

-        result[data.y[i]][data.x[i]] += 1

-

-    # row and column sums

-    rs = rowSums(result)

-    cs = colSums(result)

-

-    # fill in column sums

-    for i in [0...ncol]

-        result[nrow][i] = cs[i]

-

-    # fill in row sums

-    for i in [0...nrow]

-        result[i][ncol] = rs[i]

-

-    # fill in total

-    result[nrow][ncol] = sumArray(rs)

-

-    result

-

-# rowSums: the sums for each row

-rowSums = (mat) -> (sumArray(x) for x in mat)

-

-# transpose: matrix transpose

-transpose = (mat) -> ((mat[i][j] for i in [0...mat.length]) for j in [0...mat[0].length])

-

-# colSums = the sums for each column

-colSums = (mat) -> rowSums(transpose(mat))

-

-# log base 2

-log2 = (x) -> 

-    return(x) unless x?

-    Math.log(x)/Math.log(2.0)

-

-# log base 10

-log10 = (x) ->

-    return(x) unless x?

-    Math.log(x)/Math.log(10.0)

-

-# absolute value, preserving nulls

-abs = (x) -> 

-    return(x) unless x?

+# A variety of utility functions used by the different panel functions
+
+# determine rounding of axis labels
+formatAxis = (d, extra_digits=0) ->
+    d = d[1] - d[0]
+    ndig = Math.floor( Math.log(d % 10) / Math.log(10) )
+    ndig = 0 if ndig > 0
+    ndig = Math.abs(ndig) + extra_digits
+    d3.format(".#{ndig}f")
+
+# unique values of array (ignore nulls)
+unique = (x) ->
+    output = {}
+    output[v] = v for v in x when v
+    output[v] for v of output
+
+# Pull out a variable (column) from a two-dimensional array
+pullVarAsArray = (data, variable) ->
+    v = []
+    for i of data
+        v = v.concat data[i][variable]
+    v
+  
+  
+# reorganize lod/pos by chromosome
+# lodvarname==null    -> case for multiple LOD columns (lodheatmap)
+# lodvarname provided -> case for one LOD column (lodchart)
+reorgLodData = (data, lodvarname=null) ->
+    data.posByChr = {}
+    data.lodByChr = {}
+    
+    for chr,i in data.chrnames
+      #console.log("chr:", chr)
+      data.posByChr[chr[0]] = []
+      data.lodByChr[chr[0]] = []
+      for pos, j in data.pos
+        if data.chr[j].toString() == chr[0]
+          #console.log(data.chr[j] + " AND " + chr[0])
+          data.posByChr[chr[0]].push(pos)
+          data.lodnames = [data.lodnames] unless Array.isArray(data.lodnames)
+          lodval = (data[lodcolumn][j] for lodcolumn in data.lodnames)
+          data.lodByChr[chr[0]].push(lodval)
+
+    #console.log("data.posByChr:", data.posByChr)
+
+    if lodvarname?
+      data.markers = []
+      for marker,i in data.markernames
+        if marker != ""
+          data.markers.push({name:marker, chr:data.chr[i], pos:data.pos[i], lod:data[lodvarname][i]})
+    
+    data
+  
+# calculate chromosome start/end + scales, for heat map
+chrscales = (data, width, chrGap, leftMargin, pad4heatmap) ->
+    # start and end of chromosome positions
+    chrStart = []
+    chrEnd = []
+    chrLength = []
+    totalChrLength = 0
+    maxd = 0
+    for chr in data.chrnames
+      d = maxdiff(data.posByChr[chr[0]])
+      maxd = d if d > maxd
+  
+      rng = d3.extent(data.posByChr[chr[0]])
+      chrStart.push(rng[0])
+      chrEnd.push(rng[1])
+      L = rng[1] - rng[0]
+      chrLength.push(L)
+      totalChrLength += L
+  
+    # adjust lengths for heatmap
+    if pad4heatmap
+      data.recwidth = maxd
+      chrStart = chrStart.map (x) -> x-maxd/2
+      chrEnd = chrEnd.map (x) -> x+maxd/2
+      chrLength = chrLength.map (x) -> x+maxd
+      totalChrLength += (chrLength.length*maxd)
+  
+    # break up x axis into chromosomes by length, with gaps
+    data.chrStart = []
+    data.chrEnd = []
+    cur = leftMargin
+    cur += chrGap/2 unless pad4heatmap
+    data.xscale = {}
+    for chr,i in data.chrnames
+      data.chrStart.push(cur)
+      w = Math.round((width-chrGap*(data.chrnames.length-pad4heatmap))/totalChrLength*chrLength[i])
+      data.chrEnd.push(cur + w)
+      cur = data.chrEnd[i] + chrGap
+      # x-axis scales, by chromosome
+      data.xscale[chr[0]] = d3.scale.linear()
+                           .domain([chrStart[i], chrEnd[i]])
+                           .range([data.chrStart[i], data.chrEnd[i]])
+  
+    # return data with new stuff added
+    data
+    
+    
+# reorganize lod/pos by chromosome
+# lodvarname==null    -> case for multiple LOD columns (lodheatmap)
+# lodvarname provided -> case for one LOD column (lodchart)
+#reorgLodData = (data, lodvarname=null) ->
+#    data.posByChr = {}
+#    data.lodByChr = {}
+#
+#    #console.log("data.chr", data.chr)
+#    #console.log("data.chrnames:", data.chrnames)
+#    the_chr = "0"
+#    for chr,i in data.chrnames
+#        data.posByChr[chr] = []
+#        data.lodByChr[chr] = []
+#        for pos,j in data.pos
+#            console.log("data.chr[j][0]:", data.chr[j][0])
+#            if data.chr[j][0] == chr
+#                console.log("IS EQUAL")
+#                data.posByChr[chr].push(pos)
+#                data.lodnames = [data.lodnames] unless Array.isArray(data.lodnames)
+#                lodval = (data[lodcolumn][j] for lodcolumn in data.lodnames)
+#                data.lodByChr[chr].push(lodval)
+#
+#    if lodvarname?
+#        data.markers = []
+#        for marker,i in data.markernames
+#            if marker != ""
+#                data.markers.push({name:marker, chr:data.chr[i][0], pos:data.pos[i], lod:data[lodvarname][i]})
+#
+#    data
+
+# calculate chromosome start/end + scales, for heat map
+#chrscales = (data, width, chrGap, leftMargin, pad4heatmap) ->
+#    # start and end of chromosome positions
+#    chrStart = []
+#    chrEnd = []
+#    chrLength = []
+#    totalChrLength = 0
+#    maxd = 0
+#    for chr in data.chrnames
+#        d = maxdiff(data.posByChr[chr])
+#        maxd = d if d > maxd
+#
+#        rng = d3.extent(data.posByChr[chr])
+#        chrStart.push(rng[0])
+#        chrEnd.push(rng[1])
+#        L = rng[1] - rng[0]
+#        chrLength.push(L)
+#        totalChrLength += L
+#
+#    # adjust lengths for heatmap
+#    if pad4heatmap
+#        data.recwidth = maxd
+#        chrStart = chrStart.map (x) -> x-maxd/2
+#        chrEnd = chrEnd.map (x) -> x+maxd/2
+#        chrLength = chrLength.map (x) -> x+maxd
+#        totalChrLength += (chrLength.length*maxd)
+#
+#    # break up x axis into chromosomes by length, with gaps
+#    data.chrStart = []
+#    data.chrEnd = []
+#    cur = leftMargin
+#    cur += chrGap/2 unless pad4heatmap
+#    data.xscale = {}
+#    for chr,i in data.chrnames
+#        data.chrStart.push(cur)
+#        w = Math.round((width-chrGap*(data.chrnames.length-pad4heatmap))/totalChrLength*chrLength[i])
+#        data.chrEnd.push(cur + w)
+#        cur = data.chrEnd[i] + chrGap
+#        # x-axis scales, by chromosome
+#        data.xscale[chr] = d3.scale.linear()
+#                             .domain([chrStart[i], chrEnd[i]])
+#                             .range([data.chrStart[i], data.chrEnd[i]])
+#
+#    # return data with new stuff added
+#    data
+
+# Select a set of categorical colors
+# ngroup is positive integer
+# palette = "dark" or "pastel"
+selectGroupColors = (ngroup, palette) ->
+    return [] if ngroup == 0
+
+    if palette == "dark"
+        return ["slateblue"] if ngroup == 1
+        return ["MediumVioletRed", "slateblue"] if ngroup == 2
+        return colorbrewer.Set1[ngroup] if ngroup <= 9
+        return d3.scale.category20().range()[0...ngroup]
+    else
+        return ["#bebebe"] if ngroup == 1
+        return ["lightpink", "lightblue"] if ngroup == 2
+        return colorbrewer.Pastel1[ngroup] if ngroup <= 9
+        # below is rough attempt to make _big_ pastel palette
+        return ["#8fc7f4", "#fed7f8", "#ffbf8e", "#fffbb8",
+                "#8ce08c", "#d8ffca", "#f68788", "#ffd8d6",
+                "#d4a7fd", "#f5f0f5", "#cc968b", "#f4dcd4",
+                "#f3b7f2", "#f7f6f2", "#bfbfbf", "#f7f7f7",
+                "#fcfd82", "#fbfbcd", "#87feff", "#defaf5"][0...ngroup]
+
+# expand element/array (e.g., of colors) to a given length
+#     single elment -> array, then repeated to length n
+expand2vector = (input, n) ->
+    return input unless input? # return null if null
+    return input if Array.isArray(input) and input.length >= n
+    input = [input] unless Array.isArray(input)
+    input = (input[0] for i of d3.range(n)) if input.length == 1 and n > 1
+    input
+
+# median of a vector
+median = (x) ->
+    return null if !x? 
+    n = x.length
+    x.sort((a,b) -> a-b)
+    if n % 2 == 1
+        return x[(n-1)/2]
+    (x[n/2] + x[(n/2)-1])/2
+
+# given a vector of x's, return hash with values to left and right, and the differences
+getLeftRight = (x) ->
+    n = x.length
+    x.sort( (a,b) -> a-b )
+
+    xdif = []
+    result = {}
+    for v in x
+        result[v] = {}
+
+    for i in [1...n]
+        #console.log("result:", result)
+        xdif.push(x[i]-x[i-1])
+        result[x[i]].left = x[i-1]
+    for i in [0...(n-1)]
+        result[x[i]].right = x[i+1]
+
+    xdif = median(xdif)
+    result.mediandiff = xdif
+
+    result[x[0]].left = x[0]-xdif
+    result[x[n-1]].right = x[n-1]+xdif
+    result.extent = [x[0]-xdif/2, x[n-1]+xdif/2]
+
+    result
+
+# maximum difference between adjacent values in a vector
+maxdiff = (x) ->
+    return null if x.length < 2
+    result = x[1] - x[0]
+    return result if x.length < 3
+    for i in [2...x.length]
+        d = x[i] - x[i-1]
+        result = d if d > result
+    result
+
+# matrix extent, min max
+matrixMin = (mat) ->
+    result = mat[0][0]
+    for i of mat
+        for j of mat[i]
+            result = mat[i][j] if result > mat[i][j]
+    result      
+
+matrixMax = (mat) ->
+    result = mat[0][0]
+    for i of mat
+        for j of mat[i]
+            result = mat[i][j] if result < mat[i][j]
+    result      
+
+matrixMaxAbs = (mat) ->
+    result = Math.abs(mat[0][0])
+    for i of mat
+        for j of mat[i]
+            result = Math.abs(mat[i][j]) if result < mat[i][j]
+    result      
+
+matrixExtent = (mat) -> [matrixMin(mat), matrixMax(mat)]
+
+d3.selection.prototype.moveToFront = () ->
+    this.each () -> this.parentNode.appendChild(this)
+  
+d3.selection.prototype.moveToBack = () ->
+    this.each () ->
+        firstChild = this.parentNode.firstchild
+        this.parentNode.insertBefore(this, firstChild) if firstChild
+
+forceAsArray = (x) ->
+    return x unless x? # if null, return null
+    return x if Array.isArray(x)
+    [x]
+
+# any values in vec that appear in missing are made null
+missing2null = (vec, missingvalues=['NA', '']) ->
+    vec.map (value) -> if missingvalues.indexOf(value) > -1 then null else value
+
+# display error at top of page
+displayError = (message) ->
+    if d3.select("div.error").empty() # no errors yet
+        d3.select("body")
+          .insert("div", ":first-child")
+          .attr("class", "error")
+    d3.select("div.error")
+      .append("p")
+      .text(message)
+
+# sum values in an array
+sumArray = (vec) -> (vec.reduce (a,b) -> a+b)
+
+# calculate cross-tabulation
+calc_crosstab = (data) ->
+    nrow = data.ycat.length
+    ncol = data.xcat.length
+
+    result = ((0 for col in [0..ncol]) for row in [0..nrow]) # matrix of 0's
+
+    # count things up
+    for i of data.x
+        result[data.y[i]][data.x[i]] += 1
+
+    # row and column sums
+    rs = rowSums(result)
+    cs = colSums(result)
+
+    # fill in column sums
+    for i in [0...ncol]
+        result[nrow][i] = cs[i]
+
+    # fill in row sums
+    for i in [0...nrow]
+        result[i][ncol] = rs[i]
+
+    # fill in total
+    result[nrow][ncol] = sumArray(rs)
+
+    result
+
+# rowSums: the sums for each row
+rowSums = (mat) -> (sumArray(x) for x in mat)
+
+# transpose: matrix transpose
+transpose = (mat) -> ((mat[i][j] for i in [0...mat.length]) for j in [0...mat[0].length])
+
+# colSums = the sums for each column
+colSums = (mat) -> rowSums(transpose(mat))
+
+# log base 2
+log2 = (x) -> 
+    return(x) unless x?
+    Math.log(x)/Math.log(2.0)
+
+# log base 10
+log10 = (x) ->
+    return(x) unless x?
+    Math.log(x)/Math.log(10.0)
+
+# absolute value, preserving nulls
+abs = (x) -> 
+    return(x) unless x?
     Math.abs(x)
\ No newline at end of file
diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee
index 8d56d04b..8b73b85d 100755
--- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee
+++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee
@@ -134,7 +134,7 @@ $('input[name=display_all]').change(() =>
 )
 
 $("#pylmm_compute").click(() =>
-        $("#progress_bar_container").modal()
+        $("#progress_bar_container").modal({show:true})
         url = "/marker_regression"
         $('input[name=method]').val("pylmm")
         $('input[name=num_perm]').val($('input[name=num_perm_pylmm]').val())
diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
index e1ef8b63..92d3183a 100755
--- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
@@ -132,7 +132,9 @@ $('input[name=display_all]').change((function(_this) {
 $("#pylmm_compute").click((function(_this) {
   return function() {
     var form_data, url;
-    $("#progress_bar_container").modal();
+    $("#progress_bar_container").modal({
+      show: true
+    });
     url = "/marker_regression";
     $('input[name=method]').val("pylmm");
     $('input[name=num_perm]').val($('input[name=num_perm_pylmm]').val());
diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html
index 29fa0469..e032dfc5 100755
--- a/wqflask/wqflask/templates/base.html
+++ b/wqflask/wqflask/templates/base.html
@@ -72,7 +72,7 @@
                 </div>
 
                 <!-- Collect the nav links, forms, and other content for toggling -->
-                <div class="nav-collapse collapse">
+                <div class="nav-collapse collapse in">
                     <ul class="nav navbar-nav">
                         <li class="">
                             <a href="/">Home</a>
diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html
index eee22afe..1c94d61f 100755
--- a/wqflask/wqflask/templates/collections/view.html
+++ b/wqflask/wqflask/templates/collections/view.html
@@ -16,14 +16,42 @@
             <h2>{{ uc.name }}</h2>
             {% endif %}
 
+            <div class="form-group">
             <form action="/collections/delete" method="post">
                 {% if uc %}
                 <input type="hidden" name="uc_id" id="uc_id" value="{{ uc.id }}" />
                 {% endif %}
-                <input type="submit"
-                       class="btn btn-small btn-danger"
-                       value="Delete this collection" />
+                <div class="col-sm-2 controls">
+                    <input type="submit" class="btn btn-danger" value="Delete this collection" />
+                </div>
+            </form>
+            <form action="/corr_matrix" method="post">
+                {% if uc %}
+                <input type="hidden" name="uc_id" id="uc_id" value="{{ uc.id }}" />
+                {% endif %}
+                <input type="hidden" name="trait_list" id="trait_list" value= "
+                {% for this_trait in trait_obs %}
+                    {{ this_trait.name }}:{{ this_trait.dataset.name }},
+                {% endfor %}" >
+                <div class="col-sm-2 controls">
+                    <input type="submit" class="btn" value="Correlation Matrix" />
+                </div>
             </form>
+            <form action="/heatmap" method="post">
+                {% if uc %}
+                <input type="hidden" name="uc_id" id="uc_id" value="{{ uc.id }}" />
+                {% endif %}
+                <input type="hidden" name="trait_list" id="trait_list" value= "
+                {% for this_trait in trait_obs %}
+                    {{ this_trait.name }}:{{ this_trait.dataset.name }},
+                {% endfor %}" >
+                <div class="col-sm-2 controls">
+                    <input type="submit" class="btn" value="Heatmap" />
+                </div>
+            </form>
+            </div>
+
+            <!--
             <form action="/corr_matrix" method="post">
                 {% if uc %}
                 <input type="hidden" name="uc_id" id="uc_id" value="{{ uc.id }}" />
@@ -48,6 +76,7 @@
                        class="btn btn-small"
                        value="Heatmap" />
             </form>
+            -->
         </div>
 
 
diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html
index 277acffa..fcaabce3 100755
--- a/wqflask/wqflask/templates/index_page.html
+++ b/wqflask/wqflask/templates/index_page.html
@@ -226,8 +226,8 @@
 
                                 <div class="form-group">
                                     <label for="tfor" class="col-sm-1 control-label">Search:</label>
-                                    <div style="margin-left: 30px;" class="col-sm-6 controls">
-                                        <input name="search_terms" type="text" class="form-control search-query" id="tfor">
+                                    <div style="margin-left: 30px;" class="col-sm-10 controls">
+                                        <textarea name="search_terms" rows="2" class="form-control search-query" id="tfor"></textarea>
                                     </div>
                                 </div>
 
@@ -242,12 +242,18 @@
 
                                 <!--  SEARCH, MAKE DEFAULT, ADVANCED SEARCH -->
 
+                                <div class="form-group">
 
-                                <input id="btsearch" type="submit" class="btn btn-primary form-control" value="Search">
-
-                                <input type="button" class="btn form-control btn-default" value="Make Default" id="make_default">
-
-                                <input type="button" class="btn form-control btn-default" value="Advanced Search" onclick="javascript:window.open('/index3.html', '_self');">
+                                    <div class="col-sm-3 controls">
+                                        <input id="btsearch" type="submit" class="btn btn-primary form-control" value="Search">
+                                    </div>
+                                    <div class="col-sm-4 controls">
+                                        <input id="make_default" type="submit" class="btn btn-default form-control" value="Make Default">
+                                    </div>
+                                    <div class="col-sm-5 controls">
+                                        <input type="submit" class="btn btn-default form-control" value="Advanced Search">
+                                    </div>
+                                </div>
 
                                 <input type="hidden" name="FormID" value="searchResult" class="form-control">
                                 <!--!<input type="hidden" name="RISet" value="BXD">-->
diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html
index 4e308c75..3378846a 100755
--- a/wqflask/wqflask/templates/new_security/login_user.html
+++ b/wqflask/wqflask/templates/new_security/login_user.html
@@ -26,21 +26,18 @@
 
 
 
-            <form class="form-horizontal" action="/n/login"
-                  method="POST" name="login_user_form">
-                <fieldset>
-
-
-                    <div class="control-group">
-                        <label class="control-label" for="email_address">Email Address</label>
-                        <div class="controls">
+            <form class="form-horizontal" action="/n/login" method="POST" name="login_user_form">
+                 <fieldset>
+                    <div class="form-group">
+                        <label class="col-sm-2 control-label" for="email_address">Email&nbsp;Address</label>
+                        <div class="col-sm-3 controls">
                             <input id="email_address" class="focused" name="email_address" type="text" value="">
                         </div>
                     </div>
 
-                    <div class="control-group">
-                        <label class="control-label" for="password">Password</label>
-                        <div class="controls">
+                    <div class="form-group">
+                        <label class="col-sm-2 control-label" for="password">Password</label>
+                        <div class="col-sm-3 controls">
                             <input id="password" name="password" type="password" value="">
                             <br />
                             <a href="/n/forgot_password">Forgot your password?</a><br/>
@@ -48,22 +45,21 @@
                     </div>
 
 
-                    <div class="control-group">
-                        <div class="controls">
+                    <div class="form-group">
+                        <label class="col-sm-2 control-label" for="remember"></label>
+                        <div class="col-sm-3 controls">
                             <label class="checkbox">
                                 <input id="remember" name="remember" type="checkbox" value="y"> Remember me
                             </label>
                         </div>
+                    </div>
 
-
-                    <div class="control-group">
-                        <div class="controls">
+                    <div class="form-group">
+                        <label class="col-sm-2 control-label" for="submit"></label>
+                        <div class="col-sm-3 controls">
                             <input id="next" name="next" type="hidden" value="">
-
                             <input class="btn btn-primary" id="submit" name="submit" type="submit" value="Sign in">
                         </div>
-
-
                     </div>
               </fieldset>
 
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index af606600..5d0437df 100755
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -18,13 +18,13 @@
 
     <form method="post" action="/corr_compute" name="trait_page" id="trait_data_form"
     class="form-horizontal">
-            <div id="hidden_inputs">
-            {% for key in hddn %}
-                <input type="hidden" name="{{ key }}" value="{{ hddn[key] }}">
-            {% endfor %}
-            </div>
+        <div id="hidden_inputs">
+        {% for key in hddn %}
+            <input type="hidden" name="{{ key }}" value="{{ hddn[key] }}">
+        {% endfor %}
+        </div>
 
-            <input type="hidden" name="temp_uuid" id="temp_uuid" value="{{ temp_uuid }}">
+        <input type="hidden" name="temp_uuid" id="temp_uuid" value="{{ temp_uuid }}">
 
         <div class="container">
             <div class="page-header">
@@ -81,18 +81,19 @@
                 <div class="panel panel-default">
                     <div class="panel-heading">
                         <h3 class="panel-title">
-                            <a data-toggle="collapse" data-parent="#accordion" href="#collapseFour">
+                            <a data-toggle="collapse" data-parent="#accordion" href="#collapseFour" aria-expanded="true">
                                 Review and Edit Data
                             </a>
                         </h3>
                     </div>
-                    <div id="collapseFour" class="panel-collapse collapse in">
+                    <div id="collapseFour" class="panel-collapse collapse" aria-expanded="true">
                         <div class="panel-body">
                             {% include 'show_trait_edit_data.html' %}
                         </div>
                     </div>
                 </div>
             </div>
+            {% include 'show_trait_progress_bar.html' %}
  
         </div>
     </form>
@@ -134,7 +135,7 @@
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/dataTables.naturalSort.js"></script>
     <script language="javascript" type="text/javascript" src="/static/packages/DT_bootstrap/DT_bootstrap.js"></script>
     <script language="javascript" type="text/javascript" src="/static/packages/TableTools/media/js/TableTools.min.js"></script>
-        <script type="text/javascript" charset="utf-8">
+    <script type="text/javascript" charset="utf-8">
             
            function getValue(x) {
                 if (x.indexOf('input') >= 0) {
@@ -227,6 +228,6 @@
                     console.timeEnd("Creating table");
                 {% endif %}
             });
-        </script>
+    </script>
     
 {% endblock %}
diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html
index 88fd74d9..0afac1f7 100755
--- a/wqflask/wqflask/templates/show_trait_details.html
+++ b/wqflask/wqflask/templates/show_trait_details.html
@@ -19,11 +19,11 @@
             BLAT Specifity
         </a>
     </dt>
-    <dd>{{ "%s" % (this_trait.probe_set_specificity) }}</dd>
+    <dd>{{ "%0.3f" | format(this_trait.probe_set_specificity|float) }}</dd>
     {% endif %}
     {% if this_trait.probe_set_blat_score %}
     <dt>BLAT Score</dt>
-    <dd>{{ "%s" % (this_trait.probe_set_blat_score) }}</dd>
+    <dd>{{ "%0.3f" | format(this_trait.probe_set_blat_score|float) }}</dd>
     {% endif %}
 </dl>
 
diff --git a/wqflask/wqflask/templates/show_trait_progress_bar.html b/wqflask/wqflask/templates/show_trait_progress_bar.html
index aebf64a4..99906338 100755
--- a/wqflask/wqflask/templates/show_trait_progress_bar.html
+++ b/wqflask/wqflask/templates/show_trait_progress_bar.html
@@ -1,17 +1,40 @@
-<div id="progress_bar_container" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="progress_bar" aria-hidden="true">
-  <div class="modal-header">
-    <h3 id="progress_bar">Loading...</h3>
+<div id="progress_bar_container" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="progress_bar" aria-hidden="true">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h3 id="progress_bar">Loading...</h3>
+      </div>
+      <div class="modal-body">
+        <div class="progress">
+          <div id="marker_regression_progress" class="progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:0%;">
+          </div>
+        </div>
+        <div id="time_remaining">
+        </div>
+      </div>
+    </div>
   </div>
-    <div class="modal-body">
-        <div class="progress active">
-            <div id="marker_regression_progress" class="bar"></div>
+</div>
+
+<div id="static_progress_bar_container" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="progress_bar" aria-hidden="true">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h3 id="progress_bar">Loading... (Estimated time ~10-15m)</h3>
+      </div>
+      <div class="modal-body">
+        <div class="progress progress-striped active">
+          <div id="marker_regression_progress" class="progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:100%;">
+          </div>
         </div>
         <div id="time_remaining">
         </div>
+      </div>
     </div>
+  </div>
 </div>
 
-<div id="static_progress_bar_container" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="progress_bar" aria-hidden="true">
+<!--<div id="static_progress_bar_container" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="progress_bar" aria-hidden="true">
   <div class="modal-header">
     <h3 id="progress_bar">Loading... (Estimated time ~10-15m)</h3>
   </div>
@@ -22,4 +45,4 @@
         <div id="time_remaining">
         </div>
     </div>
-</div>
\ No newline at end of file
+</div>-->
\ No newline at end of file