Added factor loadings table and plot (though need to finish improving plot still)

Fixed issue where cM locations for R/qtl were displayed as Mb and had used Mb Chr lenghts for the scale

10 files changed, 137 insertions, 37 deletions

diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
index 2cdd989f..6bc0ef77 100755
--- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
+++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
@@ -95,35 +95,24 @@ class CorrelationMatrix(object):
                     #self.sample_data[this_trait.name].append('')
             self.sample_data.append(this_trait_vals)
 
+        self.lowest_overlap = 8 #ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning)
+
         self.corr_results = []
-        self.corr_results_for_pca = []
+        self.pca_corr_results = []
         for trait_db in self.trait_list:
             this_trait = trait_db[0]
             this_db = trait_db[1]
             
             this_db_samples = this_db.group.samplelist
-            
-            #for sample in this_db_samples:
-            #    if sample not in self.samples:
-            #        self.samples.append(sample)
-            
             this_sample_data = this_trait.data
-            print("this_sample_data", len(this_sample_data))
-            
-            #for sample in this_sample_data:
-            #    if sample not in self.all_sample_list:
-            #        self.all_sample_list.append(sample)
             
             corr_result_row = []
+            pca_corr_result_row = []
             is_spearman = False #ZS: To determine if it's above or below the diagonal
             for target in self.trait_list:
                 target_trait = target[0]
                 target_db = target[1]
                 target_samples = target_db.group.samplelist
-                
-                #if this_trait == target_trait and this_db == target_db:
-                #    corr_result_row.append(1)
-                #    continue
 
                 target_sample_data = target_trait.data
                 print("target_samples", len(target_samples))
@@ -139,19 +128,26 @@ class CorrelationMatrix(object):
                         target_vals.append(target_sample_value)
         
                 this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(this_trait_vals, target_vals)
+
+                if num_overlap < self.lowest_overlap:
+                    self.lowest_overlap = num_overlap
                 if num_overlap == 0:
                     corr_result_row.append([target_trait, 0, num_overlap])
+                    pca_corr_result_row.append(0)
                 else:
+                    pearson_r, pearson_p = scipy.stats.pearsonr(this_trait_vals, target_vals)
                     if is_spearman == False:
-                        sample_r, sample_p = scipy.stats.pearsonr(this_trait_vals, target_vals)
+                        sample_r, sample_p = pearson_r, pearson_p
                         if sample_r == 1:
                             is_spearman = True
                     else:
                         sample_r, sample_p = scipy.stats.spearmanr(this_trait_vals, target_vals)
 
                     corr_result_row.append([target_trait, sample_r, num_overlap])
+                    pca_corr_result_row.append(pearson_r)
                 
             self.corr_results.append(corr_result_row)
+            self.pca_corr_results.append(pca_corr_result_row)
 
         print("corr_results:", pf(self.corr_results))
 
@@ -159,8 +155,9 @@ class CorrelationMatrix(object):
         for sample in self.all_sample_list:
             groups.append(1)
 
-        #pca = self.calculate_pca(self.corr_results, range(len(self.traits)))
+        pca = self.calculate_pca(self.pca_corr_results, range(len(self.traits)))
 
+        self.loadings_array = self.process_loadings()
 
         self.js_data = dict(traits = [trait.name for trait in self.traits],
                             groups = groups,
@@ -211,8 +208,24 @@ class CorrelationMatrix(object):
         print("eigen:", eigen)
         pca = stats.princomp(m, cor = "TRUE")
         print("pca:", pca)
-        print("loadings:", pca.rx('loadings'))
+        self.loadings = pca.rx('loadings')
+        self.scores = pca.rx('scores')
+        self.scale = pca.rx('scale')
         print("scores:", pca.rx('scores'))
         print("scale:", pca.rx('scale'))
 
-        return pca
\ No newline at end of file
+        return pca
+
+    def process_loadings(self):
+        loadings_array = []
+        loadings_row = []
+        print("before loop:", self.loadings[0])
+        for i in range(len(self.trait_list)):
+            loadings_row = []
+            for j in range(3):
+                position = i + len(self.trait_list)*j
+                loadings_row.append(self.loadings[0][position])
+            loadings_array.append(loadings_row)
+        print("loadings:", loadings_array)
+        return loadings_array
+
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index b33efc1f..dc7ebfcc 100755
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -74,6 +74,7 @@ class MarkerRegression(object):
         self.significant = ""
         self.pair_scan = False # Initializing this since it is checked in views to determine which template to use
         self.score_type = "LRS" #ZS: LRS or LOD
+        self.mapping_scale = "megabase"
  
         self.dataset.group.get_markers()
         if self.mapping_method == "gemma":
@@ -86,6 +87,7 @@ class MarkerRegression(object):
             results = self.run_rqtl_plink()
         elif self.mapping_method == "rqtl_geno":
             self.score_type = "LOD"
+            self.mapping_scale = "centimorgan"
             if start_vars['num_perm'] == "":
                 self.num_perm = 0
             else:
@@ -136,6 +138,7 @@ class MarkerRegression(object):
                 data_set = self.dataset.name,
                 maf = self.maf,
                 manhattan_plot = self.manhattan_plot,
+                mapping_scale = self.mapping_scale,
                 qtl_results = self.qtl_results,
             )
 
@@ -191,6 +194,7 @@ class MarkerRegression(object):
                 data_set = self.dataset.name,
                 maf = self.maf,
                 manhattan_plot = self.manhattan_plot,
+                mapping_scale = self.mapping_scale,
                 chromosomes = chromosome_mb_lengths,
                 qtl_results = self.qtl_results,
             )
diff --git a/wqflask/wqflask/static/new/css/corr_matrix.css b/wqflask/wqflask/static/new/css/corr_matrix.css
index f4838f77..495ca28c 100755
--- a/wqflask/wqflask/static/new/css/corr_matrix.css
+++ b/wqflask/wqflask/static/new/css/corr_matrix.css
@@ -10,3 +10,21 @@
               width: 100px;
     float: left; 
 }
+
+.chart {
+
+}
+
+.main text {
+    font: 10px sans-serif;	
+}
+
+.axis line, .axis path {
+    shape-rendering: crispEdges;
+    stroke: black;
+    fill: none;
+}
+
+circle {
+    fill: steelblue;
+}
diff --git a/wqflask/wqflask/static/new/javascript/chr_lod_chart.js b/wqflask/wqflask/static/new/javascript/chr_lod_chart.js
index 616a89f6..60878978 100644
--- a/wqflask/wqflask/static/new/javascript/chr_lod_chart.js
+++ b/wqflask/wqflask/static/new/javascript/chr_lod_chart.js
@@ -2,11 +2,12 @@
 var Chr_Lod_Chart;
 
 Chr_Lod_Chart = (function() {
-  function Chr_Lod_Chart(plot_height, plot_width, chr, manhattanPlot) {
+  function Chr_Lod_Chart(plot_height, plot_width, chr, manhattanPlot, mappingScale) {
     this.plot_height = plot_height;
     this.plot_width = plot_width;
     this.chr = chr;
     this.manhattanPlot = manhattanPlot;
+    this.mappingScale = mappingScale;
     this.qtl_results = js_data.qtl_results;
     console.log("qtl_results are:", this.qtl_results);
     console.log("chr is:", this.chr);
@@ -65,8 +66,6 @@ Chr_Lod_Chart = (function() {
       } else {
         this_chr = result.chr;
       }
-      console.log("this_chr is:", this_chr);
-      console.log("@chr[0] is:", parseInt(this.chr[0]));
       if (this_chr > parseInt(this.chr[0])) {
         break;
       }
@@ -120,7 +119,19 @@ Chr_Lod_Chart = (function() {
   };
 
   Chr_Lod_Chart.prototype.create_scales = function() {
-    this.x_scale = d3.scale.linear().domain([0, this.chr[1]]).range([this.x_buffer, this.plot_width]);
+    if (this.mappingScale == "centimorgan") {
+        max_pos = 0
+        for (i = 0, len = this.these_results.length; i < len; i++) {
+           marker = this.these_results[i]
+           if (parseFloat(marker['Mb']) > max_pos){
+               max_pos = parseFloat(marker.Mb)
+           }
+        }
+        this.x_scale = d3.scale.linear().domain([0, max_pos]).range([this.x_buffer, this.plot_width]);
+    }
+    else {
+        this.x_scale = d3.scale.linear().domain([0, this.chr[1]]).range([this.x_buffer, this.plot_width]);
+    }
     return this.y_scale = d3.scale.linear().domain([0, this.y_max]).range([this.plot_height, this.y_buffer]);
   };
 
diff --git a/wqflask/wqflask/static/new/javascript/create_lodchart.js b/wqflask/wqflask/static/new/javascript/create_lodchart.js
index b8fcf1f8..c756d842 100644
--- a/wqflask/wqflask/static/new/javascript/create_lodchart.js
+++ b/wqflask/wqflask/static/new/javascript/create_lodchart.js
@@ -16,8 +16,8 @@
     halfh = h + margin.top + margin.bottom;
     totalh = halfh * 2;
     totalw = w + margin.left + margin.right;
-    //console.log("js_data:", js_data);
-    mychart = lodchart().lodvarname("lod.hk").height(h).width(w).margin(margin).ylab(js_data.result_score_type + " score").manhattanPlot(js_data.manhattan_plot);
+    console.log("js_data:", js_data);
+    mychart = lodchart().lodvarname("lod.hk").height(h).width(w).margin(margin).ylab(js_data.result_score_type + " score").mappingScale(js_data.mapping_scale).manhattanPlot(js_data.manhattan_plot);
     data = js_data.json_data;
     d3.select("div#topchart").datum(data).call(mychart);
     chrrect = mychart.chrSelect();
diff --git a/wqflask/wqflask/static/new/javascript/lod_chart.js b/wqflask/wqflask/static/new/javascript/lod_chart.js
index d0d39fe6..014bf59b 100644
--- a/wqflask/wqflask/static/new/javascript/lod_chart.js
+++ b/wqflask/wqflask/static/new/javascript/lod_chart.js
@@ -2,7 +2,7 @@
 var lodchart;
 
 lodchart = function() {
-  var additive, additive_ylab, additive_ylim, additive_yscale, additive_yticks, additivelinecolor, axispos, chart, chrGap, chrSelect, darkrect, height, lightrect, linewidth, lodcurve, lodlinecolor, lodvarname, manhattanPlot, margin, markerSelect, nyticks, pad4heatmap, pointcolor, pointsAtMarkers, pointsize, pointstroke, rotate_ylab, significantcolor, suggestivecolor, title, titlepos, width, xlab, xscale, ylab, ylim, yscale, yticks;
+  var additive, additive_ylab, additive_ylim, additive_yscale, additive_yticks, additivelinecolor, axispos, chart, chrGap, chrSelect, darkrect, height, lightrect, linewidth, lodcurve, lodlinecolor, lodvarname, manhattanPlot, mappingScale, margin, markerSelect, nyticks, pad4heatmap, pointcolor, pointsAtMarkers, pointsize, pointstroke, rotate_ylab, significantcolor, suggestivecolor, title, titlepos, width, xlab, xscale, ylab, ylim, yscale, yticks;
   width = 800;
   height = 500;
   margin = {
@@ -97,7 +97,7 @@ lodchart = function() {
         additive_yticks = additive_yticks != null ? additive_yticks : additive_yscale.ticks(nyticks);
       }
       reorgLodData(data, lodvarname);
-      data = chrscales(data, width, chrGap, margin.left, pad4heatmap);
+      data = chrscales(data, width, chrGap, margin.left, pad4heatmap, mappingScale);
       xscale = data.xscale;
       chrSelect = g.append("g").attr("class", "chrRect").selectAll("empty").data(data.chrnames).enter().append("rect").attr("id", function(d) {
         return "chrrect" + d[0];
@@ -133,7 +133,7 @@ lodchart = function() {
         var chr_plot;
         $('#topchart').remove();
         $('#chart_container').append('<div class="qtlcharts" id="topchart"></div>');
-        return chr_plot = new Chr_Lod_Chart(600, 1200, chr_ob, manhattanPlot);
+        return chr_plot = new Chr_Lod_Chart(600, 1200, chr_ob, manhattanPlot, mappingScale);
       };
       rotate_ylab = rotate_ylab != null ? rotate_ylab : ylab.length > 1;
       yaxis = g.append("g").attr("class", "y axis");
@@ -305,6 +305,13 @@ lodchart = function() {
     manhattanPlot = value;
     return chart;
   };
+  chart.mappingScale = function(value) {
+    if (!arguments.length) {
+      return mappingScale;
+    }
+    mappingScale = value;
+    return chart;
+  };
   chart.ylim = function(value) {
     if (!arguments.length) {
       return ylim;
diff --git a/wqflask/wqflask/static/new/javascript/panelutil.js b/wqflask/wqflask/static/new/javascript/panelutil.js
index 113512b4..5a931f7d 100644
--- a/wqflask/wqflask/static/new/javascript/panelutil.js
+++ b/wqflask/wqflask/static/new/javascript/panelutil.js
@@ -105,7 +105,7 @@ reorgLodData = function(data, lodvarname) {
   return data;
 };
 
-chrscales = function(data, width, chrGap, leftMargin, pad4heatmap) {
+chrscales = function(data, width, chrGap, leftMargin, pad4heatmap, mappingScale) {
   var L, chr, chrEnd, chrLength, chrStart, cur, d, i, maxd, rng, totalChrLength, w, _i, _j, _len, _len1, _ref, _ref1;
   chrStart = [];
   chrEnd = [];
@@ -156,7 +156,15 @@ chrscales = function(data, width, chrGap, leftMargin, pad4heatmap) {
     w = Math.round((width - chrGap * (data.chrnames.length - pad4heatmap)) / totalChrLength * chrLength[i]);
     data.chrEnd.push(cur + w);
     cur = data.chrEnd[i] + chrGap;
-    data.xscale[chr[0]] = d3.scale.linear().domain([chrStart[i], chrEnd[i]]).range([data.chrStart[i], data.chrEnd[i]]);
+
+    if (mappingScale == "centimorgan") {
+        max_pos = d3.max(data.posByChr[chr[0]])
+        console.log("max_pos:", max_pos)
+        data.xscale[chr[0]] = d3.scale.linear().domain([chrStart[i], max_pos]).range([data.chrStart[i], data.chrEnd[i]]);
+    }
+    else {
+        data.xscale[chr[0]] = d3.scale.linear().domain([chrStart[i], chrEnd[i]]).range([data.chrStart[i], data.chrEnd[i]]);
+    }
   }
   return data;
 };
diff --git a/wqflask/wqflask/templates/correlation_matrix.html b/wqflask/wqflask/templates/correlation_matrix.html
index c822b8bc..593c7bea 100755
--- a/wqflask/wqflask/templates/correlation_matrix.html
+++ b/wqflask/wqflask/templates/correlation_matrix.html
@@ -12,7 +12,9 @@
     
     {{ header("Correlation Matrix") }}
 
-   
+{% if lowest_overlap < 8 %}
+<div style="margin: 20px;" >Caution: This matrix of correlations contains some cells with small sample sizes of fewer than 8.</div>
+{% endif %}
 
 <table class="matrix" border="1" cellpadding="5" cellspacing="1" style="margin: 20px;" width="80%">
   <tbody>
@@ -42,18 +44,48 @@
   </tbody>
 </table>
 
+<br>
+
+<div id="loadings_plot"></div>
+
+<table class="table table-hover table-striped" border="1" id='trait_table' style="margin: 20px;" width="40%">
+  <thead>
+    <tr>
+      <th></th>
+      <th align="right" >Factor 1</th>
+      <th align="right" >Factor 2</th>
+      <th align="right" >Factor 2</th>
 <!--
-<div class="container">
-  <div id="chart"></div>
-</div>
+      {% for row in loadings_array %}
+      <th>Factor {{ loop.index }}</th>
+      {% endfor %}
 -->
+    </tr>
+  </thead>
+  <tbody>
+    {% for row in loadings_array %}
+    {% set row_counter = loop.index-1 %}
+    <tr>
+      <td>
+        <a href="{{ url_for('show_trait_page', trait_id = traits[loop.index-1].name, dataset = traits[loop.index-1].dataset.name) }}">
+          {{ traits[loop.index-1].name }}
+        </a>
+      </td>
+      {% for column in row %}
+      <td><span style="float: right;">{{ '%0.3f' % loadings_array[row_counter][loop.index-1]|float }}</span></td>
+      {% endfor %}
+    </tr>
+    {% endfor %}
+    
+  </tbody>
+</table>
 
 {% endblock %}
 
 {% block js %}
 
     <script>
-        js_data = {{ js_data | safe }}
+        loadings = {{ loadings_array | safe }}
     </script>
 
     <script type="text/javascript" src="http://d3js.org/d3.v3.min.js"></script>
@@ -65,7 +97,6 @@
     <script language="javascript" type="text/javascript" src="/static/packages/underscore/underscore-min.js"></script>
     <script type="text/javascript" src="/static/new/javascript/panelutil.js"></script>
     <script language="javascript" type="text/javascript" src="/static/new/js_external/chroma.js"></script>
-    <script language="javascript" type="text/javascript" src="/static/new/javascript/corr_matrix.js"></script>
-    <script language="javascript" type="text/javascript" src="/static/new/javascript/create_corr_matrix.js"></script>
+    <script language="javascript" type="text/javascript" src="/static/new/javascript/loadings_plot.js"></script>
 
 {% endblock %}
diff --git a/wqflask/wqflask/templates/marker_regression.html b/wqflask/wqflask/templates/marker_regression.html
index 2d0b6256..ad117337 100755
--- a/wqflask/wqflask/templates/marker_regression.html
+++ b/wqflask/wqflask/templates/marker_regression.html
@@ -36,7 +36,11 @@
                         <th>Index</th>
                         <th>{{ score_type }}</th>
                         <th>Chr</th>
+                        {% if mapping_scale == "centimorgan" %}
+                        <th>cM</th>
+                        {% else %}
                         <th>Mb</th>
+                        {% endif %}
                         <th>Locus</th>
                    </tr>
                 </thead>
diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html
index f29b907d..e304be1c 100755
--- a/wqflask/wqflask/templates/search_result_page.html
+++ b/wqflask/wqflask/templates/search_result_page.html
@@ -15,6 +15,7 @@
 
         <input type="hidden" name="uc_id" id="uc_id" value="{{ uc_id }}">
 
+        <!-- Need to customize text more for other types of searches -->
         <p>We searched <a href="/dbdoc/{{dataset.fullname}}">{{ dataset.fullname }}</a>
            to find all records that match
                 {% for word in search_terms %}
@@ -49,6 +50,8 @@
           <input type="text" id="select_top" class="form-control" style="width: 200px; display: inline;" placeholder="Select Top ...">
           <br />
           <br />
+<!--
+          Removing this until more options are added and work correctly
           {% if dataset.type == 'ProbeSet' %}
           <button class="btn btn-default" id="open_options">Open Extra Options</button>
           <br />
@@ -59,6 +62,7 @@
           </div>
           <br />
           <br />
+-->
           {% endif %}
           <div id="table_container">
             <table class="table table-hover table-striped" id='trait_table' {% if dataset.type == 'Geno' %}width="400px"{% endif %} style="float: left;">