From d723eb260bfd9c707f6e282037c4e42670bdf48e Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 16 Jun 2021 21:15:27 +0000
Subject: Changed views.py to call the correlation code using the GN3 API + a
 function in show_corr_results for setting other template vars (might change
 this later)

---
 wqflask/wqflask/views.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index b9181368..96f228af 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -65,7 +65,7 @@ from wqflask.comparison_bar_chart import comparison_bar_chart
 from wqflask.marker_regression import run_mapping
 from wqflask.marker_regression import display_mapping_results
 from wqflask.network_graph import network_graph
-from wqflask.correlation import show_corr_results
+from wqflask.correlation.show_corr_results import set_other_template_vars
 from wqflask.correlation.correlation_gn3_api import compute_correlation
 from wqflask.correlation_matrix import show_corr_matrix
 from wqflask.correlation import corr_scatter_plot
@@ -1082,15 +1082,9 @@ def network_graph_page():
 
 @app.route("/corr_compute", methods=('POST',))
 def corr_compute_page():
-    logger.info("In corr_compute, request.form is:", pf(request.form))
-    logger.info(request.url)
-    template_vars = show_corr_results.CorrelationResults(request.form)
-    return render_template("correlation_page.html", **template_vars.__dict__)
-
-    # to test/disable the new  correlation api uncomment these lines
-
-    # correlation_results = compute_correlation(request.form)
-    # return render_template("test_correlation_page.html", correlation_results=correlation_results)
+    correlation_results = compute_correlation(request.form, compute_all=True)
+    correlation_results = set_other_template_vars(request.form, correlation_results)
+    return render_template("correlation_page.html", **correlation_results)
 
 
 @app.route("/test_corr_compute", methods=["POST"])
-- 
cgit v1.2.3


From c327905af1ef1a0c66a453127ee8f5ba4a824573 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 16 Jun 2021 21:16:09 +0000
Subject: Changed correlation page template to account for differences in the
 output returned from the GN3 API correlation code

---
 wqflask/wqflask/templates/correlation_page.html | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html
index 4cad2749..f66eb4bd 100644
--- a/wqflask/wqflask/templates/correlation_page.html
+++ b/wqflask/wqflask/templates/correlation_page.html
@@ -17,9 +17,9 @@
             <hr style="height: 1px; background-color: #A9A9A9;">
         </div>
         <div style="max-width: 100%;">
-          <p>Values of record {{ this_trait.name }} in the <a href="http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&{% if dataset.accession_id != 'None' %}GN_AccessionId={{ dataset.accession_id }}{% else %}InfoPageName={{ dataset.name }}{% endif %}">{{ dataset.fullname }}</a>
+          <p>Values of record {{ this_trait.name }} in the <a href="http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&{% if this_dataset.accession_id != 'None' %}GN_AccessionId={{ this_dataset.accession_id }}{% else %}InfoPageName={{ this_dataset.name }}{% endif %}">{{ this_dataset.fullname }}</a>
           dataset were compared to all records in the <a href="http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&{% if target_dataset.accession_id != 'None' %}GN_AccessionId={{ target_dataset.accession_id }}{% else %}InfoPageName={{ target_dataset.name }}{% endif %}">{{ target_dataset.fullname }}</a>
-          dataset. The top {{ return_number }} correlations ranked by the {{ formatted_corr_type }} are displayed.
+          dataset. The top {{ return_results }} correlations ranked by the {{ formatted_corr_type }} are displayed.
           You can resort this list by clicking the headers. Select the Record ID to open the trait data
           and analysis page.
           </p>
@@ -30,7 +30,7 @@
             <input type="hidden" name="form_url" value="" />
             <input type="hidden" name="trait_list" id="trait_list" value= "
             {% for this_trait in trait_list %}
-                {{ this_trait.name }}:{{ this_trait.dataset }},
+                {{ this_trait }}:{{ this_dataset.name }},
             {% endfor %}" >
             {% include 'tool_buttons.html' %}
           </form>
@@ -43,7 +43,7 @@
                 <button class="btn btn-success" id="add" type="button" disabled><span class="glyphicon glyphicon-plus-sign"></span> Add</button>
                 <input type="hidden" name="database_name" id="database_name" value="None">
                 <input type="hidden" name="export_data" id="export_data" value="">
-                <input type="hidden" name="file_name" id="file_name" value="{{ this_trait.name }}_{{ dataset.name }}_correlation">
+                <input type="hidden" name="file_name" id="file_name" value="{{ this_trait.name }}_{{ this_dataset.name }}_correlation">
                 <input type="text" id="searchbox" class="form-control" style="width: 200px; display: inline;" placeholder="Search Table For ...">
                 <input type="text" id="select_top" class="form-control" style="width: 200px; display: inline;" placeholder="Select Top ...">
                 <button class="btn btn-default" id="deselect_all" type="button"><span class="glyphicon glyphicon-remove"></span> Deselect</button>
@@ -146,7 +146,7 @@
     
 
     <script type="text/javascript" charset="utf-8">
-        var table_json = {{ json_results | safe }}
+        var table_json = {{ table_json | safe }}
     </script>
 
     <script type="text/javascript" charset="utf-8">
@@ -313,7 +313,7 @@
                       'orderSequence': [ "desc", "asc"],
                       'render': function(data, type, row, meta) {
                         if (data.sample_r != "N/A") {
-                          return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if dataset.name == 'Temp' %}Temp_{{ dataset.group.name }}{% else %}{{ dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>"
+                          return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if this_dataset.name == 'Temp' %}Temp_{{ this_dataset.group }}{% else %}{{ this_dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>"
                         } else {
                           return data.sample_r
                         }
@@ -441,7 +441,7 @@
                       'orderSequence': [ "desc", "asc"],
                       'render': function(data, type, row, meta) {
                         if (data.sample_r != "N/A") {
-                          return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if dataset.name == 'Temp' %}Temp_{{ dataset.group.name }}{% else %}{{ dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>"
+                          return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if this_dataset.name== 'Temp' %}Temp_{{ this_dataset.group }}{% else %}{{ this_dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>"
                         } else {
                           return data.sample_r
                         }
@@ -495,7 +495,7 @@
                       'orderSequence': [ "desc", "asc"],
                       'render': function(data, type, row, meta) {
                         if (data.sample_r != "N/A") {
-                          return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if dataset.name == 'Temp' %}Temp_{{ dataset.group.name }}{% else %}{{ dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>"
+                          return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if this_dataset.name == 'Temp' %}Temp_{{ this_dataset.group }}{% else %}{{ this_dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>"
                         } else {
                           return data.sample_r
                         }
-- 
cgit v1.2.3


From f2e035bb4ff5a1dd5b465ae694105b1a7de956c8 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 16 Jun 2021 21:54:45 +0000
Subject: Added compute_all as parameter to compute_correlation and keyword
 arguments to docstring

---
 wqflask/wqflask/correlation/correlation_gn3_api.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index 6974dbd5..df0afba6 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -131,8 +131,14 @@ def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset):
     return (this_trait_data, results)
 
 
-def compute_correlation(start_vars, method="pearson"):
-    """compute correlation for to call gn3  api"""
+def compute_correlation(start_vars, method="pearson", compute_all=False):
+    """Compute correlations using GN3 API
+
+    Keyword arguments:
+    start_vars -- All input from form; includes things like the trait/dataset names
+    method -- Correlation method to be used (pearson, spearman, or bicor)
+    compute_all -- Include sample, tissue, and literature correlations (when applicable)
+    """
     # pylint: disable-msg=too-many-locals
 
     corr_type = start_vars['corr_type']
@@ -186,8 +192,6 @@ def compute_correlation(start_vars, method="pearson"):
 
     correlation_results = correlation_results[0:corr_return_results]
 
-    compute_all = True  # later to  be passed as argument
-
     if (compute_all):
 
         correlation_results = compute_corr_for_top_results(correlation_results,
-- 
cgit v1.2.3


From 6e1b4d1c53b96ed0bb6335deebd888c24b28d366 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 16 Jun 2021 21:57:03 +0000
Subject: Rewrote show_corr_results.py to remove all code calculating
 correlations (that was moved to correlation_gn3_api.py, which will probably
 be renamed at some point) and only include the code generating the table JSON
 and some template variables

---
 wqflask/wqflask/correlation/show_corr_results.py | 588 ++++++-----------------
 1 file changed, 139 insertions(+), 449 deletions(-)

diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 2f3df67a..6c6d8f4e 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -18,474 +18,145 @@
 #
 # This module is used by GeneNetwork project (www.genenetwork.org)
 
-import collections
 import json
-import scipy
-import numpy
-# import rpy2.robjects as ro                    # R Objects
-import utility.logger
-import utility.webqtlUtil
 
-from base.trait import create_trait
+from base.trait import create_trait, jsonable
+from base.data_set import create_dataset
 
-from base import data_set
-from utility import helper_functions
-from utility import corr_result_helpers
 from utility import hmac
 
-from wqflask.correlation import correlation_functions
-from utility.benchmark import Bench
-
-from utility.type_checking import is_str
-from utility.type_checking import get_float
-from utility.type_checking import get_int
-from utility.type_checking import get_string
-from utility.db_tools import escape
-
-from flask import g
-
-logger = utility.logger.getLogger(__name__)
-
-METHOD_LIT = "3"
-METHOD_TISSUE_PEARSON = "4"
-METHOD_TISSUE_RANK = "5"
-
-TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK]
-
-TISSUE_MOUSE_DB = 1
-
-
-class CorrelationResults:
-    def __init__(self, start_vars):
-        # get trait list from db (database name)
-        # calculate correlation with Base vector and targets
-
-        # Check parameters
-        assert('corr_type' in start_vars)
-        assert(is_str(start_vars['corr_type']))
-        assert('dataset' in start_vars)
-        # assert('group' in start_vars) permitted to be empty?
-        assert('corr_sample_method' in start_vars)
-        assert('corr_samples_group' in start_vars)
-        assert('corr_dataset' in start_vars)
-        assert('corr_return_results' in start_vars)
-        if 'loc_chr' in start_vars:
-            assert('min_loc_mb' in start_vars)
-            assert('max_loc_mb' in start_vars)
-
-        with Bench("Doing correlations"):
-            if start_vars['dataset'] == "Temp":
-                self.dataset = data_set.create_dataset(
-                    dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group'])
-                self.trait_id = start_vars['trait_id']
-                self.this_trait = create_trait(dataset=self.dataset,
-                                               name=self.trait_id,
-                                               cellid=None)
-            else:
-                helper_functions.get_species_dataset_trait(self, start_vars)
-
-            corr_samples_group = start_vars['corr_samples_group']
-
-            self.sample_data = {}
-            self.corr_type = start_vars['corr_type']
-            self.corr_method = start_vars['corr_sample_method']
-            self.min_expr = get_float(start_vars, 'min_expr')
-            self.p_range_lower = get_float(start_vars, 'p_range_lower', -1.0)
-            self.p_range_upper = get_float(start_vars, 'p_range_upper', 1.0)
-
-
-            if ('loc_chr' in start_vars
-                and 'min_loc_mb' in start_vars
-                    and 'max_loc_mb' in start_vars):
-
-                self.location_type = get_string(start_vars, 'location_type')
-                self.location_chr = get_string(start_vars, 'loc_chr')
-                self.min_location_mb = get_int(start_vars, 'min_loc_mb')
-                self.max_location_mb = get_int(start_vars, 'max_loc_mb')
-            else:
-                self.location_type = self.location_chr = self.min_location_mb = self.max_location_mb = None
-
-            self.get_formatted_corr_type()
-            self.return_number = int(start_vars['corr_return_results'])
-
-            # The two if statements below append samples to the sample list based upon whether the user
-            # rselected Primary Samples Only, Other Samples Only, or All Samples
-
-            primary_samples = self.dataset.group.samplelist
-            if self.dataset.group.parlist != None:
-                primary_samples += self.dataset.group.parlist
-            if self.dataset.group.f1list != None:
-                primary_samples += self.dataset.group.f1list
-
-            # If either BXD/whatever Only or All Samples, append all of that group's samplelist
-            if corr_samples_group != 'samples_other':
-                self.process_samples(start_vars, primary_samples)
-
-            # If either Non-BXD/whatever or All Samples, get all samples from this_trait.data and
-            # exclude the primary samples (because they would have been added in the previous
-            # if statement if the user selected All Samples)
-            if corr_samples_group != 'samples_primary':
-                if corr_samples_group == 'samples_other':
-                    primary_samples = [x for x in primary_samples if x not in (
-                        self.dataset.group.parlist + self.dataset.group.f1list)]
-                self.process_samples(start_vars, list(
-                    self.this_trait.data.keys()), primary_samples)
-
-            self.target_dataset = data_set.create_dataset(
-                start_vars['corr_dataset'])
-            self.target_dataset.get_trait_data(list(self.sample_data.keys()))
-
-            self.header_fields = get_header_fields(
-                self.target_dataset.type, self.corr_method)
-
-            if self.target_dataset.type == "ProbeSet":
-                self.filter_cols = [7, 6]
-            elif self.target_dataset.type == "Publish":
-                self.filter_cols = [6, 0]
-            else:
-                self.filter_cols = [4, 0]
-
-            self.correlation_results = []
+def set_template_vars(start_vars, correlation_data):
+    corr_type = start_vars['corr_type']
+    corr_method = start_vars['corr_sample_method']
 
-            self.correlation_data = {}
+    this_dataset_ob = create_dataset(dataset_name=start_vars['dataset'])
+    this_trait = create_trait(dataset=this_dataset_ob,
+                              name=start_vars['trait_id'])
 
-            if self.corr_type == "tissue":
-                self.trait_symbol_dict = self.dataset.retrieve_genes("Symbol")
+    correlation_data['this_trait'] = jsonable(this_trait, this_dataset_ob)
+    correlation_data['this_dataset'] = this_dataset_ob.as_dict()
 
-                tissue_corr_data = self.do_tissue_correlation_for_all_traits()
-                if tissue_corr_data != None:
-                    for trait in list(tissue_corr_data.keys())[:self.return_number]:
-                        self.get_sample_r_and_p_values(
-                            trait, self.target_dataset.trait_data[trait])
-                else:
-                    for trait, values in list(self.target_dataset.trait_data.items()):
-                        self.get_sample_r_and_p_values(trait, values)
-
-            elif self.corr_type == "lit":
-                self.trait_geneid_dict = self.dataset.retrieve_genes("GeneId")
-                lit_corr_data = self.do_lit_correlation_for_all_traits()
-
-                for trait in list(lit_corr_data.keys())[:self.return_number]:
-                    self.get_sample_r_and_p_values(
-                        trait, self.target_dataset.trait_data[trait])
-
-            elif self.corr_type == "sample":
-                for trait, values in list(self.target_dataset.trait_data.items()):
-                    self.get_sample_r_and_p_values(trait, values)
-
-            self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()),
-                                                                   key=lambda t: -abs(t[1][0])))
-
-            # ZS: Convert min/max chromosome to an int for the location range option
-            range_chr_as_int = None
-            for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()):
-                if 'loc_chr' in start_vars:
-                    if chr_info.name == self.location_chr:
-                        range_chr_as_int = order_id
-
-            for _trait_counter, trait in enumerate(list(self.correlation_data.keys())[:self.return_number]):
-                trait_object = create_trait(
-                    dataset=self.target_dataset, name=trait, get_qtl_info=True, get_sample_info=False)
-                if not trait_object:
-                    continue
-
-                chr_as_int = 0
-                for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()):
-                    if self.location_type == "highest_lod":
-                        if chr_info.name == trait_object.locus_chr:
-                            chr_as_int = order_id
-                    else:
-                        if chr_info.name == trait_object.chr:
-                            chr_as_int = order_id
-
-
-                if (float(self.correlation_data[trait][0]) >= self.p_range_lower
-                        and float(self.correlation_data[trait][0]) <= self.p_range_upper):
-
-                    if (self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Publish") and bool(trait_object.mean):
-                        if (self.min_expr != None) and (float(trait_object.mean) < self.min_expr):
-                            continue
-
-                    if range_chr_as_int != None and (chr_as_int != range_chr_as_int):
-                        continue
-                    if self.location_type == "highest_lod":
-                        if (self.min_location_mb != None) and (float(trait_object.locus_mb) < float(self.min_location_mb)):
-                            continue
-                        if (self.max_location_mb != None) and (float(trait_object.locus_mb) > float(self.max_location_mb)):
-                            continue
-                    else:
-                        if (self.min_location_mb != None) and (float(trait_object.mb) < float(self.min_location_mb)):
-                            continue
-                        if (self.max_location_mb != None) and (float(trait_object.mb) > float(self.max_location_mb)):
-                            continue
-
-                    (trait_object.sample_r,
-                     trait_object.sample_p,
-                     trait_object.num_overlap) = self.correlation_data[trait]
-
-                    # Set some sane defaults
-                    trait_object.tissue_corr = 0
-                    trait_object.tissue_pvalue = 0
-                    trait_object.lit_corr = 0
-                    if self.corr_type == "tissue" and tissue_corr_data != None:
-                        trait_object.tissue_corr = tissue_corr_data[trait][1]
-                        trait_object.tissue_pvalue = tissue_corr_data[trait][2]
-                    elif self.corr_type == "lit":
-                        trait_object.lit_corr = lit_corr_data[trait][1]
-
-                    self.correlation_results.append(trait_object)
-
-            if self.corr_type != "lit" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet":
-                self.do_lit_correlation_for_trait_list()
-
-            if self.corr_type != "tissue" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet":
-                self.do_tissue_correlation_for_trait_list()
-
-        self.json_results = generate_corr_json(
-            self.correlation_results, self.this_trait, self.dataset, self.target_dataset)
-
-############################################################################################################################################
-
-    def get_formatted_corr_type(self):
-        self.formatted_corr_type = ""
-        if self.corr_type == "lit":
-            self.formatted_corr_type += "Literature Correlation "
-        elif self.corr_type == "tissue":
-            self.formatted_corr_type += "Tissue Correlation "
-        elif self.corr_type == "sample":
-            self.formatted_corr_type += "Genetic Correlation "
-
-        if self.corr_method == "pearson":
-            self.formatted_corr_type += "(Pearson's r)"
-        elif self.corr_method == "spearman":
-            self.formatted_corr_type += "(Spearman's rho)"
-        elif self.corr_method == "bicor":
-            self.formatted_corr_type += "(Biweight r)"
-
-    def do_tissue_correlation_for_trait_list(self, tissue_dataset_id=1):
-        """Given a list of correlation results (self.correlation_results), gets the tissue correlation value for each"""
-
-        # Gets tissue expression values for the primary trait
-        primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(
-            symbol_list=[self.this_trait.symbol])
-
-        if self.this_trait.symbol.lower() in primary_trait_tissue_vals_dict:
-            primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower(
-            )]
-            gene_symbol_list = [
-                trait.symbol for trait in self.correlation_results if trait.symbol]
-
-            corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(
-                symbol_list=gene_symbol_list)
-
-            for trait in self.correlation_results:
-                if trait.symbol and trait.symbol.lower() in corr_result_tissue_vals_dict:
-                    this_trait_tissue_values = corr_result_tissue_vals_dict[trait.symbol.lower(
-                    )]
-
-                    result = correlation_functions.cal_zero_order_corr_for_tiss(primary_trait_tissue_values,
-                                                                                this_trait_tissue_values,
-                                                                                self.corr_method)
-
-                    trait.tissue_corr = result[0]
-                    trait.tissue_pvalue = result[2]
-
-    def do_tissue_correlation_for_all_traits(self, tissue_dataset_id=1):
-        # Gets tissue expression values for the primary trait
-        primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(
-            symbol_list=[self.this_trait.symbol])
-
-        if self.this_trait.symbol.lower() in primary_trait_tissue_vals_dict:
-            primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower(
-            )]
-
-            #print("trait_gene_symbols: ", pf(trait_gene_symbols.values()))
-            corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(
-                symbol_list=list(self.trait_symbol_dict.values()))
-
-            #print("corr_result_tissue_vals: ", pf(corr_result_tissue_vals_dict))
-
-            #print("trait_gene_symbols: ", pf(trait_gene_symbols))
-
-            tissue_corr_data = {}
-            for trait, symbol in list(self.trait_symbol_dict.items()):
-                if symbol and symbol.lower() in corr_result_tissue_vals_dict:
-                    this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower(
-                    )]
+    target_dataset_ob = create_dataset(correlation_data['target_dataset'])
+    correlation_data['target_dataset'] = target_dataset_ob.as_dict()
 
-                    result = correlation_functions.cal_zero_order_corr_for_tiss(primary_trait_tissue_values,
-                                                                                this_trait_tissue_values,
-                                                                                self.corr_method)
+    table_json = correlation_json_for_table(correlation_data,
+                                            correlation_data['this_trait'],
+                                            correlation_data['this_dataset'],
+                                            target_dataset_ob)
 
-                    tissue_corr_data[trait] = [symbol, result[0], result[2]]
+    correlation_data['table_json'] = table_json
 
-            tissue_corr_data = collections.OrderedDict(sorted(list(tissue_corr_data.items()),
-                                                              key=lambda t: -abs(t[1][1])))
+    if target_dataset_ob.type == "ProbeSet":
+        filter_cols = [7, 6]
+    elif target_dataset_ob.type == "Publish":
+        filter_cols = [6, 0]
+    else:
+        filter_cols = [4, 0]
 
-            return tissue_corr_data
+    correlation_data['corr_method'] = corr_method
+    correlation_data['filter_cols'] = filter_cols
+    correlation_data['header_fields'] = get_header_fields(target_dataset_ob.type, correlation_data['corr_method'])
+    correlation_data['formatted_corr_type'] = get_formatted_corr_type(corr_type, corr_method)
 
-    def do_lit_correlation_for_trait_list(self):
+    return correlation_data
 
-        input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(
-            self.dataset.group.species.lower(), self.this_trait.geneid)
+def correlation_json_for_table(correlation_data, this_trait, this_dataset, target_dataset_ob):
+    """Return JSON data for use with the DataTable in the correlation result page
 
-        for trait in self.correlation_results:
+    Keyword arguments:
+    correlation_data -- Correlation results
+    this_trait -- Trait being correlated against a dataset, as a dict
+    this_dataset -- Dataset of this_trait, as a dict
+    target_dataset_ob - Target dataset, as a Dataset ob
+    """
+    this_trait = correlation_data['this_trait']
+    this_dataset = correlation_data['this_dataset']
+    target_dataset = target_dataset_ob.as_dict()
 
-            if trait.geneid:
-                trait.mouse_gene_id = self.convert_to_mouse_gene_id(
-                    self.dataset.group.species.lower(), trait.geneid)
-            else:
-                trait.mouse_gene_id = None
-
-            if trait.mouse_gene_id and str(trait.mouse_gene_id).find(";") == -1:
-                result = g.db.execute(
-                    """SELECT value
-                       FROM LCorrRamin3
-                       WHERE GeneId1='%s' and
-                             GeneId2='%s'
-                    """ % (escape(str(trait.mouse_gene_id)), escape(str(input_trait_mouse_gene_id)))
-                ).fetchone()
-                if not result:
-                    result = g.db.execute("""SELECT value
-                       FROM LCorrRamin3
-                       WHERE GeneId2='%s' and
-                             GeneId1='%s'
-                    """ % (escape(str(trait.mouse_gene_id)), escape(str(input_trait_mouse_gene_id)))
-                    ).fetchone()
-
-                if result:
-                    lit_corr = result.value
-                    trait.lit_corr = lit_corr
-                else:
-                    trait.lit_corr = 0
-            else:
-                trait.lit_corr = 0
-
-    def do_lit_correlation_for_all_traits(self):
-        input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(
-            self.dataset.group.species.lower(), self.this_trait.geneid)
-
-        lit_corr_data = {}
-        for trait, gene_id in list(self.trait_geneid_dict.items()):
-            mouse_gene_id = self.convert_to_mouse_gene_id(
-                self.dataset.group.species.lower(), gene_id)
-
-            if mouse_gene_id and str(mouse_gene_id).find(";") == -1:
-                #print("gene_symbols:", input_trait_mouse_gene_id + " / " + mouse_gene_id)
-                result = g.db.execute(
-                    """SELECT value
-                       FROM LCorrRamin3
-                       WHERE GeneId1='%s' and
-                             GeneId2='%s'
-                    """ % (escape(mouse_gene_id), escape(input_trait_mouse_gene_id))
-                ).fetchone()
-                if not result:
-                    result = g.db.execute("""SELECT value
-                       FROM LCorrRamin3
-                       WHERE GeneId2='%s' and
-                             GeneId1='%s'
-                    """ % (escape(mouse_gene_id), escape(input_trait_mouse_gene_id))
-                    ).fetchone()
-                if result:
-                    #print("result:", result)
-                    lit_corr = result.value
-                    lit_corr_data[trait] = [gene_id, lit_corr]
+    corr_results = correlation_data['correlation_results']
+    results_list = []
+    for i, trait_dict in enumerate(corr_results):
+        trait_name = list(trait_dict.keys())[0]
+        trait = trait_dict[trait_name]
+        target_trait_ob = create_trait(dataset=target_dataset_ob,
+                                       name=trait_name,
+                                       get_qtl_info=True)
+        target_trait = jsonable(target_trait_ob, target_dataset_ob)
+        if target_trait['view'] == False:
+            continue
+        results_dict = {}
+        results_dict['index'] = i + 1
+        results_dict['trait_id'] = target_trait['name']
+        results_dict['dataset'] = target_dataset['name']
+        results_dict['hmac'] = hmac.data_hmac(
+            '{}:{}'.format(target_trait['name'], target_dataset['name']))
+        results_dict['sample_r'] = f"{float(trait['corr_coeffient']):.3f}"
+        results_dict['num_overlap'] = trait['num_overlap']
+        results_dict['sample_p'] = f"{float(trait['p_value']):.3e}"
+        if target_dataset['type'] == "ProbeSet":
+            results_dict['symbol'] = target_trait['symbol']
+            results_dict['description'] = "N/A"
+            results_dict['location'] = target_trait['location']
+            results_dict['mean'] = "N/A"
+            results_dict['additive'] = "N/A"
+            if bool(target_trait['description']):
+                results_dict['description'] = target_trait['description']
+            if bool(target_trait['mean']):
+                results_dict['mean'] = f"{float(target_trait['mean']):.3f}"
+            try:
+                results_dict['lod_score'] = f"{float(target_trait['lrs_score']) / 4.61:.1f}"
+            except:
+                results_dict['lod_score'] = "N/A"
+            results_dict['lrs_location'] = target_trait['lrs_location']
+            if bool(target_trait['additive']):
+                results_dict['additive'] = f"{float(target_trait['additive']):.3f}"
+            results_dict['lit_corr'] = "--"
+            results_dict['tissue_corr'] = "--"
+            results_dict['tissue_pvalue'] = "--"
+            if this_dataset['type'] == "ProbeSet":
+                if 'lit_corr' in trait:
+                    results_dict['lit_corr'] = f"{float(trait['lit_corr']):.3f}"
+                if 'tissue_corr' in trait:
+                    results_dict['tissue_corr'] = f"{float(trait['tissue_corr']):.3f}"
+                    results_dict['tissue_pvalue'] = f"{float(trait['tissue_p_val']):.3e}"
+        elif target_dataset['type'] == "Publish":
+            results_dict['abbreviation_display'] = "N/A"
+            results_dict['description'] = "N/A"
+            results_dict['mean'] = "N/A"
+            results_dict['authors_display'] = "N/A"
+            results_dict['additive'] = "N/A"
+            results_dict['pubmed_link'] = "N/A"
+            results_dict['pubmed_text'] = "N/A"
+
+            if bool(target_trait['abbreviation']):
+                results_dict['abbreviation_display'] = target_trait['abbreviation']
+            if bool(target_trait['description']):
+                results_dict['description'] = target_trait['description']
+            if bool(target_trait['mean']):
+                results_dict['mean'] = f"{float(target_trait['mean']):.3f}"
+            if bool(target_trait['authors']):
+                authors_list = target_trait['authors'].split(',')
+                if len(authors_list) > 6:
+                    results_dict['authors_display'] = ", ".join(
+                        authors_list[:6]) + ", et al."
                 else:
-                    lit_corr_data[trait] = [gene_id, 0]
-            else:
-                lit_corr_data[trait] = [gene_id, 0]
-
-        lit_corr_data = collections.OrderedDict(sorted(list(lit_corr_data.items()),
-                                                       key=lambda t: -abs(t[1][1])))
-
-        return lit_corr_data
-
-    def convert_to_mouse_gene_id(self, species=None, gene_id=None):
-        """If the species is rat or human, translate the gene_id to the mouse geneid
-
-        If there is no input gene_id or there's no corresponding mouse gene_id, return None
-
-        """
-        if not gene_id:
-            return None
-
-        mouse_gene_id = None
-
-        if species == 'mouse':
-            mouse_gene_id = gene_id
-
-        elif species == 'rat':
-
-            query = """SELECT mouse
-                   FROM GeneIDXRef
-                   WHERE rat='%s'""" % escape(gene_id)
-
-            result = g.db.execute(query).fetchone()
-            if result != None:
-                mouse_gene_id = result.mouse
-
-        elif species == 'human':
-
-            query = """SELECT mouse
-                   FROM GeneIDXRef
-                   WHERE human='%s'""" % escape(gene_id)
-
-            result = g.db.execute(query).fetchone()
-            if result != None:
-                mouse_gene_id = result.mouse
-
-        return mouse_gene_id
-
-    def get_sample_r_and_p_values(self, trait, target_samples):
-        """Calculates the sample r (or rho) and p-value
-
-        Given a primary trait and a target trait's sample values,
-        calculates either the pearson r or spearman rho and the p-value
-        using the corresponding scipy functions.
-
-        """
-
-        self.this_trait_vals = []
-        target_vals = []
-        for index, sample in enumerate(self.target_dataset.samplelist):
-            if sample in self.sample_data:
-                sample_value = self.sample_data[sample]
-                target_sample_value = target_samples[index]
-                self.this_trait_vals.append(sample_value)
-                target_vals.append(target_sample_value)
-
-        self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(
-            self.this_trait_vals, target_vals)
-
-        if num_overlap > 5:
-            # ZS: 2015 could add biweight correlation, see http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3465711/
-            # if self.corr_method == 'bicor':
-            #     sample_r, sample_p = do_bicor(
-            #         self.this_trait_vals, target_vals)
-            if self.corr_method == 'pearson':
-                sample_r, sample_p = scipy.stats.pearsonr(
-                    self.this_trait_vals, target_vals)
-            else:
-                sample_r, sample_p = scipy.stats.spearmanr(
-                    self.this_trait_vals, target_vals)
-
-            if numpy.isnan(sample_r):
-                pass
-            else:
-                self.correlation_data[trait] = [
-                    sample_r, sample_p, num_overlap]
-
-    def process_samples(self, start_vars, sample_names, excluded_samples=None):
-        if not excluded_samples:
-            excluded_samples = ()
+                    results_dict['authors_display'] = target_trait['authors']
+            if 'pubmed_id' in target_trait:
+                results_dict['pubmed_link'] = target_trait['pubmed_link']
+                results_dict['pubmed_text'] = target_trait['pubmed_text']
+            try:
+                results_dict['lod_score'] = f"{float(target_trait['lrs_score']) / 4.61:.1f}"
+            except:
+                results_dict['lod_score'] = "N/A"
+            results_dict['lrs_location'] = target_trait['lrs_location']
+            if bool(target_trait['additive']):
+                results_dict['additive'] = f"{float(target_trait['additive']):.3f}"
+        else:
+            results_dict['location'] = target_trait['lrs_location']
 
-        sample_val_dict = json.loads(start_vars['sample_vals'])
-        for sample in sample_names:
-            if sample not in excluded_samples:
-                value = sample_val_dict[sample]
-                if not value.strip().lower() == 'x':
-                    self.sample_data[str(sample)] = float(value)
+        results_list.append(results_dict)
 
+    return json.dumps(results_list)
 
 # def do_bicor(this_trait_vals, target_trait_vals):
 #     r_library = ro.r["library"]             # Map the library function
@@ -598,6 +269,25 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap
     return json.dumps(results_list)
 
 
+def get_formatted_corr_type(corr_type, corr_method):
+    formatted_corr_type = ""
+    if corr_type == "lit":
+        formatted_corr_type += "Literature Correlation "
+    elif corr_type == "tissue":
+        formatted_corr_type += "Tissue Correlation "
+    elif corr_type == "sample":
+        formatted_corr_type += "Genetic Correlation "
+
+    if corr_method == "pearson":
+        formatted_corr_type += "(Pearson's r)"
+    elif corr_method == "spearman":
+        formatted_corr_type += "(Spearman's rho)"
+    elif corr_method == "bicor":
+        formatted_corr_type += "(Biweight r)"
+
+    return formatted_corr_type
+
+
 def get_header_fields(data_type, corr_method):
     if data_type == "ProbeSet":
         if corr_method == "spearman":
-- 
cgit v1.2.3


From f5a7a54741cef7285e8862ceb2493fe3f451fb84 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 16 Jun 2021 21:57:23 +0000
Subject: Changed set_other_template_vars to set_template_vars

---
 wqflask/wqflask/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 96f228af..da427bfe 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -65,7 +65,7 @@ from wqflask.comparison_bar_chart import comparison_bar_chart
 from wqflask.marker_regression import run_mapping
 from wqflask.marker_regression import display_mapping_results
 from wqflask.network_graph import network_graph
-from wqflask.correlation.show_corr_results import set_other_template_vars
+from wqflask.correlation.show_corr_results import set_template_vars
 from wqflask.correlation.correlation_gn3_api import compute_correlation
 from wqflask.correlation_matrix import show_corr_matrix
 from wqflask.correlation import corr_scatter_plot
@@ -1083,7 +1083,7 @@ def network_graph_page():
 @app.route("/corr_compute", methods=('POST',))
 def corr_compute_page():
     correlation_results = compute_correlation(request.form, compute_all=True)
-    correlation_results = set_other_template_vars(request.form, correlation_results)
+    correlation_results = set_template_vars(request.form, correlation_results)
     return render_template("correlation_page.html", **correlation_results)
 
 
-- 
cgit v1.2.3


From 4f06f22071898171cb7bbc12f5dfd42b05dc3208 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 16 Jun 2021 21:58:00 +0000
Subject: Added function as_dict to data_set.py for situations where we don't
 want to pass around the whole dataset object

---
 wqflask/base/data_set.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 75ddf278..dc338971 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -559,6 +559,7 @@ class DataSet:
         self.fullname = None
         self.type = None
         self.data_scale = None  # ZS: For example log2
+        self.accession_id = None
 
         self.setup()
 
@@ -575,6 +576,17 @@ class DataSet:
             self.group.get_samplelist()
         self.species = species.TheSpecies(self)
 
+    def as_dict(self):
+        return {
+            'name': self.name,
+            'shortname': self.shortname,
+            'fullname': self.fullname,
+            'type': self.type,
+            'data_scale': self.data_scale,
+            'group': self.group.name,
+            'accession_id': self.accession_id
+        }
+
     def get_accession_id(self):
         if self.type == "Publish":
             results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where
-- 
cgit v1.2.3


From 73195c1af57cbcf0fa38f1e73e8878c02938ef4b Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 16 Jun 2021 21:58:43 +0000
Subject: Changed the function jsonable in trait.py to return a couple extra
 fields + take dataset as an optional argument (to avoid having to pointlessly
 create it)

---
 wqflask/base/trait.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index d09cfd40..10851e00 100644
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -284,17 +284,19 @@ def get_sample_data():
         return None
 
 
-def jsonable(trait):
+def jsonable(trait, dataset=None):
     """Return a dict suitable for using as json
 
     Actual turning into json doesn't happen here though"""
 
-    dataset = create_dataset(dataset_name=trait.dataset.name,
-                             dataset_type=trait.dataset.type,
-                             group_name=trait.dataset.group.name)
+    if not dataset:
+        dataset = create_dataset(dataset_name=trait.dataset.name,
+                                dataset_type=trait.dataset.type,
+                                group_name=trait.dataset.group.name)
 
     if dataset.type == "ProbeSet":
         return dict(name=trait.name,
+                    view=trait.view,
                     symbol=trait.symbol,
                     dataset=dataset.name,
                     dataset_name=dataset.shortname,
@@ -308,37 +310,44 @@ def jsonable(trait):
     elif dataset.type == "Publish":
         if trait.pubmed_id:
             return dict(name=trait.name,
+                        view=trait.view,
                         dataset=dataset.name,
                         dataset_name=dataset.shortname,
                         description=trait.description_display,
                         abbreviation=trait.abbreviation,
                         authors=trait.authors,
+                        pubmed_id=trait.pubmed_id,
                         pubmed_text=trait.pubmed_text,
                         pubmed_link=trait.pubmed_link,
+                        mean=trait.mean,
                         lrs_score=trait.LRS_score_repr,
                         lrs_location=trait.LRS_location_repr,
                         additive=trait.additive
                         )
         else:
             return dict(name=trait.name,
+                        view=trait.view,
                         dataset=dataset.name,
                         dataset_name=dataset.shortname,
                         description=trait.description_display,
                         abbreviation=trait.abbreviation,
                         authors=trait.authors,
                         pubmed_text=trait.pubmed_text,
+                        mean=trait.mean,
                         lrs_score=trait.LRS_score_repr,
                         lrs_location=trait.LRS_location_repr,
                         additive=trait.additive
                         )
     elif dataset.type == "Geno":
         return dict(name=trait.name,
+                    view=trait.view,
                     dataset=dataset.name,
                     dataset_name=dataset.shortname,
                     location=trait.location_repr
                     )
     elif dataset.name == "Temp":
         return dict(name=trait.name,
+                    view=trait.view,
                     dataset="Temp",
                     dataset_name="Temp")
     else:
-- 
cgit v1.2.3


From 55bccdd5cf8cdac52a625228b31732b5529810b5 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 16 Jun 2021 22:04:41 +0000
Subject: Fixed reference to target_trait['lrs_location'] that should have been
 target_trait['location'] for correlations against genotypes

---
 wqflask/wqflask/correlation/show_corr_results.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 6c6d8f4e..91716a6a 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -152,7 +152,7 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe
             if bool(target_trait['additive']):
                 results_dict['additive'] = f"{float(target_trait['additive']):.3f}"
         else:
-            results_dict['location'] = target_trait['lrs_location']
+            results_dict['location'] = target_trait['location']
 
         results_list.append(results_dict)
 
-- 
cgit v1.2.3


From c048491ea0ef863b5553511588a53e2ebc330617 Mon Sep 17 00:00:00 2001
From: BonfaceKilz
Date: Thu, 17 Jun 2021 08:52:17 +0300
Subject: doc: Add results of investing rpy2 blocking issue in gn2

---
 doc/rpy2-performance.org | 170 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 170 insertions(+)
 create mode 100644 doc/rpy2-performance.org

diff --git a/doc/rpy2-performance.org b/doc/rpy2-performance.org
new file mode 100644
index 00000000..29b99ba0
--- /dev/null
+++ b/doc/rpy2-performance.org
@@ -0,0 +1,170 @@
+* Python-Rpy2 performance issues with genenetwork2
+
+At one point, genenetwork2 was down. A possible cause was that it
+wrote into the log file in an infinite loop due to rpy2, so a solution
+was to empty it. Currently, as a work around, rpy2 is disabled by
+removing it's imports. This affects WGCNA/ CTL imports and commenting
+out Biweight Midcorrelation option in the trait page. See:
+
+- [[https://github.com/genenetwork/genenetwork2/commit/1baf5f7611909c651483208184c5fbf7d4a7a088][1baf5f7]]
+- [[https://github.com/genenetwork/genenetwork2/commit/afee4d625248565857df98d3510f680ae6204864][afee4d6]]
+- [[https://github.com/genenetwork/genenetwork2/commit/c458bf0ad731e5e5fd9cbd0686936b3a441bae63][c458bf0]]
+- [[https://github.com/genenetwork/genenetwork2/commit/d31f3f763471b19559ca74e73b52b3cb5e7153ce][d31f3f7]]
+
+** Reproducing the problem
+
+I went back to commit #b8408cea. With regards to logs, I never
+experienced any log issue. Perhaps it's because of how I start my
+server:
+
+: env SERVER_PORT=5004 TMPDIR=/home/bonface/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG GENENETWORK_FILES=/home/bonface/data/genotype_files/ GN2_PROFILE=/home/bonface/opt/python3-genenetwork2 ./scripts/run_debug.sh
+
+However, when loading the homepage, I occasionally ran into this trace:
+
+#+begin_export ascii
+DEBUG:wqflask.views:.check_access_permissions: @app.before_request check_access_permissions
+DEBUG:wqflask.views:.shutdown_session: remove db_session
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: Error: ignoring SIGPIPE signal
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: In addition:
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: Warning messages:
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: 1:
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: In (function (package, help, pos = 2, lib.loc = NULL, character.only = FALSE,  :
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:  library '/home/bonface/R/x86_64-unknown-linux-gnu-library/4.0' contains no packages
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: 2:
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: In (function (package, help, pos = 2, lib.loc = NULL, character.only = FALSE,  :
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:  library '/home/bonface/R/x86_64-unknown-linux-gnu-library/4.0' contains no packages
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: Fatal error: unable to initialize the JIT
+
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
+ *** caught segfault ***
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: address (nil), cause 'memory not mapped'
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
+Possible actions:
+1: abort (with core dump, if enabled)
+2: normal R exit
+3: exit R without saving workspace
+4: exit R saving workspace
+
+Selection:
+
+#+end_export
+
+This blocks the flask service. Seems to be related to: [[https://github.com/rpy2/rpy2/issues/769][rpy2-issue#769]]
+and [[https://github.com/rpy2/rpy2/issues/809][rpy2-issue#809]]. I tried to reproduce this problem using some endpoint:
+
+#+begin_src python
+@app.route("/test")
+  def test():
+      from rpy2 import robjects as ro
+      from rpy2 import rinterface
+      from threading import Thread
+
+      def rpy2_init_simple():
+          rinterface.initr_simple()
+
+      thread = Thread(target=rpy2_init_simple)
+      thread.start()
+      return "This is a test after importing rpy2"
+#+end_src
+
+which generates this trace:
+
+#+begin_export ascii
+/home/bonface/opt/python3-genenetwork2/lib/python3.8/site-packages/rpy2/rinterface.py:955: UserWarning: R is not initialized by the main thread.
+              Its taking over SIGINT cannot be reversed here, and as a
+              consequence the embedded R cannot be interrupted with Ctrl-C.
+              Consider (re)setting the signal handler of your choice from
+              the main thread.
+warnings.warn(
+DEBUG:wqflask.views:.shutdown_session: remove db_session
+
+#+end_export
+
+Modifying the endpoint to:
+
+#+begin_src python
+@app.route("/test")
+  def test():
+      import wqflask.correlation.show_corr_results
+      import wqflask.ctl.ctl_analysis
+      import time
+      from wqflask.correlation.correlation_functions import cal_zero_order_corr_for_tiss
+
+      print("Sleeping for 3 seconds")
+      time.sleep(3)
+      return "This is a test after importing rpy2"
+#+end_src
+
+and refreshing the page a couple of times, I get:
+
+#+begin_export ascii
+DEBUG:wqflask.views:.check_access_permissions: @app.before_request check_access_
+permissions
+Sleeping for 3 seconds
+DEBUG:wqflask.views:.shutdown_session: remove db_session
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: Error: ignoring SIGPI
+PE signal
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: In addition:
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: Warning messages:
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: 1:
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: In (function (package
+, help, pos = 2, lib.loc = NULL, character.only = FALSE,  :
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:  library '/home/bonfa
+ce/R/x86_64-unknown-linux-gnu-library/4.0' contains no packages
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: 2:
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: In (function (package
+, help, pos = 2, lib.loc = NULL, character.only = FALSE,  :
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:  library '/home/bonfa
+ce/R/x86_64-unknown-linux-gnu-library/4.0' contains no packages
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
+*** caught segfault ***
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: address (nil), cause
+'memory not mapped'
+
+WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
+Possible actions:
+1: abort (with core dump, if enabled)
+2: normal R exit
+3: exit R without saving workspace
+4: exit R saving workspace
+
+Selection: [2021-06-16 13:11:00 +0300] [18657] [INFO] Handling signal: winch
+[2021-06-16 13:11:00 +0300] [18657] [INFO] Handling signal: winch
+[2021-06-16 13:13:02 +0300] [18657] [INFO] Handling signal: winch
+#+end_export
+
+However, this seems to be non-deterministic, in the sense that I can't really pin what causes the above. I've tried to write a Locust Test that simulates users hitting that endpoint:
+
+#+begin_src python
+"""Load test a single trait page"""
+from locust import HttpUser, task, between
+
+
+  class LoadTest(HttpUser):
+      wait_time = between(1, 2.5)
+
+      @task
+      def fetch_trait(self):
+          """Fetch a single trait"""
+          self.client.get("/test")
+#+end_src
-- 
cgit v1.2.3


From db9b0a9df8879921217bac353eaa2c3440e8337a Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Mon, 24 May 2021 16:35:11 +0300
Subject: add query for fetching probeset data

---
 wqflask/base/data_set.py                           | 258 +++++++++++++--------
 wqflask/wqflask/correlation/correlation_gn3_api.py | 144 +++++++++++-
 2 files changed, 303 insertions(+), 99 deletions(-)

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index dc338971..7531ac81 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -166,7 +166,6 @@ class DatasetType:
         if t in ['pheno', 'other_pheno']:
             group_name = name.replace("Publish", "")
 
-
         results = g.db.execute(sql_query_mapping[t] % group_name).fetchone()
         if results:
             self.datasets[name] = dataset_name_mapping[t]
@@ -278,7 +277,7 @@ class Markers:
             filtered_markers = []
             for marker in self.markers:
                 if marker['name'] in p_values:
-                    #logger.debug("marker {} IS in p_values".format(i))
+                    # logger.debug("marker {} IS in p_values".format(i))
                     marker['p_value'] = p_values[marker['name']]
                     if math.isnan(marker['p_value']) or (marker['p_value'] <= 0):
                         marker['lod_score'] = 0
@@ -299,7 +298,7 @@ class HumanMarkers(Markers):
         self.markers = []
         for line in marker_data_fh:
             splat = line.strip().split()
-            #logger.debug("splat:", splat)
+            # logger.debug("splat:", splat)
             if len(specified_markers) > 0:
                 if splat[1] in specified_markers:
                     marker = {}
@@ -441,7 +440,7 @@ class DatasetGroup:
         # genotype_1 is Dataset Object without parents and f1
         # genotype_2 is Dataset Object with parents and f1 (not for intercross)
 
-        #genotype_1 = reaper.Dataset()
+        # genotype_1 = reaper.Dataset()
 
         # reaper barfs on unicode filenames, so here we ensure it's a string
         if self.genofile:
@@ -662,9 +661,39 @@ class DataSet:
 
 
 
-    def get_trait_data(self, sample_list=None):
+
+    def chunk_dataset(self, dataset, n):
+
+
+        results = {}
+
+        query = """
+                SELECT ProbeSetXRef.DataId,ProbeSet.Name
+                FROM ProbeSet, ProbeSetXRef, ProbeSetFreeze
+                WHERE ProbeSetFreeze.Name = '{}' AND
+                      ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND
+                      ProbeSetXRef.ProbeSetId = ProbeSet.Id
+        """.format(self.name)
+
+        # should cache this
+
+        traits_name_dict= dict(g.db.execute(query).fetchall())
+
+
+
+
+        for i in range(0, len(dataset), n):
+            matrix = list(dataset[i:i + n])
+            trait_name = traits_name_dict[matrix[0][0]]
+
+            my_values = [value for (trait_name, strain, value) in matrix]
+            results[trait_name] = my_values
+        return results
+
+    def get_probeset_data(self, sample_list=None, trait_ids=None):
         if sample_list:
             self.samplelist = sample_list
+
         else:
             self.samplelist = self.group.samplelist
 
@@ -678,27 +707,59 @@ class DataSet:
             and Strain.SpeciesId=Species.Id
             and Species.name = '{}'
             """.format(create_in_clause(self.samplelist), *mescape(self.group.species))
-        logger.sql(query)
         results = dict(g.db.execute(query).fetchall())
         sample_ids = [results[item] for item in self.samplelist]
 
+        query = """SELECT * from ProbeSetData
+                where StrainID in {}
+                and id in (SELECT ProbeSetXRef.DataId
+                FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze)
+                WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id
+                and ProbeSetFreeze.Name = '{}'
+                and ProbeSet.Id = ProbeSetXRef.ProbeSetId)""".format(create_in_clause(sample_ids),self.name)
+
+        query_results=list(g.db.execute(query).fetchall())
+
+        data_results=self.chunk_dataset(query_results, len(sample_ids))
+        self.trait_data=data_results
+
+    def get_trait_data(self, sample_list=None):
+        if sample_list:
+            self.samplelist=sample_list
+        else:
+            self.samplelist=self.group.samplelist
+
+        if self.group.parlist != None and self.group.f1list != None:
+            if (self.group.parlist + self.group.f1list) in self.samplelist:
+                self.samplelist += self.group.parlist + self.group.f1list
+
+        query="""
+            SELECT Strain.Name, Strain.Id FROM Strain, Species
+            WHERE Strain.Name IN {}
+            and Strain.SpeciesId=Species.Id
+            and Species.name = '{}'
+            """.format(create_in_clause(self.samplelist), *mescape(self.group.species))
+        logger.sql(query)
+        results=dict(g.db.execute(query).fetchall())
+        sample_ids=[results[item] for item in self.samplelist]
+
         # MySQL limits the number of tables that can be used in a join to 61,
         # so we break the sample ids into smaller chunks
         # Postgres doesn't have that limit, so we can get rid of this after we transition
-        chunk_size = 50
-        number_chunks = int(math.ceil(len(sample_ids) / chunk_size))
-        trait_sample_data = []
+        chunk_size=50
+        number_chunks=int(math.ceil(len(sample_ids) / chunk_size))
+        trait_sample_data=[]
         for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks):
             if self.type == "Publish":
-                dataset_type = "Phenotype"
+                dataset_type="Phenotype"
             else:
-                dataset_type = self.type
-            temp = ['T%s.value' % item for item in sample_ids_step]
+                dataset_type=self.type
+            temp=['T%s.value' % item for item in sample_ids_step]
             if self.type == "Publish":
-                query = "SELECT {}XRef.Id,".format(escape(self.type))
+                query="SELECT {}XRef.Id,".format(escape(self.type))
             else:
-                query = "SELECT {}.Name,".format(escape(dataset_type))
-            data_start_pos = 1
+                query="SELECT {}.Name,".format(escape(dataset_type))
+            data_start_pos=1
             query += ', '.join(temp)
             query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(dataset_type,
                                                                      self.type,
@@ -727,27 +788,27 @@ class DataSet:
                         """.format(*mescape(self.type, self.type, self.type, self.type,
                                             self.name, dataset_type, self.type, self.type, dataset_type))
 
-            results = g.db.execute(query).fetchall()
+            results=g.db.execute(query).fetchall()
             trait_sample_data.append(results)
 
-        trait_count = len(trait_sample_data[0])
-        self.trait_data = collections.defaultdict(list)
+        trait_count=len(trait_sample_data[0])
+        self.trait_data=collections.defaultdict(list)
 
         # put all of the separate data together into a dictionary where the keys are
         # trait names and values are lists of sample values
         for trait_counter in range(trait_count):
-            trait_name = trait_sample_data[0][trait_counter][0]
+            trait_name=trait_sample_data[0][trait_counter][0]
             for chunk_counter in range(int(number_chunks)):
                 self.trait_data[trait_name] += (
                     trait_sample_data[chunk_counter][trait_counter][data_start_pos:])
 
 
 class PhenotypeDataSet(DataSet):
-    DS_NAME_MAP['Publish'] = 'PhenotypeDataSet'
+    DS_NAME_MAP['Publish']='PhenotypeDataSet'
 
     def setup(self):
         # Fields in the database table
-        self.search_fields = ['Phenotype.Post_publication_description',
+        self.search_fields=['Phenotype.Post_publication_description',
                               'Phenotype.Pre_publication_description',
                               'Phenotype.Pre_publication_abbreviation',
                               'Phenotype.Post_publication_abbreviation',
@@ -760,7 +821,7 @@ class PhenotypeDataSet(DataSet):
                               'PublishXRef.Id']
 
         # Figure out what display_fields is
-        self.display_fields = ['name', 'group_code',
+        self.display_fields=['name', 'group_code',
                                'pubmed_id',
                                'pre_publication_description',
                                'post_publication_description',
@@ -778,7 +839,7 @@ class PhenotypeDataSet(DataSet):
                                'sequence', 'units', 'comments']
 
         # Fields displayed in the search results table header
-        self.header_fields = ['Index',
+        self.header_fields=['Index',
                               'Record',
                               'Description',
                               'Authors',
@@ -787,9 +848,9 @@ class PhenotypeDataSet(DataSet):
                               'Max LRS Location',
                               'Additive Effect']
 
-        self.type = 'Publish'
+        self.type='Publish'
 
-        self.query_for_group = '''
+        self.query_for_group='''
                             SELECT
                                     InbredSet.Name, InbredSet.Id, InbredSet.GeneticType
                             FROM
@@ -809,13 +870,13 @@ class PhenotypeDataSet(DataSet):
             if not this_trait.haveinfo:
                 this_trait.retrieve_info(get_qtl_info=True)
 
-            description = this_trait.post_publication_description
+            description=this_trait.post_publication_description
 
             # If the dataset is confidential and the user has access to confidential
             # phenotype traits, then display the pre-publication description instead
             # of the post-publication description
             if this_trait.confidential:
-                this_trait.description_display = ""
+                this_trait.description_display=""
                 continue   # for now, because no authorization features
 
                 if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(
@@ -823,46 +884,46 @@ class PhenotypeDataSet(DataSet):
                         userName=self.userName,
                         authorized_users=this_trait.authorized_users):
 
-                    description = this_trait.pre_publication_description
+                    description=this_trait.pre_publication_description
 
             if len(description) > 0:
-                this_trait.description_display = description.strip()
+                this_trait.description_display=description.strip()
             else:
-                this_trait.description_display = ""
+                this_trait.description_display=""
 
             if not this_trait.year.isdigit():
-                this_trait.pubmed_text = "N/A"
+                this_trait.pubmed_text="N/A"
             else:
-                this_trait.pubmed_text = this_trait.year
+                this_trait.pubmed_text=this_trait.year
 
             if this_trait.pubmed_id:
-                this_trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % this_trait.pubmed_id
+                this_trait.pubmed_link=webqtlConfig.PUBMEDLINK_URL % this_trait.pubmed_id
 
             # LRS and its location
-            this_trait.LRS_score_repr = "N/A"
-            this_trait.LRS_location_repr = "N/A"
+            this_trait.LRS_score_repr="N/A"
+            this_trait.LRS_location_repr="N/A"
 
             if this_trait.lrs:
-                query = """
+                query="""
                     select Geno.Chr, Geno.Mb from Geno, Species
                     where Species.Name = '%s' and
                         Geno.Name = '%s' and
                         Geno.SpeciesId = Species.Id
                 """ % (species, this_trait.locus)
                 logger.sql(query)
-                result = g.db.execute(query).fetchone()
+                result=g.db.execute(query).fetchone()
 
                 if result:
                     if result[0] and result[1]:
-                        LRS_Chr = result[0]
-                        LRS_Mb = result[1]
+                        LRS_Chr=result[0]
+                        LRS_Mb=result[1]
 
-                        this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs
-                        this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (
+                        this_trait.LRS_score_repr=LRS_score_repr='%3.1f' % this_trait.lrs
+                        this_trait.LRS_location_repr=LRS_location_repr='Chr%s: %.6f' % (
                             LRS_Chr, float(LRS_Mb))
 
     def retrieve_sample_data(self, trait):
-        query = """
+        query="""
                     SELECT
                             Strain.Name, PublishData.value, PublishSE.error, NStrain.count, Strain.Name2
                     FROM
@@ -880,34 +941,34 @@ class PhenotypeDataSet(DataSet):
                             Strain.Name
                     """
         logger.sql(query)
-        results = g.db.execute(query, (trait, self.id)).fetchall()
+        results=g.db.execute(query, (trait, self.id)).fetchall()
         return results
 
 
 class GenotypeDataSet(DataSet):
-    DS_NAME_MAP['Geno'] = 'GenotypeDataSet'
+    DS_NAME_MAP['Geno']='GenotypeDataSet'
 
     def setup(self):
         # Fields in the database table
-        self.search_fields = ['Name',
+        self.search_fields=['Name',
                               'Chr']
 
         # Find out what display_fields is
-        self.display_fields = ['name',
+        self.display_fields=['name',
                                'chr',
                                'mb',
                                'source2',
                                'sequence']
 
         # Fields displayed in the search results table header
-        self.header_fields = ['Index',
+        self.header_fields=['Index',
                               'ID',
                               'Location']
 
         # Todo: Obsolete or rename this field
-        self.type = 'Geno'
+        self.type='Geno'
 
-        self.query_for_group = '''
+        self.query_for_group='''
                 SELECT
                         InbredSet.Name, InbredSet.Id, InbredSet.GeneticType
                 FROM
@@ -926,11 +987,11 @@ class GenotypeDataSet(DataSet):
                 this_trait.retrieveInfo()
 
             if this_trait.chr and this_trait.mb:
-                this_trait.location_repr = 'Chr%s: %.6f' % (
+                this_trait.location_repr='Chr%s: %.6f' % (
                     this_trait.chr, float(this_trait.mb))
 
     def retrieve_sample_data(self, trait):
-        query = """
+        query="""
                     SELECT
                             Strain.Name, GenoData.value, GenoSE.error, "N/A", Strain.Name2
                     FROM
@@ -947,7 +1008,7 @@ class GenotypeDataSet(DataSet):
                             Strain.Name
                     """
         logger.sql(query)
-        results = g.db.execute(query,
+        results=g.db.execute(query,
                                (webqtlDatabaseFunction.retrieve_species_id(self.group.name),
                                 trait, self.name)).fetchall()
         return results
@@ -961,11 +1022,11 @@ class MrnaAssayDataSet(DataSet):
     platform and is far too specific.
 
     '''
-    DS_NAME_MAP['ProbeSet'] = 'MrnaAssayDataSet'
+    DS_NAME_MAP['ProbeSet']='MrnaAssayDataSet'
 
     def setup(self):
         # Fields in the database table
-        self.search_fields = ['Name',
+        self.search_fields=['Name',
                               'Description',
                               'Probe_Target_Description',
                               'Symbol',
@@ -975,7 +1036,7 @@ class MrnaAssayDataSet(DataSet):
                               'RefSeq_TranscriptId']
 
         # Find out what display_fields is
-        self.display_fields = ['name', 'symbol',
+        self.display_fields=['name', 'symbol',
                                'description', 'probe_target_description',
                                'chr', 'mb',
                                'alias', 'geneid',
@@ -995,7 +1056,7 @@ class MrnaAssayDataSet(DataSet):
                                'flag']
 
         # Fields displayed in the search results table header
-        self.header_fields = ['Index',
+        self.header_fields=['Index',
                               'Record',
                               'Symbol',
                               'Description',
@@ -1006,9 +1067,9 @@ class MrnaAssayDataSet(DataSet):
                               'Additive Effect']
 
         # Todo: Obsolete or rename this field
-        self.type = 'ProbeSet'
+        self.type='ProbeSet'
 
-        self.query_for_group = '''
+        self.query_for_group='''
                         SELECT
                                 InbredSet.Name, InbredSet.Id, InbredSet.GeneticType
                         FROM
@@ -1026,7 +1087,7 @@ class MrnaAssayDataSet(DataSet):
 
         #  Note: setting trait_list to [] is probably not a great idea.
         if not trait_list:
-            trait_list = []
+            trait_list=[]
 
         for this_trait in trait_list:
 
@@ -1034,33 +1095,33 @@ class MrnaAssayDataSet(DataSet):
                 this_trait.retrieveInfo(QTL=1)
 
             if not this_trait.symbol:
-                this_trait.symbol = "N/A"
+                this_trait.symbol="N/A"
 
             # XZ, 12/08/2008: description
             # XZ, 06/05/2009: Rob asked to add probe target description
-            description_string = str(
+            description_string=str(
                 str(this_trait.description).strip(codecs.BOM_UTF8), 'utf-8')
-            target_string = str(
+            target_string=str(
                 str(this_trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8')
 
             if len(description_string) > 1 and description_string != 'None':
-                description_display = description_string
+                description_display=description_string
             else:
-                description_display = this_trait.symbol
+                description_display=this_trait.symbol
 
             if (len(description_display) > 1 and description_display != 'N/A'
                     and len(target_string) > 1 and target_string != 'None'):
-                description_display = description_display + '; ' + target_string.strip()
+                description_display=description_display + '; ' + target_string.strip()
 
             # Save it for the jinja2 template
-            this_trait.description_display = description_display
+            this_trait.description_display=description_display
 
             if this_trait.chr and this_trait.mb:
-                this_trait.location_repr = 'Chr%s: %.6f' % (
+                this_trait.location_repr='Chr%s: %.6f' % (
                     this_trait.chr, float(this_trait.mb))
 
             # Get mean expression value
-            query = (
+            query=(
                 """select ProbeSetXRef.mean from ProbeSetXRef, ProbeSet
                 where ProbeSetXRef.ProbeSetFreezeId = %s and
                 ProbeSet.Id = ProbeSetXRef.ProbeSetId and
@@ -1068,44 +1129,45 @@ class MrnaAssayDataSet(DataSet):
             """ % (escape(str(this_trait.dataset.id)),
                    escape(this_trait.name)))
 
-            #logger.debug("query is:", pf(query))
+            # logger.debug("query is:", pf(query))
             logger.sql(query)
-            result = g.db.execute(query).fetchone()
+            result=g.db.execute(query).fetchone()
 
-            mean = result[0] if result else 0
+            mean=result[0] if result else 0
 
             if mean:
-                this_trait.mean = "%2.3f" % mean
+                this_trait.mean="%2.3f" % mean
 
             # LRS and its location
-            this_trait.LRS_score_repr = 'N/A'
-            this_trait.LRS_location_repr = 'N/A'
+            this_trait.LRS_score_repr='N/A'
+            this_trait.LRS_location_repr='N/A'
 
             # Max LRS and its Locus location
             if this_trait.lrs and this_trait.locus:
-                query = """
+                query="""
                     select Geno.Chr, Geno.Mb from Geno, Species
                     where Species.Name = '{}' and
                         Geno.Name = '{}' and
                         Geno.SpeciesId = Species.Id
                 """.format(species, this_trait.locus)
                 logger.sql(query)
-                result = g.db.execute(query).fetchone()
+                result=g.db.execute(query).fetchone()
 
                 if result:
-                    lrs_chr, lrs_mb = result
-                    this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs
-                    this_trait.LRS_location_repr = 'Chr%s: %.6f' % (
+                    lrs_chr, lrs_mb=result
+                    this_trait.LRS_score_repr='%3.1f' % this_trait.lrs
+                    this_trait.LRS_location_repr='Chr%s: %.6f' % (
                         lrs_chr, float(lrs_mb))
 
         return trait_list
 
     def retrieve_sample_data(self, trait):
-        query = """
+        query="""
                     SELECT
                             Strain.Name, ProbeSetData.value, ProbeSetSE.error, NStrain.count, Strain.Name2
                     FROM
-                            (ProbeSetData, ProbeSetFreeze, Strain, ProbeSet, ProbeSetXRef)
+                            (ProbeSetData, ProbeSetFreeze,
+                             Strain, ProbeSet, ProbeSetXRef)
                     left join ProbeSetSE on
                             (ProbeSetSE.DataId = ProbeSetData.Id AND ProbeSetSE.StrainId = ProbeSetData.StrainId)
                     left join NStrain on
@@ -1121,19 +1183,19 @@ class MrnaAssayDataSet(DataSet):
                             Strain.Name
                     """ % (escape(trait), escape(self.name))
         logger.sql(query)
-        results = g.db.execute(query).fetchall()
-        #logger.debug("RETRIEVED RESULTS HERE:", results)
+        results=g.db.execute(query).fetchall()
+        # logger.debug("RETRIEVED RESULTS HERE:", results)
         return results
 
     def retrieve_genes(self, column_name):
-        query = """
+        query="""
                     select ProbeSet.Name, ProbeSet.%s
                     from ProbeSet,ProbeSetXRef
                     where ProbeSetXRef.ProbeSetFreezeId = %s and
                     ProbeSetXRef.ProbeSetId=ProbeSet.Id;
                 """ % (column_name, escape(str(self.id)))
         logger.sql(query)
-        results = g.db.execute(query).fetchall()
+        results=g.db.execute(query).fetchall()
 
         return dict(results)
 
@@ -1141,40 +1203,40 @@ class MrnaAssayDataSet(DataSet):
 class TempDataSet(DataSet):
     '''Temporary user-generated data set'''
 
-    DS_NAME_MAP['Temp'] = 'TempDataSet'
+    DS_NAME_MAP['Temp']='TempDataSet'
 
     def setup(self):
-        self.search_fields = ['name',
+        self.search_fields=['name',
                               'description']
 
-        self.display_fields = ['name',
+        self.display_fields=['name',
                                'description']
 
-        self.header_fields = ['Name',
+        self.header_fields=['Name',
                               'Description']
 
-        self.type = 'Temp'
+        self.type='Temp'
 
         # Need to double check later how these are used
-        self.id = 1
-        self.fullname = 'Temporary Storage'
-        self.shortname = 'Temp'
+        self.id=1
+        self.fullname='Temporary Storage'
+        self.shortname='Temp'
 
 
 def geno_mrna_confidentiality(ob):
-    dataset_table = ob.type + "Freeze"
-    #logger.debug("dataset_table [%s]: %s" % (type(dataset_table), dataset_table))
+    dataset_table=ob.type + "Freeze"
+    # logger.debug("dataset_table [%s]: %s" % (type(dataset_table), dataset_table))
 
-    query = '''SELECT Id, Name, FullName, confidentiality,
+    query='''SELECT Id, Name, FullName, confidentiality,
                         AuthorisedUsers FROM %s WHERE Name = "%s"''' % (dataset_table, ob.name)
     logger.sql(query)
-    result = g.db.execute(query)
+    result=g.db.execute(query)
 
     (dataset_id,
      name,
      full_name,
      confidential,
-     authorized_users) = result.fetchall()[0]
+     authorized_users)=result.fetchall()[0]
 
     if confidential:
         return True
diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index df0afba6..9fbfee48 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -27,6 +27,34 @@ def create_target_this_trait(start_vars):
     return (this_dataset, this_trait, target_dataset, sample_data)
 
 
+
+def test_process_data(this_trait,dataset,start_vars):
+    """test function for bxd,all and other sample data"""
+
+    corr_samples_group = start_vars["corr_samples_group"]
+
+
+    primary_samples = dataset.group.samplelist
+    if dataset.group.parlist != None:
+        primary_samples += dataset.group.parlist
+    if dataset.group.f1list != None:
+        primary_samples += dataset.group.f1list
+
+    # If either BXD/whatever Only or All Samples, append all of that group's samplelist
+    if corr_samples_group != 'samples_other':
+        sample_data = process_samples(start_vars, primary_samples)
+
+    # If either Non-BXD/whatever or All Samples, get all samples from this_trait.data and
+    # exclude the primary samples (because they would have been added in the previous
+    # if statement if the user selected All Samples)
+    if corr_samples_group != 'samples_primary':
+        if corr_samples_group == 'samples_other':
+            primary_samples = [x for x in primary_samples if x not in (
+                dataset.group.parlist + dataset.group.f1list)]
+        sample_data = process_samples(start_vars, list(this_trait.data.keys()), primary_samples)
+
+    return sample_data
+
 def process_samples(start_vars, sample_names, excluded_samples=None):
     """process samples"""
     sample_data = {}
@@ -118,13 +146,22 @@ def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset):
 
     sample_data = process_samples(
         start_vars, this_dataset.group.samplelist)
-    target_dataset.get_trait_data(list(sample_data.keys()))
+
+    # sample_data = test_process_data(this_trait,this_dataset,start_vars)
+
+    if target_dataset.type =="ProbeSet":
+        # pass
+        target_dataset.get_probeset_data(list(sample_data.keys()))
+    else:
+        target_dataset.get_trait_data(list(sample_data.keys()))
     this_trait = retrieve_sample_data(this_trait, this_dataset)
     this_trait_data = {
         "trait_sample_data": sample_data,
         "trait_id": start_vars["trait_id"]
     }
 
+    # should remove this  len(samplelist) == len(strain_values)
+
     results = map_shared_keys_to_values(
         target_dataset.samplelist, target_dataset.trait_data)
 
@@ -205,6 +242,7 @@ def compute_correlation(start_vars, method="pearson", compute_all=False):
                         "target_dataset": start_vars['corr_dataset'],
                         "return_results": corr_return_results}
 
+
     return correlation_data
 
 
@@ -265,3 +303,107 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict):
         }
         return (primary_tissue_data, target_tissue_data)
     return None
+
+
+def generate_corr_data(corr_results, target_dataset):
+    counter = 0
+    results_list = []
+    for (index, trait_corr) in enumerate(corr_results):
+        trait_name = list(trait_corr.keys())[0]
+        trait = create_trait(dataset=target_dataset,
+                             name=trait_name)
+
+        trait_corr_data =  trait_corr[trait_name]
+
+        if trait.view == False:
+            continue
+        results_dict = {}
+        results_dict['index'] = index + 1
+        results_dict['trait_id'] = trait.name
+        results_dict['dataset'] = trait.dataset.name
+        # results_dict['hmac'] = hmac.data_hmac(
+        #     '{}:{}'.format(trait.name, trait.dataset.name))
+        if target_dataset.type == "ProbeSet":
+            results_dict['symbol'] = trait.symbol
+            results_dict['description'] = "N/A"
+            results_dict['location'] = trait.location_repr
+            results_dict['mean'] = "N/A"
+            results_dict['additive'] = "N/A"
+            if bool(trait.description_display):
+                results_dict['description'] = trait.description_display
+            if bool(trait.mean):
+                results_dict['mean'] = f"{float(trait.mean):.3f}"
+            try:
+                results_dict['lod_score'] = f"{float(trait.LRS_score_repr) / 4.61:.1f}"
+            except:
+                results_dict['lod_score'] = "N/A"
+            results_dict['lrs_location'] = trait.LRS_location_repr
+            if bool(trait.additive):
+                results_dict['additive'] = f"{float(trait.additive):.3f}"
+            results_dict['sample_r'] = f"{float(trait_corr_data.get('sample_r',0)):.3f}"
+            results_dict['num_overlap'] = trait.num_overlap
+            results_dict['sample_p'] = f"{float(trait_corr_data.get('sample_p',0)):.3e}"
+            results_dict['lit_corr'] = "--"
+            results_dict['tissue_corr'] = "--"
+            results_dict['tissue_pvalue'] = "--"
+            tissue_corr = trait_corr_data.get('tissue_corr',0)
+            lit_corr = trait_corr_data.get('lit_corr',0)
+            if bool(lit_corr):
+                results_dict['lit_corr'] = f"{float(trait_corr_data.get('lit_corr',0)):.3f}"
+            if bool(tissue_corr):
+                results_dict['tissue_corr'] = f"{float(trait_corr_data.get('tissue_corr',0)):.3f}"
+                results_dict['tissue_pvalue'] = f"{float(trait_corr_data.get('tissue_pvalue',0)):.3e}"
+        elif target_dataset.type == "Publish":
+            results_dict['abbreviation_display'] = "N/A"
+            results_dict['description'] = "N/A"
+            results_dict['mean'] = "N/A"
+            results_dict['authors_display'] = "N/A"
+            results_dict['additive'] = "N/A"
+            if for_api:
+                results_dict['pubmed_id'] = "N/A"
+                results_dict['year'] = "N/A"
+            else:
+                results_dict['pubmed_link'] = "N/A"
+                results_dict['pubmed_text'] = "N/A"
+
+            if bool(trait.abbreviation):
+                results_dict['abbreviation_display'] = trait.abbreviation
+            if bool(trait.description_display):
+                results_dict['description'] = trait.description_display
+            if bool(trait.mean):
+                results_dict['mean'] = f"{float(trait.mean):.3f}"
+            if bool(trait.authors):
+                authors_list = trait.authors.split(',')
+                if len(authors_list) > 6:
+                    results_dict['authors_display'] = ", ".join(
+                        authors_list[:6]) + ", et al."
+                else:
+                    results_dict['authors_display'] = trait.authors
+            if bool(trait.pubmed_id):
+                if for_api:
+                    results_dict['pubmed_id'] = trait.pubmed_id
+                    results_dict['year'] = trait.pubmed_text
+                else:
+                    results_dict['pubmed_link'] = trait.pubmed_link
+                    results_dict['pubmed_text'] = trait.pubmed_text
+            try:
+                results_dict['lod_score'] = f"{float(trait.LRS_score_repr) / 4.61:.1f}"
+            except:
+                results_dict['lod_score'] = "N/A"
+            results_dict['lrs_location'] = trait.LRS_location_repr
+            if bool(trait.additive):
+                results_dict['additive'] = f"{float(trait.additive):.3f}"
+            results_dict['sample_r'] = f"{float(trait_corr_data.get('sample_r',0)):.3f}"
+            results_dict['num_overlap'] = trait.num_overlap
+            results_dict['sample_p'] = f"{float(trait_corr_data.get('sample_p',0)):.3e}"
+        else:
+            results_dict['location'] = trait.location_repr
+            results_dict['sample_r'] = f"{float(trait_corr_data.get('sample_r',0)):.3f}"
+            results_dict['num_overlap'] = trait.num_overlap
+            results_dict['sample_p'] = f"{float(trait_corr_data.get('sample_p',0)):.3e}"
+
+        results_list.append(results_dict)
+
+    return results_list
+
+
-- 
cgit v1.2.3


From c9a133e75d32e0bb9c8e63443b0f43f084363996 Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Mon, 24 May 2021 16:37:36 +0300
Subject: pep8 formatting

---
 wqflask/base/data_set.py | 214 ++++++++++++++++++++++++-----------------------
 1 file changed, 109 insertions(+), 105 deletions(-)

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 7531ac81..39296f6a 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -659,12 +659,8 @@ class DataSet:
                 "Dataset {} is not yet available in GeneNetwork.".format(self.name))
             pass
 
-
-
-
     def chunk_dataset(self, dataset, n):
 
-
         results = {}
 
         query = """
@@ -677,10 +673,7 @@ class DataSet:
 
         # should cache this
 
-        traits_name_dict= dict(g.db.execute(query).fetchall())
-
-
-
+        traits_name_dict = dict(g.db.execute(query).fetchall())
 
         for i in range(0, len(dataset), n):
             matrix = list(dataset[i:i + n])
@@ -716,50 +709,50 @@ class DataSet:
                 FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze)
                 WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id
                 and ProbeSetFreeze.Name = '{}'
-                and ProbeSet.Id = ProbeSetXRef.ProbeSetId)""".format(create_in_clause(sample_ids),self.name)
+                and ProbeSet.Id = ProbeSetXRef.ProbeSetId)""".format(create_in_clause(sample_ids), self.name)
 
-        query_results=list(g.db.execute(query).fetchall())
+        query_results = list(g.db.execute(query).fetchall())
 
-        data_results=self.chunk_dataset(query_results, len(sample_ids))
-        self.trait_data=data_results
+        data_results = self.chunk_dataset(query_results, len(sample_ids))
+        self.trait_data = data_results
 
     def get_trait_data(self, sample_list=None):
         if sample_list:
-            self.samplelist=sample_list
+            self.samplelist = sample_list
         else:
-            self.samplelist=self.group.samplelist
+            self.samplelist = self.group.samplelist
 
         if self.group.parlist != None and self.group.f1list != None:
             if (self.group.parlist + self.group.f1list) in self.samplelist:
                 self.samplelist += self.group.parlist + self.group.f1list
 
-        query="""
+        query = """
             SELECT Strain.Name, Strain.Id FROM Strain, Species
             WHERE Strain.Name IN {}
             and Strain.SpeciesId=Species.Id
             and Species.name = '{}'
             """.format(create_in_clause(self.samplelist), *mescape(self.group.species))
         logger.sql(query)
-        results=dict(g.db.execute(query).fetchall())
-        sample_ids=[results[item] for item in self.samplelist]
+        results = dict(g.db.execute(query).fetchall())
+        sample_ids = [results[item] for item in self.samplelist]
 
         # MySQL limits the number of tables that can be used in a join to 61,
         # so we break the sample ids into smaller chunks
         # Postgres doesn't have that limit, so we can get rid of this after we transition
-        chunk_size=50
-        number_chunks=int(math.ceil(len(sample_ids) / chunk_size))
-        trait_sample_data=[]
+        chunk_size = 50
+        number_chunks = int(math.ceil(len(sample_ids) / chunk_size))
+        trait_sample_data = []
         for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks):
             if self.type == "Publish":
-                dataset_type="Phenotype"
+                dataset_type = "Phenotype"
             else:
-                dataset_type=self.type
-            temp=['T%s.value' % item for item in sample_ids_step]
+                dataset_type = self.type
+            temp = ['T%s.value' % item for item in sample_ids_step]
             if self.type == "Publish":
-                query="SELECT {}XRef.Id,".format(escape(self.type))
+                query = "SELECT {}XRef.Id,".format(escape(self.type))
             else:
-                query="SELECT {}.Name,".format(escape(dataset_type))
-            data_start_pos=1
+                query = "SELECT {}.Name,".format(escape(dataset_type))
+            data_start_pos = 1
             query += ', '.join(temp)
             query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(dataset_type,
                                                                      self.type,
@@ -788,27 +781,27 @@ class DataSet:
                         """.format(*mescape(self.type, self.type, self.type, self.type,
                                             self.name, dataset_type, self.type, self.type, dataset_type))
 
-            results=g.db.execute(query).fetchall()
+            results = g.db.execute(query).fetchall()
             trait_sample_data.append(results)
 
-        trait_count=len(trait_sample_data[0])
-        self.trait_data=collections.defaultdict(list)
+        trait_count = len(trait_sample_data[0])
+        self.trait_data = collections.defaultdict(list)
 
         # put all of the separate data together into a dictionary where the keys are
         # trait names and values are lists of sample values
         for trait_counter in range(trait_count):
-            trait_name=trait_sample_data[0][trait_counter][0]
+            trait_name = trait_sample_data[0][trait_counter][0]
             for chunk_counter in range(int(number_chunks)):
                 self.trait_data[trait_name] += (
                     trait_sample_data[chunk_counter][trait_counter][data_start_pos:])
 
 
 class PhenotypeDataSet(DataSet):
-    DS_NAME_MAP['Publish']='PhenotypeDataSet'
+    DS_NAME_MAP['Publish'] = 'PhenotypeDataSet'
 
     def setup(self):
         # Fields in the database table
-        self.search_fields=['Phenotype.Post_publication_description',
+        self.search_fields = ['Phenotype.Post_publication_description',
                               'Phenotype.Pre_publication_description',
                               'Phenotype.Pre_publication_abbreviation',
                               'Phenotype.Post_publication_abbreviation',
@@ -821,7 +814,7 @@ class PhenotypeDataSet(DataSet):
                               'PublishXRef.Id']
 
         # Figure out what display_fields is
-        self.display_fields=['name', 'group_code',
+        self.display_fields = ['name', 'group_code',
                                'pubmed_id',
                                'pre_publication_description',
                                'post_publication_description',
@@ -839,7 +832,7 @@ class PhenotypeDataSet(DataSet):
                                'sequence', 'units', 'comments']
 
         # Fields displayed in the search results table header
-        self.header_fields=['Index',
+        self.header_fields = ['Index',
                               'Record',
                               'Description',
                               'Authors',
@@ -848,9 +841,9 @@ class PhenotypeDataSet(DataSet):
                               'Max LRS Location',
                               'Additive Effect']
 
-        self.type='Publish'
+        self.type = 'Publish'
 
-        self.query_for_group='''
+        self.query_for_group = '''
                             SELECT
                                     InbredSet.Name, InbredSet.Id, InbredSet.GeneticType
                             FROM
@@ -870,13 +863,13 @@ class PhenotypeDataSet(DataSet):
             if not this_trait.haveinfo:
                 this_trait.retrieve_info(get_qtl_info=True)
 
-            description=this_trait.post_publication_description
+            description = this_trait.post_publication_description
 
             # If the dataset is confidential and the user has access to confidential
             # phenotype traits, then display the pre-publication description instead
             # of the post-publication description
             if this_trait.confidential:
-                this_trait.description_display=""
+                this_trait.description_display = ""
                 continue   # for now, because no authorization features
 
                 if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(
@@ -884,46 +877,46 @@ class PhenotypeDataSet(DataSet):
                         userName=self.userName,
                         authorized_users=this_trait.authorized_users):
 
-                    description=this_trait.pre_publication_description
+                    description = this_trait.pre_publication_description
 
             if len(description) > 0:
-                this_trait.description_display=description.strip()
+                this_trait.description_display = description.strip()
             else:
-                this_trait.description_display=""
+                this_trait.description_display = ""
 
             if not this_trait.year.isdigit():
-                this_trait.pubmed_text="N/A"
+                this_trait.pubmed_text = "N/A"
             else:
-                this_trait.pubmed_text=this_trait.year
+                this_trait.pubmed_text = this_trait.year
 
             if this_trait.pubmed_id:
-                this_trait.pubmed_link=webqtlConfig.PUBMEDLINK_URL % this_trait.pubmed_id
+                this_trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % this_trait.pubmed_id
 
             # LRS and its location
-            this_trait.LRS_score_repr="N/A"
-            this_trait.LRS_location_repr="N/A"
+            this_trait.LRS_score_repr = "N/A"
+            this_trait.LRS_location_repr = "N/A"
 
             if this_trait.lrs:
-                query="""
+                query = """
                     select Geno.Chr, Geno.Mb from Geno, Species
                     where Species.Name = '%s' and
                         Geno.Name = '%s' and
                         Geno.SpeciesId = Species.Id
                 """ % (species, this_trait.locus)
                 logger.sql(query)
-                result=g.db.execute(query).fetchone()
+                result = g.db.execute(query).fetchone()
 
                 if result:
                     if result[0] and result[1]:
-                        LRS_Chr=result[0]
-                        LRS_Mb=result[1]
+                        LRS_Chr = result[0]
+                        LRS_Mb = result[1]
 
-                        this_trait.LRS_score_repr=LRS_score_repr='%3.1f' % this_trait.lrs
-                        this_trait.LRS_location_repr=LRS_location_repr='Chr%s: %.6f' % (
+                        this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs
+                        this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (
                             LRS_Chr, float(LRS_Mb))
 
     def retrieve_sample_data(self, trait):
-        query="""
+        query = """
                     SELECT
                             Strain.Name, PublishData.value, PublishSE.error, NStrain.count, Strain.Name2
                     FROM
@@ -941,34 +934,34 @@ class PhenotypeDataSet(DataSet):
                             Strain.Name
                     """
         logger.sql(query)
-        results=g.db.execute(query, (trait, self.id)).fetchall()
+        results = g.db.execute(query, (trait, self.id)).fetchall()
         return results
 
 
 class GenotypeDataSet(DataSet):
-    DS_NAME_MAP['Geno']='GenotypeDataSet'
+    DS_NAME_MAP['Geno'] = 'GenotypeDataSet'
 
     def setup(self):
         # Fields in the database table
-        self.search_fields=['Name',
+        self.search_fields = ['Name',
                               'Chr']
 
         # Find out what display_fields is
-        self.display_fields=['name',
+        self.display_fields = ['name',
                                'chr',
                                'mb',
                                'source2',
                                'sequence']
 
         # Fields displayed in the search results table header
-        self.header_fields=['Index',
+        self.header_fields = ['Index',
                               'ID',
                               'Location']
 
         # Todo: Obsolete or rename this field
-        self.type='Geno'
+        self.type = 'Geno'
 
-        self.query_for_group='''
+        self.query_for_group = '''
                 SELECT
                         InbredSet.Name, InbredSet.Id, InbredSet.GeneticType
                 FROM
@@ -987,11 +980,11 @@ class GenotypeDataSet(DataSet):
                 this_trait.retrieveInfo()
 
             if this_trait.chr and this_trait.mb:
-                this_trait.location_repr='Chr%s: %.6f' % (
+                this_trait.location_repr = 'Chr%s: %.6f' % (
                     this_trait.chr, float(this_trait.mb))
 
     def retrieve_sample_data(self, trait):
-        query="""
+        query = """
                     SELECT
                             Strain.Name, GenoData.value, GenoSE.error, "N/A", Strain.Name2
                     FROM
@@ -1008,7 +1001,7 @@ class GenotypeDataSet(DataSet):
                             Strain.Name
                     """
         logger.sql(query)
-        results=g.db.execute(query,
+        results = g.db.execute(query,
                                (webqtlDatabaseFunction.retrieve_species_id(self.group.name),
                                 trait, self.name)).fetchall()
         return results
@@ -1022,11 +1015,11 @@ class MrnaAssayDataSet(DataSet):
     platform and is far too specific.
 
     '''
-    DS_NAME_MAP['ProbeSet']='MrnaAssayDataSet'
+    DS_NAME_MAP['ProbeSet'] = 'MrnaAssayDataSet'
 
     def setup(self):
         # Fields in the database table
-        self.search_fields=['Name',
+        self.search_fields = ['Name',
                               'Description',
                               'Probe_Target_Description',
                               'Symbol',
@@ -1036,7 +1029,7 @@ class MrnaAssayDataSet(DataSet):
                               'RefSeq_TranscriptId']
 
         # Find out what display_fields is
-        self.display_fields=['name', 'symbol',
+        self.display_fields = ['name', 'symbol',
                                'description', 'probe_target_description',
                                'chr', 'mb',
                                'alias', 'geneid',
@@ -1056,7 +1049,7 @@ class MrnaAssayDataSet(DataSet):
                                'flag']
 
         # Fields displayed in the search results table header
-        self.header_fields=['Index',
+        self.header_fields = ['Index',
                               'Record',
                               'Symbol',
                               'Description',
@@ -1067,9 +1060,9 @@ class MrnaAssayDataSet(DataSet):
                               'Additive Effect']
 
         # Todo: Obsolete or rename this field
-        self.type='ProbeSet'
+        self.type = 'ProbeSet'
 
-        self.query_for_group='''
+        self.query_for_group = '''
                         SELECT
                                 InbredSet.Name, InbredSet.Id, InbredSet.GeneticType
                         FROM
@@ -1087,7 +1080,7 @@ class MrnaAssayDataSet(DataSet):
 
         #  Note: setting trait_list to [] is probably not a great idea.
         if not trait_list:
-            trait_list=[]
+            trait_list = []
 
         for this_trait in trait_list:
 
@@ -1095,33 +1088,33 @@ class MrnaAssayDataSet(DataSet):
                 this_trait.retrieveInfo(QTL=1)
 
             if not this_trait.symbol:
-                this_trait.symbol="N/A"
+                this_trait.symbol = "N/A"
 
             # XZ, 12/08/2008: description
             # XZ, 06/05/2009: Rob asked to add probe target description
-            description_string=str(
+            description_string = str(
                 str(this_trait.description).strip(codecs.BOM_UTF8), 'utf-8')
-            target_string=str(
+            target_string = str(
                 str(this_trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8')
 
             if len(description_string) > 1 and description_string != 'None':
-                description_display=description_string
+                description_display = description_string
             else:
-                description_display=this_trait.symbol
+                description_display = this_trait.symbol
 
             if (len(description_display) > 1 and description_display != 'N/A'
                     and len(target_string) > 1 and target_string != 'None'):
-                description_display=description_display + '; ' + target_string.strip()
+                description_display = description_display + '; ' + target_string.strip()
 
             # Save it for the jinja2 template
-            this_trait.description_display=description_display
+            this_trait.description_display = description_display
 
             if this_trait.chr and this_trait.mb:
-                this_trait.location_repr='Chr%s: %.6f' % (
+                this_trait.location_repr = 'Chr%s: %.6f' % (
                     this_trait.chr, float(this_trait.mb))
 
             # Get mean expression value
-            query=(
+            query = (
                 """select ProbeSetXRef.mean from ProbeSetXRef, ProbeSet
                 where ProbeSetXRef.ProbeSetFreezeId = %s and
                 ProbeSet.Id = ProbeSetXRef.ProbeSetId and
@@ -1131,38 +1124,38 @@ class MrnaAssayDataSet(DataSet):
 
             # logger.debug("query is:", pf(query))
             logger.sql(query)
-            result=g.db.execute(query).fetchone()
+            result = g.db.execute(query).fetchone()
 
-            mean=result[0] if result else 0
+            mean = result[0] if result else 0
 
             if mean:
-                this_trait.mean="%2.3f" % mean
+                this_trait.mean = "%2.3f" % mean
 
             # LRS and its location
-            this_trait.LRS_score_repr='N/A'
-            this_trait.LRS_location_repr='N/A'
+            this_trait.LRS_score_repr = 'N/A'
+            this_trait.LRS_location_repr = 'N/A'
 
             # Max LRS and its Locus location
             if this_trait.lrs and this_trait.locus:
-                query="""
+                query = """
                     select Geno.Chr, Geno.Mb from Geno, Species
                     where Species.Name = '{}' and
                         Geno.Name = '{}' and
                         Geno.SpeciesId = Species.Id
                 """.format(species, this_trait.locus)
                 logger.sql(query)
-                result=g.db.execute(query).fetchone()
+                result = g.db.execute(query).fetchone()
 
                 if result:
-                    lrs_chr, lrs_mb=result
-                    this_trait.LRS_score_repr='%3.1f' % this_trait.lrs
-                    this_trait.LRS_location_repr='Chr%s: %.6f' % (
+                    lrs_chr, lrs_mb = result
+                    this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs
+                    this_trait.LRS_location_repr = 'Chr%s: %.6f' % (
                         lrs_chr, float(lrs_mb))
 
         return trait_list
 
     def retrieve_sample_data(self, trait):
-        query="""
+        query = """
                     SELECT
                             Strain.Name, ProbeSetData.value, ProbeSetSE.error, NStrain.count, Strain.Name2
                     FROM
@@ -1183,19 +1176,19 @@ class MrnaAssayDataSet(DataSet):
                             Strain.Name
                     """ % (escape(trait), escape(self.name))
         logger.sql(query)
-        results=g.db.execute(query).fetchall()
+        results = g.db.execute(query).fetchall()
         # logger.debug("RETRIEVED RESULTS HERE:", results)
         return results
 
     def retrieve_genes(self, column_name):
-        query="""
+        query = """
                     select ProbeSet.Name, ProbeSet.%s
                     from ProbeSet,ProbeSetXRef
                     where ProbeSetXRef.ProbeSetFreezeId = %s and
                     ProbeSetXRef.ProbeSetId=ProbeSet.Id;
                 """ % (column_name, escape(str(self.id)))
         logger.sql(query)
-        results=g.db.execute(query).fetchall()
+        results = g.db.execute(query).fetchall()
 
         return dict(results)
 
@@ -1203,40 +1196,51 @@ class MrnaAssayDataSet(DataSet):
 class TempDataSet(DataSet):
     '''Temporary user-generated data set'''
 
-    DS_NAME_MAP['Temp']='TempDataSet'
+    DS_NAME_MAP['Temp'] = 'TempDataSet'
 
     def setup(self):
-        self.search_fields=['name',
+        self.search_fields = ['name',
                               'description']
 
-        self.display_fields=['name',
+        self.display_fields = ['name',
                                'description']
 
-        self.header_fields=['Name',
+        self.header_fields = ['Name',
                               'Description']
 
-        self.type='Temp'
+        self.type = 'Temp'
 
         # Need to double check later how these are used
-        self.id=1
-        self.fullname='Temporary Storage'
-        self.shortname='Temp'
+        self.id = 1
+        self.fullname = 'Temporary Storage'
+        self.shortname = 'Temp'
 
 
 def geno_mrna_confidentiality(ob):
-    dataset_table=ob.type + "Freeze"
+    dataset_table = ob.type + "Freeze"
     # logger.debug("dataset_table [%s]: %s" % (type(dataset_table), dataset_table))
 
-    query='''SELECT Id, Name, FullName, confidentiality,
+    query = '''SELECT Id, Name, FullName, confidentiality,
                         AuthorisedUsers FROM %s WHERE Name = "%s"''' % (dataset_table, ob.name)
     logger.sql(query)
-    result=g.db.execute(query)
+    result = g.db.execute(query)
+
+    (dataset_id,
+     name,
+     full_name,
+     confidential,
+     authorized_users) = result.fetchall()[0]
+
+    if confidential:
+        return True
+uery)
+    result = g.db.execute(query)
 
     (dataset_id,
      name,
      full_name,
      confidential,
-     authorized_users)=result.fetchall()[0]
+     authorized_users) = result.fetchall()[0]
 
     if confidential:
         return True
-- 
cgit v1.2.3


From d8d3a8311106ea97c1fba4ef4606929eb9b0fdcb Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Mon, 24 May 2021 16:43:45 +0300
Subject: minor fix

---
 wqflask/base/data_set.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 39296f6a..4a150e86 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -1233,7 +1233,6 @@ def geno_mrna_confidentiality(ob):
 
     if confidential:
         return True
-uery)
     result = g.db.execute(query)
 
     (dataset_id,
-- 
cgit v1.2.3


From aaacfe3a0abc7ca4fe5bdb486651e018cdc7aba0 Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Wed, 9 Jun 2021 07:25:03 +0300
Subject: remove unused functions + minor fixes

---
 wqflask/base/data_set.py                           |   2 +
 wqflask/wqflask/correlation/correlation_gn3_api.py | 115 +--------------------
 2 files changed, 7 insertions(+), 110 deletions(-)

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 4a150e86..4d54cfae 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -684,6 +684,8 @@ class DataSet:
         return results
 
     def get_probeset_data(self, sample_list=None, trait_ids=None):
+
+        # improvement of get trait data--->>>
         if sample_list:
             self.samplelist = sample_list
 
diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index 9fbfee48..5fa33027 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -27,13 +27,11 @@ def create_target_this_trait(start_vars):
     return (this_dataset, this_trait, target_dataset, sample_data)
 
 
-
-def test_process_data(this_trait,dataset,start_vars):
+def test_process_data(this_trait, dataset, start_vars):
     """test function for bxd,all and other sample data"""
 
     corr_samples_group = start_vars["corr_samples_group"]
 
-
     primary_samples = dataset.group.samplelist
     if dataset.group.parlist != None:
         primary_samples += dataset.group.parlist
@@ -51,10 +49,12 @@ def test_process_data(this_trait,dataset,start_vars):
         if corr_samples_group == 'samples_other':
             primary_samples = [x for x in primary_samples if x not in (
                 dataset.group.parlist + dataset.group.f1list)]
-        sample_data = process_samples(start_vars, list(this_trait.data.keys()), primary_samples)
+        sample_data = process_samples(start_vars, list(
+            this_trait.data.keys()), primary_samples)
 
     return sample_data
 
+
 def process_samples(start_vars, sample_names, excluded_samples=None):
     """process samples"""
     sample_data = {}
@@ -149,7 +149,7 @@ def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset):
 
     # sample_data = test_process_data(this_trait,this_dataset,start_vars)
 
-    if target_dataset.type =="ProbeSet":
+    if target_dataset.type == "ProbeSet":
         # pass
         target_dataset.get_probeset_data(list(sample_data.keys()))
     else:
@@ -242,7 +242,6 @@ def compute_correlation(start_vars, method="pearson", compute_all=False):
                         "target_dataset": start_vars['corr_dataset'],
                         "return_results": corr_return_results}
 
-
     return correlation_data
 
 
@@ -303,107 +302,3 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict):
         }
         return (primary_tissue_data, target_tissue_data)
     return None
-
-
-def generate_corr_data(corr_results, target_dataset):
-    counter = 0
-    results_list = []
-    for (index, trait_corr) in enumerate(corr_results):
-        trait_name = list(trait_corr.keys())[0]
-        trait = create_trait(dataset=target_dataset,
-                             name=trait_name)
-
-        trait_corr_data =  trait_corr[trait_name]
-
-        if trait.view == False:
-            continue
-        results_dict = {}
-        results_dict['index'] = index + 1
-        results_dict['trait_id'] = trait.name
-        results_dict['dataset'] = trait.dataset.name
-        # results_dict['hmac'] = hmac.data_hmac(
-        #     '{}:{}'.format(trait.name, trait.dataset.name))
-        if target_dataset.type == "ProbeSet":
-            results_dict['symbol'] = trait.symbol
-            results_dict['description'] = "N/A"
-            results_dict['location'] = trait.location_repr
-            results_dict['mean'] = "N/A"
-            results_dict['additive'] = "N/A"
-            if bool(trait.description_display):
-                results_dict['description'] = trait.description_display
-            if bool(trait.mean):
-                results_dict['mean'] = f"{float(trait.mean):.3f}"
-            try:
-                results_dict['lod_score'] = f"{float(trait.LRS_score_repr) / 4.61:.1f}"
-            except:
-                results_dict['lod_score'] = "N/A"
-            results_dict['lrs_location'] = trait.LRS_location_repr
-            if bool(trait.additive):
-                results_dict['additive'] = f"{float(trait.additive):.3f}"
-            results_dict['sample_r'] = f"{float(trait_corr_data.get('sample_r',0)):.3f}"
-            results_dict['num_overlap'] = trait.num_overlap
-            results_dict['sample_p'] = f"{float(trait_corr_data.get('sample_p',0)):.3e}"
-            results_dict['lit_corr'] = "--"
-            results_dict['tissue_corr'] = "--"
-            results_dict['tissue_pvalue'] = "--"
-            tissue_corr = trait_corr_data.get('tissue_corr',0)
-            lit_corr = trait_corr_data.get('lit_corr',0)
-            if bool(lit_corr):
-                results_dict['lit_corr'] = f"{float(trait_corr_data.get('lit_corr',0)):.3f}"
-            if bool(tissue_corr):
-                results_dict['tissue_corr'] = f"{float(trait_corr_data.get('tissue_corr',0)):.3f}"
-                results_dict['tissue_pvalue'] = f"{float(trait_corr_data.get('tissue_pvalue',0)):.3e}"
-        elif target_dataset.type == "Publish":
-            results_dict['abbreviation_display'] = "N/A"
-            results_dict['description'] = "N/A"
-            results_dict['mean'] = "N/A"
-            results_dict['authors_display'] = "N/A"
-            results_dict['additive'] = "N/A"
-            if for_api:
-                results_dict['pubmed_id'] = "N/A"
-                results_dict['year'] = "N/A"
-            else:
-                results_dict['pubmed_link'] = "N/A"
-                results_dict['pubmed_text'] = "N/A"
-
-            if bool(trait.abbreviation):
-                results_dict['abbreviation_display'] = trait.abbreviation
-            if bool(trait.description_display):
-                results_dict['description'] = trait.description_display
-            if bool(trait.mean):
-                results_dict['mean'] = f"{float(trait.mean):.3f}"
-            if bool(trait.authors):
-                authors_list = trait.authors.split(',')
-                if len(authors_list) > 6:
-                    results_dict['authors_display'] = ", ".join(
-                        authors_list[:6]) + ", et al."
-                else:
-                    results_dict['authors_display'] = trait.authors
-            if bool(trait.pubmed_id):
-                if for_api:
-                    results_dict['pubmed_id'] = trait.pubmed_id
-                    results_dict['year'] = trait.pubmed_text
-                else:
-                    results_dict['pubmed_link'] = trait.pubmed_link
-                    results_dict['pubmed_text'] = trait.pubmed_text
-            try:
-                results_dict['lod_score'] = f"{float(trait.LRS_score_repr) / 4.61:.1f}"
-            except:
-                results_dict['lod_score'] = "N/A"
-            results_dict['lrs_location'] = trait.LRS_location_repr
-            if bool(trait.additive):
-                results_dict['additive'] = f"{float(trait.additive):.3f}"
-            results_dict['sample_r'] = f"{float(trait_corr_data.get('sample_r',0)):.3f}"
-            results_dict['num_overlap'] = trait.num_overlap
-            results_dict['sample_p'] = f"{float(trait_corr_data.get('sample_p',0)):.3e}"
-        else:
-            results_dict['location'] = trait.location_repr
-            results_dict['sample_r'] = f"{float(trait_corr_data.get('sample_r',0)):.3f}"
-            results_dict['num_overlap'] = trait.num_overlap
-            results_dict['sample_p'] = f"{float(trait_corr_data.get('sample_p',0)):.3e}"
-
-        results_list.append(results_dict)
-
-    return results_list
-
-
-- 
cgit v1.2.3


From 481b1fede3c815d5b9afbb07f0f69dcb7363143e Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Mon, 14 Jun 2021 00:52:10 +0300
Subject: remove print statements

---
 wqflask/utility/helper_functions.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py
index 50e00421..27dd0729 100644
--- a/wqflask/utility/helper_functions.py
+++ b/wqflask/utility/helper_functions.py
@@ -21,15 +21,11 @@ def get_species_dataset_trait(self, start_vars):
             self.dataset = data_set.create_dataset(start_vars['dataset'])
     else:
         self.dataset = data_set.create_dataset(start_vars['dataset'])
-    logger.debug("After creating dataset")
     self.species = TheSpecies(dataset=self.dataset)
-    logger.debug("After creating species")
     self.this_trait = create_trait(dataset=self.dataset,
                                    name=start_vars['trait_id'],
                                    cellid=None,
                                    get_qtl_info=True)
-    logger.debug("After creating trait")
-
 
 def get_trait_db_obs(self, trait_db_list):
     if isinstance(trait_db_list, str):
-- 
cgit v1.2.3


From 48e66dc230292a9c0b66da946106d8c9f611074f Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Mon, 14 Jun 2021 00:54:02 +0300
Subject: sort sample name by sample_ids

---
 wqflask/base/data_set.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 4d54cfae..991c9fee 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -705,6 +705,9 @@ class DataSet:
         results = dict(g.db.execute(query).fetchall())
         sample_ids = [results[item] for item in self.samplelist]
 
+        sorted_samplelist = [strain_name for strain_name, strain_id in sorted(
+            results.items(), key=lambda item: item[1])]
+
         query = """SELECT * from ProbeSetData
                 where StrainID in {}
                 and id in (SELECT ProbeSetXRef.DataId
@@ -714,9 +717,10 @@ class DataSet:
                 and ProbeSet.Id = ProbeSetXRef.ProbeSetId)""".format(create_in_clause(sample_ids), self.name)
 
         query_results = list(g.db.execute(query).fetchall())
-
         data_results = self.chunk_dataset(query_results, len(sample_ids))
+        self.samplelist = sorted_samplelist
         self.trait_data = data_results
+        
 
     def get_trait_data(self, sample_list=None):
         if sample_list:
-- 
cgit v1.2.3


From beea9ac15e531c32b2368e04261e521d8a03c8c8 Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Mon, 14 Jun 2021 00:54:55 +0300
Subject: minor refactoring

---
 wqflask/wqflask/correlation/correlation_gn3_api.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index 5fa33027..d0ee7a8b 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -146,11 +146,7 @@ def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset):
 
     sample_data = process_samples(
         start_vars, this_dataset.group.samplelist)
-
-    # sample_data = test_process_data(this_trait,this_dataset,start_vars)
-
     if target_dataset.type == "ProbeSet":
-        # pass
         target_dataset.get_probeset_data(list(sample_data.keys()))
     else:
         target_dataset.get_trait_data(list(sample_data.keys()))
@@ -159,9 +155,6 @@ def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset):
         "trait_sample_data": sample_data,
         "trait_id": start_vars["trait_id"]
     }
-
-    # should remove this  len(samplelist) == len(strain_values)
-
     results = map_shared_keys_to_values(
         target_dataset.samplelist, target_dataset.trait_data)
 
-- 
cgit v1.2.3


From c499f3418efe0a8bdb8e4800426ed6a06f9bb579 Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Mon, 14 Jun 2021 01:15:11 +0300
Subject: add f1list and parlist to primary_samples

---
 wqflask/wqflask/correlation/correlation_gn3_api.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index d0ee7a8b..86531fa2 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -146,6 +146,10 @@ def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset):
 
     sample_data = process_samples(
         start_vars, this_dataset.group.samplelist)
+
+    sample_data = test_process_data(this_trait, this_dataset, start_vars)
+
+
     if target_dataset.type == "ProbeSet":
         target_dataset.get_probeset_data(list(sample_data.keys()))
     else:
-- 
cgit v1.2.3


From 8521ba973ccf14b6918948a93cfde07d6cf1e27a Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Mon, 14 Jun 2021 09:28:46 +0300
Subject: delete loggers and comments

---
 wqflask/base/data_set.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 991c9fee..4cb82665 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -440,7 +440,6 @@ class DatasetGroup:
         # genotype_1 is Dataset Object without parents and f1
         # genotype_2 is Dataset Object with parents and f1 (not for intercross)
 
-        # genotype_1 = reaper.Dataset()
 
         # reaper barfs on unicode filenames, so here we ensure it's a string
         if self.genofile:
@@ -1128,7 +1127,6 @@ class MrnaAssayDataSet(DataSet):
             """ % (escape(str(this_trait.dataset.id)),
                    escape(this_trait.name)))
 
-            # logger.debug("query is:", pf(query))
             logger.sql(query)
             result = g.db.execute(query).fetchone()
 
@@ -1183,7 +1181,6 @@ class MrnaAssayDataSet(DataSet):
                     """ % (escape(trait), escape(self.name))
         logger.sql(query)
         results = g.db.execute(query).fetchall()
-        # logger.debug("RETRIEVED RESULTS HERE:", results)
         return results
 
     def retrieve_genes(self, column_name):
@@ -1224,7 +1221,6 @@ class TempDataSet(DataSet):
 
 def geno_mrna_confidentiality(ob):
     dataset_table = ob.type + "Freeze"
-    # logger.debug("dataset_table [%s]: %s" % (type(dataset_table), dataset_table))
 
     query = '''SELECT Id, Name, FullName, confidentiality,
                         AuthorisedUsers FROM %s WHERE Name = "%s"''' % (dataset_table, ob.name)
-- 
cgit v1.2.3


From 2f88a976ce4c732f2e8c06939dda325e80fb1f89 Mon Sep 17 00:00:00 2001
From: BonfaceKilz
Date: Thu, 17 Jun 2021 10:09:38 +0300
Subject: doc: rpy2-performance: Replace begin_export with begin_src

---
 doc/rpy2-performance.org | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/doc/rpy2-performance.org b/doc/rpy2-performance.org
index 29b99ba0..ddcac81a 100644
--- a/doc/rpy2-performance.org
+++ b/doc/rpy2-performance.org
@@ -21,7 +21,7 @@ server:
 
 However, when loading the homepage, I occasionally ran into this trace:
 
-#+begin_export ascii
+#+begin_src
 DEBUG:wqflask.views:.check_access_permissions: @app.before_request check_access_permissions
 DEBUG:wqflask.views:.shutdown_session: remove db_session
 WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: Error: ignoring SIGPIPE signal
@@ -58,7 +58,7 @@ Possible actions:
 
 Selection:
 
-#+end_export
+#+end_src
 
 This blocks the flask service. Seems to be related to: [[https://github.com/rpy2/rpy2/issues/769][rpy2-issue#769]]
 and [[https://github.com/rpy2/rpy2/issues/809][rpy2-issue#809]]. I tried to reproduce this problem using some endpoint:
@@ -80,7 +80,7 @@ and [[https://github.com/rpy2/rpy2/issues/809][rpy2-issue#809]]. I tried to repr
 
 which generates this trace:
 
-#+begin_export ascii
+#+begin_src
 /home/bonface/opt/python3-genenetwork2/lib/python3.8/site-packages/rpy2/rinterface.py:955: UserWarning: R is not initialized by the main thread.
               Its taking over SIGINT cannot be reversed here, and as a
               consequence the embedded R cannot be interrupted with Ctrl-C.
@@ -89,7 +89,7 @@ which generates this trace:
 warnings.warn(
 DEBUG:wqflask.views:.shutdown_session: remove db_session
 
-#+end_export
+#+end_src
 
 Modifying the endpoint to:
 
@@ -108,7 +108,7 @@ Modifying the endpoint to:
 
 and refreshing the page a couple of times, I get:
 
-#+begin_export ascii
+#+begin_src
 DEBUG:wqflask.views:.check_access_permissions: @app.before_request check_access_
 permissions
 Sleeping for 3 seconds
@@ -136,7 +136,7 @@ WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:  library '/home/bonfa
 ce/R/x86_64-unknown-linux-gnu-library/4.0' contains no packages
 
 WARNING:rpy2.rinterface_lib.callbacks:R[write to console]:
-*** caught segfault ***
+\*** caught segfault ***
 
 WARNING:rpy2.rinterface_lib.callbacks:R[write to console]: address (nil), cause
 'memory not mapped'
@@ -151,9 +151,11 @@ Possible actions:
 Selection: [2021-06-16 13:11:00 +0300] [18657] [INFO] Handling signal: winch
 [2021-06-16 13:11:00 +0300] [18657] [INFO] Handling signal: winch
 [2021-06-16 13:13:02 +0300] [18657] [INFO] Handling signal: winch
-#+end_export
+#+end_src
 
-However, this seems to be non-deterministic, in the sense that I can't really pin what causes the above. I've tried to write a Locust Test that simulates users hitting that endpoint:
+However, this seems to be non-deterministic, in the sense that I can't
+really pin what causes the above. I've tried to write a Locust Test
+that simulates users hitting that endpoint:
 
 #+begin_src python
 """Load test a single trait page"""
@@ -168,3 +170,4 @@ from locust import HttpUser, task, between
           """Fetch a single trait"""
           self.client.get("/test")
 #+end_src
+
-- 
cgit v1.2.3


From de645c34b18e14b0661b50410072f2f0f08355f2 Mon Sep 17 00:00:00 2001
From: BonfaceKilz
Date: Thu, 17 Jun 2021 10:11:53 +0300
Subject: docs: rpy2-performance: Indicate version of python-rpy2 being used

---
 doc/rpy2-performance.org | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/rpy2-performance.org b/doc/rpy2-performance.org
index ddcac81a..177c427e 100644
--- a/doc/rpy2-performance.org
+++ b/doc/rpy2-performance.org
@@ -1,10 +1,10 @@
 * Python-Rpy2 performance issues with genenetwork2
 
 At one point, genenetwork2 was down. A possible cause was that it
-wrote into the log file in an infinite loop due to rpy2, so a solution
-was to empty it. Currently, as a work around, rpy2 is disabled by
-removing it's imports. This affects WGCNA/ CTL imports and commenting
-out Biweight Midcorrelation option in the trait page. See:
+wrote into the log file in an infinite loop due to rpy2(v3.4.4), so a
+solution was to empty it. Currently, as a work around, rpy2 is
+disabled by removing it's imports. This affects WGCNA/ CTL imports and
+commenting out Biweight Midcorrelation option in the trait page. See:
 
 - [[https://github.com/genenetwork/genenetwork2/commit/1baf5f7611909c651483208184c5fbf7d4a7a088][1baf5f7]]
 - [[https://github.com/genenetwork/genenetwork2/commit/afee4d625248565857df98d3510f680ae6204864][afee4d6]]
-- 
cgit v1.2.3


From 4a6536e7a9b5830a91e17e9171ef699b68a8ddb5 Mon Sep 17 00:00:00 2001
From: BonfaceKilz
Date: Thu, 17 Jun 2021 10:20:39 +0300
Subject: doc: rpy2-performance: Add a section exploring possible solutions

---
 doc/rpy2-performance.org | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/rpy2-performance.org b/doc/rpy2-performance.org
index 177c427e..8f917ca0 100644
--- a/doc/rpy2-performance.org
+++ b/doc/rpy2-performance.org
@@ -171,3 +171,12 @@ from locust import HttpUser, task, between
           self.client.get("/test")
 #+end_src
 
+
+** A possible solution
+
+From this [[https://github.com/rpy2/rpy2/issues/809#issuecomment-845923975][comment]], a possible reason for the above traces, is that
+from Flask's end, a [[https://tldp.org/LDP/lpg/node20.html][SIGPIPE]] is somehow generated by our Python
+code. However, at this particular point, the R thread just happens to
+be running, and R can't handle this correctly. This seems to have been
+fixed in this [[https://github.com/rpy2/rpy2/pull/810][PR]] with a this [[https://github.com/rpy2/rpy2/issues/809#issuecomment-851618215][explanation]]. On our end, to have these
+changes, we have to update our python-rpy2 version.
-- 
cgit v1.2.3


From 9080fa91f072956fb0137e36b9ef1b45e7e03e34 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 9 Jul 2021 21:32:38 +0000
Subject: Changed compute_corr_for_top_results to also do sample correlation
 when literature or tissue correlation are selected

---
 wqflask/wqflask/correlation/correlation_gn3_api.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index 86531fa2..aea91220 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -227,8 +227,8 @@ def compute_correlation(start_vars, method="pearson", compute_all=False):
     correlation_results = correlation_results[0:corr_return_results]
 
     if (compute_all):
-
-        correlation_results = compute_corr_for_top_results(correlation_results,
+        correlation_results = compute_corr_for_top_results(start_vars,
+                                                           correlation_results,
                                                            this_trait,
                                                            this_dataset,
                                                            target_dataset,
@@ -242,7 +242,8 @@ def compute_correlation(start_vars, method="pearson", compute_all=False):
     return correlation_data
 
 
-def compute_corr_for_top_results(correlation_results,
+def compute_corr_for_top_results(start_vars,
+                                 correlation_results,
                                  this_trait,
                                  this_dataset,
                                  target_dataset,
@@ -265,8 +266,12 @@ def compute_corr_for_top_results(correlation_results,
             correlation_results = merge_correlation_results(
                 correlation_results, lit_result)
 
-    if corr_type != "sample":
-        pass
+    if corr_type != "sample" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet":
+        sample_result = sample_for_trait_lists(
+            correlation_results, target_dataset, this_trait, this_dataset, start_vars)
+        if sample_result:
+            correlation_results = merge_correlation_results(
+                correlation_results, sample_result)
 
     return correlation_results
 
-- 
cgit v1.2.3


From fd726da54c9e65c0721de6f8181c2dde93831a7a Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 9 Jul 2021 21:35:13 +0000
Subject: Only show literature and tissue correlation options if the trait's
 dataset is type ProbeSet

---
 wqflask/wqflask/templates/show_trait_calculate_correlations.html | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html
index e623a968..ffbe313c 100644
--- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html
+++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html
@@ -7,8 +7,10 @@
             <div class="col-xs-3 controls">
                 <select name="corr_type" class="form-control">
                     <option value="sample">Sample r</option>
+                    {% if dataset.type == 'ProbeSet' %}
                     <option value="lit">Literature r</option>
                     <option value="tissue">Tissue r</option>
+                    {% endif %}
                 </select>
             </div>
         </div>
-- 
cgit v1.2.3


From 8dcb6972e4be972a37defdf96415fda7ffaac3db Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 9 Jul 2021 21:41:11 +0000
Subject: Remove commented out function do_bicor

---
 wqflask/wqflask/correlation/show_corr_results.py | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 91716a6a..bebef9e7 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -158,23 +158,6 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe
 
     return json.dumps(results_list)
 
-# def do_bicor(this_trait_vals, target_trait_vals):
-#     r_library = ro.r["library"]             # Map the library function
-#     r_options = ro.r["options"]             # Map the options function
-
-#     r_library("WGCNA")
-#     r_bicor = ro.r["bicorAndPvalue"]        # Map the bicorAndPvalue function
-
-#     r_options(stringsAsFactors=False)
-
-#     this_vals = ro.Vector(this_trait_vals)
-#     target_vals = ro.Vector(target_trait_vals)
-
-#     the_r, the_p, _fisher_transform, _the_t, _n_obs = [
-#         numpy.asarray(x) for x in r_bicor(x=this_vals, y=target_vals)]
-
-#     return the_r, the_p
-
 
 def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_api=False):
     results_list = []
-- 
cgit v1.2.3