From 64ce38b45839b6305b009f6e28b0f852409e9bda Mon Sep 17 00:00:00 2001
From: Muriithi Frederick Muriuki
Date: Tue, 31 Aug 2021 10:45:11 +0300
Subject: Parse QTLReaper outputs

Issue:
https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi

* gn3/computations/qtlreaper.py: pass output files
* tests/unit/computations/data/qtlreaper/main_output_sample.txt: sample test
  data
* tests/unit/computations/data/qtlreaper/permu_output_sample.txt: sample test
  data
* tests/unit/computations/test_qtlreaper.py: add tests

  Add code to parse the QTLReaper output data files.
---
 tests/unit/computations/test_qtlreaper.py | 74 +++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 tests/unit/computations/test_qtlreaper.py

(limited to 'tests/unit/computations/test_qtlreaper.py')

diff --git a/tests/unit/computations/test_qtlreaper.py b/tests/unit/computations/test_qtlreaper.py
new file mode 100644
index 0000000..ec23664
--- /dev/null
+++ b/tests/unit/computations/test_qtlreaper.py
@@ -0,0 +1,74 @@
+"""Module contains tests for gn3.computations.qtlreaper"""
+import os
+from unittest import TestCase
+from gn3.computations.qtlreaper import (
+    parse_reaper_main_results, parse_reaper_permutation_results)
+
+class TestQTLReaper(TestCase):
+    """Class for testing qtlreaper interface functions."""
+
+    def test_parse_reaper_main_results(self):
+        self.assertEqual(
+            parse_reaper_main_results(
+                "tests/unit/computations/data/qtlreaper/main_output_sample.txt"),
+            [
+                {
+                    "ID": "T1", "Locus": "rs31443144", "Chr": 1, "cM": 1.500,
+                    "Mb": 3.010, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs6269442", "Chr": 1, "cM": 1.500,
+                    "Mb": 3.492, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs32285189", "Chr": 1, "cM": 1.630,
+                    "Mb": 3.511, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs258367496", "Chr": 1, "cM": 1.630,
+                    "Mb": 3.660, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs32430919", "Chr": 1, "cM": 1.750,
+                    "Mb": 3.777, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs36251697", "Chr": 1, "cM": 1.880,
+                    "Mb": 3.812, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs30658298", "Chr": 1, "cM": 2.010,
+                    "Mb": 4.431, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs51852623", "Chr": 1, "cM": 2.010,
+                    "Mb": 4.447, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs31879829", "Chr": 1, "cM": 2.140,
+                    "Mb": 4.519, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs36742481", "Chr": 1, "cM": 2.140,
+                    "Mb": 4.776, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                }
+            ])
+
+    def test_parse_reaper_permutation_results(self):
+        self.assertEqual(
+            parse_reaper_permutation_results(
+            "tests/unit/computations/data/qtlreaper/permu_output_sample.txt"),
+            [4.44174, 5.03825, 5.08167, 5.18119, 5.18578, 5.24563, 5.24619,
+             5.24619, 5.27961, 5.28228, 5.43903, 5.50188, 5.51694, 5.56830,
+             5.63874, 5.71346, 5.71936, 5.74275, 5.76764, 5.79815, 5.81671,
+             5.82775, 5.89659, 5.92117, 5.93396, 5.93396, 5.94957])
-- 
cgit v1.2.3


From b5e1d1176f1bf4f7c0b68b27beb15e99418f1650 Mon Sep 17 00:00:00 2001
From: Muriithi Frederick Muriuki
Date: Tue, 31 Aug 2021 11:16:29 +0300
Subject: Fix linting errors, minor bugs and reorganise code

* Fix some linting errors and some minor bugs caught by the linter.
  Move the `random_string` function to separate module for use in multiple
  places in the code.
---
 gn3/computations/heatmap.py               |  7 ++++---
 gn3/computations/qtlreaper.py             | 27 ++++++++++++++-------------
 gn3/db/traits.py                          |  5 ++++-
 gn3/heatmaps/heatmaps.py                  | 25 +++++++++++++++++++------
 gn3/random.py                             | 11 +++++++++++
 tests/unit/computations/test_qtlreaper.py |  5 +++--
 6 files changed, 55 insertions(+), 25 deletions(-)
 create mode 100644 gn3/random.py

(limited to 'tests/unit/computations/test_qtlreaper.py')

diff --git a/gn3/computations/heatmap.py b/gn3/computations/heatmap.py
index 92014cf..1143450 100644
--- a/gn3/computations/heatmap.py
+++ b/gn3/computations/heatmap.py
@@ -6,6 +6,7 @@ generate various kinds of heatmaps.
 from functools import reduce
 from typing import Any, Dict, Sequence
 from gn3.computations.slink import slink
+from gn3.computations.qtlreaper import generate_traits_file
 from gn3.computations.correlations2 import compute_correlation
 from gn3.db.genotypes import build_genotype_file, load_genotype_samples
 from gn3.db.traits import (
@@ -155,14 +156,14 @@ def heatmap_data(traits_names, conn: Any):
         for fullname in traits_names]
     traits_list = tuple(x[0] for x in traits_details)
     traits_data_list = [x[1] for x in traits_details]
-    exported_traits_data_list = tuple(
-        export_trait_data(td, strainlist) for td in traits_data_list)
     genotype_filename = build_genotype_file(traits_list[0]["riset"])
     strainlist = load_genotype_samples(genotype_filename)
+    exported_traits_data_list = tuple(
+        export_trait_data(td, strainlist) for td in traits_data_list)
     slink_data = slink(cluster_traits(exported_traits_data_list))
     ordering_data = compute_heatmap_order(slink_data)
     strains_and_values = retrieve_strains_and_values(
-        orders, strainlist, exported_traits_data_list)
+        ordering_data, strainlist, exported_traits_data_list)
     strains_values = strains_and_values[0][1]
     trait_values = [t[2] for t in strains_and_values]
     traits_filename = generate_traits_filename()
diff --git a/gn3/computations/qtlreaper.py b/gn3/computations/qtlreaper.py
index 3b8e4db..30c7051 100644
--- a/gn3/computations/qtlreaper.py
+++ b/gn3/computations/qtlreaper.py
@@ -3,17 +3,10 @@ This module contains functions to interact with the `qtlreaper` utility for
 computation of QTLs.
 """
 import os
-import random
-import string
 import subprocess
+from gn3.random import random_string
 from gn3.settings import TMPDIR, REAPER_COMMAND
 
-def random_string(length):
-    """Generate a random string of length `length`."""
-    return "".join(
-        random.choices(
-            string.ascii_letters + string.digits, k=length))
-
 def generate_traits_file(strains, trait_values, traits_filename):
     """
     Generate a traits file for use with `qtlreaper`.
@@ -25,11 +18,13 @@ def generate_traits_file(strains, trait_values, traits_filename):
         computation of QTLs.
     """
     header = "Trait\t{}\n".format("\t".join(strains))
-    data = [header] + [
-        "T{}\t{}\n".format(i+1, "\t".join([str(i) for i in t]))
-        for i, t in enumerate(trait_values[:-1])] + [
-        "T{}\t{}".format(len(trait_values), "\t".join([str(i) for i in t]))
-        for t in trait_values[-1:]]
+    data = (
+        [header] +
+        ["T{}\t{}\n".format(i+1, "\t".join([str(i) for i in t]))
+         for i, t in enumerate(trait_values[:-1])] +
+        ["T{}\t{}".format(
+            len(trait_values), "\t".join([str(i) for i in t]))
+         for t in trait_values[-1:]])
     with open(traits_filename, "w") as outfile:
         outfile.writelines(data)
 
@@ -93,6 +88,9 @@ def run_reaper(
 
 
 def parse_reaper_main_results(results_file):
+    """
+    Parse the results file of running QTLReaper into a list of dicts.
+    """
     with open(results_file, "r") as infile:
         lines = infile.readlines()
 
@@ -104,6 +102,9 @@ def parse_reaper_main_results(results_file):
     return [dict(zip(header, __parse_line(line))) for line in lines[1:]]
 
 def parse_reaper_permutation_results(results_file):
+    """
+    Parse the results QTLReaper permutations into a list of values.
+    """
     with open(results_file, "r") as infile:
         lines = infile.readlines()
 
diff --git a/gn3/db/traits.py b/gn3/db/traits.py
index ccb101a..bfe887e 100644
--- a/gn3/db/traits.py
+++ b/gn3/db/traits.py
@@ -1,6 +1,8 @@
 """This class contains functions relating to trait data manipulation"""
-from gn3.settings import TMPDIR
+import os
 from typing import Any, Dict, Union, Sequence
+from gn3.settings import TMPDIR
+from gn3.random import random_string
 from gn3.function_helpers import compose
 from gn3.db.datasets import retrieve_trait_dataset
 
@@ -669,5 +671,6 @@ def retrieve_trait_data(trait: dict, conn: Any, strainlist: Sequence[str] = tupl
     return {}
 
 def generate_traits_filename(base_path: str = TMPDIR):
+    """Generate a unique filename for use with generated traits files."""
     return "{}/traits_test_file_{}.txt".format(
         os.path.abspath(base_path), random_string(10))
diff --git a/gn3/heatmaps/heatmaps.py b/gn3/heatmaps/heatmaps.py
index 3bf7917..88f546d 100644
--- a/gn3/heatmaps/heatmaps.py
+++ b/gn3/heatmaps/heatmaps.py
@@ -14,6 +14,19 @@ def generate_random_data(data_stop: float = 2, width: int = 10, height: int = 30
     return [[random.uniform(0,data_stop) for i in range(0, width)]
             for j in range(0, height)]
 
+def generate_random_data2(data_stop: float = 2, width: int = 10, height: int = 30):
+    """
+    This is mostly a utility function to be used to generate random data, useful
+    for development of the heatmap generation code, without access to the actual
+    database data.
+    """
+    return [
+        [{
+            "value": item,
+            "category": random.choice(["C57BL/6J +", "DBA/2J +"])}
+         for item in axis]
+        for axis in generate_random_data(data_stop, width, height)]
+
 def heatmap_x_axis_names():
     return [
         "UCLA_BXDBXH_CARTILAGE_V2::ILM103710672",
@@ -30,13 +43,14 @@ def heatmap_x_axis_names():
 
 # Grey + Blue + Red
 def generate_heatmap():
-    rows = 20
-    data = generate_random_data(height=rows)
-    y = (["%s"%x for x in range(1, rows+1)][:-1] + ["X"]) #replace last item with x for now
+    cols = 20
+    y_axis = (["%s"%x for x in range(1, cols+1)][:-1] + ["X"]) #replace last item with x for now
+    x_axis = heatmap_x_axis_names()
+    data = generate_random_data(height=cols, width=len(x_axis))
     fig = px.imshow(
         data,
-        x=heatmap_x_axis_names(),
-        y=y,
+        x=x_axis,
+        y=y_axis,
         width=500)
     fig.update_traces(xtype="array")
     fig.update_traces(ytype="array")
@@ -49,6 +63,5 @@ def generate_heatmap():
         coloraxis_colorscale=[
             [0.0, '#3B3B3B'], [0.4999999999999999, '#ABABAB'],
             [0.5, '#F5DE11'], [1.0, '#FF0D00']])
-
     fig.write_html("%s/%s"%(heatmap_dir, "test_image.html"))
     return fig
diff --git a/gn3/random.py b/gn3/random.py
new file mode 100644
index 0000000..f0ba574
--- /dev/null
+++ b/gn3/random.py
@@ -0,0 +1,11 @@
+"""
+Functions to generate complex random data.
+"""
+import random
+import string
+
+def random_string(length):
+    """Generate a random string of length `length`."""
+    return "".join(
+        random.choices(
+            string.ascii_letters + string.digits, k=length))
diff --git a/tests/unit/computations/test_qtlreaper.py b/tests/unit/computations/test_qtlreaper.py
index ec23664..6c3b64d 100644
--- a/tests/unit/computations/test_qtlreaper.py
+++ b/tests/unit/computations/test_qtlreaper.py
@@ -1,5 +1,4 @@
 """Module contains tests for gn3.computations.qtlreaper"""
-import os
 from unittest import TestCase
 from gn3.computations.qtlreaper import (
     parse_reaper_main_results, parse_reaper_permutation_results)
@@ -8,6 +7,7 @@ class TestQTLReaper(TestCase):
     """Class for testing qtlreaper interface functions."""
 
     def test_parse_reaper_main_results(self):
+        """Test that the main results file is parsed correctly."""
         self.assertEqual(
             parse_reaper_main_results(
                 "tests/unit/computations/data/qtlreaper/main_output_sample.txt"),
@@ -65,9 +65,10 @@ class TestQTLReaper(TestCase):
             ])
 
     def test_parse_reaper_permutation_results(self):
+        """Test that the permutations results file is parsed correctly."""
         self.assertEqual(
             parse_reaper_permutation_results(
-            "tests/unit/computations/data/qtlreaper/permu_output_sample.txt"),
+                "tests/unit/computations/data/qtlreaper/permu_output_sample.txt"),
             [4.44174, 5.03825, 5.08167, 5.18119, 5.18578, 5.24563, 5.24619,
              5.24619, 5.27961, 5.28228, 5.43903, 5.50188, 5.51694, 5.56830,
              5.63874, 5.71346, 5.71936, 5.74275, 5.76764, 5.79815, 5.81671,
-- 
cgit v1.2.3


From 679a1af832ad9585c7cf72996043edb08e1b0d10 Mon Sep 17 00:00:00 2001
From: Muriithi Frederick Muriuki
Date: Mon, 6 Sep 2021 08:06:14 +0300
Subject: Leave "Chr" value as string when parsing

Issue:
https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi

* The "Chr" value seems to be mostly a name of some sort, despite it being,
  seemingly an number. This commit parses the "Chr" value as a string.
  It also updates the tests to expec a string, rather than a number for "Chr"
  values.
---
 gn3/computations/qtlreaper.py             |  5 +++--
 tests/unit/computations/test_qtlreaper.py | 20 ++++++++++----------
 2 files changed, 13 insertions(+), 12 deletions(-)

(limited to 'tests/unit/computations/test_qtlreaper.py')

diff --git a/gn3/computations/qtlreaper.py b/gn3/computations/qtlreaper.py
index eff2a80..9b20309 100644
--- a/gn3/computations/qtlreaper.py
+++ b/gn3/computations/qtlreaper.py
@@ -94,7 +94,7 @@ def parse_reaper_main_results(results_file):
     with open(results_file, "r") as infile:
         lines = infile.readlines()
 
-    def __parse_column_value(value):
+    def __parse_column_float_value(value):
         try:
             return float(value)
         except:
@@ -102,7 +102,8 @@ def parse_reaper_main_results(results_file):
 
     def __parse_line(line):
         items = line.strip().split("\t")
-        return items[0:2] + [__parse_column_value(item) for item in items[2:]]
+        return items[0:3] + [
+            __parse_column_float_value(item) for item in items[3:]]
 
     header = lines[0].strip().split("\t")
     return [dict(zip(header, __parse_line(line))) for line in lines[1:]]
diff --git a/tests/unit/computations/test_qtlreaper.py b/tests/unit/computations/test_qtlreaper.py
index 6c3b64d..fd3434a 100644
--- a/tests/unit/computations/test_qtlreaper.py
+++ b/tests/unit/computations/test_qtlreaper.py
@@ -13,52 +13,52 @@ class TestQTLReaper(TestCase):
                 "tests/unit/computations/data/qtlreaper/main_output_sample.txt"),
             [
                 {
-                    "ID": "T1", "Locus": "rs31443144", "Chr": 1, "cM": 1.500,
+                    "ID": "T1", "Locus": "rs31443144", "Chr": "1", "cM": 1.500,
                     "Mb": 3.010, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs6269442", "Chr": 1, "cM": 1.500,
+                    "ID": "T1", "Locus": "rs6269442", "Chr": "1", "cM": 1.500,
                     "Mb": 3.492, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs32285189", "Chr": 1, "cM": 1.630,
+                    "ID": "T1", "Locus": "rs32285189", "Chr": "1", "cM": 1.630,
                     "Mb": 3.511, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs258367496", "Chr": 1, "cM": 1.630,
+                    "ID": "T1", "Locus": "rs258367496", "Chr": "1", "cM": 1.630,
                     "Mb": 3.660, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs32430919", "Chr": 1, "cM": 1.750,
+                    "ID": "T1", "Locus": "rs32430919", "Chr": "1", "cM": 1.750,
                     "Mb": 3.777, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs36251697", "Chr": 1, "cM": 1.880,
+                    "ID": "T1", "Locus": "rs36251697", "Chr": "1", "cM": 1.880,
                     "Mb": 3.812, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs30658298", "Chr": 1, "cM": 2.010,
+                    "ID": "T1", "Locus": "rs30658298", "Chr": "1", "cM": 2.010,
                     "Mb": 4.431, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs51852623", "Chr": 1, "cM": 2.010,
+                    "ID": "T1", "Locus": "rs51852623", "Chr": "1", "cM": 2.010,
                     "Mb": 4.447, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs31879829", "Chr": 1, "cM": 2.140,
+                    "ID": "T1", "Locus": "rs31879829", "Chr": "1", "cM": 2.140,
                     "Mb": 4.519, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs36742481", "Chr": 1, "cM": 2.140,
+                    "ID": "T1", "Locus": "rs36742481", "Chr": "1", "cM": 2.140,
                     "Mb": 4.776, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 }
-- 
cgit v1.2.3


From d4943f1d01d89a3928c905f80914a23144126c8e Mon Sep 17 00:00:00 2001
From: Muriithi Frederick Muriuki
Date: Mon, 6 Sep 2021 08:09:20 +0300
Subject: Provide function to organise parsed QTLReaper results

* gn3/computations/qtlreaper.py: Provide a function to organise the results by
  trait for easier use down the line.

* tests/unit/computations/test_qtlreaper.py: provide a test to ensure that the
  organising function works as expected.
---
 gn3/computations/qtlreaper.py             |  25 +++++++
 tests/unit/computations/test_qtlreaper.py | 105 +++++++++++++++++++++++++++++-
 2 files changed, 129 insertions(+), 1 deletion(-)

(limited to 'tests/unit/computations/test_qtlreaper.py')

diff --git a/gn3/computations/qtlreaper.py b/gn3/computations/qtlreaper.py
index 9b20309..8c0e6de 100644
--- a/gn3/computations/qtlreaper.py
+++ b/gn3/computations/qtlreaper.py
@@ -86,6 +86,31 @@ def run_reaper(
     subprocess.run(command_list, check=True)
     return (output_filename, permu_output_filename)
 
+def organise_reaper_main_results(parsed_results):
+    def __organise_by_chromosome(chr_name, items):
+        chr_items = [item for item in items if item["Chr"] == chr_name]
+        return {
+            "Chr": str(chr_name),
+            "loci": [{
+                "Locus": locus["Locus"],
+                "cM": locus["cM"],
+                "Mb": locus["Mb"],
+                "LRS": locus["LRS"],
+                "Additive": locus["Additive"],
+                "pValue": locus["pValue"]
+            } for locus in chr_items]}
+
+    def __organise_by_id(identifier, items):
+        id_items = [item for item in items if item["ID"] == identifier]
+        unique_chromosomes = {item["Chr"] for item in id_items}
+        return {
+            "ID": identifier,
+            "chromosomes": [
+                __organise_by_chromosome(chromo, id_items)
+                for chromo in sorted(unique_chromosomes)]}
+
+    unique_ids = {res["ID"] for res in parsed_results}
+    return [__organise_by_id(_id, parsed_results) for _id in sorted(unique_ids)]
 
 def parse_reaper_main_results(results_file):
     """
diff --git a/tests/unit/computations/test_qtlreaper.py b/tests/unit/computations/test_qtlreaper.py
index fd3434a..1d7347f 100644
--- a/tests/unit/computations/test_qtlreaper.py
+++ b/tests/unit/computations/test_qtlreaper.py
@@ -1,7 +1,9 @@
 """Module contains tests for gn3.computations.qtlreaper"""
 from unittest import TestCase
 from gn3.computations.qtlreaper import (
-    parse_reaper_main_results, parse_reaper_permutation_results)
+    parse_reaper_main_results,
+    organise_reaper_main_results,
+    parse_reaper_permutation_results)
 
 class TestQTLReaper(TestCase):
     """Class for testing qtlreaper interface functions."""
@@ -73,3 +75,104 @@ class TestQTLReaper(TestCase):
              5.24619, 5.27961, 5.28228, 5.43903, 5.50188, 5.51694, 5.56830,
              5.63874, 5.71346, 5.71936, 5.74275, 5.76764, 5.79815, 5.81671,
              5.82775, 5.89659, 5.92117, 5.93396, 5.93396, 5.94957])
+
+    def test_organise_reaper_main_results(self):
+        self.assertEqual(
+            organise_reaper_main_results([
+                {
+                    "ID": "T1", "Locus": "rs31443144", "Chr": 1, "cM": 1.500,
+                    "Mb": 3.010, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs6269442", "Chr": 1, "cM": 1.500,
+                    "Mb": 3.492, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs32285189", "Chr": 1, "cM": 1.630,
+                    "Mb": 3.511, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs258367496", "Chr": 1, "cM": 1.630,
+                    "Mb": 3.660, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs32430919", "Chr": 1, "cM": 1.750,
+                    "Mb": 3.777, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs36251697", "Chr": 1, "cM": 1.880,
+                    "Mb": 3.812, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs30658298", "Chr": 1, "cM": 2.010,
+                    "Mb": 4.431, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs51852623", "Chr": 2, "cM": 2.010,
+                    "Mb": 4.447, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs31879829", "Chr": 2, "cM": 2.140,
+                    "Mb": 4.519, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                },
+                {
+                    "ID": "T1", "Locus": "rs36742481", "Chr": 2, "cM": 2.140,
+                    "Mb": 4.776, "LRS": 0.500, "Additive": -0.074,
+                    "pValue": 1.000
+                }
+            ]),
+            [{"ID": "T1",
+              "chromosomes": [
+                  {"Chr": "1",
+                   "loci": [
+                       {
+                           "Locus": "rs31443144",  "cM": 1.500, "Mb": 3.010,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       },
+                       {
+                           "Locus": "rs6269442", "cM": 1.500, "Mb": 3.492,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       },
+                       {
+                           "Locus": "rs32285189", "cM": 1.630, "Mb": 3.511,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       },
+                       {
+                           "Locus": "rs258367496", "cM": 1.630, "Mb": 3.660,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       },
+                       {
+                           "Locus": "rs32430919", "cM": 1.750, "Mb": 3.777,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       },
+                       {
+                           "Locus": "rs36251697", "cM": 1.880, "Mb": 3.812,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       },
+                       {
+                           "Locus": "rs30658298", "cM": 2.010, "Mb": 4.431,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       }]},
+                  {"Chr": "2",
+                   "loci": [
+                       {
+                           "Locus": "rs51852623", "cM": 2.010, "Mb": 4.447,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       },
+                       {
+                           "Locus": "rs31879829", "cM": 2.140, "Mb": 4.519,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       },
+                       {
+                           "Locus": "rs36742481", "cM": 2.140, "Mb": 4.776,
+                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                       }]}]}])
-- 
cgit v1.2.3


From a718069c757bea9f7ecbaee25e23bd581750f906 Mon Sep 17 00:00:00 2001
From: Muriithi Frederick Muriuki
Date: Wed, 8 Sep 2021 10:56:56 +0300
Subject: Ease search for traits and chromosomes

Issue:
https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi

* Return a dict of values rather than list for the traits and chromosomes to
  ease searching through the data.
---
 gn3/computations/qtlreaper.py             |  9 ++-
 tests/unit/computations/test_qtlreaper.py | 92 +++++++++++++++----------------
 2 files changed, 52 insertions(+), 49 deletions(-)

(limited to 'tests/unit/computations/test_qtlreaper.py')

diff --git a/gn3/computations/qtlreaper.py b/gn3/computations/qtlreaper.py
index 02d6572..5180853 100644
--- a/gn3/computations/qtlreaper.py
+++ b/gn3/computations/qtlreaper.py
@@ -110,12 +110,15 @@ def organise_reaper_main_results(parsed_results):
         unique_chromosomes = {item["Chr"] for item in id_items}
         return {
             "ID": identifier,
-            "chromosomes": [
+            "chromosomes": {_chr["Chr"]: _chr for _chr in [
                 __organise_by_chromosome(chromo, id_items)
-                for chromo in sorted(unique_chromosomes)]}
+                for chromo in sorted(
+                        unique_chromosomes, key=chromosome_sorter_key_fn)]}}
 
     unique_ids = {res["ID"] for res in parsed_results}
-    return [__organise_by_id(_id, parsed_results) for _id in sorted(unique_ids)]
+    return {
+        trait["ID"]: trait for trait in
+        [__organise_by_id(_id, parsed_results) for _id in sorted(unique_ids)]}
 
 def parse_reaper_main_results(results_file):
     """
diff --git a/tests/unit/computations/test_qtlreaper.py b/tests/unit/computations/test_qtlreaper.py
index 1d7347f..495ed97 100644
--- a/tests/unit/computations/test_qtlreaper.py
+++ b/tests/unit/computations/test_qtlreaper.py
@@ -130,49 +130,49 @@ class TestQTLReaper(TestCase):
                     "pValue": 1.000
                 }
             ]),
-            [{"ID": "T1",
-              "chromosomes": [
-                  {"Chr": "1",
-                   "loci": [
-                       {
-                           "Locus": "rs31443144",  "cM": 1.500, "Mb": 3.010,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       },
-                       {
-                           "Locus": "rs6269442", "cM": 1.500, "Mb": 3.492,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       },
-                       {
-                           "Locus": "rs32285189", "cM": 1.630, "Mb": 3.511,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       },
-                       {
-                           "Locus": "rs258367496", "cM": 1.630, "Mb": 3.660,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       },
-                       {
-                           "Locus": "rs32430919", "cM": 1.750, "Mb": 3.777,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       },
-                       {
-                           "Locus": "rs36251697", "cM": 1.880, "Mb": 3.812,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       },
-                       {
-                           "Locus": "rs30658298", "cM": 2.010, "Mb": 4.431,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       }]},
-                  {"Chr": "2",
-                   "loci": [
-                       {
-                           "Locus": "rs51852623", "cM": 2.010, "Mb": 4.447,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       },
-                       {
-                           "Locus": "rs31879829", "cM": 2.140, "Mb": 4.519,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       },
-                       {
-                           "Locus": "rs36742481", "cM": 2.140, "Mb": 4.776,
-                           "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                       }]}]}])
+            {"T1": {"ID": "T1",
+                    "chromosomes": {
+                        1: {"Chr": 1,
+                            "loci": [
+                                {
+                                    "Locus": "rs31443144",  "cM": 1.500, "Mb": 3.010,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                },
+                                {
+                                    "Locus": "rs6269442", "cM": 1.500, "Mb": 3.492,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                },
+                                {
+                                    "Locus": "rs32285189", "cM": 1.630, "Mb": 3.511,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                },
+                                {
+                                    "Locus": "rs258367496", "cM": 1.630, "Mb": 3.660,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                },
+                                {
+                                    "Locus": "rs32430919", "cM": 1.750, "Mb": 3.777,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                },
+                                {
+                                    "Locus": "rs36251697", "cM": 1.880, "Mb": 3.812,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                },
+                                {
+                                    "Locus": "rs30658298", "cM": 2.010, "Mb": 4.431,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                }]},
+                        2: {"Chr": 2,
+                            "loci": [
+                                {
+                                    "Locus": "rs51852623", "cM": 2.010, "Mb": 4.447,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                },
+                                {
+                                    "Locus": "rs31879829", "cM": 2.140, "Mb": 4.519,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                },
+                                {
+                                    "Locus": "rs36742481", "cM": 2.140, "Mb": 4.776,
+                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                                }]}}}})
-- 
cgit v1.2.3


From f17b489c8eb94050b81b1a59fb43954d036f7c38 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Wed, 15 Sep 2021 06:01:44 +0300
Subject: Fix format of arguments and expected values

* tests/unit/computations/test_heatmap.py: ordering is not longer provided as
  a list of tuples; the ordering values are just a list of numbers now. This
  commit updates the test to take this into consideration.

* tests/unit/computations/test_qtlreaper.py: the 'Chr' value if numeric, is
  represented by an actual number, not a string. This commit updates the code
  to take this into consideration.
---
 tests/unit/computations/test_heatmap.py   |  8 ++++----
 tests/unit/computations/test_qtlreaper.py | 20 ++++++++++----------
 2 files changed, 14 insertions(+), 14 deletions(-)

(limited to 'tests/unit/computations/test_qtlreaper.py')

diff --git a/tests/unit/computations/test_heatmap.py b/tests/unit/computations/test_heatmap.py
index f1bbefc..156af45 100644
--- a/tests/unit/computations/test_heatmap.py
+++ b/tests/unit/computations/test_heatmap.py
@@ -165,22 +165,22 @@ class TestHeatmap(TestCase):
         """Test retrieval of strains and values."""
         for orders, slist, tdata, expected in [
                 [
-                    [(60, 2)],
+                    [2],
                     ["s1", "s2", "s3", "s4"],
                     [[2, 9, 6, None, 4],
                      [7, 5, None, None, 4],
                      [9, None, 5, 4, 7],
                      [6, None, None, 4, None]],
-                    [[(60, 2), ["s1", "s3", "s4"], [9, 5, 4]]]
+                    [[2, ["s1", "s3", "s4"], [9, 5, 4]]]
                 ],
                 [
-                    [(60, 3)],
+                    [3],
                     ["s1", "s2", "s3", "s4", "s5"],
                     [[2, 9, 6, None, 4],
                      [7, 5, None, None, 4],
                      [9, None, 5, 4, 7],
                      [6, None, None, 4, None]],
-                    [[(60, 3), ["s1", "s4"], [6, 4]]]
+                    [[3, ["s1", "s4"], [6, 4]]]
                 ]]:
             with self.subTest(strainlist=slist, traitdata=tdata):
                 self.assertEqual(
diff --git a/tests/unit/computations/test_qtlreaper.py b/tests/unit/computations/test_qtlreaper.py
index 495ed97..1d67827 100644
--- a/tests/unit/computations/test_qtlreaper.py
+++ b/tests/unit/computations/test_qtlreaper.py
@@ -15,52 +15,52 @@ class TestQTLReaper(TestCase):
                 "tests/unit/computations/data/qtlreaper/main_output_sample.txt"),
             [
                 {
-                    "ID": "T1", "Locus": "rs31443144", "Chr": "1", "cM": 1.500,
+                    "ID": "T1", "Locus": "rs31443144", "Chr": 1, "cM": 1.500,
                     "Mb": 3.010, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs6269442", "Chr": "1", "cM": 1.500,
+                    "ID": "T1", "Locus": "rs6269442", "Chr": 1, "cM": 1.500,
                     "Mb": 3.492, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs32285189", "Chr": "1", "cM": 1.630,
+                    "ID": "T1", "Locus": "rs32285189", "Chr": 1, "cM": 1.630,
                     "Mb": 3.511, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs258367496", "Chr": "1", "cM": 1.630,
+                    "ID": "T1", "Locus": "rs258367496", "Chr": 1, "cM": 1.630,
                     "Mb": 3.660, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs32430919", "Chr": "1", "cM": 1.750,
+                    "ID": "T1", "Locus": "rs32430919", "Chr": 1, "cM": 1.750,
                     "Mb": 3.777, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs36251697", "Chr": "1", "cM": 1.880,
+                    "ID": "T1", "Locus": "rs36251697", "Chr": 1, "cM": 1.880,
                     "Mb": 3.812, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs30658298", "Chr": "1", "cM": 2.010,
+                    "ID": "T1", "Locus": "rs30658298", "Chr": 1, "cM": 2.010,
                     "Mb": 4.431, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs51852623", "Chr": "1", "cM": 2.010,
+                    "ID": "T1", "Locus": "rs51852623", "Chr": 1, "cM": 2.010,
                     "Mb": 4.447, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs31879829", "Chr": "1", "cM": 2.140,
+                    "ID": "T1", "Locus": "rs31879829", "Chr": 1, "cM": 2.140,
                     "Mb": 4.519, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs36742481", "Chr": "1", "cM": 2.140,
+                    "ID": "T1", "Locus": "rs36742481", "Chr": 1, "cM": 2.140,
                     "Mb": 4.776, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 }
-- 
cgit v1.2.3


From 1e2357049adc72808fbf8eaac3da9411d3c78c66 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Fri, 17 Sep 2021 11:20:16 +0300
Subject: Fix a number of linting issues

Issue:
https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi
---
 gn3/computations/qtlreaper.py             |  7 ++--
 gn3/db/genotypes.py                       |  2 +-
 gn3/heatmaps.py                           | 54 ++++++++++++-------------------
 tests/unit/computations/test_qtlreaper.py |  3 +-
 tests/unit/test_heatmaps.py               |  6 ++--
 5 files changed, 32 insertions(+), 40 deletions(-)

(limited to 'tests/unit/computations/test_qtlreaper.py')

diff --git a/gn3/computations/qtlreaper.py b/gn3/computations/qtlreaper.py
index 5180853..377db9b 100644
--- a/gn3/computations/qtlreaper.py
+++ b/gn3/computations/qtlreaper.py
@@ -110,9 +110,10 @@ def organise_reaper_main_results(parsed_results):
         unique_chromosomes = {item["Chr"] for item in id_items}
         return {
             "ID": identifier,
-            "chromosomes": {_chr["Chr"]: _chr for _chr in [
-                __organise_by_chromosome(chromo, id_items)
-                for chromo in sorted(
+            "chromosomes": {
+                _chr["Chr"]: _chr for _chr in [
+                    __organise_by_chromosome(chromo, id_items)
+                    for chromo in sorted(
                         unique_chromosomes, key=chromosome_sorter_key_fn)]}}
 
     unique_ids = {res["ID"] for res in parsed_results}
diff --git a/gn3/db/genotypes.py b/gn3/db/genotypes.py
index b03d55c..9d052d9 100644
--- a/gn3/db/genotypes.py
+++ b/gn3/db/genotypes.py
@@ -174,7 +174,7 @@ def parse_genotype_file(filename: str, parlist: tuple = tuple()):
     geno_obj = dict(labels + header)
     markers = tuple(
         [parse_genotype_marker(line, geno_obj, parlist)
-        for line in data_lines[1:]])
+         for line in data_lines[1:]])
     chromosomes = tuple(
         dict(chromosome) for chromosome in
         build_genotype_chromosomes(geno_obj, markers))
diff --git a/gn3/heatmaps.py b/gn3/heatmaps.py
index 2859dde..c4fc67d 100644
--- a/gn3/heatmaps.py
+++ b/gn3/heatmaps.py
@@ -3,13 +3,13 @@ This module will contain functions to be used in computation of the data used to
 generate various kinds of heatmaps.
 """
 
+from typing import Any, Dict, Sequence
 import numpy as np
 from functools import reduce
 from gn3.settings import TMPDIR
 import plotly.graph_objects as go
 import plotly.figure_factory as ff
 from gn3.random import random_string
-from typing import Any, Dict, Sequence
 from gn3.computations.slink import slink
 from plotly.subplots import make_subplots
 from gn3.computations.correlations2 import compute_correlation
@@ -165,7 +165,7 @@ def build_heatmap(traits_names, conn: Any):
         for fullname in traits_names]
     traits_data_list = [retrieve_trait_data(t, conn) for t in traits]
     genotype_filename = build_genotype_file(traits[0]["riset"])
-    genotype = parse_genotype_file(genotype_filename)
+    # genotype = parse_genotype_file(genotype_filename)
     strains = load_genotype_samples(genotype_filename)
     exported_traits_data_list = [
         export_trait_data(td, strains) for td in traits_data_list]
@@ -183,22 +183,21 @@ def build_heatmap(traits_names, conn: Any):
         [t[2] for t in strains_and_values],
         traits_filename)
 
-    main_output, permutations_output = run_reaper(
+    main_output, _permutations_output = run_reaper(
         genotype_filename, traits_filename, separate_nperm_output=True)
 
     qtlresults = parse_reaper_main_results(main_output)
-    permudata = parse_reaper_permutation_results(permutations_output)
+    # permudata = parse_reaper_permutation_results(permutations_output)
     organised = organise_reaper_main_results(qtlresults)
 
     traits_ids = [# sort numerically, but retain the ids as strings
         str(i) for i in sorted({int(row["ID"]) for row in qtlresults})]
     chromosome_names = sorted(
-        {row["Chr"] for row in qtlresults}, key = chromosome_sorter_key_fn)
-    loci_names = sorted({row["Locus"] for row in qtlresults})
-    ordered_traits_names = {
-        res_id: trait for res_id, trait in
+        {row["Chr"] for row in qtlresults}, key=chromosome_sorter_key_fn)
+    # loci_names = sorted({row["Locus"] for row in qtlresults})
+    ordered_traits_names = dict(
         zip(traits_ids,
-            [traits[idx]["trait_fullname"] for idx in traits_order])}
+            [traits[idx]["trait_fullname"] for idx in traits_order]))
 
     return generate_clustered_heatmap(
         process_traits_data_for_heatmap(
@@ -207,22 +206,11 @@ def build_heatmap(traits_names, conn: Any):
         "single_heatmap_{}".format(random_string(10)),
         y_axis=tuple(
             ordered_traits_names[traits_ids[order]]
-                for order in traits_order),
+            for order in traits_order),
         y_label="Traits",
-        x_axis=[chromo for chromo in chromosome_names],
+        x_axis=chromosome_names,
         x_label="Chromosomes")
 
-    return {
-        "slink_data": slink_data,
-        "ordering_data": ordering_data,
-        "strainlist": strainlist,
-        "genotype_filename": genotype_filename,
-        "traits_list": traits_list,
-        "traits_data_list": traits_data_list,
-        "exported_traits_data_list": exported_traits_data_list,
-        "traits_filename": traits_filename
-    }
-
 def compute_traits_order(slink_data, neworder: tuple = tuple()):
     """
     Compute the order of the traits for clustering from `slink_data`.
@@ -314,7 +302,7 @@ def get_nearest_marker(traits_list, genotype):
     https://github.com/genenetwork/genenetwork1/blob/master/web/webqtl/heatmap/Heatmap.py#L419-L438
     """
     if not genotype["Mbmap"]:
-        return [None] * len(trait_list)
+        return [None] * len(traits_list)
 
     marker_finder = nearest_marker_finder(genotype)
     return [marker_finder(trait) for trait in traits_list]
@@ -340,10 +328,10 @@ def process_traits_data_for_heatmap(data, trait_names, chromosome_names):
     return hdata
 
 def generate_clustered_heatmap(
-        data, clustering_data, image_filename_prefix, x_axis = None,
-        x_label: str = "", y_axis = None, y_label: str = "",
+        data, clustering_data, image_filename_prefix, x_axis=None,
+        x_label: str = "", y_axis=None, y_label: str = "",
         output_dir: str = TMPDIR,
-        colorscale = (
+        colorscale=(
             (0.0, '#5D5D5D'), (0.4999999999999999, '#ABABAB'),
             (0.5, '#F5DE11'), (1.0, '#FF0D00'))):
     """
@@ -357,15 +345,15 @@ def generate_clustered_heatmap(
         shared_yaxes="rows",
         horizontal_spacing=0.001,
         subplot_titles=["distance"] + x_axis,
-        figure = ff.create_dendrogram(
+        figure=ff.create_dendrogram(
             np.array(clustering_data), orientation="right", labels=y_axis))
     hms = [go.Heatmap(
         name=chromo,
-        y = y_axis,
-        z = data_array,
+        y=y_axis,
+        z=data_array,
         showscale=False) for chromo, data_array in zip(x_axis, data)]
-    for i, hm in enumerate(hms):
-        fig.add_trace(hm, row=1, col=(i + 2))
+    for i, heatmap in enumerate(hms):
+        fig.add_trace(heatmap, row=1, col=(i + 2))
 
     fig.update_layout(
         {
@@ -380,8 +368,8 @@ def generate_clustered_heatmap(
     x_axes_layouts = {
         "xaxis{}".format(i+1 if i > 0 else ""): {
             "mirror": False,
-            "showticklabels": True if i==0 else False,
-            "ticks": "outside" if i==0 else ""
+            "showticklabels": True if i == 0 else False,
+            "ticks": "outside" if i == 0 else ""
         }
         for i in range(num_cols)}
 
diff --git a/tests/unit/computations/test_qtlreaper.py b/tests/unit/computations/test_qtlreaper.py
index 1d67827..d420470 100644
--- a/tests/unit/computations/test_qtlreaper.py
+++ b/tests/unit/computations/test_qtlreaper.py
@@ -77,6 +77,7 @@ class TestQTLReaper(TestCase):
              5.82775, 5.89659, 5.92117, 5.93396, 5.93396, 5.94957])
 
     def test_organise_reaper_main_results(self):
+        """Check that results are organised correctly."""
         self.assertEqual(
             organise_reaper_main_results([
                 {
@@ -135,7 +136,7 @@ class TestQTLReaper(TestCase):
                         1: {"Chr": 1,
                             "loci": [
                                 {
-                                    "Locus": "rs31443144",  "cM": 1.500, "Mb": 3.010,
+                                    "Locus": "rs31443144", "cM": 1.500, "Mb": 3.010,
                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
                                 },
                                 {
diff --git a/tests/unit/test_heatmaps.py b/tests/unit/test_heatmaps.py
index f3a81c5..c0a496b 100644
--- a/tests/unit/test_heatmaps.py
+++ b/tests/unit/test_heatmaps.py
@@ -189,6 +189,7 @@ class TestHeatmap(TestCase):
                     retrieve_strains_and_values(orders, slist, tdata), expected)
 
     def test_get_lrs_from_chr(self):
+        """Check that function gets correct LRS values"""
         for trait, chromosome, expected in [
                 [{"chromosomes": {}}, 3, [None]],
                 [{"chromosomes": {3: {"loci": [
@@ -202,6 +203,7 @@ class TestHeatmap(TestCase):
                 self.assertEqual(get_lrs_from_chr(trait, chromosome), expected)
 
     def test_process_traits_data_for_heatmap(self):
+        """Check for correct processing of data for heatmap generation."""
         self.assertEqual(
             process_traits_data_for_heatmap(
                 {"1": {
@@ -210,7 +212,7 @@ class TestHeatmap(TestCase):
                         1: {"Chr": 1,
                             "loci": [
                                 {
-                                    "Locus": "rs31443144",  "cM": 1.500, "Mb": 3.010,
+                                    "Locus": "rs31443144", "cM": 1.500, "Mb": 3.010,
                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
                                 },
                                 {
@@ -257,7 +259,7 @@ class TestHeatmap(TestCase):
                          1: {"Chr": 1,
                              "loci": [
                                  {
-                                     "Locus": "rs31443144",  "cM": 1.500, "Mb": 3.010,
+                                     "Locus": "rs31443144", "cM": 1.500, "Mb": 3.010,
                                      "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
                                  },
                                  {
-- 
cgit v1.2.3


From 95c5c0e73bffbf0287a17309e703063ee54d25ba Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Thu, 23 Sep 2021 03:45:19 +0300
Subject: Refactor: Move common sample data to separate file

Issue:
https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi

* Move common sample test data into a separate file where it can be imported
  from, to prevent pylint error R0801 which proved tricky to silence in any
  other way.
---
 tests/unit/computations/test_qtlreaper.py |  68 ++++--------------
 tests/unit/db/test_traits.py              |  15 ++--
 tests/unit/sample_test_data.py            | 111 ++++++++++++++++++++++++++++++
 tests/unit/test_heatmaps.py               |  96 +-------------------------
 4 files changed, 134 insertions(+), 156 deletions(-)
 create mode 100644 tests/unit/sample_test_data.py

(limited to 'tests/unit/computations/test_qtlreaper.py')

diff --git a/tests/unit/computations/test_qtlreaper.py b/tests/unit/computations/test_qtlreaper.py
index d420470..742d106 100644
--- a/tests/unit/computations/test_qtlreaper.py
+++ b/tests/unit/computations/test_qtlreaper.py
@@ -4,6 +4,7 @@ from gn3.computations.qtlreaper import (
     parse_reaper_main_results,
     organise_reaper_main_results,
     parse_reaper_permutation_results)
+from tests.unit.sample_test_data import organised_trait_1
 
 class TestQTLReaper(TestCase):
     """Class for testing qtlreaper interface functions."""
@@ -81,99 +82,54 @@ class TestQTLReaper(TestCase):
         self.assertEqual(
             organise_reaper_main_results([
                 {
-                    "ID": "T1", "Locus": "rs31443144", "Chr": 1, "cM": 1.500,
+                    "ID": "1", "Locus": "rs31443144", "Chr": 1, "cM": 1.500,
                     "Mb": 3.010, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs6269442", "Chr": 1, "cM": 1.500,
+                    "ID": "1", "Locus": "rs6269442", "Chr": 1, "cM": 1.500,
                     "Mb": 3.492, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs32285189", "Chr": 1, "cM": 1.630,
+                    "ID": "1", "Locus": "rs32285189", "Chr": 1, "cM": 1.630,
                     "Mb": 3.511, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs258367496", "Chr": 1, "cM": 1.630,
+                    "ID": "1", "Locus": "rs258367496", "Chr": 1, "cM": 1.630,
                     "Mb": 3.660, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs32430919", "Chr": 1, "cM": 1.750,
+                    "ID": "1", "Locus": "rs32430919", "Chr": 1, "cM": 1.750,
                     "Mb": 3.777, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs36251697", "Chr": 1, "cM": 1.880,
+                    "ID": "1", "Locus": "rs36251697", "Chr": 1, "cM": 1.880,
                     "Mb": 3.812, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs30658298", "Chr": 1, "cM": 2.010,
+                    "ID": "1", "Locus": "rs30658298", "Chr": 1, "cM": 2.010,
                     "Mb": 4.431, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs51852623", "Chr": 2, "cM": 2.010,
+                    "ID": "1", "Locus": "rs51852623", "Chr": 2, "cM": 2.010,
                     "Mb": 4.447, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs31879829", "Chr": 2, "cM": 2.140,
+                    "ID": "1", "Locus": "rs31879829", "Chr": 2, "cM": 2.140,
                     "Mb": 4.519, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 },
                 {
-                    "ID": "T1", "Locus": "rs36742481", "Chr": 2, "cM": 2.140,
+                    "ID": "1", "Locus": "rs36742481", "Chr": 2, "cM": 2.140,
                     "Mb": 4.776, "LRS": 0.500, "Additive": -0.074,
                     "pValue": 1.000
                 }
             ]),
-            {"T1": {"ID": "T1",
-                    "chromosomes": {
-                        1: {"Chr": 1,
-                            "loci": [
-                                {
-                                    "Locus": "rs31443144", "cM": 1.500, "Mb": 3.010,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs6269442", "cM": 1.500, "Mb": 3.492,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs32285189", "cM": 1.630, "Mb": 3.511,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs258367496", "cM": 1.630, "Mb": 3.660,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs32430919", "cM": 1.750, "Mb": 3.777,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs36251697", "cM": 1.880, "Mb": 3.812,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs30658298", "cM": 2.010, "Mb": 4.431,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                }]},
-                        2: {"Chr": 2,
-                            "loci": [
-                                {
-                                    "Locus": "rs51852623", "cM": 2.010, "Mb": 4.447,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs31879829", "cM": 2.140, "Mb": 4.519,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs36742481", "cM": 2.140, "Mb": 4.776,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                }]}}}})
+            organised_trait_1)
diff --git a/tests/unit/db/test_traits.py b/tests/unit/db/test_traits.py
index baa2af3..8af8e82 100644
--- a/tests/unit/db/test_traits.py
+++ b/tests/unit/db/test_traits.py
@@ -170,12 +170,15 @@ class TestTraitsDBFunctions(TestCase):
         db_mock = mock.MagicMock()
 
         STRAIN_ID_SQL: str = "UPDATE Strain SET Name = %s WHERE Id = %s"
-        PUBLISH_DATA_SQL: str = ("UPDATE PublishData SET value = %s "
-                                 "WHERE StrainId = %s AND Id = %s")
-        PUBLISH_SE_SQL: str = ("UPDATE PublishSE SET error = %s "
-                               "WHERE StrainId = %s AND DataId = %s")
-        N_STRAIN_SQL: str = ("UPDATE NStrain SET count = %s "
-                             "WHERE StrainId = %s AND DataId = %s")
+        PUBLISH_DATA_SQL: str = (
+            "UPDATE PublishData SET value = %s "
+            "WHERE StrainId = %s AND Id = %s")
+        PUBLISH_SE_SQL: str = (
+            "UPDATE PublishSE SET error = %s "
+            "WHERE StrainId = %s AND DataId = %s")
+        N_STRAIN_SQL: str = (
+            "UPDATE NStrain SET count = %s "
+            "WHERE StrainId = %s AND DataId = %s")
 
         with db_mock.cursor() as cursor:
             type(cursor).rowcount = 1
diff --git a/tests/unit/sample_test_data.py b/tests/unit/sample_test_data.py
new file mode 100644
index 0000000..407d074
--- /dev/null
+++ b/tests/unit/sample_test_data.py
@@ -0,0 +1,111 @@
+"""
+This module holds a collection of sample data variables, used in more than one
+ test.
+
+This is mostly to avoid the `duplicate-code` pylint error that gets raised if
+the same data is defined in more than one file. It has been found that adding
+the `# pylint: disable=R0801` or `# pylint: disable=duplicate-code` to the top
+of the file seems to not work as expected.
+
+Adding these same declarations to .pylintrc is not an option, since that,
+seemingly, would deactivate the warnings for all code in the project: We do not
+want that.
+"""
+
+organised_trait_1 = {
+    "1": {
+        "ID": "1",
+        "chromosomes": {
+            1: {"Chr": 1,
+                "loci": [
+                    {
+                        "Locus": "rs31443144", "cM": 1.500, "Mb": 3.010,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs6269442", "cM": 1.500, "Mb": 3.492,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs32285189", "cM": 1.630, "Mb": 3.511,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs258367496", "cM": 1.630, "Mb": 3.660,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs32430919", "cM": 1.750, "Mb": 3.777,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs36251697", "cM": 1.880, "Mb": 3.812,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs30658298", "cM": 2.010, "Mb": 4.431,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    }]},
+            2: {"Chr": 2,
+                "loci": [
+                    {
+                        "Locus": "rs51852623", "cM": 2.010, "Mb": 4.447,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs31879829", "cM": 2.140, "Mb": 4.519,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs36742481", "cM": 2.140, "Mb": 4.776,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    }]}}}}
+
+organised_trait_2 = {
+    "2": {
+        "ID": "2",
+        "chromosomes": {
+            1: {"Chr": 1,
+                "loci": [
+                    {
+                        "Locus": "rs31443144", "cM": 1.500, "Mb": 3.010,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs6269442", "cM": 1.500, "Mb": 3.492,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs32285189", "cM": 1.630, "Mb": 3.511,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs258367496", "cM": 1.630, "Mb": 3.660,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs32430919", "cM": 1.750, "Mb": 3.777,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs36251697", "cM": 1.880, "Mb": 3.812,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs30658298", "cM": 2.010, "Mb": 4.431,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    }]},
+            2: {"Chr": 2,
+                "loci": [
+                    {
+                        "Locus": "rs51852623", "cM": 2.010, "Mb": 4.447,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs31879829", "cM": 2.140, "Mb": 4.519,
+                        "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
+                    },
+                    {
+                        "Locus": "rs36742481", "cM": 2.140, "Mb": 4.776,
+                        "LRS": 0.579, "Additive": -0.074, "pValue": 1.000
+                    }]}}}}
diff --git a/tests/unit/test_heatmaps.py b/tests/unit/test_heatmaps.py
index c0a496b..fd91cf9 100644
--- a/tests/unit/test_heatmaps.py
+++ b/tests/unit/test_heatmaps.py
@@ -7,6 +7,7 @@ from gn3.heatmaps import (
     compute_traits_order,
     retrieve_strains_and_values,
     process_traits_data_for_heatmap)
+from tests.unit.sample_test_data import organised_trait_1, organised_trait_2
 
 strainlist = ["B6cC3-1", "BXD1", "BXD12", "BXD16", "BXD19", "BXD2"]
 trait_data = {
@@ -206,100 +207,7 @@ class TestHeatmap(TestCase):
         """Check for correct processing of data for heatmap generation."""
         self.assertEqual(
             process_traits_data_for_heatmap(
-                {"1": {
-                    "ID": "T1",
-                    "chromosomes": {
-                        1: {"Chr": 1,
-                            "loci": [
-                                {
-                                    "Locus": "rs31443144", "cM": 1.500, "Mb": 3.010,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs6269442", "cM": 1.500, "Mb": 3.492,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs32285189", "cM": 1.630, "Mb": 3.511,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs258367496", "cM": 1.630, "Mb": 3.660,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs32430919", "cM": 1.750, "Mb": 3.777,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs36251697", "cM": 1.880, "Mb": 3.812,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs30658298", "cM": 2.010, "Mb": 4.431,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                }]},
-                        2: {"Chr": 2,
-                            "loci": [
-                                {
-                                    "Locus": "rs51852623", "cM": 2.010, "Mb": 4.447,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs31879829", "cM": 2.140, "Mb": 4.519,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                },
-                                {
-                                    "Locus": "rs36742481", "cM": 2.140, "Mb": 4.776,
-                                    "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                }]}}},
-                 "2": {
-                     "ID": "T1",
-                     "chromosomes": {
-                         1: {"Chr": 1,
-                             "loci": [
-                                 {
-                                     "Locus": "rs31443144", "cM": 1.500, "Mb": 3.010,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 },
-                                 {
-                                     "Locus": "rs6269442", "cM": 1.500, "Mb": 3.492,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 },
-                                 {
-                                     "Locus": "rs32285189", "cM": 1.630, "Mb": 3.511,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 },
-                                 {
-                                     "Locus": "rs258367496", "cM": 1.630, "Mb": 3.660,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 },
-                                 {
-                                     "Locus": "rs32430919", "cM": 1.750, "Mb": 3.777,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 },
-                                 {
-                                     "Locus": "rs36251697", "cM": 1.880, "Mb": 3.812,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 },
-                                 {
-                                     "Locus": "rs30658298", "cM": 2.010, "Mb": 4.431,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 }]},
-                         2: {"Chr": 2,
-                             "loci": [
-                                 {
-                                     "Locus": "rs51852623", "cM": 2.010, "Mb": 4.447,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 },
-                                 {
-                                     "Locus": "rs31879829", "cM": 2.140, "Mb": 4.519,
-                                     "LRS": 0.500, "Additive": -0.074, "pValue": 1.000
-                                 },
-                                 {
-                                     "Locus": "rs36742481", "cM": 2.140, "Mb": 4.776,
-                                     "LRS": 0.579, "Additive": -0.074, "pValue": 1.000
-                                 }]}}}},
+                {**organised_trait_1, **organised_trait_2},
                 ["2", "1"],
                 [1, 2]),
             [[[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5],
-- 
cgit v1.2.3