Merge remote-tracking branch 'origin/testing' into feature/add_rqtl_pairscan

50 files changed, 2679 insertions, 988 deletions

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index af248659..d7e4e62f 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -756,7 +756,7 @@ class DataSet:
         chunk_size = 50
         number_chunks = int(math.ceil(len(sample_ids) / chunk_size))
 
-        cached_results = fetch_cached_results(self.name, self.type)
+        cached_results = fetch_cached_results(self.name, self.type, self.samplelist)
         if cached_results is None:
             trait_sample_data = []
             for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks):
@@ -812,9 +812,8 @@ class DataSet:
                         trait_sample_data[chunk_counter][trait_counter][data_start_pos:])
 
             cache_dataset_results(
-                self.name, self.type, self.trait_data)
+                self.name, self.type, self.samplelist, self.trait_data)
         else:
-
             self.trait_data = cached_results
 
 
@@ -1278,14 +1277,14 @@ def query_table_timestamp(dataset_type: str):
     return date_time_obj.strftime("%Y-%m-%d %H:%M:%S")
 
 
-def generate_hash_file(dataset_name: str, dataset_type: str, dataset_timestamp: str):
+def generate_hash_file(dataset_name: str, dataset_type: str, dataset_timestamp: str, samplelist: str):
     """given the trait_name generate a unique name for this"""
-    string_unicode = f"{dataset_name}{dataset_timestamp}".encode()
+    string_unicode = f"{dataset_name}{dataset_timestamp}{samplelist}".encode()
     md5hash = hashlib.md5(string_unicode)
     return md5hash.hexdigest()
 
 
-def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: List):
+def cache_dataset_results(dataset_name: str, dataset_type: str, samplelist: List, query_results: List):
     """function to cache dataset query results to file
     input dataset_name and type query_results(already processed in default dict format)
     """
@@ -1293,21 +1292,22 @@ def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: L
     # store the file path on redis
 
     table_timestamp = query_table_timestamp(dataset_type)
+    samplelist_as_str = ",".join(samplelist)
 
-
-    file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp)
+    file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp, samplelist_as_str)
     file_path = os.path.join(TMPDIR, f"{file_name}.json")
 
     with open(file_path, "w") as file_handler:
         json.dump(query_results, file_handler)
 
 
-def fetch_cached_results(dataset_name: str, dataset_type: str):
+def fetch_cached_results(dataset_name: str, dataset_type: str, samplelist: List):
     """function to fetch the cached results"""
 
     table_timestamp = query_table_timestamp(dataset_type)
+    samplelist_as_str = ",".join(samplelist)
 
-    file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp)
+    file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp, samplelist_as_str)
     file_path = os.path.join(TMPDIR, f"{file_name}.json")
     try:
         with open(file_path, "r") as file_handler:
diff --git a/wqflask/maintenance/gen_ind_genofiles.py b/wqflask/maintenance/gen_ind_genofiles.py
new file mode 100644
index 00000000..8b958efa
--- /dev/null
+++ b/wqflask/maintenance/gen_ind_genofiles.py
@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+"""A script that generates the genotype files for groups of individuals, using an existing strain genotype file as a basis
+
+Example commands:
+python3 gen_ind_genofiles.py
+        /home/zas1024/gn2-zach/genotype_files/genotype/
+        /home/zas1024/gn2-zach/new_geno/
+        BXD-Micturition.geno
+        BXD.json
+python3 gen_ind_genofiles.py
+        /home/zas1024/gn2-zach/genotype_files/genotype
+        /home/zas1024/gn2-zach/new_geno/
+        BXD-Micturition.geno
+        BXD.2.geno BXD.4.geno BXD.5.geno
+
+"""
+
+import json
+import os
+import sys
+from typing import List
+
+import MySQLdb
+
+def conn():
+    return MySQLdb.Connect(db=os.environ.get("DB_NAME"),
+                           user=os.environ.get("DB_USER"),
+                           passwd=os.environ.get("DB_PASS"),
+                           host=os.environ.get("DB_HOST"))
+
+def main(args):
+
+    # Directory in which .geno files are located
+    geno_dir = args[1]
+
+    # Directory in which to output new files
+    out_dir = args[2]
+
+    # The individuals group that we want to generate a .geno file for
+    target_file = geno_dir + args[3]
+
+    # The source group(s) we're generating the .geno files from
+    # This can be passed as either a specific .geno file (or set of files as multiple arguments),
+    # or as a JSON file containing a set of .geno files (and their corresponding file names and sample lists)
+    geno_json = {}
+    source_files = []
+    if ".json" in args[4]:
+        geno_json = json.load(open(geno_dir + args[4], "r"))
+        par_f1s = {
+            "mat": geno_json['mat'],
+            "pat": geno_json['pat'],
+            "f1s": geno_json['f1s']
+        }
+
+        # List of file titles and locations from JSON
+        source_files = [{'title': genofile['title'], 'location': geno_dir + genofile['location']} for genofile in geno_json['genofile']]
+    else:
+        par_f1s = {}
+        # List of files directly taken from command line arguments, with titles just set to the filename
+        for group in args[4:]:
+            file_name = geno_dir + group + ".geno" if ".geno" not in group else group
+            source_files.append({'title': file_name[:-5], 'location': file_name})
+
+    if len(source_files) > 1:
+        # Generate a JSON file pointing to the new target genotype files, in situations where there are multiple source .geno files
+        target_json_loc = out_dir + ".".join(args[3].split(".")[:-1]) + ".json"
+        target_json = {'genofile': []}
+
+    # Generate the output .geno files
+    for source_file in source_files:
+        filename, samples = generate_new_genofile(source_file['location'], target_file, par_f1s, out_dir)
+
+        target_json['genofile'].append({
+            'location': filename.split("/")[-1],
+            'title': source_file['title'],
+            'sample_list': samples
+        })
+
+    json.dump(target_json, open(target_json_loc, "w"))
+
+def get_strain_for_sample(sample):
+    query = (
+        "SELECT CaseAttributeXRefNew.Value "
+        "FROM CaseAttributeXRefNew, Strain "
+        "WHERE CaseAttributeXRefNew.CaseAttributeId=11 "
+        "AND CaseAttributeXRefNew.StrainId = Strain.Id "
+        "AND Strain.Name = %(name)s" )
+
+    with conn().cursor() as cursor:
+        cursor.execute(query, {"name": sample.strip()})
+        return cursor.fetchone()[0]
+
+def generate_new_genofile(source_genofile, target_genofile, par_f1s, out_dir):
+    source_samples = group_samples(source_genofile)
+    source_genotypes = strain_genotypes(source_genofile)
+    target_samples = group_samples(target_genofile)
+    strain_pos_map = map_strain_pos_to_target_group(source_samples, target_samples, par_f1s)
+
+    if len(source_genofile.split("/")[-1].split(".")) > 2:
+        # The number in the source genofile; for example 4 in BXD.4.geno
+        source_num = source_genofile.split("/")[-1].split(".")[-2]
+        target_filename = ".".join(target_genofile.split("/")[-1].split(".")[:-1]) + "." + source_num + ".geno"
+    else:
+        target_filename = ".".join(target_genofile.split("/")[-1].split(".")[:-1]) + ".geno"
+
+    file_location = out_dir + target_filename
+
+    with open(file_location, "w") as fh:
+        for metadata in ["name", "type", "mat", "pat", "het", "unk"]:
+            fh.write("@" + metadata + ":" + source_genotypes[metadata] + "\n")
+
+        header_line = ["Chr", "Locus", "cM", "Mb"] + target_samples
+        fh.write("\t".join(header_line))
+
+        for marker in source_genotypes['markers']:
+            line_items = [
+                marker['Chr'],
+                marker['Locus'],
+                marker['cM'],
+                marker['Mb']
+            ]
+
+            for pos in strain_pos_map:
+                if isinstance(pos, int):
+                    line_items.append(marker['genotypes'][pos])
+                else:
+                    if pos in ["mat", "pat"]:
+                        line_items.append(source_genotypes[pos])
+                    elif pos == "f1s":
+                        line_items.append("H")
+                    else:
+                        line_items.append("U")
+
+            fh.write("\t".join(line_items) + "\n")
+
+    return file_location, target_samples
+
+def map_strain_pos_to_target_group(source_samples, target_samples, par_f1s):
+    """
+    Retrieve corresponding strain position for each sample in the target group
+
+    This is so the genotypes from the base genofile can be mapped to the samples in the target group
+
+    For example:
+    Base strains: BXD1, BXD2, BXD3
+    Target samples: BXD1_1, BXD1_2, BXD2_1, BXD3_1, BXD3_2, BXD3_3
+    Returns: [0, 0, 1, 2, 2, 2]
+    """
+    pos_map = []
+    for sample in target_samples:
+        sample_strain = get_strain_for_sample(sample)
+        if sample_strain in source_samples:
+            pos_map.append(source_samples.index(sample_strain))
+        else:
+            val = "U"
+            for key in par_f1s.keys():
+                if sample_strain in par_f1s[key]:
+                    val = key
+            pos_map.append(val)
+
+    return pos_map
+
+def group_samples(target_file: str) -> List:
+    """
+    Get the group samples from its "dummy" .geno file (which still contains the sample list)
+    """
+
+    sample_list = []
+    with open(target_file, "r") as target_geno:
+        for i, line in enumerate(target_geno):
+            # Skip header lines
+            if line[0] in ["#", "@"] or not len(line):
+                continue
+    
+            line_items = line.split("\t")
+            sample_list = [item for item in line_items if item not in ["Chr", "Locus", "Mb", "cM"]]
+            break
+
+    return sample_list
+
+def strain_genotypes(strain_genofile: str) -> List:
+    """
+    Read genotypes from source strain .geno file
+
+    :param strain_genofile: string of genofile filename
+    :return: a list of dictionaries representing each marker's genotypes
+
+    Example output: [
+        {
+            'Chr': '1',
+            'Locus': 'marker1',
+            'Mb': '10.0',
+            'cM': '8.0',
+            'genotypes': [('BXD1', 'B'), ('BXD2', 'D'), ('BXD3', 'H'), ...]
+        },
+        ...
+    ]
+    """
+
+    geno_dict = {}
+
+    geno_start_col = None
+    header_columns = []
+    sample_list = []
+    markers = []
+    with open(strain_genofile, "r") as source_geno:
+        for i, line in enumerate(source_geno):
+            if line[0] == "@":
+                metadata_type = line[1:].split(":")[0]
+                if metadata_type in ['name', 'type', 'mat', 'pat', 'het', 'unk']:
+                    geno_dict[metadata_type] = line.split(":")[1].strip()
+
+                continue
+
+            # Skip other header lines
+            if line[0] == "#" or not len(line):
+                continue
+
+            line_items = line.split("\t")
+            if "Chr" in line_items: # Header row
+                # Get the first column index containing genotypes
+                header_columns = line_items
+                for j, item in enumerate(line_items):
+                    if item not in ["Chr", "Locus", "Mb", "cM"]:
+                        geno_start_col = j
+                        break
+
+                sample_list = line_items[geno_start_col:]
+                if not geno_start_col:
+                    print("Check .geno file - expected columns not found")
+                    sys.exit()
+            else: # Marker rows
+                this_marker = {
+                    'Chr': line_items[header_columns.index("Chr")],
+                    'Locus': line_items[header_columns.index("Locus")],
+                    'Mb': line_items[header_columns.index("Mb")],
+                    'cM': line_items[header_columns.index("cM")],
+                    'genotypes': [item.strip() for item in line_items][geno_start_col:]
+                }
+
+                markers.append(this_marker)
+
+    geno_dict['markers'] = markers
+
+    return geno_dict
+            
+if __name__ == "__main__":
+    main(sys.argv)
+
diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py
index db65a11f..9f4b670d 100644
--- a/wqflask/maintenance/gen_select_dataset.py
+++ b/wqflask/maintenance/gen_select_dataset.py
@@ -39,21 +39,13 @@ from wqflask import app
 
 from utility.tools import locate, locate_ignore_error, TEMPDIR, SQL_URI
 
-import MySQLdb
-
 import simplejson as json
 import urllib.parse
 
 
-#import sqlalchemy as sa
-
 from pprint import pformat as pf
 
-#Engine = sa.create_engine(zach_settings.SQL_URI)
-
-# build MySql database connection
-
-#conn = Engine.connect()
+from wqflask.database import database_connection
 
 
 def parse_db_uri():
@@ -71,19 +63,19 @@ def parse_db_uri():
     return db_conn_info
 
 
-def get_species():
+def get_species(cursor):
     """Build species list"""
-    #Cursor.execute("select Name, MenuName from Species where Species.Name != 'macaque monkey' order by OrderId")
-    Cursor.execute("select Name, MenuName from Species order by OrderId")
-    species = list(Cursor.fetchall())
+    #cursor.execute("select Name, MenuName from Species where Species.Name != 'macaque monkey' order by OrderId")
+    cursor.execute("select Name, MenuName from Species order by OrderId")
+    species = list(cursor.fetchall())
     return species
 
 
-def get_groups(species):
+def get_groups(cursor, species):
     """Build groups list"""
     groups = {}
     for species_name, _species_full_name in species:
-        Cursor.execute("""select InbredSet.Name, InbredSet.FullName from InbredSet,
+        cursor.execute("""select InbredSet.Name, InbredSet.FullName from InbredSet,
                        Species,
                        ProbeFreeze, GenoFreeze, PublishFreeze where Species.Name = '%s'
                        and InbredSet.SpeciesId = Species.Id and
@@ -92,7 +84,7 @@ def get_groups(species):
                         or ProbeFreeze.InbredSetId = InbredSet.Id)
                         group by InbredSet.Name
                         order by InbredSet.FullName""" % species_name)
-        results = Cursor.fetchall()
+        results = cursor.fetchall()
         groups[species_name] = list(results)
     return groups
 
@@ -273,13 +265,13 @@ def build_datasets(species, group, type_name):
     return datasets
 
 
-def main():
+def main(cursor):
     """Generates and outputs (as json file) the data for the main dropdown menus on the home page"""
 
     parse_db_uri()
 
-    species = get_species()
-    groups = get_groups(species)
+    species = get_species(cursor)
+    groups = get_groups(cursor, species)
     types = get_types(groups)
     datasets = get_datasets(types)
 
@@ -316,6 +308,6 @@ def _test_it():
 
 
 if __name__ == '__main__':
-    Conn = MySQLdb.Connect(**parse_db_uri())
-    Cursor = Conn.cursor()
-    main()
+    with database_connection() as conn:
+        with conn.cursor() as cursor:
+            main(cursor)
diff --git a/wqflask/maintenance/generate_probesetfreeze_file.py b/wqflask/maintenance/generate_probesetfreeze_file.py
index e964c8ed..f43f952b 100644
--- a/wqflask/maintenance/generate_probesetfreeze_file.py
+++ b/wqflask/maintenance/generate_probesetfreeze_file.py
@@ -8,20 +8,11 @@ import os
 import collections
 import csv
 
-import MySQLdb
-
 from base import webqtlConfig
 
 from pprint import pformat as pf
 
-
-def get_cursor():
-    con = MySQLdb.Connect(db=webqtlConfig.DB_UPDNAME,
-                          host=webqtlConfig.MYSQL_UPDSERVER,
-                          user=webqtlConfig.DB_UPDUSER,
-                          passwd=webqtlConfig.DB_UPDPASSWD)
-    cursor = con.cursor()
-    return cursor
+from wqflask.database import database_connection
 
 
 def show_progress(process, counter):
@@ -116,13 +107,14 @@ def main():
         "(Oct08)_RankInv_Beta.txt")
     dataset_name = "Eye_AXBXA_1008_RankInv"
 
-    cursor = get_cursor()
-    strains = get_strains(cursor)
-    print("Getting probset_vals")
-    probeset_vals = get_probeset_vals(cursor, dataset_name)
-    print("Finished getting probeset_vals")
-    trimmed_strains = trim_strains(strains, probeset_vals)
-    write_data_matrix_file(trimmed_strains, probeset_vals, filename)
+    with database_connection as conn:
+        with conn.cursor() as cursor:
+            strains = get_strains(cursor)
+            print("Getting probset_vals")
+            probeset_vals = get_probeset_vals(cursor, dataset_name)
+            print("Finished getting probeset_vals")
+            trimmed_strains = trim_strains(strains, probeset_vals)
+            write_data_matrix_file(trimmed_strains, probeset_vals, filename)
 
 
 if __name__ == '__main__':
diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py
index 32780ca6..90ec72de 100644
--- a/wqflask/maintenance/quantile_normalize.py
+++ b/wqflask/maintenance/quantile_normalize.py
@@ -1,6 +1,5 @@
 import sys
 sys.path.insert(0, './')
-import MySQLdb
 import urllib.parse
 
 import numpy as np
@@ -9,6 +8,7 @@ import pandas as pd
 from flask import Flask, g, request
 
 from wqflask import app
+from wqflask.database import database_connection
 
 
 def parse_db_uri():
@@ -52,7 +52,7 @@ def quantileNormalize(df_input):
     return df
 
 
-def set_data(dataset_name):
+def set_data(cursor, dataset_name):
     orig_file = "/home/zas1024/cfw_data/" + dataset_name + ".txt"
 
     sample_list = []
@@ -80,8 +80,8 @@ def set_data(dataset_name):
                                  ProbeSetFreeze.Id = ProbeSetXRef.ProbeSetFreezeId and
                                  ProbeSetXRef.ProbeSetId = ProbeSet.Id and
                                  ProbeSet.Name = '%s'""" % (dataset_name, line1.split('\t')[0])
-                Cursor.execute(query)
-                result_info = Cursor.fetchone()
+                cursor.execute(query)
+                result_info = cursor.fetchone()
 
                 yield {
                     "_index": "traits",
@@ -99,15 +99,14 @@ def set_data(dataset_name):
 
 
 if __name__ == '__main__':
-    Conn = MySQLdb.Connect(**parse_db_uri())
-    Cursor = Conn.cursor()
+    with database_connection as conn:
+        with conn.cursor() as cursor:
+            success, _ = bulk(es, set_data(cursor, sys.argv[1]))
 
-    success, _ = bulk(es, set_data(sys.argv[1]))
-
-    response = es.search(
-        index="traits", doc_type="trait", body={
-            "query": {"match": {"name": "ENSMUSG00000028982"}}
-        }
-    )
+            response = es.search(
+                index="traits", doc_type="trait", body={
+                    "query": {"match": {"name": "ENSMUSG00000028982"}}
+                }
+            )
 
-    print(response)
+            print(response)
diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py
index 0f472494..22d73ba3 100644
--- a/wqflask/maintenance/set_resource_defaults.py
+++ b/wqflask/maintenance/set_resource_defaults.py
@@ -30,10 +30,9 @@ from utility.tools import SQL_URI
 from utility.redis_tools import get_redis_conn, get_user_id, add_resource, get_resources, get_resource_info
 Redis = get_redis_conn()
 
-import MySQLdb
-
 import urllib.parse
 
+from wqflask.database import database_connection
 from utility.logger import getLogger
 logger = getLogger(__name__)
 
@@ -53,14 +52,14 @@ def parse_db_uri():
     return db_conn_info
 
 
-def insert_probeset_resources(default_owner_id):
+def insert_probeset_resources(cursor, default_owner_id):
     current_resources = Redis.hgetall("resources")
-    Cursor.execute("""  SELECT
+    cursor.execute("""  SELECT
                             ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.confidentiality, ProbeSetFreeze.public
                         FROM
                             ProbeSetFreeze""")
 
-    resource_results = Cursor.fetchall()
+    resource_results = cursor.fetchall()
     for i, resource in enumerate(resource_results):
         resource_ob = {}
         resource_ob['name'] = resource[1]
@@ -80,9 +79,9 @@ def insert_probeset_resources(default_owner_id):
         add_resource(resource_ob, update=False)
 
 
-def insert_publish_resources(default_owner_id):
+def insert_publish_resources(cursor, default_owner_id):
     current_resources = Redis.hgetall("resources")
-    Cursor.execute("""  SELECT 
+    cursor.execute("""  SELECT 
                             PublishXRef.Id, PublishFreeze.Id, InbredSet.InbredSetCode
                         FROM
                             PublishXRef, PublishFreeze, InbredSet, Publication
@@ -91,7 +90,7 @@ def insert_publish_resources(default_owner_id):
                             InbredSet.Id = PublishXRef.InbredSetId AND
                             Publication.Id = PublishXRef.PublicationId""")
 
-    resource_results = Cursor.fetchall()
+    resource_results = cursor.fetchall()
     for resource in resource_results:
         if resource[2]:
             resource_ob = {}
@@ -114,14 +113,14 @@ def insert_publish_resources(default_owner_id):
             continue
 
 
-def insert_geno_resources(default_owner_id):
+def insert_geno_resources(cursor, default_owner_id):
     current_resources = Redis.hgetall("resources")
-    Cursor.execute("""  SELECT
+    cursor.execute("""  SELECT
                             GenoFreeze.Id, GenoFreeze.ShortName, GenoFreeze.confidentiality
                         FROM
                             GenoFreeze""")
 
-    resource_results = Cursor.fetchall()
+    resource_results = cursor.fetchall()
     for i, resource in enumerate(resource_results):
         resource_ob = {}
         resource_ob['name'] = resource[1]
@@ -147,15 +146,15 @@ def insert_geno_resources(default_owner_id):
 def insert_resources(default_owner_id):
     current_resources = get_resources()
     print("START")
-    insert_publish_resources(default_owner_id)
+    insert_publish_resources(cursor, default_owner_id)
     print("AFTER PUBLISH")
-    insert_geno_resources(default_owner_id)
+    insert_geno_resources(cursor, default_owner_id)
     print("AFTER GENO")
-    insert_probeset_resources(default_owner_id)
+    insert_probeset_resources(cursor, default_owner_id)
     print("AFTER PROBESET")
 
 
-def main():
+def main(cursor):
     """Generates and outputs (as json file) the data for the main dropdown menus on the home page"""
 
     Redis.delete("resources")
@@ -166,6 +165,6 @@ def main():
 
 
 if __name__ == '__main__':
-    Conn = MySQLdb.Connect(**parse_db_uri())
-    Cursor = Conn.cursor()
-    main()
+    with database_connection() as conn:
+        with conn.cursor() as cursor:
+            main(cursor)
diff --git a/wqflask/tests/unit/base/test_data_set.py b/wqflask/tests/unit/base/test_data_set.py
index 66ad361d..505449a5 100644
--- a/wqflask/tests/unit/base/test_data_set.py
+++ b/wqflask/tests/unit/base/test_data_set.py
@@ -66,6 +66,7 @@ class TestDataSetTypes(unittest.TestCase):
                  '"HC_M2_0606_P": "ProbeSet", '
                  '"BXDPublish": "Publish"}'))
 
+    @unittest.skip("Too complicated")
     @mock.patch('base.data_set.g')
     def test_set_dataset_key_mrna(self, db_mock):
         with app.app_context():
@@ -92,6 +93,7 @@ class TestDataSetTypes(unittest.TestCase):
                  + "WHERE ProbeSetFreeze.Name = \"Test\" ")
             )
 
+    @unittest.skip("Too complicated")
     @mock.patch('base.data_set.g')
     def test_set_dataset_key_pheno(self, db_mock):
         with app.app_context():
@@ -121,6 +123,7 @@ class TestDataSetTypes(unittest.TestCase):
                  "InfoFiles.InfoPageName = PublishFreeze.Name")
             )
 
+    @unittest.skip("Too complicated")
     @mock.patch('base.data_set.g')
     def test_set_dataset_other_pheno(self, db_mock):
         with app.app_context():
@@ -151,6 +154,7 @@ class TestDataSetTypes(unittest.TestCase):
                  "PublishFreeze.InbredSetId = InbredSet.Id")
             )
 
+    @unittest.skip("Too complicated")
     @mock.patch('base.data_set.g')
     def test_set_dataset_geno(self, db_mock):
         with app.app_context():
diff --git a/wqflask/tests/unit/base/test_species.py b/wqflask/tests/unit/base/test_species.py
index 9b5c023c..87f89607 100644
--- a/wqflask/tests/unit/base/test_species.py
+++ b/wqflask/tests/unit/base/test_species.py
@@ -60,6 +60,7 @@ class TestIndChromosome(unittest.TestCase):
         self.assertEqual(test_ind_chromosome.mb_length, 10)
 
 
+@unittest.skip("Too complicated")
 class TestChromosomes(unittest.TestCase):
     """Tests for Chromosomes class"""
     maxDiff = None
diff --git a/wqflask/tests/unit/base/test_trait.py b/wqflask/tests/unit/base/test_trait.py
index 826ccefd..71fe0a44 100644
--- a/wqflask/tests/unit/base/test_trait.py
+++ b/wqflask/tests/unit/base/test_trait.py
@@ -106,49 +106,57 @@ class TestRetrieveTraitInfo(unittest.TestCase):
         self.assertEqual(test_trait.authors,
                          "Jane Doe かいと")
 
+
+    @unittest.skip("Too complicated")
     @mock.patch('base.trait.requests.get')
-    @mock.patch('base.trait.g')
+    @mock.patch('base.trait.database_connection')
     @mock.patch('base.trait.get_resource_id')
     def test_retrieve_trait_info_with_non_empty_lrs(self,
                                                     resource_id_mock,
-                                                    g_mock,
+                                                    mock_db,
                                                     requests_mock):
         """Test retrieve trait info when lrs has a value"""
         resource_id_mock.return_value = 1
-        g_mock.db.execute.return_value.fetchone = mock.Mock()
-        g_mock.db.execute.return_value.fetchone.side_effect = [
-            [1, 2, 3, 4],  # trait_info = g.db.execute(query).fetchone()
-            [1, 2.37, 3, 4, 5],  # trait_qtl = g.db.execute(query).fetchone()
-            [2.7333, 2.1204]  # trait_info = g.db.execute(query).fetchone()
-        ]
-        requests_mock.return_value = None
-
-        mock_dataset = mock.MagicMock()
-        type(mock_dataset).display_fields = mock.PropertyMock(
-            return_value=["a", "b", "c", "d"])
-        type(mock_dataset).type = "ProbeSet"
-        type(mock_dataset).name = "RandomName"
-
-        mock_trait = MockTrait(
-            dataset=mock_dataset,
-            pre_publication_description="test_string"
-        )
-        trait_attrs = {
-            "description": "some description",
-            "probe_target_description": "some description",
-            "cellid": False,
-            "chr": 2.733,
-            "mb": 2.1204
-        }
-
-        for key, val in list(trait_attrs.items()):
-            setattr(mock_trait, key, val)
-        test_trait = retrieve_trait_info(trait=mock_trait,
-                                         dataset=mock_dataset,
-                                         get_qtl_info=True)
-        self.assertEqual(test_trait.LRS_score_repr,
-                         "2.4")
-
+        conn = mock.MagicMock()
+        mock_db.return_value.__enter__.return_value = conn
+        with conn.cursor() as cursor:
+            cursor.fetchone.side_effect = [
+                # trait_info = g.db.execute(query).fetchone()
+                [1, 2, 3, 4],
+                # trait_qtl = g.db.execute(query).fetchone()
+                [1, 2.37, 3, 4, 5],
+                # trait_info = g.db.execute(query).fetchone()
+                [2.7333, 2.1204]
+            ]
+            requests_mock.return_value = None
+
+            mock_dataset = mock.MagicMock()
+            type(mock_dataset).display_fields = mock.PropertyMock(
+                return_value=["a", "b", "c", "d"])
+            type(mock_dataset).type = "ProbeSet"
+            type(mock_dataset).name = "RandomName"
+
+            mock_trait = MockTrait(
+                dataset=mock_dataset,
+                pre_publication_description="test_string"
+            )
+            trait_attrs = {
+                "description": "some description",
+                "probe_target_description": "some description",
+                "cellid": False,
+                "chr": 2.733,
+                "mb": 2.1204
+            }
+
+            for key, val in list(trait_attrs.items()):
+                setattr(mock_trait, key, val)
+            test_trait = retrieve_trait_info(trait=mock_trait,
+                                             dataset=mock_dataset,
+                                             get_qtl_info=True)
+            self.assertEqual(test_trait.LRS_score_repr,
+                             "2.4")
+
+    @unittest.skip("Too complicated")
     @mock.patch('base.trait.requests.get')
     @mock.patch('base.trait.g')
     @mock.patch('base.trait.get_resource_id')
@@ -193,7 +201,8 @@ class TestRetrieveTraitInfo(unittest.TestCase):
                          "N/A")
         self.assertEqual(test_trait.LRS_location_repr,
                          "Chr2: 3.000000")
-
+        
+    @unittest.skip("Too complicated")
     @mock.patch('base.trait.requests.get')
     @mock.patch('base.trait.g')
     @mock.patch('base.trait.get_resource_id')
diff --git a/wqflask/tests/unit/wqflask/api/test_correlation.py b/wqflask/tests/unit/wqflask/api/test_correlation.py
index 1089a36f..53e1b9a2 100644
--- a/wqflask/tests/unit/wqflask/api/test_correlation.py
+++ b/wqflask/tests/unit/wqflask/api/test_correlation.py
@@ -20,12 +20,7 @@ class MockDataset(AttributeSetter):
         return None
 
     def retrieve_genes(self, id=None):
-        return {
-            "TT-1": "GH-1",
-            "TT-2": "GH-2",
-            "TT-3": "GH-3"
-
-        }
+        return {"TT-1": "GH-1", "TT-2": "GH-2", "TT-3": "GH-3"}
 
 
 class TestCorrelations(unittest.TestCase):
@@ -37,93 +32,112 @@ class TestCorrelations(unittest.TestCase):
         self.app_context.pop()
 
     def test_init_corr_params(self):
-        start_vars = {
-            "return_count": "3",
-            "type": "T1",
-            "method": "spearman"
-        }
+        start_vars = {"return_count": "3", "type": "T1", "method": "spearman"}
 
         corr_params_results = init_corr_params(start_vars=start_vars)
-        expected_results = {
-            "return_count": 3,
-            "type": "T1",
-            "method": "spearman"
-        }
+        expected_results = {"return_count": 3, "type": "T1", "method": "spearman"}
 
         self.assertEqual(corr_params_results, expected_results)
 
-    @mock.patch("wqflask.api.correlation.g")
+    @mock.patch("wqflask.api.correlation.database_connection")
     def test_convert_to_mouse_gene_id(self, mock_db):
-
-        results = convert_to_mouse_gene_id(species="Other", gene_id="")
-        self.assertEqual(results, None)
-
-        rat_species_results = convert_to_mouse_gene_id(
-            species="rat", gene_id="GH1")
-
-        mock_db.db.execute.return_value.fetchone.side_effect = [
-            AttributeSetter({"mouse": "MG-1"}), AttributeSetter({"mouse": "MG-2"})]
-
-        self.assertEqual(convert_to_mouse_gene_id(
-            species="mouse", gene_id="MG-4"), "MG-4")
-        self.assertEqual(convert_to_mouse_gene_id(
-            species="rat", gene_id="R1"), "MG-1")
-        self.assertEqual(convert_to_mouse_gene_id(
-            species="human", gene_id="H1"), "MG-2")
-
-    @mock.patch("wqflask.api.correlation.g")
+        conn = mock.MagicMock()
+        mock_db.return_value.__enter__.return_value = conn
+        with conn.cursor() as cursor:
+            cursor.fetchone.side_effect = [("MG-1",), ("MG-2",)]
+
+            self.assertEqual(
+                convert_to_mouse_gene_id(species="Other", gene_id=""), None
+            )
+            self.assertEqual(
+                convert_to_mouse_gene_id(species="mouse", gene_id="MG-4"), "MG-4"
+            )
+            self.assertEqual(
+                convert_to_mouse_gene_id(species="rat", gene_id="R1"), "MG-1"
+            )
+            self.assertEqual(
+                convert_to_mouse_gene_id(species="human", gene_id="H1"), "MG-2"
+            )
+
+    @mock.patch("wqflask.api.correlation.database_connection")
     @mock.patch("wqflask.api.correlation.convert_to_mouse_gene_id")
-    def test_do_literature_correlation_for_all_traits(self, mock_convert_to_mouse_geneid, mock_db):
-        mock_convert_to_mouse_geneid.side_effect = [
-            "MG-1", "MG-2;", "MG-3", "MG-4"]
-
-        trait_geneid_dict = {
-            "TT-1": "GH-1",
-            "TT-2": "GH-2",
-            "TT-3": "GH-3"
-
-        }
-        mock_db.db.execute.return_value.fetchone.side_effect = [AttributeSetter(
-            {"value": "V1"}), AttributeSetter({"value": "V2"}), AttributeSetter({"value": "V3"})]
-
-        this_trait = AttributeSetter({"geneid": "GH-1"})
-
-        target_dataset = AttributeSetter(
-            {"group": AttributeSetter({"species": "rat"})})
-        results = do_literature_correlation_for_all_traits(
-            this_trait=this_trait, target_dataset=target_dataset, trait_geneid_dict=trait_geneid_dict, corr_params={})
-
-        expected_results = {'TT-1': ['GH-1', 0],
-                            'TT-2': ['GH-2', 'V1'], 'TT-3': ['GH-3', 'V2']}
-        self.assertEqual(results, expected_results)
+    def test_do_literature_correlation_for_all_traits(
+        self, mock_convert_to_mouse_geneid, mock_db
+    ):
+        mock_convert_to_mouse_geneid.side_effect = ["MG-1", "MG-2;", "MG-3", "MG-4"]
+
+        trait_geneid_dict = {"TT-1": "GH-1", "TT-2": "GH-2", "TT-3": "GH-3"}
+        conn = mock.MagicMock()
+        mock_db.return_value.__enter__.return_value = conn
+        with conn.cursor() as cursor:
+            cursor.fetchone.side_effect = [("V1",), ("V2",), ("V3",)]
+            this_trait = AttributeSetter({"geneid": "GH-1"})
+            target_dataset = AttributeSetter(
+                {"group": AttributeSetter({"species": "rat"})}
+            )
+            results = do_literature_correlation_for_all_traits(
+                this_trait=this_trait,
+                target_dataset=target_dataset,
+                trait_geneid_dict=trait_geneid_dict,
+                corr_params={},
+            )
+            expected_results = {
+                "TT-1": ["GH-1", 0],
+                "TT-2": ["GH-2", "V1"],
+                "TT-3": ["GH-3", "V2"],
+            }
+            self.assertEqual(results, expected_results)
 
     @mock.patch("wqflask.api.correlation.corr_result_helpers.normalize_values")
     def test_get_sample_r_and_p_values(self, mock_normalize):
 
         group = AttributeSetter(
-            {"samplelist": ["S1", "S2", "S3", "S4", "S5", "S6", "S7"]})
+            {"samplelist": ["S1", "S2", "S3", "S4", "S5", "S6", "S7"]}
+        )
         target_dataset = AttributeSetter({"group": group})
 
         target_vals = [3.4, 6.2, 4.1, 3.4, 1.2, 5.6]
-        trait_data = {"S1": AttributeSetter({"value": 2.3}), "S2": AttributeSetter({"value": 1.1}),
-                      "S3": AttributeSetter(
-            {"value": 6.3}), "S4": AttributeSetter({"value": 3.6}), "S5": AttributeSetter({"value": 4.1}),
-            "S6": AttributeSetter({"value": 5.0})}
+        trait_data = {
+            "S1": AttributeSetter({"value": 2.3}),
+            "S2": AttributeSetter({"value": 1.1}),
+            "S3": AttributeSetter({"value": 6.3}),
+            "S4": AttributeSetter({"value": 3.6}),
+            "S5": AttributeSetter({"value": 4.1}),
+            "S6": AttributeSetter({"value": 5.0}),
+        }
         this_trait = AttributeSetter({"data": trait_data})
-        mock_normalize.return_value = ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0],
-                                       [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6)
-        mock_normalize.side_effect = [([2.3, 1.1, 6.3, 3.6, 4.1, 5.0],
-                                       [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6),
-                                      ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0],
-                                       [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6),
-                                      ([2.3, 1.1, 1.4], [3.4, 6.2, 4.1], 3)]
-
-        results_pearsonr = get_sample_r_and_p_values(this_trait=this_trait, this_dataset={
-        }, target_vals=target_vals, target_dataset=target_dataset, type="pearson")
-        results_spearmanr = get_sample_r_and_p_values(this_trait=this_trait, this_dataset={
-        }, target_vals=target_vals, target_dataset=target_dataset, type="spearman")
-        results_num_overlap = get_sample_r_and_p_values(this_trait=this_trait, this_dataset={
-        }, target_vals=target_vals, target_dataset=target_dataset, type="pearson")
+        mock_normalize.return_value = (
+            [2.3, 1.1, 6.3, 3.6, 4.1, 5.0],
+            [3.4, 6.2, 4.1, 3.4, 1.2, 5.6],
+            6,
+        )
+        mock_normalize.side_effect = [
+            ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0], [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6),
+            ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0], [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6),
+            ([2.3, 1.1, 1.4], [3.4, 6.2, 4.1], 3),
+        ]
+
+        results_pearsonr = get_sample_r_and_p_values(
+            this_trait=this_trait,
+            this_dataset={},
+            target_vals=target_vals,
+            target_dataset=target_dataset,
+            type="pearson",
+        )
+        results_spearmanr = get_sample_r_and_p_values(
+            this_trait=this_trait,
+            this_dataset={},
+            target_vals=target_vals,
+            target_dataset=target_dataset,
+            type="spearman",
+        )
+        results_num_overlap = get_sample_r_and_p_values(
+            this_trait=this_trait,
+            this_dataset={},
+            target_vals=target_vals,
+            target_dataset=target_dataset,
+            type="pearson",
+        )
         expected_pearsonr = [-0.21618688834430866, 0.680771605997119, 6]
         expected_spearmanr = [-0.11595420713048969, 0.826848213385815, 6]
         for i, val in enumerate(expected_pearsonr):
@@ -136,18 +150,26 @@ class TestCorrelations(unittest.TestCase):
     def test_calculate_results(self, literature_correlation):
 
         literature_correlation.return_value = {
-            'TT-1': ['GH-1', 0], 'TT-2': ['GH-2', 3], 'TT-3': ['GH-3', 1]}
+            "TT-1": ["GH-1", 0],
+            "TT-2": ["GH-2", 3],
+            "TT-3": ["GH-3", 1],
+        }
 
-        this_dataset = MockDataset(
-            {"group": AttributeSetter({"species": "rat"})})
-        target_dataset = MockDataset(
-            {"group": AttributeSetter({"species": "rat"})})
+        this_dataset = MockDataset({"group": AttributeSetter({"species": "rat"})})
+        target_dataset = MockDataset({"group": AttributeSetter({"species": "rat"})})
         this_trait = AttributeSetter({"geneid": "GH-1"})
         corr_params = {"type": "literature"}
         sorted_results = calculate_results(
-            this_trait=this_trait, this_dataset=this_dataset, target_dataset=target_dataset, corr_params=corr_params)
-        expected_results = {'TT-2': ['GH-2', 3],
-                            'TT-3': ['GH-3', 1], 'TT-1': ['GH-1', 0]}
+            this_trait=this_trait,
+            this_dataset=this_dataset,
+            target_dataset=target_dataset,
+            corr_params=corr_params,
+        )
+        expected_results = {
+            "TT-2": ["GH-2", 3],
+            "TT-3": ["GH-3", 1],
+            "TT-1": ["GH-1", 0],
+        }
 
         self.assertTrue(isinstance(sorted_results, OrderedDict))
         self.assertEqual(dict(sorted_results), expected_results)
diff --git a/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py b/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py
index 89442c47..a09d1538 100644
--- a/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py
+++ b/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py
@@ -17,89 +17,190 @@ class TestSnpBrowser(unittest.TestCase):
         self.app_context.pop()
 
     def test_get_header_list(self):
-        empty_columns = {"snp_source": "false", "conservation_score": "true", "gene_name": "false",
-                         "transcript": "false", "exon": "false", "domain_2": "true", "function": "false", "function_details": "true"}
+        empty_columns = {
+            "snp_source": "false",
+            "conservation_score": "true",
+            "gene_name": "false",
+            "transcript": "false",
+            "exon": "false",
+            "domain_2": "true",
+            "function": "false",
+            "function_details": "true",
+        }
         strains = {"mouse": ["S1", "S2", "S3", "S4", "S5"], "rat": []}
-        expected_results = ([['Index', 'SNP ID', 'Chr', 'Mb', 'Alleles', 'ConScore',
-                              'Domain 1', 'Domain 2', 'Details'],
-                             ['S1', 'S2', 'S3', 'S4', 'S5']], 5,
-                            ['index', 'snp_name', 'chr', 'mb_formatted', 'alleles',
-                             'conservation_score', 'domain_1', 'domain_2',
-                             'function_details', 'S1', 'S2', 'S3', 'S4', 'S5'])
+        expected_results = (
+            [
+                [
+                    "Index",
+                    "SNP ID",
+                    "Chr",
+                    "Mb",
+                    "Alleles",
+                    "ConScore",
+                    "Domain 1",
+                    "Domain 2",
+                    "Details",
+                ],
+                ["S1", "S2", "S3", "S4", "S5"],
+            ],
+            5,
+            [
+                "index",
+                "snp_name",
+                "chr",
+                "mb_formatted",
+                "alleles",
+                "conservation_score",
+                "domain_1",
+                "domain_2",
+                "function_details",
+                "S1",
+                "S2",
+                "S3",
+                "S4",
+                "S5",
+            ],
+        )
 
         results_with_snp = get_header_list(
-            variant_type="SNP", strains=strains, species="Mouse", empty_columns=empty_columns)
+            variant_type="SNP",
+            strains=strains,
+            species="Mouse",
+            empty_columns=empty_columns,
+        )
         results_with_indel = get_header_list(
-            variant_type="InDel", strains=strains, species="rat", empty_columns=[])
+            variant_type="InDel", strains=strains, species="rat", empty_columns=[]
+        )
         expected_results_with_indel = (
-            ['Index', 'ID', 'Type', 'InDel Chr', 'Mb Start',
-             'Mb End', 'Strand', 'Size', 'Sequence', 'Source'], 0,
-            ['index', 'indel_name', 'indel_type', 'indel_chr', 'indel_mb_s',
-             'indel_mb_e', 'indel_strand', 'indel_size', 'indel_sequence', 'source_name'])
+            [
+                "Index",
+                "ID",
+                "Type",
+                "InDel Chr",
+                "Mb Start",
+                "Mb End",
+                "Strand",
+                "Size",
+                "Sequence",
+                "Source",
+            ],
+            0,
+            [
+                "index",
+                "indel_name",
+                "indel_type",
+                "indel_chr",
+                "indel_mb_s",
+                "indel_mb_e",
+                "indel_strand",
+                "indel_size",
+                "indel_sequence",
+                "source_name",
+            ],
+        )
 
         self.assertEqual(expected_results, results_with_snp)
         self.assertEqual(expected_results_with_indel, results_with_indel)
 
-    @mock.patch("wqflask.snp_browser.snp_browser.g")
+    @mock.patch("wqflask.snp_browser.snp_browser.database_connection")
     def test_get_gene_id(self, mock_db):
-        mock_db.db.execute.return_value.fetchone.return_value = "517d729f-aa13-4413-a885-40a3f7ff768a"
-        db_query_value = """
-                SELECT
-                        geneId
-                FROM
-                        GeneList
-                WHERE
-                        SpeciesId = c9c0f59e-1259-4cba-91e6-831ef1a99c83 AND geneSymbol = 'INSR'
-            """
-        results = get_gene_id(
-            species_id="c9c0f59e-1259-4cba-91e6-831ef1a99c83", gene_name="INSR")
-        mock_db.db.execute.assert_called_once_with(db_query_value)
-        self.assertEqual(results, "517d729f-aa13-4413-a885-40a3f7ff768a")
+        db_query_value = (
+            "SELECT geneId FROM GeneList WHERE " "SpeciesId = %s AND geneSymbol = %s"
+        )
+        conn = mock.MagicMock()
+        mock_db.return_value.__enter__.return_value = conn
+        with conn.cursor() as cursor:
+            cursor.fetchone.return_value = (
+                ("517d729f-aa13-4413" "-a885-40a3f7ff768a"),
+            )
 
-    @mock.patch("wqflask.snp_browser.snp_browser.g")
+            results = get_gene_id(
+                species_id="c9c0f59e-1259-4cba-91e6-831ef1a99c83", gene_name="INSR"
+            )
+            cursor.execute.assert_called_once_with(
+                db_query_value, ("c9c0f59e-1259-4cba-91e6-831ef1a99c83", "INSR")
+            )
+            self.assertEqual(results, "517d729f-aa13-4413-a885-40a3f7ff768a")
+
+    @mock.patch("wqflask.snp_browser.snp_browser.database_connection")
     def test_gene_id_name_dict(self, mock_db):
         no_gene_names = []
-        self.assertEqual("", get_gene_id_name_dict(
-            species_id="fregb343bui43g4", gene_name_list=no_gene_names))
-        gene_name_list = ["GH1", "GH2", "GH3"]
-        mock_db.db.execute.return_value.fetchall.side_effect = [[], [("fsdf43-fseferger-f22", "GH1"), ("1sdf43-fsewferger-f22", "GH2"),
-                                                                     ("fwdj43-fstferger-f22", "GH3")]]
-        no_results = get_gene_id_name_dict(
-            species_id="ret3-32rf32", gene_name_list=gene_name_list)
-        results_found = get_gene_id_name_dict(
-            species_id="ret3-32rf32", gene_name_list=gene_name_list)
-        expected_found = {'GH1': 'fsdf43-fseferger-f22',
-                          'GH2': '1sdf43-fsewferger-f22', 'GH3': 'fwdj43-fstferger-f22'}
-        db_query_value = """
-                SELECT
-                        geneId, geneSymbol
-                FROM
-                        GeneList
-                WHERE
-                        SpeciesId = ret3-32rf32 AND geneSymbol in ('GH1','GH2','GH3')
-            """
-        mock_db.db.execute.assert_called_with(db_query_value)
-        self.assertEqual(results_found, expected_found)
-        self.assertEqual(no_results, {})
+        conn = mock.MagicMock()
+        mock_db.return_value.__enter__.return_value = conn
+        with conn.cursor() as cursor:
+            cursor.fetchall.side_effect = [
+                [],
+                [
+                    ("fsdf43-fseferger-f22", "GH1"),
+                    ("1sdf43-fsewferger-f22", "GH2"),
+                    ("fwdj43-fstferger-f22", "GH3"),
+                ],
+            ]
+            self.assertEqual(
+                "",
+                get_gene_id_name_dict(
+                    species_id="fregb343bui43g4", gene_name_list=no_gene_names
+                ),
+            )
+            gene_name_list = ["GH1", "GH2", "GH3"]
+            no_results = get_gene_id_name_dict(
+                species_id="ret3-32rf32", gene_name_list=gene_name_list
+            )
+            results_found = get_gene_id_name_dict(
+                species_id="ret3-32rf32", gene_name_list=gene_name_list
+            )
+            expected_found = {
+                "GH1": "fsdf43-fseferger-f22",
+                "GH2": "1sdf43-fsewferger-f22",
+                "GH3": "fwdj43-fstferger-f22",
+            }
+            db_query_value = (
+                "SELECT geneId, geneSymbol FROM GeneList WHERE "
+                "SpeciesId = %s AND geneSymbol in (%s, %s, %s)"
+            )
+            cursor.execute.assert_called_with(
+                db_query_value, ("ret3-32rf32", "GH1", "GH2", "GH3")
+            )
+            self.assertEqual(results_found, expected_found)
+            self.assertEqual(no_results, {})
 
-    @mock.patch("wqflask.snp_browser.snp_browser.g")
+    @mock.patch("wqflask.snp_browser.snp_browser.database_connection")
     def test_check_if_in_gene(self, mock_db):
-        mock_db.db.execute.return_value.fetchone.side_effect = [
-            ("fsdf-232sdf-sdf", "GHA"), ""]
-        results_found = check_if_in_gene(
-            species_id="517d729f-aa13-4413-a885-40a3f7ff768a", chr="CH1", mb=12.09)
-        db_query_value = """SELECT geneId, geneSymbol
-                   FROM GeneList
-                   WHERE SpeciesId = 517d729f-aa13-4413-a885-40a3f7ff768a AND chromosome = 'CH1' AND
-                        (txStart < 12.09 AND txEnd > 12.09); """
-        gene_not_found = check_if_in_gene(
-            species_id="517d729f-aa13-4413-a885-40a3f7ff768a", chr="CH1", mb=12.09)
-        mock_db.db.execute.assert_called_with(db_query_value)
-        self.assertEqual(gene_not_found, "")
+        conn = mock.MagicMock()
+        mock_db.return_value.__enter__.return_value = conn
+        with conn.cursor() as cursor:
+            cursor.fetchone.side_effect = [("fsdf-232sdf-sdf", "GHA"), ""]
+            results_found = check_if_in_gene(
+                species_id="517d729f-aa13-4413-a885-40a3f7ff768a", chr_="CH1", mb=12.09
+            )
+            self.assertEqual(results_found, ["fsdf-232sdf-sdf", "GHA"])
+            db_query_value = (
+                "SELECT geneId, geneSymbol FROM GeneList "
+                "WHERE SpeciesId = %s AND chromosome = %s "
+                "AND (txStart < %s AND txEnd > %s)"
+            )
+            gene_not_found = check_if_in_gene(
+                species_id="517d729f-aa13-4413-a885-40a3f7ff768a", chr_="CH1", mb=12.09
+            )
+            cursor.execute.assert_has_calls(
+                [
+                    mock.call(
+                        db_query_value,
+                        ("517d729f-aa13-4413-a885-40a3f7ff768a", "CH1", 12.09, 12.09),
+                    ),
+                    mock.call(
+                        db_query_value,
+                        ("517d729f-aa13-4413-a885-40a3f7ff768a", "CH1", 12.09, 12.09),
+                    ),
+                ]
+            )
+            self.assertEqual(gene_not_found, "")
 
-    @mock.patch("wqflask.snp_browser.snp_browser.g")
+    @mock.patch("wqflask.snp_browser.snp_browser.database_connection")
     def test_get_browser_sample_lists(self, mock_db):
-        mock_db.db.execute.return_value.fetchall.return_value = []
-
-        results = get_browser_sample_lists(species_id="12")
-        self.assertEqual(results, {'mouse': [], 'rat': []})
+        conn = mock.MagicMock()
+        mock_db.return_value.__enter__.return_value = conn
+        with conn.cursor() as cursor:
+            cursor.execute.return_value.fetchall.return_value = []
+            results = get_browser_sample_lists(species_id="12")
+            self.assertEqual(results, {"mouse": [], "rat": []})
diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py
index eddb97da..912cd04c 100644
--- a/wqflask/utility/svg.py
+++ b/wqflask/utility/svg.py
@@ -108,12 +108,6 @@ if use_dom_implementation != 0:
 # The implementation is used for the creating the XML document.
 # The prettyprint module is used for converting the xml document object to a xml file
 
-assert sys.version_info[0] >= 2
-if sys.version_info[1] < 2:
-    True = 1
-    False = 0
-    file = open
-
 sys.setrecursionlimit = 50
 # The recursion limit is set conservative so mistakes like s=svg() s.addElement(s)
 # won't eat up too much processor time.
diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py
index 05e040ed..118a7ff3 100644
--- a/wqflask/wqflask/__init__.py
+++ b/wqflask/wqflask/__init__.py
@@ -11,6 +11,8 @@ from utility import formatting
 
 from gn3.authentication import DataRole, AdminRole
 
+from wqflask.database import parse_db_url
+
 from wqflask.group_manager import group_management
 from wqflask.resource_manager import resource_management
 from wqflask.metadata_edits import metadata_edit
@@ -29,17 +31,6 @@ from wqflask.jupyter_notebooks import jupyter_notebooks
 app = Flask(__name__)
 
 
-# Helper function for getting the SQL objects
-def parse_db_url(sql_uri: str) -> Tuple:
-    """Parse SQL_URI env variable from an sql URI
-    e.g. 'mysql://user:pass@host_name/db_name'
-
-    """
-    parsed_db = urlparse(sql_uri)
-    return (parsed_db.hostname, parsed_db.username,
-            parsed_db.password, parsed_db.path[1:])
-
-
 # See http://flask.pocoo.org/docs/config/#configuring-from-files
 # Note no longer use the badly named WQFLASK_OVERRIDES (nyi)
 app.config.from_envvar('GN2_SETTINGS')
@@ -98,3 +89,4 @@ from wqflask import user_login
 from wqflask import user_session
 
 import wqflask.views
+import wqflask.partial_correlations_views
diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py
index 9b875c99..ab1e772a 100644
--- a/wqflask/wqflask/api/correlation.py
+++ b/wqflask/wqflask/api/correlation.py
@@ -3,11 +3,9 @@ import scipy
 
 from base import data_set
 from base.trait import create_trait, retrieve_sample_data
-from flask import g
 from utility import corr_result_helpers
-from utility.db_tools import escape
 from wqflask.correlation import correlation_functions
-
+from wqflask.database import database_connection
 
 def do_correlation(start_vars):
     assert('db' in start_vars)
@@ -125,22 +123,24 @@ def do_literature_correlation_for_all_traits(this_trait, target_dataset, trait_g
             target_dataset.group.species.lower(), gene_id)
 
         if mouse_gene_id and str(mouse_gene_id).find(";") == -1:
-            result = g.db.execute(
-                """SELECT value
-                    FROM LCorrRamin3
-                    WHERE GeneId1='%s' and
-                            GeneId2='%s'
-                """ % (escape(mouse_gene_id), escape(input_trait_mouse_gene_id))
-            ).fetchone()
-            if not result:
-                result = g.db.execute("""SELECT value
-                    FROM LCorrRamin3
-                    WHERE GeneId2='%s' and
-                            GeneId1='%s'
-                """ % (escape(mouse_gene_id), escape(input_trait_mouse_gene_id))
-                ).fetchone()
+            result = ""
+            with database_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(
+                        ("SELECT value FROM LCorrRamin3 "
+                         "WHERE GeneId1=%s AND GeneId2=%s"),
+                        (mouse_gene_id,
+                         input_trait_mouse_gene_id))
+                    result = cursor.fetchone()
+                    if not result:
+                        cursor.execute(
+                            ("SELECT value FROM LCorrRamin3 "
+                             "WHERE GeneId2=%s AND GeneId1=%s"),
+                            (mouse_gene_id,
+                             input_trait_mouse_gene_id))
+                        result = cursor.fetchone()
             if result:
-                lit_corr = result.value
+                lit_corr = result[0]
                 lit_corr_data[trait] = [gene_id, lit_corr]
             else:
                 lit_corr_data[trait] = [gene_id, 0]
@@ -195,30 +195,24 @@ def convert_to_mouse_gene_id(species=None, gene_id=None):
         return None
 
     mouse_gene_id = None
-
-    if species == 'mouse':
-        mouse_gene_id = gene_id
-
-    elif species == 'rat':
-
-        query = """SELECT mouse
-                FROM GeneIDXRef
-                WHERE rat='%s'""" % escape(gene_id)
-
-        result = g.db.execute(query).fetchone()
-        if result != None:
-            mouse_gene_id = result.mouse
-
-    elif species == 'human':
-
-        query = """SELECT mouse
-                FROM GeneIDXRef
-                WHERE human='%s'""" % escape(gene_id)
-
-        result = g.db.execute(query).fetchone()
-        if result != None:
-            mouse_gene_id = result.mouse
-
+    with database_connection() as conn:
+        with conn.cursor() as cursor:
+            if species == 'mouse':
+                mouse_gene_id = gene_id
+            elif species == 'rat':
+                cursor.execute(
+                    ("SELECT mouse FROM GeneIDXRef "
+                     "WHERE rat=%s"), gene_id)
+                result = cursor.fetchone()
+                if result:
+                    mouse_gene_id = result[0]
+            elif species == 'human':
+                cursor.execute(
+                    "SELECT mouse FROM GeneIDXRef "
+                    "WHERE human=%s", gene_id)
+                result = cursor.fetchone()
+                if result:
+                    mouse_gene_id = result[0]
     return mouse_gene_id
 
 
diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py
index a739e5a9..3d33cc87 100644
--- a/wqflask/wqflask/api/router.py
+++ b/wqflask/wqflask/api/router.py
@@ -6,7 +6,6 @@ import csv
 import json
 import datetime
 import requests
-import MySQLdb
 
 from zipfile import ZipFile, ZIP_DEFLATED
 
@@ -24,6 +23,8 @@ from wqflask.api import correlation, mapping, gen_menu
 
 from utility.tools import flat_files
 
+from wqflask.database import database_connection
+
 import utility.logger
 logger = utility.logger.getLogger(__name__)
 
@@ -847,11 +848,8 @@ def get_genotypes(group_name, file_format="csv", dataset_name=None):
 
 @app.route("/api/v_{}/gen_dropdown".format(version), methods=("GET",))
 def gen_dropdown_menu():
-    conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
-                           user=current_app.config.get("DB_USER"),
-                           passwd=current_app.config.get("DB_PASS"),
-                           host=current_app.config.get("DB_HOST"))
-    results = gen_menu.gen_dropdown_json(conn)
+    with database_connection() as conn:
+        results = gen_menu.gen_dropdown_json(conn)
 
     if len(results) > 0:
         return flask.jsonify(results)
diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py
index 76ef5ca4..815bb7c1 100644
--- a/wqflask/wqflask/collect.py
+++ b/wqflask/wqflask/collect.py
@@ -189,6 +189,30 @@ def delete_collection():
     return redirect(url_for('list_collections'))
 
 
+def trait_info_str(trait):
+    """Provide a string representation for given trait"""
+    def __trait_desc(trt):
+        if trait.dataset.type == "Geno":
+            return f"Marker: {trt.name}"
+        return trt.description_display or "N/A"
+
+    def __symbol(trt):
+        return (trt.symbol or trt.abbreviation or "N/A")[:20]
+
+    def __lrs(trt):
+        return (
+            f"{float(trait.LRS_score_repr):0.3f}" if float(trait.LRS_score_repr) > 0
+            else f"{trait.LRS_score_repr}")
+
+    def __location(trt):
+        if hasattr(trt, "location_repr"):
+            return trt.location_repr
+        return None
+
+    return "{}|||{}|||{}|||{}|||{}|||{:0.3f}|||{}|||{}".format(
+        trait.name, trait.dataset.name, __trait_desc(trait), __symbol(trait),
+        __location(trait), trait.mean, __lrs(trait), trait.LRS_location_repr)
+
 @app.route("/collections/view")
 def view_collection():
     params = request.args
@@ -222,14 +246,15 @@ def view_collection():
     collection_info = dict(
         trait_obs=trait_obs,
         uc=uc,
-        heatmap_data_url=f"{GN_SERVER_URL}heatmaps/clustered")
+        heatmap_data_url=f"{GN_SERVER_URL}api/heatmaps/clustered")
 
     if "json" in params:
         return json.dumps(json_version)
     else:
-        return render_template("collections/view.html",
-                               **collection_info
-                               )
+        return render_template(
+            "collections/view.html",
+            trait_info_str=trait_info_str,
+            **collection_info)
 
 
 @app.route("/collections/change_name", methods=('POST',))
diff --git a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py
index cb88eb53..438d2276 100644
--- a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py
+++ b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py
@@ -26,8 +26,6 @@ from utility import webqtlUtil, helper_functions, corr_result_helpers
 import utility.webqtlUtil  # this is for parallel computing only.
 from wqflask.correlation import correlation_functions
 
-from MySQLdb import escape_string as escape
-
 from flask import Flask, g
 
 
diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py
index 975a53b8..cb2f4470 100644
--- a/wqflask/wqflask/correlation/pre_computes.py
+++ b/wqflask/wqflask/correlation/pre_computes.py
@@ -6,29 +6,40 @@ from pathlib import Path
 from base.data_set import query_table_timestamp
 from base.webqtlConfig import TMPDIR
 
+from json.decoder import JSONDecodeError
+
 
 def fetch_all_cached_metadata(dataset_name):
     """in a gvein dataset fetch all the traits metadata"""
     file_name = generate_filename(dataset_name, suffix="metadata")
 
-    file_path = os.path.join(TMPDIR, file_name)
+    file_path = Path(TMPDIR, file_name)
 
     try:
         with open(file_path, "r+") as file_handler:
             dataset_metadata = json.load(file_handler)
+
             return (file_path, dataset_metadata)
 
     except FileNotFoundError:
-        Path(file_path).touch(exist_ok=True)
-        return (file_path, {})
+        pass
+
+    except JSONDecodeError:
+        file_path.unlink()
+
+    file_path.touch(exist_ok=True)
+
+    return (file_path, {})
 
 
 def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_path: str):
     """function to cache the new traits metadata"""
 
-    if bool(new_traits_metadata):
-        dataset_metadata.update(new_traits_metadata)
-             
+    if (dataset_metadata == {} and new_traits_metadata == {}):
+        return
+
+    dataset_metadata.update(new_traits_metadata)
+
     with open(file_path, "w+") as file_handler:
         json.dump(dataset_metadata, file_handler)
 
diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
index e7b16e77..cf37b9e9 100644
--- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
+++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
@@ -19,27 +19,26 @@
 # This module is used by GeneNetwork project (www.genenetwork.org)
 
 import datetime
-import math
 import random
 import string
-
-import rpy2.robjects as ro
-from rpy2.robjects.packages import importr
-
 import numpy as np
 import scipy
 
-from base import data_set
+from base.data_set import create_dataset
 from base.webqtlConfig import GENERATED_TEXT_DIR
-from functools import reduce
-from functools import cmp_to_key
-from utility import webqtlUtil
-from utility import helper_functions
-from utility import corr_result_helpers
+
+
+from utility.helper_functions import get_trait_db_obs
+from utility.corr_result_helpers import normalize_values
 from utility.redis_tools import get_redis_conn
 
-Redis = get_redis_conn()
-THIRTY_DAYS = 60 * 60 * 24 * 30
+
+from gn3.computations.pca import compute_pca
+from gn3.computations.pca import process_factor_loadings_tdata
+from gn3.computations.pca import generate_pca_temp_traits
+from gn3.computations.pca import cache_pca_dataset
+from gn3.computations.pca import generate_scree_plot_data
+
 
 class CorrelationMatrix:
 
@@ -47,11 +46,10 @@ class CorrelationMatrix:
         trait_db_list = [trait.strip()
                          for trait in start_vars['trait_list'].split(',')]
 
-        helper_functions.get_trait_db_obs(self, trait_db_list)
+        get_trait_db_obs(self, trait_db_list)
 
         self.all_sample_list = []
         self.traits = []
-        self.insufficient_shared_samples = False
         self.do_PCA = True
         # ZS: Getting initial group name before verifying all traits are in the same group in the following loop
         this_group = self.trait_list[0][1].group.name
@@ -116,7 +114,7 @@ class CorrelationMatrix:
                         if sample in self.shared_samples_list:
                             self.shared_samples_list.remove(sample)
 
-                this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(
+                this_trait_vals, target_vals, num_overlap = normalize_values(
                     this_trait_vals, target_vals)
 
                 if num_overlap < self.lowest_overlap:
@@ -165,16 +163,14 @@ class CorrelationMatrix:
 
         self.pca_works = "False"
         try:
-            corr_result_eigen = np.linalg.eig(np.array(self.pca_corr_results))
-            corr_eigen_value, corr_eigen_vectors = sortEigenVectors(
-                corr_result_eigen)
 
-            if self.do_PCA == True:
+            if self.do_PCA:
                 self.pca_works = "True"
                 self.pca_trait_ids = []
-                pca = self.calculate_pca(
-                    list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors)
-                self.loadings_array = self.process_loadings()
+                pca = self.calculate_pca()
+                self.loadings_array = process_factor_loadings_tdata(
+                    factor_loadings=self.loadings, traits_num=len(self.trait_list))
+
             else:
                 self.pca_works = "False"
         except:
@@ -182,71 +178,45 @@ class CorrelationMatrix:
 
         self.js_data = dict(traits=[trait.name for trait in self.traits],
                             groups=groups,
+                            scree_data = self.scree_data,
                             cols=list(range(len(self.traits))),
                             rows=list(range(len(self.traits))),
                             samples=self.all_sample_list,
                             sample_data=self.sample_data,)
 
-    def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors):
-        base = importr('base')
-        stats = importr('stats')
-
-        corr_results_to_list = ro.FloatVector(
-            [item for sublist in self.pca_corr_results for item in sublist])
-
-        m = ro.r.matrix(corr_results_to_list, nrow=len(cols))
-        eigen = base.eigen(m)
-        pca = stats.princomp(m, cor="TRUE")
-        self.loadings = pca.rx('loadings')
-        self.scores = pca.rx('scores')
-        self.scale = pca.rx('scale')
+    def calculate_pca(self):
 
-        trait_array = zScore(self.trait_data_array)
-        trait_array_vectors = np.dot(corr_eigen_vectors, trait_array)
+        pca = compute_pca(self.pca_corr_results)
 
-        pca_traits = []
-        for i, vector in enumerate(trait_array_vectors):
-            # ZS: Check if below check is necessary
-            # if corr_eigen_value[i-1] > 100.0/len(self.trait_list):
-            pca_traits.append((vector * -1.0).tolist())
+        self.loadings = pca["components"]
+        self.scores = pca["scores"]
+        self.pca_obj = pca["pca"]
 
         this_group_name = self.trait_list[0][1].group.name
-        temp_dataset = data_set.create_dataset(
-            dataset_name="Temp", dataset_type="Temp", group_name=this_group_name)
+        temp_dataset = create_dataset(
+            dataset_name="Temp", dataset_type="Temp",
+            group_name=this_group_name)
         temp_dataset.group.get_samplelist()
-        for i, pca_trait in enumerate(pca_traits):
-            trait_id = "PCA" + str(i + 1) + "_" + temp_dataset.group.species + "_" + \
-                this_group_name + "_" + datetime.datetime.now().strftime("%m%d%H%M%S")
-            this_vals_string = ""
-            position = 0
-            for sample in temp_dataset.group.all_samples_ordered():
-                if sample in self.shared_samples_list:
-                    this_vals_string += str(pca_trait[position])
-                    this_vals_string += " "
-                    position += 1
-                else:
-                    this_vals_string += "x "
-            this_vals_string = this_vals_string[:-1]
 
-            Redis.set(trait_id, this_vals_string, ex=THIRTY_DAYS)
-            self.pca_trait_ids.append(trait_id)
+        pca_temp_traits = generate_pca_temp_traits(species=temp_dataset.group.species, group=this_group_name,
+                                                   traits_data=self.trait_data_array, corr_array=self.pca_corr_results,
+                                                   dataset_samples=temp_dataset.group.all_samples_ordered(),
+                                                   shared_samples=self.shared_samples_list,
+                                                   create_time=datetime.datetime.now().strftime("%m%d%H%M%S"))
 
-        return pca
+        cache_pca_dataset(redis_conn=get_redis_conn(
+        ), exp_days=60 * 60 * 24 * 30, pca_trait_dict=pca_temp_traits)
 
-    def process_loadings(self):
-        loadings_array = []
-        loadings_row = []
-        for i in range(len(self.trait_list)):
-            loadings_row = []
-            if len(self.trait_list) > 2:
-                the_range = 3
-            else:
-                the_range = 2
-            for j in range(the_range):
-                position = i + len(self.trait_list) * j
-                loadings_row.append(self.loadings[0][position])
-            loadings_array.append(loadings_row)
-        return loadings_array
+        self.pca_trait_ids = list(pca_temp_traits.keys())
+
+        x_coord, y_coord = generate_scree_plot_data(
+            list(self.pca_obj.explained_variance_ratio_))
+
+        self.scree_data = {
+            "x_coord": x_coord,
+            "y_coord": y_coord
+        }
+        return pca
 
 
 def export_corr_matrix(corr_results):
@@ -261,11 +231,11 @@ def export_corr_matrix(corr_results):
         output_file.write("\n")
         output_file.write("Correlation ")
         for i, item in enumerate(corr_results[0]):
-            output_file.write("Trait" + str(i + 1) + ": " + \
+            output_file.write("Trait" + str(i + 1) + ": " +
                               str(item[0].dataset.name) + "::" + str(item[0].name) + "\t")
         output_file.write("\n")
         for i, row in enumerate(corr_results):
-            output_file.write("Trait" + str(i + 1) + ": " + \
+            output_file.write("Trait" + str(i + 1) + ": " +
                               str(row[0][0].dataset.name) + "::" + str(row[0][0].name) + "\t")
             for item in row:
                 output_file.write(str(item[1]) + "\t")
@@ -275,57 +245,14 @@ def export_corr_matrix(corr_results):
         output_file.write("\n")
         output_file.write("N ")
         for i, item in enumerate(corr_results[0]):
-            output_file.write("Trait" + str(i) + ": " + \
+            output_file.write("Trait" + str(i) + ": " +
                               str(item[0].dataset.name) + "::" + str(item[0].name) + "\t")
         output_file.write("\n")
         for i, row in enumerate(corr_results):
-            output_file.write("Trait" + str(i) + ": " + \
+            output_file.write("Trait" + str(i) + ": " +
                               str(row[0][0].dataset.name) + "::" + str(row[0][0].name) + "\t")
             for item in row:
                 output_file.write(str(item[2]) + "\t")
             output_file.write("\n")
 
     return corr_matrix_filename, matrix_export_path
-
-
-def zScore(trait_data_array):
-    NN = len(trait_data_array[0])
-    if NN < 10:
-        return trait_data_array
-    else:
-        i = 0
-        for data in trait_data_array:
-            N = len(data)
-            S = reduce(lambda x, y: x + y, data, 0.)
-            SS = reduce(lambda x, y: x + y * y, data, 0.)
-            mean = S / N
-            var = SS - S * S / N
-            stdev = math.sqrt(var / (N - 1))
-            if stdev == 0:
-                stdev = 1e-100
-            data2 = [(x - mean) / stdev for x in data]
-            trait_data_array[i] = data2
-            i += 1
-        return trait_data_array
-
-
-def sortEigenVectors(vector):
-    try:
-        eigenValues = vector[0].tolist()
-        eigenVectors = vector[1].T.tolist()
-        combines = []
-        i = 0
-        for item in eigenValues:
-            combines.append([eigenValues[i], eigenVectors[i]])
-            i += 1
-        sorted(combines, key=cmp_to_key(webqtlUtil.cmpEigenValue))
-        A = []
-        B = []
-        for item in combines:
-            A.append(item[0])
-            B.append(item[1])
-        sum = reduce(lambda x, y: x + y, A, 0.0)
-        A = [x * 100.0 / sum for x in A]
-        return [A, B]
-    except:
-        return []
diff --git a/wqflask/wqflask/ctl/gn3_ctl_analysis.py b/wqflask/wqflask/ctl/gn3_ctl_analysis.py
new file mode 100644
index 00000000..8f790597
--- /dev/null
+++ b/wqflask/wqflask/ctl/gn3_ctl_analysis.py
@@ -0,0 +1,132 @@
+import requests
+import itertools
+
+from utility import genofile_parser
+from utility.tools import GN3_LOCAL_URL
+from utility.tools import locate
+
+from base.trait import create_trait
+from base.trait import retrieve_sample_data
+from base import data_set
+
+
+def process_significance_data(dataset):
+    col_names = ["trait", "marker", "trait_2", "LOD", "dcor"]
+    dataset_rows = [[] for _ in range(len(dataset["trait"]))]
+    for col in col_names:
+        for (index, col_data) in enumerate(dataset[col]):
+            if col in ["dcor", "LOD"]:
+                dataset_rows[index].append(round(float(col_data), 2))
+            else:
+                dataset_rows[index].append(col_data)
+
+    return {
+        "col_names": col_names,
+        "data_set_rows": dataset_rows
+    }
+
+
+def parse_geno_data(dataset_group_name) -> dict:
+    """
+    Args:
+        dataset_group_name: string name
+
+    @returns : dict with keys genotypes,markernames & individuals
+    """
+    genofile_location = locate(dataset_group_name + ".geno", "genotype")
+    parser = genofile_parser.ConvertGenoFile(genofile_location)
+    parser.process_csv()
+    markers = []
+    markernames = []
+    for marker in parser.markers:
+        markernames.append(marker["name"])
+        markers.append(marker["genotypes"])
+
+    return {
+
+        "genotypes": list(itertools.chain(*markers)),
+        "markernames": markernames,
+        "individuals": parser.individuals
+
+
+    }
+
+
+def parse_phenotype_data(trait_list, dataset, individuals):
+    """
+    Args:
+        trait_list:list contains the traits
+        dataset:  object
+        individuals:a list contains the individual vals
+    Returns:
+           traits_db_List:parsed list of traits 
+           traits: list contains trait names
+           individuals
+
+    """
+
+    traits = []
+    for trait in trait_list:
+        if trait != "":
+            ts = trait.split(':')
+            gt = create_trait(name=ts[0], dataset_name=ts[1])
+            gt = retrieve_sample_data(gt, dataset, individuals)
+            for ind in individuals:
+                if ind in list(gt.data.keys()):
+                    traits.append(gt.data[ind].value)
+                else:
+                    traits.append("-999")
+
+    return {
+        "trait_db_list": trait_list,
+        "traits": traits,
+        "individuals": individuals
+    }
+
+
+def parse_form_data(form_data: dict):
+
+    trait_db_list = [trait.strip()
+                     for trait in form_data['trait_list'].split(',')]
+
+    form_data["trait_db_list"] = [x for x in trait_db_list if x]
+    form_data["nperm"] = int(form_data["nperm"])
+    form_data["significance"] = float(form_data["significance"])
+    form_data["strategy"] = form_data["strategy"].capitalize()
+
+    return form_data
+
+
+def run_ctl(requestform):
+    """function to make an api call
+    to gn3 and run ctl"""
+    ctl_api = f"{GN3_LOCAL_URL}/api/ctl/run_ctl"
+
+    form_data = parse_form_data(requestform.to_dict())
+    trait_db_list = form_data["trait_db_list"]
+    dataset = data_set.create_dataset(trait_db_list[0].split(":")[1])
+    geno_data = parse_geno_data(dataset.group.name)
+    pheno_data = parse_phenotype_data(
+        trait_db_list, dataset, geno_data["individuals"])
+
+    try:
+
+        response = requests.post(ctl_api, json={
+
+            "genoData": geno_data,
+            "phenoData": pheno_data,
+            **form_data,
+
+        })
+        if response.status_code != 200:
+            return {"error": response.json()}
+        response = response.json()["results"]
+        response["significance_data"] = process_significance_data(
+            response["significance_data"])
+
+        return response
+
+    except requests.exceptions.ConnectionError:
+        return {
+            "error": "A connection error to perform computation occurred"
+        }
diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py
index 11f8d287..9f659ae6 100644
--- a/wqflask/wqflask/database.py
+++ b/wqflask/wqflask/database.py
@@ -1,12 +1,33 @@
 # Module to initialize sqlalchemy with flask
+import os
+import sys
+from string import Template
+from typing import Tuple
+from urllib.parse import urlparse
+import importlib
+
+import MySQLdb
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.ext.declarative import declarative_base
-from utility.tools import SQL_URI
 
+def read_from_pyfile(pyfile, setting):
+    orig_sys_path = sys.path[:]
+    sys.path.insert(0, os.path.dirname(pyfile))
+    module = importlib.import_module(os.path.basename(pyfile).strip(".py"))
+    sys.path = orig_sys_path[:]
+    return module.__dict__.get(setting)
+
+def sql_uri():
+    """Read the SQL_URI from the environment or settings file."""
+    return os.environ.get(
+        "SQL_URI", read_from_pyfile(
+            os.environ.get(
+                "GN2_SETTINGS", os.path.abspath("../etc/default_settings.py")),
+            "SQL_URI"))
 
-engine = create_engine(SQL_URI, encoding="latin1")
+engine = create_engine(sql_uri(), encoding="latin1")
 
 db_session = scoped_session(sessionmaker(autocommit=False,
                                          autoflush=False,
@@ -16,3 +37,17 @@ Base.query = db_session.query_property()
 
 # Initialise the db
 Base.metadata.create_all(bind=engine)
+
+def parse_db_url(sql_uri: str) -> Tuple:
+    """
+    Parse SQL_URI env variable from an sql URI
+    e.g. 'mysql://user:pass@host_name/db_name'
+    """
+    parsed_db = urlparse(sql_uri)
+    return (parsed_db.hostname, parsed_db.username,
+            parsed_db.password, parsed_db.path[1:])
+
+def database_connection():
+    """Returns a database connection"""
+    host, user, passwd, db_name = parse_db_url(sql_uri())
+    return MySQLdb.Connect(db=db_name, user=user, passwd=passwd, host=host)
diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py
index 99272ee3..1e245d6a 100644
--- a/wqflask/wqflask/do_search.py
+++ b/wqflask/wqflask/do_search.py
@@ -1,6 +1,7 @@
-import string
-import requests
 import json
+import re
+import requests
+import string
 
 from flask import Flask, g
 
@@ -82,7 +83,7 @@ class MrnaAssaySearch(DoSearch):
     DoSearch.search_types['ProbeSet'] = "MrnaAssaySearch"
 
     base_query = """
-                SELECT
+                SELECT DISTINCT
                 ProbeSetFreeze.`Name`,
                 ProbeSetFreeze.`FullName`,
                 ProbeSet.`Name`,
@@ -137,15 +138,17 @@ class MrnaAssaySearch(DoSearch):
         search_string = escape(self.search_term[0])
 
         if self.search_term[0] != "*":
-            match_clause = """((MATCH (ProbeSet.Name,
+            if re.search("\w{1,2}\-\w+|\w+\-\w{1,2}", self.search_term[0]):
+                search_string = f'"{search_string}*"'
+
+            match_clause = f"""((MATCH (ProbeSet.Name,
                         ProbeSet.description,
                         ProbeSet.symbol,
                         alias,
                         GenbankId,
                         UniGeneId,
                         Probe_Target_Description)
-                        AGAINST ('%s' IN BOOLEAN MODE))) AND
-                                """ % (search_string)
+                        AGAINST ('{search_string}' IN BOOLEAN MODE))) AND """
         else:
             match_clause = ""
 
@@ -343,7 +346,7 @@ class GenotypeSearch(DoSearch):
                 GenoFreeze.createtime as thistable,
                 Geno.Name as Geno_Name,
                 Geno.Source2 as Geno_Source2,
-                Geno.chr_num as Geno_chr_num,
+                Geno.Chr as Geno_Chr,
                 Geno.Mb as Geno_Mb
                 FROM GenoXRef, GenoFreeze, Geno """
 
@@ -609,9 +612,6 @@ class PhenotypeLrsSearch(LrsSearch, PhenotypeSearch):
 
 class CisTransLrsSearch(DoSearch):
 
-    def get_from_clause(self):
-        return ", Geno"
-
     def get_where_clause(self, cis_trans):
         self.mb_buffer = 5  # default
         chromosome = None
@@ -989,8 +989,6 @@ def get_aliases(symbol, species):
 if __name__ == "__main__":
     # Usually this will be used as a library, but call it from the command line for testing
     # And it runs the code below
-
-    import MySQLdb
     import sys
 
     from base import webqtlConfig
@@ -998,15 +996,11 @@ if __name__ == "__main__":
     from utility import webqtlUtil
     from db import webqtlDatabaseFunction
 
-    db_conn = MySQLdb.Connect(db=webqtlConfig.DB_NAME,
-                              host=webqtlConfig.MYSQL_SERVER,
-                              user=webqtlConfig.DB_USER,
-                              passwd=webqtlConfig.DB_PASSWD)
-    cursor = db_conn.cursor()
-
-    dataset_name = "HC_M2_0606_P"
-    dataset = create_dataset(db_conn, dataset_name)
+    from wqflask.database import database_connection
 
-    results = PvalueSearch(['0.005'], '<', dataset, cursor, db_conn).run()
+    with database_connection() as db_conn:
+        with db_conn.cursor() as cursor:
+            dataset_name = "HC_M2_0606_P"
+            dataset = create_dataset(db_conn, dataset_name)
 
-    db_conn.close()
+            results = PvalueSearch(['0.005'], '<', dataset, cursor, db_conn).run()
diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py
index 7d76828e..a6d06af0 100644
--- a/wqflask/wqflask/jupyter_notebooks.py
+++ b/wqflask/wqflask/jupyter_notebooks.py
@@ -1,17 +1,31 @@
 from flask import Blueprint, render_template
 
-jupyter_notebooks = Blueprint('jupyter_notebooks', __name__)
+jupyter_notebooks = Blueprint("jupyter_notebooks", __name__)
+
 
 @jupyter_notebooks.route("/launcher", methods=("GET",))
 def launcher():
     links = (
-    {
-        "main_url": "http://notebook.genenetwork.org/34301/notebooks/genenetwork-api-using-r.ipynb",
-        "notebook_name": "R notebook showing how to query the GeneNetwork API.",
-        "src_link_url": "https://github.com/jgarte/genenetwork-api-r-jupyter-notebook"},
-    {
-        "main_url": "http://notebook.genenetwork.org/57675/notebooks/genenetwork.ipynb",
-        "notebook_name": "Querying the GeneNetwork API declaratively with python.",
-        "src_link_url": "https://github.com/jgarte/genenetwork-jupyter-notebook-example"})
+        {
+            "main_url": "http://notebook.genenetwork.org/58965/notebooks/2020-05-08/solberg-rat-analysis.ipynb",
+            "notebook_name": "Quantitative Genetics Tools for Mapping Trait Variation to Mechanisms, Therapeutics, and Interventions - Webinar Series",
+            "src_link_url": "https://github.com/senresearch/quant-genetics-webinars",
+        },
+        {
+            "main_url": "http://notebook.genenetwork.org/58163/notebooks/BXD%20Analysis.ipynb",
+            "notebook_name": "This shows how to model BXD mouse weight data using an AR(1) process.",
+            "src_link_url": "https://github.com/BonfaceKilz/tsaf-analysis-of-bxd-mouse-colonies",
+        },
+        {
+            "main_url": "http://notebook.genenetwork.org/46649/notebooks/genenetwork.ipynb",
+            "notebook_name": "Querying the GeneNetwork API declaratively with python.",
+            "src_link_url": "https://github.com/jgarte/genenetwork-jupyter-notebook-example",
+        },
+        {
+            "main_url": "http://notebook.genenetwork.org/37279/notebooks/genenetwork-api-using-r.ipynb",
+            "notebook_name": "R notebook showing how to query the GeneNetwork API.",
+            "src_link_url": "https://github.com/jgarte/genenetwork-api-r-jupyter-notebook",
+        },
+    )
 
     return render_template("jupyter_notebooks.html", links=links)
diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py
index 65896e06..c7a3451a 100644
--- a/wqflask/wqflask/marker_regression/rqtl_mapping.py
+++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py
@@ -1,6 +1,7 @@
 import csv
 import hashlib
 import io
+import json
 import requests
 import shutil
 from typing import Dict
@@ -10,6 +11,8 @@ from typing import TextIO
 
 import numpy as np
 
+from flask import g
+
 from base.webqtlConfig import TMPDIR
 from base.trait import create_trait
 from utility.tools import locate, GN3_LOCAL_URL
@@ -39,6 +42,10 @@ def run_rqtl(trait_name, vals, samples, dataset, pair_scan, mapping_scale, model
     if pair_scan:
         post_data["pairscan"] = True
 
+    if cofactors:
+        covarstruct_file = write_covarstruct_file(cofactors)
+        post_data["covarstruct"] = covarstruct_file
+
     if do_control == "true" and control_marker:
         post_data["control"] = control_marker
 
@@ -67,6 +74,32 @@ def get_hash_of_textio(the_file: TextIO) -> str:
     return hash_of_file
 
 
+def write_covarstruct_file(cofactors: str) -> str:
+    """
+    Given list of cofactors (as comma-delimited string), write
+    a comma-delimited file where the first column consists of cofactor names
+    and the second column indicates whether they're numerical or categorical
+    """
+    datatype_query = "SELECT value FROM TraitMetadata WHERE type='trait_data_type'"
+    trait_datatype_json = json.loads(g.db.execute(datatype_query).fetchone()[0])
+
+    covar_struct_file = io.StringIO()
+    writer = csv.writer(covar_struct_file, delimiter="\t", quoting = csv.QUOTE_NONE)
+    for cofactor in cofactors.split(","):
+        datatype = trait_datatype_json[cofactor] if cofactor in trait_datatype_json else "numerical"
+        cofactor_name = cofactor.split(":")[0]
+        writer.writerow([cofactor_name, datatype])
+
+    hash_of_file = get_hash_of_textio(covar_struct_file)
+    file_path = TMPDIR + hash_of_file + ".csv"
+
+    with open(file_path, "w") as fd:
+        covar_struct_file.seek(0)
+        shutil.copyfileobj(covar_struct_file, fd)
+
+    return file_path
+
+
 def write_phenotype_file(trait_name: str,
                          samples: List[str],
                          vals: List,
diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py
index dc738f88..30acf4d4 100644
--- a/wqflask/wqflask/metadata_edits.py
+++ b/wqflask/wqflask/metadata_edits.py
@@ -1,13 +1,11 @@
 import datetime
 import json
 import os
-import re
 
 from collections import namedtuple
 from itertools import groupby
 from typing import Dict
 
-import MySQLdb
 import difflib
 import redis
 
@@ -21,15 +19,18 @@ from flask import render_template
 from flask import request
 from flask import url_for
 
+from wqflask.database import database_connection
 from wqflask.decorators import edit_access_required
 from wqflask.decorators import edit_admins_access_required
 from wqflask.decorators import login_required
 
 from gn3.authentication import AdminRole
-from gn3.authentication import DataRole
 from gn3.authentication import get_highest_user_access_role
-from gn3.authentication import get_user_membership
-from gn3.commands import run_cmd
+from gn3.csvcmp import create_dirs_if_not_exists
+from gn3.csvcmp import csv_diff
+from gn3.csvcmp import extract_invalid_csv_headers
+from gn3.csvcmp import get_allowable_sampledata_headers
+from gn3.csvcmp import remove_insignificant_edits
 from gn3.db import diff_from_dict
 from gn3.db import fetchall
 from gn3.db import fetchone
@@ -41,10 +42,10 @@ from gn3.db.phenotypes import Probeset
 from gn3.db.phenotypes import Publication
 from gn3.db.phenotypes import PublishXRef
 from gn3.db.phenotypes import probeset_mapping
-from gn3.db.traits import get_trait_csv_sample_data
-from gn3.db.traits import update_sample_data
-from gn3.db.traits import delete_sample_data
-from gn3.db.traits import insert_sample_data
+from gn3.db.sample_data import delete_sample_data
+from gn3.db.sample_data import get_trait_csv_sample_data
+from gn3.db.sample_data import insert_sample_data
+from gn3.db.sample_data import update_sample_data
 
 
 metadata_edit = Blueprint('metadata_edit', __name__)
@@ -178,50 +179,41 @@ def edit_probeset(conn, name):
 @edit_access_required
 @login_required
 def display_phenotype_metadata(dataset_id: str, name: str):
-    conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
-                           user=current_app.config.get("DB_USER"),
-                           passwd=current_app.config.get("DB_PASS"),
-                           host=current_app.config.get("DB_HOST"))
-    _d = edit_phenotype(conn=conn, name=name, dataset_id=dataset_id)
-    return render_template(
-        "edit_phenotype.html",
-        diff=_d.get("diff"),
-        publish_xref=_d.get("publish_xref"),
-        phenotype=_d.get("phenotype"),
-        publication=_d.get("publication"),
-        dataset_id=dataset_id,
-        resource_id=request.args.get("resource-id"),
-        version=os.environ.get("GN_VERSION"),
-    )
+    with database_connection() as conn:
+        _d = edit_phenotype(conn=conn, name=name, dataset_id=dataset_id)
+        return render_template(
+            "edit_phenotype.html",
+            diff=_d.get("diff"),
+            publish_xref=_d.get("publish_xref"),
+            phenotype=_d.get("phenotype"),
+            publication=_d.get("publication"),
+            dataset_id=dataset_id,
+            resource_id=request.args.get("resource-id"),
+            headers=get_allowable_sampledata_headers(conn),
+            version=os.environ.get("GN_VERSION"),
+        )
 
 
 @metadata_edit.route("/traits/<name>")
 @edit_access_required
 @login_required
 def display_probeset_metadata(name: str):
-    conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
-                           user=current_app.config.get("DB_USER"),
-                           passwd=current_app.config.get("DB_PASS"),
-                           host=current_app.config.get("DB_HOST"))
-    _d = edit_probeset(conn=conn, name=name)
-    return render_template(
-        "edit_probeset.html",
-        diff=_d.get("diff"),
-        probeset=_d.get("probeset"),
-        name=name,
-        resource_id=request.args.get("resource-id"),
-        version=os.environ.get("GN_VERSION"),
-    )
+    with database_connection as conn:
+        _d = edit_probeset(conn=conn, name=name)
+        return render_template(
+            "edit_probeset.html",
+            diff=_d.get("diff"),
+            probeset=_d.get("probeset"),
+            name=name,
+            resource_id=request.args.get("resource-id"),
+            version=os.environ.get("GN_VERSION"),
+        )
 
 
 @metadata_edit.route("/<dataset_id>/traits/<name>", methods=("POST",))
 @edit_access_required
 @login_required
 def update_phenotype(dataset_id: str, name: str):
-    conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
-                           user=current_app.config.get("DB_USER"),
-                           passwd=current_app.config.get("DB_PASS"),
-                           host=current_app.config.get("DB_HOST"))
     data_ = request.form.to_dict()
     TMPDIR = current_app.config.get("TMPDIR")
     author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8")
@@ -230,52 +222,68 @@ def update_phenotype(dataset_id: str, name: str):
     if not (file_ := request.files.get("file")):
         flash("No sample-data has been uploaded", "warning")
     else:
-        if not os.path.exists(SAMPLE_DATADIR := os.path.join(TMPDIR, "sample-data")):
-            os.makedirs(SAMPLE_DATADIR)
-        if not os.path.exists(os.path.join(SAMPLE_DATADIR,
-                                           "diffs")):
-            os.makedirs(os.path.join(SAMPLE_DATADIR,
-                                     "diffs"))
-        if not os.path.exists(os.path.join(SAMPLE_DATADIR,
-                                           "updated")):
-            os.makedirs(os.path.join(SAMPLE_DATADIR,
-                                     "updated"))
+        create_dirs_if_not_exists([
+            SAMPLE_DATADIR := os.path.join(TMPDIR, "sample-data"),
+            DIFF_DATADIR := os.path.join(SAMPLE_DATADIR, "diffs"),
+            UPLOAD_DATADIR := os.path.join(SAMPLE_DATADIR, "updated")
+        ])
+
         current_time = str(datetime.datetime.now().isoformat())
         _file_name = (f"{author}.{request.args.get('resource-id')}."
                       f"{current_time}")
-        new_file_name = (os.path.join(TMPDIR,
-                                      f"sample-data/updated/{_file_name}.csv"))
-        uploaded_file_name = (os.path.join(
-            TMPDIR, "sample-data/updated/",
-            f"{_file_name}.csv.uploaded"))
-        file_.save(new_file_name)
-        with open(uploaded_file_name, "w") as f_:
-            f_.write(get_trait_csv_sample_data(
-                conn=conn,
-                trait_name=str(name),
-                phenotype_id=str(phenotype_id)))
-        r = run_cmd(cmd=("csvdiff "
-                         f"'{uploaded_file_name}' '{new_file_name}' "
-                         "--format json"))
-
+        diff_data = {}
+        with database_connection() as conn:
+            diff_data = remove_insignificant_edits(
+                diff_data=csv_diff(
+                    base_csv=(base_csv := get_trait_csv_sample_data(
+                        conn=conn,
+                        trait_name=str(name),
+                        phenotype_id=str(phenotype_id))),
+                    delta_csv=(delta_csv := file_.read().decode()),
+                    tmp_dir=TMPDIR),
+                epsilon=0.001)
+            headers = get_allowable_sampledata_headers(conn)
+            invalid_headers = extract_invalid_csv_headers(
+                    allowed_headers=headers,
+                    csv_text=delta_csv)
+            if invalid_headers:
+                flash("You have invalid headers: "
+                      f"""{', '.join(invalid_headers)}.  Valid headers """
+                      f"""are: {', '.join(headers)}""",
+                  "warning")
+                return redirect(
+                    f"/datasets/{dataset_id}/traits/{name}"
+                    f"?resource-id={request.args.get('resource-id')}")
         # Edge case where the csv file has not been edited!
-        if not any(json.loads(r.get("output")).values()):
-            flash(f"You have not modified the csv file you downloaded!",
+        if not any(diff_data.values()):
+            flash("You have not modified the csv file you downloaded!",
                   "warning")
             return redirect(f"/datasets/{dataset_id}/traits/{name}"
                             f"?resource-id={request.args.get('resource-id')}")
-        diff_output = (f"{TMPDIR}/sample-data/diffs/"
-                       f"{_file_name}.json")
-        with open(diff_output, "w") as f:
-            dict_ = json.loads(r.get("output"))
-            dict_.update({
+
+        with open(os.path.join(
+                UPLOAD_DATADIR,
+                f"{_file_name}.csv"), "w") as f_:
+            f_.write(base_csv)
+        with open(os.path.join(
+                UPLOAD_DATADIR,
+                f"{_file_name}.delta.csv"), "w") as f_:
+            f_.write(delta_csv)
+
+        with open(os.path.join(DIFF_DATADIR,
+                               f"{_file_name}.json"), "w") as f:
+            diff_data.update({
                 "trait_name": str(name),
                 "phenotype_id": str(phenotype_id),
+                "dataset_id": name,
+                "resource_id": request.args.get('resource-id'),
                 "author": author,
-                "timestamp": datetime.datetime.now().strftime(
-                    "%Y-%m-%d %H:%M:%S")
+                "timestamp": (datetime
+                              .datetime
+                              .now()
+                              .strftime("%Y-%m-%d %H:%M:%S")),
             })
-            f.write(json.dumps(dict_))
+            f.write(json.dumps(diff_data))
         flash("Sample-data has been successfully uploaded", "success")
     # Run updates:
     phenotype_ = {
@@ -290,10 +298,12 @@ def update_phenotype(dataset_id: str, name: str):
         "owner": data_.get("owner"),
         "authorized_users": data_.get("authorized-users"),
     }
-    updated_phenotypes = update(
-        conn, "Phenotype",
-        data=Phenotype(**phenotype_),
-        where=Phenotype(id_=data_.get("phenotype-id")))
+    updated_phenotypes = ""
+    with database_connection() as conn:
+        updated_phenotypes = update(
+            conn, "Phenotype",
+            data=Phenotype(**phenotype_),
+            where=Phenotype(id_=data_.get("phenotype-id")))
     diff_data = {}
     if updated_phenotypes:
         diff_data.update({"Phenotype": diff_from_dict(old={
@@ -309,11 +319,13 @@ def update_phenotype(dataset_id: str, name: str):
         "month": data_.get("month"),
         "year": data_.get("year")
     }
-    updated_publications = update(
-        conn, "Publication",
-        data=Publication(**publication_),
-        where=Publication(id_=data_.get("pubmed-id",
-                                        data_.get("old_id_"))))
+    updated_publications = ""
+    with database_connection() as conn:
+        updated_publications = update(
+            conn, "Publication",
+            data=Publication(**publication_),
+            where=Publication(id_=data_.get("pubmed-id",
+                                            data_.get("old_id_"))))
     if updated_publications:
         diff_data.update({"Publication": diff_from_dict(old={
             k: data_.get(f"old_{k}") for k, v in publication_.items()
@@ -329,11 +341,12 @@ def update_phenotype(dataset_id: str, name: str):
                           .now()
                           .strftime("%Y-%m-%d %H:%M:%S")),
         })
-        insert(conn,
-               table="metadata_audit",
-               data=MetadataAudit(dataset_id=name,
-                                  editor=author,
-                                  json_data=json.dumps(diff_data)))
+        with database_connection() as conn:
+            insert(conn,
+                   table="metadata_audit",
+                   data=MetadataAudit(dataset_id=name,
+                                      editor=author,
+                                      json_data=json.dumps(diff_data)))
         flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success")
     return redirect(f"/datasets/{dataset_id}/traits/{name}"
                     f"?resource-id={request.args.get('resource-id')}")
@@ -343,76 +356,71 @@ def update_phenotype(dataset_id: str, name: str):
 @edit_access_required
 @login_required
 def update_probeset(name: str):
-    conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
-                           user=current_app.config.get("DB_USER"),
-                           passwd=current_app.config.get("DB_PASS"),
-                           host=current_app.config.get("DB_HOST"))
-    data_ = request.form.to_dict()
-    probeset_ = {
-        "id_": data_.get("id"),
-        "symbol": data_.get("symbol"),
-        "description": data_.get("description"),
-        "probe_target_description": data_.get("probe_target_description"),
-        "chr_": data_.get("chr"),
-        "mb": data_.get("mb"),
-        "alias": data_.get("alias"),
-        "geneid": data_.get("geneid"),
-        "homologeneid": data_.get("homologeneid"),
-        "unigeneid": data_.get("unigeneid"),
-        "omim": data_.get("OMIM"),
-        "refseq_transcriptid": data_.get("refseq_transcriptid"),
-        "blatseq": data_.get("blatseq"),
-        "targetseq": data_.get("targetseq"),
-        "strand_probe": data_.get("Strand_Probe"),
-        "probe_set_target_region": data_.get("probe_set_target_region"),
-        "probe_set_specificity": data_.get("probe_set_specificity"),
-        "probe_set_blat_score": data_.get("probe_set_blat_score"),
-        "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"),
-        "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"),
-        "probe_set_strand": data_.get("probe_set_strand"),
-        "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"),
-        "flag": data_.get("flag")
-    }
-    diff_data = {}
-    author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8")
-              or g.user_session.record.get("user_id") or "")
-    if (updated_probeset := update(
-            conn, "ProbeSet",
-            data=Probeset(**probeset_),
-            where=Probeset(id_=data_.get("id")))):
-        diff_data.update({"Probeset": diff_from_dict(old={
-            k: data_.get(f"old_{k}") for k, v in probeset_.items()
-            if v is not None}, new=probeset_)})
-    if diff_data:
-        diff_data.update({"probeset_name": data_.get("probeset_name")})
-        diff_data.update({"author": author})
-        diff_data.update({"resource_id": request.args.get('resource-id')})
-        diff_data.update({"timestamp": datetime.datetime.now().strftime(
-            "%Y-%m-%d %H:%M:%S")})
-        insert(conn,
-               table="metadata_audit",
-               data=MetadataAudit(dataset_id=data_.get("id"),
-                                  editor=author,
-                                  json_data=json.dumps(diff_data)))
-    return redirect(f"/datasets/traits/{name}"
-                    f"?resource-id={request.args.get('resource-id')}")
+    with database_connection as conn:
+        data_ = request.form.to_dict()
+        probeset_ = {
+            "id_": data_.get("id"),
+            "symbol": data_.get("symbol"),
+            "description": data_.get("description"),
+            "probe_target_description": data_.get("probe_target_description"),
+            "chr_": data_.get("chr"),
+            "mb": data_.get("mb"),
+            "alias": data_.get("alias"),
+            "geneid": data_.get("geneid"),
+            "homologeneid": data_.get("homologeneid"),
+            "unigeneid": data_.get("unigeneid"),
+            "omim": data_.get("OMIM"),
+            "refseq_transcriptid": data_.get("refseq_transcriptid"),
+            "blatseq": data_.get("blatseq"),
+            "targetseq": data_.get("targetseq"),
+            "strand_probe": data_.get("Strand_Probe"),
+            "probe_set_target_region": data_.get("probe_set_target_region"),
+            "probe_set_specificity": data_.get("probe_set_specificity"),
+            "probe_set_blat_score": data_.get("probe_set_blat_score"),
+            "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"),
+            "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"),
+            "probe_set_strand": data_.get("probe_set_strand"),
+            "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"),
+            "flag": data_.get("flag")
+        }
+        diff_data = {}
+        author = ((g.user_session.record.get(b"user_id")
+                   or b"").decode("utf-8")
+                  or g.user_session.record.get("user_id") or "")
+        if update(conn, "ProbeSet",
+                  data=Probeset(**probeset_),
+                  where=Probeset(id_=data_.get("id"))):
+            diff_data.update({"Probeset": diff_from_dict(old={
+                k: data_.get(f"old_{k}") for k, v in probeset_.items()
+                if v is not None}, new=probeset_)})
+        if diff_data:
+            diff_data.update({"probeset_name": data_.get("probeset_name")})
+            diff_data.update({"author": author})
+            diff_data.update({"resource_id": request.args.get('resource-id')})
+            diff_data.update({"timestamp": datetime.datetime.now().strftime(
+                "%Y-%m-%d %H:%M:%S")})
+            insert(conn,
+                   table="metadata_audit",
+                   data=MetadataAudit(dataset_id=data_.get("id"),
+                                      editor=author,
+                                      json_data=json.dumps(diff_data)))
+        return redirect(f"/datasets/traits/{name}"
+                        f"?resource-id={request.args.get('resource-id')}")
 
 
 @metadata_edit.route("/<dataset_id>/traits/<phenotype_id>/csv")
 @login_required
-def get_sample_data_as_csv(dataset_id: str, phenotype_id:     int):
-    return Response(
-        get_trait_csv_sample_data(
-            conn=MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
-                                 user=current_app.config.get("DB_USER"),
-                                 passwd=current_app.config.get("DB_PASS"),
-                                 host=current_app.config.get("DB_HOST")),
-            trait_name=str(dataset_id),
-            phenotype_id=str(phenotype_id)),
-        mimetype="text/csv",
-        headers={"Content-disposition":
-                 "attachment; filename=myplot.csv"}
-    )
+def get_sample_data_as_csv(dataset_id: str, phenotype_id: int):
+    with database_connection() as conn:
+        return Response(
+            get_trait_csv_sample_data(
+                conn=conn,
+                trait_name=str(dataset_id),
+                phenotype_id=str(phenotype_id)),
+            mimetype="text/csv",
+            headers={"Content-disposition":
+                     f"attachment; filename=sample-data-{dataset_id}.csv"}
+        )
 
 
 @metadata_edit.route("/diffs")
@@ -470,88 +478,77 @@ def reject_data(resource_id: str, file_name: str):
 @metadata_edit.route("<resource_id>/diffs/<file_name>/approve")
 @edit_admins_access_required
 @login_required
-def approve_data(resource_id:str, file_name: str):
+def approve_data(resource_id: str, file_name: str):
     sample_data = {file_name: str}
-    conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
-                           user=current_app.config.get("DB_USER"),
-                           passwd=current_app.config.get("DB_PASS"),
-                           host=current_app.config.get("DB_HOST"))
     TMPDIR = current_app.config.get("TMPDIR")
     with open(os.path.join(f"{TMPDIR}/sample-data/diffs",
                            file_name), 'r') as myfile:
         sample_data = json.load(myfile)
-    for modification in (
-            modifications := [d for d in sample_data.get("Modifications")]):
-        if modification.get("Current"):
-            (strain_name,
-             value, se, count) = modification.get("Current").split(",")
-            update_sample_data(
-                conn=conn,
-                trait_name=sample_data.get("trait_name"),
-                strain_name=strain_name,
-                phenotype_id=int(sample_data.get("phenotype_id")),
-                value=value,
-                error=se,
-                count=count)
+    with database_connection() as conn:
+        for modification in (
+                modifications := [d for d in
+                                  sample_data.get("Modifications")]):
+            if modification.get("Current"):
+                update_sample_data(
+                    conn=conn,
+                    trait_name=sample_data.get("trait_name"),
+                    original_data=modification.get("Original"),
+                    updated_data=modification.get("Current"),
+                    csv_header=sample_data.get("Columns",
+                                               "Strain Name,Value,SE,Count"),
+                    phenotype_id=int(sample_data.get("phenotype_id")))
 
     n_deletions = 0
-    for deletion in (deletions := [d for d in sample_data.get("Deletions")]):
-        strain_name, _, _, _ = deletion.split(",")
-        __deletions, _, _ = delete_sample_data(
-            conn=conn,
-            trait_name=sample_data.get("trait_name"),
-            strain_name=strain_name,
-            phenotype_id=int(sample_data.get("phenotype_id")))
-        if __deletions:
-            n_deletions += 1
-        # Remove any data that already exists from sample_data deletes
-        else:
-            sample_data.get("Deletions").remove(deletion)
+    with database_connection() as conn:
+        for data in [d for d in sample_data.get("Deletions")]:
+            __deletions = delete_sample_data(
+                conn=conn,
+                trait_name=sample_data.get("trait_name"),
+                data=data,
+                csv_header=sample_data.get("Columns",
+                                           "Strain Name,Value,SE,Count"),
+                phenotype_id=int(sample_data.get("phenotype_id")))
+            if __deletions:
+                n_deletions += 1
+            # Remove any data that already exists from sample_data deletes
+            else:
+                sample_data.get("Deletions").remove(data)
 
     n_insertions = 0
-    for insertion in (
-            insertions := [d for d in sample_data.get("Additions")]):
-        (strain_name,
-         value, se, count) = insertion.split(",")
-        __insertions, _, _ = insert_sample_data(
-            conn=conn,
-            trait_name=sample_data.get("trait_name"),
-            strain_name=strain_name,
-            phenotype_id=int(sample_data.get("phenotype_id")),
-            value=value,
-            error=se,
-            count=count)
-        if __insertions:
-            n_insertions += 1
-        # Remove any data that already exists from sample_data inserts
-        else:
-            sample_data.get("Additions").remove(insertion)
+    with database_connection() as conn:
+        for data in [d for d in sample_data.get("Additions")]:
+            if insert_sample_data(
+                    conn=conn,
+                    trait_name=sample_data.get("trait_name"),
+                    data=data,
+                    csv_header=sample_data.get("Columns",
+                                               "Strain Name,Value,SE,Count"),
+                    phenotype_id=int(sample_data.get("phenotype_id"))):
+                n_insertions += 1
     if any([sample_data.get("Additions"),
             sample_data.get("Modifications"),
             sample_data.get("Deletions")]):
-        insert(conn,
-               table="metadata_audit",
-               data=MetadataAudit(
-                   dataset_id=sample_data.get("trait_name"),
-                   editor=sample_data.get("author"),
-                   json_data=json.dumps(sample_data)))
+        with database_connection() as conn:
+            insert(conn,
+                   table="metadata_audit",
+                   data=MetadataAudit(
+                       dataset_id=sample_data.get("trait_name"),
+                       editor=sample_data.get("author"),
+                       json_data=json.dumps(sample_data)))
         # Once data is approved, rename it!
         os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name),
                   os.path.join(f"{TMPDIR}/sample-data/diffs",
                                f"{file_name}.approved"))
-        message = ""
         if n_deletions:
             flash(f"# Deletions: {n_deletions}", "success")
         if n_insertions:
-            flash("# Additions: {len(modifications)", "success")
+            flash(f"# Additions: {len(modifications)}", "success")
         if len(modifications):
-            flash("# Modifications: {len(modifications)}", "success")
+            flash(f"# Modifications: {len(modifications)}", "success")
     else:  # Edge case where you need to automatically reject the file
         os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name),
                   os.path.join(f"{TMPDIR}/sample-data/diffs",
                                f"{file_name}.rejected"))
         flash(("Automatically rejecting this file since no "
                "changes could be applied."), "warning")
-
     return redirect(url_for('metadata_edit.list_diffs'))
-
diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py
index bd1c4407..7a808ac9 100644
--- a/wqflask/wqflask/parser.py
+++ b/wqflask/wqflask/parser.py
@@ -33,7 +33,7 @@ def parse(pstring):
     pstring = re.split(r"""(?:(\w+\s*=\s*[\('"\[][^)'"]*[\)\]'"])  |  # LRS=(1 2 3), cisLRS=[4 5 6], etc
                        (\w+\s*[=:\>\<][\w\*]+)  |  # wiki=bar, GO:foobar, etc
                        (".*?") | ('.*?') | # terms in quotes, i.e. "brain weight"
-                       ([\w\*\?]+))  # shh, brain, etc """, pstring,
+                       ([\w\*\?\-]+))  # shh, brain, etc """, pstring,
                        flags=re.VERBOSE)
 
     pstring = [item.strip() for item in pstring if item and item.strip()]
diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py
new file mode 100644
index 00000000..659b49e9
--- /dev/null
+++ b/wqflask/wqflask/partial_correlations_views.py
@@ -0,0 +1,285 @@
+import json
+import math
+import requests
+from functools import reduce
+from typing import Union, Tuple
+
+from flask import (
+    flash,
+    request,
+    url_for,
+    redirect,
+    current_app,
+    render_template)
+
+from wqflask import app
+from utility.tools import GN_SERVER_URL
+from wqflask.database import database_connection
+from gn3.db.partial_correlations import traits_info
+
+def publish_target_databases(conn, groups, threshold):
+    query = (
+        "SELECT PublishFreeze.FullName,PublishFreeze.Name "
+        "FROM PublishFreeze, InbredSet "
+        "WHERE PublishFreeze.InbredSetId = InbredSet.Id "
+        f"AND InbredSet.Name IN ({', '.join(['%s'] * len(groups))}) "
+        "AND PublishFreeze.public > %s")
+    with conn.cursor() as cursor:
+        cursor.execute(query, tuple(groups) + (threshold,))
+        res = cursor.fetchall()
+        if res:
+            return tuple(
+                dict(zip(("description", "value"), row)) for row in res)
+
+    return tuple()
+
+def geno_target_databases(conn, groups, threshold):
+    query = (
+        "SELECT GenoFreeze.FullName,GenoFreeze.Name "
+        "FROM GenoFreeze, InbredSet "
+        "WHERE GenoFreeze.InbredSetId = InbredSet.Id "
+        f"AND InbredSet.Name IN ({', '.join(['%s'] * len(groups))}) "
+        "AND GenoFreeze.public > %s")
+    with conn.cursor() as cursor:
+        cursor.execute(query, tuple(groups) + (threshold,))
+        res = cursor.fetchall()
+        if res:
+            return tuple(
+                dict(zip(("description", "value"), row)) for row in res)
+
+    return tuple()
+
+def probeset_target_databases(conn, groups, threshold):
+    query1 = "SELECT Id, Name FROM Tissue order by Name"
+    with conn.cursor() as cursor:
+        cursor.execute(query1)
+        tissue_res = cursor.fetchall()
+        if tissue_res:
+            tissue_ids = tuple(row[0] for row in tissue_res)
+            groups_clauses = ["InbredSet.Name like %s"] * len(groups)
+            query2 = (
+                "SELECT ProbeFreeze.TissueId, ProbeSetFreeze.FullName, "
+                "ProbeSetFreeze.Name "
+                "FROM ProbeSetFreeze, ProbeFreeze, InbredSet "
+                "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id "
+                "AND ProbeFreeze.TissueId IN "
+                f"({', '.join(['%s'] * len(tissue_ids))}) "
+                "AND ProbeSetFreeze.public > %s "
+                "AND ProbeFreeze.InbredSetId = InbredSet.Id "
+                f"AND ({' OR '.join(groups_clauses)}) "
+                "ORDER BY ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId")
+            cursor.execute(query2, tissue_ids + (threshold,) + tuple(groups))
+            db_res = cursor.fetchall()
+            if db_res:
+                databases = tuple(
+                    dict(zip(("tissue_id", "description", "value"), row))
+                    for row in db_res)
+                return tuple(
+                    {tissue_name: tuple(
+                        {
+                            "value": item["value"],
+                            "description": item["description"]
+                         } for item in databases
+                        if item["tissue_id"] == tissue_id)}
+                    for tissue_id, tissue_name in tissue_res)
+
+    return tuple()
+
+def target_databases(conn, traits, threshold):
+    """
+    Retrieves the names of possible target databases from the database.
+    """
+    trait_info = traits_info(
+        conn, threshold,
+        tuple(f"{trait['dataset']}::{trait['trait_name']}" for trait in traits))
+    groups = tuple(set(row["db"]["group"] for row in trait_info))
+    return (
+        publish_target_databases(conn, groups, threshold) +
+        geno_target_databases(conn, groups, threshold) +
+        probeset_target_databases(conn, groups, threshold))
+
+def primary_error(args):
+    if len(args["primary_trait"]) == 0 or len(args["primary_trait"]) > 1:
+        return {
+            **args,
+            "errors": (args.get("errors", tuple()) +
+                       ("You must provide one, and only one primary trait",))}
+    return args
+
+def controls_error(args):
+    if len(args["control_traits"]) == 0 or len(args["control_traits"]) > 3:
+        return {
+            **args,
+            "errors": (
+                args.get("errors", tuple()) +
+                (("You must provide at least one control trait, and a maximum "
+                  "of three control traits"),))}
+    return args
+
+def target_db_error(args):
+    if not args["target_db"]:
+        return {
+            **args,
+            "errors": (
+                args.get("errors", tuple()) +
+                ("The target database must be provided",))}
+    return args
+
+def method_error(args):
+    methods = (
+        "genetic correlation, pearson's r",
+        "genetic correlation, spearman's rho",
+        "sgo literature correlation",
+        "tissue correlation, pearson's r",
+        "tissue correlation, spearman's rho")
+    if not args["method"] or args["method"].lower() not in methods:
+        return {
+            **args,
+            "errors": (
+                args.get("errors", tuple()) +
+                ("Invalid correlation method provided",))}
+    return args
+
+def criteria_error(args):
+    try:
+        int(args.get("criteria", "invalid"))
+        return args
+    except ValueError:
+        return {
+            **args,
+            "errors": (
+                args.get("errors", tuple()) +
+                ("Invalid return number provided",))}
+
+def errors(args):
+    return criteria_error(method_error(target_db_error(controls_error(
+        primary_error(args)))))
+
+def __classify_args(acc, item):
+    if item[1].startswith("primary_"):
+        return {
+            **acc,
+            "primary_trait": (acc.get("primary_trait", tuple()) + (item,))} 
+    if item[1].startswith("controls_"):
+        return {**acc, "control_traits": (acc.get("control_traits", tuple()) + (item,))}
+    if item[0] == "target_db":
+        return {**acc, "target_db": item[1]}
+    if item[0] == "method":
+        return {**acc, "method": item[1]}
+    if item[0] == "criteria":
+        return {**acc, "criteria": item[1]}
+    return acc
+
+def __build_args(raw_form, traits):
+    args = reduce(__classify_args, raw_form.items(), {})
+    return {
+        **args,
+        "primary_trait": [
+            item for item in traits if item["trait_name"] in
+            (name[1][8:] for name in args["primary_trait"])],
+        "control_traits": [
+            item for item in traits if item["trait_name"] in
+            (name[1][9:] for name in args["control_traits"])]
+    }
+
+def parse_trait(trait_str):
+    return dict(zip(
+        ("trait_name", "dataset", "description", "symbol", "location", "mean",
+         "lrs", "lrs_location"),
+        trait_str.strip().split("|||")))
+
+def response_error_message(response):
+    error_messages = {
+        404: ("We could not connect to the API server at this time. "
+              "Try again later."),
+        500: ("The API server experienced a problem. We will be working on a "
+              "fix. Please try again later.")
+    }
+    return error_messages.get(
+        response.status_code,
+        "General API server error!!")
+
+def render_error(error_message):
+    return render_template(
+        "partial_correlations/pcorrs_error.html",
+        message = error_message)
+
+def handle_200_response(response):
+    if response["status"] == "success":
+        return redirect(
+            url_for(
+                "poll_partial_correlation_results",
+                command_id=response["results"]),
+            code=303)
+    return render_error(response["results"])
+
+def handle_response(response):
+    if response.status_code != 200:
+        return render_template(
+            "partial_correlations/pcorrs_error.html",
+            message = response_error_message(response))
+    return handle_200_response(response.json())
+
+@app.route("/partial_correlations", methods=["POST"])
+def partial_correlations():
+    form = request.form
+    traits = tuple(
+        parse_trait(trait) for trait in
+        form.get("trait_list").split(";;;"))
+
+    if form.get("submit") == "Run Partial Correlations":
+        args = errors(__build_args(form, traits))
+        if len(args.get("errors", [])) == 0:
+            post_data = {
+                **args,
+                "primary_trait": args["primary_trait"][0]
+            }
+            return handle_response(requests.post(
+                url=f"{GN_SERVER_URL}api/correlation/partial",
+                json=json.dumps(post_data)))
+
+        for error in args["errors"]:
+            flash(error, "alert-danger")
+
+    with database_connection() as conn:
+        target_dbs = target_databases(conn, traits, threshold=0)
+        return render_template(
+            "partial_correlations/pcorrs_select_operations.html",
+            trait_list_str=form.get("trait_list"),
+            traits=traits,
+            target_dbs=target_dbs)
+
+def process_pcorrs_command_output(result):
+    if result["status"] == "success":
+        def __format_number(num):
+            if num is None or math.isnan(num):
+                return ""
+            if abs(num) <= 1.04E-4:
+                return f"{num:.2e}"
+            return f"{num:.5f}"
+
+        return render_template(
+            "partial_correlations/pcorrs_results_presentation.html",
+            primary=result["results"]["primary_trait"],
+            controls=result["results"]["control_traits"],
+            correlations=result["results"]["correlations"],
+            dataset_type=result["results"]["dataset_type"],
+            method=result["results"]["method"],
+            format_number=__format_number)
+    if result["status"] == "error":
+        return render_error(
+            "The partial correlations computation failed with an error")
+
+@app.route("/partial_correlations/<command_id>", methods=["GET"])
+def poll_partial_correlation_results(command_id):
+    response = requests.get(
+        url=f"{GN_SERVER_URL}api/async_commands/state/{command_id}")
+    if response.status_code == 200:
+        data = response.json()
+        if data["status"] == "error":
+            return render_error(response["result"])
+        if data["status"] == "success":
+            return process_pcorrs_command_output(json.loads(data["result"]))
+        return render_template(
+            "partial_correlations/pcorrs_poll_results.html",
+            command_id = command_id)
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index cf2905c9..858ca56d 100644
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -148,7 +148,7 @@ class SearchResultPage:
                 trait_dict['name'] = trait_dict['display_name'] = str(result[0])
                 trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset']))
                 permissions = check_resource_availability(self.dataset, trait_dict['display_name'])
-                if "view" not in permissions['data']:
+                if not any(x in permissions['data'] for x in ["view", "edit"]):
                     continue
 
                 if result[10]:
@@ -203,8 +203,8 @@ class SearchResultPage:
             for i, trait in enumerate(trait_list):
                 for key in trait.keys():
                     if key == "authors":
-                        authors_string = ",".join(str(trait[key]).split(",")[:6]) + ", et al."
-                        self.max_widths[key] = max(len(authors_string), self.max_widths[key]) if key in self.max_widths else len(str(trait[key]))
+                        authors_string = ",".join(str(trait[key]).split(",")[:2]) + ", et al."
+                        self.max_widths[key] = max(len(authors_string), self.max_widths[key]) if key in self.max_widths else len(str(authors_string))
                     else:
                         self.max_widths[key] = max(len(str(trait[key])), self.max_widths[key]) if key in self.max_widths else len(str(trait[key]))
 
@@ -360,7 +360,8 @@ def get_aliases(symbol_list, species):
 
     filtered_aliases = []
     response = requests.get(
-        GN2_BASE_URL + "/gn3/gene/aliases2/" + symbols_string)
+        GN2_BASE_URL + "gn3/gene/aliases/" + symbols_string)
+
     if response:
         alias_lists = json.loads(response.content)
         seen = set()
diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py
index c4d0e135..43bb55b5 100644
--- a/wqflask/wqflask/snp_browser/snp_browser.py
+++ b/wqflask/wqflask/snp_browser/snp_browser.py
@@ -3,12 +3,11 @@ from flask import Flask, g, url_for
 import string
 from PIL import (Image)
 
-from utility.logger import getLogger
-logger = getLogger(__name__)
-
 from base import species
 from base import webqtlConfig
 
+from wqflask.database import database_connection
+
 
 class SnpBrowser:
 
@@ -660,32 +659,27 @@ class SnpBrowser:
         x_scale = plot_width / (self.end_mb - self.start_mb)
 
         # draw clickable image map at some point
-
         n_click = 80.0
         click_step = plot_width / n_click
         click_mb_step = (self.end_mb - self.start_mb) / n_click
 
-        # for i in range(n_click):
-        #    href = url_for('snp_browser', first_run="false", chosen_strains_mouse=self.chosen_strains_mouse, chosen_strains_rat=self.chosen_strains_rat, variant=self.variant_type, species=self.species_name, gene_name=self.gene_name, chr=self.chr, start_mb=self.start_mb, end_mb=self.end_mb, limit_strains=self.limit_strains, domain=self.domain, function=self.function, criteria=self.criteria, score=self.score, diff_alleles=self.diff_alleles)
-
 
 def get_browser_sample_lists(species_id=1):
     strain_lists = {}
     mouse_strain_list = []
-    query = "SHOW COLUMNS FROM SnpPattern;"
-    results = g.db.execute(query).fetchall()
-    for result in results[1:]:
-        mouse_strain_list.append(result[0])
-
     rat_strain_list = []
-    query = "SHOW COLUMNS FROM RatSnpPattern;"
-    results = g.db.execute(query).fetchall()
-    for result in results[2:]:
-        rat_strain_list.append(result[0])
-
-    strain_lists['mouse'] = mouse_strain_list
-    strain_lists['rat'] = rat_strain_list
-
+    with database_connection() as conn:
+        with conn.cursor() as cursor:
+            cursor.execute("SHOW COLUMNS FROM SnpPattern")
+            _mouse_snp_pattern = cursor.fetchall()
+            cursor.execute("SHOW COLUMNS FROM RatSnpPattern")
+            _rats_snp_pattern = cursor.fetchall()
+            for result in _mouse_snp_pattern[1:]:
+                mouse_strain_list.append(result[0])
+            for result in _rats_snp_pattern[2:]:
+                rat_strain_list.append(result[0])
+            strain_lists['mouse'] = mouse_strain_list
+            strain_lists['rat'] = rat_strain_list
     return strain_lists
 
 
@@ -891,64 +885,51 @@ def get_effect_info(effect_list):
 
 
 def get_gene_id(species_id, gene_name):
-    query = """
-                SELECT
-                        geneId
-                FROM
-                        GeneList
-                WHERE
-                        SpeciesId = %s AND geneSymbol = '%s'
-            """ % (species_id, gene_name)
-
-    result = g.db.execute(query).fetchone()
-
-    if len(result) > 0:
-        return result
-    else:
-        return ""
+    query = ("SELECT geneId FROM GeneList WHERE "
+             "SpeciesId = %s AND geneSymbol = %s")
+
+    with database_connection() as conn:
+        with conn.cursor() as cursor:
+            cursor.execute(query, (species_id, gene_name))
+            if (result := cursor.fetchone()):
+                return result[0]
+    return ""
 
 
 def get_gene_id_name_dict(species_id, gene_name_list):
     gene_id_name_dict = {}
     if len(gene_name_list) == 0:
         return ""
-    gene_name_str_list = ["'" + gene_name + \
-                          "'" for gene_name in gene_name_list]
-    gene_name_str = ",".join(gene_name_str_list)
-
-    query = """
-                SELECT
-                        geneId, geneSymbol
-                FROM
-                        GeneList
-                WHERE
-                        SpeciesId = %s AND geneSymbol in (%s)
-            """ % (species_id, gene_name_str)
-
-    results = g.db.execute(query).fetchall()
-
-    if len(results) > 0:
-        for item in results:
-            gene_id_name_dict[item[1]] = item[0]
-
+    query = ("SELECT geneId, geneSymbol FROM "
+             "GeneList WHERE SpeciesId = %s AND "
+             f"geneSymbol in ({', '.join(['%s'] * len(gene_name_list))})")
+    with database_connection() as conn:
+        with conn.cursor() as cursor:
+            cursor.execute(query, (species_id, *gene_name_list))
+            results = cursor.fetchall()
+            if results:
+                for item in results:
+                    gene_id_name_dict[item[1]] = item[0]
     return gene_id_name_dict
 
 
-def check_if_in_gene(species_id, chr, mb):
-    if species_id != 0:  # ZS: Check if this is necessary
-        query = """SELECT geneId, geneSymbol
-                   FROM GeneList
-                   WHERE SpeciesId = {0} AND chromosome = '{1}' AND
-                        (txStart < {2} AND txEnd > {2}); """.format(species_id, chr, mb)
-    else:
-        query = """SELECT geneId,geneSymbol
-                   FROM GeneList
-                   WHERE chromosome = '{0}' AND
-                        (txStart < {1} AND txEnd > {1}); """.format(chr, mb)
-
-    result = g.db.execute(query).fetchone()
-
-    if result:
-        return [result[0], result[1]]
-    else:
-        return ""
+def check_if_in_gene(species_id, chr_, mb):
+    with database_connection() as conn:
+        with conn.cursor() as cursor:
+            if species_id != 0:  # ZS: Check if this is necessary
+                cursor.execute(
+                    "SELECT geneId, geneSymbol "
+                    "FROM GeneList WHERE "
+                    "SpeciesId = %s AND chromosome = %s "
+                    "AND (txStart < %s AND txEnd > %s)",
+                    (species_id, chr_, mb, mb))
+            else:
+                cursor.execute(
+                    "SELECT geneId,geneSymbol "
+                    "FROM GeneList WHERE "
+                    "chromosome = %s AND "
+                    "(txStart < %s AND txEnd > %s)",
+                    (chr_, mb, mb))
+            if (result := cursor.fetchone()):
+                return [result[0], result[1]]
+    return ""
diff --git a/wqflask/wqflask/static/new/css/partial_correlations.css b/wqflask/wqflask/static/new/css/partial_correlations.css
new file mode 100644
index 00000000..84a0877f
--- /dev/null
+++ b/wqflask/wqflask/static/new/css/partial_correlations.css
@@ -0,0 +1,109 @@
+#partial-correlations-form {
+    width: 100%;
+    display: grid;
+    grid-column-gap: 1em;
+}
+
+#main-form {
+    grid-column-start: 1;
+    grid-column-end: 2;
+    text-align: left;
+}
+
+#form-display-area {
+    grid-column-start: 2;
+    grid-column-end: 3;
+}
+
+#part-corr-success {
+    grid-column-start: 1;
+    grid-column-end: 3;
+}
+
+td, th {
+    border: 1px solid;
+    text-align: left;
+    padding: 0.2em 0.5em 0.2em 0.7em;
+}
+
+tr:nth-of-type(2n) {
+    background: #F9F9F9;
+}
+
+.with-trait {
+    margin-left: 0.7em;
+    position: relative;
+    display: grid;
+    width: 100%;
+    grid-column-gap: 1em;
+    grid-template-columns: 1em 1fr;
+    text-align: left;
+}
+
+.with-trait:nth-of-type(2n) {
+    background: #E5E5FF;
+}
+
+.with-trait .selector-element {
+    grid-column: 1 / 2;
+    grid-row: 1 / 1;
+}
+
+.with-trait:first-of-type {
+    padding-top: 2.5em;
+}
+
+.with-trait:first-of-type label *:before{
+    position: absolute;
+    top: 0px;
+    
+    text-transform: capitalize;
+    font-weight: bolder;
+    content: attr(data-title);
+    background: #336699; /*#FFCCCC;*/
+    color: #FFFFFF;
+    padding: 0.5em;
+}
+
+.with-trait .label-element {
+    display: grid;
+    grid-column-gap: 0.5em;
+    grid-template-columns: 4fr 2fr 2fr 9fr 2fr 1fr 2fr;
+    grid-column: 2 / 2;
+    grid-row: 1 / 1;
+}
+
+.with-trait .label-element .trait-dataset {
+    grid-column: 1;
+    grid-row: 1 / 1;
+}
+
+.with-trait .label-element .trait-name {
+    grid-column: 2;
+    grid-row: 1 / 1;
+}
+
+.with-trait .label-element .trait-symbol {
+    grid-column: 3;
+    grid-row: 1 / 1;
+}
+
+.with-trait .label-element .trait-description {
+    grid-column: 4;
+    grid-row: 1 / 1;
+}
+
+.with-trait .label-element .trait-location {
+    grid-column: 5;
+    grid-row: 1 / 1;
+}
+
+.with-trait .label-element .trait-mean-expr {
+    grid-column: 6;
+    grid-row: 1 / 1;
+}
+
+.with-trait .label-element .trait-max-lrs {
+    grid-column: 7;
+    grid-row: 1 / 1;
+}
diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js
new file mode 100644
index 00000000..5de1204c
--- /dev/null
+++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js
@@ -0,0 +1,26 @@
+function selected_traits() {
+    traits = $("#trait_table input:checked").map(function() {
+      return $(this).attr("data-trait-info");
+    }).get();
+    if (traits.length == 0){
+      num_traits = $("#trait_table input").length
+      if (num_traits <= 100){
+        traits = $("#trait_table input").map(function() {
+          return $(this).attr("data-trait-info");
+        }).get();
+      }
+    }
+    return traits
+}
+
+$("#partial-correlations").on("click", function() {
+    // Submit the form to the `partial_correlations` endpoint
+    url = $(this).data("url")
+    traits = selected_traits();
+    $("#trait_list").val(traits.reduce(function(acc, str) {
+	return acc.concat(";;;".concat(str));
+    }));
+    $("input[name=tool_used]").val("Partial Correlation")
+    $("input[name=form_url]").val(url)
+    return submit_special(url)
+})
diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js
index 48b9b7be..ff2452af 100644
--- a/wqflask/wqflask/static/new/javascript/search_results.js
+++ b/wqflask/wqflask/static/new/javascript/search_results.js
@@ -25,10 +25,9 @@ change_buttons = function(check_node = 0) {
 };
 
 $(function() {
-  var add, checked_traits, deselect_all, invert, remove, removed_traits, select_all;
+  let selectAll, deselectAll, invert;
 
-  checked_traits = null;
-  select_all = function() {
+  selectAll = function() {
     table_api = $('#trait_table').DataTable();
 
     check_cells = table_api.column(0).nodes().to$();
@@ -44,7 +43,7 @@ $(function() {
     change_buttons();
   };
 
-  deselect_all = function() {
+  deselectAll = function() {
     table_api = $('#trait_table').DataTable();
 
     check_cells = table_api.column(0).nodes().to$();
@@ -93,48 +92,91 @@ $(function() {
       $('#trait_table').DataTable().search($(this).val()).draw();
   });
 
-  $('#select_top').keyup(function(){
-      num_rows = $(this).val()
-
-      if (num_rows = parseInt(num_rows)){
-          table_api = $('#trait_table').DataTable();
+  /**
+   * parseIndexString takes a string consisting of integers,
+   * hyphens, and/or commas to indicate range(s) of indices
+   * to select a rows and returns the corresponding set of indices
+   * For example - "1, 5-10, 15" would return a set of 8 rows
+   * @return {Set} The list of indices as a Set
+   */
+  parseIndexString = function(idx_string) {
+    index_list = [];
+
+    _ref = idx_string.split(",");
+    for (_i = 0; _i < _ref.length; _i++) {
+      index_set = _ref[_i];
+      if (!/^ *([0-9]+$) *| *([0-9]+ *- *[0-9]+$) *|(^$)$/.test(index_set)) {
+        $('#select_samples_invalid').show();
+        break
+      } else {
+        $('#select_samples_invalid').hide();
+      }
+      if (index_set.indexOf('-') !== -1) {
+          start_index = parseInt(index_set.split("-")[0]);
+          end_index = parseInt(index_set.split("-")[1]);
 
-          check_cells = table_api.column(0).nodes().to$();
-          for (let i = 0; i < num_rows; i++) {
-            check_cells[i].childNodes[0].checked = true;
+          // If start index is higher than end index (for example is the string "10-5" exists) swap values so it'll be interpreted as "5-10" instead
+          if (start_index > end_index) {
+            [start_index, end_index] = [end_index, start_index]
           }
 
-          check_rows = table_api.rows().nodes();
-          for (let i=0; i < num_rows; i++) {
-            if (check_rows[i].classList.contains("selected")){
-              continue
-            } else {
-              check_rows[i].classList.add("selected")
-            }
-          }
-          for (let i = num_rows; i < check_rows.length; i++){
-            check_cells[i].childNodes[0].checked = false;
-            if (check_rows[i].classList.contains("selected")){
-              check_rows[i].classList.remove("selected")
-            }
+          for (index = start_index; index <= end_index; index++) {
+            index_list.push(index);
           }
+      } else {
+        index = parseInt(index_set);
+        index_list.push(index);
       }
-      else {
-        for (let i = 0; i < check_rows.length; i++){
-          check_cells[i].childNodes[0].checked = false;
-          if (check_rows[i].classList.contains("selected")){
-            check_rows[i].classList.remove("selected")
-          }
-        }
+    }
+    return new Set(index_list)
+  }
+
+  filterByIndex = function() {
+    indexString = $('#select_top').val()
+    indexSet = parseIndexString(indexString)
+
+    tableApi = $('#trait_table').DataTable();
+    checkNodes = tableApi.column(0).nodes().to$();
+    checkNodes.each(function(index) {
+      if (indexSet.has(index + 1)){
+        $(this)[0].childNodes[0].checked = true
       }
-      change_buttons();
+    })
+
+    checkRows = tableApi.rows().nodes().to$();
+    checkRows.each(function(index) {
+      if (indexSet.has(index + 1)){
+        $(this)[0].classList.add("selected");
+      }
+    })
+  }
+
+  $(window).keydown(function(event){
+    if((event.keyCode == 13)) {
+      event.preventDefault();
+      return false;
+    }
+  });
+
+  $('#select_top').keyup(function(event){
+    if (event.keyCode === 13) {
+      filterByIndex()
+    }
   });
 
-  add_to_collection = function() {
+  $('#select_top').blur(function() {
+    filterByIndex()
+  });
+
+  addToCollection = function() {
     var traits;
-    traits = $("#trait_table input:checked").map(function() {
-      return $(this).val();
-    }).get();
+    table_api = $('#trait_table').DataTable();
+    check_nodes = table_api.column(0).nodes().to$();
+    traits = Array.from(check_nodes.map(function() {
+      if ($(this)[0].childNodes[0].checked){
+        return $(this)[0].childNodes[0].value
+      }
+    }))
 
     var traits_hash = md5(traits.toString());
 
@@ -153,19 +195,15 @@ $(function() {
 
   };
 
-  removed_traits = function() {
-    return checked_traits.closest("tr").fadeOut();
-  };
-
-  submit_bnw = function() {
-    trait_data = submit_traits_to_export_or_bnw("trait_table", "submit_bnw")
+  submitBnw = function() {
+    trait_data = submitTraitsToExportOrBnw("trait_table", "submit_bnw")
   }
 
-  export_traits = function() {
-    trait_data = submit_traits_to_export_or_bnw("trait_table", "export_csv")
+  exportTraits = function() {
+    trait_data = submitTraitsToExportOrBnw("trait_table", "export_csv")
   };
 
-  submit_traits_to_export_or_bnw = function(table_name, destination) {
+  submitTraitsToExportOrBnw = function(table_name, destination) {
     trait_table = $('#'+table_name);
     table_dict = {};
 
@@ -206,7 +244,7 @@ $(function() {
     $('#export_form').submit();
   };
 
-  get_traits_from_table = function(){
+  getTraitsFromTable = function(){
     traits = $("#trait_table input:checked").map(function() {
       return $(this).val();
     }).get();
@@ -222,42 +260,42 @@ $(function() {
   }
 
   $("#corr_matrix").on("click", function() {
-      traits = get_traits_from_table()
+      traits = getTraitsFromTable()
       $("#trait_list").val(traits)
       $("input[name=tool_used]").val("Correlation Matrix")
       $("input[name=form_url]").val($(this).data("url"))
       return submit_special("/loading")
   });
   $("#network_graph").on("click", function() {
-      traits = get_traits_from_table()
+      traits = getTraitsFromTable()
       $("#trait_list").val(traits)
       $("input[name=tool_used]").val("Network Graph")
       $("input[name=form_url]").val($(this).data("url"))
       return submit_special("/loading")
   });
   $("#wgcna_setup").on("click", function() {
-      traits = get_traits_from_table()
+      traits = getTraitsFromTable()
       $("#trait_list").val(traits)
       $("input[name=tool_used]").val("WGCNA Setup")
       $("input[name=form_url]").val($(this).data("url"))
       return submit_special("/loading")
   });
   $("#ctl_setup").on("click", function() {
-      traits = get_traits_from_table()
+      traits = getTraitsFromTable()
       $("#trait_list").val(traits)
       $("input[name=tool_used]").val("CTL Setup")
       $("input[name=form_url]").val($(this).data("url"))
       return submit_special("/loading")
   });
   $("#heatmap").on("click", function() {
-      traits = get_traits_from_table()
+      traits = getTraitsFromTable()
       $("#trait_list").val(traits)
       $("input[name=tool_used]").val("Heatmap")
       $("input[name=form_url]").val($(this).data("url"))
       return submit_special("/loading")
   });
   $("#comp_bar_chart").on("click", function() {
-      traits = get_traits_from_table()
+      traits = getTraitsFromTable()
       $("#trait_list").val(traits)
       $("input[name=tool_used]").val("Comparison Bar Chart")
       $("input[name=form_url]").val($(this).data("url"))
@@ -265,32 +303,32 @@ $(function() {
   });
 
   $("#send_to_webgestalt, #send_to_bnw, #send_to_geneweaver").on("click", function() {
-      traits = get_traits_from_table()
+      traits = getTraitsFromTable()
       $("#trait_list").val(traits)
       url = $(this).data("url")
       return submit_special(url)
   });
 
 
-  $("#select_all").click(select_all);
-  $("#deselect_all").click(deselect_all);
+  $("#select_all").click(selectAll);
+  $("#deselect_all").click(deselectAll);
   $("#invert").click(invert);
-  $("#add").click(add_to_collection);
-  $("#submit_bnw").click(submit_bnw);
-  $("#export_traits").click(export_traits);
+  $("#add").click(addToCollection);
+  $("#submit_bnw").click(submitBnw);
+  $("#export_traits").click(exportTraits);
 
   let naturalAsc = $.fn.dataTableExt.oSort["natural-ci-asc"]
   let naturalDesc = $.fn.dataTableExt.oSort["natural-ci-desc"]
 
   let na_equivalent_vals = ["N/A", "--", ""]; //ZS: Since there are multiple values that should be treated the same as N/A
 
-  function extract_inner_text(the_string){
+  function extractInnerText(the_string){
     var span = document.createElement('span');
     span.innerHTML = the_string;
     return span.textContent || span.innerText;
   }
 
-  function sort_NAs(a, b, sort_function){
+  function sortNAs(a, b, sort_function){
     if ( na_equivalent_vals.includes(a) && na_equivalent_vals.includes(b)) {
       return 0;
     }
@@ -305,10 +343,10 @@ $(function() {
 
   $.extend( $.fn.dataTableExt.oSort, {
     "natural-minus-na-asc": function (a, b) {
-      return sort_NAs(extract_inner_text(a), extract_inner_text(b), naturalAsc)
+      return sortNAs(extractInnerText(a), extractInnerText(b), naturalAsc)
     },
     "natural-minus-na-desc": function (a, b) {
-      return sort_NAs(extract_inner_text(a), extract_inner_text(b), naturalDesc)
+      return sortNAs(extractInnerText(a), extractInnerText(b), naturalDesc)
     }
   });
 
@@ -326,7 +364,7 @@ $(function() {
       } );
   }
 
-  apply_default = function() {
+  applyDefault = function() {
     let default_collection_id = $.cookie('default_collection');
     if (default_collection_id) {
       let the_option = $('[name=existing_collection] option').filter(function() {
@@ -335,6 +373,6 @@ $(function() {
       the_option.prop('selected', true);
     }
   }
-  apply_default();
+  applyDefault();
 
-});
\ No newline at end of file
+});
diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html
index 0ded66a6..6f1a9680 100644
--- a/wqflask/wqflask/templates/collections/view.html
+++ b/wqflask/wqflask/templates/collections/view.html
@@ -33,31 +33,45 @@
 
           </form>
         </div>
+        <div style="display: flex;">
+            <form id="heatmaps_form">
+                <button id="clustered-heatmap"
+                    class="btn btn-primary"
+                    data-url="{{heatmap_data_url}}"
+                    title="Generate heatmap from this collection" style="margin-top: 10px; margin-bottom: 10px;">
+                        Generate Heatmap
+                </button>
+                <br>
+                <div id="heatmap-options" style="display: none;">
+                    <div style="margin-bottom: 10px;">
+                        <b>Heatmap Orientation: </b>
+                        <br>
+                        Vertical
+                        <input id="heatmap-orient-vertical"
+                                type="radio"
+                                name="vertical"
+                                value="true" checked="checked"/>
+                        Horizontal
+                        <input id="heatmap-orient-horizontal"
+                                type="radio"
+                                name="vertical"
+                                value="false" />
+                    </div>
+                    <div style="margin-bottom: 10px;">
+                        <button id="clear-heatmap"
+                            class="btn btn-danger"
+                            title="Clear Heatmap">
+                            Clear Heatmap
+                        </button>
+                    </div>
+                </div>
+            </form>
+            &nbsp;
+        </div>
 
         <div>
+            <div id="clustered-heatmap-image-area"></div>
             <br />
-	    <form id="heatmaps_form">
-	      <fieldset>
-		<legend>Heatmap Orientation</legend>
-		<label for="heatmap-orient-vertical">Vertical</label>
-		<input id="heatmap-orient-vertical"
-		       type="radio"
-		       name="vertical"
-		       value="true" />
-		<label for="heatmap-orient-horizontal">Horizontal</label>
-		<input id="heatmap-orient-horizontal"
-		       type="radio"
-		       name="vertical"
-		       value="false" />
-	      </fieldset>
-	      <button id="clustered-heatmap"
-		      class="btn btn-primary"
-		      data-url="{{heatmap_data_url}}"
-		      title="Generate heatmap from this collection">
-		Generate Heatmap
-	      </button>
-	    </form>
-
             <div class="collection-table-options">
                 <form id="export_form" method="POST" action="/export_traits_csv">
                     <button class="btn btn-default" id="select_all" type="button"><span class="glyphicon glyphicon-ok"></span> Select All</button>
@@ -74,8 +88,6 @@
                     <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" type="button" title="Delete this collection" > Delete Collection</button>
                 </form>
             </div>
-	    <div id="clustered-heatmap-image-area">
-	    </div>
             <div style="margin-top: 10px; margin-bottom: 5px;">
                 <b>Show/Hide Columns:</b>
             </div>
@@ -100,7 +112,13 @@
                     <tbody>
                     {% for this_trait in trait_obs %}
                         <TR id="trait:{{ this_trait.name }}:{{ this_trait.dataset.name }}">
-                            <TD align="center" style="padding: 0px;"><INPUT TYPE="checkbox" NAME="searchResult" class="checkbox trait_checkbox" VALUE="{{ data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) }}"></TD>
+                          <TD align="center" style="padding: 0px;">
+			    <input type="checkbox"
+				   name="searchResult"
+				   class="checkbox trait_checkbox"
+				   value="{{data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name))}}"
+				   data-trait-info="{{trait_info_str(this_trait)}}">
+			  </TD>
                             <TD data-export="{{ loop.index }}" align="right">{{ loop.index }}</TD>
                             <TD title="{{ this_trait.dataset.fullname }}" data-export="{{ this_trait.dataset.fullname }}">{{ this_trait.dataset.fullname }}</TD>
                             <TD data-export="{{ this_trait.name }}">
@@ -165,6 +183,9 @@
 
     <script type="text/javascript" src="{{ url_for('js', filename='plotly/plotly.min.js') }}"></script>
 
+    <script type="text/javascript"
+	    src="/static/new/javascript/partial_correlations.js"></script>
+
 
     <script language="javascript" type="text/javascript">
         $(document).ready( function () {
@@ -322,6 +343,7 @@
 
 	    $("#clustered-heatmap").on("click", function() {
 		clear_heatmap_area();
+        $("#heatmap-options").show();
 		intv = window.setInterval(generate_progress_indicator(), 300);
 		vert_element = document.getElementById("heatmap-orient-vertical");
 		vert_true = vert_element == null ? false : vert_element.checked;
@@ -348,6 +370,12 @@
 		    }
 		});
 	    });
+
+        $("#clear-heatmap").on("click", function() {
+            clear_heatmap_area();
+            $("#heatmap-options").hide();
+        });
+
         });
     </script>
 
diff --git a/wqflask/wqflask/templates/correlation_matrix.html b/wqflask/wqflask/templates/correlation_matrix.html
index 3da6981c..e90accf2 100644
--- a/wqflask/wqflask/templates/correlation_matrix.html
+++ b/wqflask/wqflask/templates/correlation_matrix.html
@@ -71,6 +71,12 @@
   <button class="btn btn-default" id="export">Download <span class="glyphicon glyphicon-download"></span></button>
 </form>
 <br>
+
+<div style="margin:20x;">
+  {% include 'pca_scree_plot.html' %}
+  
+</div>
+
 {% if pca_works == "True" %}
 <h2>PCA Traits</h2>
 <div style="margin-bottom: 20px; overflow:hidden; width: 450px;">
diff --git a/wqflask/wqflask/templates/ctl_setup.html b/wqflask/wqflask/templates/ctl_setup.html
index a7ad5759..f5b0baf8 100644
--- a/wqflask/wqflask/templates/ctl_setup.html
+++ b/wqflask/wqflask/templates/ctl_setup.html
@@ -1,76 +1,70 @@
 {% extends "base.html" %}
 {% block title %}CTL analysis{% endblock %}
-
-{% block content %} <!-- Start of body -->
+{% block content %}
+<!-- Start of body -->
 <div class="container">
-  {% if request.form['trait_list'].split(",")|length < 2 %}
-  <div class="alert alert-danger" role="alert">
-    <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
-    <span class="sr-only">Error:</span>
-    <h2>Too few traits as input</h2>
-    Please make sure you select enough traits to perform CTL. Your collection needs to contain at least 2 different traits. You provided {{request.form['trait_list'].split(',')|length}} traits as input.
-  </div>
-  {% else %}
-  <h1>CTL analysis</h1>
-  CTL analysis is published as open source software, if you are using this method in your publications, please cite:<br><br>
-  Arends D, Li Y, Brockmann GA, Jansen RC, Williams RW, Prins P<br>
-  Correlation trait locus (CTL) mapping: Phenotype network inference subject to genotype.<br>
-  The Journal of Open Source Software (2016)<br>
-  Published in <a href="http://joss.theoj.org/papers/10.21105/joss.00087"><img src="http://joss.theoj.org/papers/10.21105/joss.00087/status.svg"></a>
-  <br><br>
-  {{(request.form['trait_list'].split(',')|length)}} traits as input
-
-  <form action="/ctl_results" method="post" class="form-horizontal">
-    <input type="hidden" name="trait_list" id="trait_list" value= "{{request.form['trait_list']}}">
-    <div class="dropdown">
-      <label for="Strategy">Strategy</label>
-      <div class="col-sm-10">
-        <select  name="strategy" id="strategy">
-          <option value="Exact">Exact</option>
-          <option value="Full">Full</option>
-          <option value="Pairwise">Pairwise</option>
-        </select>
-      </div>
+    {% if request.form['trait_list'].split(",")|length < 2 %} <div class="alert alert-danger" role="alert">
+        <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
+        <span class="sr-only">Error:</span>
+        <h2>Too few traits as input</h2>
+        Please make sure you select enough traits to perform CTL. Your collection needs to contain at least 2 different traits. You provided {{request.form['trait_list'].split(',')|length}} traits as input.
+</div>
+{% else %}
+<h1>CTL analysis</h1>
+CTL analysis is published as open source software, if you are using this method in your publications, please cite:<br><br>
+Arends D, Li Y, Brockmann GA, Jansen RC, Williams RW, Prins P<br>
+Correlation trait locus (CTL) mapping: Phenotype network inference subject to genotype.<br>
+The Journal of Open Source Software (2016)<br>
+Published in <a href="http://joss.theoj.org/papers/10.21105/joss.00087"><img src="http://joss.theoj.org/papers/10.21105/joss.00087/status.svg"></a>
+<br><br>
+<form class="col-md-8" action="/ctl_results" method="post" class="form-horizontal" id="ctl_form">
+    <input type="hidden" name="trait_list" id="trait_list" value="{{request.form['trait_list']}}">
+    <div class="form-group row">
+        <label for="Strategy" class="col-md-3 col-form-label col-form-label-sm">Strategy</label>
+        <div class="col-md-9">
+            <select class="form-control col-md-9" name="strategy" id="strategy">
+                <option value="Exact" selected="selected">Exact</option>
+                <option value="Full">Full</option>
+                <option value="Pairwise">Pairwise</option>
+            </select>
+        </div>
     </div>
-
-    <div class="dropdown">
-      <label for="Permutations">Number of permutation (Used when strategy is Full or Pairwise)</label>
-      <div class="col-sm-10">
-        <select  name="nperm" id="nperm">
-          <option value="100">100</option>
-          <option value="1000" selected="selected">1000</option>
-          <option value="10000">10000</option>
-        </select>
-      </div>
+    <div class="form-group row">
+        <label for="corType" class="col-md-3 col-form-label col-form-label-sm">Perform parametric analysis</label>
+        <div class="col-md-9">
+            <select class="form-control  col-md-9" name="parametric" id="parametric">
+                <option value="True" selected="selected">True</option>
+                <option value="False">False</option>
+            </select>
+        </div>
     </div>
-
-    <div class="dropdown">
-      <label for="Coefficient">Type of correlation coefficient</label>
-      <div class="col-sm-10">
-        <select  name="parametric" id="parametric">
-          <option value="False">Spearman</option>
-          <option value="True">Pearson</option>
-        </select>
-      </div>
+    <div class="form-group row">
+        <label for="Permutations" class="col-md-3 col-form-label col-form-label-sm">Number of permutation <span style="color:red;">(Used when strategy is Full or Pairwise)</span></label>
+        <div class="col-md-9">
+            <select class="form-control" name="nperm" id="nperm">
+                <option value="100">100</option>
+                <option value="1000" selected="selected">1000</option>
+                <option value="10000">10000</option>
+            </select>
+        </div>
     </div>
-
-    <div class="dropdown">
-      <label for="Significance">Significance level</label>
-      <div class="col-sm-10">
-        <select  name="significance" id="significance">
-          <option value="0.1">0.1</option>
-          <option value="0.05" selected="selected">0.05</option>
-          <option value="0.001">0.001</option>
-        </select>
-      </div>
+    <div class="form-group row">
+        <label for="Significance" class="col-md-3 col-form-label col-form-label-sm"> Significance level</label>
+        <div class="col-md-9">
+            <select class="form-control" name="significance" id="significance">
+                <option value="0.1">0.1</option>
+                <option value="0.05" selected="selected">0.05</option>
+                <option value="0.001">0.001</option>
+            </select>
+        </div>
     </div>
-    <br>
     <div class="form-group">
-      <div class="col-sm-10">
-        <input type="submit" class="btn btn-primary" value="Run CTL using these settings" />
-      </div>
+        <div class="text-center">
+            <input type="submit" class="btn btn-primary" value="Run CTL using these settings" />
+        </div>
     </div>
 </form>
 {% endif %}
 </div>
 {% endblock %}
+
diff --git a/wqflask/wqflask/templates/display_diffs.html b/wqflask/wqflask/templates/display_diffs.html
index e787e468..ce50c1b4 100644
--- a/wqflask/wqflask/templates/display_diffs.html
+++ b/wqflask/wqflask/templates/display_diffs.html
@@ -11,14 +11,14 @@
     {% set additions = diff.get("Additions") %}
     {% set modifications = diff.get("Modifications") %}
     {% set deletions = diff.get("Deletions") %}
-
+    {% set header = diff.get("Columns", "Strain Name,Value,SE,Count") %}
     {% if additions %}
     <h2>Additions Data:</h2>
     <div class="row">
         <div class="col-md-8">
             <table class="table-responsive table-hover table-striped cell-border" id="table-additions">
                 <thead>
-                    <th scope="col">Added Data</</th>
+                    <th scope="col">Added Data ({{ header }})</th>
                 </thead>
                 <tbody>
                     {% for data in additions %}
@@ -39,9 +39,9 @@
         <div class="col-md-8">
             <table class="table-responsive table-hover table-striped cell-border" id="table-modifications">
                 <thead>
-                    <th scope="col">Original</</th>
-                    <th scope="col">Current</</th>
-                    <th scope="col">Diff</</th>
+                    <th scope="col">Original</th>
+                    <th scope="col">Current</th>
+                    <th scope="col">Diff ({{ header }})</th>
                 </thead>
                 <tbody>
                     {% for data in modifications %}
diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html
index 0daea51d..5458247e 100644
--- a/wqflask/wqflask/templates/edit_phenotype.html
+++ b/wqflask/wqflask/templates/edit_phenotype.html
@@ -218,12 +218,17 @@
     </div>
     <div style="margin-left: 13%;">
         <a href="/datasets/{{ publish_xref.id_ }}/traits/{{ publish_xref.phenotype_id }}/csv?resource-id={{ resource_id }}" class="btn btn-link btn-sm">
-            Sample Data(CSV Download)
+            Click to Download CSV Sample Data
         </a>
     </div>
     <div class="form-group">
         <input type = "file" class="col-sm-4 control-label" name = "file" />
     </div>
+    <div class="col-xs-6">
+        <p>
+            Note: Current allowable sample headers are: {{ ', '.join(headers) }}
+        </p>
+    </div>
     <div class="controls center-block" style="width: max-content;">
         <input name="inbred-set-id" class="changed" type="hidden" value="{{ publish_xref.inbred_set_id }}"/>
         <input name="phenotype-id" class="changed" type="hidden" value="{{ publish_xref.phenotype_id }}"/>
diff --git a/wqflask/wqflask/templates/gn3_ctl_results.html b/wqflask/wqflask/templates/gn3_ctl_results.html
new file mode 100644
index 00000000..c42707f6
--- /dev/null
+++ b/wqflask/wqflask/templates/gn3_ctl_results.html
@@ -0,0 +1,101 @@
+{% extends "base.html" %}
+{% block title %}Ctl results{% endblock %}
+{% block content %}
+<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
+<link REL="stylesheet" TYPE="text/css" href="{{ url_for('css', filename='bootstrap/css/bootstrap.css') }}" />
+<link rel="stylesheet" href="https://cdn.datatables.net/1.11.3/css/jquery.dataTables.min.css">
+<style type="text/css">
+.carousel-control-next,
+.carousel-control-prev
+
+/*, .carousel-indicators */
+    {
+    filter: invert(100%);
+}
+</style>
+<div style="margin-top:10px">
+
+{% if error %}
+    <div>
+        <h4 style="text-align:center;color:red">{{error}}</h4>
+    </div> 
+
+{% else %}
+    <div>
+    <div>
+        <div style="text-align: center;">
+            <h3>CTL/QTL Plots for Individual Traits</h3>
+            <h4> {{ctl_plots|length}} phenotypes as input</h4>
+        </div>
+        <div id="carouselExampleControls" class="carousel slide" data-interval="false">
+            <div class="carousel-inner">
+                {% for ctl_plot in ctl_plots %}
+                <div class="item{% if loop.index == 1 %} active{% endif %}">
+                    <img style="width:1000px;height: 600px;" class="center-block" src="data:image/jpeg;base64,{{ ctl_plot | safe }}" alt="First slide">
+                </div>
+                {% endfor %}
+                <a class="carousel-control-prev" href="#carouselExampleControls" role="button" data-slide="prev">
+                    <span class="carousel-control-prev-icon" aria-hidden="true"></span>
+                    <span class="sr-only">Previous</span>
+                </a>
+                <a class="carousel-control-next" href="#carouselExampleControls" role="button" data-slide="next">
+                    <span class="carousel-control-next-icon" aria-hidden="true"></span>
+                    <span class="sr-only">Next</span>
+                </a>
+            </div>
+        </div>
+    </div>
+    <div>
+        <div style="text-align:center;">
+            <h2>Ctl line plot</h2>
+            <h4>Plot the CTL for genome-wide CTL on all traits (the output of CTLscan).</h4>
+        </div>
+        <div class="row">
+            <div class="col-8">
+                <img style="width:100%;height: 600px;" class="center-block" src="data:image/jpeg;base64,{{ image_data | safe }}">
+            </div>
+            <div class="col-4">
+                <ol style="height: 100%;display:flex;flex-direction: column;align-items: center;justify-content: center;">
+                    {% for trait in phenotypes %}
+                    {% set trait_data = trait.split(':') %}
+                    <li><a href="/show_trait?trait_id={{trait_data[0]}}&dataset={{trait_data[1]}}">{{trait_data[0]}}</a></li>
+                    {% endfor %}
+                </ol>
+            </div>
+        </div>
+    </div>
+    <h2 style="text-align:center">Significant CTL </h2>
+    <table id="significance" width="80vw"></table>
+    <div style="text-align: center;margin-top: 20px;">
+        <h3 style="text-align:center;">Network figure</h3>
+        <div style="margin-top: 20px;">
+            <p>Use tools like cytoscape to visualize the network</p>
+            <a href="/ctl_network_files/{{network_file_name}}/sif" class="btn btn-secondary btn-lg mx-2">Download Sif file</a>
+            <a href="/ctl_network_files/{{network_file_name}}/nodes" class="btn btn-secondary btn-lg mx-2">Download Node file</a>
+        </div>
+    </div>
+</div>
+
+{% endif %}
+</div>
+
+<!-- <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script> -->
+<script src="{{ url_for('js', filename='bootstrap/js/bootstrap.min.js') }}" type="text/javascript"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script>
+<script type="text/javascript">
+let { data_set_rows, col_names } = {{ significance_data | safe }}
+
+
+$('#significance').DataTable({
+    data: data_set_rows,
+    columns: col_names.map((name) => {
+        return {
+            title: name
+        }
+    })
+});
+</script>
+{% endblock %}
+
diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_error.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_error.html
new file mode 100644
index 00000000..36847f57
--- /dev/null
+++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_error.html
@@ -0,0 +1,54 @@
+{% extends "base.html" %}
+{% block title %}Error: {{message}}{% endblock %}
+{% block content %}
+<!-- Start of body -->
+
+<div class="col-md-8">
+<div class="form-group has-error">
+  <div class="control-label" for="inputError1">
+
+    <img src="/static/gif/error/{{ error_image }}">
+
+    <h1>ERROR</h1>
+
+    <p>
+      This error is not what we wanted to see. Unfortunately errors
+      are part of all software systems and we need to resolve this
+      together.
+    </p>
+    <p>
+      <b>It is important to report this ERROR so we can fix it for everyone</b>.
+    </p>
+
+    <p>
+      Report to the GeneNetwork team by recording the steps you take
+      to reproduce this ERROR. Next to those steps, copy-paste below
+      stack trace, either as
+      a <a href="https://github.com/genenetwork/genenetwork2/issues/new">new
+      issue</a> or E-mail this full page to one of the developers
+      directly.
+    </p>
+  </div>
+
+  <p>
+  (GeneNetwork error: {{message[:128]}})
+  </p>
+
+  <p>
+    To check if this already a known issue, search the
+    <a href="https://github.com/genenetwork/genenetwork2/issues">issue
+      tracker</a>.
+  </p>
+
+  <a href="#Stack" class="btn btn-default" data-toggle="collapse">Toggle full stack trace</a>
+  <div id="Stack" class="collapse">
+    <pre>
+      GeneNetwork {{ version }} {% for line in stack %} {{ line }}
+      {% endfor %}
+    </pre>
+  </div>
+</div>
+</div>
+
+
+{% endblock %}
diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_poll_results.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_poll_results.html
new file mode 100644
index 00000000..47acc294
--- /dev/null
+++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_poll_results.html
@@ -0,0 +1,15 @@
+{%extends "base.html"%}
+
+{%block title%}Partial Correlations:{%endblock%}
+
+{%block css%}
+<meta http-equiv="refresh"
+      content="10;URL=/partial_correlations/{{command_id}}">
+{%endblock%}
+
+{%block content%}
+
+<p>Computing partial correlations...</p>
+<img src="/static/gif/waitAnima2.gif"
+     alt="Image indicating computation of partial correlations is ongoing" />
+{%endblock%}
diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_results_presentation.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_results_presentation.html
new file mode 100644
index 00000000..c109f24b
--- /dev/null
+++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_results_presentation.html
@@ -0,0 +1,249 @@
+{%extends "base.html"%}
+
+{%block title%}Partial Correlations:{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" type="text/css" href="/static/new/css/partial_correlations.css" />
+<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" />
+<link rel="stylesheet" type="text/css" href="/static/new/css/trait_list.css" />
+<link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" />
+{%endblock%}
+
+{%block content%}
+<p>
+  <strong>Primary Trait</strong><br /><br />
+  <a href="{{url_for(
+	   'show_trait_page',
+	   trait_id=primary['trait_name'],
+	   dataset=primary['dataset_name'])}}"
+     title="Link to trait data for trait {{primary['trait_name']}}">
+    {{primary["dataset_type"]}}/{{primary["trait_name"]}}
+    [{{primary["symbol"] }} on Chr {{primary["chr"]}} @ {{primary["mb"]}}]:
+    {{primary["description"]}}
+  </a> --- FROM: {{primary["dataset_name"]}}
+</p>
+<p><strong>Control Traits</strong><br /><br />
+  {%for trait in controls:%}
+  <a href="{{url_for(
+	   'show_trait_page',
+	   trait_id=trait['trait_name'],
+	   dataset=trait['dataset_name'])}}"
+     title="Link to trait data for trait {{trait['trait_name']}}">
+    {{trait["dataset_type"]}}/{{trait["trait_name"]}}
+    [{{trait["symbol"] }} on Chr {{trait["chr"]}} @ {{trait["mb"]}}]:
+    {{trait["description"]}}
+  </a> --- FROM: {{trait["dataset_name"]}}<br />
+  {%endfor%}
+</p>
+
+<div id="partial-correlation-results">
+  {%if dataset_type == "Publish":%}
+  <table id="part-corr-results-publish"
+	 class="table-hover table-striped cell-border"
+	 style="float: left;">
+    <thead>
+      <tr>
+	<th></th>
+	<th>Index</th>
+	<th>Record</th>
+	<th>Phenotype</th>
+	<th>Authors</th>
+	<th>Year</th>
+	<th>N</th>
+	<th>Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+	<th>p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th>
+	<th>{%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+	<th>p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th>
+	<th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      {%for trait in correlations:%}
+      <tr class="results-row">
+	<td></td>
+	<td data-column-heading="Index"></td>
+	<td data-column-heading="Record">
+	  <a href="{{url_for(
+		   'show_trait_page',
+		   trait_id=trait['trait_name'],
+		   dataset=trait['dataset_name'])}}"
+	     title="Link to trait data for trait {{trait['trait_name']}}">
+	    {{trait["trait_name"]}}
+	  </a>
+	</td>
+	<td data-column-heading="Phenotype">
+	  {{trait["post_publication_description"]}}</td>
+	<td data-column-heading="Authors">{{trait["authors"]}}</td>
+	<td data-column-heading="Year">{{trait["year"]}}</td>
+	<td data-column-heading="N">{{trait["noverlap"]}}</td>
+	<td data-column-heading="Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("partial_corr"))}}
+	</td>
+	<td data-column-heading="p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})">
+	  {{format_number(trait.get("partial_corr_p_value"))}}
+	</td>
+	<td data-column-heading="{%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("corr"))}}
+	</td>
+	<td data-column-heading="p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})">
+	  {{format_number(trait.get("corr_p_value"))}}
+	</td>
+	<td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("delta"))}}
+	</td>
+      </tr>
+      {%endfor%}
+    </tbody>
+  </table>
+  {%endif%}
+
+  {%if dataset_type == "Geno":%}
+  <table id="part-corr-results-geno"
+	 class="table-hover table-striped cell-border"
+	 style="float: left;">
+    <thead>
+      <tr>
+	<th></th>
+	<th>Index</th>
+	<th>Locus</th>
+	<th>Chr</th>
+	<th>Megabase</th>
+	<th>N</th>
+	<th>Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+	<th>p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th>
+	<th>{%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+	<th>p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th>
+	<th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      {%for trait in correlations:%}
+      <tr class="results-row">
+	<td></td>
+	<td data-column-heading="Index"></td>
+	<td data-column-heading="Locus">
+	  <a href="{{url_for(
+		   'show_trait_page',
+		   trait_id=trait['trait_name'],
+		   dataset=trait['dataset_name'])}}"
+	     title="Link to trait data for trait {{trait['trait_name']}}">
+	    {{trait["trait_name"]}}
+	  </a>
+	</td>
+	<td data-column-heading="Chr">{{trait["chr"]}}</td>
+	<td data-column-heading="Megabase">{{trait["mb"]}}</td>
+	<td data-column-heading="N">{{trait["noverlap"]}}</td>
+	<td data-column-heading="Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("partial_corr"))}}
+	</td>
+	<td data-column-heading="p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})">
+	  {{format_number(trait.get("partial_corr_p_value"))}}
+	</td>
+	<td data-column-heading="{%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("corr"))}}
+	</td>
+	<td data-column-heading="p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})">
+	  {{format_number(trait.get("corr_p_value"))}}
+	</td>
+	<td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("delta"))}}
+	</td>
+      </tr>
+      {%endfor%}
+    </tbody>
+  </table>
+  {%endif%}
+
+  {%if dataset_type == "ProbeSet":%}
+  <table id="part-corr-results-probeset"
+	 class="table-hover table-striped cell-border"
+	 style="float: left;">
+    <thead>
+      <tr>
+	<th></th>
+	<th>Index</th>
+	<th>Record</th>
+	<th>Gene ID</th>
+	<th>Homologene ID</th>
+	<th>Symbol</th>
+	<th>Description</th>
+	<th>Chr</th>
+	<th>Megabase</th>
+	<th>Mean Expr</th>
+	<th>N</th>
+	<th>Sample Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+	<th>Sample p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th>
+	<th>Sample {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+	<th>Sample p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th>
+	<th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+	<th>Lit Corr</th>
+	<th>Tissue {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th>
+	<th>Tissue p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      {%for trait in correlations:%}
+      <tr class="results-row">
+	<td></td>
+	<td data-column-heading="Index"></td>
+	<td data-column-heading="Record">
+	  <a href="{{url_for(
+		   'show_trait_page',
+		   trait_id=trait['trait_name'],
+		   dataset=trait['dataset_name'])}}"
+	     title="Link to trait data for trait {{trait['trait_name']}}">
+	    {{trait["trait_name"]}}
+	  </a>
+	</td>
+	<td data-column-heading="Gene ID">{{trait["geneid"]}}</td>
+	<td data-column-heading="Homologene ID">{{trait["homologeneid"]}}</td>
+	<td data-column-heading="Symbol">{{trait["symbol"]}}</td>
+	<td data-column-heading="Description">{{trait["description"]}}</td>
+	<td data-column-heading="Chr">{{trait["chr"]}}</td>
+	<td data-column-heading="Megabase">{{trait["mb"]}}</td>
+	<td data-column-heading="Mean Expr">{{trait["mean_expr"]}}</td>
+	<td data-column-heading="N">{{trait["noverlap"]}}</td>
+	<td data-column-heading="Sample Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("partial_corr"))}}
+	</td>
+	<td data-column-heading="Sample p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})">
+	  {{format_number(trait.get("partial_corr_p_value"))}}
+	</td>
+	<td data-column-heading="Sample {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("corr"))}}
+	</td>
+	<td data-column-heading="Sample p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})">
+	  {{format_number(trait.get("corr_p_value"))}}
+	</td>
+	<td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("delta"))}}
+	</td>
+	<td data-column-heading="Lit Corr">
+	  {{format_number(trait.get("l_corr"))}}
+	</td>
+	<td data-column-heading="Tissue {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}">
+	  {{format_number(trait.get("tissue_corr"))}}
+	</td>
+	<td data-column-heading="Tissue p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})">
+	  {{format_number(trait.get("tissue_p_value"))}}
+	</td>
+      </tr>
+      {%endfor%}
+    </tbody>
+  </table>
+  {%endif%}
+
+</div>
+{%endblock%}
+
+{%block js%}
+{%if step == "select-corr-method":%}
+<script type="text/javascript"
+	src="/static/new/javascript/partial_correlations.js"></script>
+<script language="javascript" type="text/javascript"
+	src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script>
+{%endif%}
+{%endblock%}
diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html
new file mode 100644
index 00000000..e541f31b
--- /dev/null
+++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html
@@ -0,0 +1,146 @@
+{%extends "base.html"%}
+
+{%block title%}Partial Correlations:{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" type="text/css"
+      href="{{url_for('css', filename='DataTables/css/jquery.dataTables.css')}}" />
+<link rel="stylesheet" type="text/css"
+      href="{{url_for('js', filename='DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css')}}">
+<link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" />
+<link rel="stylesheet" type="text/css" href="/static/new/css/trait_list.css" />
+<link rel="stylesheet" type="text/css"
+      href="/static/new/css/partial_correlations.css" />
+{%endblock%}
+
+{%block content%}
+<form id="pcorrs-form"
+      method="POST"
+      action="{{url_for('partial_correlations')}}">
+  {%with messages = get_flashed_messages(with_categories=true)%}
+  {%if messages:%}
+  <ul class=flashes>
+    {%for category, message in messages:%}
+    <li class="{{category}}">{{message}}</li>
+    {%endfor%}
+  </ul>
+  {%endif%}
+  {%endwith%}
+
+  <input type="hidden" value="{{trait_list_str}}" name="trait_list">
+  <table id="pcorrs_traits_table" class="table-hover table-striped cell-border">
+    <thead>
+      <tr>
+	<th>Primary (X)</th>
+	<th>Controls (Z)</th>
+	<th>Ignored</th>
+	<th>Dataset</th>
+	<th>Trait ID</th>
+	<th>Symbol</th>
+	<th>Description</th>
+	<th>Location</th>
+	<th>Mean</th>
+	<th>Max LRS</th>
+	<th>Max LRS Location Chr and Mb</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      {%for trait in traits:%}
+      <tr>
+	<td>
+	  <input type="radio" name="trait_{{trait['trait_name']}}"
+		 id="trait_{{trait['trait_name']}}"
+		 value="primary_{{trait['trait_name']}}">
+	</td>
+	<td>
+	  <input type="radio" name="trait_{{trait['trait_name']}}"
+		 id="trait_{{trait['trait_name']}}"
+		 value="controls_{{trait['trait_name']}}">
+	</td>
+	<td>
+	  <input type="radio" name="trait_{{trait['trait_name']}}"
+		 id="trait_{{trait['trait_name']}}"
+		 value="ignored_{{trait['trait_name']}}" checked="checked">
+	</td>
+	<td>{{trait.get("dataset", "_")}}
+	<td>{{trait.get("trait_name", "_")}}</td>
+	<td>{{trait.get("symbol", "_")}}</td>
+	<td>{{trait.get("description", "_")}}</td>
+	<td>{{trait.get("location", "_")}}</td>
+	<td>{{trait.get("mean", "_")}}</td>
+	<td>{{trait.get("lrs", "_")}}</td>
+	<td>{{trait.get("lrs_location", "_")}}</td>
+      </tr>
+      {%endfor%}
+    </tbody>
+  </table>
+
+  <div class="form-group">
+    <label for="corr-method-input" class="form-label">Compute</label>
+    <select id="corr-method-input" required="required" name="method"
+	    class="form-control">
+      <option value="Genetic Correlation, Pearson's r">
+	Genetic Correlation, Pearson's r</option>
+      <option value="Genetic Correlation, Spearman's rho">
+	Genetic Correlation, Spearman's rho</option>
+      <option value="SGO Literature Correlation">
+	SGO Literature Correlation</option>
+      <option value="Tissue Correlation, Pearson's r">
+	Tissue Correlation, Pearson's r</option>
+      <option value="Tissue Correlation, Spearman's rho">
+	Tissue Correlation, Spearman's rho</option>
+    </select>
+  </div>
+
+  <div class="form-group">
+    <label for="target-db-input" class="form-label">Choose Database</label>
+    <select id="target-db-input" required="required" name="target_db"
+	    class="form-control">
+      {%if target_dbs:%}
+      {%for item in target_dbs:%}
+      {%if "description" in item.keys():%}
+      <option value="{{item['value']}}">{{item['description']}}</option>
+      {%else:%}
+      {%for group, opts in item.items()%}
+      {%if opts | length > 0:%}
+      <optgroup label="{{group}} ------">
+	{%for item2 in opts:%}
+	<option value="{{item2['value']}}">{{item2['description']}}</option>
+	{%endfor%}
+      </optgroup>
+      {%endif%}
+      {%endfor%}
+      {%endif%}
+      {%endfor%}
+      {%endif%}
+    </select>
+  </div>
+
+  <div class="form-group">
+      <label for="criteria-input" class="form-label">Return</label>
+      <select id="criteria-input" required="required" name="criteria" size="1"
+	      class="form-control">
+	<option value="100">top 100</option>
+	<option value="200">top 200</option>
+	<option value="500" selected="selected">top 500</option>
+	<option value="1000">top 1000</option>
+	<option value="2000">top 2000</option>
+	<option value="5000">top 5000</option>
+	<option value="10000">top 10000</option>
+	<option value="15000">top 15000</option>
+	<option value="20000">top 20000</option>
+      </select>
+    </div>
+
+  <button type="submit" class="btn btn-primary" name="submit"
+	  value="Run Partial Correlations">
+    Run Partial Correlations
+  </button>
+</form>
+{%endblock%}
+
+{%block js%}
+<script type="text/javascript"
+	src="/static/new/javascript/partial_correlations.js"></script>
+{%endblock%}
diff --git a/wqflask/wqflask/templates/pca_scree_plot.html b/wqflask/wqflask/templates/pca_scree_plot.html
new file mode 100644
index 00000000..11bfebff
--- /dev/null
+++ b/wqflask/wqflask/templates/pca_scree_plot.html
@@ -0,0 +1,94 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title></title>
+</head>
+
+<body>
+    <div>
+        
+        <div id="scree_plot" style="width:700px;height:600px;"></div>
+    </div>
+</body>
+<script type="text/javascript" src="{{ url_for('js', filename='plotly/plotly.min.js') }}"></script>
+<script type="text/javascript">
+js_data = {{ js_data | safe }}
+
+let { x_coord, y_coord } = js_data["scree_data"]
+
+
+const layout = {
+
+    title: {
+        text: "<b>Pca Scree Plot</b>",
+        font: {
+            "size": 24,
+            "family": "Arial",
+            "color": "#000000"
+        }
+    },
+
+    yaxis: {
+        title: {
+            text: "Percent of total variance %",
+            font: {
+                "size": 18,
+                "color": ""
+
+            }
+        }
+    },
+
+    xaxis: {
+        title: {
+            text: "PCA components",
+            font: {
+                "size": 18,
+                "color": ""
+
+            }
+        }
+    },
+
+}
+
+const data = [{
+    x: x_coord,
+    y: y_coord,
+    marker: {
+
+        color: 'rgb(17, 157, 255)',
+        size: 5,
+        line: {
+            color: 'rgb(255, 0, 0)',
+            width: 3
+        }
+
+    }
+}]
+
+
+let custom_configs = (filename, download_format, modebar = true) => {
+
+    return {
+        displayModeBar: modebar,
+        scrollZoom: false,
+        toImageButtonOptions: {
+            filename,
+            format:download_format,
+            height: 600,
+            width: 700,
+            scale: 1
+        }
+    }
+
+}
+
+Plotly.newPlot(document.getElementById("scree_plot"), data, layout,
+	custom_configs(file_name = "scree_plot", download_format = "svg"));
+</script>
+
+</html>
diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html
index f73cba17..b9859229 100644
--- a/wqflask/wqflask/templates/search_result_page.html
+++ b/wqflask/wqflask/templates/search_result_page.html
@@ -40,6 +40,8 @@
                     with <u>{{ word.key|upper }}</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong>{% if loop.last %}.{% else %} and {% endif %}
                     {% elif word.search_term|length == 3 %}
                     with <u>{{ word.key|upper }}</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[2] }}</strong>{% if loop.last %}.{% else %} and {% endif %}
+                    {% elif word.search_term|length == 4 %}
+                    with <u>{{ word.key|upper }}</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[3] }}</strong> with an exclusion zone of <strong>{{ word.search_term[2] }}</strong> Mb
                     {% elif word.search_term|length == 5 %}
                     with <u>{{ word.key|upper }}</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[2] }}</strong> between <strong>{{ word.search_term[3] }}</strong> and <strong>{{ word.search_term[4] }}</strong> Mb{% if loop.last %}.{% else %} and {% endif %}
                     {% endif %}
@@ -94,19 +96,22 @@
               <input type="hidden" name="accession_id" id="accession_id" value="{{ dataset.accession_id }}">
               {% endif %}
               <input type="hidden" name="export_data" id="export_data" value="">
-              <button class="btn btn-default" id="select_all" type="button"><span class="glyphicon glyphicon-ok"></span> Select</button>
-              <button class="btn btn-default" id="invert" type="button"><span class="glyphicon glyphicon-adjust"></span> Invert</button>
+              <input type="text" id="searchbox" class="form-control" style="width: 200px; display: inline;" placeholder="Search For...">
               <button class="btn btn-success" id="add" type="button" disabled><span class="glyphicon glyphicon-plus-sign"></span> Add</button>
-              <button class="btn btn-default" id="export_traits">Download <span class="glyphicon glyphicon-download"></span></button>
-              <input type="text" id="searchbox" class="form-control" style="width: 200px; display: inline;" placeholder="Search This Table For ...">
-              <input type="text" id="select_top" class="form-control" style="width: 200px; display: inline;" placeholder="Select Top ...">
+              <button class="btn btn-default" id="select_all" type="button"><span class="glyphicon glyphicon-ok"></span> Select All</button>
+              <input type="text" id="select_top" class="form-control" style="width: 200px; display: inline;" placeholder="Select Rows (1-5, 11)">
+              <button class="btn btn-default" id="export_traits"><span class="glyphicon glyphicon-download"></span> Download</button>
+              <button class="btn btn-default" id="invert" type="button"><span class="glyphicon glyphicon-adjust"></span> Invert</button>
               <button class="btn btn-default" id="deselect_all" type="button"><span class="glyphicon glyphicon-remove"></span> Deselect</button>
             </div>
+            <div id="select_samples_invalid" class="alert alert-error" style="display:none;">
+              Please check that your syntax includes only a combination of integers, dashes, and commas of a format 
+              similar to <strong>1,5,10</strong> or <strong>2, 5-10, 15</strong>, etc.
+            </div>
           </form>
           {% if dataset.type != 'Geno' %}
           <div class="show-hide-container">
-            <b>Show/Hide Columns:</b>
-            <br>
+            <b>Show/Hide Columns:&nbsp;&nbsp;</b>
             {% if dataset.type == 'ProbeSet' %}
             <button class="toggle-vis" data-column="3">Symbol</button>
             <button class="toggle-vis" data-column="4">Description</button>
@@ -126,7 +131,7 @@
             {% endif %}
           </div>
           {% endif %}
-          <div id="table_container" style="width: {% if dataset.type == 'Geno' %}270{% else %}100%{% endif %}px;">
+          <div id="table_container" style="width: {% if dataset.type == 'Geno' %}375px;{% else %}100%; min-width: 1400px;{% endif %}">
             <table class="table-hover table-striped cell-border" id='trait_table' style="float: left;">
                 <tbody>
                  <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td>
@@ -195,6 +200,8 @@
               'title': "Index",
               'type': "natural",
               'width': "35px",
+              "searchable": false,
+              "orderable": false,
               'targets': 1,
               'data': "index"
             }
@@ -232,7 +239,7 @@
             {
               'title': "<div style='text-align: right;'>Location</div>",
               'type': "natural-minus-na",
-              'width': "125px",
+              'width': "130px",
               'targets': 5,
               'data': "location"
             },
@@ -245,7 +252,7 @@
               'orderSequence': [ "desc", "asc"]
             },
             {
-              'title': "<div style='text-align: right;'>Peak <a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\">&nbsp;<i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>LOD&ensp;&emsp;</div>",
+              'title': "<div style='text-align: right; padding-right: 10px;'>Peak</div> <div style='text-align: right;'>LOD <a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"><sup style='color: #FF0000;'><i>?</i></sup></a></div>",
               'type': "natural-minus-na",
               'data': "lod_score",
               'width': "60px",
@@ -255,12 +262,12 @@
             {
               'title': "<div style='text-align: right;'>Peak Location</div>",
               'type': "natural-minus-na",
-              'width': "125px",
+              'width': "130px",
               'targets': 8,
               'data': "lrs_location"
             },
             {
-              'title': "<div style='text-align: right;'>Effect <a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\">&nbsp;<i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>Size&ensp;&emsp;</div>",
+              'title': "<div style='text-align: right; padding-right: 10px;'>Effect</div> <div style='text-align: right;'>Size <a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"><sup style='color: #FF0000;'><i>?</i></sup></a></div>",
               'type': "natural-minus-na",
               'data': "additive",
               'width': "65px",
@@ -306,8 +313,8 @@
             {
               'title': "Authors",
               'type': "natural",
-              {% if (max_widths.authors * 7) < 500 %}
-              'width': "{{ max_widths.authors * 7 }}px",
+              {% if (max_widths.authors * 5) < 500 %}
+              'width': "{{ max_widths.authors * 5 }}px",
               {% else %}
               'width': "500px",
               {% endif %}
@@ -315,8 +322,8 @@
               'targets': 5,
               'render': function(data, type, row, meta) {
                 author_list = data.authors.split(",")
-                if (author_list.length >= 6) {
-                  author_string = author_list.slice(0, 6).join(",") + ", et al."
+                if (author_list.length >= 2) {
+                  author_string = author_list.slice(0, 2).join(",") + ", et al."
                 } else{
                   author_string = data.authors
                 }
@@ -339,7 +346,7 @@
               'orderSequence': [ "desc", "asc"]
             },
             {
-              'title': "<div style='text-align: right;'>Peak <a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\">&nbsp;<i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>LOD&ensp;&emsp;</div>",
+              'title': "<div style='text-align: right; padding-right: 10px;'>Peak</div> <div style='text-align: right;'>LOD <a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"><sup style='color: #FF0000;'><i>?</i></sup></a></div>",
               'type': "natural-minus-na",
               'data': "lod_score",
               'targets': 7,
@@ -349,12 +356,12 @@
             {
               'title': "<div style='text-align: right;'>Peak Location</div>",
               'type': "natural-minus-na",
-              'width': "120px",
+              'width': "125px",
               'targets': 8,
               'data': "lrs_location"
             },
             {
-              'title': "<div style='text-align: right;'>Effect <a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\">&nbsp;<i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>Size&ensp;&emsp;</div>",
+              'title': "<div style='text-align: right; padding-right: 10px;'>Effect</div> <div style='text-align: right;'>Size <a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"><sup style='color: #FF0000;'><i>?</i></sup></a></div>",
               'type': "natural-minus-na",
               'width': "60px",
               'data': "additive",
@@ -374,7 +381,7 @@
             {
               'title': "<div style='text-align: right;'>Location</div>",
               'type': "natural-minus-na",
-              'width': "120px",
+              'width': "125px",
               'targets': 2,
               'data': "location"
             }{% endif %}
@@ -447,7 +454,7 @@
                 "destroy": true,
                 "autoWidth": false,
                 "bSortClasses": false,
-                "scrollY": "500px",
+                "scrollY": "1000px",
                 "scrollCollapse": true,
                 {% if trait_list|length > 5 %}
                 "scroller":  true,
@@ -491,6 +498,12 @@
               }
             }
 
+            trait_table.on( 'order.dt search.dt', function () {
+              trait_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) {
+                cell.innerHTML = i+1;
+              } );
+            } ).draw();
+
             window.addEventListener('resize', function(){
               trait_table.columns.adjust();
             });
diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html
index 16a819fa..2f98f14e 100644
--- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html
+++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html
@@ -119,9 +119,6 @@
               <div class="col-xs-3 controls">
                   <input type="button" class="btn corr_compute submit_special btn-success" data-url="/corr_compute" title="Compute Correlation" value="Compute">
               </div>
-                  <div class="col-xs-3 controls">
-                  <input type="button" class="btn test_corr_compute submit_special btn-success" data-url="/test_corr_compute" title="Compute Correlation" value="Test Compute">
-              </div>
           </div>
       </div>
     </div>
diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html
index 4e9ea0fb..1250d728 100644
--- a/wqflask/wqflask/templates/show_trait_details.html
+++ b/wqflask/wqflask/templates/show_trait_details.html
@@ -242,9 +242,6 @@
         {% if this_trait.dataset.type == 'ProbeSet' %}
         <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/datasets/traits/{{ this_trait.name }}?resource-id={{ resource_id }}', '_blank')">Edit</button>
         {% endif %}
-        {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %}
-        <button type="button" id="edit_resource" class="btn btn-success" title="Edit Privileges" onclick="window.open('/resource-management/resources/{{ resource_id }}', '_blank')">Edit Privileges</button>
-        {% endif %}
         {% endif %}
     </div>
 </div>
diff --git a/wqflask/wqflask/templates/tool_buttons.html b/wqflask/wqflask/templates/tool_buttons.html
index 3ee5be19..c3927495 100644
--- a/wqflask/wqflask/templates/tool_buttons.html
+++ b/wqflask/wqflask/templates/tool_buttons.html
@@ -33,3 +33,10 @@
 <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" >
     Comparison Bar Chart
 </button>
+
+<button id="partial-correlations"
+	class="btn btn-primary submit_special"
+	data-url="{{url_for('partial_correlations')}}"
+	title="Run partial correlations with the selected traits">
+    Partial Correlations
+</button>
diff --git a/wqflask/wqflask/templates/with-trait-items.html b/wqflask/wqflask/templates/with-trait-items.html
new file mode 100644
index 00000000..66d6fd22
--- /dev/null
+++ b/wqflask/wqflask/templates/with-trait-items.html
@@ -0,0 +1,18 @@
+{%for trait in traits_list:%}
+<div class="with-trait">
+  <input type="{%if step=='select-primary':%}radio{%else:%}checkbox{%endif%}"
+	 name="{%if step=='select-primary':%}primary_trait{%else:%}control_traits[]{%endif%}"
+	 value="{{trait['name']}}:::{{trait['dataset']}}:::{{trait['symbol']}}:::{{trait['description']}}:::{{trait['location']}}:::{{trait['mean_expr']}}:::{{trait['max_lrs']}}:::{{trait['data_hmac']}}"
+	 id="trait_{{trait['data_hmac']}}"
+	 class="selector-element" />
+  <label for="trait_{{trait['data_hmac']}}" class="label-element">
+    <span class="trait-dataset" data-title="dataset">{{trait["dataset"]}}</span>
+    <span class="trait-name" data-title="name">{{trait["name"]}}</span>
+    <span class="trait-symbol" data-title="symbol">{{trait["symbol"]}}</span>
+    <span class="trait-description" data-title="description">{{trait["description"]}}</span>
+    <span class="trait-locatin" data-title="location">{{trait["location"]}}</span>
+    <span class="trait-mean-expr" data-title="mean">{{"%0.3f" % trait["mean_expr"]|float}}</span>
+    <span class="trait-max-lrs" data-title="max LRS">{{trait["max_lrs"]}}</span>
+  </label>
+</div>
+{%endfor%}
diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py
index ff77982f..45a12f77 100644
--- a/wqflask/wqflask/user_login.py
+++ b/wqflask/wqflask/user_login.py
@@ -177,8 +177,6 @@ def verify_email():
 @app.route("/n/login", methods=('GET', 'POST'))
 def login():
     params = request.form if request.form else request.args
-    logger.debug("in login params are:", params)
-
     if not params:  # ZS: If coming to page for first time
         from utility.tools import GITHUB_AUTH_URL, GITHUB_CLIENT_ID, ORCID_AUTH_URL, ORCID_CLIENT_ID
         external_login = {}
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index be3d9238..b2cf95fb 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -1,6 +1,4 @@
 """Main routing table for GN2"""
-
-import MySQLdb
 import array
 import base64
 import csv
@@ -45,6 +43,7 @@ from flask import send_from_directory
 from flask import redirect
 from flask import url_for
 from flask import send_file
+from flask import jsonify
 
 # Some of these (like collect) might contain endpoints, so they're still used.
 # Blueprints should probably be used instead.
@@ -68,7 +67,9 @@ from wqflask.correlation.correlation_gn3_api import compute_correlation
 from wqflask.correlation_matrix import show_corr_matrix
 from wqflask.correlation import corr_scatter_plot
 # from wqflask.wgcna import wgcna_analysis
-# from wqflask.ctl import ctl_analysis
+from wqflask.ctl import ctl_analysis
+from wqflask.ctl.gn3_ctl_analysis import run_ctl
+
 from wqflask.wgcna.gn3_wgcna import run_wgcna
 from wqflask.snp_browser import snp_browser
 from wqflask.search_results import SearchResultPage
@@ -120,6 +121,7 @@ def shutdown_session(exception=None):
     db = getattr(g, '_database', None)
     if db is not None:
         db_session.remove()
+        g.db.dispose()
         g.db = None
 
 
@@ -354,6 +356,22 @@ def ctl_setup():
     return render_template("ctl_setup.html", **request.form)
 
 
+
+@app.route("/ctl_results", methods=["POST"])
+def ctl_results():
+
+    ctl_results = run_ctl(request.form)
+    return render_template("gn3_ctl_results.html", **ctl_results)
+
+
+@app.route("/ctl_network_files/<file_name>/<file_type>")
+def fetch_network_files(file_name, file_type):
+    file_path = f"{file_name}.{file_type}"
+
+    file_path  = os.path.join("/tmp/",file_path)
+
+    return send_file(file_path)
+
 @app.route("/intro")
 def intro():
     doc = Docs("intro", request.args)
@@ -1071,4 +1089,3 @@ def display_diffs_users():
     return render_template("display_files_user.html",
                            files=files)
 
-