From a3365dae23f204e489939d3defc55edc1b4872d8 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Mon, 1 Oct 2018 16:09:47 +0000
Subject: - Can now remove cofactors from correlation scatterplot and select
 them by just clicking their row in collection

- Cofactor color picker now works in Safari/Macs

- Displays N for relevant samples in trait page sample table

- Don't show bar chart when N>256

- Mapping loading page contents better centered

- Anonymous collections timeout correctly listed as 30 days now

- Minor allele frequency can actually be changed for GEMMA now (previously didn't work)

- Fixed transcript position marker location for mapping results

- Notifies user if their e-mail isn't associated with an account when they attempt to request forgotten password

- Users can now map with submitted traits

- Histogram width changes depending upon number of bins (need to improve this still)

- Improved Q-q plot (previously called "probability plot")
---
 wqflask/utility/helper_functions.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py
index cf16879f..1c8dad10 100644
--- a/wqflask/utility/helper_functions.py
+++ b/wqflask/utility/helper_functions.py
@@ -14,7 +14,13 @@ logger = logging.getLogger(__name__ )
 
 def get_species_dataset_trait(self, start_vars):
     #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype"
-    self.dataset = data_set.create_dataset(start_vars['dataset'])
+    if "temp_trait" in start_vars.keys():
+      if start_vars['temp_trait'] == "True":
+        self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group'])
+      else:
+        self.dataset = data_set.create_dataset(start_vars['dataset'])
+    else:
+      self.dataset = data_set.create_dataset(start_vars['dataset'])
     logger.debug("After creating dataset")
     self.species = TheSpecies(dataset=self.dataset)
     logger.debug("After creating species")
-- 
cgit v1.2.3


From 61c13a09dba95958f183dc55f3d59c8856b5f753 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 13 Feb 2019 12:41:45 +0000
Subject: Removed pylmm references and related functions

---
 bin/genenetwork2                                 |   1 -
 etc/default_settings.py                          |   1 -
 test/requests/mapping_tests.py                   |  10 --
 wqflask/base/data_set.py                         |   2 +-
 wqflask/utility/tools.py                         |   4 -
 wqflask/wqflask/heatmap/heatmap.py               |   5 +-
 wqflask/wqflask/marker_regression/run_mapping.py | 204 +----------------------
 7 files changed, 4 insertions(+), 223 deletions(-)

(limited to 'wqflask/utility')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 21f0db13..7c875274 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -131,7 +131,6 @@ else
     export LC_ALL=C # FIXME
     export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2"
     export PLINK_COMMAND="$GN2_PROFILE/bin/plink2"
-    export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis"
     export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma"
     if [ -z $GEMMA_WRAPPER_COMMAND ]; then
         export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 3e54ad1f..a1fe81e5 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -89,7 +89,6 @@ GENENETWORK_FILES   = HOME+"/genotype_files"  # base dir for all static data fil
 JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript"
 
 # ---- GN2 Executables (overwrite for testing only)
-# PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read())
 # PLINK_COMMAND = str.strip(os.popen("which plink2").read())
 # GEMMA_COMMAND = str.strip(os.popen("which gemma").read())
 # GEMMA_WRAPPER_COMMAND = str.strip(os.popen("which gemma-wrapper").read())
diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py
index 8eb19de7..6de81bfe 100644
--- a/test/requests/mapping_tests.py
+++ b/test/requests/mapping_tests.py
@@ -11,16 +11,6 @@ def load_data_from_file():
     file_data = json.loads(file_handle.read().encode("utf-8"))
     return file_data
 
-def check_pylmm_tool_selection(host, data):
-    print("")
-    print("pylmm mapping tool selection")
-    data["method"] = "pylmm"
-    page = requests.post(host+"/marker_regression", data=data)
-    doc = fromstring(page.text)
-    form = doc.forms[1]
-    assert form.fields["dataset"] == "HC_M2_0606_P"
-    assert form.fields["value:BXD1"] == "15.034" # Check value in the file
-
 def check_R_qtl_tool_selection(host, data):
     print("")
     print("R/qtl mapping tool selection")
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 79f72390..ca6621e9 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -317,7 +317,7 @@ class DatasetGroup(object):
 
         mapping_id = g.db.execute("select MappingMethodId from InbredSet where Name= '%s'" % self.name).fetchone()[0]
         if mapping_id == "1":
-            mapping_names = ["QTLReaper", "PYLMM", "R/qtl"]
+            mapping_names = ["QTLReaper", "R/qtl"]
         elif mapping_id == "2":
             mapping_names = ["GEMMA"]
         elif mapping_id == "4":
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index ea216a35..86ef2e1e 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -107,9 +107,6 @@ def js_path(module=None):
         return try_guix
     raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)"
 
-def pylmm_command(guess=None):
-    return assert_bin(get_setting("PYLMM_COMMAND",guess))
-
 def gemma_command(guess=None):
     return assert_bin(get_setting("GEMMA_COMMAND",guess))
 
@@ -276,7 +273,6 @@ SMTP_CONNECT = get_setting('SMTP_CONNECT')
 SMTP_USERNAME = get_setting('SMTP_USERNAME')
 SMTP_PASSWORD = get_setting('SMTP_PASSWORD')
 
-PYLMM_COMMAND      = app_set("PYLMM_COMMAND",pylmm_command())
 GEMMA_COMMAND      = app_set("GEMMA_COMMAND",gemma_command())
 assert(GEMMA_COMMAND is not None)
 PLINK_COMMAND      = app_set("PLINK_COMMAND",plink_command())
diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py
index ff589693..1bdf252b 100644
--- a/wqflask/wqflask/heatmap/heatmap.py
+++ b/wqflask/wqflask/heatmap/heatmap.py
@@ -24,12 +24,9 @@ import reaper
 from base.trait import GeneralTrait
 from base import data_set
 from base import species
-# from wqflask.my_pylmm.pyLMM import lmm
-# from wqflask.my_pylmm.pyLMM import input
 from utility import helper_functions
 from utility import Plot, Bunch
 from utility import temp_data
-from utility.tools import PYLMM_COMMAND
 
 from MySQLdb import escape_string as escape
 
@@ -144,4 +141,4 @@ class Heatmap(object):
                 if qtl.additive > 0:
                     self.trait_results[this_trait.name].append(-float(qtl.lrs))
                 else:
-                    self.trait_results[this_trait.name].append(float(qtl.lrs))
\ No newline at end of file
+                    self.trait_results[this_trait.name].append(float(qtl.lrs))
diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py
index 3057e340..73d985b8 100644
--- a/wqflask/wqflask/marker_regression/run_mapping.py
+++ b/wqflask/wqflask/marker_regression/run_mapping.py
@@ -38,7 +38,7 @@ from utility import temp_data
 from utility.benchmark import Bench
 from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping
 
-from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR
+from utility.tools import locate, locate_ignore_error, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR
 from utility.external import shell
 from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR
 
@@ -239,11 +239,6 @@ class RunMapping(object):
             self.manhattan_plot = True
             results = plink_mapping.run_plink(self.this_trait, self.dataset, self.species, self.vals, self.maf)
             #results = self.run_plink()
-        elif self.mapping_method == "pylmm":
-            logger.debug("RUNNING PYLMM")
-            if self.num_perm > 0:
-                self.run_permutations(str(temp_uuid))
-            results = self.gen_data(str(temp_uuid))
         else:
             logger.debug("RUNNING NOTHING")
 
@@ -354,201 +349,6 @@ class RunMapping(object):
 
         count, p_values = self.parse_rqtl_output(plink_output_filename)
 
-    def run_permutations(self, temp_uuid):
-        """Runs permutations and gets significant and suggestive LOD scores"""
-
-        top_lod_scores = []
-
-        #logger.debug("self.num_perm:", self.num_perm)
-
-        for permutation in range(self.num_perm):
-
-            pheno_vector = np.array([val == "x" and np.nan or float(val) for val in self.vals])
-            np.random.shuffle(pheno_vector)
-
-            key = "pylmm:input:" + temp_uuid
-
-            if self.dataset.group.species == "human":
-                p_values, t_stats = self.gen_human_results(pheno_vector, key, temp_uuid)
-            else:
-                genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers]
-
-                no_val_samples = self.identify_empty_samples()
-                trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples)
-
-                genotype_matrix = np.array(trimmed_genotype_data).T
-
-                params = dict(pheno_vector = pheno_vector.tolist(),
-                            genotype_matrix = genotype_matrix.tolist(),
-                            restricted_max_likelihood = True,
-                            refit = False,
-                            temp_uuid = temp_uuid,
-
-                            # meta data
-                            timestamp = datetime.datetime.now().isoformat(),
-                            )
-
-                json_params = json.dumps(params)
-                Redis.set(key, json_params)
-                Redis.expire(key, 60*60)
-
-                command = PYLMM_COMMAND+' --key {} --species {}'.format(key,"other")
-                shell(command)
-
-                json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60)
-                results = json.loads(json_results[1])
-                p_values = [float(result) for result in results['p_values']]
-
-                lowest_p_value = 1
-                for p_value in p_values:
-                    if p_value < lowest_p_value:
-                        lowest_p_value = p_value
-
-                #logger.debug("lowest_p_value:", lowest_p_value)
-                top_lod_scores.append(-math.log10(lowest_p_value))
-
-        #logger.debug("top_lod_scores:", top_lod_scores)
-
-        self.suggestive = np.percentile(top_lod_scores, 67)
-        self.significant = np.percentile(top_lod_scores, 95)
-
-    def gen_data(self, temp_uuid):
-        """Generates p-values for each marker"""
-
-        logger.debug("self.vals is:", self.vals)
-        pheno_vector = np.array([(val == "x" or val == "") and np.nan or float(val) for val in self.vals])
-
-        #lmm_uuid = str(uuid.uuid4())
-
-        key = "pylmm:input:" + temp_uuid
-        logger.debug("key is:", pf(key))
-        #with Bench("Loading cache"):
-        #    result = Redis.get(key)
-
-        if self.dataset.group.species == "human":
-            p_values, t_stats = self.gen_human_results(pheno_vector, key, temp_uuid)
-
-        else:
-            logger.debug("NOW CWD IS:", os.getcwd())
-            genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers]
-
-            no_val_samples = self.identify_empty_samples()
-            trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples)
-
-            genotype_matrix = np.array(genotype_data).T
-
-            #logger.debug("pheno_vector: ", pf(pheno_vector))
-            #logger.debug("genotype_matrix: ", pf(genotype_matrix))
-            #logger.debug("genotype_matrix.shape: ", pf(genotype_matrix.shape))
-
-            #params = {"pheno_vector": pheno_vector,
-            #            "genotype_matrix": genotype_matrix,
-            #            "restricted_max_likelihood": True,
-            #            "refit": False,
-            #            "temp_data": tempdata}
-
-            # logger.debug("genotype_matrix:", str(genotype_matrix.tolist()))
-            # logger.debug("pheno_vector:", str(pheno_vector.tolist()))
-
-            params = dict(pheno_vector = pheno_vector.tolist(),
-                        genotype_matrix = genotype_matrix.tolist(),
-                        restricted_max_likelihood = True,
-                        refit = False,
-                        temp_uuid = temp_uuid,
-
-                        # meta data
-                        timestamp = datetime.datetime.now().isoformat(),
-                        )
-
-            json_params = json.dumps(params)
-            #logger.debug("json_params:", json_params)
-            Redis.set(key, json_params)
-            Redis.expire(key, 60*60)
-            logger.debug("before printing command")
-
-            command = PYLMM_COMMAND + ' --key {} --species {}'.format(key, "other")
-            logger.debug("command is:", command)
-            logger.debug("after printing command")
-
-            shell(command)
-
-            #t_stats, p_values = lmm.run(key)
-            #lmm.run(key)
-
-            json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60)
-            results = json.loads(json_results[1])
-            p_values = [float(result) for result in results['p_values']]
-            t_stats = results['t_stats']
-
-            #t_stats, p_values = lmm.run(
-            #    pheno_vector,
-            #    genotype_matrix,
-            #    restricted_max_likelihood=True,
-            #    refit=False,
-            #    temp_data=tempdata
-            #)
-            #logger.debug("p_values:", p_values)
-
-        self.dataset.group.markers.add_pvalues(p_values)
-
-        return self.dataset.group.markers.markers
-
-    def gen_human_results(self, pheno_vector, key, temp_uuid):
-        file_base = locate(self.dataset.group.name,"mapping")
-
-        plink_input = input.plink(file_base, type='b')
-        input_file_name = os.path.join(webqtlConfig.SNP_PATH, self.dataset.group.name + ".snps.gz")
-
-        pheno_vector = pheno_vector.reshape((len(pheno_vector), 1))
-        covariate_matrix = np.ones((pheno_vector.shape[0],1))
-        kinship_matrix = np.fromfile(open(file_base + '.kin','r'),sep=" ")
-        kinship_matrix.resize((len(plink_input.indivs),len(plink_input.indivs)))
-
-        logger.debug("Before creating params")
-
-        params = dict(pheno_vector = pheno_vector.tolist(),
-                    covariate_matrix = covariate_matrix.tolist(),
-                    input_file_name = input_file_name,
-                    kinship_matrix = kinship_matrix.tolist(),
-                    refit = False,
-                    temp_uuid = temp_uuid,
-
-                    # meta data
-                    timestamp = datetime.datetime.now().isoformat(),
-                    )
-
-        logger.debug("After creating params")
-
-        json_params = json.dumps(params)
-        Redis.set(key, json_params)
-        Redis.expire(key, 60*60)
-
-        logger.debug("Before creating the command")
-
-        command = PYLMM_COMMAND+' --key {} --species {}'.format(key, "human")
-
-        logger.debug("command is:", command)
-
-        os.system(command)
-
-        json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60)
-        results = json.loads(json_results[1])
-        t_stats = results['t_stats']
-        p_values = results['p_values']
-
-
-        #p_values, t_stats = lmm.run_human(key)
-
-        #p_values, t_stats = lmm.run_human(
-        #        pheno_vector,
-        #        covariate_matrix,
-        #        input_file_name,
-        #        kinship_matrix,
-        #        loading_progress=tempdata
-        #    )
-
-        return p_values, t_stats
-
     def identify_empty_samples(self):
         no_val_samples = []
         for sample_count, val in enumerate(self.vals):
@@ -657,4 +457,4 @@ def trim_markers_for_table(markers):
         trimmed_sorted_markers = sorted_markers[:2000]
         return trimmed_sorted_markers
     else:
-        return sorted_markers
\ No newline at end of file
+        return sorted_markers
-- 
cgit v1.2.3


From 496759ad08efb02b4268ea7f3bbb7905974237e9 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 18 Feb 2019 09:39:41 +0000
Subject: Updated installation instructions and SERVER_PORT for single flask
 server

---
 doc/README.org           | 71 ++++++++++++++++++++++++++++++++++--------------
 wqflask/runserver.py     |  8 ++++--
 wqflask/utility/tools.py |  1 +
 3 files changed, 57 insertions(+), 23 deletions(-)

(limited to 'wqflask/utility')

diff --git a/doc/README.org b/doc/README.org
index 5dc9e994..620c946c 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -12,13 +12,15 @@
    - [[#load-the-small-database-in-mysql][Load the small database in MySQL]]
  - [[#gn2-dependency-graph][GN2 Dependency Graph]]
  - [[#working-with-the-gn2-source-code][Working with the GN2 source code]]
+ - [[#running-elasticsearch][Running ElasticSearch]]
+   - [[#systemd][SystemD]]
+ - [[#read-more][Read more]]
  - [[#trouble-shooting][Trouble shooting]]
    - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]]
-   - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]]
+   - [[#error-can-not-find-directory-homegn2_data-or-can-not-find-directory-homegenotype_filesgenotype][ERROR: 'can not find directory $HOME/gn2_data' or 'can not find directory $HOME/genotype_files/genotype']]
    - [[#cant-run-a-module][Can't run a module]]
    - [[#rpy2-error-show-now-found][Rpy2 error 'show' now found]]
    - [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]]
- - [[#read-more][Read more]]
  - [[#irc-session][IRC session]]
 
 * Introduction
@@ -100,11 +102,13 @@ mysql (which comes as part of the GNU Guix genenetwork2 install).
 
 As root configure and run
 
-:  adduser mysql && addgroup mysql
-:  mysqld --datadir=/var/mysql --initialize-insecure
-:  mkdir -p /var/run/mysqld
-:  chown mysql.mysql ~/mysql /var/run/mysqld
-:  mysqld -u mysql --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048"
+#+BEGIN_SRC bash
+adduser mysql && addgroup mysql
+mysqld --datadir=/var/mysql --initialize-insecure
+mkdir -p /var/run/mysqld
+chown mysql.mysql ~/mysql /var/run/mysqld
+mysqld -u mysql --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048"
+#+END_SRC
 
 If you want to run as root you may have to set
 
@@ -192,6 +196,41 @@ http://biogems.info/contrib/genenetwork/gn2.svg
 
 See [[development.org]].
 
+* Running ElasticSearch
+
+In order to start up elasticsearch:
+Penguin - change user to "elasticsearch" and use the following command: "env JAVA_HOME=/opt/jdk-9.0.4 /opt/elasticsearch-6.2.1/bin/elasticsearch"
+
+
+** SystemD
+
+New server - as root run "systemctl restart elasticsearch"
+
+#+BEGIN_SRC
+tux01:/etc/systemd/system# cat elasticsearch.service
+[Unit]
+Description=Run Elasticsearch
+
+[Service]
+ExecStart=/opt/elasticsearch-6.2.1/bin/elasticsearch
+Environment=JAVA_HOME=/opt/jdk-9.0.4
+Environment="ES_JAVA_OPTS=-Xms1g -Xmx8g"
+Environment="PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/opt/jdk-9.0.4/bin"
+LimitNOFILE=65536
+StandardOutput=syslog
+StandardError=syslog
+User=elasticsearch
+
+[Install]
+WantedBy=multi-user.target
+#+END_SRC
+
+* Read more
+
+If you want to understand the architecture of GN2 read
+[[Architecture.org]].  The rest of this document is mostly on deployment
+of GN2.
+
 * Trouble shooting
 
 ** ImportError: No module named jinja2
@@ -210,13 +249,17 @@ On one system:
 : export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0"
 
 and perhaps a few more.
-** ERROR: can not find directory $HOME/gn2_data
+** ERROR: 'can not find directory $HOME/gn2_data' or 'can not find directory $HOME/genotype_files/genotype'
 
 The default settings file looks in your $HOME/gn2_data. Since these
 files come with a Guix installation you should take a hint from the
 values in the installed version of default_settings.py (see above in
 this document).
 
+You can use the GENENETWORK_FILES switch to set the datadir, for example
+
+: env GN2_PROFILE=~/opt/gn-latest GENENETWORK_FILES=/gnu/data/gn2_data ./bin/genenetwork2
+
 ** Can't run a module
 
 In rare cases, development modules are not brought in with Guix
@@ -257,18 +300,6 @@ if that works run genenetwork after setting SQL_URI to something like
 
 : export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s
 
-* Running ElasticSearch
-
-In order to start up elasticsearch:
-Penguin - change user to "elasticsearch" and use the following command: "env JAVA_HOME=/opt/jdk-9.0.4 /opt/elasticsearch-6.2.1/bin/elasticsearch"
-
-New server - as root run "systemctl restart elasticsearch"
-
-* Read more
-
-If you want to understand the architecture of GN2 read
-[[Architecture.org]].  The rest of this document is mostly on deployment
-of GN2.
 
 * IRC session
 
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index 5f41d04d..7c06356b 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -27,9 +27,11 @@ app_config()
 
 werkzeug_logger = logging.getLogger('werkzeug')
 
+from utility.tools import WEBSERVER_MODE, SERVER_PORT
+
 if WEBSERVER_MODE == 'DEBUG':
     app.run(host='0.0.0.0',
-            port=port,
+            port=SERVER_PORT,
             debug=True,
             use_debugger=False,
             threaded=False,
@@ -38,7 +40,7 @@ if WEBSERVER_MODE == 'DEBUG':
 elif WEBSERVER_MODE == 'DEV':
     werkzeug_logger.setLevel(logging.WARNING)
     app.run(host='0.0.0.0',
-            port=port,
+            port=SERVER_PORT,
             debug=False,
             use_debugger=False,
             threaded=False,
@@ -46,7 +48,7 @@ elif WEBSERVER_MODE == 'DEV':
             use_reloader=True)
 else: # staging/production modes
     app.run(host='0.0.0.0',
-            port=port,
+            port=SERVER_PORT,
             debug=False,
             use_debugger=False,
             threaded=True,
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 86ef2e1e..8b2260f5 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -232,6 +232,7 @@ GN_VERSION         = get_setting('GN_VERSION')
 HOME               = get_setting('HOME')
 WEBSERVER_MODE     = get_setting('WEBSERVER_MODE')
 GN_SERVER_URL      = get_setting('GN_SERVER_URL')
+SERVER_PORT        = get_setting_int('SERVER_PORT')
 SQL_URI            = get_setting('SQL_URI')
 LOG_LEVEL          = get_setting('LOG_LEVEL')
 LOG_LEVEL_DEBUG    = get_setting_int('LOG_LEVEL_DEBUG')
-- 
cgit v1.2.3


From 4b4007f00f26afadc42d451c78e1253235ad65f3 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 22 Feb 2019 17:07:58 -0600
Subject: Fixed some appearance issues with several tables

Fixed issue where cofactor selection screens were broken

Changed case attributes to be applied across groups, though need to discuss this with Rob since it shouldn't always be done this way

Added new inversion transformation
---
 wqflask/base/trait.py                              |  4 ++
 wqflask/utility/Plot.py                            |  3 -
 wqflask/wqflask/search_results.py                  | 11 ++-
 wqflask/wqflask/show_trait/SampleList.py           | 32 ++++-----
 wqflask/wqflask/show_trait/show_trait.py           | 22 ++++++
 .../new/javascript/dataset_menu_structure.json     | 79 ++++++++++++----------
 .../javascript/get_covariates_from_collection.js   | 76 ++++++++++++++++++---
 .../wqflask/static/new/javascript/show_trait.js    | 65 ++++++------------
 wqflask/wqflask/templates/search_result_page.html  | 22 +++---
 wqflask/wqflask/templates/show_trait.html          | 30 +++++++-
 .../templates/show_trait_mapping_tools.html        |  2 +-
 .../templates/show_trait_transform_and_filter.html |  1 +
 12 files changed, 221 insertions(+), 126 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index e57d4176..9f76306f 100644
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -206,6 +206,7 @@ def jsonable(trait):
         return dict(name=trait.name,
                     symbol=trait.symbol,
                     dataset=dataset.name,
+                    dataset_name = dataset.shortname,
                     description=trait.description_display,
                     mean=trait.mean,
                     location=trait.location_repr,
@@ -217,6 +218,7 @@ def jsonable(trait):
         if trait.pubmed_id:
             return dict(name=trait.name,
                         dataset=dataset.name,
+                        dataset_name = dataset.shortname,
                         description=trait.description_display,
                         authors=trait.authors,
                         pubmed_text=trait.pubmed_text,
@@ -228,6 +230,7 @@ def jsonable(trait):
         else:
             return dict(name=trait.name,
                         dataset=dataset.name,
+                        dataset_name = dataset.shortname,
                         description=trait.description_display,
                         authors=trait.authors,
                         pubmed_text=trait.pubmed_text,
@@ -238,6 +241,7 @@ def jsonable(trait):
     elif dataset.type == "Geno":
         return dict(name=trait.name,
                     dataset=dataset.name,
+                    dataset_name = dataset.shortname,
                     location=trait.location_repr
                     )
     else:
diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py
index 529cd117..cce8435d 100644
--- a/wqflask/utility/Plot.py
+++ b/wqflask/utility/Plot.py
@@ -98,8 +98,6 @@ def find_outliers(vals):
 
     """
 
-    logger.debug("xerxes vals is:", pf(vals))
-
     if vals:
         #logger.debug("vals is:", pf(vals))
         stats = corestats.Stats(vals)
@@ -114,7 +112,6 @@ def find_outliers(vals):
         upper_bound = None
         lower_bound = None
 
-    logger.debug(pf(locals()))
     return upper_bound, lower_bound
 
 # parameter: data is either object returned by reaper permutation function (called by MarkerRegressionPage.py)
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index c67063b0..893fd172 100644
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -112,7 +112,7 @@ views.py).
             trait_dict['hmac'] = user_manager.data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name))
             if this_trait.dataset.type == "ProbeSet":
                 trait_dict['symbol'] = this_trait.symbol
-                trait_dict['description'] = this_trait.description_display
+                trait_dict['description'] = insert_newlines(this_trait.description_display)
                 trait_dict['location'] = this_trait.location_repr
                 trait_dict['mean'] = "N/A"
                 trait_dict['additive'] = "N/A"
@@ -125,7 +125,7 @@ views.py).
             elif this_trait.dataset.type == "Geno":
                 trait_dict['location'] = this_trait.location_repr
             elif this_trait.dataset.type == "Publish":
-                trait_dict['description'] = this_trait.description_display
+                trait_dict['description'] = insert_newlines(this_trait.description_display)
                 trait_dict['authors'] = this_trait.authors
                 trait_dict['pubmed_id'] = "N/A"
                 if this_trait.pubmed_id:
@@ -222,3 +222,10 @@ views.py).
             return the_search
         else:
             return None
+
+def insert_newlines(string, every=64):
+    """ This is because it is seemingly impossible to change the width of the description column, so I'm just manually adding line breaks """
+    lines = []
+    for i in xrange(0, len(string), every):
+        lines.append(string[i:i+every])
+    return '\n'.join(lines)
diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py
index 50d7b6c0..50026bba 100644
--- a/wqflask/wqflask/show_trait/SampleList.py
+++ b/wqflask/wqflask/show_trait/SampleList.py
@@ -39,7 +39,7 @@ class SampleList(object):
 
         #self.sample_qnorm = get_transform_vals(self.dataset, this_trait)
 
-        if self.this_trait and self.dataset and self.dataset.type == 'ProbeSet':
+        if self.this_trait and self.dataset:
             self.get_extra_attribute_values()
 
         for counter, sample_name in enumerate(sample_names, 1):
@@ -72,10 +72,9 @@ class SampleList(object):
 
             self.sample_list.append(sample)
 
-        logger.debug("self.attributes is", pf(self.attributes))
+        #logger.debug("attribute vals are", pf(self.sample_attribute_values))
 
         self.do_outliers()
-        logger.debug("*the_samples are [%i]: %s" % (len(self.sample_list), pf(self.sample_list)))
 
     def __repr__(self):
         return "<SampleList> --> %s" % (pf(self.__dict__))
@@ -98,11 +97,11 @@ class SampleList(object):
 
         # Get attribute names and distinct values for each attribute
         results = g.db.execute('''
-                        SELECT DISTINCT CaseAttribute.Id, CaseAttribute.Name, CaseAttributeXRef.Value
-                        FROM CaseAttribute, CaseAttributeXRef
-                        WHERE CaseAttributeXRef.CaseAttributeId = CaseAttribute.Id
-                        AND CaseAttributeXRef.ProbeSetFreezeId = %s
-                        ORDER BY CaseAttribute.Name''', (str(self.dataset.id),))
+                        SELECT DISTINCT CaseAttribute.Id, CaseAttribute.Name, CaseAttributeXRefNew.Value
+                        FROM CaseAttribute, CaseAttributeXRefNew
+                        WHERE CaseAttributeXRefNew.CaseAttributeId = CaseAttribute.Id
+                        AND CaseAttributeXRefNew.InbredSetId = %s
+                        ORDER BY CaseAttribute.Name''', (str(self.dataset.group.id),))
 
         self.attributes = {}
         for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name)):
@@ -115,16 +114,17 @@ class SampleList(object):
 
     def get_extra_attribute_values(self):
         if self.attributes:
-            results = g.db.execute('''
-                        SELECT Strain.Name AS SampleName, CaseAttributeId AS Id, CaseAttributeXRef.Value
-                        FROM Strain, StrainXRef, InbredSet, CaseAttributeXRef
+            query = '''
+                        SELECT Strain.Name AS SampleName, CaseAttributeId AS Id, CaseAttributeXRefNew.Value
+                        FROM Strain, StrainXRef, InbredSet, CaseAttributeXRefNew
                         WHERE StrainXRef.StrainId = Strain.Id
                         AND InbredSet.Id = StrainXRef.InbredSetId
-                        AND CaseAttributeXRef.StrainId = Strain.Id
-                        AND InbredSet.Name = %s
-                        AND CaseAttributeXRef.ProbeSetFreezeId = %s
-                        ORDER BY SampleName''',
-                       (self.dataset.group.name, self.this_trait.dataset.id))
+                        AND CaseAttributeXRefNew.StrainId = Strain.Id
+                        AND InbredSet.Id = CaseAttributeXRefNew.InbredSetId
+                        AND CaseAttributeXRefNew.InbredSetId = %s
+                        ORDER BY SampleName''' % self.dataset.group.id
+
+            results = g.db.execute(query)
 
             for sample_name, items in itertools.groupby(results.fetchall(), lambda row: row.SampleName):
                 attribute_values = {}
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index 0d1ed345..9b4470fe 100644
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -350,6 +350,28 @@ def quantile_normalize_vals(sample_groups):
 
     return qnorm_by_group
 
+def get_z_scores(sample_groups):
+    zscore_by_group = []
+    for sample_type in sample_groups:
+        trait_vals = []
+        for sample in sample_type.sample_list:
+            try:
+                trait_vals.append(float(sample.value))
+            except:
+                continue
+
+        qnorm_vals = normf(trait_vals)
+        qnorm_vals_with_x = []
+        counter = 0
+        for sample in sample_type.sample_list:
+            if sample.display_value == "x":
+                qnorm_vals_with_x.append("x")
+            else:
+                qnorm_vals_with_x.append(qnorm_vals[counter])
+                counter += 1
+
+        qnorm_by_group.append(qnorm_vals_with_x)
+
 def get_nearest_marker(this_trait, this_db):
     this_chr = this_trait.locus_chr
     logger.debug("this_chr:", this_chr)
diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
index 3f2673c1..7d00d509 100644
--- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
+++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
@@ -1560,7 +1560,7 @@
          "AIL-LGSM-F34-A": {
             "Phenotypes": [
                [
-                  "None",
+                  "655",
                   "AIL-LGSM-F34-APublish",
                   "AIL-LGSM-F34-A Phenotypes"
                ]
@@ -1569,7 +1569,7 @@
          "AIL-LGSM-F34-F39-43-GBS": {
             "Phenotypes": [
                [
-                  "None",
+                  "654",
                   "AIL-LGSM-F34-F39-43-GBSPublish",
                   "AIL-LGSM-F34-F39-43-GBS Phenotypes"
                ]
@@ -1578,7 +1578,7 @@
          "AIL-LGSM-F34-GBS": {
             "Phenotypes": [
                [
-                  "None",
+                  "656",
                   "AIL-LGSM-F34-GBSPublish",
                   "AIL-LGSM-F34-GBS Phenotypes"
                ]
@@ -1587,7 +1587,7 @@
          "AIL-LGSM-F39-43-GBS": {
             "Phenotypes": [
                [
-                  "None",
+                  "657",
                   "AIL-LGSM-F39-43-GBSPublish",
                   "AIL-LGSM-F39-43-GBS Phenotypes"
                ]
@@ -1712,11 +1712,6 @@
          },
          "B6D2F2": {
             "Brain mRNA": [
-               [
-                  "77",
-                  "BRF2_M_0805_R",
-                  "OHSU/VA B6D2F2 Brain mRNA M430 (Aug05) RMA"
-               ],
                [
                   "76",
                   "BRF2_M_0805_M",
@@ -1727,6 +1722,11 @@
                   "BRF2_M_0805_P",
                   "OHSU/VA B6D2F2 Brain mRNA M430 (Aug05) PDNN"
                ],
+               [
+                  "77",
+                  "BRF2_M_0805_R",
+                  "OHSU/VA B6D2F2 Brain mRNA M430 (Aug05) RMA"
+               ],
                [
                   "33",
                   "BRF2_M_0304_P",
@@ -2081,16 +2081,16 @@
                ]
             ],
             "Brain mRNA": [
-               [
-                  "164",
-                  "UTHSC_BXD_WB_RNASeq1112",
-                  "UTHSC Mouse BXD Whole Brain RNA Sequence (Nov12) RPKM Untrimmed"
-               ],
                [
                   "590",
                   "UTHSC_BXD_WB_RNASeqtrim1_1112",
                   "UTHSC Mouse BXD Whole Brain RNA Sequence (Nov12) RPKM Trimmed 1.0"
                ],
+               [
+                  "164",
+                  "UTHSC_BXD_WB_RNASeq1112",
+                  "UTHSC Mouse BXD Whole Brain RNA Sequence (Nov12) RPKM Untrimmed"
+               ],
                [
                   "394",
                   "UTHSC_BXD_WB_RNASeqEx1112",
@@ -2232,11 +2232,6 @@
                   "Eye_M2_0908_R_ND",
                   "Eye M430v2 WT Gpnmb (Sep08) RMA"
                ],
-               [
-                  "279",
-                  "Eye_M2_0908_R_WT",
-                  "Eye M430v2 WT Tyrp1 (Sep08) RMA"
-               ],
                [
                   "278",
                   "Eye_M2_0908_R_MT",
@@ -2247,6 +2242,11 @@
                   "Eye_M2_0908_WTWT",
                   "Eye M430v2 WT WT (Sep08) RMA"
                ],
+               [
+                  "279",
+                  "Eye_M2_0908_R_WT",
+                  "Eye M430v2 WT Tyrp1 (Sep08) RMA"
+               ],
                [
                   "400",
                   "DBA2J-ONH-1212",
@@ -2530,16 +2530,16 @@
                ]
             ],
             "Kidney mRNA": [
-               [
-                  "239",
-                  "MA_M2F_0706_R",
-                  "Mouse kidney M430v2 Female (Aug06) RMA"
-               ],
                [
                   "240",
                   "MA_M2M_0706_R",
                   "Mouse kidney M430v2 Male (Aug06) RMA"
                ],
+               [
+                  "239",
+                  "MA_M2F_0706_R",
+                  "Mouse kidney M430v2 Female (Aug06) RMA"
+               ],
                [
                   "118",
                   "MA_M2_0806_R",
@@ -2691,6 +2691,11 @@
                   "EPFLMouseLiverHFDRMA0413",
                   "EPFL/LISP BXD HFD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA"
                ],
+               [
+                  "849",
+                  "EPFLMouseLiverCDEx0413",
+                  "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA Exon Level"
+               ],
                [
                   "848",
                   "EPFLMouseLiverHFCEx0413",
@@ -2701,11 +2706,6 @@
                   "EPFLMouseLiverCDRMA0413",
                   "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA"
                ],
-               [
-                  "849",
-                  "EPFLMouseLiverCDEx0413",
-                  "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA Exon Level"
-               ],
                [
                   "433",
                   "EPFLMouseLiverBothExRMA0413",
@@ -3121,6 +3121,13 @@
             ]
          },
          "BXD-Harvested": {
+            "Hippocampus mRNA": [
+               [
+                  "873",
+                  "JAX-BXD-Hip-Pro-0219",
+                  "JAX BXD Hippocampal Proteome (Feb19)"
+               ]
+            ],
             "Liver mRNA": [
                [
                   "843",
@@ -3538,11 +3545,6 @@
                ]
             ],
             "Hippocampus mRNA": [
-               [
-                  "213",
-                  "Illum_LXS_Hipp_NOS_1008",
-                  "Hippocampus Illumina NOS (Oct08) RankInv beta"
-               ],
                [
                   "219",
                   "Illum_LXS_Hipp_NON_1008",
@@ -3563,6 +3565,11 @@
                   "Illum_LXS_Hipp_RSS_1008",
                   "Hippocampus Illumina RSS (Oct08) RankInv beta"
                ],
+               [
+                  "213",
+                  "Illum_LXS_Hipp_NOS_1008",
+                  "Hippocampus Illumina NOS (Oct08) RankInv beta"
+               ],
                [
                   "143",
                   "Illum_LXS_Hipp_loess0807",
@@ -5332,6 +5339,10 @@
                "Phenotypes",
                "Phenotypes"
             ],
+            [
+               "Hippocampus mRNA",
+               "Hippocampus mRNA"
+            ],
             [
                "Liver mRNA",
                "Liver mRNA"
diff --git a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js
index f2e694d8..fc6e5a78 100644
--- a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js
+++ b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js
@@ -2,12 +2,65 @@
 var add_trait_data, assemble_into_json, back_to_collections, collection_click, collection_list, color_by_trait, create_trait_data_csv, get_this_trait_vals, get_trait_data, process_traits, selected_traits, submit_click, this_trait_data, trait_click,
   __indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; };
 
-collection_list = null;
-
 this_trait_data = null;
 
 selected_traits = {};
 
+$('#collections_list').attr("style", "width: 100%;");
+$('#trait_table').dataTable( {
+    "drawCallback": function( settings ) {
+         $('#trait_table tr').click(function(event) {
+             if (event.target.type !== 'checkbox') {
+                 $(':checkbox', this).trigger('click');
+             }
+         });
+    },
+    "columns": [
+        { "type": "natural", "width": "3%" },
+        { "type": "natural", "width": "8%" },
+        { "type": "natural", "width": "20%" },
+        { "type": "natural", "width": "25%" },
+        { "type": "natural", "width": "25%" },
+        { "type": "natural", "width": "15%" }
+    ],
+    "columnDefs": [ {
+        "targets": 0,
+        "orderable": false
+    } ],
+    "order": [[1, "asc" ]],
+    "sDom": "RZtr",
+    "iDisplayLength": -1,
+    "autoWidth": true,
+    "bDeferRender": true,
+    "bSortClasses": false,
+    "paging": false,
+    "orderClasses": true
+} );
+
+$('#collection_table').dataTable( {
+    "createdRow": function ( row, data, index ) {
+        if ($('td', row).eq(2).text().length > 40) {
+            $('td', row).eq(2).text($('td', row).eq(2).text().substring(0, 40));
+            $('td', row).eq(2).text($('td', row).eq(2).text() + '...')
+        }
+        if ($('td', row).eq(4).text().length > 50) {
+            $('td', row).eq(4).text($('td', row).eq(4).text().substring(0, 50));
+            $('td', row).eq(4).text($('td', row).eq(4).text() + '...')
+        }
+    },
+    "columnDefs": [ {
+        "targets": 0,
+        "orderable": false
+    } ],
+    "order": [[1, "asc" ]],
+    "sDom": "ZRtr",
+    "iDisplayLength": -1,
+    "autoWidth": true,
+    "bSortClasses": false,
+    "paging": false,
+    "orderClasses": true
+} );
+
 collection_click = function() {
   var this_collection_url;
   console.log("Clicking on:", $(this));
@@ -28,16 +81,20 @@ submit_click = function() {
   $('#collections_holder').find('input[type=checkbox]:checked').each(function() {
     var this_dataset, this_trait;
     this_trait = $(this).parents('tr').find('.trait').text();
+    this_description = $(this).parents('tr').find('.description').text();
     console.log("this_trait is:", this_trait);
-    this_dataset = $(this).parents('tr').find('.dataset').text();
+    this_dataset = $(this).parents('tr').find('.dataset').data("dataset");
     console.log("this_dataset is:", this_dataset);
     covariates_string += this_trait + ":" + this_dataset + ","
-    covariates_display_string += this_trait + "\n"
+    this_covariate_display_string = this_trait + ": " + this_description
+    if (this_covariate_display_string.length > 50) {
+      this_covariate_display_string = this_covariate_display_string.substring(0, 45) + "..."
+    }
+    covariates_display_string += this_covariate_display_string + "\n"
   });
   // Trim the last comma
   covariates_string = covariates_string.substring(0, covariates_string.length - 1)
   //covariates_display_string = covariates_display_string.substring(0, covariates_display_string.length - 2)
-  console.log("COVARIATES:", covariates_string)
 
   $("input[name=covariates]").val(covariates_string)
   $(".selected_covariates").val(covariates_display_string)
@@ -125,17 +182,16 @@ process_traits = function(trait_data, textStatus, jqXHR) {
   the_html = "<button id='back_to_collections' class='btn btn-inverse btn-small'>";
   the_html += "<i class='icon-white icon-arrow-left'></i> Back </button>";
   the_html += "    <button id='submit' class='btn btn-primary btn-small'> Submit </button>";
-  the_html += "<table class='table table-hover'>";
-  the_html += "<thead><tr><th></th><th>Record</th><th>Data Set</th><th>Description</th><th>Mean</th></tr></thead>";
+  the_html += "<table id='collection_table' style='padding-top: 10px;' class='table table-hover'>";
+  the_html += "<thead><tr><th></th><th>Record</th><th>Data Set</th><th>Description</th></tr></thead>";
   the_html += "<tbody>";
   for (_i = 0, _len = trait_data.length; _i < _len; _i++) {
     trait = trait_data[_i];
     the_html += "<tr class='trait_line'>";
     the_html += "<td class='select_trait'><input type='checkbox' name='selectCheck' class='checkbox edit_sample_checkbox'></td>";
     the_html += "<td class='trait'>" + trait.name + "</td>";
-    the_html += "<td class='dataset'>" + trait.dataset + "</td>";
-    the_html += "<td>" + trait.description + "</td>";
-    the_html += "<td>" + (trait.mean || '&nbsp;') + "</td></tr>";
+    the_html += "<td class='dataset' data-dataset='" + trait.dataset + "'>" + trait.dataset_name + "</td>";
+    the_html += "<td class='description'>" + trait.description + "</td>";
   }
   the_html += "</tbody>";
   the_html += "</table>";
diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js
index 210945f0..bcb67527 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait.js
@@ -2,8 +2,6 @@ var Stat_Table_Rows, is_number,
   __hasProp = {}.hasOwnProperty,
   __slice = [].slice;
 
-console.log("start_b");
-
 is_number = function(o) {
   return !isNaN((o - 0) && o !== null);
 };
@@ -65,7 +63,6 @@ var add, block_by_attribute_value, block_by_index, block_outliers, change_stats_
 add = function() {
   var trait;
   trait = $("input[name=trait_hmac]").val();
-  console.log("trait is:", trait);
   return $.colorbox({
     href: "/collections/add?traits=" + trait
   });
@@ -100,9 +97,7 @@ open_trait_selection = function() {
         inline: true,
         href: "#collections_holder",
         onComplete: function(){
-            console.log("before get script")
             $.getScript("/static/new/javascript/get_traits_from_collection.js");
-            console.log("after get script")
         }
       });
       return $('a.collection_name').attr('onClick', 'return false');
@@ -115,10 +110,10 @@ open_covariate_selection = function() {
       $.colorbox({
         inline: true,
         href: "#collections_holder",
+        width: "1000px",
+        height: "700px",
         onComplete: function(){
-            console.log("before get cov script")
             $.getScript("/static/new/javascript/get_covariates_from_collection.js");
-            console.log("after get cov script")
         }
       });
       return $('a.collection_name').attr('onClick', 'return false');
@@ -142,21 +137,16 @@ stats_mdp_change = function() {
 change_stats_value = function(sample_sets, category, value_type, decimal_places, effects) {
   var current_value, id, in_box, the_value, title_value;
   id = "#" + process_id(category, value_type);
-  console.log("the_id:", id);
   in_box = $(id).html;
   current_value = parseFloat($(in_box)).toFixed(decimal_places);
   the_value = sample_sets[category][value_type]();
-  console.log("After running sample_sets, the_value is:", the_value);
   if (decimal_places > 0) {
     title_value = the_value.toFixed(decimal_places * 2);
     the_value = the_value.toFixed(decimal_places);
   } else {
     title_value = null;
   }
-  console.log("*-* the_value:", the_value);
-  console.log("*-* current_value:", current_value);
   if (the_value !== current_value) {
-    console.log("object:", $(id).html(the_value));
     if (effects) {
       $(id).html(the_value).effect("highlight");
     } else {
@@ -179,7 +169,6 @@ update_stat_values = function(sample_sets) {
       _results1 = [];
       for (_j = 0, _len1 = Stat_Table_Rows.length; _j < _len1; _j++) {
         row = Stat_Table_Rows[_j];
-        console.log("Calling change_stats_value");
         _results1.push(change_stats_value(sample_sets, category, row.vn, row.digits, show_effects));
       }
       return _results1;
@@ -356,7 +345,6 @@ process_id = function() {
   processed = "";
   for (_i = 0, _len = values.length; _i < _len; _i++) {
     value = values[_i];
-    console.log("value:", value);
     value = value.replace(" ", "_");
     if (processed.length) {
       processed += "-";
@@ -378,7 +366,6 @@ edit_data_change = function() {
     samples_other: {},
     samples_all: {}
   };
-  console.log("at beginning:", sample_sets);
   tables = ['samples_primary', 'samples_other'];
   for (_i = 0, _len = tables.length; _i < _len; _i++) {
     table = tables[_i];
@@ -412,39 +399,30 @@ edit_data_change = function() {
       }
     }
   }
-  console.log("towards end:", sample_sets);
   update_stat_values(sample_sets);
 
   if ($('#histogram').hasClass('js-plotly-plot')){
-    console.log("redrawing histogram");
     redraw_histogram();
   }
   if ($('#bar_chart').hasClass('js-plotly-plot')){
-    console.log("redrawing bar chart");
     redraw_bar_chart();
   }
   if ($('#box_plot').hasClass('js-plotly-plot')){
-    console.log("redrawing box plot");
     redraw_box_plot();
   }
   if ($('#violin_plot').hasClass('js-plotly-plot')){
-    console.log("redrawing violin plot");
     redraw_violin_plot();
   }
   if ($('#prob_plot_div').hasClass('js-plotly-plot')){
-    console.log("redrawing probability plot");
     return redraw_prob_plot();
   }
 };
 show_hide_outliers = function() {
   var label;
-  console.log("FOOBAR in beginning of show_hide_outliers");
   label = $('#show_hide_outliers').val();
-  console.log("lable is:", label);
   if (label === "Hide Outliers") {
     return $('#show_hide_outliers').val("Show Outliers");
   } else if (label === "Show Outliers") {
-    console.log("Found Show Outliers");
     $('#show_hide_outliers').val("Hide Outliers");
     return console.log("Should be now Hide Outliers");
   }
@@ -452,7 +430,6 @@ show_hide_outliers = function() {
 on_corr_method_change = function() {
   var corr_method;
   corr_method = $('select[name=corr_type]').val();
-  console.log("corr_method is:", corr_method);
   $('.correlation_desc').hide();
   $('#' + corr_method + "_r_desc").show().effect("highlight");
   if (corr_method === "lit") {
@@ -487,7 +464,6 @@ create_value_dropdown = function(value) {
 };
 populate_sample_attributes_values_dropdown = function() {
   var attribute_info, key, sample_attributes, selected_attribute, value, _i, _len, _ref, _ref1, _results;
-  console.log("in beginning of psavd");
   $('#attribute_values').empty();
   sample_attributes = {};
   _ref = js_data.attribute_names;
@@ -496,9 +472,7 @@ populate_sample_attributes_values_dropdown = function() {
     attribute_info = _ref[key];
     sample_attributes[attribute_info.name] = attribute_info.distinct_values;
   }
-  console.log("[visa] attributes is:", sample_attributes);
   selected_attribute = $('#exclude_menu').val().replace("_", " ");
-  console.log("selected_attribute is:", selected_attribute);
   _ref1 = sample_attributes[selected_attribute];
   _results = [];
   for (_i = 0, _len = _ref1.length; _i < _len; _i++) {
@@ -507,7 +481,7 @@ populate_sample_attributes_values_dropdown = function() {
   }
   return _results;
 };
-if (js_data.attribute_names.length > 0) {
+if (Object.keys(js_data.attribute_names).length > 0) {
   populate_sample_attributes_values_dropdown();
 }
 $('#exclude_menu').change(populate_sample_attributes_values_dropdown);
@@ -547,21 +521,15 @@ block_by_index = function() {
       }
     } else {
       index = parseInt(index_set);
-      console.log("index:", index);
       index_list.push(index);
     }
   }
-  console.log("index_list:", index_list);
   _results = [];
   for (_k = 0, _len1 = index_list.length; _k < _len1; _k++) {
     index = index_list[_k];
     if ($('#block_group').val() === "primary") {
-      console.log("block_group:", $('#block_group').val());
-      console.log("row:", $('#Primary_' + index.toString()));
       _results.push($('#Primary_' + index.toString()).find('.trait_value_input').val("x"));
     } else if ($('#block_group').val() === "other") {
-      console.log("block_group:", $('#block_group').val());
-      console.log("row:", $('#Other_' + index.toString()));
       _results.push($('#Other_' + index.toString()).find('.trait_value_input').val("x"));
     } else {
       _results.push(void 0);
@@ -592,9 +560,7 @@ reset_samples_table = function() {
   $('input[name="transform"]').val("");
   return $('.trait_value_input').each((function(_this) {
     return function(_index, element) {
-      console.log("value is:", $(element).val());
       $(element).val($(element).data('value'));
-      console.log("data-value is:", $(element).data('value'));
       return $(element).parents('.value_se').show();
     };
   })(this));
@@ -629,6 +595,21 @@ sqrt_normalize_data = function() {
   })(this));
 };
 
+invert_data = function() {
+  return $('.edit_sample_value').each((function(_this) {
+    return function(_index, element) {
+      current_value = parseFloat($(element).val());
+      if(isNaN(current_value)) {
+        return current_value
+      } else {
+        $(element).val(-(current_value));
+        return -(current_value)
+      }
+    };
+  })(this));
+};
+
+
 qnorm_data = function() {
   return $('.edit_sample_value').each((function(_this) {
     return function(_index, element) {
@@ -656,6 +637,9 @@ normalize_data = function() {
       $('input[name="transform"]').val("sqrt")
     }
   }
+  else if ($('#norm_method option:selected').val() == 'invert'){
+    invert_data()
+  }
   else if ($('#norm_method option:selected').val() == 'qnorm'){
     if ($('input[name="transform"]').val() != "qnorm") {
       qnorm_data()
@@ -696,7 +680,6 @@ get_sample_table_data = function(table_name) {
         attribute_info = _ref[key];
         row_data[attribute_info.name] = $.trim($(element).find('.column_name-' + attribute_info.name.replace(" ", "_")).text());
       }
-      console.log("row_data is:", row_data);
       return samples.push(row_data);
     };
   })(this));
@@ -707,18 +690,14 @@ export_sample_table_data = function() {
   sample_data = {};
   sample_data.primary_samples = get_sample_table_data('samples_primary');
   sample_data.other_samples = get_sample_table_data('samples_other');
-  console.log("sample_data is:", sample_data);
   json_sample_data = JSON.stringify(sample_data);
-  console.log("json_sample_data is:", json_sample_data);
   $('input[name=export_data]').val(json_sample_data);
-  console.log("export_data is", $('input[name=export_data]').val());
   format = $('input[name=export_format]').val();
   if (format === "excel") {
     $('#trait_data_form').attr('action', '/export_trait_excel');
   } else {
     $('#trait_data_form').attr('action', '/export_trait_csv');
   }
-  console.log("action is:", $('#trait_data_form').attr('action'));
   return $('#trait_data_form').submit();
 };
 
@@ -728,9 +707,7 @@ $('.export_format').change(function() {
 });
 
 $('.export').click(export_sample_table_data);
-console.log("before registering block_outliers");
 $('#block_outliers').click(block_outliers);
-console.log("after registering block_outliers");
 _.mixin(_.str.exports());
 
 get_sample_vals = function(sample_list) {
diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html
index 36a25665..31a5b94e 100644
--- a/wqflask/wqflask/templates/search_result_page.html
+++ b/wqflask/wqflask/templates/search_result_page.html
@@ -117,7 +117,7 @@
           </form>
           <br />
 
-          <div id="table_container">
+          <div id="table_container" style="width: 1400px;">
             <table class="table-hover table-striped cell-border" id='trait_table' style="float: left;">
                 <tbody>
                  <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td>
@@ -224,10 +224,6 @@
                     }
                     $('td', row).eq(4).attr('title', $('td', row).eq(4).text());
                     $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text());
-                    if ($('td', row).eq(4).text().length > 55) {
-                        $('td', row).eq(4).text($('td', row).eq(4).text().substring(0, 55));
-                        $('td', row).eq(4).text($('td', row).eq(4).text() + '...')
-                    }
                     $('td', row).slice(6,10).attr("align", "right");
                     $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text());
                     $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text());
@@ -237,16 +233,8 @@
                     {% elif dataset.type == 'Publish' %}
                     $('td', row).eq(3).attr('title', $('td', row).eq(3).text());
                     $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text());
-                    if ($('td', row).eq(3).text().length > 20) {
-                        $('td', row).eq(3).text($('td', row).eq(3).text().substring(0, 20));
-                        $('td', row).eq(3).text($('td', row).eq(3).text() + '...')
-                    }
                     $('td', row).eq(4).attr('title', $('td', row).eq(4).text());
                     $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text());
-                    if ($('td', row).eq(4).text().length > 55) {
-                        $('td', row).eq(4).text($('td', row).eq(4).text().substring(0, 55));
-                        $('td', row).eq(4).text($('td', row).eq(4).text() + '...')
-                    }
                     $('td', row).slice(6,9).attr("align", "right");
                     $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text());
                     $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text());
@@ -288,6 +276,7 @@
                     {
                       'title': "Description",
                       'type': "natural",
+                      'width': "500px",
                       'data': "description"
                     },
                     {
@@ -321,11 +310,13 @@
                     {
                       'title': "Description",
                       'type': "natural",
+                      'width': "500px",
                       'data': "description"
                     },
                     {
                       'title': "Authors",
                       'type': "natural",
+                      'width': "300px",
                       'data': "authors"
                     },
                     {
@@ -351,17 +342,20 @@
                     {
                       'title': "Max LRS Location",
                       'type': "natural",
+                      'width': "160px",
                       'data': "lrs_location"
                     },
                     {
-                      'title': "Additive Effect<a href=\"http://genenetwork.org//glossary.html#A\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>",
+                      'title': "Additive<br>Effect<a href=\"http://genenetwork.org//glossary.html#A\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>",
                       'type': "natural",
+                      'width': "100px",
                       'data': "additive",
                       'orderSequence': [ "desc", "asc"]
                     }{% elif dataset.type == 'Geno' %},
                     {
                       'title': "Location",
                       'type': "natural",
+                      'width': "160px",
                       'data': "location"
                     }{% endif %}
                 ],
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index def40f5d..9d2cb7ed 100644
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -10,7 +10,6 @@
     <link rel="stylesheet" type="text/css" href="/static/new/packages/DataTables/css/jquery.dataTables.css" />
     <link rel="stylesheet" type="text/css" href="/static/new/packages/noUiSlider/nouislider.css" />
     <link rel="stylesheet" type="text/css" href="/static/new/packages/noUiSlider/nouislider.pips.css" />
-    <link rel="stylesheet" type="text/css" href="/static/packages/DT_bootstrap/DT_bootstrap.css" />
     <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" />
 
 {% endblock %}
@@ -142,7 +141,6 @@
     <script type="text/javascript" src="/static/new/javascript/scatter-matrix.js"></script>
     <script type="text/javascript" src="/static/new/javascript/plotly_probability_plot.js"></script>
     <script type="text/javascript" src="/static/new/javascript/compare_traits_scatterplot.js"></script>
-    <script language="javascript" type="text/javascript" src="/static/new/javascript/get_covariates_from_collection.js"></script>
 
     <script type="text/javascript" src="/static/new/javascript/show_trait_mapping_tools.js"></script>
     <script type="text/javascript" src="/static/new/javascript/show_trait.js"></script>
@@ -152,6 +150,8 @@
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/dataTables.scientific.js"></script>
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/dataTables.naturalSort.js"></script>
     <script language="javascript" type="text/javascript" src="/static/new/packages/noUiSlider/nouislider.js"></script>
+    <script language="javascript" type="text/javascript" src="/static/new/javascript/get_covariates_from_collection.js"></script>
+
     <script type="text/javascript" charset="utf-8">
 
             $.fn.dataTable.ext.order['dom-checkbox'] = function  ( settings, col )
@@ -177,6 +177,22 @@
                    }
                 });
 
+                //ZS: This variable is just created to get the column position of the first case attribute, since it's needed to set the row classes in createdRow for the DataTable
+                {% if sample_groups[0].attributes|length > 0 %}
+                {% if sample_groups[0].se_exists() %}
+                {% if has_num_cases %}
+                attribute_start_pos = 6
+                {% else %}
+                attribute_start_pos = 5
+                {% endif %}
+                {% else %}
+                {% if has_num_cases %}
+                attribute_start_pos = 4
+                {% else %}
+                attribute_start_pos = 3
+                {% endif %}
+                {% endif %}
+                {% endif %}
 
                 $('#samples_primary').DataTable( {
                   'initComplete': function(settings, json) {
@@ -210,6 +226,11 @@
                     $('td', row).eq(4).addClass("column_name-num_cases")
                     $('td', row).eq(4).attr("style", "text-align: right; padding-top: 2px; padding-bottom: 0px;");
                     {% endif %} {% endif %}
+                    {% if sample_groups[0].attributes|length > 0 %}
+                    {% for attribute in sample_groups[0].attributes|sort() %}
+                    $('td', row).eq(attribute_start_pos + {{ loop.index }}).addClass("column_name-{{ sample_groups[0].attributes[attribute].name }}")
+                    {% endfor %}
+                    {% endif %}
                   },
                   'data': js_data['sample_lists'][0],
                   'columns': [
@@ -335,6 +356,11 @@
                     $('td', row).eq(4).addClass("column_name-num_cases")
                     $('td', row).eq(4).attr("style", "text-align: right; padding-top: 2px; padding-bottom: 0px;");
                     {% endif %} {% endif %}
+                    {% if sample_groups[1].attributes|length > 0 %}
+                    {% for attribute in sample_groups[1].attributes|sort() %}
+                    $('td', row).eq(attribute_start_pos + {{ loop.index }}).addClass("column_name-{{ sample_groups[1].attributes[attribute].name }}")
+                    {% endfor %}
+                    {% endif %}
                   },
                   'data': js_data['sample_lists'][1],
                   'columns': [
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index e7f3eca3..e79cf1ab 100644
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -89,7 +89,7 @@
                                 <button type="button" id="select_covariates" class="btn btn-default">Select</button>
                                 <button type="button" id="remove_covariates" class="btn btn-default">Remove</button>
                               </div>
-                              <textarea rows="3" cols="20" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none;" class="selected_covariates"></textarea>
+                              <textarea rows="3" cols="50" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none;" class="selected_covariates"></textarea>
                               {% endif %}
                               {% elif g.cookie_session.display_num_collections() == "" %}
                               No collections available. Please add traits to a collection to use them as covariates.
diff --git a/wqflask/wqflask/templates/show_trait_transform_and_filter.html b/wqflask/wqflask/templates/show_trait_transform_and_filter.html
index d1765778..04ceaeaf 100644
--- a/wqflask/wqflask/templates/show_trait_transform_and_filter.html
+++ b/wqflask/wqflask/templates/show_trait_transform_and_filter.html
@@ -54,6 +54,7 @@
                 <option value="log2">Log2</option>
                 <option value="qnorm">Quantile</option>
                 <option value="sqrt">Square Root</option>
+                <option value="invert">Invert +/-</option>
               </select>
               </div>
               <!--
-- 
cgit v1.2.3


From d515061c4878b448f5b866e32eee7e37301ecdb7 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Mon, 18 Mar 2019 17:03:30 -0500
Subject: - Added PCA traits to correlation matrix. You can't change their
 names yet or add them to a collection from the corr matrix page, but you can
 click them and access them from the trait page. I'll add the option to access
 them from the corr matrix page as well, but adding the option to change their
 names might be trickier since they're currently used as their Redis keys. I
 need some better way of passing the Redis key around so it can be stored in
 collections, but this is tricky without changing the structure in
 ElasticSearch

- Fixed the way temp traits work so you can use them with various functions, like mapping, correlations, network graph, etc

- Fixed some appearance issue where the network graph options were too wide if a trait name was too long
---
 wqflask/base/data_set.py                           |  3 +-
 wqflask/base/trait.py                              | 40 +++++++++++++++-------
 wqflask/utility/helper_functions.py                |  5 ++-
 wqflask/wqflask/collect.py                         |  5 ++-
 wqflask/wqflask/correlation/show_corr_results.py   |  2 +-
 .../wqflask/correlation_matrix/show_corr_matrix.py | 37 ++++++++++++++++----
 .../marker_regression/display_mapping_results.py   | 11 +++---
 wqflask/wqflask/templates/correlation_matrix.html  |  7 ++++
 wqflask/wqflask/templates/network_graph.html       | 15 ++++++--
 9 files changed, 95 insertions(+), 30 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index ca6621e9..beb2a8a2 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -277,7 +277,6 @@ class DatasetGroup(object):
     """
     def __init__(self, dataset, name=None):
         """This sets self.group and self.group_id"""
-        #logger.debug("DATASET NAME2:", dataset.name)
         if name == None:
             self.name, self.id, self.genetic_type = fetchone(dataset.query_for_group)
         else:
@@ -500,7 +499,7 @@ class DataSet(object):
         self.setup()
 
         if self.type == "Temp": #Need to supply group name as input if temp trait
-            self.group = DatasetGroup(self, group_name)   # sets self.group and self.group_id and gets genotype
+            self.group = DatasetGroup(self, name=group_name)   # sets self.group and self.group_id and gets genotype
         else:
             self.check_confidentiality()
             self.retrieve_other_names()
diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index 79aa196f..0689e950 100644
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -6,6 +6,9 @@ import codecs
 
 from htmlgen import HTMLgen2 as HT
 
+import redis
+Redis = redis.StrictRedis()
+
 from base import webqtlConfig
 from base.webqtlCaseData import webqtlCaseData
 from base.data_set import create_dataset
@@ -35,13 +38,15 @@ class GeneralTrait(object):
     def __init__(self, get_qtl_info=False, get_sample_info=True, **kw):
         # xor assertion
         assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name";
+        self.name = kw.get('name')                 # Trait ID, ProbeSet ID, Published ID, etc.
         if kw.get('dataset_name'):
-            self.dataset = create_dataset(kw.get('dataset_name'))
-            #print(" in GeneralTrait created dataset:", self.dataset)
+            if kw.get('dataset_name') == "Temp":
+                temp_group = self.name.split("_")[2]
+                self.dataset = create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = temp_group)
+            else:
+                self.dataset = create_dataset(kw.get('dataset_name'))
         else:
             self.dataset = kw.get('dataset')
-        self.name = kw.get('name')                 # Trait ID, ProbeSet ID, Published ID, etc.
-        #print("THE NAME IS:", self.name)
         self.cellid = kw.get('cellid')
         self.identification = kw.get('identification', 'un-named trait')
         self.haveinfo = kw.get('haveinfo', False)
@@ -73,8 +78,8 @@ class GeneralTrait(object):
         # So we could add a simple if statement to short-circuit this if necessary
         if self.dataset.type != "Temp":
             self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info)
-            if get_sample_info != False:
-                self = retrieve_sample_data(self, self.dataset)
+        if get_sample_info != False:
+            self = retrieve_sample_data(self, self.dataset)
 
     def export_informative(self, include_variance=0):
         """
@@ -154,18 +159,27 @@ def retrieve_sample_data(trait, dataset, samplelist=None):
     if samplelist == None:
         samplelist = []
 
-    results = dataset.retrieve_sample_data(trait.name)
+    if dataset.type == "Temp":
+        results = Redis.get(trait.name).split()
+    else:
+        results = dataset.retrieve_sample_data(trait.name)
 
     # Todo: is this necessary? If not remove
     trait.data.clear()
 
-    all_samples_ordered = dataset.group.all_samples_ordered()
-
     if results:
-        for item in results:
-            name, value, variance, num_cases, name2 = item
-            if not samplelist or (samplelist and name in samplelist):
-                trait.data[name] = webqtlCaseData(*item)   #name, value, variance, num_cases)
+        if dataset.type == "Temp":
+            all_samples_ordered = dataset.group.all_samples_ordered()
+            for i, item in enumerate(results):
+                try:
+                    trait.data[all_samples_ordered[i]] = webqtlCaseData(all_samples_ordered[i], float(item))
+                except:
+                    pass
+        else:
+            for item in results:
+                name, value, variance, num_cases, name2 = item
+                if not samplelist or (samplelist and name in samplelist):
+                    trait.data[name] = webqtlCaseData(*item)   #name, value, variance, num_cases)
     return trait
 
 @app.route("/trait/get_sample_data")
diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py
index 1c8dad10..6980af4e 100644
--- a/wqflask/utility/helper_functions.py
+++ b/wqflask/utility/helper_functions.py
@@ -45,7 +45,10 @@ def get_trait_db_obs(self, trait_db_list):
         data = data.strip()
         assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?"
         trait_name, dataset_name = data.split(":")
-        dataset_ob = data_set.create_dataset(dataset_name)
+        if dataset_name == "Temp":
+            dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2])
+        else:
+            dataset_ob = data_set.create_dataset(dataset_name)
         trait_ob = GeneralTrait(dataset=dataset_ob,
                                name=trait_name,
                                cellid=None)
diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py
index eb0e2726..6e1ac592 100644
--- a/wqflask/wqflask/collect.py
+++ b/wqflask/wqflask/collect.py
@@ -233,7 +233,10 @@ def collections_new():
             collection_id = params['existing_collection'].split(":")[0]
             collection_name = params['existing_collection'].split(":")[1]
         if g.user_session.logged_in:
-            unprocessed_traits = Redis.get(params['hash'])
+            if "hash" in params:
+                unprocessed_traits = Redis.get(params['hash'])
+            else:
+                unprocessed_traits = params['traits']
             traits = list(process_traits(unprocessed_traits))
             g.user_session.add_traits_to_collection(collection_id, traits)
             return redirect(url_for('view_collection', uc_id=collection_id))
diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index c15c3579..0c6b8a2b 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -93,7 +93,7 @@ class CorrelationResults(object):
         with Bench("Doing correlations"):
             if start_vars['dataset'] == "Temp":
                 self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group'])
-                self.trait_id = "Temp"
+                self.trait_id = start_vars['trait_id']
                 self.this_trait = GeneralTrait(dataset=self.dataset,
                                            name=self.trait_id,
                                            cellid=None)
diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
index 4bb4d65d..007e8e47 100644
--- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
+++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
@@ -26,6 +26,7 @@ import sys
 import string
 import cPickle
 import os
+import datetime
 import time
 import pp
 import math
@@ -45,6 +46,9 @@ from pprint import pformat as pf
 from htmlgen import HTMLgen2 as HT
 import reaper
 
+import redis
+Redis = redis.StrictRedis()
+
 from utility.THCell import THCell
 from utility.TDCell import TDCell
 from base.trait import GeneralTrait
@@ -59,8 +63,10 @@ from MySQLdb import escape_string as escape
 
 from pprint import pformat as pf
 
-from flask import Flask, g
+from flask import Flask, g, url_for
 
+import utility.logger
+logger = utility.logger.getLogger(__name__ )
 
 class CorrelationMatrix(object):
 
@@ -111,6 +117,7 @@ class CorrelationMatrix(object):
             self.corr_results = []
             self.pca_corr_results = []
             self.trait_data_array = []
+            self.shared_samples_list = self.all_sample_list
             for trait_db in self.trait_list:
                 this_trait = trait_db[0]
                 this_db = trait_db[1]
@@ -138,6 +145,8 @@ class CorrelationMatrix(object):
                     target_vals = []
                     for index, sample in enumerate(target_samples):
                         if (sample in this_sample_data) and (sample in target_sample_data):
+                            if sample not in self.shared_samples_list:
+                                self.shared_samples_list.remove(sample)
                             sample_value = this_sample_data[sample].value
                             target_sample_value = target_sample_data[sample].value
                             this_trait_vals.append(sample_value)
@@ -174,6 +183,7 @@ class CorrelationMatrix(object):
 
             try:
                 self.pca_works = "True"
+                self.pca_trait_ids = []
                 pca = self.calculate_pca(range(len(self.traits)), corr_eigen_value, corr_eigen_vectors)
                 self.loadings_array = self.process_loadings()
             except:
@@ -201,9 +211,6 @@ class CorrelationMatrix(object):
                                    cellid=None)
             self.trait_list.append((trait_ob, dataset_ob))
 
-        #print("trait_list:", self.trait_list)
-
-
     def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors):
         base = importr('base')
         stats = importr('stats')
@@ -224,9 +231,27 @@ class CorrelationMatrix(object):
         pca_traits = []
         for i, vector in enumerate(trait_array_vectors):
             if corr_eigen_value[i-1] < 100.0/len(self.trait_list):
-                pca_traits.append(vector*-1.0)
+                pca_traits.append((vector*-1.0).tolist())
+
+        this_group_name = self.trait_list[0][1].group.name
+        temp_dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = this_group_name)
+        temp_dataset.group.get_samplelist()
+        for i, pca_trait in enumerate(pca_traits):
+            trait_id = "PCA" + str(i+1) + "_" + temp_dataset.group.species + "_" + this_group_name + "_" + datetime.datetime.now().strftime("%m%d%H%M%S")
+            this_vals_string = ""
+            position = 0
+            for sample in temp_dataset.group.all_samples_ordered():
+                if sample in self.shared_samples_list:
+                    this_vals_string += str(pca_trait[position])
+                    this_vals_string += " "
+                    position += 1
+                else:
+                    this_vals_string += "x "
+            this_vals_string = this_vals_string[:-1]
+
+            Redis.set(trait_id, this_vals_string)
+            self.pca_trait_ids.append(trait_id)
 
-        print("pca_traits:", pca_traits)
         return pca
 
     def process_loadings(self):
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index 41d5c9d0..39e0d712 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -932,10 +932,13 @@ class DisplayMappingResults(object):
 
             if self.this_trait.symbol:
                 identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.symbol)
-            elif self.this_trait.post_publication_abbreviation:
-                identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.post_publication_abbreviation)
-            elif self.this_trait.pre_publication_abbreviation:
-                identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.pre_publication_abbreviation)
+            elif self.dataset.type == "Publish":
+                if self.this_trait.post_publication_abbreviation:
+                    identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.post_publication_abbreviation)
+                elif self.this_trait.pre_publication_abbreviation:
+                    identification += "Trait: %s - %s" % (self.this_trait.name, self.this_trait.pre_publication_abbreviation)
+                else:
+                    identification += "Trait: %s" % (self.this_trait.name)
             else:
                 identification += "Trait: %s" % (self.this_trait.name)
             identification += " with %s samples" % (self.n_samples)
diff --git a/wqflask/wqflask/templates/correlation_matrix.html b/wqflask/wqflask/templates/correlation_matrix.html
index 8698b710..9c790780 100644
--- a/wqflask/wqflask/templates/correlation_matrix.html
+++ b/wqflask/wqflask/templates/correlation_matrix.html
@@ -60,6 +60,13 @@
 {% if pca_works == "True" %}
 <br>
 <br>
+<h2>PCA Traits</h2>
+<ul>
+  {% for this_trait_id in pca_trait_ids %}
+  <li><a href="{{ url_for('show_trait_page', trait_id = pca_trait_ids[loop.index - 1], dataset = "Temp") }}">{{ pca_trait_ids[loop.index - 1] }}</a></li>
+  {% endfor %}
+</ul>
+<br>
 <h2>Factor Loadings Plot</h2>
 <div id="loadings_plot" style="margin-top: 20px; margin-bottom: 20px; width: 980px; border-style: solid; border-width: 1px;"></div>
 <h2>Factor Loadings Table</h2>
diff --git a/wqflask/wqflask/templates/network_graph.html b/wqflask/wqflask/templates/network_graph.html
index 6bee11e7..4492dd3f 100644
--- a/wqflask/wqflask/templates/network_graph.html
+++ b/wqflask/wqflask/templates/network_graph.html
@@ -34,7 +34,11 @@
                         <select name="focus_select">
                           <option disabled selected value>Select Trait</option>
                           {% for trait in traits %}
+                          {% if trait.symbol == None %}
+                          <option value="{{ trait.name }}:{{ trait.dataset.name }}">{{ trait.name }}</option>
+                          {% else %}
                           <option value="{{ trait.name }}:{{ trait.dataset.name }}">{{ trait.symbol }} ({{ trait.name }})</option>
+                          {% endif %}
                           {% endfor %}
                         </select>
                       </td>
@@ -46,11 +50,18 @@
                     </tr>
                     <tr>
                       <td colspan="1">
+                        <div style="text-align: center;">
+                          <div style="float: left;"><font size="2"><b>-1</b></font></div>
+                          <font size="2"><b>0</b></font>
+                          <div style="float: right;"><font size="2"><b>1</b></font></div>
+                        </div>
+                        <!--
                         <font size="2"><b>-1 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
                                             &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
                                             &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; 1</b></font><br>
-                        <input type="range" id="neg_slide" min="-1" max="0" value="0" step="0.001" list="corr_range" style="display: inline; width: 45%">
-                        <input type="range" id="pos_slide" min="0" max="1" value="0" step="0.001" list="corr_range" style="display: inline; width: 45%">
+                        -->
+                        <input type="range" id="neg_slide" min="-1" max="0" value="0" step="0.001" list="corr_range" style="display: inline; width: 49%">
+                        <input type="range" id="pos_slide" min="0" max="1" value="0" step="0.001" list="corr_range" style="display: inline; width: 49%">
                       </td>
                     </tr>
                     <tr>
-- 
cgit v1.2.3


From 3afaa7cfad3ffa3d0bb033dd1b984d3443a65fa0 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Mon, 6 May 2019 12:02:49 -0500
Subject: Removed some unused imports of htmlgen

Fixed a bug that sometimes occurred in search results

Changed order of items on index page
---
 wqflask/base/trait.py                              |  2 --
 wqflask/utility/webqtlUtil.py                      |  2 --
 .../comparison_bar_chart/comparison_bar_chart.py   |  1 -
 wqflask/wqflask/correlation/show_corr_results.py   |  1 -
 .../wqflask/correlation_matrix/show_corr_matrix.py |  1 -
 wqflask/wqflask/heatmap/heatmap.py                 |  1 -
 wqflask/wqflask/network_graph/network_graph.py     |  1 -
 wqflask/wqflask/search_results.py                  |  2 +-
 wqflask/wqflask/snp_browser/snp_browser.py         |  6 ++--
 wqflask/wqflask/templates/index_page_orig.html     | 41 ++++++++++------------
 10 files changed, 22 insertions(+), 36 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index 0689e950..6fecf725 100644
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -4,8 +4,6 @@ import string
 import resource
 import codecs
 
-from htmlgen import HTMLgen2 as HT
-
 import redis
 Redis = redis.StrictRedis()
 
diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py
index 83fa90b7..67a476e3 100644
--- a/wqflask/utility/webqtlUtil.py
+++ b/wqflask/utility/webqtlUtil.py
@@ -30,8 +30,6 @@ import re
 import math
 from math import *
 
-from htmlgen import HTMLgen2 as HT
-
 from base import webqtlConfig
 
 # NL, 07/27/2010. moved from webqtlForm.py
diff --git a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py
index b9e6f450..21eb1493 100644
--- a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py
+++ b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py
@@ -35,7 +35,6 @@ import resource
 
 from pprint import pformat as pf
 
-from htmlgen import HTMLgen2 as HT
 import reaper
 
 from base.trait import GeneralTrait
diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 031c7a16..1aa7945d 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -37,7 +37,6 @@ import numpy
 
 from pprint import pformat as pf
 
-from htmlgen import HTMLgen2 as HT
 import reaper
 
 from base import webqtlConfig
diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
index 5a1f464b..09213d1f 100644
--- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
+++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
@@ -41,7 +41,6 @@ import rpy2.robjects as robjects
 
 from pprint import pformat as pf
 
-from htmlgen import HTMLgen2 as HT
 import reaper
 
 import redis
diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py
index 1bdf252b..9d33961c 100644
--- a/wqflask/wqflask/heatmap/heatmap.py
+++ b/wqflask/wqflask/heatmap/heatmap.py
@@ -18,7 +18,6 @@ import numpy as np
 
 from pprint import pformat as pf
 
-from htmlgen import HTMLgen2 as HT
 import reaper
 
 from base.trait import GeneralTrait
diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py
index 63273a29..a332db46 100644
--- a/wqflask/wqflask/network_graph/network_graph.py
+++ b/wqflask/wqflask/network_graph/network_graph.py
@@ -40,7 +40,6 @@ import rpy2.robjects as robjects
 
 from pprint import pformat as pf
 
-from htmlgen import HTMLgen2 as HT
 import reaper
 
 from utility.THCell import THCell
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index 4a224de6..2c8e07dd 100644
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -116,7 +116,7 @@ views.py).
                 trait_dict['location'] = this_trait.location_repr
                 trait_dict['mean'] = "N/A"
                 trait_dict['additive'] = "N/A"
-                if this_trait.mean != "":
+                if this_trait.mean != "" and this_trait.mean != None:
                     trait_dict['mean'] = '%.3f' % this_trait.mean
                 trait_dict['lrs_score'] = this_trait.LRS_score_repr
                 trait_dict['lrs_location'] = this_trait.LRS_location_repr
diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py
index 1e28add8..360e6b28 100644
--- a/wqflask/wqflask/snp_browser/snp_browser.py
+++ b/wqflask/wqflask/snp_browser/snp_browser.py
@@ -2,8 +2,6 @@ from __future__ import absolute_import, print_function, division
 
 from flask import Flask, g, url_for
 
-from htmlgen import HTMLgen2 as HT
-
 import string
 import piddle as pid
 
@@ -639,8 +637,8 @@ class SnpBrowser(object):
 
         x_scale = plot_width/(self.end_mb - self.start_mb)
 
-        #draw clickable image map
-        #gifmap = HT.Map
+        #draw clickable image map at some point
+
         n_click = 80.0
         click_step = plot_width/n_click
         click_mb_step = (self.end_mb - self.start_mb)/n_click
diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html
index e32b1c82..4bf1e85f 100755
--- a/wqflask/wqflask/templates/index_page_orig.html
+++ b/wqflask/wqflask/templates/index_page_orig.html
@@ -176,24 +176,33 @@
             </div>
 
             <div style="padding-left:120px" class="col-xs-4" style="width: 600px !important;">
+	        <section id="affiliates">
+	            <div class="page-header">
+                        <h1>Affiliates</h1>
+                        <ul>
+                          <li><a href="http://ucscbrowser.genenetwork.org/">Genome browser</a> at UTHSC</li>
+                          <li><a href="http://galaxy.genenetwork.org/">Galaxy</a> at UTHSC</li>
+                        </ul>
+                    </div>
+                </section>
                 <section id="websites">
                     <div class="page-header">
                         <h1>Github</h1>
+                        <ul>
+                          <li><a href="https://github.com/genenetwork/genenetwork2">GN2 Source Code</a></li>
+                          <li><a href="https://github.com/genenetwork/gn-docs/wiki">GN2 Document Wiki</a></li>
+                          <li><a href="https://github.com/genenetwork/sysmaintenance">System Maintenance Code</a></li>
+                          <li><a href="https://github.com/genenetwork/genenetwork">GN1 Source Code</a></li>
+                        </ul>
                     </div>
-                    <ul>
-                      <li><a href="https://github.com/genenetwork/genenetwork2">GN2 Source Code</a></li>
-                      <li><a href="https://github.com/genenetwork/gn-docs/wiki">GN2 Document Wiki</a></li>
-                      <li><a href="https://github.com/genenetwork/sysmaintenance">System Maintenance Code</a></li>
-                      <li><a href="https://github.com/genenetwork/genenetwork">GN1 Source Code</a></li>
-                    </ul>
                </section>
                 <section id="news-section">
                     <div class="page-header">
                         <h1>News</h1>
-                    </div>
-                    <div id="tweets"></div>
-                    <div align="right">
-                      <a href="https://twitter.com/GeneNetwork2">more news items...</a>
+                        <div id="tweets"></div>
+                        <div align="right">
+                          <a href="https://twitter.com/GeneNetwork2">more news items...</a>
+                        </div>
                     </div>
               <!--
                 <section id="tour-info">
@@ -240,18 +249,6 @@
                         <li><a href="http://ec2.genenetwork.org/">Amazon
                             Cloud (EC2)</a></li>
                         </ul>
-                      <h3>Affiliates</h3>
-                      <ul>
-                        <li><a href="http://ucscbrowser.genenetwork.org/">Genome
-                        browser</a> at UTHSC</li>
-
-                        <li><a href="http://galaxy.genenetwork.org/">Galaxy</a> at
-                        UTHSC</li>
-
-                    </ul>
-
-
-
                </section>
 
                 <!--<section id="getting-started">
-- 
cgit v1.2.3


From e304c2cfdf1b38da483efa607a2acd72b66ad0de Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 24 May 2019 11:35:12 -0500
Subject: Replaced qtlreaper Dataset object with python object

---
 wqflask/base/data_set.py       |   7 ++-
 wqflask/utility/gen_geno_ob.py | 135 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 140 insertions(+), 2 deletions(-)
 create mode 100644 wqflask/utility/gen_geno_ob.py

(limited to 'wqflask/utility')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index beb2a8a2..b324ac74 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -43,6 +43,7 @@ from db import webqtlDatabaseFunction
 from utility import webqtlUtil
 from utility.benchmark import Bench
 from utility import chunks
+from utility import gen_geno_ob
 from utility.tools import locate, locate_ignore_error, flat_files
 
 from maintenance import get_group_samplelists
@@ -388,14 +389,16 @@ class DatasetGroup(object):
         #genotype_1 is Dataset Object without parents and f1
         #genotype_2 is Dataset Object with parents and f1 (not for intercross)
 
-        genotype_1 = reaper.Dataset()
+        #genotype_1 = reaper.Dataset()
 
         # reaper barfs on unicode filenames, so here we ensure it's a string
         if self.genofile:
             full_filename = str(locate(self.genofile, 'genotype'))
         else:
             full_filename = str(locate(self.name + '.geno', 'genotype'))
-        genotype_1.read(full_filename)
+        #genotype_1.read(full_filename)
+
+        genotype_1 = gen_geno_ob.genotype(full_filename)
 
         if genotype_1.type == "group" and self.parlist:
             genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1])       #, F1=_f1)
diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
new file mode 100644
index 00000000..5824b0b3
--- /dev/null
+++ b/wqflask/utility/gen_geno_ob.py
@@ -0,0 +1,135 @@
+from __future__ import absolute_import, division, print_function
+
+class genotype(object):
+    """
+    Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure
+    """
+
+    def __init__(self, filename):
+        self.group = None
+        self.type = "riset"
+        self.prgy = []
+        self.nprgy = 0
+        self.mat = -1
+        self.pat = 1
+        self.het = 0
+        self.unk = "U"
+        self.filler = False
+        self.mb_exists = False
+
+        #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary
+        self.cm_column = 2
+        self.mb_column = 3
+
+        self.chromosomes = []
+
+        self.read_file(filename)
+
+    def __iter__(self):
+        return iter(self.chromosomes)
+
+    def __getitem__(self, index):
+        return self.chromosomes[index]
+
+    def __len__(self):
+        return len(self.chromosomes)
+
+    def read_file(self, filename):
+
+        with open(filename, 'r') as geno_file:
+            lines = geno_file.readlines()
+
+            this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers
+            chr_ob = None
+            for line in lines:
+                if line[0] == "#":
+                    continue
+                elif line[0] == "@":
+                    label = line.split(":")[0][1:]
+                    if label == "name":
+                        self.group = line.split(":")[1]
+                    elif label == "filler":
+                        if line.split(":")[1] == "yes":
+                            self.filler = True
+                    elif label == "type":
+                        self.type = line.split(":")[1]
+                    elif label == "mat":
+                        self.mat = line.split(":")[1]
+                    elif label == "pat":
+                        self.pat = line.split(":")[1]
+                    elif label == "het":
+                        self.het = line.split(":")[1]
+                    elif label == "unk":
+                        self.unk = line.split(":")[1]
+                    else:
+                        continue
+                elif line[:3] == "Chr":
+                    header_row = line.split("\t")
+                    if header_row[2] == "Mb":
+                        self.mb_exists = True
+                        self.mb_column = 2
+                        self.cm_column = 3
+                    elif header_row[3] == "Mb":
+                        self.mb_exists = True
+                        self.mb_column = 3
+                    elif header_row[2] == "cM":
+                        self.cm_column = 2
+
+                    if self.mb_exists:
+                        self.prgy = header_row[4:]
+                    else:
+                        self.prgy = header_row[3:]
+                    self.nprgy = len(self.prgy)
+                else:
+                    if line.split("\t")[0] != this_chr:
+                        if this_chr != "":
+                            self.chromosomes.append(chr_ob)
+                        this_chr = line.split("\t")[0]
+                        chr_ob = Chr(line.split("\t")[0], self)
+                    chr_ob.add_marker(line.split("\t"))
+                    
+class Chr(object):
+    def __init__(self, name, geno_ob):
+        self.name = name
+        self.loci = []
+        self.mb_exists = geno_ob.mb_exists
+        self.cm_column = geno_ob.cm_column
+        self.mb_column = geno_ob.mb_column
+        self.geno_ob = geno_ob
+
+    def __iter__(self):
+        return iter(self.loci)
+
+    def __getitem__(self, index):
+        return self.loci[index]
+
+    def __len__(self):
+        return len(self.loci)
+    
+    def add_marker(self, marker_row):
+        self.loci.append(Locus(marker_row, self.geno_ob))
+
+class Locus(object):
+    def __init__(self, marker_row, geno_ob):
+        self.chr = marker_row[0]
+        self.name = marker_row[1]
+        self.cM = float(marker_row[geno_ob.cm_column])
+        self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
+
+        geno_table = {
+            geno_ob.mat: -1,
+            geno_ob.pat: 1,
+            geno_ob.het: 0,
+            geno_ob.unk: "U"
+        }
+
+        self.genotype = []
+        if geno_ob.mb_exists:
+            start_pos = 4
+        else:
+            start_pos = 3
+        for allele in marker_row[start_pos:]:
+            if allele in geno_table.keys():
+                self.genotype.append(geno_table[allele])
+            else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown
+                self.genotype.append("U")
\ No newline at end of file
-- 
cgit v1.2.3


From 66c6bbfcbbd5fb23145ec09956f4809e5f701bec Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 5 Jun 2019 13:01:36 -0500
Subject: Fixed issue that caused interval mapping to not work because the
 python implementation of the reaper Dataset object doesn't include the
 addinterval method (so for those situations I still use reaper)

Fixed issue where the last chromosome wasn't displayed for mapping results (though still need to fix issue where points are drawn too far to the right when a specific range is viewed)
---
 wqflask/base/data_set.py                           |   9 +-
 wqflask/utility/gen_geno_ob.py                     | 270 +++++++++++----------
 .../marker_regression/display_mapping_results.py   |   6 +-
 .../wqflask/marker_regression/qtlreaper_mapping.py |   2 +-
 4 files changed, 147 insertions(+), 140 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index b324ac74..1fd1792e 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -384,7 +384,7 @@ class DatasetGroup(object):
         [result.extend(l) for l in lists if l]
         return result
 
-    def read_genotype_file(self):
+    def read_genotype_file(self, use_reaper=False):
         '''Read genotype from .geno file instead of database'''
         #genotype_1 is Dataset Object without parents and f1
         #genotype_2 is Dataset Object with parents and f1 (not for intercross)
@@ -396,9 +396,12 @@ class DatasetGroup(object):
             full_filename = str(locate(self.genofile, 'genotype'))
         else:
             full_filename = str(locate(self.name + '.geno', 'genotype'))
-        #genotype_1.read(full_filename)
 
-        genotype_1 = gen_geno_ob.genotype(full_filename)
+        if use_reaper:
+            genotype_1 = reaper.Dataset()
+            genotype_1.read(full_filename)
+        else:
+            genotype_1 = gen_geno_ob.genotype(full_filename)
 
         if genotype_1.type == "group" and self.parlist:
             genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1])       #, F1=_f1)
diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
index 5824b0b3..5172369f 100644
--- a/wqflask/utility/gen_geno_ob.py
+++ b/wqflask/utility/gen_geno_ob.py
@@ -1,135 +1,137 @@
-from __future__ import absolute_import, division, print_function
-
-class genotype(object):
-    """
-    Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure
-    """
-
-    def __init__(self, filename):
-        self.group = None
-        self.type = "riset"
-        self.prgy = []
-        self.nprgy = 0
-        self.mat = -1
-        self.pat = 1
-        self.het = 0
-        self.unk = "U"
-        self.filler = False
-        self.mb_exists = False
-
-        #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary
-        self.cm_column = 2
-        self.mb_column = 3
-
-        self.chromosomes = []
-
-        self.read_file(filename)
-
-    def __iter__(self):
-        return iter(self.chromosomes)
-
-    def __getitem__(self, index):
-        return self.chromosomes[index]
-
-    def __len__(self):
-        return len(self.chromosomes)
-
-    def read_file(self, filename):
-
-        with open(filename, 'r') as geno_file:
-            lines = geno_file.readlines()
-
-            this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers
-            chr_ob = None
-            for line in lines:
-                if line[0] == "#":
-                    continue
-                elif line[0] == "@":
-                    label = line.split(":")[0][1:]
-                    if label == "name":
-                        self.group = line.split(":")[1]
-                    elif label == "filler":
-                        if line.split(":")[1] == "yes":
-                            self.filler = True
-                    elif label == "type":
-                        self.type = line.split(":")[1]
-                    elif label == "mat":
-                        self.mat = line.split(":")[1]
-                    elif label == "pat":
-                        self.pat = line.split(":")[1]
-                    elif label == "het":
-                        self.het = line.split(":")[1]
-                    elif label == "unk":
-                        self.unk = line.split(":")[1]
-                    else:
-                        continue
-                elif line[:3] == "Chr":
-                    header_row = line.split("\t")
-                    if header_row[2] == "Mb":
-                        self.mb_exists = True
-                        self.mb_column = 2
-                        self.cm_column = 3
-                    elif header_row[3] == "Mb":
-                        self.mb_exists = True
-                        self.mb_column = 3
-                    elif header_row[2] == "cM":
-                        self.cm_column = 2
-
-                    if self.mb_exists:
-                        self.prgy = header_row[4:]
-                    else:
-                        self.prgy = header_row[3:]
-                    self.nprgy = len(self.prgy)
-                else:
-                    if line.split("\t")[0] != this_chr:
-                        if this_chr != "":
-                            self.chromosomes.append(chr_ob)
-                        this_chr = line.split("\t")[0]
-                        chr_ob = Chr(line.split("\t")[0], self)
-                    chr_ob.add_marker(line.split("\t"))
-                    
-class Chr(object):
-    def __init__(self, name, geno_ob):
-        self.name = name
-        self.loci = []
-        self.mb_exists = geno_ob.mb_exists
-        self.cm_column = geno_ob.cm_column
-        self.mb_column = geno_ob.mb_column
-        self.geno_ob = geno_ob
-
-    def __iter__(self):
-        return iter(self.loci)
-
-    def __getitem__(self, index):
-        return self.loci[index]
-
-    def __len__(self):
-        return len(self.loci)
-    
-    def add_marker(self, marker_row):
-        self.loci.append(Locus(marker_row, self.geno_ob))
-
-class Locus(object):
-    def __init__(self, marker_row, geno_ob):
-        self.chr = marker_row[0]
-        self.name = marker_row[1]
-        self.cM = float(marker_row[geno_ob.cm_column])
-        self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
-
-        geno_table = {
-            geno_ob.mat: -1,
-            geno_ob.pat: 1,
-            geno_ob.het: 0,
-            geno_ob.unk: "U"
-        }
-
-        self.genotype = []
-        if geno_ob.mb_exists:
-            start_pos = 4
-        else:
-            start_pos = 3
-        for allele in marker_row[start_pos:]:
-            if allele in geno_table.keys():
-                self.genotype.append(geno_table[allele])
-            else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown
+from __future__ import absolute_import, division, print_function
+
+class genotype(object):
+    """
+    Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure
+    """
+
+    def __init__(self, filename):
+        self.group = None
+        self.type = "riset"
+        self.prgy = []
+        self.nprgy = 0
+        self.mat = -1
+        self.pat = 1
+        self.het = 0
+        self.unk = "U"
+        self.filler = False
+        self.mb_exists = False
+
+        #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary
+        self.cm_column = 2
+        self.mb_column = 3
+
+        self.chromosomes = []
+
+        self.read_file(filename)
+
+    def __iter__(self):
+        return iter(self.chromosomes)
+
+    def __getitem__(self, index):
+        return self.chromosomes[index]
+
+    def __len__(self):
+        return len(self.chromosomes)
+
+    def read_file(self, filename):
+
+        with open(filename, 'r') as geno_file:
+            lines = geno_file.readlines()
+
+            this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers
+            chr_ob = None
+            for line in lines:
+                if line[0] == "#":
+                    continue
+                elif line[0] == "@":
+                    label = line.split(":")[0][1:]
+                    if label == "name":
+                        self.group = line.split(":")[1]
+                    elif label == "filler":
+                        if line.split(":")[1] == "yes":
+                            self.filler = True
+                    elif label == "type":
+                        self.type = line.split(":")[1]
+                    elif label == "mat":
+                        self.mat = line.split(":")[1]
+                    elif label == "pat":
+                        self.pat = line.split(":")[1]
+                    elif label == "het":
+                        self.het = line.split(":")[1]
+                    elif label == "unk":
+                        self.unk = line.split(":")[1]
+                    else:
+                        continue
+                elif line[:3] == "Chr":
+                    header_row = line.split("\t")
+                    if header_row[2] == "Mb":
+                        self.mb_exists = True
+                        self.mb_column = 2
+                        self.cm_column = 3
+                    elif header_row[3] == "Mb":
+                        self.mb_exists = True
+                        self.mb_column = 3
+                    elif header_row[2] == "cM":
+                        self.cm_column = 2
+
+                    if self.mb_exists:
+                        self.prgy = header_row[4:]
+                    else:
+                        self.prgy = header_row[3:]
+                    self.nprgy = len(self.prgy)
+                else:
+                    if line.split("\t")[0] != this_chr:
+                        if this_chr != "":
+                            self.chromosomes.append(chr_ob)
+                        this_chr = line.split("\t")[0]
+                        chr_ob = Chr(line.split("\t")[0], self)
+                    chr_ob.add_marker(line.split("\t"))
+
+            self.chromosomes.append(chr_ob)
+
+class Chr(object):
+    def __init__(self, name, geno_ob):
+        self.name = name
+        self.loci = []
+        self.mb_exists = geno_ob.mb_exists
+        self.cm_column = geno_ob.cm_column
+        self.mb_column = geno_ob.mb_column
+        self.geno_ob = geno_ob
+
+    def __iter__(self):
+        return iter(self.loci)
+
+    def __getitem__(self, index):
+        return self.loci[index]
+
+    def __len__(self):
+        return len(self.loci)
+
+    def add_marker(self, marker_row):
+        self.loci.append(Locus(marker_row, self.geno_ob))
+
+class Locus(object):
+    def __init__(self, marker_row, geno_ob):
+        self.chr = marker_row[0]
+        self.name = marker_row[1]
+        self.cM = float(marker_row[geno_ob.cm_column])
+        self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
+
+        geno_table = {
+            geno_ob.mat: -1,
+            geno_ob.pat: 1,
+            geno_ob.het: 0,
+            geno_ob.unk: "U"
+        }
+
+        self.genotype = []
+        if geno_ob.mb_exists:
+            start_pos = 4
+        else:
+            start_pos = 3
+        for allele in marker_row[start_pos:]:
+            if allele in geno_table.keys():
+                self.genotype.append(geno_table[allele])
+            else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown
                 self.genotype.append("U")
\ No newline at end of file
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index 993fc2d9..e53e5279 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -236,9 +236,11 @@ class DisplayMappingResults(object):
         self.selectedChr = int(start_vars['selected_chr'])
 
         self.strainlist = start_vars['samples']
-        self.genotype = self.dataset.group.read_genotype_file()
+
         if self.mapping_method == "reaper" and self.manhattan_plot != True:
-            self.genotype = self.genotype.addinterval()
+            self.genotype = self.dataset.group.read_genotype_file(use_reaper=True)
+        else:
+            self.genotype = self.dataset.group.read_genotype_file()
 
         #Darwing Options
         try:
diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
index 35bed8d8..d58c59c8 100644
--- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
+++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
@@ -2,7 +2,7 @@ import utility.logger
 logger = utility.logger.getLogger(__name__ )
 
 def gen_reaper_results(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot):
-    genotype = dataset.group.read_genotype_file()
+    genotype = dataset.group.read_genotype_file(use_reaper=True)
 
     if manhattan_plot != True:
         genotype = genotype.addinterval()
-- 
cgit v1.2.3


From 3659ceb0ee7058e41cd0b789558b07d7e8bf415e Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 7 Jun 2019 13:08:47 -0500
Subject: Global search and correlation results can handle null expression now

---
 wqflask/utility/type_checking.py                 | 2 +-
 wqflask/wqflask/correlation/show_corr_results.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/type_checking.py b/wqflask/utility/type_checking.py
index 220e5f62..f15b17e2 100644
--- a/wqflask/utility/type_checking.py
+++ b/wqflask/utility/type_checking.py
@@ -27,7 +27,7 @@ def get_float(vars,name,default=None):
     if name in vars:
         if is_float(vars[name]):
             return float(vars[name])
-    return None
+    return default
 
 def get_int(vars,name,default=None):
     if name in vars:
diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 1aa7945d..6e9abcd6 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -83,7 +83,7 @@ class CorrelationResults(object):
         assert('corr_sample_method' in start_vars)
         assert('corr_samples_group' in start_vars)
         assert('corr_dataset' in start_vars)
-        assert('min_expr' in start_vars)
+        #assert('min_expr' in start_vars)
         assert('corr_return_results' in start_vars)
         if 'loc_chr' in start_vars:
             assert('min_loc_mb' in start_vars)
-- 
cgit v1.2.3


From 5f5adb62bf13ace887c03522b0b8e83181cd6503 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 23 Jul 2019 11:24:56 -0500
Subject: Added change to automatically update datasets list using rest api
 instead of json file

Removed option to edit certain html pages, like news, policies, etc
---
 etc/default_settings.py                            |  1 +
 wqflask/base/data_set.py                           |  4 +-
 wqflask/utility/tools.py                           |  4 ++
 .../marker_regression/display_mapping_results.py   |  1 -
 wqflask/wqflask/marker_regression/run_mapping.py   | 17 +++++++-
 .../new/javascript/dataset_menu_structure.json     | 51 ++++++++++++----------
 wqflask/wqflask/templates/docs.html                |  4 +-
 7 files changed, 53 insertions(+), 29 deletions(-)

(limited to 'wqflask/utility')

diff --git a/etc/default_settings.py b/etc/default_settings.py
index a1fe81e5..a16ac3ad 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -91,4 +91,5 @@ JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript"
 # ---- GN2 Executables (overwrite for testing only)
 # PLINK_COMMAND = str.strip(os.popen("which plink2").read())
 # GEMMA_COMMAND = str.strip(os.popen("which gemma").read())
+REAPER_COMMAND = HOME + "/gn2-zach/rust-qtlreaper/target/release/qtlreaper"
 # GEMMA_WRAPPER_COMMAND = str.strip(os.popen("which gemma-wrapper").read())
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index d766e284..41de8492 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -46,6 +46,8 @@ from utility import chunks
 from utility import gen_geno_ob
 from utility.tools import locate, locate_ignore_error, flat_files
 
+from wqflask.api import gen_menu
+
 from maintenance import get_group_samplelists
 
 from MySQLdb import escape_string as escape
@@ -92,7 +94,7 @@ Publish or ProbeSet. E.g.
         """
         self.datasets = {}
         if USE_GN_SERVER:
-            data = menu_main()
+            data = gen_menu.gen_dropdown_json()
         else:
             file_name = "wqflask/static/new/javascript/dataset_menu_structure.json"
             with open(file_name, 'r') as fh:
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 8b2260f5..31ab2046 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -107,6 +107,9 @@ def js_path(module=None):
         return try_guix
     raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)"
 
+def reaper_command(guess=None):
+    return get_setting("REAPER_COMMAND",guess)
+
 def gemma_command(guess=None):
     return assert_bin(get_setting("GEMMA_COMMAND",guess))
 
@@ -274,6 +277,7 @@ SMTP_CONNECT = get_setting('SMTP_CONNECT')
 SMTP_USERNAME = get_setting('SMTP_USERNAME')
 SMTP_PASSWORD = get_setting('SMTP_PASSWORD')
 
+REAPER_COMMAND     = app_set("REAPER_COMMAND",reaper_command())
 GEMMA_COMMAND      = app_set("GEMMA_COMMAND",gemma_command())
 assert(GEMMA_COMMAND is not None)
 PLINK_COMMAND      = app_set("PLINK_COMMAND",plink_command())
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index 3bcd613f..d9601405 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -1790,7 +1790,6 @@ class DisplayMappingResults(object):
             m = 0
             thisLRSColor = self.colorCollection[0]
             if qtlresult['chr'] != previous_chr and self.selectedChr == -1:
-
                 if self.manhattan_plot != True:
                     canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
 
diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py
index 2bde2b53..6e9fe85c 100644
--- a/wqflask/wqflask/marker_regression/run_mapping.py
+++ b/wqflask/wqflask/marker_regression/run_mapping.py
@@ -36,7 +36,7 @@ from utility import helper_functions
 from utility import Plot, Bunch
 from utility import temp_data
 from utility.benchmark import Bench
-from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping
+from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping, rust_reaper_mapping
 
 from utility.tools import locate, locate_ignore_error, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR
 from utility.external import shell
@@ -242,7 +242,8 @@ class RunMapping(object):
             self.control_marker = start_vars['control_marker']
             self.do_control = start_vars['do_control']
             logger.info("Running qtlreaper")
-            results, self.json_data, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = qtlreaper_mapping.gen_reaper_results(self.this_trait,
+
+            results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = rust_reaper_mapping.run_reaper(self.this_trait,
                                                                                                                                                         self.dataset,
                                                                                                                                                         self.samples,
                                                                                                                                                         self.vals,
@@ -253,6 +254,18 @@ class RunMapping(object):
                                                                                                                                                         self.do_control,
                                                                                                                                                         self.control_marker,
                                                                                                                                                         self.manhattan_plot)
+
+            # results, self.json_data, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = qtlreaper_mapping.gen_reaper_results(self.this_trait,
+            #                                                                                                                                             self.dataset,
+            #                                                                                                                                             self.samples,
+            #                                                                                                                                             self.vals,
+            #                                                                                                                                             self.json_data,
+            #                                                                                                                                             self.num_perm,
+            #                                                                                                                                             self.bootCheck,
+            #                                                                                                                                             self.num_bootstrap,
+            #                                                                                                                                             self.do_control,
+            #                                                                                                                                             self.control_marker,
+            #                                                                                                                                             self.manhattan_plot)
         elif self.mapping_method == "plink":
             self.score_type = "-log(p)"
             self.manhattan_plot = True
diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
index 0d73213d..6531f5a0 100644
--- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
+++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
@@ -1440,7 +1440,7 @@
                [
                   "None",
                   "HSBPublish",
-                  "HSB Phenotypes"
+                  "HSB Published Phenotypes"
                ]
             ],
             "Posterior Inferior Parietal Cortex mRNA": [
@@ -1721,7 +1721,7 @@
                [
                   "None",
                   "B6D2Publish",
-                  "B6D2 Phenotypes"
+                  "UTHSC-Glaucoma-Aged-Retina Phenotypes"
                ]
             ]
          },
@@ -2247,6 +2247,11 @@
                   "Eye_M2_0908_R_ND",
                   "Eye M430v2 WT Gpnmb (Sep08) RMA"
                ],
+               [
+                  "382",
+                  "Eye_M2_0908_WTWT",
+                  "Eye M430v2 WT WT (Sep08) RMA"
+               ],
                [
                   "279",
                   "Eye_M2_0908_R_WT",
@@ -2257,11 +2262,6 @@
                   "Eye_M2_0908_R_MT",
                   "Eye M430v2 Mutant Tyrp1 (Sep08) RMA"
                ],
-               [
-                  "382",
-                  "Eye_M2_0908_WTWT",
-                  "Eye M430v2 WT WT (Sep08) RMA"
-               ],
                [
                   "400",
                   "DBA2J-ONH-1212",
@@ -2500,6 +2500,11 @@
                   "245",
                   "UT_ILM_BXD_hipp_RSE_0909",
                   "UTHSC Hippocampus Illumina v6.1 RSE (Sep09) RankInv"
+               ],
+               [
+                  "711",
+                  "UTHSC_BXD_AgeHipp0515",
+                  "UTHSC BXD Aged Hippocampus Affy MoGene1.0 ST (May15) RMA Gene Level"
                ]
             ],
             "Hypothalamus mRNA": [
@@ -3028,16 +3033,6 @@
                ]
             ],
             "Striatum mRNA": [
-               [
-                  "376",
-                  "DevStriatum_ILM6.2P3RInv_1111",
-                  "BIDMC/UTHSC Dev Striatum P3 ILMv6.2 (Nov11) RankInv"
-               ],
-               [
-                  "377",
-                  "DevStriatum_ILM6.2P14RInv_1111",
-                  "BIDMC/UTHSC Dev Striatum P14 ILMv6.2 (Nov11) RankInv"
-               ],
                [
                   "399",
                   "Striatum_Exon_1212",
@@ -3048,6 +3043,16 @@
                   "Striatum_Exon_0209",
                   "HQF Striatum Affy Mouse Exon 1.0ST Exon Level (Dec09) RMA"
                ],
+               [
+                  "376",
+                  "DevStriatum_ILM6.2P3RInv_1111",
+                  "BIDMC/UTHSC Dev Striatum P3 ILMv6.2 (Nov11) RankInv"
+               ],
+               [
+                  "377",
+                  "DevStriatum_ILM6.2P14RInv_1111",
+                  "BIDMC/UTHSC Dev Striatum P14 ILMv6.2 (Nov11) RankInv"
+               ],
                [
                   "285",
                   "UTHSC_Striatum_RankInv_1210",
@@ -3554,6 +3559,11 @@
                ]
             ],
             "Hippocampus mRNA": [
+               [
+                  "211",
+                  "Illum_LXS_Hipp_RSS_1008",
+                  "Hippocampus Illumina RSS (Oct08) RankInv beta"
+               ],
                [
                   "213",
                   "Illum_LXS_Hipp_NOS_1008",
@@ -3574,11 +3584,6 @@
                   "Illum_LXS_Hipp_NOE_1008",
                   "Hippocampus Illumina NOE (Oct08) RankInv beta"
                ],
-               [
-                  "211",
-                  "Illum_LXS_Hipp_RSS_1008",
-                  "Hippocampus Illumina RSS (Oct08) RankInv beta"
-               ],
                [
                   "143",
                   "Illum_LXS_Hipp_loess0807",
@@ -3835,7 +3840,7 @@
                [
                   "None",
                   "HSNIH-RGSMCPublish",
-                  "HSNIH-RGSMC Phenotypes"
+                  "HSNIH Published Phenotypes"
                ]
             ]
          },
diff --git a/wqflask/wqflask/templates/docs.html b/wqflask/wqflask/templates/docs.html
index c485f757..1a241ce7 100644
--- a/wqflask/wqflask/templates/docs.html
+++ b/wqflask/wqflask/templates/docs.html
@@ -6,9 +6,9 @@
 <div class="container">
 	<h3>{{title}}</h3>
 	<div style="text-align: right;">
-		<a href="docedit?entry={{entry}}">
+		<!--<a href="docedit?entry={{entry}}">-->
 			<img style="width: 16px;" src="/static/new/images/edit.gif">
-		</a>
+		<!--</a>-->
 	</div>
 	{{content|safe}}
 </div>
-- 
cgit v1.2.3


From 4bd1db34b109bff3b9abdcc161472885e9ae700d Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Fri, 6 Sep 2019 03:29:12 -0500
Subject: Fix name of javascript-twitter-post-fetcher

---
 wqflask/utility/tools.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 31ab2046..ec9f01bd 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -289,11 +289,14 @@ assert_dir(TEMPDIR)
 JS_GUIX_PATH = get_setting("JS_GUIX_PATH")
 assert_dir(JS_GUIX_PATH)
 assert_dir(JS_GUIX_PATH+'/cytoscape-panzoom')
+
 CSS_PATH = "UNKNOWN"
 # assert_dir(JS_PATH)
-JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher"))
+
+JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("javascript-twitter-post-fetcher"))
 assert_dir(JS_TWITTER_POST_FETCHER_PATH)
 assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js")
+
 JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH",js_path("cytoscape"))
 assert_dir(JS_CYTOSCAPE_PATH)
 assert_file(JS_CYTOSCAPE_PATH+'/cytoscape.min.js')
-- 
cgit v1.2.3


From 233ed8a88c3f5681ed984a965080be2388e4ebbf Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 24 Sep 2019 12:17:08 -0500
Subject: Fixed issue that caused GEMMA to not work with genotype files without
 cM, though this issue still needs to be dealt with to get the new qtlreaper
 working

---
 wqflask/utility/gen_geno_ob.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
index 5172369f..e8780879 100644
--- a/wqflask/utility/gen_geno_ob.py
+++ b/wqflask/utility/gen_geno_ob.py
@@ -115,7 +115,10 @@ class Locus(object):
     def __init__(self, marker_row, geno_ob):
         self.chr = marker_row[0]
         self.name = marker_row[1]
-        self.cM = float(marker_row[geno_ob.cm_column])
+        try:
+            self.cM = float(marker_row[geno_ob.cm_column])
+        except:
+            self.cM = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else 0
         self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
 
         geno_table = {
-- 
cgit v1.2.3


From bd13472a05b947b809a8199584d6a3cf5dfad703 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 8 Oct 2019 14:28:42 -0500
Subject: Fixed serious issue that caused output to be wrong for new reaper
 mapping

Haplotype analyst now works for GEMMA

Increased margin a bit to fix minor issue with buttons getting cut off on certain Safari versions for bar chart
---
 wqflask/utility/gen_geno_ob.py                          | 15 ++++++++-------
 .../marker_regression/display_mapping_results.py        |  2 +-
 wqflask/wqflask/marker_regression/qtlreaper_mapping.py  | 17 ++++++++++++++++-
 wqflask/wqflask/templates/mapping_results.html          |  2 +-
 wqflask/wqflask/templates/show_trait_statistics.html    |  2 +-
 5 files changed, 27 insertions(+), 11 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
index e8780879..44e2722f 100644
--- a/wqflask/utility/gen_geno_ob.py
+++ b/wqflask/utility/gen_geno_ob.py
@@ -47,20 +47,20 @@ class genotype(object):
                 elif line[0] == "@":
                     label = line.split(":")[0][1:]
                     if label == "name":
-                        self.group = line.split(":")[1]
+                        self.group = line.split(":")[1].strip()
                     elif label == "filler":
-                        if line.split(":")[1] == "yes":
+                        if line.split(":")[1].strip() == "yes":
                             self.filler = True
                     elif label == "type":
-                        self.type = line.split(":")[1]
+                        self.type = line.split(":")[1].strip()
                     elif label == "mat":
-                        self.mat = line.split(":")[1]
+                        self.mat = line.split(":")[1].strip()
                     elif label == "pat":
-                        self.pat = line.split(":")[1]
+                        self.pat = line.split(":")[1].strip()
                     elif label == "het":
-                        self.het = line.split(":")[1]
+                        self.het = line.split(":")[1].strip()
                     elif label == "unk":
-                        self.unk = line.split(":")[1]
+                        self.unk = line.split(":")[1].strip()
                     else:
                         continue
                 elif line[:3] == "Chr":
@@ -133,6 +133,7 @@ class Locus(object):
             start_pos = 4
         else:
             start_pos = 3
+
         for allele in marker_row[start_pos:]:
             if allele in geno_table.keys():
                 self.genotype.append(geno_table[allele])
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index f3755758..a2f9651b 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -1341,7 +1341,7 @@ class DisplayMappingResults(object):
 
         for i, _chr in enumerate(self.genotype):
             if _chr.name == self.ChrList[self.selectedChr][0]:
-                for j, _geno in enumerate(_chr[1]):
+                for j, _geno in enumerate(_chr[1].genotype):
 
                     plotbxd=0
                     for item in smd:
diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
index 50e6bd7e..fe216166 100644
--- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
+++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
@@ -1,4 +1,4 @@
-import os, math, string, random, json
+import os, math, string, random, json, re
 
 from base import webqtlConfig
 from base.trait import GeneralTrait
@@ -105,6 +105,9 @@ def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename):
                 marker['additive'] = float(line.split("\t")[6])
                 marker_obs.append(marker)
 
+    #ZS: Results have to be reordered because the new reaper returns results sorted alphabetically by chr for some reason, resulting in chr 1 being followed by 10, etc
+    sorted_indices = natural_sort(marker_obs)
+
     permu_vals = []
     if permu_filename:
         with open("{}{}.txt".format(webqtlConfig.GENERATED_IMAGE_DIR, permu_filename)) as permu_file:
@@ -117,6 +120,9 @@ def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename):
             for line in bootstrap_file:
                 bootstrap_vals.append(int(line))
 
+    marker_obs = [marker_obs[i] for i in sorted_indices]
+    bootstrap_vals = [bootstrap_vals[i] for i in sorted_indices]
+
     return marker_obs, permu_vals, bootstrap_vals
 
 def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot):
@@ -210,3 +216,12 @@ def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_da
                "cM":reaper_locus.cM, "name":reaper_locus.name, "additive":qtl.additive, "dominance":qtl.dominance}
         qtl_results.append(qtl)
     return qtl_results, json_data, perm_output, suggestive, significant, bootstrap_results
+
+def natural_sort(marker_list):
+    """
+    Function to naturally sort numbers + strings, adopted from user Mark Byers here: https://stackoverflow.com/questions/4836710/does-python-have-a-built-in-function-for-string-natural-sort
+    Changed to return indices instead of values, though, since the same reordering needs to be applied to bootstrap results
+    """
+    convert = lambda text: int(text) if text.isdigit() else text.lower()
+    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', str(marker_list[key]['chr'])) ]
+    return sorted(range(len(marker_list)), key = alphanum_key)
\ No newline at end of file
diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html
index 1fc792d8..8422b1d2 100644
--- a/wqflask/wqflask/templates/mapping_results.html
+++ b/wqflask/wqflask/templates/mapping_results.html
@@ -144,7 +144,7 @@
                     <span style="color:red;">*</span>
                   <br>
                   <input type="checkbox" name="showGenes" class="checkbox" style="display: inline; margin-top: 0px;" {% if geneChecked|upper == "ON" %}value="ON" checked{% endif %}> <span style="font-size: 12px;">Gene Track </span> <span style="color:red;">*</span><br>
-                  {% if plotScale != "morgan" and mapping_method != "gemma" and mapping_method != "plink" %}
+                  {% if plotScale != "morgan" %}
                   <input type="checkbox" name="haplotypeAnalystCheck" class="checkbox" style="display: inline; margin-top: 0px;" {% if haplotypeAnalystChecked|upper == "ON" %}value="ON" checked{% endif %}> <span style="font-size: 12px;">Haplotype Analyst </span> <span style="color:red;">*</span><br>
                   {% endif %}
                   <input type="checkbox" name="viewLegend" class="checkbox" style="display: inline; margin-top: 0px;" {% if legendChecked|upper == "ON" %}value="ON" checked{% endif %}> <span style="font-size: 12px;">Legend </span>
diff --git a/wqflask/wqflask/templates/show_trait_statistics.html b/wqflask/wqflask/templates/show_trait_statistics.html
index 1093e1bb..4653b398 100644
--- a/wqflask/wqflask/templates/show_trait_statistics.html
+++ b/wqflask/wqflask/templates/show_trait_statistics.html
@@ -70,7 +70,7 @@
                             </select>
                         </div>
                         {% endif %}
-                        <div id="update_bar_chart" class="btn-group">
+                        <div id="update_bar_chart" class="btn-group" style="margin-bottom: 5px;">
                             <button type="button" class="btn btn-default sort_by_name" value="name">
                             <i class="icon-resize-horizontal"></i> Sort By Name
                             </button>
-- 
cgit v1.2.3


From acf114fb60ce493287dc852265a91ef12c854b0b Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 18 Oct 2019 14:35:17 -0500
Subject: Fixed issue with collections ordering

---
 wqflask/utility/webqtlUtil.py                  | 5 +++--
 wqflask/wqflask/api/gen_menu.py                | 6 ++++++
 wqflask/wqflask/collect.py                     | 2 +-
 wqflask/wqflask/templates/index_page_orig.html | 1 +
 wqflask/wqflask/user_manager.py                | 5 ++++-
 5 files changed, 15 insertions(+), 4 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py
index 67a476e3..53661ae4 100644
--- a/wqflask/utility/webqtlUtil.py
+++ b/wqflask/utility/webqtlUtil.py
@@ -52,11 +52,12 @@ ParInfo ={
 'AXB':['ABF1', 'BAF1', 'C57BL/6J', 'A/J'],
 'BXA':['BAF1', 'ABF1', 'C57BL/6J', 'A/J'],
 'LXS':['LSF1', 'SLF1', 'ISS', 'ILS'],
-'HXBBXH':['HSRBNF1', 'BNHSRF1', 'BN', 'HSR'],
+'HXBBXH':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'],
 'BayXSha':['BayXShaF1', 'ShaXBayF1', 'Bay-0','Shahdara'],
 'ColXBur':['ColXBurF1', 'BurXColF1', 'Col-0','Bur-0'],
 'ColXCvi':['ColXCviF1', 'CviXColF1', 'Col-0','Cvi'],
-'SXM':['SMF1', 'MSF1', 'Steptoe','Morex']
+'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'],
+'HRDP':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv']
 }
 
 #########################################
diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py
index d7a59f29..77ca6525 100644
--- a/wqflask/wqflask/api/gen_menu.py
+++ b/wqflask/wqflask/api/gen_menu.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division
 
 import sys
+import json
 
 from flask import g
 
@@ -35,6 +36,11 @@ def gen_dropdown_json():
                 types=types,
                 datasets=datasets)
 
+    output_file = """./wqflask/static/new/javascript/dataset_menu_structure.json"""
+
+    with open(output_file, 'w') as fh:
+        json.dump(data, fh, indent=3, sort_keys=True)
+
     return data
 
 def get_species():
diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py
index 73269c4d..c273af59 100644
--- a/wqflask/wqflask/collect.py
+++ b/wqflask/wqflask/collect.py
@@ -100,7 +100,7 @@ class AnonCollection(object):
                     collection_exists = True
                     break
             if collection_exists:
-                collections_list[collection_position]['members'].extend(self.traits)
+                collections_list[collection_position]['members'] += list(set(self.traits) - set(collections_list[collection_position]['members']))
                 collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members'])
                 collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
             else:
diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html
index f8ea3c7c..bdee6a12 100755
--- a/wqflask/wqflask/templates/index_page_orig.html
+++ b/wqflask/wqflask/templates/index_page_orig.html
@@ -193,6 +193,7 @@
                           <li><a href="http://bnw.genenetwork.org/">Bayesian Network Web Server</a> at UTHSC</li>
                           <li><a href="http://genenetwork.org">GeneNetwork 1</a> at UTHSC</li>
                           <li><a href="https://www.geneweaver.org/">GeneWeaver</a></li>
+                          <li><a href="https://phenogen.org/">PhenoGen</a> at University of Colorado</li>
                           <li><a href="http://www.webgestalt.org/">WebGestalt</a> at Baylor</li>
                         </ul>
                     </div>
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index b2c7a914..3298c244 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -99,6 +99,8 @@ class AnonUser(object):
             for collection in collections:
                 collection['created_timestamp'] = datetime.datetime.strptime(collection['created_timestamp'], '%b %d %Y %I:%M%p')
                 collection['changed_timestamp'] = datetime.datetime.strptime(collection['changed_timestamp'], '%b %d %Y %I:%M%p')
+
+            collections = sorted(collections, key = lambda i: i['changed_timestamp'], reverse = True)
             return collections
 
     def import_traits_to_user(self):
@@ -234,7 +236,8 @@ class UserSession(object):
         user_info = response['hits']['hits'][0]['_source']
         if 'collections' in user_info.keys():
             if len(user_info['collections']) > 0:
-                return json.loads(user_info['collections'])
+                collection_list = json.loads(user_info['collections'])
+                return sorted(collection_list, key = lambda i: datetime.datetime.strptime(i['changed_timestamp'], '%b %d %Y %I:%M%p'), reverse=True)
             else:
                 return []
         else:
-- 
cgit v1.2.3


From 81afc6fd60d2b267c9041835a93088f7a5e4c3c4 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Thu, 23 Jan 2020 11:08:27 -0600
Subject: Forgot to commit new file containing Redis functions

---
 wqflask/utility/redis_tools.py | 73 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 wqflask/utility/redis_tools.py

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py
new file mode 100644
index 00000000..ffd263e8
--- /dev/null
+++ b/wqflask/utility/redis_tools.py
@@ -0,0 +1,73 @@
+from __future__ import print_function, division, absolute_import
+
+import simplejson as json
+
+import redis # used for collections
+Redis = redis.StrictRedis()
+
+import logging
+
+from flask import (render_template, flash)
+
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+def get_user_id(column_name, column_value):
+    user_list = Redis.hgetall("users")
+    for key in user_list:
+        user_ob = json.loads(user_list[key])
+        if user_ob[column_name] == column_value:
+            return key
+
+    return None
+
+def get_user_by_unique_column(column_name, column_value):
+    item_details = None
+
+    user_list = Redis.hgetall("users")
+    if column_name != "user_id":
+        for key in user_list:
+            user_ob = json.loads(user_list[key])
+            if user_ob[column_name] == column_value:
+                item_details = user_ob
+    else:
+        item_details = json.loads(user_list[column_value])
+
+    return item_details
+
+def set_user_attribute(user_id, column_name, column_value):
+    user_info = json.loads(Redis.hget("users", user_id))
+    user_info[column_name] = column_value
+
+    Redis.hset("users", user_id, json.dumps(user_info))
+
+def get_user_collections(user_id):
+    collections = None
+    collections = Redis.hget("collections", user_id)
+
+    if collections:
+        return json.loads(collections)
+    else:
+        return []
+
+def save_user(user, user_id):
+    Redis.hset("users", user_id, json.dumps(user))
+
+def save_collections(user_id, collections_ob):
+    Redis.hset("collections", user_id, collections_ob)
+
+def save_verification_code(user_email, code):
+    Redis.hset("verification_codes", code, user_email)
+
+def check_verification_code(code):
+    email_address = None
+    user_details = None
+    email_address = Redis.hget("verification_codes", code)
+    return email_address
+
+    if email_address:
+        user_details = get_user_by_unique_column('email_address', email_address)
+        return user_details
+    else:
+        return None
+        #flash("Invalid code: Password reset code does not exist or might have expired!", "error")
-- 
cgit v1.2.3


From bce4926ec6cb0082fe4d9cd46884859ebd1b1f39 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 12 Feb 2020 12:55:16 -0600
Subject: Fixed issue that broke some logins with the new redis user code

---
 wqflask/utility/redis_tools.py | 153 +++++++++++++++++++++--------------------
 1 file changed, 80 insertions(+), 73 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py
index ffd263e8..0754e16f 100644
--- a/wqflask/utility/redis_tools.py
+++ b/wqflask/utility/redis_tools.py
@@ -1,73 +1,80 @@
-from __future__ import print_function, division, absolute_import
-
-import simplejson as json
-
-import redis # used for collections
-Redis = redis.StrictRedis()
-
-import logging
-
-from flask import (render_template, flash)
-
-from utility.logger import getLogger
-logger = getLogger(__name__)
-
-def get_user_id(column_name, column_value):
-    user_list = Redis.hgetall("users")
-    for key in user_list:
-        user_ob = json.loads(user_list[key])
-        if user_ob[column_name] == column_value:
-            return key
-
-    return None
-
-def get_user_by_unique_column(column_name, column_value):
-    item_details = None
-
-    user_list = Redis.hgetall("users")
-    if column_name != "user_id":
-        for key in user_list:
-            user_ob = json.loads(user_list[key])
-            if user_ob[column_name] == column_value:
-                item_details = user_ob
-    else:
-        item_details = json.loads(user_list[column_value])
-
-    return item_details
-
-def set_user_attribute(user_id, column_name, column_value):
-    user_info = json.loads(Redis.hget("users", user_id))
-    user_info[column_name] = column_value
-
-    Redis.hset("users", user_id, json.dumps(user_info))
-
-def get_user_collections(user_id):
-    collections = None
-    collections = Redis.hget("collections", user_id)
-
-    if collections:
-        return json.loads(collections)
-    else:
-        return []
-
-def save_user(user, user_id):
-    Redis.hset("users", user_id, json.dumps(user))
-
-def save_collections(user_id, collections_ob):
-    Redis.hset("collections", user_id, collections_ob)
-
-def save_verification_code(user_email, code):
-    Redis.hset("verification_codes", code, user_email)
-
-def check_verification_code(code):
-    email_address = None
-    user_details = None
-    email_address = Redis.hget("verification_codes", code)
-    return email_address
-
-    if email_address:
-        user_details = get_user_by_unique_column('email_address', email_address)
-        return user_details
-    else:
-        return None
-        #flash("Invalid code: Password reset code does not exist or might have expired!", "error")
+from __future__ import print_function, division, absolute_import
+
+import simplejson as json
+
+import redis # used for collections
+Redis = redis.StrictRedis()
+
+import logging
+
+from flask import (render_template, flash)
+
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+def is_redis_available():
+    try:
+        Redis.ping()
+    except:
+        return False
+    return True
+
+def get_user_id(column_name, column_value):
+    user_list = Redis.hgetall("users")
+    for key in user_list:
+        user_ob = json.loads(user_list[key])
+        if column_name in user_ob and user_ob[column_name] == column_value:
+            return key
+
+    return None
+
+def get_user_by_unique_column(column_name, column_value):
+    item_details = None
+
+    user_list = Redis.hgetall("users")
+    if column_name != "user_id":
+        for key in user_list:
+            user_ob = json.loads(user_list[key])
+            if column_name in user_ob and user_ob[column_name] == column_value:
+                item_details = user_ob
+    else:
+        item_details = json.loads(user_list[column_value])
+
+    return item_details
+
+def set_user_attribute(user_id, column_name, column_value):
+    user_info = json.loads(Redis.hget("users", user_id))
+    user_info[column_name] = column_value
+
+    Redis.hset("users", user_id, json.dumps(user_info))
+
+def get_user_collections(user_id):
+    collections = None
+    collections = Redis.hget("collections", user_id)
+
+    if collections:
+        return json.loads(collections)
+    else:
+        return []
+
+def save_user(user, user_id):
+    Redis.hset("users", user_id, json.dumps(user))
+
+def save_collections(user_id, collections_ob):
+    Redis.hset("collections", user_id, collections_ob)
+
+def save_verification_code(user_email, code):
+    Redis.hset("verification_codes", code, user_email)
+
+def check_verification_code(code):
+    email_address = None
+    user_details = None
+    email_address = Redis.hget("verification_codes", code)
+    return email_address
+
+    if email_address:
+        user_details = get_user_by_unique_column('email_address', email_address)
+        return user_details
+    else:
+        return None
+        flash("Invalid code: Password reset code does not exist or might have expired!", "error")
-- 
cgit v1.2.3


From f9849394e3a252b5a1ac59c78a06728d20ca69ed Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 3 Mar 2020 15:08:48 -0600
Subject: Refactored all the anonymous (not logged-in) user stuff so that both
 logged-in and anonymous users share the same code

---
 wqflask/utility/tools.py                       |   3 +-
 wqflask/wqflask/static/new/images/CITGLogo.png | Bin 0 -> 11962 bytes
 wqflask/wqflask/user_login.py                  | 467 +++++++++++++++++++++++++
 wqflask/wqflask/user_session.py                | 272 ++++++++++++++
 wqflask/wqflask/views.py                       |   7 +-
 5 files changed, 744 insertions(+), 5 deletions(-)
 create mode 100644 wqflask/wqflask/static/new/images/CITGLogo.png
 create mode 100644 wqflask/wqflask/user_login.py
 create mode 100644 wqflask/wqflask/user_session.py

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index ec9f01bd..75bddb24 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -254,7 +254,6 @@ JS_GN_PATH         = get_setting('JS_GN_PATH')
 
 GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID')
 GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET')
-GITHUB_AUTH_URL = None
 if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET:
     GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \
                       GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET
@@ -264,7 +263,7 @@ ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID')
 ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET')
 ORCID_AUTH_URL = None
 if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET:
-    ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \
+    ORCID_AUTH_URL = "https://orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \
                       ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET
     ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL')
 
diff --git a/wqflask/wqflask/static/new/images/CITGLogo.png b/wqflask/wqflask/static/new/images/CITGLogo.png
new file mode 100644
index 00000000..ae99fedb
Binary files /dev/null and b/wqflask/wqflask/static/new/images/CITGLogo.png differ
diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py
new file mode 100644
index 00000000..05885e2c
--- /dev/null
+++ b/wqflask/wqflask/user_login.py
@@ -0,0 +1,467 @@
+from __future__ import print_function, division, absolute_import
+
+import os
+import hashlib
+import datetime
+import time
+import logging
+import uuid
+import hashlib
+import hmac
+import base64
+import requests
+
+import simplejson as json
+
+import redis # used for collections
+Redis = redis.StrictRedis()
+
+from flask import (Flask, g, render_template, url_for, request, make_response,
+                   redirect, flash, abort)
+
+from wqflask import app
+from wqflask import pbkdf2
+from wqflask.hmac_func import hmac_creation
+from wqflask.user_session import UserSession
+
+from utility.redis_tools import is_redis_available, get_user_id, get_user_by_unique_column, set_user_attribute, save_user, save_verification_code, check_verification_code, get_user_collections, save_collections
+
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+from smtplib import SMTP
+from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_ALCHEMY
+
+THREE_DAYS = 60 * 60 * 24 * 3
+
+def timestamp():
+    return datetime.datetime.utcnow().isoformat()
+
+def basic_info():
+    return dict(timestamp = timestamp(),
+                ip_address = request.remote_addr,
+                user_agent = request.headers.get('User-Agent'))
+
+def encode_password(pass_gen_fields):
+    hashfunc = getattr(hashlib, pass_gen_fields['hashfunc'])
+
+    salt = base64.b64decode(pass_gen_fields['salt'])
+    password = pbkdf2.pbkdf2_hex(str(pass_gen_fields['unencrypted_password']), 
+                                 pass_gen_fields['salt'], 
+                                 pass_gen_fields['iterations'], 
+                                 pass_gen_fields['keylength'], 
+                                 hashfunc)
+
+    return password
+
+def set_password(password):
+    pass_gen_fields = {
+      "unencrypted_password": password,
+      "algorithm": "pbkdf2",
+      "hashfunc": "sha256",
+      "salt": base64.b64encode(os.urandom(32)),
+      "iterations": 100000,
+      "keylength": 32,
+      "created_timestamp": timestamp()
+    }
+
+    assert len(password) >= 6, "Password shouldn't be shorter than 6 characters"
+
+    encoded_password = encode_password(pass_gen_fields)
+
+    return encoded_password
+
+def encrypt_password(unencrypted_password, pwfields):
+        hashfunc = getattr(hashlib, pwfields['hashfunc'])
+        salt = base64.b64decode(pwfields['salt'])
+        iterations = pwfields['iterations']
+        keylength = pwfields['keylength']
+        encrypted_password = pbkdf2.pbkdf2_hex(str(unencrypted_password),
+                                               salt, iterations, keylength, hashfunc)
+        return encrypted_password
+
+def get_signed_session_id(user):
+    session_id = str(uuid.uuid4())
+
+    session_id_signature = hmac_creation(session_id)
+    session_id_signed = session_id + ":" + session_id_signature
+
+    #ZS: Need to check if this is ever actually used or exists
+    if 'user_id' not in user:
+        user['user_id'] = str(uuid.uuid4())
+        save_user(user, user['user_id'])
+
+    if 'github_id' in user:
+        session = dict(login_time = time.time(),
+                    user_type = "github",
+                    user_id = user['user_id'],
+                    github_id = user['github_id'],
+                    user_name = user['name'],
+                    user_url = user['user_url'])
+    elif 'orcid' in user:
+        session = dict(login_time = time.time(),
+                    user_type = "orcid",
+                    user_id = user['user_id'],
+                    github_id = user['orcid'],
+                    user_name = user['name'],
+                    user_url = user['user_url'])
+    else:
+        session = dict(login_time = time.time(),
+                       user_type = "gn2",
+                       user_id = user['user_id'],
+                       user_name = user['full_name'],
+                       user_email_address = user['email_address'])
+
+    key = UserSession.user_cookie_name + ":" + session_id
+    Redis.hmset(key, session)
+    Redis.expire(key, THREE_DAYS)
+    
+    return session_id_signed
+
+def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"):
+    """Send an E-mail through SMTP_CONNECT host. If SMTP_USERNAME is not
+    'UNKNOWN' TLS is used
+
+    """
+    if SMTP_USERNAME == 'UNKNOWN':
+        server = SMTP(SMTP_CONNECT)
+        server.sendmail(fromaddr, toaddr, msg)
+    else:
+        server = SMTP(SMTP_CONNECT)
+        server.starttls()
+        server.login(SMTP_USERNAME, SMTP_PASSWORD)
+        server.sendmail(fromaddr, toaddr, msg)
+        server.quit()
+    logger.info("Successfully sent email to "+toaddr)
+
+def send_verification_email(user_details, template_name = "email/verification.txt", key_prefix = "verification_code", subject = "GeneNetwork email verification"):
+    verification_code = str(uuid.uuid4())
+    key = key_prefix + ":" + verification_code
+
+    data = json.dumps(dict(id=user_details['user_id'], timestamp = timestamp()))
+
+    Redis.set(key, data)
+    Redis.expire(key, THREE_DAYS)
+
+    recipient = user_details['email_address']
+    body = render_template(template_name, verification_code = verification_code)
+    send_email(recipient, subject, body)
+    return {"recipient": recipient, "subject": subject, "body": body}
+
+@app.route("/n/login", methods=('GET', 'POST'))
+def login():
+    params = request.form if request.form else request.args
+    logger.debug("in login params are:", params)
+
+    if not params: #ZS: If coming to page for first time
+        from utility.tools import GITHUB_AUTH_URL, GITHUB_CLIENT_ID, ORCID_AUTH_URL, ORCID_CLIENT_ID
+        external_login = {}
+        if GITHUB_AUTH_URL and GITHUB_CLIENT_ID != 'UNKNOWN':
+            external_login["github"] = GITHUB_AUTH_URL
+        if ORCID_AUTH_URL and ORCID_CLIENT_ID != 'UNKNOWN':
+            external_login["orcid"] = ORCID_AUTH_URL
+        return render_template("new_security/login_user.html", external_login = external_login, redis_is_available=is_redis_available())
+    else: #ZS: After clicking sign-in
+        if 'type' in params and 'uid' in params:
+            user_details = get_user_by_unique_column("user_id", params['uid'])
+            if user_details:
+                session_id_signed = get_signed_session_id(user_details)
+                if 'name' in user_details and user_details['name'] != "None":
+                    display_id = user_details['name']
+                elif 'github_id' in user_details:
+                    display_id = user_details['github_id']
+                elif 'orcid' in user_details:
+                    display_id = user_details['orcid']
+                else:
+                    display_id = ""
+                flash("Thank you for logging in {}.".format(display_id), "alert-success")
+                response = make_response(redirect(url_for('index_page')))
+                response.set_cookie(UserSession.user_cookie_name, session_id_signed, max_age=None)
+            else:
+                flash("Something went unexpectedly wrong.", "alert-danger")
+                response = make_response(redirect(url_for('index_page')))  
+            return response
+        else:
+            user_details = get_user_by_unique_column("email_address", params['email_address'])
+            password_match = False
+            if user_details:
+                submitted_password = params['password']
+                pwfields = json.loads(user_details['password'])
+                encrypted_pass = encrypt_password(submitted_password, pwfields)
+                password_match = pbkdf2.safe_str_cmp(encrypted_pass, pwfields['password'])
+            else: # Invalid e-mail
+                flash("Invalid e-mail address. Please try again.", "alert-danger")
+                response = make_response(redirect(url_for('login')))
+
+                return response
+            if password_match: # If password correct
+                if user_details['confirmed']: # If account confirmed
+                    import_col = "false"
+                    if 'import_collections' in params:
+                        import_col = "true"
+
+                    session_id_signed = get_signed_session_id(user_details)
+                    flash("Thank you for logging in {}.".format(user_details['full_name']), "alert-success")
+                    response = make_response(redirect(url_for('index_page', import_collections = import_col)))
+                    response.set_cookie(UserSession.user_cookie_name, session_id_signed, max_age=None)
+                    return response
+                else:
+                    email_ob = send_verification_email(user_details)
+                    return render_template("newsecurity/verification_still_needed.html", subject=email_ob['subject'])
+            else: # Incorrect password
+                #ZS: It previously seemed to store that there was an incorrect log-in attempt here, but it did so in the MySQL DB so this might need to be reproduced with Redis
+                flash("Invalid password. Please try again.", "alert-danger")
+                response = make_response(redirect(url_for('login')))
+
+                return response
+
+@app.route("/n/login/github_oauth2", methods=('GET', 'POST'))
+def github_oauth2():
+    from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET, GITHUB_AUTH_URL
+    code = request.args.get("code")
+    data = {
+        "client_id": GITHUB_CLIENT_ID,
+        "client_secret": GITHUB_CLIENT_SECRET,
+        "code": code
+    }
+    logger.debug("LOGIN DATA:", data)
+    result = requests.post("https://github.com/login/oauth/access_token", json=data)
+    result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]}
+
+    github_user = get_github_user_details(result_dict["access_token"])
+
+    user_details = get_user_by_unique_column("github_id", github_user["id"])
+    if user_details == None:
+        user_details = {
+            "user_id": str(uuid.uuid4()), 
+            "name": github_user["name"].encode("utf-8") if github_user["name"] else "None", 
+            "github_id": github_user["id"],
+            "user_url": github_user["html_url"].encode("utf-8"), 
+            "login_type": "github", 
+            "organization": "", 
+            "active": 1, 
+            "confirmed": 1
+        }
+        save_user(user_details, user_details["user_id"])
+
+    url = "/n/login?type=github&uid="+user_details["user_id"]
+    return redirect(url)
+
+def get_github_user_details(access_token):
+    from utility.tools import GITHUB_API_URL
+    result = requests.get(GITHUB_API_URL, headers = {'Authorization':'token ' + access_token }).content
+
+    return json.loads(result)
+
+@app.route("/n/login/orcid_oauth2", methods=('GET', 'POST'))
+def orcid_oauth2():
+    from uuid import uuid4
+    from utility.tools import ORCID_CLIENT_ID, ORCID_CLIENT_SECRET, ORCID_TOKEN_URL, ORCID_AUTH_URL
+    code = request.args.get("code")
+    error = request.args.get("error")
+    url = "/n/login"
+    if code:
+        data = {
+            "client_id": ORCID_CLIENT_ID, 
+            "client_secret": ORCID_CLIENT_SECRET, 
+            "grant_type": "authorization_code", 
+            "code": code
+        }
+        result = requests.post(ORCID_TOKEN_URL, data=data)
+        result_dict = json.loads(result.text.encode("utf-8"))
+
+        user_details = get_user_by_unique_column("orcid", result_dict["orcid"])
+        if user_details == None:
+            user_details = {
+                "user_id": str(uuid4()), 
+                "name": result_dict["name"], 
+                "orcid": result_dict["orcid"], 
+                "user_url": "%s/%s" % ("/".join(ORCID_AUTH_URL.split("/")[:-2]), result_dict["orcid"]), 
+                "login_type": "orcid", 
+                "organization": "", 
+                "active": 1, 
+                "confirmed": 1
+            }
+            save_user(user_details, user_details["user_id"])
+
+        url = "/n/login?type=orcid&uid="+user_details["user_id"]
+    else:
+        flash("There was an error getting code from ORCID")
+    return redirect(url)
+
+def get_github_user_details(access_token):
+    from utility.tools import GITHUB_API_URL
+    result = requests.get(GITHUB_API_URL, headers = {'Authorization':'token ' + access_token }).content
+
+    return json.loads(result)
+
+
+@app.route("/n/logout")
+def logout():
+    logger.debug("Logging out...")
+    UserSession().delete_session()
+    flash("You are now logged out. We hope you come back soon!")
+    response = make_response(redirect(url_for('index_page')))
+    # Delete the cookie
+    response.set_cookie(UserSession.user_cookie_name, '', expires=0)
+    return response
+
+@app.route("/n/forgot_password", methods=['GET'])
+def forgot_password():
+    """Entry point for forgotten password"""
+    print("ARGS: ", request.args)
+    errors = {"no-email": request.args.get("no-email")}
+    print("ERRORS: ", errors)
+    return render_template("new_security/forgot_password.html", errors=errors)
+
+def send_forgot_password_email(verification_email):
+    from email.MIMEMultipart import MIMEMultipart
+    from email.MIMEText import MIMEText
+
+    template_name  = "email/forgot_password.txt"
+    key_prefix = "forgot_password_code"
+    subject = "GeneNetwork password reset"
+    fromaddr = "no-reply@genenetwork.org"
+    
+    verification_code = str(uuid.uuid4())
+    key = key_prefix + ":" + verification_code
+
+    data = {
+        "verification_code": verification_code,
+        "email_address": verification_email,
+        "timestamp": timestamp()
+    }
+
+    save_verification_code(verification_email, verification_code)
+
+    body = render_template(template_name, verification_code = verification_code)
+
+    msg = MIMEMultipart()
+    msg["To"] = verification_email
+    msg["Subject"] = subject
+    msg["From"] = fromaddr
+    msg.attach(MIMEText(body, "plain"))
+
+    send_email(verification_email, msg.as_string())
+
+    return subject
+
+@app.route("/n/forgot_password_submit", methods=('POST',))
+def forgot_password_submit():
+    """When a forgotten password form is submitted we get here"""
+    params = request.form
+    email_address = params['email_address']
+    next_page = None
+    if email_address != "":
+        logger.debug("Wants to send password E-mail to ", email_address)
+        user_details = get_user_by_unique_column("email_address", email_address)
+        if user_details:
+            email_subject = send_forgot_password_email(user_details["email_address"])
+            return render_template("new_security/forgot_password_step2.html",
+                                   subject=email_subject)
+        else:
+            flash("The e-mail entered is not associated with an account.", "alert-danger")
+            return redirect(url_for("forgot_password"))
+
+    else:
+        flash("You MUST provide an email", "alert-danger")
+        return redirect(url_for("forgot_password"))
+
+@app.route("/n/password_reset", methods=['GET'])
+def password_reset():
+    """Entry point after user clicks link in E-mail"""
+    logger.debug("in password_reset request.url is:", request.url)
+
+    verification_code = request.args.get('code')
+    hmac = request.args.get('hm')
+
+    if verification_code:
+        user_email = check_verification_code(verification_code)
+        if user_email:
+            user_details = get_user_by_unique_column('email_address', user_email)
+            if user_details:
+                return render_template(
+                    "new_security/password_reset.html", user_encode=user_details["email_address"])
+            else:
+                flash("Invalid code: User no longer exists!", "error")
+        else:
+            flash("Invalid code: Password reset code does not exist or might have expired!", "error")
+    else:
+        return redirect(url_for("login"))
+
+@app.route("/n/password_reset_step2", methods=('POST',))
+def password_reset_step2():
+    """Handle confirmation E-mail for password reset"""
+    logger.debug("in password_reset request.url is:", request.url)
+
+    errors = []
+    user_email = request.form['user_encode']
+
+    password = request.form['password']
+    encoded_password = set_password(password)
+
+    set_user_attribute(user_id, "password", encoded_password)
+
+    flash("Password changed successfully. You can now sign in.", "alert-info")
+    response = make_response(redirect(url_for('login')))
+
+    return response
+
+def register_user(params):
+        thank_you_mode = False
+        errors = []
+        user_details = {}
+
+        user_details['email_address'] = params.get('email_address', '').encode("utf-8").strip()
+        if not (5 <= len(user_details['email_address']) <= 50):
+            errors.append('Email Address needs to be between 5 and 50 characters.')
+        else:
+            email_exists = get_user_by_unique_column("email_address", user_details['email_address'])
+            if email_exists:
+                errors.append('User already exists with that email')
+
+        user_details['full_name'] = params.get('full_name', '').encode("utf-8").strip()
+        if not (5 <= len(user_details['full_name']) <= 50):
+            errors.append('Full Name needs to be between 5 and 50 characters.')
+
+        user_details['organization'] = params.get('organization', '').encode("utf-8").strip()
+        if user_details['organization'] and not (5 <= len(user_details['organization']) <= 50):
+            errors.append('Organization needs to be empty or between 5 and 50 characters.')
+
+        password = str(params.get('password', ''))
+        if not (6 <= len(password)):
+            errors.append('Password needs to be at least 6 characters.')
+
+        if params.get('password_confirm') != password:
+            errors.append("Passwords don't match.")
+
+        if errors:
+            return errors
+
+        user_details['password'] = set_password(password)
+        user_details['user_id'] = str(uuid.uuid4())
+        user_details['confirmed'] = 1
+
+        user_details['registration_info'] = json.dumps(basic_info(), sort_keys=True)
+        save_user(user_details, user_details['user_id'])
+
+@app.route("/n/register", methods=('GET', 'POST'))
+def register():
+    errors = None
+
+    params = request.form if request.form else request.args
+    params = params.to_dict(flat=True)
+
+    if params:
+        logger.debug("Attempting to register the user...")
+        errors = register_user(params)
+
+        if len(errors) == 0:
+            flash("Registration successful. You may login with your new account", "alert-info")
+            return redirect(url_for("login"))
+
+    return render_template("new_security/register_user.html", values=params, errors=errors)
+
+@app.errorhandler(401)
+def unauthorized(error):
+    return redirect(url_for('login'))
\ No newline at end of file
diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py
new file mode 100644
index 00000000..1f3e6558
--- /dev/null
+++ b/wqflask/wqflask/user_session.py
@@ -0,0 +1,272 @@
+from __future__ import print_function, division, absolute_import
+
+import datetime
+import time
+import uuid
+
+import simplejson as json
+
+import redis # used for collections
+Redis = redis.StrictRedis()
+
+from flask import (Flask, g, render_template, url_for, request, make_response,
+                   redirect, flash, abort)
+
+from wqflask import app
+from wqflask.hmac_func import hmac_creation
+
+#from utility.elasticsearch_tools import get_elasticsearch_connection
+from utility.redis_tools import get_user_id, get_user_by_unique_column, get_user_collections, save_collections
+
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+THREE_DAYS = 60 * 60 * 24 * 3
+THIRTY_DAYS = 60 * 60 * 24 * 30
+
+def verify_cookie(cookie):
+    the_uuid, separator, the_signature = cookie.partition(':')
+    assert len(the_uuid) == 36, "Is session_id a uuid?"
+    assert separator == ":", "Expected a : here"
+    assert the_signature == hmac_creation(the_uuid), "Uh-oh, someone tampering with the cookie?"
+    return the_uuid
+
+def create_signed_cookie():
+    the_uuid = str(uuid.uuid4())
+    signature = hmac_creation(the_uuid)
+    uuid_signed = the_uuid + ":" + signature
+    logger.debug("uuid_signed:", uuid_signed)
+    return the_uuid, uuid_signed
+
+class UserSession(object):
+    """Logged in user handling"""
+
+    user_cookie_name = 'session_id_v1'
+    anon_cookie_name = 'anon_user_v1'
+
+    def __init__(self):
+        user_cookie = request.cookies.get(self.user_cookie_name)
+        if not user_cookie:
+            self.logged_in = False
+            anon_cookie = request.cookies.get(self.anon_cookie_name)
+            self.cookie_name = self.anon_cookie_name
+            if anon_cookie:
+                self.cookie = anon_cookie
+                session_id = verify_cookie(self.cookie)
+            else:
+                session_id, self.cookie = create_signed_cookie()
+        else:
+            self.cookie_name = self.user_cookie_name
+            self.cookie = user_cookie
+            session_id = verify_cookie(self.cookie)
+
+        self.redis_key = self.cookie_name + ":" + session_id
+        self.session_id = session_id
+        self.record = Redis.hgetall(self.redis_key)
+
+        #ZS: If user correctled logged in but their session expired
+        #ZS: Need to test this by setting the time-out to be really short or something
+        if not self.record:
+            if user_cookie:
+                self.logged_in = False
+
+                ########### Grrr...this won't work because of the way flask handles cookies
+                # Delete the cookie
+                response = make_response(redirect(url_for('login')))
+                #response.set_cookie(self.cookie_name, '', expires=0)
+                flash("Due to inactivity your session has expired. If you'd like please login again.")
+                return response
+                #return
+            else:
+                self.record = dict(login_time = time.time(),
+                                    user_type = "anon",
+                                    user_id = str(uuid.uuid4()))
+
+                Redis.hmset(self.redis_key, self.record)
+                Redis.expire(self.redis_key, THIRTY_DAYS)
+        else:
+            if user_cookie:
+                self.logged_in = True
+
+        if user_cookie:
+            session_time = THREE_DAYS
+        else:
+            session_time = THIRTY_DAYS
+
+        if Redis.ttl(self.redis_key) < session_time:
+            # (Almost) everytime the user does something we extend the session_id in Redis...
+            logger.debug("Extending ttl...")
+            Redis.expire(self.redis_key, session_time)
+
+    @property
+    def user_id(self):
+        """Shortcut to the user_id"""
+        if 'user_id' in self.record:
+            return self.record['user_id']
+        else:
+            return ''
+
+    @property
+    def redis_user_id(self):
+        """User id from Redis (need to check if this is the same as the id stored in self.records)"""
+
+        #ZS: This part is a bit weird. Some accounts used to not have saved user ids, and in the process of testing I think I created some duplicate accounts for myself.
+        #ZS: Accounts should automatically generate user_ids if they don't already have one now, so this might not be necessary for anything other than my account's collections
+
+        if 'user_email_address' in self.record:
+            user_email = self.record['user_email_address']
+
+            #ZS: Get user's collections if they exist
+            user_id = None
+            user_id = get_user_id("email_address", user_email)
+        elif 'user_id' in self.record:
+            user_id = self.record['user_id']
+        elif 'github_id' in self.record:
+            user_github_id = self.record['github_id']
+            user_id = None
+            user_id = get_user_id("github_id", user_github_id)
+        else: #ZS: Anonymous user
+            return None
+
+        return user_id
+
+    @property
+    def user_name(self):
+        """Shortcut to the user_name"""
+        if 'user_name' in self.record:
+            return self.record['user_name']
+        else:
+            return ''
+
+    @property
+    def user_collections(self):
+        """List of user's collections"""
+
+        #ZS: Get user's collections if they exist
+        collections = get_user_collections(self.redis_user_id)
+        return collections
+
+    @property
+    def num_collections(self):
+        """Number of user's collections"""
+
+        return len(self.user_collections)
+
+    def add_collection(self, collection_name, traits):
+        """Add collection into ElasticSearch"""
+
+        collection_dict = {'id': unicode(uuid.uuid4()),
+                           'name': collection_name,
+                           'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
+                           'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
+                           'num_members': len(traits),
+                           'members': list(traits) }
+
+        current_collections = self.user_collections
+        current_collections.append(collection_dict)
+        self.update_collections(current_collections)
+
+        return collection_dict['id']
+
+    def change_collection_name(self, collection_id, new_name):
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                collection['name'] = new_name
+                break
+
+        return new_name
+
+    def delete_collection(self, collection_id):
+        """Remove collection with given ID"""
+
+        updated_collections = []
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                continue
+            else:
+                updated_collections.append(collection)
+
+        self.update_collections(updated_collections)
+
+        return collection['name']
+
+    def add_traits_to_collection(self, collection_id, traits_to_add):
+        """Add specified traits to a collection"""
+
+        this_collection = self.get_collection_by_id(collection_id)
+
+        updated_collection = this_collection
+        updated_traits = this_collection['members'] + traits_to_add
+
+        updated_collection['members'] = updated_traits
+        updated_collection['num_members'] = len(updated_traits)
+        updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
+
+        updated_collections = []
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                updated_collections.append(updated_collection)
+            else:
+                updated_collections.append(collection)
+
+        self.update_collections(updated_collections)
+
+    def remove_traits_from_collection(self, collection_id, traits_to_remove):
+        """Remove specified traits from a collection"""
+
+        this_collection = self.get_collection_by_id(collection_id)
+
+        updated_collection = this_collection
+        updated_traits = []
+        for trait in this_collection['members']:
+            if trait in traits_to_remove:
+                continue
+            else:
+                updated_traits.append(trait)
+
+        updated_collection['members'] = updated_traits
+        updated_collection['num_members'] = len(updated_traits)
+        updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
+
+        updated_collections = []
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                updated_collections.append(updated_collection)
+            else:
+                updated_collections.append(collection)
+
+        self.update_collections(updated_collections)
+
+        return updated_traits
+
+    def get_collection_by_id(self, collection_id):
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                return collection
+
+    def get_collection_by_name(self, collection_name):
+        for collection in self.user_collections:
+            if collection['name'] == collection_name:
+                return collection
+
+        return None
+
+    def update_collections(self, updated_collections):
+        collection_body = json.dumps(updated_collections)
+
+        save_collections(self.redis_user_id, collection_body)
+
+    def delete_session(self):
+        # And more importantly delete the redis record
+        Redis.delete(self.redis_key)
+        self.logged_in = False
+
+@app.before_request
+def before_request():
+    g.user_session = UserSession()
+
+@app.after_request
+def set_cookie(response):
+    if not request.cookies.get(g.user_session.cookie_name):
+        response.set_cookie(g.user_session.cookie_name, g.user_session.cookie)
+    return response
\ No newline at end of file
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index a84a7480..436ebc91 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -65,9 +65,8 @@ from utility.benchmark import Bench
 
 from pprint import pformat as pf
 
-#from wqflask import user_login
-#from wqflask import user_session
-from wqflask import user_manager
+from wqflask import user_login
+from wqflask import user_session
 
 from wqflask import collect
 from wqflask.database import db_session
@@ -839,6 +838,8 @@ def browser_inputs():
 
     return flask.jsonify(file_contents)
 
+
+
 ##########################################################################
 
 def json_default_handler(obj):
-- 
cgit v1.2.3


From 8839637fbcb1b083e19367801ca7646962944d30 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 3 Mar 2020 15:16:31 -0600
Subject: was missing a couple files in last commit

---
 wqflask/utility/hmac.py      | 18 ++++++++++++++++++
 wqflask/wqflask/hmac_func.py | 19 +++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 wqflask/utility/hmac.py
 create mode 100644 wqflask/wqflask/hmac_func.py

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py
new file mode 100644
index 00000000..47001e54
--- /dev/null
+++ b/wqflask/utility/hmac.py
@@ -0,0 +1,18 @@
+from __future__ import print_function, division, absolute_import
+
+import hmac
+
+from wqflask import app
+
+def hmac_creation(stringy):
+    """Helper function to create the actual hmac"""
+
+    secret = app.config['SECRET_HMAC_CODE']
+
+    hmaced = hmac.new(secret, stringy, hashlib.sha1)
+    hm = hmaced.hexdigest()
+    # ZS: Leaving the below comment here to ask Pjotr about
+    # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output."
+    # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html
+    hm = hm[:20]
+    return hm
\ No newline at end of file
diff --git a/wqflask/wqflask/hmac_func.py b/wqflask/wqflask/hmac_func.py
new file mode 100644
index 00000000..361c35b6
--- /dev/null
+++ b/wqflask/wqflask/hmac_func.py
@@ -0,0 +1,19 @@
+from __future__ import print_function, division, absolute_import
+
+import hashlib
+import hmac
+
+from wqflask import app
+
+def hmac_creation(stringy):
+    """Helper function to create the actual hmac"""
+
+    secret = app.config['SECRET_HMAC_CODE']
+
+    hmaced = hmac.new(secret, stringy, hashlib.sha1)
+    hm = hmaced.hexdigest()
+    # ZS: Leaving the below comment here to ask Pjotr about
+    # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output."
+    # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html
+    hm = hm[:20]
+    return hm
\ No newline at end of file
-- 
cgit v1.2.3


From 2ed0e4ed2ff000c0378ebfe828d015a662ab52a9 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 3 Mar 2020 15:40:34 -0600
Subject: Missed one more reference in helper_functions

---
 wqflask/utility/helper_functions.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py
index 6980af4e..971dd3cc 100644
--- a/wqflask/utility/helper_functions.py
+++ b/wqflask/utility/helper_functions.py
@@ -4,7 +4,7 @@ from base.trait import GeneralTrait
 from base import data_set
 from base.species import TheSpecies
 
-from wqflask import user_manager
+from utility import hmac
 
 from flask import Flask, g
 
@@ -43,7 +43,7 @@ def get_trait_db_obs(self, trait_db_list):
     for trait in trait_db_list:
         data, _separator, hmac = trait.rpartition(':')
         data = data.strip()
-        assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?"
+        assert hmac==hmac.hmac_creation(data), "Data tampering?"
         trait_name, dataset_name = data.split(":")
         if dataset_name == "Temp":
             dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2])
-- 
cgit v1.2.3


From 04b32f95279602da4cfb00b4f356a04eee48af51 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Mon, 9 Mar 2020 11:54:34 -0500
Subject: I think this should complete consolidating all the collections code.

---
 wqflask/base/trait.py                              |   8 +-
 wqflask/utility/hmac.py                            |  56 ++-
 wqflask/wqflask/collect.py                         | 247 ++-------
 wqflask/wqflask/correlation/corr_scatter_plot.py   |   9 +-
 wqflask/wqflask/gsearch.py                         |   4 +-
 wqflask/wqflask/search_results.py                  |   2 +-
 wqflask/wqflask/templates/base.html                |   4 -
 wqflask/wqflask/templates/collections/list.html    |  24 -
 wqflask/wqflask/templates/collections/view.html    |   9 +-
 .../wqflask/templates/new_security/login_user.html |   1 +
 .../templates/show_trait_mapping_tools.html        |  11 -
 .../wqflask/templates/show_trait_statistics.html   |   6 -
 wqflask/wqflask/user_login.py                      |   8 +-
 wqflask/wqflask/user_session.py                    | 551 +++++++++++----------
 wqflask/wqflask/views.py                           |   2 +-
 15 files changed, 375 insertions(+), 567 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index c9dfe780..8e779a11 100644
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -303,7 +303,7 @@ def jsonable_table_row(trait, dataset_name, index):
             additive = "N/A"
         else:
             additive = "%.3f" % round(float(trait.additive), 2)
-        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.hmac_creation('{}:{}'.format(str(trait.name), dataset.name)) + '">',
+        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                 index,
                 '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                 trait.symbol,
@@ -319,7 +319,7 @@ def jsonable_table_row(trait, dataset_name, index):
         else:
             additive = "%.2f" % round(float(trait.additive), 2)
         if trait.pubmed_id:
-            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.hmac_creation('{}:{}'.format(str(trait.name), dataset.name)) + '">',
+            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                     index,
                     '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                     trait.description_display,
@@ -329,7 +329,7 @@ def jsonable_table_row(trait, dataset_name, index):
                     trait.LRS_location_repr,
                     additive]
         else:
-            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.hmac_creation('{}:{}'.format(str(trait.name), dataset.name)) + '">',
+            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                     index,
                     '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                     trait.description_display,
@@ -339,7 +339,7 @@ def jsonable_table_row(trait, dataset_name, index):
                     trait.LRS_location_repr,
                     additive]
     elif dataset.type == "Geno":
-        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.hmac_creation('{}:{}'.format(str(trait.name), dataset.name)) + '">',
+        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                 index,
                 '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                 trait.location_repr]
diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py
index 47001e54..d8a0eace 100644
--- a/wqflask/utility/hmac.py
+++ b/wqflask/utility/hmac.py
@@ -1,18 +1,38 @@
-from __future__ import print_function, division, absolute_import
-
-import hmac
-
-from wqflask import app
-
-def hmac_creation(stringy):
-    """Helper function to create the actual hmac"""
-
-    secret = app.config['SECRET_HMAC_CODE']
-
-    hmaced = hmac.new(secret, stringy, hashlib.sha1)
-    hm = hmaced.hexdigest()
-    # ZS: Leaving the below comment here to ask Pjotr about
-    # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output."
-    # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html
-    hm = hm[:20]
-    return hm
\ No newline at end of file
+from __future__ import print_function, division, absolute_import
+
+import hmac
+import hashlib
+
+from wqflask import app
+
+def hmac_creation(stringy):
+    """Helper function to create the actual hmac"""
+
+    secret = app.config['SECRET_HMAC_CODE']
+
+    hmaced = hmac.new(secret, stringy, hashlib.sha1)
+    hm = hmaced.hexdigest()
+    # ZS: Leaving the below comment here to ask Pjotr about
+    # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output."
+    # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html
+    hm = hm[:20]
+    return hm
+
+def data_hmac(stringy):
+    """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with"""
+    return stringy + ":" + hmac_creation(stringy)
+
+def url_for_hmac(endpoint, **values):
+    """Like url_for but adds an hmac at the end to insure the url hasn't been tampered with"""
+
+    url = url_for(endpoint, **values)
+
+    hm = hmac_creation(url)
+    if '?' in url:
+        combiner = "&"
+    else:
+        combiner = "?"
+    return url + combiner + "hm=" + hm
+
+app.jinja_env.globals.update(url_for_hmac=url_for_hmac,
+                             data_hmac=data_hmac)
\ No newline at end of file
diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py
index c273af59..e5c24873 100644
--- a/wqflask/wqflask/collect.py
+++ b/wqflask/wqflask/collect.py
@@ -8,7 +8,6 @@ import time
 
 import uuid
 import hashlib
-import hmac
 import base64
 
 import urlparse
@@ -28,9 +27,8 @@ from pprint import pformat as pf
 from wqflask.database import db_session
 
 from wqflask import model
-from wqflask import user_manager
 
-from utility import Bunch, Struct
+from utility import Bunch, Struct, hmac
 from utility.formatting import numify
 
 from base import trait
@@ -40,119 +38,15 @@ import logging
 from utility.logger import getLogger
 logger = getLogger(__name__)
 
-
-class AnonCollection(object):
-    """User is not logged in"""
-    def __init__(self, collection_name):
-        anon_user = user_manager.AnonUser()
-        self.key = anon_user.key
-        self.name = collection_name
-        self.id = None
-        self.created_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
-        self.changed_timestamp = self.created_timestamp #ZS: will be updated when changes are made
-
-        #ZS: Find id and set it if the collection doesn't already exist
-        if Redis.get(self.key) == "None" or Redis.get(self.key) == None:
-            Redis.set(self.key, "None") #ZS: For some reason I get the error "Operation against a key holding the wrong kind of value" if I don't do this
-        else:
-            collections_list = json.loads(Redis.get(self.key))
-            collection_position = 0 #ZS: Position of collection in collection_list, if it exists
-            collection_exists = False
-            for i, collection in enumerate(collections_list):
-                if collection['name'] == self.name:
-                    collection_position = i
-                    collection_exists = True
-                    self.id = collection['id']
-                    break
-
-        if self.id == None:
-            self.id = str(uuid.uuid4())
-
-    def get_members(self):
-        traits = []
-        collections_list = json.loads(Redis.get(self.key))
-        for collection in collections_list:
-            if collection['id'] == self.id:
-                traits = collection['members']
-        return traits
-
-    @property
-    def num_members(self):
-        num_members = 0
-        collections_list = json.loads(Redis.get(self.key))
-        for collection in collections_list:
-            if collection['id'] == self.id:
-                num_members = collection['num_members']
-        return num_members
-
-    def add_traits(self, unprocessed_traits):
-        #assert collection_name == "Default", "Unexpected collection name for anonymous user"
-        self.traits = list(process_traits(unprocessed_traits))
-        existing_collections = Redis.get(self.key)
-        logger.debug("existing_collections:", existing_collections)
-        if existing_collections != None and existing_collections != "None":
-            collections_list = json.loads(existing_collections)
-            collection_position = 0 #ZS: Position of collection in collection_list, if it exists
-            collection_exists = False
-            for i, collection in enumerate(collections_list):
-                if collection['id'] == self.id:
-                    collection_position = i
-                    collection_exists = True
-                    break
-            if collection_exists:
-                collections_list[collection_position]['members'] += list(set(self.traits) - set(collections_list[collection_position]['members']))
-                collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members'])
-                collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
-            else:
-                collection_dict = {"id" : self.id,
-                                   "name" : self.name,
-                                   "created_timestamp" : self.created_timestamp,
-                                   "changed_timestamp" : self.changed_timestamp,
-                                   "num_members" : len(self.traits),
-                                   "members" : self.traits}
-                collections_list.append(collection_dict)
-        else:
-            collections_list = []
-            collection_dict = {"id" : self.id,
-                               "name" : self.name,
-                               "created_timestamp" : self.created_timestamp,
-                               "changed_timestamp" : self.changed_timestamp,
-                               "num_members" : len(self.traits),
-                               "members" : self.traits}
-            collections_list.append(collection_dict)
-
-        Redis.set(self.key, json.dumps(collections_list))
-
-    def remove_traits(self, params):
-        traits_to_remove = [(":").join(trait.split(":")[:2]) for trait in params.getlist('traits[]')]
-        existing_collections = Redis.get(self.key)
-        collection_position = 0
-        collections_list = json.loads(existing_collections)
-        for i, collection in enumerate(collections_list):
-            if collection['id'] == self.id:
-                collection_position = i
-                collection_exists = True
-                break
-        collections_list[collection_position]['members'] = [trait for trait in collections_list[collection_position]['members'] if trait not in traits_to_remove]
-        collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members'])
-        collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
-        len_now = collections_list[collection_position]['num_members']
-        Redis.set(self.key, json.dumps(collections_list))
-
-        # We need to return something so we'll return this...maybe in the future
-        # we can use it to check the results
-        return str(len_now)
-
 def process_traits(unprocessed_traits):
     if isinstance(unprocessed_traits, basestring):
         unprocessed_traits = unprocessed_traits.split(",")
     traits = set()
     for trait in unprocessed_traits:
-        #print("trait is:", trait)
         data, _separator, hmac = trait.rpartition(':')
         data = data.strip()
         if g.user_session.logged_in:
-          assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?"
+          assert hmac == hmac.data_hmac(data), "Data tampering?"
         traits.add(str(data))
 
     return traits
@@ -160,8 +54,6 @@ def process_traits(unprocessed_traits):
 def report_change(len_before, len_now):
     new_length = len_now - len_before
     if new_length:
-        logger.debug("We've added {} to your collection.".format(
-            numify(new_length, 'new trait', 'new traits')))
         flash("We've added {} to your collection.".format(
             numify(new_length, 'new trait', 'new traits')))
     else:
@@ -180,18 +72,14 @@ def store_traits_list():
 
 @app.route("/collections/add")
 def collections_add():
-    if g.user_session.logged_in:
+
+    collections = g.user_session.user_collections
+    if len(collections) < 1:
+        collection_name = "Default Collection"
+        uc_id = g.user_session.add_collection(collection_name, set())
         collections = g.user_session.user_collections
-        if len(collections) < 1:
-            collection_name = "Default Collection"
-            uc_id = g.user_session.add_collection(collection_name, set())
-            collections = g.user_session.user_collections
-    else:
-        anon_collections = user_manager.AnonUser().get_collections()
-        collections = []
-        for collection in anon_collections:
-            collections.append({'id':collection['id'], 'name':collection['name']})
 
+    #ZS: One of these might be unnecessary
     if 'traits' in request.args:
         traits=request.args['traits']
         return render_template("collections/add.html",
@@ -212,19 +100,13 @@ def collections_new():
     if "sign_in" in params:
         return redirect(url_for('login'))
     if "create_new" in params:
-        logger.debug("in create_new")
         collection_name = params['new_collection']
         if collection_name.strip() == "":
             collection_name = datetime.datetime.utcnow().strftime('Collection_%b_%d_%H:%M')
         return create_new(collection_name)
     elif "add_to_existing" in params:
-        logger.debug("in add to existing")
         if 'existing_collection' not in params:
-            default_collection_exists = False
-            if g.user_session.logged_in:
-                collections = g.user_session.user_collections
-            else:
-                collections = user_manager.AnonUser().get_collections()
+            collections = g.user_session.user_collections
             for collection in collections:
                 if collection["name"] == "Default Collection":
                     collection_id = collection["id"]
@@ -235,22 +117,14 @@ def collections_new():
         else:
             collection_id = params['existing_collection'].split(":")[0]
             collection_name = params['existing_collection'].split(":")[1]
-        if g.user_session.logged_in:
-            if "hash" in params:
-                unprocessed_traits = Redis.get(params['hash'])
-            else:
-                unprocessed_traits = params['traits']
-            traits = list(process_traits(unprocessed_traits))
-            g.user_session.add_traits_to_collection(collection_id, traits)
-            return redirect(url_for('view_collection', uc_id=collection_id))
+
+        if "hash" in params:
+            unprocessed_traits = Redis.get(params['hash'])
         else:
-            ac = AnonCollection(collection_name)
-            if "hash" in params:
-                unprocessed_traits = Redis.get(params['hash'])
-            else:
-                unprocessed_traits = params['traits']
-            ac.add_traits(unprocessed_traits)
-            return redirect(url_for('view_collection', collection_id=ac.id))
+            unprocessed_traits = params['traits']
+        traits = list(process_traits(unprocessed_traits))
+        g.user_session.add_traits_to_collection(collection_id, traits)
+        return redirect(url_for('view_collection', uc_id=collection_id))
     else:
         CauseAnError
 
@@ -265,50 +139,30 @@ def create_new(collection_name):
 
     traits = process_traits(unprocessed_traits)
 
-    if g.user_session.logged_in:
-        uc_id = g.user_session.add_collection(collection_name, traits)
+    uc_id = g.user_session.add_collection(collection_name, traits)
 
-        return redirect(url_for('view_collection', uc_id=uc_id))
-    else:
-        ac = AnonCollection(collection_name)
-        ac.changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
-        ac.add_traits(unprocessed_traits)
-        return redirect(url_for('view_collection', collection_id=ac.id))
+    return redirect(url_for('view_collection', uc_id=uc_id))
 
 @app.route("/collections/list")
 def list_collections():
     params = request.args
 
-    if g.user_session.logged_in:
-        user_collections = list(g.user_session.user_collections)
-        #logger.debug("user_collections are:", user_collections)
-        return render_template("collections/list.html",
-                               params = params,
-                               collections = user_collections,
-                               )
-    else:
-        anon_collections = user_manager.AnonUser().get_collections()
-        #logger.debug("anon_collections are:", anon_collections)
-        return render_template("collections/list.html",
-                               params = params,
-                               collections = anon_collections)
-
+    user_collections = list(g.user_session.user_collections)
+    return render_template("collections/list.html",
+                            params = params,
+                            collections = user_collections,
+                            )
 
 @app.route("/collections/remove", methods=('POST',))
 def remove_traits():
     params = request.form
 
-    if "uc_id" in params:
-        uc_id = params['uc_id']
-        traits_to_remove = params.getlist('traits[]')
-        traits_to_remove = process_traits(traits_to_remove)
-        logger.debug("\n\n  after processing, traits_to_remove:", traits_to_remove)
-
-        members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove)
-    else:
-        collection_name = params['collection_name']
-        members_now = AnonCollection(collection_name).remove_traits(params)
+    uc_id = params['uc_id']
+    traits_to_remove = params.getlist('traits[]')
+    traits_to_remove = process_traits(traits_to_remove)
+    logger.debug("\n\n  after processing, traits_to_remove:", traits_to_remove)
 
+    members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove)
 
     # We need to return something so we'll return this...maybe in the future
     # we can use it to check the results
@@ -319,21 +173,13 @@ def remove_traits():
 def delete_collection():
     params = request.form
     uc_id = ""
-    if g.user_session.logged_in:
-        uc_id = params['uc_id']
-        if len(uc_id.split(":")) > 1:
-            for this_uc_id in uc_id.split(":"):
-                collection_name = g.user_session.delete_collection(this_uc_id)
-        else:
-            collection_name = g.user_session.delete_collection(uc_id)
+
+    uc_id = params['uc_id']
+    if len(uc_id.split(":")) > 1:
+        for this_uc_id in uc_id.split(":"):
+            collection_name = g.user_session.delete_collection(this_uc_id)
     else:
-        if "collection_name" in params:
-            collection_name = params['collection_name']
-            user_manager.AnonUser().delete_collection(collection_name)
-        else:
-            uc_id = params['uc_id']
-            for this_collection in uc_id.split(":"):
-                user_manager.AnonUser().delete_collection(this_collection)
+        collection_name = g.user_session.delete_collection(uc_id)
 
     if uc_id != "":
         if len(uc_id.split(":")) > 1:
@@ -350,19 +196,9 @@ def delete_collection():
 def view_collection():
     params = request.args
 
-    if g.user_session.logged_in and "uc_id" in params:
-        uc_id = params['uc_id']
-        uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next()
-        traits = uc["members"]
-    else:
-        user_collections = json.loads(Redis.get(user_manager.AnonUser().key))
-        this_collection = {}
-        for collection in user_collections:
-            if collection['id'] == params['collection_id']:
-                this_collection = collection
-                break
-
-        traits = this_collection['members']
+    uc_id = params['uc_id']
+    uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next()
+    traits = uc["members"]
 
     trait_obs = []
     json_version = []
@@ -381,12 +217,9 @@ def view_collection():
 
         json_version.append(trait.jsonable(trait_ob))
 
-    if "uc_id" in params:
-        collection_info = dict(trait_obs=trait_obs,
-                               uc = uc)
-    else:
-        collection_info = dict(trait_obs=trait_obs,
-                               collection_name=this_collection['name'])
+    collection_info = dict(trait_obs=trait_obs,
+                           uc = uc)
+
     if "json" in params:
         return json.dumps(json_version)
     else:
diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py
index f8eca386..dfb81c54 100644
--- a/wqflask/wqflask/correlation/corr_scatter_plot.py
+++ b/wqflask/wqflask/correlation/corr_scatter_plot.py
@@ -71,15 +71,8 @@ class CorrScatterPlot(object):
 
         sr_intercept_coords = get_intercept_coords(srslope, srintercept, sr_range, sr_range)
 
-        #vals_3 = []
-        #for sample in self.trait_3.data:
-        #    vals_3.append(self.trait_3.data[sample].value)
-
         self.collections_exist = "False"
-        if g.user_session.logged_in:
-            if g.user_session.num_collections > 0:
-                self.collections_exist = "True"
-        elif g.cookie_session.display_num_collections() != "":
+        if g.user_session.num_collections > 0:
             self.collections_exist = "True"
 
         self.js_data = dict(
diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py
index ed3b74b8..12813e9a 100644
--- a/wqflask/wqflask/gsearch.py
+++ b/wqflask/wqflask/gsearch.py
@@ -70,7 +70,7 @@ class GSearch(object):
                     this_trait['name'] = line[5]
                     this_trait['dataset'] = line[3]
                     this_trait['dataset_fullname'] = line[4]
-                    this_trait['hmac'] = hmac.hmac_creation('{}:{}'.format(line[5], line[3]))
+                    this_trait['hmac'] = hmac.data_hmac('{}:{}'.format(line[5], line[3]))
                     this_trait['species'] = line[0]
                     this_trait['group'] = line[1]
                     this_trait['tissue'] = line[2]
@@ -174,7 +174,7 @@ class GSearch(object):
                         this_trait['display_name'] = this_trait['name']
                     this_trait['dataset'] = line[2]
                     this_trait['dataset_fullname'] = line[3]
-                    this_trait['hmac'] = hmac.hmac_creation('{}:{}'.format(line[4], line[2]))
+                    this_trait['hmac'] = hmac.data_hmac('{}:{}'.format(line[4], line[2]))
                     this_trait['species'] = line[0]
                     this_trait['group'] = line[1]
                     if line[9] != None and line[6] != None:
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index 902317a4..34c647f4 100644
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -120,7 +120,7 @@ views.py).
             else:
                 trait_dict['display_name'] = this_trait.name
             trait_dict['dataset'] = this_trait.dataset.name
-            trait_dict['hmac'] = hmac.hmac_creation('{}:{}'.format(this_trait.name, this_trait.dataset.name))
+            trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name))
             if this_trait.dataset.type == "ProbeSet":
                 trait_dict['symbol'] = this_trait.symbol
                 trait_dict['description'] = this_trait.description_display.decode('utf-8', 'replace')
diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html
index 04cf718a..2c95bf5e 100644
--- a/wqflask/wqflask/templates/base.html
+++ b/wqflask/wqflask/templates/base.html
@@ -73,11 +73,7 @@
                         </li>
                         <li class="">
                             <a href="/collections/list">Collections
-                                {% if g.user_session.logged_in %}
                                 <span class="badge badge-info">{{ g.user_session.num_collections }}</span>
-                                {% else %}
-                                <span class="badge badge-info">{{ g.cookie_session.display_num_collections() }}</span>
-                                {% endif %}
                             </a>
                         </li>
                         <li class="">
diff --git a/wqflask/wqflask/templates/collections/list.html b/wqflask/wqflask/templates/collections/list.html
index 19aec8ae..eb3fc061 100644
--- a/wqflask/wqflask/templates/collections/list.html
+++ b/wqflask/wqflask/templates/collections/list.html
@@ -15,17 +15,6 @@
         <h1>Your Collections</h1>
         {% endif %}
 
-        <!-- <hr style="height: 1px; background-color: #A9A9A9;">-->
-
-        <!--
-        <div class="page-header">
-            {% if g.user_session.logged_in %}
-            <h1>Collections owned by {{ g.user_session.user_name }}</h1>
-            {% else %}
-            <h1>Your Collections</h1>
-            {% endif %}
-        </div>
-        -->
         <div>
             <form id="collections_form" action="/delete" method="post">
               <input type="hidden" name="uc_id" id="uc_id" value="" />
@@ -55,24 +44,11 @@
             <tbody>
             {% for uc in collections %}
                 <tr class="collection_line">
-                    {% if g.user_session.logged_in %}
                     <td align="center" style="padding: 0px;"><INPUT TYPE="checkbox" NAME="collection" class="checkbox trait_checkbox" VALUE="{{ uc.id }}"></td>
-                    {% else %}
-                    <td align="center" style="padding: 0px;"><INPUT TYPE="checkbox" NAME="collection" class="checkbox trait_checkbox" VALUE="{{ uc.name }}"></td>
-                    {% endif %}
                     <td align="right">{{ loop.index }}
-                    {% if g.user_session.logged_in %}
                     <td><a class="collection_name" href="{{ url_for('view_collection', uc_id=uc.id) }}">{{ uc.name }}</a></td>
-                    {% else %}
-                    <td><a class="collection_name" href="{{ url_for('view_collection', collection_id=uc.id) }}">{{ uc.name }}</a></td>
-                    {% endif %}
-                    {% if g.user_session.logged_in %}
                     <td>{{ uc.created_timestamp }}</td>
                     <td>{{ uc.changed_timestamp }}</td>
-                    {% else %}
-                    <td>{{ timeago(uc.created_timestamp.isoformat() + "Z") }}</td>
-                    <td>{{ timeago(uc.changed_timestamp.isoformat() + "Z") }}</td>
-                    {% endif %}
                     <td align="right">{{ uc.num_members }}</td>
                 </tr>
             {% endfor %}
diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html
index 640eb561..fd0025d9 100644
--- a/wqflask/wqflask/templates/collections/view.html
+++ b/wqflask/wqflask/templates/collections/view.html
@@ -9,20 +9,15 @@
 <!-- Start of body -->
 
     <div class="container" style="min-width: 1250px;">
-        {% if uc %}
-        <h1>{{ uc.name }}</h1>
-        {% else %}
-        <h1>{{ collection_name }}</h1>
-        {% endif %}
+        <h1><input type="text" id="change_collection_name" style="display: none; width: 500px;" class="form-control" placeholder="{{ uc.name }}"></h1>
+        <h1>{{ uc.name }} <!--<button class="btn btn-default" style="margin-bottom: 5px;" id="change_collection_name">Change Collection Name</button>--></h1>
         <h3>This collection has {{ '{}'.format(numify(trait_obs|count, "record", "records")) }}</h3>
 
         <!--<hr style="height: 1px; background-color: #A9A9A9;">-->
 
         <div>
           <form id="collection_form" action="/delete" method="post">
-            {% if uc %}
             <input type="hidden" name="uc_id" id="uc_id" value="{{ uc.id }}" />
-            {% endif %}
             <input type="hidden" name="collection_name" id="collection_name" value="{{ collection_name }}" />
             <input type="hidden" name="tool_used" value="" />
             <input type="hidden" name="form_url" value="" />
diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html
index 9df0e16a..80fed82a 100644
--- a/wqflask/wqflask/templates/new_security/login_user.html
+++ b/wqflask/wqflask/templates/new_security/login_user.html
@@ -10,6 +10,7 @@
 
 	    {% if redis_is_available: %}
             <form class="form-horizontal" action="/n/login" method="POST" name="login_user_form" id="loginUserForm">
+                 <input name="anon_id" type="hidden" value="{{ g.user_session.user_id }}">
                  <fieldset>
                     <div class="form-group">
                         <label style="text-align:left;" class="col-xs-1 control-label" for="email_address">Email&nbsp;Address</label>
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index 841d6ad5..a806a8b3 100644
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -81,7 +81,6 @@
                             <font size="2">Select covariate(s) from a collection</font>
                             </div>
                             <div style="margin-left:20px;" class="col-xs-7">
-                              {% if g.user_session.logged_in %}
                               {% if g.user_session.num_collections < 1 %}
                               No collections available. Please add traits to a collection to use them as covariates.
                               {% else %}
@@ -91,16 +90,6 @@
                               </div>
                               <textarea rows="3" cols="50" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none; width: 200px;" class="selected_covariates"></textarea>
                               {% endif %}
-                              {% elif g.cookie_session.display_num_collections() == "" %}
-                              No collections available. Please add traits to a collection to use them as covariates.
-                              {% else %}
-                              <div style="margin-bottom: 10px;" class="btn-group" role="group">
-                                <button type="button" id="select_covariates" class="btn btn-default">Select</button>
-                                <button type="button" id="remove_covariates" class="btn btn-default">Remove</button>
-                              </div>
-                              <br>
-                              <textarea rows="3" cols="30" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none;" class="selected_covariates"></textarea>
-                              {% endif %}
                             </div>
                         </div>
                         <div class="mapping_method_fields form-group">
diff --git a/wqflask/wqflask/templates/show_trait_statistics.html b/wqflask/wqflask/templates/show_trait_statistics.html
index 4653b398..974081d3 100644
--- a/wqflask/wqflask/templates/show_trait_statistics.html
+++ b/wqflask/wqflask/templates/show_trait_statistics.html
@@ -21,11 +21,9 @@
                 <li>
                     <a href="#violin_plot_tab" class="violin_plot_tab" data-toggle="tab">Violin Plot</a>
                 </li>
-                {% if g.user_session.logged_in %}
                 <li>
                     <a href="#scatterplot_matrix" data-toggle="tab">Scatterplot Matrix</a>
                 </li>
-                {% endif %}
             </ul>
 
             <div class="tab-content">
@@ -78,13 +76,11 @@
                             <i class="icon-signal"></i> Sort By Value
                             </button>
                         </div>
-                        {% if g.user_session.logged_in %}
                         <div class="btn-group">
                             <button type="button" class="btn btn-default" id="color_by_trait">
                                 <i class="icon-tint"></i> Color by Trait
                             </button>
                         </div>
-                        {% endif %}
                         <div id="bar_chart_container">
                             <div id="bar_chart"></div>
                         </div>
@@ -126,7 +122,6 @@
                         <div id="violin_plot"></div>
                     </div>
                 </div>
-                {% if g.user_session.logged_in %}
                 <div class="tab-pane" id="scatterplot_matrix">
                     
                     <div style="margin-bottom:40px;" class="btn-group">
@@ -141,7 +136,6 @@
                         <div id="comparison_scatterplot" class="qtlcharts"></div>
                     </div>
                 </div>
-                {% endif %}
             </div>
         </div>
     <!--</div>-->
diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py
index 05885e2c..da3cc504 100644
--- a/wqflask/wqflask/user_login.py
+++ b/wqflask/wqflask/user_login.py
@@ -21,9 +21,9 @@ from flask import (Flask, g, render_template, url_for, request, make_response,
 
 from wqflask import app
 from wqflask import pbkdf2
-from wqflask.hmac_func import hmac_creation
 from wqflask.user_session import UserSession
 
+from utility import hmac
 from utility.redis_tools import is_redis_available, get_user_id, get_user_by_unique_column, set_user_attribute, save_user, save_verification_code, check_verification_code, get_user_collections, save_collections
 
 from utility.logger import getLogger
@@ -83,7 +83,7 @@ def encrypt_password(unencrypted_password, pwfields):
 def get_signed_session_id(user):
     session_id = str(uuid.uuid4())
 
-    session_id_signature = hmac_creation(session_id)
+    session_id_signature = hmac.hmac_creation(session_id)
     session_id_signed = session_id + ":" + session_id_signature
 
     #ZS: Need to check if this is ever actually used or exists
@@ -197,12 +197,14 @@ def login():
             if password_match: # If password correct
                 if user_details['confirmed']: # If account confirmed
                     import_col = "false"
+                    anon_id = ""
                     if 'import_collections' in params:
                         import_col = "true"
+                        anon_id = params['anon_id']
 
                     session_id_signed = get_signed_session_id(user_details)
                     flash("Thank you for logging in {}.".format(user_details['full_name']), "alert-success")
-                    response = make_response(redirect(url_for('index_page', import_collections = import_col)))
+                    response = make_response(redirect(url_for('index_page', import_collections = import_col, anon_id = anon_id)))
                     response.set_cookie(UserSession.user_cookie_name, session_id_signed, max_age=None)
                     return response
                 else:
diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py
index 1f3e6558..e19ec9df 100644
--- a/wqflask/wqflask/user_session.py
+++ b/wqflask/wqflask/user_session.py
@@ -1,272 +1,281 @@
-from __future__ import print_function, division, absolute_import
-
-import datetime
-import time
-import uuid
-
-import simplejson as json
-
-import redis # used for collections
-Redis = redis.StrictRedis()
-
-from flask import (Flask, g, render_template, url_for, request, make_response,
-                   redirect, flash, abort)
-
-from wqflask import app
-from wqflask.hmac_func import hmac_creation
-
-#from utility.elasticsearch_tools import get_elasticsearch_connection
-from utility.redis_tools import get_user_id, get_user_by_unique_column, get_user_collections, save_collections
-
-from utility.logger import getLogger
-logger = getLogger(__name__)
-
-THREE_DAYS = 60 * 60 * 24 * 3
-THIRTY_DAYS = 60 * 60 * 24 * 30
-
-def verify_cookie(cookie):
-    the_uuid, separator, the_signature = cookie.partition(':')
-    assert len(the_uuid) == 36, "Is session_id a uuid?"
-    assert separator == ":", "Expected a : here"
-    assert the_signature == hmac_creation(the_uuid), "Uh-oh, someone tampering with the cookie?"
-    return the_uuid
-
-def create_signed_cookie():
-    the_uuid = str(uuid.uuid4())
-    signature = hmac_creation(the_uuid)
-    uuid_signed = the_uuid + ":" + signature
-    logger.debug("uuid_signed:", uuid_signed)
-    return the_uuid, uuid_signed
-
-class UserSession(object):
-    """Logged in user handling"""
-
-    user_cookie_name = 'session_id_v1'
-    anon_cookie_name = 'anon_user_v1'
-
-    def __init__(self):
-        user_cookie = request.cookies.get(self.user_cookie_name)
-        if not user_cookie:
-            self.logged_in = False
-            anon_cookie = request.cookies.get(self.anon_cookie_name)
-            self.cookie_name = self.anon_cookie_name
-            if anon_cookie:
-                self.cookie = anon_cookie
-                session_id = verify_cookie(self.cookie)
-            else:
-                session_id, self.cookie = create_signed_cookie()
-        else:
-            self.cookie_name = self.user_cookie_name
-            self.cookie = user_cookie
-            session_id = verify_cookie(self.cookie)
-
-        self.redis_key = self.cookie_name + ":" + session_id
-        self.session_id = session_id
-        self.record = Redis.hgetall(self.redis_key)
-
-        #ZS: If user correctled logged in but their session expired
-        #ZS: Need to test this by setting the time-out to be really short or something
-        if not self.record:
-            if user_cookie:
-                self.logged_in = False
-
-                ########### Grrr...this won't work because of the way flask handles cookies
-                # Delete the cookie
-                response = make_response(redirect(url_for('login')))
-                #response.set_cookie(self.cookie_name, '', expires=0)
-                flash("Due to inactivity your session has expired. If you'd like please login again.")
-                return response
-                #return
-            else:
-                self.record = dict(login_time = time.time(),
-                                    user_type = "anon",
-                                    user_id = str(uuid.uuid4()))
-
-                Redis.hmset(self.redis_key, self.record)
-                Redis.expire(self.redis_key, THIRTY_DAYS)
-        else:
-            if user_cookie:
-                self.logged_in = True
-
-        if user_cookie:
-            session_time = THREE_DAYS
-        else:
-            session_time = THIRTY_DAYS
-
-        if Redis.ttl(self.redis_key) < session_time:
-            # (Almost) everytime the user does something we extend the session_id in Redis...
-            logger.debug("Extending ttl...")
-            Redis.expire(self.redis_key, session_time)
-
-    @property
-    def user_id(self):
-        """Shortcut to the user_id"""
-        if 'user_id' in self.record:
-            return self.record['user_id']
-        else:
-            return ''
-
-    @property
-    def redis_user_id(self):
-        """User id from Redis (need to check if this is the same as the id stored in self.records)"""
-
-        #ZS: This part is a bit weird. Some accounts used to not have saved user ids, and in the process of testing I think I created some duplicate accounts for myself.
-        #ZS: Accounts should automatically generate user_ids if they don't already have one now, so this might not be necessary for anything other than my account's collections
-
-        if 'user_email_address' in self.record:
-            user_email = self.record['user_email_address']
-
-            #ZS: Get user's collections if they exist
-            user_id = None
-            user_id = get_user_id("email_address", user_email)
-        elif 'user_id' in self.record:
-            user_id = self.record['user_id']
-        elif 'github_id' in self.record:
-            user_github_id = self.record['github_id']
-            user_id = None
-            user_id = get_user_id("github_id", user_github_id)
-        else: #ZS: Anonymous user
-            return None
-
-        return user_id
-
-    @property
-    def user_name(self):
-        """Shortcut to the user_name"""
-        if 'user_name' in self.record:
-            return self.record['user_name']
-        else:
-            return ''
-
-    @property
-    def user_collections(self):
-        """List of user's collections"""
-
-        #ZS: Get user's collections if they exist
-        collections = get_user_collections(self.redis_user_id)
-        return collections
-
-    @property
-    def num_collections(self):
-        """Number of user's collections"""
-
-        return len(self.user_collections)
-
-    def add_collection(self, collection_name, traits):
-        """Add collection into ElasticSearch"""
-
-        collection_dict = {'id': unicode(uuid.uuid4()),
-                           'name': collection_name,
-                           'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
-                           'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
-                           'num_members': len(traits),
-                           'members': list(traits) }
-
-        current_collections = self.user_collections
-        current_collections.append(collection_dict)
-        self.update_collections(current_collections)
-
-        return collection_dict['id']
-
-    def change_collection_name(self, collection_id, new_name):
-        for collection in self.user_collections:
-            if collection['id'] == collection_id:
-                collection['name'] = new_name
-                break
-
-        return new_name
-
-    def delete_collection(self, collection_id):
-        """Remove collection with given ID"""
-
-        updated_collections = []
-        for collection in self.user_collections:
-            if collection['id'] == collection_id:
-                continue
-            else:
-                updated_collections.append(collection)
-
-        self.update_collections(updated_collections)
-
-        return collection['name']
-
-    def add_traits_to_collection(self, collection_id, traits_to_add):
-        """Add specified traits to a collection"""
-
-        this_collection = self.get_collection_by_id(collection_id)
-
-        updated_collection = this_collection
-        updated_traits = this_collection['members'] + traits_to_add
-
-        updated_collection['members'] = updated_traits
-        updated_collection['num_members'] = len(updated_traits)
-        updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
-
-        updated_collections = []
-        for collection in self.user_collections:
-            if collection['id'] == collection_id:
-                updated_collections.append(updated_collection)
-            else:
-                updated_collections.append(collection)
-
-        self.update_collections(updated_collections)
-
-    def remove_traits_from_collection(self, collection_id, traits_to_remove):
-        """Remove specified traits from a collection"""
-
-        this_collection = self.get_collection_by_id(collection_id)
-
-        updated_collection = this_collection
-        updated_traits = []
-        for trait in this_collection['members']:
-            if trait in traits_to_remove:
-                continue
-            else:
-                updated_traits.append(trait)
-
-        updated_collection['members'] = updated_traits
-        updated_collection['num_members'] = len(updated_traits)
-        updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
-
-        updated_collections = []
-        for collection in self.user_collections:
-            if collection['id'] == collection_id:
-                updated_collections.append(updated_collection)
-            else:
-                updated_collections.append(collection)
-
-        self.update_collections(updated_collections)
-
-        return updated_traits
-
-    def get_collection_by_id(self, collection_id):
-        for collection in self.user_collections:
-            if collection['id'] == collection_id:
-                return collection
-
-    def get_collection_by_name(self, collection_name):
-        for collection in self.user_collections:
-            if collection['name'] == collection_name:
-                return collection
-
-        return None
-
-    def update_collections(self, updated_collections):
-        collection_body = json.dumps(updated_collections)
-
-        save_collections(self.redis_user_id, collection_body)
-
-    def delete_session(self):
-        # And more importantly delete the redis record
-        Redis.delete(self.redis_key)
-        self.logged_in = False
-
-@app.before_request
-def before_request():
-    g.user_session = UserSession()
-
-@app.after_request
-def set_cookie(response):
-    if not request.cookies.get(g.user_session.cookie_name):
-        response.set_cookie(g.user_session.cookie_name, g.user_session.cookie)
+from __future__ import print_function, division, absolute_import
+
+import datetime
+import time
+import uuid
+
+import simplejson as json
+
+import redis # used for collections
+Redis = redis.StrictRedis()
+
+from flask import (Flask, g, render_template, url_for, request, make_response,
+                   redirect, flash, abort)
+
+from wqflask import app
+from utility import hmac
+
+#from utility.elasticsearch_tools import get_elasticsearch_connection
+from utility.redis_tools import get_user_id, get_user_by_unique_column, get_user_collections, save_collections
+
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+THREE_DAYS = 60 * 60 * 24 * 3
+THIRTY_DAYS = 60 * 60 * 24 * 30
+
+def verify_cookie(cookie):
+    the_uuid, separator, the_signature = cookie.partition(':')
+    assert len(the_uuid) == 36, "Is session_id a uuid?"
+    assert separator == ":", "Expected a : here"
+    assert the_signature == hmac.hmac_creation(the_uuid), "Uh-oh, someone tampering with the cookie?"
+    return the_uuid
+
+def create_signed_cookie():
+    the_uuid = str(uuid.uuid4())
+    signature = hmac.hmac_creation(the_uuid)
+    uuid_signed = the_uuid + ":" + signature
+    logger.debug("uuid_signed:", uuid_signed)
+    return the_uuid, uuid_signed
+
+class UserSession(object):
+    """Logged in user handling"""
+
+    user_cookie_name = 'session_id_v1'
+    anon_cookie_name = 'anon_user_v1'
+
+    def __init__(self):
+        user_cookie = request.cookies.get(self.user_cookie_name)
+        if not user_cookie:
+            self.logged_in = False
+            anon_cookie = request.cookies.get(self.anon_cookie_name)
+            self.cookie_name = self.anon_cookie_name
+            if anon_cookie:
+                self.cookie = anon_cookie
+                session_id = verify_cookie(self.cookie)
+            else:
+                session_id, self.cookie = create_signed_cookie()
+        else:
+            self.cookie_name = self.user_cookie_name
+            self.cookie = user_cookie
+            session_id = verify_cookie(self.cookie)
+
+        self.redis_key = self.cookie_name + ":" + session_id
+        self.session_id = session_id
+        self.record = Redis.hgetall(self.redis_key)
+
+        #ZS: If user correctled logged in but their session expired
+        #ZS: Need to test this by setting the time-out to be really short or something
+        if not self.record:
+            if user_cookie:
+                self.logged_in = False
+
+                ########### Grrr...this won't work because of the way flask handles cookies
+                # Delete the cookie
+                response = make_response(redirect(url_for('login')))
+                #response.set_cookie(self.cookie_name, '', expires=0)
+                flash("Due to inactivity your session has expired. If you'd like please login again.")
+                return response
+                #return
+            else:
+                self.record = dict(login_time = time.time(),
+                                    user_type = "anon",
+                                    user_id = str(uuid.uuid4()))
+
+                Redis.hmset(self.redis_key, self.record)
+                Redis.expire(self.redis_key, THIRTY_DAYS)
+        else:
+            if user_cookie:
+                self.logged_in = True
+
+        if user_cookie:
+            session_time = THREE_DAYS
+        else:
+            session_time = THIRTY_DAYS
+
+        if Redis.ttl(self.redis_key) < session_time:
+            # (Almost) everytime the user does something we extend the session_id in Redis...
+            logger.debug("Extending ttl...")
+            Redis.expire(self.redis_key, session_time)
+
+    @property
+    def user_id(self):
+        """Shortcut to the user_id"""
+        if 'user_id' in self.record:
+            return self.record['user_id']
+        else:
+            return ''
+
+    @property
+    def redis_user_id(self):
+        """User id from Redis (need to check if this is the same as the id stored in self.records)"""
+
+        #ZS: This part is a bit weird. Some accounts used to not have saved user ids, and in the process of testing I think I created some duplicate accounts for myself.
+        #ZS: Accounts should automatically generate user_ids if they don't already have one now, so this might not be necessary for anything other than my account's collections
+
+        if 'user_email_address' in self.record:
+            user_email = self.record['user_email_address']
+
+            #ZS: Get user's collections if they exist
+            user_id = None
+            user_id = get_user_id("email_address", user_email)
+        elif 'user_id' in self.record:
+            user_id = self.record['user_id']
+        elif 'github_id' in self.record:
+            user_github_id = self.record['github_id']
+            user_id = None
+            user_id = get_user_id("github_id", user_github_id)
+        else: #ZS: Anonymous user
+            return None
+
+        return user_id
+
+    @property
+    def user_name(self):
+        """Shortcut to the user_name"""
+        if 'user_name' in self.record:
+            return self.record['user_name']
+        else:
+            return ''
+
+    @property
+    def user_collections(self):
+        """List of user's collections"""
+
+        #ZS: Get user's collections if they exist
+        collections = get_user_collections(self.redis_user_id)
+        return collections
+
+    @property
+    def num_collections(self):
+        """Number of user's collections"""
+
+        return len(self.user_collections)
+
+    def add_collection(self, collection_name, traits):
+        """Add collection into Redis"""
+
+        collection_dict = {'id': unicode(uuid.uuid4()),
+                           'name': collection_name,
+                           'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
+                           'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
+                           'num_members': len(traits),
+                           'members': list(traits) }
+
+        current_collections = self.user_collections
+        current_collections.append(collection_dict)
+        self.update_collections(current_collections)
+
+        return collection_dict['id']
+
+    def change_collection_name(self, collection_id, new_name):
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                collection['name'] = new_name
+                break
+
+        return new_name
+
+    def delete_collection(self, collection_id):
+        """Remove collection with given ID"""
+
+        updated_collections = []
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                continue
+            else:
+                updated_collections.append(collection)
+
+        self.update_collections(updated_collections)
+
+        return collection['name']
+
+    def add_traits_to_collection(self, collection_id, traits_to_add):
+        """Add specified traits to a collection"""
+
+        this_collection = self.get_collection_by_id(collection_id)
+
+        updated_collection = this_collection
+        updated_traits = this_collection['members'] + traits_to_add
+
+        updated_collection['members'] = updated_traits
+        updated_collection['num_members'] = len(updated_traits)
+        updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
+
+        updated_collections = []
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                updated_collections.append(updated_collection)
+            else:
+                updated_collections.append(collection)
+
+        self.update_collections(updated_collections)
+
+    def remove_traits_from_collection(self, collection_id, traits_to_remove):
+        """Remove specified traits from a collection"""
+
+        this_collection = self.get_collection_by_id(collection_id)
+
+        updated_collection = this_collection
+        updated_traits = []
+        for trait in this_collection['members']:
+            if trait in traits_to_remove:
+                continue
+            else:
+                updated_traits.append(trait)
+
+        updated_collection['members'] = updated_traits
+        updated_collection['num_members'] = len(updated_traits)
+        updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
+
+        updated_collections = []
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                updated_collections.append(updated_collection)
+            else:
+                updated_collections.append(collection)
+
+        self.update_collections(updated_collections)
+
+        return updated_traits
+
+    def get_collection_by_id(self, collection_id):
+        for collection in self.user_collections:
+            if collection['id'] == collection_id:
+                return collection
+
+    def get_collection_by_name(self, collection_name):
+        for collection in self.user_collections:
+            if collection['name'] == collection_name:
+                return collection
+
+        return None
+
+    def update_collections(self, updated_collections):
+        collection_body = json.dumps(updated_collections)
+
+        save_collections(self.redis_user_id, collection_body)
+
+    def import_traits_to_user(self, anon_id):
+        collections = get_user_collections(anon_id)
+        for collection in collections:
+            collection_exists = self.get_collection_by_name(collection['name'])
+            if collection_exists:
+                continue
+            else:
+                self.add_collection(collection['name'], collection['members'])
+
+    def delete_session(self):
+        # And more importantly delete the redis record
+        Redis.delete(self.redis_key)
+        self.logged_in = False
+
+@app.before_request
+def before_request():
+    g.user_session = UserSession()
+
+@app.after_request
+def set_cookie(response):
+    if not request.cookies.get(g.user_session.cookie_name):
+        response.set_cookie(g.user_session.cookie_name, g.user_session.cookie)
     return response
\ No newline at end of file
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 436ebc91..8a80c26c 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -126,7 +126,7 @@ def index_page():
     if 'import_collections' in params:
         import_collections = params['import_collections']
         if import_collections == "true":
-            g.cookie_session.import_traits_to_user()
+            g.user_session.import_traits_to_user(params['anon_id'])
     #if USE_GN_SERVER:
     #    # The menu is generated using GN_SERVER
     #    return render_template("index_page.html", gn_server_url = GN_SERVER_URL, version=GN_VERSION)
-- 
cgit v1.2.3


From b3240e39af690c240011f7c74b462fc28c79b528 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 10 Mar 2020 14:35:16 -0500
Subject: Fixed issue that caused some problems when verifying hmac

---
 wqflask/utility/helper_functions.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py
index 971dd3cc..e7c04fef 100644
--- a/wqflask/utility/helper_functions.py
+++ b/wqflask/utility/helper_functions.py
@@ -41,9 +41,9 @@ def get_trait_db_obs(self, trait_db_list):
 
     self.trait_list = []
     for trait in trait_db_list:
-        data, _separator, hmac = trait.rpartition(':')
+        data, _separator, hmac_string = trait.rpartition(':')
         data = data.strip()
-        assert hmac==hmac.hmac_creation(data), "Data tampering?"
+        assert hmac_string==hmac.hmac_creation(data), "Data tampering?"
         trait_name, dataset_name = data.split(":")
         if dataset_name == "Temp":
             dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2])
-- 
cgit v1.2.3


From 0e240cbfc6034b65b42c2d21f59ae2663dd2f6ec Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 27 Mar 2020 16:32:08 -0500
Subject: Added integration for using RData files with R/qtl, plus some other
 changes to how it decides which mapping methods to display on the trait page

---
 wqflask/base/data_set.py                           | 11 ++-
 wqflask/utility/gen_geno_ob.py                     | 91 +++++++++++++++-------
 .../marker_regression/display_mapping_results.py   | 86 ++++++++++----------
 wqflask/wqflask/marker_regression/rqtl_mapping.py  | 40 +++++++---
 wqflask/wqflask/marker_regression/run_mapping.py   |  3 +-
 wqflask/wqflask/templates/collections/view.html    |  4 +-
 .../templates/show_trait_mapping_tools.html        | 39 +++++-----
 7 files changed, 168 insertions(+), 106 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 7fe9a8ac..abfdd277 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -307,9 +307,11 @@ class DatasetGroup(object):
 
         mapping_id = g.db.execute("select MappingMethodId from InbredSet where Name= '%s'" % self.name).fetchone()[0]
         if mapping_id == "1":
-            mapping_names = ["QTLReaper", "R/qtl"]
+            mapping_names = ["GEMMA", "QTLReaper", "R/qtl"]
         elif mapping_id == "2":
-            mapping_names = ["GEMMA"]
+            mapping_names = []
+        elif mapping_id == "3":
+            mapping_names = ["R/qtl"]
         elif mapping_id == "4":
             mapping_names = ["GEMMA", "PLINK"]
         else:
@@ -392,7 +394,10 @@ class DatasetGroup(object):
 
         # reaper barfs on unicode filenames, so here we ensure it's a string
         if self.genofile:
-            full_filename = str(locate(self.genofile, 'genotype'))
+            if "RData" in self.genofile: #ZS: This is a temporary fix; I need to change the way the JSON files that point to multiple genotype files are structured to point to other file types like RData
+                full_filename = str(locate(self.genofile.split(".")[0] + ".geno", 'genotype'))
+            else:
+                full_filename = str(locate(self.genofile, 'genotype'))
         else:
             full_filename = str(locate(self.name + '.geno', 'genotype'))
 
diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
index 44e2722f..aa5b27c4 100644
--- a/wqflask/utility/gen_geno_ob.py
+++ b/wqflask/utility/gen_geno_ob.py
@@ -1,5 +1,8 @@
 from __future__ import absolute_import, division, print_function
 
+import utility.logger
+logger = utility.logger.getLogger(__name__ )
+
 class genotype(object):
     """
     Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure
@@ -34,8 +37,34 @@ class genotype(object):
     def __len__(self):
         return len(self.chromosomes)
 
-    def read_file(self, filename):
+    def read_rdata_output(self, qtl_results):
+        #ZS: This is necessary because R/qtl requires centimorgan marker positions, which it normally gets from the .geno file, but that doesn't exist for HET3-ITP (which only has RData), so it needs to read in the marker cM positions from the results
+        self.chromosomes = [] #ZS: Overwriting since the .geno file's contents are just placeholders
+
+        this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers
+        chr_ob = None
+        for marker in qtl_results:
+            locus = Locus(self)
+            if str(marker['chr']) != this_chr:
+                if this_chr != "":
+                    self.chromosomes.append(chr_ob)
+                this_chr = str(marker['chr'])
+                chr_ob = Chr(this_chr, self)
+            if 'chr' in marker:
+                locus.chr = str(marker['chr'])
+            if 'name' in marker:
+                locus.name = marker['name']
+            if 'Mb' in marker:
+                locus.Mb = marker['Mb']
+            if 'cM' in marker:
+                locus.cM = marker['cM']
+            chr_ob.loci.append(locus)
+
+        self.chromosomes.append(chr_ob)
+
+        return self
 
+    def read_file(self, filename):
         with open(filename, 'r') as geno_file:
             lines = geno_file.readlines()
 
@@ -109,33 +138,39 @@ class Chr(object):
         return len(self.loci)
 
     def add_marker(self, marker_row):
-        self.loci.append(Locus(marker_row, self.geno_ob))
+        self.loci.append(Locus(self.geno_ob, marker_row))
 
 class Locus(object):
-    def __init__(self, marker_row, geno_ob):
-        self.chr = marker_row[0]
-        self.name = marker_row[1]
-        try:
-            self.cM = float(marker_row[geno_ob.cm_column])
-        except:
-            self.cM = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else 0
-        self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
-
-        geno_table = {
-            geno_ob.mat: -1,
-            geno_ob.pat: 1,
-            geno_ob.het: 0,
-            geno_ob.unk: "U"
-        }
-
+    def __init__(self, geno_ob, marker_row = None):
+        self.chr = None
+        self.name = None
+        self.cM = None
+        self.Mb = None
         self.genotype = []
-        if geno_ob.mb_exists:
-            start_pos = 4
-        else:
-            start_pos = 3
-
-        for allele in marker_row[start_pos:]:
-            if allele in geno_table.keys():
-                self.genotype.append(geno_table[allele])
-            else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown
-                self.genotype.append("U")
\ No newline at end of file
+        if marker_row:
+            self.chr = marker_row[0]
+            self.name = marker_row[1]
+            try:
+                self.cM = float(marker_row[geno_ob.cm_column])
+            except:
+                self.cM = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else 0
+            self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
+
+            geno_table = {
+                geno_ob.mat: -1,
+                geno_ob.pat: 1,
+                geno_ob.het: 0,
+                geno_ob.unk: "U"
+            }
+
+            self.genotype = []
+            if geno_ob.mb_exists:
+                start_pos = 4
+            else:
+                start_pos = 3
+
+            for allele in marker_row[start_pos:]:
+                if allele in geno_table.keys():
+                    self.genotype.append(geno_table[allele])
+                else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown
+                    self.genotype.append("U")
\ No newline at end of file
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index 66b5e25e..cf4508dd 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -247,11 +247,29 @@ class DisplayMappingResults(object):
 
         self.strainlist = start_vars['samples']
 
+        self.traitList = []
+        thisTrait = start_vars['this_trait']
+        self.traitList.append(thisTrait)
+
+        ################################################################
+        # Calculations QTL goes here
+        ################################################################
+        self.multipleInterval = len(self.traitList) > 1
+        self.qtlresults = start_vars['qtl_results']
+
+        if self.multipleInterval:
+            self.colorCollection = Plot.colorSpectrum(len(self.qtlresults))
+        else:
+            self.colorCollection = [self.LRS_COLOR]
+
         if self.mapping_method == "reaper" and self.manhattan_plot != True:
             self.genotype = self.dataset.group.read_genotype_file(use_reaper=True)
         else:
             self.genotype = self.dataset.group.read_genotype_file()
 
+        if self.mapping_method == "rqtl_geno" and self.genotype.filler == True:
+            self.genotype = self.genotype.read_rdata_output(self.qtlresults)
+
         #Darwing Options
         try:
            if self.selectedChr > -1:
@@ -346,10 +364,6 @@ class DisplayMappingResults(object):
         else:
             self.GraphInterval = self.cMGraphInterval #cM
 
-        self.traitList = []
-        thisTrait = start_vars['this_trait']
-        self.traitList.append(thisTrait)
-
 ## BEGIN HaplotypeAnalyst
 ## count the amount of individuals to be plotted, and increase self.graphHeight
         if self.haplotypeAnalystChecked and self.selectedChr > -1:
@@ -371,16 +385,7 @@ class DisplayMappingResults(object):
             self.graphHeight = self.graphHeight + 2 * (self.NR_INDIVIDUALS+10) * self.EACH_GENE_HEIGHT
 ## END HaplotypeAnalyst
 
-        ################################################################
-        # Calculations QTL goes here
-        ################################################################
-        self.multipleInterval = len(self.traitList) > 1
-        self.qtlresults = start_vars['qtl_results']
 
-        if self.multipleInterval:
-            self.colorCollection = Plot.colorSpectrum(len(self.qtlresults))
-        else:
-            self.colorCollection = [self.LRS_COLOR]
 
 
         #########################
@@ -1713,12 +1718,12 @@ class DisplayMappingResults(object):
         LRSLODFont=pid.Font(ttf="verdana", size=18*zoom*1.5, bold=0)
 
         yZero = yTopOffset + plotHeight
-        #LRSHeightThresh = drawAreaHeight
-        #AdditiveHeightThresh = drawAreaHeight/2
-        #DominanceHeightThresh = drawAreaHeight/2
-        LRSHeightThresh = (yZero - yTopOffset + 30*(zoom - 1))
-        AdditiveHeightThresh = LRSHeightThresh/2
-        DominanceHeightThresh = LRSHeightThresh/2
+        LRSHeightThresh = drawAreaHeight
+        AdditiveHeightThresh = drawAreaHeight/2
+        DominanceHeightThresh = drawAreaHeight/2
+        # LRSHeightThresh = (yZero - yTopOffset + 30*(zoom - 1))
+        # AdditiveHeightThresh = LRSHeightThresh/2
+        # DominanceHeightThresh = LRSHeightThresh/2
 
         if LRS_LOD_Max > 100:
             LRSScale = 20.0
@@ -1756,8 +1761,8 @@ class DisplayMappingResults(object):
             if LRS_LOD_Max == 0.0:
                 LRS_LOD_Max = 0.000001
             yTopOffset + 30*(zoom - 1)
-            #yLRS = yZero - (item/LRS_LOD_Max) * LRSHeightThresh
-            yLRS = yZero - (item/LRSAxisList[-1]) * LRSHeightThresh
+            yLRS = yZero - (item/LRS_LOD_Max) * LRSHeightThresh
+            #yLRS = yZero - (item/LRSAxisList[-1]) * LRSHeightThresh
             canvas.drawLine(xLeftOffset, yLRS, xLeftOffset - 4, yLRS, color=self.LRS_COLOR, width=1*zoom)
             if all_int:
                 scaleStr = "%d" % item
@@ -1767,10 +1772,10 @@ class DisplayMappingResults(object):
             canvas.drawString(scaleStr, xLeftOffset-4-canvas.stringWidth(scaleStr, font=LRSScaleFont)-5, yLRS+3, font=LRSScaleFont, color=self.LRS_COLOR)
 
         if self.permChecked and self.nperm > 0 and not self.multipleInterval:
-            # significantY = yZero - self.significant*LRSHeightThresh/LRS_LOD_Max
-            # suggestiveY = yZero - self.suggestive*LRSHeightThresh/LRS_LOD_Max
-            significantY = yZero - self.significant*LRSHeightThresh/LRSAxisList[-1]
-            suggestiveY = yZero - self.suggestive*LRSHeightThresh/LRSAxisList[-1]
+            significantY = yZero - self.significant*LRSHeightThresh/LRS_LOD_Max
+            suggestiveY = yZero - self.suggestive*LRSHeightThresh/LRS_LOD_Max
+            # significantY = yZero - self.significant*LRSHeightThresh/LRSAxisList[-1]
+            # suggestiveY = yZero - self.suggestive*LRSHeightThresh/LRSAxisList[-1]
             startPosX = xLeftOffset
 
             #"Significant" and "Suggestive" Drawing Routine
@@ -1887,36 +1892,35 @@ class DisplayMappingResults(object):
                 # fix the over limit LRS graph issue since genotype trait may give infinite LRS;
                 # for any lrs is over than 460(LRS max in this system), it will be reset to 460
 
-                yLRS = yZero - (item/LRSAxisList[-1]) * (yZero - yTopOffset + 30*(zoom - 1))
-                #yLRS = yZero - (item/LRS_LOD_Max) * LRSHeightThresh
+                yLRS = yZero - (item/LRS_LOD_Max) * LRSHeightThresh
 
 
                 if 'lrs_value' in qtlresult:
                     if self.LRS_LOD == "LOD" or self.LRS_LOD == "-log(p)":
                         if qtlresult['lrs_value'] > 460 or qtlresult['lrs_value']=='inf':
-                            Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/(LRSAxisList[-1]*self.LODFACTOR)
-                            #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/(LRS_LOD_Max*self.LODFACTOR)
+                            #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/(LRSAxisList[-1]*self.LODFACTOR)
+                            Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/(LRS_LOD_Max*self.LODFACTOR)
                         else:
-                            Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/(LRSAxisList[-1]*self.LODFACTOR)
-                            #Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/(LRS_LOD_Max*self.LODFACTOR)
+                            #Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/(LRSAxisList[-1]*self.LODFACTOR)
+                            Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/(LRS_LOD_Max*self.LODFACTOR)
                     else:
                         if qtlresult['lrs_value'] > 460 or qtlresult['lrs_value']=='inf':
-                            Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRSAxisList[-1]
-                            #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRS_LOD_Max
+                            #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRSAxisList[-1]
+                            Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRS_LOD_Max
                         else:
-                            Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/LRSAxisList[-1]
-                            #Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/LRS_LOD_Max
+                            #Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/LRSAxisList[-1]
+                            Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/LRS_LOD_Max
                 else:
                     if qtlresult['lod_score'] > 100 or qtlresult['lod_score']=='inf':
-                        Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRSAxisList[-1]
-                        #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRS_LOD_Max
+                        #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRSAxisList[-1]
+                        Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRS_LOD_Max
                     else:
                         if self.LRS_LOD == "LRS":
-                            Yc = yZero - qtlresult['lod_score']*self.LODFACTOR*LRSHeightThresh/LRSAxisList[-1]
-                            #Yc = yZero - qtlresult['lod_score']*self.LODFACTOR*LRSHeightThresh/LRS_LOD_Max
+                            #Yc = yZero - qtlresult['lod_score']*self.LODFACTOR*LRSHeightThresh/LRSAxisList[-1]
+                            Yc = yZero - qtlresult['lod_score']*self.LODFACTOR*LRSHeightThresh/LRS_LOD_Max
                         else:
-                            Yc = yZero - qtlresult['lod_score']*LRSHeightThresh/LRSAxisList[-1]
-                            #Yc = yZero - qtlresult['lod_score']*LRSHeightThresh/LRS_LOD_Max
+                            #Yc = yZero - qtlresult['lod_score']*LRSHeightThresh/LRSAxisList[-1]
+                            Yc = yZero - qtlresult['lod_score']*LRSHeightThresh/LRS_LOD_Max
 
                 if self.manhattan_plot == True:
                     if self.selectedChr == -1 and (previous_chr_as_int % 2 == 1):
diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py
index 41d67012..2e3ea406 100644
--- a/wqflask/wqflask/marker_regression/rqtl_mapping.py
+++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py
@@ -9,8 +9,6 @@ import utility.logger
 logger = utility.logger.getLogger(__name__ )
 
 def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan):
-    geno_to_rqtl_function(dataset)
-
     ## Get pointers to some common R functions
     r_library     = ro.r["library"]                 # Map the library function
     r_c           = ro.r["c"]                       # Map the c function
@@ -21,16 +19,24 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
     print(r_library("qtl"))                         # Load R/qtl
 
     ## Get pointers to some R/qtl functions
-    scanone         = ro.r["scanone"]               # Map the scanone function
-    scantwo         = ro.r["scantwo"]               # Map the scantwo function
-    calc_genoprob   = ro.r["calc.genoprob"]         # Map the calc.genoprob function
-    GENOtoCSVR      = ro.r["GENOtoCSVR"]            # Map the local GENOtoCSVR function
+    scanone                    = ro.r["scanone"]               # Map the scanone function
+    scantwo                    = ro.r["scantwo"]               # Map the scantwo function
+    calc_genoprob              = ro.r["calc.genoprob"]         # Map the calc.genoprob function
 
     crossname = dataset.group.name
-    genofilelocation  = locate(crossname + ".geno", "genotype")
-    crossfilelocation = TMPDIR + crossname + ".cross"
-
-    cross_object = GENOtoCSVR(genofilelocation, crossfilelocation)                            # TODO: Add the SEX if that is available
+    try:
+        generate_cross_from_rdata(dataset)
+        read_cross_from_rdata      = ro.r["generate_cross_from_rdata"] # Map the local read_cross_from_rdata function
+        genofilelocation  = locate(crossname + ".RData", "genotype/rdata")
+        cross_object = read_cross_from_rdata(genofilelocation)  # Map the local GENOtoCSVR function
+    except:
+        generate_cross_from_geno(dataset)
+        GENOtoCSVR                 = ro.r["GENOtoCSVR"]            # Map the local GENOtoCSVR function
+        crossfilelocation = TMPDIR + crossname + ".cross"
+        genofilelocation  = locate(crossname + ".geno", "genotype")
+
+        GENOtoCSVR      = ro.r["GENOtoCSVR"]                                # Map the local GENOtoCSVR function
+        cross_object = GENOtoCSVR(genofilelocation, crossfilelocation)      # TODO: Add the SEX if that is available
 
     if manhattan_plot:
         cross_object = calc_genoprob(cross_object)
@@ -71,7 +77,18 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
         else:
             return process_rqtl_results(result_data_frame)
 
-def geno_to_rqtl_function(dataset):        # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly
+def generate_cross_from_rdata(dataset):
+    rdata_location  = locate(dataset.group.name + ".RData", "genotype/rdata")
+    ro.r("""
+       generate_cross_from_rdata <- function(filename = '%s') {
+           load(file=filename)
+           cross = cunique
+           return(cross)
+       }
+    """ % (rdata_location))
+
+def generate_cross_from_geno(dataset):        # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly
+
     ro.r("""
        trim <- function( x ) { gsub("(^[[:space:]]+|[[:space:]]+$)", "", x) }
 
@@ -170,6 +187,7 @@ def process_rqtl_results(result):        # TODO: how to make this a one liner an
         marker = {}
         marker['name'] = result.rownames[i]
         marker['chr'] = output[i][0]
+        marker['cM'] = output[i][1]
         marker['Mb'] = output[i][1]
         marker['lod_score'] = output[i][2]
         qtl_results.append(marker)
diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py
index ba41ffc3..f03b046e 100644
--- a/wqflask/wqflask/marker_regression/run_mapping.py
+++ b/wqflask/wqflask/marker_regression/run_mapping.py
@@ -81,7 +81,6 @@ class RunMapping(object):
                         self.vals.append(value)
         else:
             self.samples = []
-
             for sample in self.dataset.group.samplelist: # sample is actually the name of an individual
                 if (len(genofile_samplelist) == 0) or (sample in genofile_samplelist):
                     in_trait_data = False
@@ -186,7 +185,7 @@ class RunMapping(object):
             self.showGenes = "ON"
             self.viewLegend = "ON"
 
-        self.dataset.group.get_markers()
+        #self.dataset.group.get_markers()
         if self.mapping_method == "gemma":
             self.first_run = True
             self.output_files = None
diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html
index 97e46756..1be6539d 100644
--- a/wqflask/wqflask/templates/collections/view.html
+++ b/wqflask/wqflask/templates/collections/view.html
@@ -161,10 +161,10 @@
 {% endblock %}
 
 {% block js %}
+    <script language="javascript" type="text/javascript" src="/static/new/js_external/jszip.min.js"></script>
+    <script language="javascript" type="text/javascript" src="/static/new/js_external/md5.min.js"></script>
     <script type="text/javascript" src="/static/new/javascript/search_results.js"></script>
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/jquery.dataTables.min.js"></script>
-    <script language="javascript" type="text/javascript" src="/static/new/js_external/md5.min.js"></script>
-    <script language="javascript" type="text/javascript" src="/static/new/js_external/jszip.min.js"></script>
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/dataTables.naturalSort.js"></script>
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/extensions/dataTables.colResize.js"></script>
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/extensions/dataTables.colReorder.js"></script>
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index a806a8b3..ad7412b2 100644
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -4,25 +4,18 @@
         <div class="tabbable"> <!-- Only required for left/right tabs -->
 
             <ul class="nav nav-pills">
-                {% if dataset.group.mapping_id == "1" %}
-                <li class="gemma_tab mapping_tab active">
-                    <a href="#gemma" data-toggle="tab">GEMMA</a>
-                </li>
-                <li class="reaper_tab mapping_tab">
-                    <a href="#interval_mapping" data-toggle="tab">Haley-Knott Regression</a>
-                </li>
-                <li class="rqtl_tab mapping_tab rqtl">
-                    <a href="#rqtl_geno" data-toggle="tab">R/qtl</a>
-                </li>
-                {% endif %}
                 {% for mapping_method in dataset.group.mapping_names %}
                 {% if mapping_method == "GEMMA" %}
-                <li class="gemma_tab mapping_tab active">
+                <li class="gemma_tab mapping_tab {% if dataset.group.mapping_id == '1' %}active{% endif %}">
                     <a href="#gemma" data-toggle="tab">GEMMA</a>
                 </li>
-                {% elif mapping_method == "PLINK" %}
-                <li class="plink_tab mapping_tab">
-                    <a href="#plink" data-toggle="tab">PLINK</a>
+                {% elif mapping_method == "R/qtl" %}
+                <li class="rqtl_geno_tab mapping_tab {% if dataset.group.mapping_id == '3' %}active{% endif %}">
+                    <a href="#rqtl_geno" data-toggle="tab">R/qtl</a>
+                </li>
+                {% elif mapping_method == "QTLReaper" %}
+                <li class="reaper_tab mapping_tab">
+                    <a href="#interval_mapping" data-toggle="tab">Haley-Knott Regression</a>
                 </li>
                 {% endif %}
                 {% endfor %}
@@ -32,7 +25,9 @@
             </ul>
 
             <div class="tab-content">
-                <div class="tab-pane active" id="gemma">
+                {% for mapping_method in dataset.group.mapping_names %}
+                {% if mapping_method == "GEMMA" %}
+                <div class="tab-pane {% if dataset.group.mapping_id == '1' %}active{% endif %}" id="gemma">
                     <div style="padding-top: 20px;" class="form-horizontal">
                         <div class="mapping_method_fields form-group">
                             <label for="chr_select" style="text-align: right;" class="col-xs-3 control-label">Chromosome</label>
@@ -102,7 +97,7 @@
                         </div>
                     </div>
                 </div>
-                {% if dataset.group.mapping_id == "1" %}
+                {% elif mapping_method == "QTLReaper" %}
                 <div class="tab-pane" id="interval_mapping">
                     <div style="margin-top: 20px" class="form-horizontal">
                         <div class="mapping_method_fields form-group">
@@ -217,7 +212,8 @@
                         </div>
                     </div>
                 </div>
-                <div class="tab-pane" id="rqtl_geno">
+                {% elif mapping_method == "R/qtl" %}
+                <div class="tab-pane {% if dataset.group.mapping_id == '3' %}active{% endif %}" id="rqtl_geno">
                     <div style="margin-top: 20px" class="form-horizontal">
                         <div class="mapping_method_fields form-group">
                             <label for="chr_select" style="text-align: right;" class="col-xs-3 control-label">Chromosome</label>
@@ -341,19 +337,24 @@
                     </div>
                 </div>
                 {% endif %}
+                {% endfor %}
             </div>
         </div>
     </div>
     <div class="col-xs-7">
         <dl style="width: 500px;">
+             {% for mapping_method in dataset.group.mapping_names %}
+             {% if mapping_method == "GEMMA" %}
              <dt style="padding-top: 20px;">GEMMA</dt>
              <dd>Maps traits with correction for kinship among samples using a linear mixed model method, and also allows users to fit multiple covariates such as sex, age, treatment, and genetic markers (<a href="https://www.ncbi.nlm.nih.gov/pubmed/24531419">PMID: 2453419</a>, and <a href="https://github.com/genetics-statistics/GEMMA"> GitHub code</a>). GEMMA incorporates the Leave One Chromosome Out (LOCO) method to ensure that the correction for kinship does not remove useful genetic variance near each marker. Markers can be filtered to include only those with minor allele frequencies (MAF) above a threshold. The default MAF is 0.05.</dd>
-             {% if dataset.group.mapping_id == "1" %}
+             {% elif mapping_method == "R/qtl" %}
              <dt style="margin-top: 20px;">R/qtl</dt>
              <dd>Major upgrade of R/qtl that supports most experimental populations including those with complex admixture and two or more parental lines as well as large omic data sets (<a href="https://www.ncbi.nlm.nih.gov/pubmed/30591514">PMID: 30591514</a>). Both R/qtl and R/qtl2 are available as stand-alone R packages (<a href="https://kbroman.org/pages/software.html">R suite</a>).</dd>
+             {% elif mapping_method == "QTLReaper" %}
              <dt style="margin-top: 20px;">Haley-Knott Regression</dt>
              <dd>Fast linear mapping method (<a href="https://www.ncbi.nlm.nih.gov/pubmed/16718932">PMID 16718932</a>) works well with F2 intercrosses and backcrosses, but that is not recommended for complex or admixed populations (e.g., GWAS or heterogeneous stock studies) or for advanced intercrosses, recombinant inbred families, or diallel crosses. Interactive plots in GeneNetwork have relied on the fast HK mapping for two decades and we still use this method for mapping omics data sets and computing genome-wide permutation threshold (<a href="https://github.com/pjotrp/QTLReaper">QTL Reaper code</a>).</dd>
              {% endif %}
+             {% endfor %}
         </dl>
         <div class="rqtl_description" style="padding-top: 40px; display: none;">
         More information on R/qtl mapping models and methods can be found <a href="http://www.rqtl.org/tutorials/rqtltour.pdf">here</a>.
-- 
cgit v1.2.3


From 897f3291f9d943b4deaef7bc924197c268bc8b3f Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 3 Apr 2020 16:57:59 -0500
Subject: Added option to select cofactors from collection for R/qtl

Fixed some issues with displaying the R/qtl results from the RData HET3-ITP dataa
---
 wqflask/utility/gen_geno_ob.py                     |   4 +-
 .../marker_regression/display_mapping_results.py   |  36 ++++---
 wqflask/wqflask/marker_regression/rqtl_mapping.py  | 116 ++++++++++++++++-----
 wqflask/wqflask/marker_regression/run_mapping.py   |   4 +-
 .../wqflask/static/new/javascript/show_trait.js    |   7 ++
 .../templates/show_trait_mapping_tools.html        |  18 +++-
 6 files changed, 140 insertions(+), 45 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
index aa5b27c4..db40f6ea 100644
--- a/wqflask/utility/gen_geno_ob.py
+++ b/wqflask/utility/gen_geno_ob.py
@@ -45,10 +45,12 @@ class genotype(object):
         chr_ob = None
         for marker in qtl_results:
             locus = Locus(self)
-            if str(marker['chr']) != this_chr:
+            if (str(marker['chr']) != this_chr) and this_chr != "X": #ZS: This is really awkward but works as a temporary fix
                 if this_chr != "":
                     self.chromosomes.append(chr_ob)
                 this_chr = str(marker['chr'])
+                if this_chr == "20":
+                    this_chr = "X"
                 chr_ob = Chr(this_chr, self)
             if 'chr' in marker:
                 locus.chr = str(marker['chr'])
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index 302a4ff9..b8f84721 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -334,6 +334,10 @@ class DisplayMappingResults(object):
         ################################################################
         self.ChrList = [("All", -1)]
         for i, indChr in enumerate(self.genotype):
+            if self.dataset.group.species == "mouse" and indChr.name == "20":
+                self.ChrList.append(("X", i))
+            elif self.dataset.group.species == "rat" and indChr.name == "21":
+                self.ChrList.append(("X", i))
             self.ChrList.append((indChr.name, i))
 
         self.ChrLengthMbList = g.db.execute("""
@@ -356,10 +360,7 @@ class DisplayMappingResults(object):
 
         self.ChrLengthCMList = []
         for i, _chr in enumerate(self.genotype):
-            if self.mapping_method == "rqtl_geno" and self.genotype.filler == True:
-                self.ChrLengthCMList.append(_chr[-1].cM)
-            else:
-                self.ChrLengthCMList.append(_chr[-1].cM - _chr[0].cM)
+            self.ChrLengthCMList.append(_chr[-1].cM - _chr[0].cM)
 
         self.ChrLengthCMSum = reduce(lambda x, y:x+y, self.ChrLengthCMList, 0.0)
 
@@ -1568,10 +1569,7 @@ class DisplayMappingResults(object):
             if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes
                 for i, _chr in enumerate(self.genotype):
                     thisChr = []
-                    if self.mapping_method == "rqtl_geno" and self.genotype.filler == True:
-                        Locus0CM = 0
-                    else:
-                        Locus0CM = _chr[0].cM
+                    Locus0CM = _chr[0].cM
                     nLoci = len(_chr)
                     if  nLoci <= 8:
                         for _locus in _chr:
@@ -1593,10 +1591,7 @@ class DisplayMappingResults(object):
                 for i, _chr in enumerate(self.genotype):
                     if _chr.name == self.ChrList[self.selectedChr][0]:
                         thisChr = []
-                        if self.mapping_method == "rqtl_geno" and self.genotype.filler == True:
-                            Locus0CM = 0
-                        else:
-                            Locus0CM = _chr[0].cM
+                        Locus0CM = _chr[0].cM
                         for _locus in _chr:
                             if _locus.name != ' - ':
                                 if _locus.cM != preLpos:
@@ -1892,12 +1887,21 @@ class DisplayMappingResults(object):
                     oldStartPosX = newStartPosX
 
             #ZS: This is beause the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used
-            if self.plotScale == "physic":
-                this_chr = str(self.ChrList[self.selectedChr][0])
-            else:
+            this_chr = str(self.ChrList[self.selectedChr][0])
+            if self.plotScale != "physic":
                 this_chr = str(self.ChrList[self.selectedChr][1]+1)
+
             if self.selectedChr == -1 or str(qtlresult['chr']) == this_chr:
-                Xc = startPosX + (qtlresult['Mb']-startMb)*plotXScale
+                if self.plotScale != "physic" and self.genotype.filler == True:
+                    if self.selectedChr != -1:
+                        start_cm = self.genotype[self.selectedChr - 1][0].cM
+                        Xc = startPosX + (qtlresult['Mb'] - start_cm)*plotXScale
+                    else:
+                        start_cm = self.genotype[previous_chr_as_int][0].cM
+                        Xc = startPosX + ((qtlresult['Mb']-start_cm-startMb)*plotXScale)*(((qtlresult['Mb']-start_cm-startMb)*plotXScale)/((qtlresult['Mb']-start_cm-startMb+self.GraphInterval)*plotXScale))
+                else:
+                    Xc = startPosX + (qtlresult['Mb']-startMb)*plotXScale
+
                 # updated by NL 06-18-2011:
                 # fix the over limit LRS graph issue since genotype trait may give infinite LRS;
                 # for any lrs is over than 460(LRS max in this system), it will be reset to 460
diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py
index 2e3ea406..d76f3812 100644
--- a/wqflask/wqflask/marker_regression/rqtl_mapping.py
+++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py
@@ -1,14 +1,17 @@
 import rpy2.robjects as ro
+import rpy2.robjects.numpy2ri as np2r
 import numpy as np
 
 from base.webqtlConfig import TMPDIR
+from base.trait import GeneralTrait
+from base.data_set import create_dataset
 from utility import webqtlUtil
 from utility.tools import locate, TEMPDIR
 
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
 
-def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan):
+def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan, samples, cofactors):
     ## Get pointers to some common R functions
     r_library     = ro.r["library"]                 # Map the library function
     r_c           = ro.r["c"]                       # Map the c function
@@ -43,14 +46,22 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
     else:
         cross_object = calc_genoprob(cross_object, step=1, stepwidth="max")
 
-    cross_object = add_phenotype(cross_object, sanitize_rqtl_phenotype(vals))                 # Add the phenotype
+    cross_object = add_phenotype(cross_object, sanitize_rqtl_phenotype(vals), "the_pheno")                 # Add the phenotype
 
     # Scan for QTLs
-    covar = create_covariates(control_marker, cross_object)                                   # Create the additive covariate matrix
+    marker_covars = create_marker_covariates(control_marker, cross_object)  # Create the additive covariate markers
+
+    if cofactors != "":
+        cross_object, trait_covars = add_cofactors(cross_object, dataset, cofactors, samples)                            # Create the covariates from selected traits
+        ro.r('all_covars <- cbind(marker_covars, trait_covars)')
+    else:
+        ro.r('all_covars <- marker_covars')
+
+    covars = ro.r['all_covars']
 
     if pair_scan:
         if do_control == "true":
-            logger.info("Using covariate"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method, n_cluster = 16)
+            logger.info("Using covariate"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", addcovar = covars, model=model, method=method, n_cluster = 16)
         else:
             logger.info("No covariates"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", model=model, method=method, n_cluster = 16)
 
@@ -61,14 +72,14 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
 
         return process_pair_scan_results(result_data_frame)
     else:
-        if do_control == "true":
-            logger.info("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method)
+        if do_control == "true" or cofactors != "":
+            logger.info("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covars, model=model, method=method)
         else:
             logger.info("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method)
 
         if num_perm > 0 and permCheck == "ON":                                                                   # Do permutation (if requested by user)
-            if do_control == "true":
-                perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covar, n_perm = num_perm, model=model, method=method)
+            if do_control == "true" or cofactors != "":
+                perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = num_perm, model=model, method=method)
             else:
                 perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method)
 
@@ -118,23 +129,6 @@ def generate_cross_from_geno(dataset):        # TODO: Need to figure out why som
       }
     """ % (dataset.group.name + ".geno"))
 
-def add_phenotype(cross, pheno_as_string):
-    ro.globalenv["the_cross"] = cross
-    ro.r('the_cross$pheno <- cbind(pull.pheno(the_cross), the_pheno = '+ pheno_as_string +')')
-    return ro.r["the_cross"]
-
-def create_covariates(control_marker, cross):
-    ro.globalenv["the_cross"] = cross
-    ro.r('genotypes <- pull.geno(the_cross)')                             # Get the genotype matrix
-    userinputS = control_marker.replace(" ", "").split(",")               # TODO: sanitize user input, Never Ever trust a user
-    covariate_names = ', '.join('"{0}"'.format(w) for w in userinputS)
-    ro.r('covnames <- c(' + covariate_names + ')')
-    ro.r('covInGeno <- which(covnames %in% colnames(genotypes))')
-    ro.r('covnames <- covnames[covInGeno]')
-    ro.r("cat('covnames (purged): ', covnames,'\n')")
-    ro.r('covariates <- genotypes[,covnames]')                            # Get the covariate matrix by using the marker name as index to the genotype file
-    return ro.r["covariates"]
-
 def sanitize_rqtl_phenotype(vals):
     pheno_as_string = "c("
     for i, val in enumerate(vals):
@@ -151,6 +145,78 @@ def sanitize_rqtl_phenotype(vals):
     pheno_as_string += ")"
     return pheno_as_string
 
+def add_phenotype(cross, pheno_as_string, col_name):
+    ro.globalenv["the_cross"] = cross
+    ro.r('the_cross$pheno <- cbind(pull.pheno(the_cross), ' + col_name + ' = '+ pheno_as_string +')')
+    return ro.r["the_cross"]
+
+def pull_covar(cross, covar_name_string):
+    ro.globalenv["the_cross"] = cross
+    ro.r('trait_covars <- pull.pheno(the_cross, ' + covar_name_string + ')')
+
+    return ro.r["trait_covars"]
+
+def add_cofactors(cross, this_dataset, covariates, samples):
+    ro.numpy2ri.activate()
+
+    covariate_list = covariates.split(",")
+    covar_name_string = "c("
+    for i, covariate in enumerate(covariate_list):
+        this_covar_data = []
+        covar_as_string = "c("
+        trait_name = covariate.split(":")[0]
+        dataset_ob = create_dataset(covariate.split(":")[1])
+        trait_ob = GeneralTrait(dataset=dataset_ob,
+                                name=trait_name,
+                                cellid=None)
+
+        this_dataset.group.get_samplelist()
+        trait_samples = this_dataset.group.samplelist
+        trait_sample_data = trait_ob.data
+        for index, sample in enumerate(trait_samples):
+            if sample in samples:
+                if sample in trait_sample_data:
+                    sample_value = trait_sample_data[sample].value
+                    this_covar_data.append(sample_value)
+                else:
+                    this_covar_data.append("NA")
+
+        for j, item in enumerate(this_covar_data):
+            if j < (len(this_covar_data) - 1):
+                covar_as_string += str(item) + ","
+            else:
+                covar_as_string += str(item)
+
+        covar_as_string += ")"
+
+        col_name = "covar_" + str(i)
+
+        if i < (len(covariate_list) - 1):
+            covar_name_string += '"' + col_name + '", '
+        else:
+            covar_name_string += '"' + col_name + '"'
+
+        cross = add_phenotype(cross, covar_as_string, col_name)
+
+    covar_name_string += ")"
+
+    covars = pull_covar(cross, covar_name_string)
+
+    return cross, covars
+
+def create_marker_covariates(control_marker, cross):
+    ro.globalenv["the_cross"] = cross
+    ro.r('genotypes <- pull.geno(the_cross)')                             # Get the genotype matrix
+    userinputS = control_marker.replace(" ", "").split(",")               # TODO: sanitize user input, Never Ever trust a user
+    covariate_names = ', '.join('"{0}"'.format(w) for w in userinputS)
+    ro.r('covnames <- c(' + covariate_names + ')')
+    ro.r('covInGeno <- which(covnames %in% colnames(genotypes))')
+    ro.r('covnames <- covnames[covInGeno]')
+    ro.r("cat('covnames (purged): ', covnames,'\n')")
+    ro.r('marker_covars <- genotypes[,covnames]')                            # Get the covariate matrix by using the marker name as index to the genotype file
+
+    return ro.r["marker_covars"]
+
 def process_pair_scan_results(result):
     pair_scan_results = []
 
diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py
index f03b046e..3006c4ff 100644
--- a/wqflask/wqflask/marker_regression/run_mapping.py
+++ b/wqflask/wqflask/marker_regression/run_mapping.py
@@ -216,9 +216,9 @@ class RunMapping(object):
             #if start_vars['pair_scan'] == "true":
             #    self.pair_scan = True
             if self.permCheck and self.num_perm > 0:
-                self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan)
+                self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.samples, self.covariates)
             else:
-                results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan)
+                results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.samples, self.covariates)
         elif self.mapping_method == "reaper":
             if "startMb" in start_vars: #ZS: Check if first time page loaded, so it can default to ON
                 if "additiveCheck" in start_vars:
diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js
index 0162f858..a2fa37e0 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait.js
@@ -107,6 +107,13 @@ $("#remove_covariates").click(function () {
     $("input[name=covariates]").val("")
     $(".selected_covariates").val("")
 });
+$(".select_covariates").click(function () {
+  open_covariate_selection();
+});
+$(".remove_covariates").click(function () {
+  $("input[name=covariates]").val("")
+  $(".selected_covariates").val("")
+});
 d3.select("#clear_compare_trait").on("click", (function(_this) {
   return function() {
     return $('.scatter-matrix-container').remove();
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index 6da25a5d..66903c31 100644
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -325,7 +325,23 @@
                                </label>
                             </div>
                         </div>
-
+                        <div class="mapping_method_fields form-group">
+                            <div class="col-xs-3" style="text-align: right;">
+                            <label class="control-label">Covariates</label>
+                            <font size="2">Select covariate(s) from a collection</font>
+                            </div>
+                            <div style="margin-left:20px;" class="col-xs-7">
+                              {% if g.user_session.num_collections < 1 %}
+                              No collections available. Please add traits to a collection to use them as covariates.
+                              {% else %}
+                              <div style="margin-bottom: 10px;">
+                                <button type="button" class="btn btn-default select_covariates" style="width: 80px; padding-right: 10px;">Select</button>
+                                <button type="button" class="btn btn-default remove_covariates" style="width: 80px;">Remove</button>
+                              </div>
+                              <textarea rows="3" cols="50" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none; width: 200px;" class="selected_covariates"></textarea>
+                              {% endif %}
+                            </div>
+                        </div>
                         <div class="mapping_method_fields form-group">
                             <label class="col-xs-3 control-label"></label>
                             <div style="margin-left:20px;" class="col-xs-6">
-- 
cgit v1.2.3


From 39d37ecde31f682013c7635e4f97853edc256b01 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Thu, 16 Apr 2020 13:48:08 -0500
Subject: Many changes, including:

- Added permutation strata option for R/qtl
- Made a variety of aesthetic changes to collections-related pages
- Made a variety of aesthetic changes to the multi-trait tool options on the search result and correlation pages
- Made some functional changes to collections that prevent duplicate traits and ensure new traits are at the top
- The "Default Collection" is now always at the bottom of the collection list and renamed to "Your Default Collection"
---
 wqflask/utility/gen_geno_ob.py                     |  5 +-
 wqflask/wqflask/collect.py                         |  4 +-
 .../marker_regression/display_mapping_results.py   | 12 ++-
 wqflask/wqflask/marker_regression/rqtl_mapping.py  | 90 ++++++++++++++--------
 wqflask/wqflask/marker_regression/run_mapping.py   | 84 +++++++++++++++++---
 wqflask/wqflask/show_trait/SampleList.py           |  4 +-
 wqflask/wqflask/show_trait/show_trait.py           | 23 +++++-
 .../new/javascript/show_trait_mapping_tools.js     |  3 +-
 wqflask/wqflask/templates/collections/add.html     | 28 +++----
 wqflask/wqflask/templates/collections/list.html    |  2 +-
 wqflask/wqflask/templates/collections/view.html    | 15 +++-
 wqflask/wqflask/templates/correlation_page.html    | 58 +++++++-------
 wqflask/wqflask/templates/loading.html             |  2 +-
 wqflask/wqflask/templates/search_result_page.html  | 49 ++++++------
 wqflask/wqflask/templates/show_trait.html          |  2 +-
 .../templates/show_trait_mapping_tools.html        | 17 +++-
 wqflask/wqflask/user_session.py                    |  1 +
 wqflask/wqflask/views.py                           | 16 ++--
 18 files changed, 278 insertions(+), 137 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
index db40f6ea..23b0b650 100644
--- a/wqflask/utility/gen_geno_ob.py
+++ b/wqflask/utility/gen_geno_ob.py
@@ -156,7 +156,10 @@ class Locus(object):
                 self.cM = float(marker_row[geno_ob.cm_column])
             except:
                 self.cM = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else 0
-            self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
+            try:
+                self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
+            except:
+                self.Mb = self.cM
 
             geno_table = {
                 geno_ob.mat: -1,
diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py
index 74eb869f..fa6e03b4 100644
--- a/wqflask/wqflask/collect.py
+++ b/wqflask/wqflask/collect.py
@@ -75,7 +75,7 @@ def collections_add():
 
     collections = g.user_session.user_collections
     if len(collections) < 1:
-        collection_name = "Default Collection"
+        collection_name = "Your Default Collection"
         uc_id = g.user_session.add_collection(collection_name, set())
         collections = g.user_session.user_collections
 
@@ -113,7 +113,7 @@ def collections_new():
                     collection_name = collection["name"]
                     default_collection_exists = True
             if not default_collection_exists:
-                return create_new("Default Collection")
+                return create_new("Your Default Collection")
         else:
             collection_id = params['existing_collection'].split(":")[0]
             collection_name = params['existing_collection'].split(":")[1]
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index b8f84721..a7e11738 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -267,8 +267,8 @@ class DisplayMappingResults(object):
         else:
             self.genotype = self.dataset.group.read_genotype_file()
 
-        if self.mapping_method == "rqtl_geno" and self.genotype.filler == True:
-            self.genotype = self.genotype.read_rdata_output(self.qtlresults)
+        #if self.mapping_method == "rqtl_geno" and self.genotype.filler == True:
+        #    self.genotype = self.genotype.read_rdata_output(self.qtlresults)
 
         #Darwing Options
         try:
@@ -935,10 +935,14 @@ class DisplayMappingResults(object):
                 string3 += 'no cofactors'
         elif self.mapping_method == "rqtl_plink" or self.mapping_method == "rqtl_geno":
             string3 = 'Using R/qtl mapping method with '
-            if self.controlLocus and self.doControl != "false":
+            if self.covariates != "":
+                string3 += 'the cofactors below:'
+                cofactor_names = ", ".join([covar.split(":")[0] for covar in self.covariates.split(",")])
+                string4 = cofactor_names
+            elif self.controlLocus and self.doControl != "false":
                 string3 += '%s as control' % self.controlLocus
             else:
-                string3 += 'no control for other QTLs'
+                string3 += 'no cofactors'
         else:
             string3 = 'Using Haldane mapping function with '
             if self.controlLocus and self.doControl != "false":
diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py
index d76f3812..aae8e602 100644
--- a/wqflask/wqflask/marker_regression/rqtl_mapping.py
+++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py
@@ -11,7 +11,7 @@ from utility.tools import locate, TEMPDIR
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
 
-def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan, samples, cofactors):
+def run_rqtl_geno(vals, samples, dataset, method, model, permCheck, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, pair_scan, cofactors):
     ## Get pointers to some common R functions
     r_library     = ro.r["library"]                 # Map the library function
     r_c           = ro.r["c"]                       # Map the c function
@@ -27,26 +27,26 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
     calc_genoprob              = ro.r["calc.genoprob"]         # Map the calc.genoprob function
 
     crossname = dataset.group.name
-    try:
-        generate_cross_from_rdata(dataset)
-        read_cross_from_rdata      = ro.r["generate_cross_from_rdata"] # Map the local read_cross_from_rdata function
-        genofilelocation  = locate(crossname + ".RData", "genotype/rdata")
-        cross_object = read_cross_from_rdata(genofilelocation)  # Map the local GENOtoCSVR function
-    except:
-        generate_cross_from_geno(dataset)
-        GENOtoCSVR                 = ro.r["GENOtoCSVR"]            # Map the local GENOtoCSVR function
-        crossfilelocation = TMPDIR + crossname + ".cross"
-        genofilelocation  = locate(crossname + ".geno", "genotype")
-
-        GENOtoCSVR      = ro.r["GENOtoCSVR"]                                # Map the local GENOtoCSVR function
-        cross_object = GENOtoCSVR(genofilelocation, crossfilelocation)      # TODO: Add the SEX if that is available
+    #try:
+    #    generate_cross_from_rdata(dataset)
+    #    read_cross_from_rdata      = ro.r["generate_cross_from_rdata"] # Map the local read_cross_from_rdata function
+    #    genofilelocation  = locate(crossname + ".RData", "genotype/rdata")
+    #    cross_object = read_cross_from_rdata(genofilelocation)  # Map the local GENOtoCSVR function
+    #except:
+    generate_cross_from_geno(dataset)
+    GENOtoCSVR                 = ro.r["GENOtoCSVR"]            # Map the local GENOtoCSVR function
+    crossfilelocation = TMPDIR + crossname + ".cross"
+    genofilelocation  = locate(dataset.group.genofile, "genotype")
+    cross_object = GENOtoCSVR(genofilelocation, crossfilelocation)      # TODO: Add the SEX if that is available
 
     if manhattan_plot:
         cross_object = calc_genoprob(cross_object)
     else:
         cross_object = calc_genoprob(cross_object, step=1, stepwidth="max")
 
-    cross_object = add_phenotype(cross_object, sanitize_rqtl_phenotype(vals), "the_pheno")                 # Add the phenotype
+    pheno_string = sanitize_rqtl_phenotype(vals)
+
+    cross_object = add_phenotype(cross_object, pheno_string, "the_pheno")                 # Add the phenotype
 
     # Scan for QTLs
     marker_covars = create_marker_covariates(control_marker, cross_object)  # Create the additive covariate markers
@@ -78,15 +78,22 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
             logger.info("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method)
 
         if num_perm > 0 and permCheck == "ON":                                                                   # Do permutation (if requested by user)
-            if do_control == "true" or cofactors != "":
-                perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = num_perm, model=model, method=method)
+            if len(perm_strata_list) > 0: #ZS: The strata list would only be populated if "Stratified" was checked on before mapping
+                cross_object, strata_ob = add_perm_strata(cross_object, perm_strata_list)
+                if do_control == "true" or cofactors != "":
+                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = int(num_perm), perm_strata = strata_ob, model=model, method=method)
+                else:
+                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, perm_strata = strata_ob, model=model, method=method)
             else:
-                perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method)
+                if do_control == "true" or cofactors != "":
+                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = int(num_perm), model=model, method=method)
+                else:
+                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method)
 
             perm_output, suggestive, significant = process_rqtl_perm_results(num_perm, perm_data_frame)          # Functions that sets the thresholds for the webinterface
-            return perm_output, suggestive, significant, process_rqtl_results(result_data_frame)
+            return perm_output, suggestive, significant, process_rqtl_results(result_data_frame, dataset.group.species)
         else:
-            return process_rqtl_results(result_data_frame)
+            return process_rqtl_results(result_data_frame, dataset.group.species)
 
 def generate_cross_from_rdata(dataset):
     rdata_location  = locate(dataset.group.name + ".RData", "genotype/rdata")
@@ -112,8 +119,12 @@ def generate_cross_from_geno(dataset):        # TODO: Need to figure out why som
          header = readLines(genotypes, 40)                                                                                 # Assume a geno header is not longer than 40 lines
          toskip = which(unlist(lapply(header, function(x){ length(grep("Chr\t", x)) })) == 1)-1                            # Major hack to skip the geno headers
 
-         genocodes <- c(getGenoCode(header, 'mat'), getGenoCode(header, 'het'), getGenoCode(header, 'pat'))                # Get the genotype codes
          type <- getGenoCode(header, 'type')
+         if(type == '4-way'){
+            genocodes <- c('1','2','3','4')
+         } else {
+            genocodes <- c(getGenoCode(header, 'mat'), getGenoCode(header, 'het'), getGenoCode(header, 'pat'))                # Get the genotype codes
+         }
          genodata <- read.csv(genotypes, sep='\t', skip=toskip, header=TRUE, na.strings=getGenoCode(header,'unk'), colClasses='character', comment.char = '#')
          cat('Genodata:', toskip, " ", dim(genodata), genocodes, '\n')
          if(is.null(phenotype)) phenotype <- runif((ncol(genodata)-4))                                                     # If there isn't a phenotype, generate a random one
@@ -127,7 +138,21 @@ def generate_cross_from_geno(dataset):        # TODO: Need to figure out why som
          if(type == 'riset') cross <- convert2riself(cross)                                                                # If its a RIL, convert to a RIL in R/qtl
          return(cross)
       }
-    """ % (dataset.group.name + ".geno"))
+    """ % (dataset.group.genofile))
+
+def add_perm_strata(cross, perm_strata):
+    col_string = 'c("the_strata")'
+    perm_strata_string = "c("
+    for item in perm_strata:
+        perm_strata_string += str(item) + ","
+
+    perm_strata_string = perm_strata_string[:-1] + ")"
+
+    cross = add_phenotype(cross, perm_strata_string, "the_strata")
+
+    strata_ob = pull_var("perm_strata", cross, col_string)
+
+    return cross, strata_ob
 
 def sanitize_rqtl_phenotype(vals):
     pheno_as_string = "c("
@@ -143,6 +168,7 @@ def sanitize_rqtl_phenotype(vals):
             else:
                 pheno_as_string += str(val)
     pheno_as_string += ")"
+
     return pheno_as_string
 
 def add_phenotype(cross, pheno_as_string, col_name):
@@ -150,11 +176,11 @@ def add_phenotype(cross, pheno_as_string, col_name):
     ro.r('the_cross$pheno <- cbind(pull.pheno(the_cross), ' + col_name + ' = '+ pheno_as_string +')')
     return ro.r["the_cross"]
 
-def pull_covar(cross, covar_name_string):
+def pull_var(var_name, cross, var_string):
     ro.globalenv["the_cross"] = cross
-    ro.r('trait_covars <- pull.pheno(the_cross, ' + covar_name_string + ')')
+    ro.r(var_name +' <- pull.pheno(the_cross, ' + var_string + ')')
 
-    return ro.r["trait_covars"]
+    return ro.r[var_name]
 
 def add_cofactors(cross, this_dataset, covariates, samples):
     ro.numpy2ri.activate()
@@ -190,19 +216,18 @@ def add_cofactors(cross, this_dataset, covariates, samples):
         covar_as_string += ")"
 
         col_name = "covar_" + str(i)
+        cross = add_phenotype(cross, covar_as_string, col_name)
 
         if i < (len(covariate_list) - 1):
             covar_name_string += '"' + col_name + '", '
         else:
             covar_name_string += '"' + col_name + '"'
 
-        cross = add_phenotype(cross, covar_as_string, col_name)
-
     covar_name_string += ")"
 
-    covars = pull_covar(cross, covar_name_string)
+    covars_ob = pull_var("trait_covars", cross, covar_name_string)
 
-    return cross, covars
+    return cross, covars_ob
 
 def create_marker_covariates(control_marker, cross):
     ro.globalenv["the_cross"] = cross
@@ -245,14 +270,17 @@ def process_rqtl_perm_results(num_perm, results):
 
     return perm_output, suggestive, significant
 
-def process_rqtl_results(result):        # TODO: how to make this a one liner and not copy the stuff in a loop
+def process_rqtl_results(result, species_name):        # TODO: how to make this a one liner and not copy the stuff in a loop
     qtl_results = []
     output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)]
 
     for i, line in enumerate(result.iter_row()):
         marker = {}
         marker['name'] = result.rownames[i]
-        marker['chr'] = output[i][0]
+        if species_name == "mouse" and output[i][0] == 20: #ZS: This is awkward, but I'm not sure how to change the 20s to Xs in the RData file
+            marker['chr'] = "X"
+        else:
+            marker['chr'] = output[i][0]
         marker['cM'] = output[i][1]
         marker['Mb'] = output[i][1]
         marker['lod_score'] = output[i][2]
diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py
index 3006c4ff..e191902c 100644
--- a/wqflask/wqflask/marker_regression/run_mapping.py
+++ b/wqflask/wqflask/marker_regression/run_mapping.py
@@ -38,6 +38,7 @@ from utility import Plot, Bunch
 from utility import temp_data
 from utility.benchmark import Bench
 from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping
+from wqflask.show_trait.SampleList import SampleList
 
 from utility.tools import locate, locate_ignore_error, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR
 from utility.external import shell
@@ -74,15 +75,41 @@ class RunMapping(object):
         self.vals = []
         if 'samples' in start_vars:
             self.samples = start_vars['samples'].split(",")
-            for sample in self.samples:
-                if (len(genofile_samplelist) == 0) or (sample in genofile_samplelist):
+            if (len(genofile_samplelist) != 0):
+                for sample in genofile_samplelist:
+                    if sample in self.samples:
+                        value = start_vars.get('value:' + sample)
+                        if value:
+                            self.vals.append(value)
+                    else:
+                        self.vals.append("x")
+            else:
+                for sample in self.samples:
                     value = start_vars.get('value:' + sample)
                     if value:
                         self.vals.append(value)
         else:
             self.samples = []
-            for sample in self.dataset.group.samplelist: # sample is actually the name of an individual
-                if (len(genofile_samplelist) == 0) or (sample in genofile_samplelist):
+            if (len(genofile_samplelist) != 0):
+                for sample in genofile_samplelist:
+                    if sample in self.dataset.group.samplelist:
+                        in_trait_data = False
+                        for item in self.this_trait.data:
+                            if self.this_trait.data[item].name == sample:
+                                value = start_vars['value:' + self.this_trait.data[item].name]
+                                self.samples.append(self.this_trait.data[item].name)
+                                self.vals.append(value)
+                                in_trait_data = True
+                                break
+                        if not in_trait_data:
+                            value = start_vars.get('value:' + sample)
+                            if value:
+                                self.samples.append(sample)
+                                self.vals.append(value)
+                    else:
+                        self.vals.append("x")
+            else:
+                for sample in self.dataset.group.samplelist: # sample is actually the name of an individual
                     in_trait_data = False
                     for item in self.this_trait.data:
                         if self.this_trait.data[item].name == sample:
@@ -204,8 +231,17 @@ class RunMapping(object):
         elif self.mapping_method == "rqtl_plink":
             results = self.run_rqtl_plink()
         elif self.mapping_method == "rqtl_geno":
+            perm_strata = []
+            if "perm_strata" in start_vars and "categorical_vars" in start_vars:
+                self.categorical_vars = start_vars["categorical_vars"].split(",")
+                if len(self.categorical_vars) and start_vars["perm_strata"] == "True":
+                    primary_samples = SampleList(dataset = self.dataset,
+                                                 sample_names = self.samples,
+                                                 this_trait = self.this_trait)
+
+                    perm_strata = get_perm_strata(self.this_trait, primary_samples, self.categorical_vars, self.samples)
             self.score_type = "LOD"
-            self.mapping_scale = "morgan"
+            #self.mapping_scale = "morgan"
             self.control_marker = start_vars['control_marker']
             self.do_control = start_vars['do_control']
             if 'mapmethod_rqtl_geno' in start_vars:
@@ -216,9 +252,9 @@ class RunMapping(object):
             #if start_vars['pair_scan'] == "true":
             #    self.pair_scan = True
             if self.permCheck and self.num_perm > 0:
-                self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.samples, self.covariates)
+                self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl_geno(self.vals, self.samples, self.dataset, self.method, self.model, self.permCheck, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates)
             else:
-                results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.samples, self.covariates)
+                results = rqtl_mapping.run_rqtl_geno(self.vals, self.samples, self.dataset, self.method, self.model, self.permCheck, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates)
         elif self.mapping_method == "reaper":
             if "startMb" in start_vars: #ZS: Check if first time page loaded, so it can default to ON
                 if "additiveCheck" in start_vars:
@@ -429,8 +465,8 @@ class RunMapping(object):
                       chr_lengths = chr_lengths,
                       num_perm = self.num_perm,
                       perm_results = self.perm_output,
-                      browser_files = browser_files,
                       significant = significant_for_browser,
+                      browser_files = browser_files,
                       selected_chr = this_chr
                   )
               else:
@@ -617,9 +653,14 @@ def get_chr_lengths(mapping_scale, dataset, qtl_results):
         this_chr = 1
         highest_pos = 0
         for i, result in enumerate(qtl_results):
-            if int(result['chr']) > this_chr or i == (len(qtl_results) - 1):
+            chr_as_num = 0
+            try:
+                chr_as_num = int(result['chr'])
+            except:
+                chr_as_num = 20
+            if chr_as_num > this_chr or i == (len(qtl_results) - 1):
                 chr_lengths.append({ "chr": str(this_chr), "size": str(highest_pos)})
-                this_chr = int(result['chr'])
+                this_chr = chr_as_num
                 highest_pos = 0
             else:
                 if float(result['Mb']) > highest_pos:
@@ -635,4 +676,25 @@ def get_genofile_samplelist(dataset):
         if genofile['location'] == dataset.group.genofile and 'sample_list' in genofile:
             genofile_samplelist = genofile['sample_list']
 
-    return genofile_samplelist
\ No newline at end of file
+    return genofile_samplelist
+
+def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples):
+    perm_strata_strings = []
+    for sample in used_samples:
+        if sample in sample_list.sample_attribute_values.keys():
+            combined_string = ""
+            for var in categorical_vars:
+                if var in sample_list.sample_attribute_values[sample].keys():
+                    combined_string += str(sample_list.sample_attribute_values[sample][var])
+                else:
+                    combined_string += "NA"
+        else:
+            combined_string = "NA"
+
+        perm_strata_strings.append(combined_string)
+
+    d = dict([(y,x+1) for x,y in enumerate(sorted(set(perm_strata_strings)))])
+    list_to_numbers = [d[x] for x in perm_strata_strings]
+    perm_strata = list_to_numbers
+
+    return perm_strata
\ No newline at end of file
diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py
index 451be50b..7e126a36 100644
--- a/wqflask/wqflask/show_trait/SampleList.py
+++ b/wqflask/wqflask/show_trait/SampleList.py
@@ -24,8 +24,8 @@ class SampleList(object):
                  dataset,
                  sample_names,
                  this_trait,
-                 sample_group_type,
-                 header):
+                 sample_group_type = "primary",
+                 header = "Samples"):
 
         self.dataset = dataset
         self.this_trait = this_trait
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index 10ce38a7..64deb942 100644
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -159,6 +159,8 @@ class ShowTrait(object):
             self.sample_group_types['samples_primary'] = self.dataset.group.name
         sample_lists = [group.sample_list for group in self.sample_groups]
 
+        categorical_var_list = get_categorical_variables(self.this_trait, self.sample_groups[0]) #ZS: Only using first samplelist, since I think mapping only uses those samples
+
         #ZS: Get list of chromosomes to select for mapping
         self.chr_list = [["All", -1]]
         for i, this_chr in enumerate(self.dataset.species.chromosomes.chromosomes):
@@ -226,6 +228,7 @@ class ShowTrait(object):
         hddn['mapping_display_all'] = True
         hddn['suggestive'] = 0
         hddn['num_perm'] = 0
+        hddn['categorical_vars'] = ""
         hddn['manhattan_plot'] = ""
         hddn['control_marker'] = ""
         if not self.temp_trait:
@@ -250,6 +253,7 @@ class ShowTrait(object):
                        sample_group_types = self.sample_group_types,
                        sample_lists = sample_lists,
                        attribute_names = self.sample_groups[0].attributes,
+                       categorical_vars = ",".join(categorical_var_list),
                        num_values = self.num_values,
                        qnorm_values = self.qnorm_vals,
                        zscore_values = self.z_scores,
@@ -570,10 +574,25 @@ def get_ncbi_summary(this_trait):
         #ZS: Need to switch this try/except to something that checks the output later
         try:
             response = requests.get("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=gene&id=%s&retmode=json" % this_trait.geneid)
-            logger.debug("NCBI:", json.loads(response.content)['result'][this_trait.geneid])
             summary = json.loads(response.content)['result'][this_trait.geneid]['summary']
             return summary
         except:
             return None
     else:
-        return None
\ No newline at end of file
+        return None
+
+def get_categorical_variables(this_trait, sample_list):
+    categorical_var_list = []
+
+    if len(sample_list.attributes) > 0:
+        for attribute in sample_list.attributes:
+            attribute_vals = []
+            for sample_name in this_trait.data.keys():
+                attribute_vals.append(this_trait.data[sample_name].extra_attributes[sample_list.attributes[attribute].name])
+
+            num_distinct = len(set(attribute_vals))
+
+            if num_distinct < 10:
+                categorical_var_list.append(sample_list.attributes[attribute].name)
+
+    return categorical_var_list
\ No newline at end of file
diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
index b26110d8..478ed87e 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
@@ -153,7 +153,7 @@
 
   //ZS: This is a list of inputs to be passed to the loading page, since not all inputs on the trait page are relevant to mapping
   var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale',
-                            'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'num_bootstrap', 'bootCheck', 'bootstrap_results',
+                            'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results',
                             'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'control_marker_db', 'do_control', 'genofile', 
                             'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 
                             'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples']
@@ -167,6 +167,7 @@
         $('input[name=selected_chr]').val($('#chr_rqtl_geno').val());
         $('input[name=genofile]').val($('#genofile_rqtl_geno').val());
         $('input[name=num_perm]').val($('input[name=num_perm_rqtl_geno]').val());
+        $('input[name=categorical_vars]').val(js_data.categorical_vars)
         $('input[name=manhattan_plot]').val($('input[name=manhattan_plot_rqtl]:checked').val());
         $('input[name=control_marker]').val($('input[name=control_rqtl_geno]').val());
         $('input[name=do_control]').val($('input[name=do_control_rqtl]:checked').val());
diff --git a/wqflask/wqflask/templates/collections/add.html b/wqflask/wqflask/templates/collections/add.html
index 058e269c..825dfb84 100644
--- a/wqflask/wqflask/templates/collections/add.html
+++ b/wqflask/wqflask/templates/collections/add.html
@@ -11,22 +11,9 @@
             {% else %}
             <input type="hidden" name="hash" value="{{ hash }}" />
             {% endif %}
-            <fieldset>
-              <legend>1. Create a new collection</legend>
-              <div style="margin-left: 20px;">
-                <!--<label>Collection name:</label>-->
-                <input type="text" name="new_collection" placeholder=" Name of new collection..."
-                    data-trigger="change" data-minlength="5" data-maxlength="50" style="width: 100%">
-                <button type="submit" name="create_new" class="btn btn-primary" style="margin-top: 20px;">Create collection</button>
-                {% if uc is not defined %}
-                <span class="help-block" style="color:red;">This collection will be saved to your computer for a year (or until you clear your cache).</span>
-                {% endif %}
-              </div>
-            </fieldset>
             {% if collections|length > 0 %}
-            <hr />
             <fieldset>
-              <legend>2. Add to an existing collection</legend>
+              <legend>1. Add to an existing collection</legend>
               <div style="margin-left: 20px;">  
                 <!--<label>Existing collection name:</label>-->
                 <select name="existing_collection" class="form-control">
@@ -44,6 +31,19 @@
               </div>
             </fieldset>
             {% endif %}
+            <hr />
+            <fieldset>
+              <legend>{% if collections|length > 0 %}2. {% else %}{% endif %}Create a new collection</legend>
+              <div style="margin-left: 20px;">
+                <!--<label>Collection name:</label>-->
+                <input type="text" name="new_collection" placeholder=" Name of new collection..."
+                    data-trigger="change" data-minlength="5" data-maxlength="50" style="width: 100%">
+                <button type="submit" name="create_new" class="btn btn-primary" style="margin-top: 20px;">Create collection</button>
+                {% if uc is not defined %}
+                <span class="help-block">This collection will be saved to your computer for a year (or until you clear your cache).</span>
+                {% endif %}
+              </div>
+            </fieldset>
         </form>
     </div>
 </div>
diff --git a/wqflask/wqflask/templates/collections/list.html b/wqflask/wqflask/templates/collections/list.html
index a2f1a1f7..3829b950 100644
--- a/wqflask/wqflask/templates/collections/list.html
+++ b/wqflask/wqflask/templates/collections/list.html
@@ -28,7 +28,7 @@
         <br>
         <div id="collections_list" style="width:50%; margin-top: 20px; margin-bottom: 20px;">
         {% if collections|length > 0 %}
-        <table class="table-hover table-striped cell-border" id='trait_table'>
+        <table class="table-hover table-striped cell-border" id='trait_table' style="float: left;">
             <thead>
                 <tr>
                     <th></th>
diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html
index 3b7f9671..59936a8e 100644
--- a/wqflask/wqflask/templates/collections/view.html
+++ b/wqflask/wqflask/templates/collections/view.html
@@ -88,8 +88,9 @@
             <button class="btn btn-danger" id="remove" disabled="disabled" type="button"><i class="icon-minus-sign"></i> Delete Rows</button>
             <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" title="Delete this collection" > Delete Collection</button>
             </form>
-            <br />
-            <br />
+            <div style="margin-top: 10px; margin-bottom: 5px;">
+                <b>Show/Hide Columns:</b>
+            </div>
             <div>
                 <table class="table-hover table-striped cell-border" id='trait_table' style="float: left;">
                     <thead>
@@ -127,6 +128,8 @@
                             </TD>
                             {% if this_trait.symbol %}
                             <TD data-export="{{ this_trait.symbol }}">{{ this_trait.symbol }}</TD>
+                            {% elif this_trait.abbreviation %}
+                            <TD data-export="{{ this_trait.abbreviation }}">{{ this_trait.abbreviation }}</TD>
                             {% else %}
                             <TD data-export="N/A">N/A</TD>
                             {% endif %}
@@ -139,9 +142,17 @@
                             {% endif %}
                             <TD data-export="{{ this_trait.location_repr }}">{{ this_trait.location_repr }}</TD>
                             <TD data-export="{{ this_trait.mean }}" align="right">{{ '%0.3f' % this_trait.mean|float }}</TD>
+                            {% if this_trait.LRS_score_repr|float > 0 %}
                             <TD data-export="{{ this_trait.LRS_score_repr }}" align="right">{{ '%0.3f' % this_trait.LRS_score_repr|float }}</TD>
+                            {% else %}
+                            <TD data-export="{{ this_trait.LRS_score_repr }}" align="right">N/A</TD>
+                            {% endif %}
                             <TD data-export="{{ this_trait.LRS_location_repr }}">{{ this_trait.LRS_location_repr }}</TD>
+                            {% if this_trait.additive|float > 0 %}
                             <TD data-export="{{ this_trait.additive }}" align="right">{{ '%0.3f' % this_trait.additive|float }}</TD>
+                            {% else %}
+                            <TD data-export="{{ this_trait.additive }}" align="right">N/A</TD>
+                            {% endif %}
                         </TR>
                     {% endfor %}
                     </tbody>
diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html
index 3e8baab6..03b03aa7 100644
--- a/wqflask/wqflask/templates/correlation_page.html
+++ b/wqflask/wqflask/templates/correlation_page.html
@@ -29,46 +29,42 @@
                 {{ this_trait.name }}:{{ this_trait.dataset }},
             {% endfor %}" >
 
+            <button id="corr_matrix" class="btn btn-primary submit_special" data-url="/corr_matrix" title="Correlation Matrix" >
+                Correlations
+            </button>
 
-                <button id="corr_matrix" class="btn btn-primary submit_special" data-url="/corr_matrix" title="Correlation Matrix" >
-                    Correlation Matrix
-                </button>
+            <button id="network_graph" class="btn btn-primary submit_special" data-url="/network_graph" title="Network Graph" >
+                Networks
+            </button>
 
+            <button id="send_to_webgestalt" class="btn btn-primary submit_special" data-url="/webgestalt_page" title="WebGestalt" >
+              WebGestalt
+            </button>
 
-                <button id="network_graph" class="btn btn-primary submit_special" data-url="/network_graph" title="Network Graph" >
-                    Network Graph
-                </button>
+            <button id="send_to_geneweaver" class="btn btn-primary submit_special" data-url="/geneweaver_page" title="GeneWeaver" >
+              GeneWeaver
+            </button>
 
+            <button id="send_to_bnw" class="btn btn-primary submit_special" data-url="/bnw_page" title="Bayesian Network" >
+              BNW
+           </button>
 
-                <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
-                    WGCNA Analysis
-                </button>
+            <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
+                WGCNA
+            </button>
 
+            <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
+                CTL Maps
+            </button>
 
-                <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
-                    CTL Analysis
-                </button>
+            <button id="heatmap" class="btn btn-primary submit_special" data-url="/heatmap" title="Heatmap" >
+                MultiMap
+            </button>
 
+            <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" >
+                Comparison Bar Chart
+            </button>
 
-                <button id="heatmap" class="btn btn-primary submit_special" data-url="/heatmap" title="Heatmap" >
-                    Heatmap
-                </button>
-
-                <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" >
-                    Comparison Bar Chart
-                </button>
-
-                <button id="send_to_webgestalt" class="btn btn-primary submit_special" data-url="/webgestalt_page" title="WebGestalt" >
-                    WebGestalt
-                </button>
-
-                <button id="send_to_geneweaver" class="btn btn-primary submit_special" data-url="/geneweaver_page" title="GeneWeaver" >
-                    GeneWeaver
-                </button>
-
-                <button id="send_to_bnw" class="btn btn-primary submit_special" data-url="/bnw_page" title="Bayesian Network" >
-                    BNW
-                </button>
           </form>
         </div>
         <br />
diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html
index bc614e01..49bcbff7 100644
--- a/wqflask/wqflask/templates/loading.html
+++ b/wqflask/wqflask/templates/loading.html
@@ -9,7 +9,7 @@
       <div style="min-height: 80vh; display: flex; align-items: center; text-align: center;">
         <div style="margin-bottom: 5px; left: 50%; margin-right: -50%; margin-top: 10%; transform: translate(-50%, -50%); position: absolute;">
         {% if start_vars.tool_used == "Mapping" %}
-        <h1>Computing the Map</h1>
+        <h1>Computing the Maps</h1>
         <br>
         <i>n</i> = {{ start_vars.num_vals }}
         <br>
diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html
index 162bde08..33221e0f 100644
--- a/wqflask/wqflask/templates/search_result_page.html
+++ b/wqflask/wqflask/templates/search_result_page.html
@@ -8,7 +8,7 @@
 {% endblock %}
 {% block content %}
 <!-- Start of body -->
-    <div style="padding-left: 10px; min-width: 1200px;">
+    <div style="padding-left: 10px; {% if dataset.type == 'Publish' %}min-width: 1000px;{% else %}min-width: 1200px;{% endif %}">
         <input type="hidden" name="uc_id" id="uc_id" value="{{ uc_id }}">
 
         <div style="padding-top: 10px; padding-bottom: 10px; font-size: 16px;">
@@ -68,45 +68,41 @@
 
 
                 <button id="corr_matrix" class="btn btn-primary submit_special" data-url="/corr_matrix" title="Correlation Matrix" >
-                    Correlation Matrix
+                    Correlations
                 </button>
 
-
                 <button id="network_graph" class="btn btn-primary submit_special" data-url="/network_graph" title="Network Graph" >
-                    Network Graph
+                    Networks
                 </button>
 
+                <button id="send_to_webgestalt" class="btn btn-primary submit_special" data-url="/webgestalt_page" title="WebGestalt" >
+                  WebGestalt
+                </button>
 
-                <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
-                    WGCNA Analysis
+                <button id="send_to_geneweaver" class="btn btn-primary submit_special" data-url="/geneweaver_page" title="GeneWeaver" >
+                  GeneWeaver
                 </button>
 
+                <button id="send_to_bnw" class="btn btn-primary submit_special" data-url="/bnw_page" title="Bayesian Network" >
+                  BNW
+               </button>
 
-                <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
-                    CTL Analysis
+                <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
+                    WGCNA
                 </button>
 
+                <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
+                    CTL Maps
+                </button>
 
                 <button id="heatmap" class="btn btn-primary submit_special" data-url="/heatmap" title="Heatmap" >
-                    Heatmap
+                    MultiMap
                 </button>
 
                 <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" >
                     Comparison Bar Chart
                 </button>
 
-                <button id="send_to_webgestalt" class="btn btn-primary submit_special" data-url="/webgestalt_page" title="WebGestalt" >
-                    WebGestalt
-                </button>
-
-                <button id="send_to_geneweaver" class="btn btn-primary submit_special" data-url="/geneweaver_page" title="GeneWeaver" >
-                    GeneWeaver
-                </button>
-
-                <button id="send_to_bnw" class="btn btn-primary submit_special" data-url="/bnw_page" title="Bayesian Network" >
-                    BNW
-                </button>
-
           </form>
         </div>
 
@@ -252,11 +248,13 @@
                     $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text());
                     $('td', row).eq(4).attr('title', $('td', row).eq(4).text());
                     $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text());
-                    $('td', row).slice(6,9).attr("align", "right");
+                    $('td', row).eq(4).attr('align', 'right');
+                    $('td', row).slice(6,10).attr("align", "right");
                     $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text());
                     $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text());
                     $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text());
                     $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text());
+                    $('td', row).eq(9).attr('data-export', $('td', row).eq(8).text());
                     {% elif dataset.type == 'Geno' %}
                     $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text());
                     {% endif %}
@@ -339,7 +337,7 @@
                     {
                       'title': "Description",
                       'type': "natural",
-                      'width': "25%",
+                      'width': "800px",
                       'data': null,
                       'render': function(data, type, row, meta) {
 			                  try {
@@ -359,7 +357,7 @@
                     {
                       'title': "Authors",
                       'type': "natural",
-                      'width': "25%",
+                      'width': "500px",
                       'data': null,
                       'render': function(data, type, row, meta) {
                         author_list = data.authors.split(",")
@@ -398,7 +396,7 @@
                     {
                       'title': "Max LRS Location",
                       'type': "natural",
-                      'width': "160px",
+                      'width': "200px",
                       'data': "lrs_location"
                     },
                     {
@@ -430,7 +428,6 @@
                     }
                 ],
                 'sDom': "Bitir",
-                'autoWidth': false,
                 'deferRender': true,
                 'paging': false,
                 'orderClasses': true,
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index 81661f86..27c3e398 100644
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -364,7 +364,7 @@
                   }
                 } );
 
-                primary_table.on( 'order.dt search.dt', function () {
+                primary_table.on( 'order.dt search.dt draw.dt', function () {
                   primary_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) {
                     cell.innerHTML = i+1;
                   } );
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index 01d90d21..777d4a2d 100644
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -243,6 +243,21 @@
                                 <input name="num_perm_rqtl_geno" value="200" type="text" class="form-control">
                             </div>
                         </div>
+                        {% if sample_groups[0].attributes|length > 0 %}
+                        <div class="mapping_method_fields form-group">
+                            <label style="text-align:right;" class="col-xs-3 control-label">Stratified</label>
+                            <div style="margin-left:20px;" class="col-xs-6 controls">
+                                <label class="radio-inline">
+                                    <input type="radio" name="perm_strata" value="True" checked=""> 
+                                    Yes
+                                </label>
+                                <label class="radio-inline">
+                                    <input type="radio" name="perm_strata" value="False" >
+                                    No
+                               </label>
+                            </div>
+                        </div>
+                        {% endif %}
                         <div class="mapping_method_fields form-group">
                             <label style="text-align:right;" for="control_for" class="col-xs-3 control-label">Control&nbsp;for</label>
                             <div style="margin-left:20px;" class="col-xs-6 controls">
@@ -365,7 +380,7 @@
              <dd>Maps traits with correction for kinship among samples using a linear mixed model method, and also allows users to fit multiple covariates such as sex, age, treatment, and genetic markers (<a href="https://www.ncbi.nlm.nih.gov/pubmed/24531419">PMID: 2453419</a>, and <a href="https://github.com/genetics-statistics/GEMMA"> GitHub code</a>). GEMMA incorporates the Leave One Chromosome Out (LOCO) method to ensure that the correction for kinship does not remove useful genetic variance near each marker. Markers can be filtered to include only those with minor allele frequencies (MAF) above a threshold. The default MAF is 0.05.</dd>
              {% elif mapping_method == "R/qtl" %}
              <dt style="margin-top: 20px;">R/qtl</dt>
-             <dd>The original R/qtl mapping code that supports most classic experimental crosses provided that they  do not have complex kinship or admixture (<a href="https://www.ncbi.nlm.nih.gov/pubmed/30591514">PMID: 30591514</a>). Both R/qtl, implemented here, and R/qtl2 are available as stand-alone R packages (<a href="https://kbroman.org/pages/software.html">R suite</a>).</dd>
+             <dd>The original R/qtl mapping package that supports classic experimental crosses including 4-parent F2 intercrosses (e.g., NIA ITP UM-HET3). R/qtl is ideal for populations that do not have complex kinship or admixture (<a href="https://www.ncbi.nlm.nih.gov/pubmed/12724300">PMID: 12724300</a>). Both R/qtl as implemented here, and R/qtl2 (<a href="https://www.ncbi.nlm.nih.gov/pubmed/30591514">PMID: 30591514</a>) are available as <a href="https://kbroman.org/pages/software.html">R suites</a>.</dd>
              {% elif mapping_method == "QTLReaper" %}
              <dt style="margin-top: 20px;">Haley-Knott Regression</dt>
              <dd>Fast linear mapping method (<a href="https://www.ncbi.nlm.nih.gov/pubmed/16718932">PMID 16718932</a>) works well with F2 intercrosses and backcrosses, but that is not recommended for complex or admixed populations (e.g., GWAS or heterogeneous stock studies) or for advanced intercrosses, recombinant inbred families, or diallel crosses. Interactive plots in GeneNetwork have relied on the fast HK mapping for two decades and we still use this method for mapping omics data sets and computing genome-wide permutation threshold (<a href="https://github.com/pjotrp/QTLReaper">QTL Reaper code</a>).</dd>
diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py
index 71572c03..d75a03df 100644
--- a/wqflask/wqflask/user_session.py
+++ b/wqflask/wqflask/user_session.py
@@ -144,6 +144,7 @@ class UserSession(object):
 
         #ZS: Get user's collections if they exist
         collections = get_user_collections(self.redis_user_id)
+        collections = [item for item in collections if item['name'] != "Your Default Collection"] + [item for item in collections if item['name'] == "Your Default Collection"] #ZS: Ensure Default Collection is last in list
         return collections
 
     @property
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 0fbe370d..923c89bd 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -15,6 +15,7 @@ import xlsxwriter
 import StringIO  # Todo: Use cStringIO?
 
 import gc
+import numpy as np
 
 import cPickle as pickle
 import uuid
@@ -392,9 +393,9 @@ def export_perm_data():
 
     buff = StringIO.StringIO()
     writer = csv.writer(buff)
-    writer.writerow(["Suggestive LRS (p=0.63) = " + str(perm_data[int(num_perm*0.37-1)])])
-    writer.writerow(["Significant LRS (p=0.05) = " + str(perm_data[int(num_perm*0.95-1)])])
-    writer.writerow(["Highly Significant LRS (p=0.01) = " + str(perm_data[int(num_perm*0.99-1)])])
+    writer.writerow(["Suggestive LRS (p=0.63) = " + str(np.percentile(np.array(perm_data), 67))])
+    writer.writerow(["Significant LRS (p=0.05) = " + str(np.percentile(np.array(perm_data), 95))])
+    writer.writerow(["Highly Significant LRS (p=0.01) = " + str(np.percentile(np.array(perm_data), 99))])
     writer.writerow("")
     writer.writerow([str(num_perm) + " Permutations"])
     writer.writerow("")
@@ -595,7 +596,7 @@ def loading_page():
 @app.route("/run_mapping", methods=('POST',))
 def mapping_results_page():
     initial_start_vars = request.form
-    logger.debug("Mapping called with initial_start_vars:", initial_start_vars.items())
+    #logger.debug("Mapping called with initial_start_vars:", initial_start_vars.items())
     logger.info(request.url)
     temp_uuid = initial_start_vars['temp_uuid']
     wanted = (
@@ -620,6 +621,9 @@ def mapping_results_page():
         'significant',
         'num_perm',
         'permCheck',
+        'perm_strata',
+        'strat_var',
+        'categorical_vars',
         'perm_output',
         'num_bootstrap',
         'bootCheck',
@@ -654,11 +658,11 @@ def mapping_results_page():
     for key, value in initial_start_vars.iteritems():
         if key in wanted or key.startswith(('value:')):
             start_vars[key] = value
-    logger.debug("Mapping called with start_vars:", start_vars)
+    #logger.debug("Mapping called with start_vars:", start_vars)
 
     version = "v3"
     key = "mapping_results:{}:".format(version) + json.dumps(start_vars, sort_keys=True)
-    logger.info("key is:", pf(key))
+    #logger.info("key is:", pf(key))
     with Bench("Loading cache"):
         result = None # Just for testing
         #result = Redis.get(key)
-- 
cgit v1.2.3


From 6178a48d29cd83fd3beb70854721070826e230e3 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 22 Apr 2020 16:03:07 -0500
Subject: Fixed a variety of issues related to users registering and logging in

GN2 should now work when Redis is empty
---
 wqflask/utility/startup_config.py        |  4 +-
 wqflask/utility/tools.py                 |  8 +--
 wqflask/wqflask/collect.py               |  2 +-
 wqflask/wqflask/show_trait/SampleList.py | 84 ++++++++++++++++----------------
 wqflask/wqflask/show_trait/show_trait.py |  4 +-
 wqflask/wqflask/user_login.py            | 44 ++++++++---------
 wqflask/wqflask/user_session.py          | 13 +++--
 7 files changed, 80 insertions(+), 79 deletions(-)

(limited to 'wqflask/utility')

diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py
index 5a62cc50..817284dd 100644
--- a/wqflask/utility/startup_config.py
+++ b/wqflask/utility/startup_config.py
@@ -33,7 +33,7 @@ def app_config():
         if page.status_code != 200:
             raise Exception("API server not found!")
 
-    import utility.elasticsearch_tools as es
-    es.test_elasticsearch_connection()
+    # import utility.elasticsearch_tools as es
+    # es.test_elasticsearch_connection()
 
     print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL")))
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 75bddb24..0fbedccb 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -267,10 +267,10 @@ if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET:
                       ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET
     ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL')
 
-ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
-ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
-import utility.elasticsearch_tools as es
-es.test_elasticsearch_connection()
+# ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
+# ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
+# import utility.elasticsearch_tools as es
+# es.test_elasticsearch_connection()
 
 SMTP_CONNECT = get_setting('SMTP_CONNECT')
 SMTP_USERNAME = get_setting('SMTP_USERNAME')
diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py
index fa6e03b4..1d74b699 100644
--- a/wqflask/wqflask/collect.py
+++ b/wqflask/wqflask/collect.py
@@ -108,7 +108,7 @@ def collections_new():
         if 'existing_collection' not in params:
             collections = g.user_session.user_collections
             for collection in collections:
-                if collection["name"] == "Default Collection":
+                if collection["name"] == "Your Default Collection":
                     collection_id = collection["id"]
                     collection_name = collection["name"]
                     default_collection_exists = True
diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py
index 7e126a36..ad78ebcc 100644
--- a/wqflask/wqflask/show_trait/SampleList.py
+++ b/wqflask/wqflask/show_trait/SampleList.py
@@ -14,8 +14,6 @@ import simplejson as json
 
 import itertools
 
-from utility.elasticsearch_tools import get_elasticsearch_connection
-
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
 
@@ -158,47 +156,47 @@ class SampleList(object):
 
         return any(sample.variance for sample in self.sample_list)
 
-def get_transform_vals(dataset, trait):
-    es = get_elasticsearch_connection(for_user=False)
-
-    logger.info("DATASET NAME:", dataset.name)
-
-    query = '{"bool": {"must": [{"match": {"name": "%s"}}, {"match": {"dataset": "%s"}}]}}' % (trait.name, dataset.name)
-
-    es_body = {
-          "query": {
-            "bool": {
-              "must": [
-                {
-                  "match": {
-                    "name": "%s" % (trait.name)
-                  }
-                },
-                {
-                  "match": {
-                    "dataset": "%s" % (dataset.name)
-                  }
-                }
-              ]
-            }
-          }
-    }
-
-    response = es.search( index = "traits", doc_type = "trait", body = es_body )
-    logger.info("THE RESPONSE:", response)
-    results = response['hits']['hits']
-
-    if len(results) > 0:
-        samples = results[0]['_source']['samples']
-
-        sample_dict = {}
-        for sample in samples:
-            sample_dict[sample['name']] = sample['qnorm']
-
-        #logger.info("SAMPLE DICT:", sample_dict)
-        return sample_dict
-    else:
-        return None
+# def get_transform_vals(dataset, trait):
+#     es = get_elasticsearch_connection(for_user=False)
+
+#     logger.info("DATASET NAME:", dataset.name)
+
+#     query = '{"bool": {"must": [{"match": {"name": "%s"}}, {"match": {"dataset": "%s"}}]}}' % (trait.name, dataset.name)
+
+#     es_body = {
+#           "query": {
+#             "bool": {
+#               "must": [
+#                 {
+#                   "match": {
+#                     "name": "%s" % (trait.name)
+#                   }
+#                 },
+#                 {
+#                   "match": {
+#                     "dataset": "%s" % (dataset.name)
+#                   }
+#                 }
+#               ]
+#             }
+#           }
+#     }
+
+#     response = es.search( index = "traits", doc_type = "trait", body = es_body )
+#     logger.info("THE RESPONSE:", response)
+#     results = response['hits']['hits']
+
+#     if len(results) > 0:
+#         samples = results[0]['_source']['samples']
+
+#         sample_dict = {}
+#         for sample in samples:
+#             sample_dict[sample['name']] = sample['qnorm']
+
+#         #logger.info("SAMPLE DICT:", sample_dict)
+#         return sample_dict
+#     else:
+#         return None
 
 def natural_sort_key(x):
     """Get expected results when using as a key for sort - ints or strings are sorted properly"""
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index d35ba749..8883e627 100644
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -364,8 +364,8 @@ class ShowTrait(object):
                 if self.dataset:
                     dataset_menu_selected = self.dataset.name
 
-                return_results_menu = (100, 200, 500, 1000, 2000, 5000, 10000, 15000, 20000)
-                return_results_menu_selected = 500
+            return_results_menu = (100, 200, 500, 1000, 2000, 5000, 10000, 15000, 20000)
+            return_results_menu_selected = 500
 
             self.corr_tools = dict(dataset_menu = dataset_menu,
                                           dataset_menu_selected = dataset_menu_selected,
diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py
index da3cc504..40d9925c 100644
--- a/wqflask/wqflask/user_login.py
+++ b/wqflask/wqflask/user_login.py
@@ -6,7 +6,6 @@ import datetime
 import time
 import logging
 import uuid
-import hashlib
 import hmac
 import base64
 import requests
@@ -42,17 +41,23 @@ def basic_info():
                 ip_address = request.remote_addr,
                 user_agent = request.headers.get('User-Agent'))
 
-def encode_password(pass_gen_fields):
+def encode_password(pass_gen_fields, unencrypted_password):
+    logger.debug("THE TYPE:", type(pass_gen_fields))
+    logger.debug("pass_gen_fields:", pass_gen_fields)
+    logger.debug("hashfunc:", pass_gen_fields['hashfunc'])
     hashfunc = getattr(hashlib, pass_gen_fields['hashfunc'])
 
     salt = base64.b64decode(pass_gen_fields['salt'])
-    password = pbkdf2.pbkdf2_hex(str(pass_gen_fields['unencrypted_password']), 
+    encrypted_password = pbkdf2.pbkdf2_hex(str(unencrypted_password), 
                                  pass_gen_fields['salt'], 
                                  pass_gen_fields['iterations'], 
                                  pass_gen_fields['keylength'], 
                                  hashfunc)
 
-    return password
+    pass_gen_fields.pop("unencrypted_password", None)
+    pass_gen_fields["password"] = encrypted_password
+
+    return pass_gen_fields
 
 def set_password(password):
     pass_gen_fields = {
@@ -67,19 +72,10 @@ def set_password(password):
 
     assert len(password) >= 6, "Password shouldn't be shorter than 6 characters"
 
-    encoded_password = encode_password(pass_gen_fields)
+    encoded_password = encode_password(pass_gen_fields, pass_gen_fields['unencrypted_password'])
 
     return encoded_password
 
-def encrypt_password(unencrypted_password, pwfields):
-        hashfunc = getattr(hashlib, pwfields['hashfunc'])
-        salt = base64.b64decode(pwfields['salt'])
-        iterations = pwfields['iterations']
-        keylength = pwfields['keylength']
-        encrypted_password = pbkdf2.pbkdf2_hex(str(unencrypted_password),
-                                               salt, iterations, keylength, hashfunc)
-        return encrypted_password
-
 def get_signed_session_id(user):
     session_id = str(uuid.uuid4())
 
@@ -186,9 +182,12 @@ def login():
             password_match = False
             if user_details:
                 submitted_password = params['password']
-                pwfields = json.loads(user_details['password'])
-                encrypted_pass = encrypt_password(submitted_password, pwfields)
-                password_match = pbkdf2.safe_str_cmp(encrypted_pass, pwfields['password'])
+                pwfields = user_details['password']
+                if type(pwfields) is str:
+                    pwfields = json.loads(pwfields)
+                encrypted_pass_fields = encode_password(pwfields, submitted_password)
+                password_match = pbkdf2.safe_str_cmp(encrypted_pass_fields['password'], pwfields['password'])
+
             else: # Invalid e-mail
                 flash("Invalid e-mail address. Please try again.", "alert-danger")
                 response = make_response(redirect(url_for('login')))
@@ -226,7 +225,7 @@ def github_oauth2():
         "client_secret": GITHUB_CLIENT_SECRET,
         "code": code
     }
-    logger.debug("LOGIN DATA:", data)
+
     result = requests.post("https://github.com/login/oauth/access_token", json=data)
     result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]}
 
@@ -437,19 +436,18 @@ def register_user(params):
         if params.get('password_confirm') != password:
             errors.append("Passwords don't match.")
 
-        if errors:
-            return errors
-
         user_details['password'] = set_password(password)
         user_details['user_id'] = str(uuid.uuid4())
         user_details['confirmed'] = 1
 
-        user_details['registration_info'] = json.dumps(basic_info(), sort_keys=True)
+        user_details['registration_info'] = basic_info()
         save_user(user_details, user_details['user_id'])
 
+        return errors
+
 @app.route("/n/register", methods=('GET', 'POST'))
 def register():
-    errors = None
+    errors = []
 
     params = request.form if request.form else request.args
     params = params.to_dict(flat=True)
diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py
index d75a03df..fd1779fb 100644
--- a/wqflask/wqflask/user_session.py
+++ b/wqflask/wqflask/user_session.py
@@ -9,6 +9,7 @@ import simplejson as json
 import redis # used for collections
 Redis = redis.StrictRedis()
 
+
 from flask import (Flask, g, render_template, url_for, request, make_response,
                    redirect, flash, abort)
 
@@ -69,19 +70,23 @@ class UserSession(object):
         if not self.record or self.record == []:
             if user_cookie:
                 self.logged_in = False
+                self.record = dict(login_time = time.time(),
+                                    user_type = "anon",
+                                    user_id = str(uuid.uuid4()))
+                Redis.hmset(self.redis_key, self.record)
+                Redis.expire(self.redis_key, THIRTY_DAYS)
+                response = make_response(redirect(url_for('login')))
+                response.set_cookie(self.user_cookie_name, '', expires=0)
 
                 ########### Grrr...this won't work because of the way flask handles cookies
                 # Delete the cookie
-                response = make_response(redirect(url_for('login')))
-                #response.set_cookie(self.cookie_name, '', expires=0)
                 flash("Due to inactivity your session has expired. If you'd like please login again.")
-                #return response
+                return response
                 #return
             else:
                 self.record = dict(login_time = time.time(),
                                     user_type = "anon",
                                     user_id = str(uuid.uuid4()))
-
                 Redis.hmset(self.redis_key, self.record)
                 Redis.expire(self.redis_key, THIRTY_DAYS)
         else:
-- 
cgit v1.2.3


From be09d207ee4e2705e358102f8bdbcd1da7f70ca2 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Fri, 24 Apr 2020 16:06:36 -0500
Subject: Replaced hard-coded instances of GN2 urls with ones pulled from
 settings file

---
 doc/API_readme.md                                  | 42 +++++++++++-----------
 wqflask/base/data_set.py                           |  4 +--
 wqflask/base/trait.py                              |  7 ++--
 wqflask/utility/tools.py                           |  2 ++
 wqflask/wqflask/ctl/ctl_analysis.py                |  4 ++-
 wqflask/wqflask/do_search.py                       |  3 +-
 wqflask/wqflask/network_graph/network_graph.py     |  2 ++
 wqflask/wqflask/search_results.py                  |  3 +-
 wqflask/wqflask/static/new/javascript/ctl_graph.js |  8 +----
 .../new/javascript/dataset_select_menu_orig.js     |  1 -
 .../wqflask/static/new/javascript/network_graph.js |  4 +--
 wqflask/wqflask/templates/ctl_results.html         |  1 +
 wqflask/wqflask/templates/index_page.html          |  4 +--
 wqflask/wqflask/templates/index_page_orig.html     |  4 +--
 wqflask/wqflask/templates/network_graph.html       |  1 +
 15 files changed, 47 insertions(+), 43 deletions(-)

(limited to 'wqflask/utility')

diff --git a/doc/API_readme.md b/doc/API_readme.md
index 652376a0..be6668dc 100644
--- a/doc/API_readme.md
+++ b/doc/API_readme.md
@@ -6,17 +6,17 @@
 
 To get a list of species with data available in GN (and their associated names and ids):
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/species
+curl http://genenetwork.org/api/v_pre1/species
 [ { "FullName": "Mus musculus", "Id": 1, "Name": "mouse", "TaxonomyId": 10090 }, ... { "FullName": "Populus trichocarpa", "Id": 10, "Name": "poplar", "TaxonomyId": 3689 } ]
 ```
 
 Or to get a single species info:
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/species/mouse
+curl http://genenetwork.org/api/v_pre1/species/mouse
 ``` 
 OR 
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/species/mouse.json
+curl http://genenetwork.org/api/v_pre1/species/mouse.json
 ```
 
 *For all queries where the last field is a user-specified name/ID, there will be the option to append a file format type. Currently there is only JSON (and it will default to JSON if none is provided), but other formats will be added later*
@@ -26,33 +26,33 @@ curl http://gn2.genenetwork.org/api/v_pre1/species/mouse.json
 This query can optionally filter by species:
 
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/groups (for all species)
+curl http://genenetwork.org/api/v_pre1/groups (for all species)
 ```
 OR
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/groups/mouse (for just mouse groups/RISets)
+curl http://genenetwork.org/api/v_pre1/groups/mouse (for just mouse groups/RISets)
 [ { "DisplayName": "BXD", "FullName": "BXD RI Family", "GeneticType": "riset", "Id": 1, "MappingMethodId": "1", "Name": "BXD", "SpeciesId": 1, "public": 2 }, ... { "DisplayName": "AIL LGSM F34 and F39-43 (GBS)", "FullName": "AIL LGSM F34 and F39-43 (GBS)", "GeneticType": "intercross", "Id": 72, "MappingMethodId": "2", "Name": "AIL-LGSM-F34-F39-43-GBS", "SpeciesId": 1, "public": 2 } ]
 ```
 
 ## Fetch Genotypes for Group/RISet ##
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/genotypes/bimbam/BXD
-curl http://gn2.genenetwork.org/api/v_pre1/genotypes/BXD.bimbam
+curl http://genenetwork.org/api/v_pre1/genotypes/bimbam/BXD
+curl http://genenetwork.org/api/v_pre1/genotypes/BXD.bimbam
 ```
 Returns a group's genotypes in one of several formats - bimbam, rqtl2, or geno (a format used by qtlreaper which is just a CSV file consisting of marker positions and genotypes)
 
 Rqtl2 genotype queries can also include the dataset name and will return a zip of the genotypes, phenotypes, and gene map (marker names/positions). For example:
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/genotypes/rqtl2/BXD/HC_M2_0606_P.zip
+curl http://genenetwork.org/api/v_pre1/genotypes/rqtl2/BXD/HC_M2_0606_P.zip
 ```
 
 ## Fetch Datasets ##
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/datasets/bxd
+curl http://genenetwork.org/api/v_pre1/datasets/bxd
 ```
 OR
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/datasets/mouse/bxd
+curl http://genenetwork.org/api/v_pre1/datasets/mouse/bxd
 [ { "AvgID": 1, "CreateTime": "Fri, 01 Aug 2003 00:00:00 GMT", "DataScale": "log2", "FullName": "UTHSC/ETHZ/EPFL BXD Liver Polar Metabolites Extraction A, CD Cohorts (Mar 2017) log2", "Id": 1, "Long_Abbreviation": "BXDMicroArray_ProbeSet_August03", "ProbeFreezeId": 3, "ShortName": "Brain U74Av2 08/03 MAS5", "Short_Abbreviation": "Br_U_0803_M", "confidentiality": 0, "public": 0 }, ... { "AvgID": 3, "CreateTime": "Tue, 14 Aug 2018 00:00:00 GMT", "DataScale": "log2", "FullName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Id": 859, "Long_Abbreviation": "EPFLMouseLiverCDRMAApr18", "ProbeFreezeId": 181, "ShortName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Short_Abbreviation": "EPFLMouseLiverCDRMA0818", "confidentiality": 0, "public": 1 } ]
 ```
 (I added the option to specify species just in case we end up with the same group name across multiple species at some point, though it's currently unnecessary)
@@ -61,11 +61,11 @@ curl http://gn2.genenetwork.org/api/v_pre1/datasets/mouse/bxd
 ### For mRNA Assay/"ProbeSet" ###
 
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/dataset/HC_M2_0606_P
+curl http://genenetwork.org/api/v_pre1/dataset/HC_M2_0606_P
 ```
 OR
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/dataset/bxd/HC_M2_0606_P```
+curl http://genenetwork.org/api/v_pre1/dataset/bxd/HC_M2_0606_P```
 { "confidential": 0, "data_scale": "log2", "dataset_type": "mRNA expression", "full_name": "Hippocampus Consortium M430v2 (Jun06) PDNN", "id": 112, "name": "HC_M2_0606_P", "public": 2, "short_name": "Hippocampus M430v2 BXD 06/06 PDNN", "tissue": "Hippocampus mRNA", "tissue_id": 9 }
 ```
 (This also has the option to specify group/riset)
@@ -73,26 +73,26 @@ curl http://gn2.genenetwork.org/api/v_pre1/dataset/bxd/HC_M2_0606_P```
 ### For "Phenotypes" (basically non-mRNA Expression; stuff like weight, sex, etc) ###
 For these traits, the query fetches publication info and takes the group and phenotype 'ID' as input. For example:
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/dataset/bxd/10001
+curl http://genenetwork.org/api/v_pre1/dataset/bxd/10001
 { "dataset_type": "phenotype", "description": "Central nervous system, morphology: Cerebellum weight, whole, bilateral in adults of both sexes [mg]", "id": 10001, "name": "CBLWT2", "pubmed_id": 11438585, "title": "Genetic control of the mouse cerebellum: identification of quantitative trait loci modulating size and architecture", "year": "2001" }
 ```
 
 ## Fetch Sample Data for Dataset ##
 ``` 
-curl http://gn2.genenetwork.org/api/v_pre1/sample_data/HSNIH-PalmerPublish.csv
+curl http://genenetwork.org/api/v_pre1/sample_data/HSNIH-PalmerPublish.csv
 ```
 
 Returns a CSV file with sample/strain names as the columns and trait IDs as rows
 
 ## Fetch Sample Data for Single Trait ##
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/sample_data/HC_M2_0606_P/1436869_at
+curl http://genenetwork.org/api/v_pre1/sample_data/HC_M2_0606_P/1436869_at
 [ { "data_id": 23415463, "sample_name": "129S1/SvImJ", "sample_name_2": "129S1/SvImJ", "se": 0.123, "value": 8.201 }, { "data_id": 23415463, "sample_name": "A/J", "sample_name_2": "A/J", "se": 0.046, "value": 8.413 }, { "data_id": 23415463, "sample_name": "AKR/J", "sample_name_2": "AKR/J", "se": 0.134, "value": 8.856 }, ... ]
 ```
 
 ## Fetch Trait List for Dataset ##
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/traits/HXBBXHPublish.json
+curl http://genenetwork.org/api/v_pre1/traits/HXBBXHPublish.json
 [ { "Additive": 0.0499967532467532, "Id": 10001, "LRS": 16.2831307029479, "Locus": "rs106114574", "PhenotypeId": 1449, "PublicationId": 319, "Sequence": 1 }, ... ]
 ```
 
@@ -101,7 +101,7 @@ Both JSON and CSV formats can be specified, with JSON as default. There is also
 ## Fetch Trait Info (Name, Description, Location, etc) ##
 ### For mRNA Expression/"ProbeSet" ###
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/trait/HC_M2_0606_P/1436869_at
+curl http://genenetwork.org/api/v_pre1/trait/HC_M2_0606_P/1436869_at
 { "additive": -0.214087568058076, "alias": "HHG1; HLP3; HPE3; SMMCI; Dsh; Hhg1", "chr": "5", "description": "sonic hedgehog (hedgehog)", "id": 99602, "locus": "rs8253327", "lrs": 12.7711275309832, "mb": 28.457155, "mean": 9.27909090909091, "name": "1436869_at", "p_value": 0.306, "se": null, "symbol": "Shh" }
 ```
 
@@ -110,7 +110,7 @@ For phenotypes this just gets the  max LRS, its location, and additive effect (a
 
 Since each group/riset only has one phenotype "dataset", this query takes either the group/riset name or the group/riset name + "Publish" (for example "BXDPublish", which is the dataset name in the DB) as input
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/trait/BXD/10001
+curl http://genenetwork.org/api/v_pre1/trait/BXD/10001
 { "additive": 2.39444435069444, "id": 4, "locus": "rs48756159", "lrs": 13.4974911471087 }
 ```
 
@@ -130,7 +130,7 @@ Each method's query takes the following parameters respectively (more will be ad
 
 Example query:
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/mapping?trait_id=10015&db=BXDPublish&method=gemma&use_loco=true
+curl http://genenetwork.org/api/v_pre1/mapping?trait_id=10015&db=BXDPublish&method=gemma&use_loco=true
 ```
 
 ### R/qtl ###
@@ -146,7 +146,7 @@ curl http://gn2.genenetwork.org/api/v_pre1/mapping?trait_id=10015&db=BXDPublish&
 
 Example query:
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/mapping?trait_id=1418701_at&db=HC_M2_0606_P&method=rqtl&num_perm=100
+curl http://genenetwork.org/api/v_pre1/mapping?trait_id=1418701_at&db=HC_M2_0606_P&method=rqtl&num_perm=100
 ```
 
 Some combinations of methods/models may not make sense. The R/qtl manual should be referred to for any questions on its use (specifically the scanone function in this case)
@@ -164,6 +164,6 @@ This query currently takes the following parameters (though more will be added):
 
 Example query:
 ```
-curl http://gn2.genenetwork.org/api/v_pre1/correlation?trait_id=1427571_at&db=HC_M2_0606_P&target_db=BXDPublish&type=sample&return_count=100
+curl http://genenetwork.org/api/v_pre1/correlation?trait_id=1427571_at&db=HC_M2_0606_P&target_db=BXDPublish&type=sample&return_count=100
 [ { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20511 }, { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20724 }, { "#_strains": 12, "p_value": 1.8288943424888848e-05, "sample_r": -0.9233615170820528, "trait": 13536 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": 0.8928571428571429, "trait": 10157 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": -0.8928571428571429, "trait": 20392 }, ... ]
 ```
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index fae62875..8652e6b7 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -56,7 +56,7 @@ from pprint import pformat as pf
 from db.gn_server import menu_main
 from db.call import fetchall,fetchone,fetch1
 
-from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists
+from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL
 from utility.logger import getLogger
 logger = getLogger(__name__ )
 
@@ -94,7 +94,7 @@ Publish or ProbeSet. E.g.
         """
         self.datasets = {}
         if rebuild: #ZS: May make this the only option
-            data = json.loads(requests.get("http://gn2.genenetwork.org/api/v_pre1/gen_dropdown").content)
+            data = json.loads(requests.get(GN2_BASE_URL + "/api/v_pre1/gen_dropdown").content)
             #data = gen_menu.gen_dropdown_json()
         else:
             file_name = "wqflask/static/new/javascript/dataset_menu_structure.json"
diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index 5525472e..e454c593 100644
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -14,6 +14,7 @@ from base.data_set import create_dataset
 from db import webqtlDatabaseFunction
 from utility import webqtlUtil
 from utility import hmac
+from utility.tools import GN2_BASE_URL
 
 from wqflask import app
 
@@ -135,9 +136,9 @@ class GeneralTrait(object):
 
         alias = 'Not available'
         if self.symbol:
-            human_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.upper())
-            mouse_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.capitalize())
-            other_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.lower())
+            human_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.upper())
+            mouse_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.capitalize())
+            other_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.lower())
 
             if human_response and mouse_response and other_response:
                 alias_list = json.loads(human_response.content) + json.loads(mouse_response.content) + json.loads(other_response.content)
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 0fbedccb..9354ece6 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -234,6 +234,8 @@ def show_settings():
 GN_VERSION         = get_setting('GN_VERSION')
 HOME               = get_setting('HOME')
 WEBSERVER_MODE     = get_setting('WEBSERVER_MODE')
+GN2_BASE_URL       = get_setting('GN2_BASE_URL')
+GN2_BRANCH_URL     = get_setting('GN2_BRANCH_URL')
 GN_SERVER_URL      = get_setting('GN_SERVER_URL')
 SERVER_PORT        = get_setting_int('SERVER_PORT')
 SQL_URI            = get_setting('SQL_URI')
diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py
index 6fda02fd..4415b86a 100644
--- a/wqflask/wqflask/ctl/ctl_analysis.py
+++ b/wqflask/wqflask/ctl/ctl_analysis.py
@@ -20,7 +20,7 @@ from base import data_set
 from base import trait as TRAIT
 
 from utility import helper_functions
-from utility.tools import locate
+from utility.tools import locate, GN2_BRANCH_URL
 
 from rpy2.robjects.packages import importr
 
@@ -56,6 +56,8 @@ class CTL(object):
         self.edges_list = []
         logger.info("Obtained pointers to CTL functions")
 
+        self.gn2_url = GN2_BRANCH_URL
+
     def addNode(self, gt):
         node_dict = { 'data' : {'id' : str(gt.name) + ":" + str(gt.dataset.name),
                                 'sid' : str(gt.name), 
diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py
index 05caa100..b0ca5ced 100644
--- a/wqflask/wqflask/do_search.py
+++ b/wqflask/wqflask/do_search.py
@@ -13,6 +13,7 @@ import sys
 # sys.path.append("..") Never in a running webserver
 
 from db import webqtlDatabaseFunction
+from utility.tools import GN2_BASE_URL
 
 import logging
 from utility.logger import getLogger
@@ -919,7 +920,7 @@ def get_aliases(symbol, species):
         return []
 
     filtered_aliases = []
-    response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + symbol_string)
+    response = requests.get(GN2_BASE_URL + "/gn3/gene/aliases/" + symbol_string)
     if response:
         alias_list = json.loads(response.content)
 
diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py
index a332db46..152e4168 100644
--- a/wqflask/wqflask/network_graph/network_graph.py
+++ b/wqflask/wqflask/network_graph/network_graph.py
@@ -47,6 +47,7 @@ from utility.TDCell import TDCell
 from base.trait import GeneralTrait
 from base import data_set
 from utility import webqtlUtil, helper_functions, corr_result_helpers
+from utility.tools import GN2_BRANCH_URL
 from db import webqtlDatabaseFunction
 import utility.webqtlUtil #this is for parallel computing only.
 from wqflask.correlation import correlation_functions
@@ -195,6 +196,7 @@ class NetworkGraph(object):
             self.nodes_list.append(node_dict)
 
         self.elements = json.dumps(self.nodes_list + self.edges_list)
+        self.gn2_url = GN2_BRANCH_URL
 
         groups = []
         for sample in self.all_sample_list:
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index 698389ab..8f702d58 100644
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -28,6 +28,7 @@ from flask import render_template, Flask, g
 
 from utility import formatting
 from utility import hmac
+from utility.tools import GN2_BASE_URL
 from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string
 
 from utility.logger import getLogger
@@ -295,7 +296,7 @@ def get_aliases(symbol_list, species):
     symbols_string = ",".join(updated_symbols)
 
     filtered_aliases = []
-    response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases2/" + symbols_string)
+    response = requests.get(GN2_BASE_URL + "/gn3/gene/aliases2/" + symbols_string)
     if response:
         alias_lists = json.loads(response.content)
         seen = set()
diff --git a/wqflask/wqflask/static/new/javascript/ctl_graph.js b/wqflask/wqflask/static/new/javascript/ctl_graph.js
index 94bd7e9d..bd950592 100644
--- a/wqflask/wqflask/static/new/javascript/ctl_graph.js
+++ b/wqflask/wqflask/static/new/javascript/ctl_graph.js
@@ -82,18 +82,12 @@ window.onload=function() {
     function create_qtips(cy){
         cy.nodes().qtip({
                             content: function(){
-                                gn_link = '<b>'+'<a href="http://gn2.genenetwork.org/show_trait?trait_id=' + this.data().sid + '&dataset=' + this.data().dataset + '" >'+this.data().id +'</a>'+'</b><br>'
+                                gn_link = '<b>'+'<a href="' + gn2_url + '/show_trait?trait_id=' + this.data().sid + '&dataset=' + this.data().dataset + '" >'+this.data().id +'</a>'+'</b><br>'
                                 ncbi_link = '<a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=' + this.data().geneid + '" >NCBI<a>'+'<br>' 
                                 omim_link = '<a href="http://www.ncbi.nlm.nih.gov/omim/' + this.data().omim + '" >OMIM<a>'+'<br>' 
                                 qtip_content = gn_link + ncbi_link + omim_link
                                 return qtip_content
-                                //return '<b>'+'<a href="http://gn2.genenetwork.org/show_trait?trait_id=' + this.data().id + '&dataset=' + this.data().dataset + '" >'+this.data().id +'<a>'+'</b>' 
                             },
-                            // content: {
-                                // title: '<b>'+'<a href="http://gn2.genenetwork.org/show_trait?trait_id=' + this.target() + '&dataset=' + this.dataset() + '" >'+this.target() +'<a>'+'</b>',
-                                // text: this.target,
-                                // button: true
-                            // },
                             position: {
                                 my: 'top center',
                                 at: 'bottom center'
diff --git a/wqflask/wqflask/static/new/javascript/dataset_select_menu_orig.js b/wqflask/wqflask/static/new/javascript/dataset_select_menu_orig.js
index fad600d2..794804f4 100644
--- a/wqflask/wqflask/static/new/javascript/dataset_select_menu_orig.js
+++ b/wqflask/wqflask/static/new/javascript/dataset_select_menu_orig.js
@@ -74,7 +74,6 @@ redo_dropdown = function(dropdown, items) {
     this_opt_group = null
     for (_i = 0, _len = group_family_list.length; _i < _len; _i++) {
       item = group_family_list[_i];
-      console.log("THE ITEM:", item)
       if (item[2] != "None" && current_family == ""){
         current_family = item[2]
         this_opt_group = $("<optgroup label=\"" + item[2] + "\">")
diff --git a/wqflask/wqflask/static/new/javascript/network_graph.js b/wqflask/wqflask/static/new/javascript/network_graph.js
index 4d507a18..02c3b817 100644
--- a/wqflask/wqflask/static/new/javascript/network_graph.js
+++ b/wqflask/wqflask/static/new/javascript/network_graph.js
@@ -85,7 +85,7 @@ window.onload=function() {
         cy.nodes().qtip({
                             content: function(){
                                 qtip_content = ''
-                                gn_link = '<b>'+'<a href="http://gn2.genenetwork.org/show_trait?trait_id=' + this.data().id.split(":")[0] + '&dataset=' + this.data().id.split(":")[1] + '" >'+this.data().id +'</a>'+'</b><br>'
+                                gn_link = '<b>'+'<a href="' + gn2_url + '/show_trait?trait_id=' + this.data().id.split(":")[0] + '&dataset=' + this.data().id.split(":")[1] + '" >'+this.data().id +'</a>'+'</b><br>'
                                 qtip_content += gn_link
                                 if (typeof(this.data().geneid) !== 'undefined'){
                                     ncbi_link = '<a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=' + this.data().geneid + '" >NCBI<a>'+'<br>'
@@ -115,7 +115,7 @@ window.onload=function() {
                                 correlation_line = '<b>Sample r: ' + this.data().correlation + '</b><br>'
                                 p_value_line = 'Sample p(r): ' + this.data().p_value + '<br>'
                                 overlap_line = 'Overlap: ' + this.data().overlap + '<br>'
-                                scatter_plot = '<a href="http://gn2-zach.genenetwork.org/corr_scatter_plot?dataset_1=' + this.data().source.split(":")[1] + '&dataset_2=' + this.data().target.split(":")[1] + '&trait_1=' + this.data().source.split(":")[0] + '&trait_2=' + this.data().target.split(":")[0] + '" >View Scatterplot</a>'
+                                scatter_plot = '<a href="' + gn2_url + '/corr_scatter_plot?dataset_1=' + this.data().source.split(":")[1] + '&dataset_2=' + this.data().target.split(":")[1] + '&trait_1=' + this.data().source.split(":")[0] + '&trait_2=' + this.data().target.split(":")[0] + '" >View Scatterplot</a>'
                                 return correlation_line + p_value_line + overlap_line + scatter_plot
                             },
                             position: {
diff --git a/wqflask/wqflask/templates/ctl_results.html b/wqflask/wqflask/templates/ctl_results.html
index d85075a9..0108d93a 100644
--- a/wqflask/wqflask/templates/ctl_results.html
+++ b/wqflask/wqflask/templates/ctl_results.html
@@ -61,6 +61,7 @@
 
     <script>
         elements_list = {{ elements | safe }}
+        gn2_url = "{{ gn2_url | safe }}"
     </script>
 
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/jquery.js"></script>
diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html
index 0116245d..f8720d39 100644
--- a/wqflask/wqflask/templates/index_page.html
+++ b/wqflask/wqflask/templates/index_page.html
@@ -219,9 +219,9 @@
                     <h3>GN1 Mirror and development sites</h3>
 
                     <ul>
-                        <li><a href="http://www.genenetwork.org/">Main GN1 site at UTHSC</a> (main site)</li>
+                        <li><a href="http://gn1.genenetwork.org/">Main GN1 site at UTHSC</a> (main site)</li>
                         <li><a href="http://genenetwork.helmholtz-hzi.de/">Germany at the HZI</a></li>
-                        <li><a href="http://gn2.genenetwork.org/">Memphis at the U of M</a></li>
+                        <li><a href="http://genenetwork.org/">Memphis at the U of M</a></li>
                     </ul>
                 </section>
 
diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html
index 963531cb..06b71f53 100755
--- a/wqflask/wqflask/templates/index_page_orig.html
+++ b/wqflask/wqflask/templates/index_page_orig.html
@@ -254,8 +254,8 @@
                     </div>
                     <h3>GeneNetwork v2:</h3>
                       <ul>
-                        <li><a href="http://gn2.genenetwork.org/">Main website</a> at UTHSC</li>
-                        <!--<li><a href="http://test-gn2.genenetwork.org/">Testing website</a> at UTHSC</li>-->
+                        <li><a href="http://genenetwork.org/">Main website</a> at UTHSC</li>
+                        <!--<li><a href="http://test-genenetwork.org/">Testing website</a> at UTHSC</li>-->
                       </ul>
                     <h3>GeneNetwork v1:</h3>
                     <ul>
diff --git a/wqflask/wqflask/templates/network_graph.html b/wqflask/wqflask/templates/network_graph.html
index 4492dd3f..25af7bb1 100644
--- a/wqflask/wqflask/templates/network_graph.html
+++ b/wqflask/wqflask/templates/network_graph.html
@@ -137,6 +137,7 @@
 
     <script>
         elements_list = {{ elements | safe }}
+        gn2_url       = "{{ gn2_url | safe }}"
     </script>
 
     <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/jquery.js"></script>
-- 
cgit v1.2.3