Merge master and testingmaster

9 files changed, 334 insertions, 25 deletions

diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py
index 529cd117..cce8435d 100644
--- a/wqflask/utility/Plot.py
+++ b/wqflask/utility/Plot.py
@@ -98,8 +98,6 @@ def find_outliers(vals):
 
     """
 
-    logger.debug("xerxes vals is:", pf(vals))
-
     if vals:
         #logger.debug("vals is:", pf(vals))
         stats = corestats.Stats(vals)
@@ -114,7 +112,6 @@ def find_outliers(vals):
         upper_bound = None
         lower_bound = None
 
-    logger.debug(pf(locals()))
     return upper_bound, lower_bound
 
 # parameter: data is either object returned by reaper permutation function (called by MarkerRegressionPage.py)
diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
new file mode 100644
index 00000000..23b0b650
--- /dev/null
+++ b/wqflask/utility/gen_geno_ob.py
@@ -0,0 +1,181 @@
+from __future__ import absolute_import, division, print_function
+
+import utility.logger
+logger = utility.logger.getLogger(__name__ )
+
+class genotype(object):
+    """
+    Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure
+    """
+
+    def __init__(self, filename):
+        self.group = None
+        self.type = "riset"
+        self.prgy = []
+        self.nprgy = 0
+        self.mat = -1
+        self.pat = 1
+        self.het = 0
+        self.unk = "U"
+        self.filler = False
+        self.mb_exists = False
+
+        #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary
+        self.cm_column = 2
+        self.mb_column = 3
+
+        self.chromosomes = []
+
+        self.read_file(filename)
+
+    def __iter__(self):
+        return iter(self.chromosomes)
+
+    def __getitem__(self, index):
+        return self.chromosomes[index]
+
+    def __len__(self):
+        return len(self.chromosomes)
+
+    def read_rdata_output(self, qtl_results):
+        #ZS: This is necessary because R/qtl requires centimorgan marker positions, which it normally gets from the .geno file, but that doesn't exist for HET3-ITP (which only has RData), so it needs to read in the marker cM positions from the results
+        self.chromosomes = [] #ZS: Overwriting since the .geno file's contents are just placeholders
+
+        this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers
+        chr_ob = None
+        for marker in qtl_results:
+            locus = Locus(self)
+            if (str(marker['chr']) != this_chr) and this_chr != "X": #ZS: This is really awkward but works as a temporary fix
+                if this_chr != "":
+                    self.chromosomes.append(chr_ob)
+                this_chr = str(marker['chr'])
+                if this_chr == "20":
+                    this_chr = "X"
+                chr_ob = Chr(this_chr, self)
+            if 'chr' in marker:
+                locus.chr = str(marker['chr'])
+            if 'name' in marker:
+                locus.name = marker['name']
+            if 'Mb' in marker:
+                locus.Mb = marker['Mb']
+            if 'cM' in marker:
+                locus.cM = marker['cM']
+            chr_ob.loci.append(locus)
+
+        self.chromosomes.append(chr_ob)
+
+        return self
+
+    def read_file(self, filename):
+        with open(filename, 'r') as geno_file:
+            lines = geno_file.readlines()
+
+            this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers
+            chr_ob = None
+            for line in lines:
+                if line[0] == "#":
+                    continue
+                elif line[0] == "@":
+                    label = line.split(":")[0][1:]
+                    if label == "name":
+                        self.group = line.split(":")[1].strip()
+                    elif label == "filler":
+                        if line.split(":")[1].strip() == "yes":
+                            self.filler = True
+                    elif label == "type":
+                        self.type = line.split(":")[1].strip()
+                    elif label == "mat":
+                        self.mat = line.split(":")[1].strip()
+                    elif label == "pat":
+                        self.pat = line.split(":")[1].strip()
+                    elif label == "het":
+                        self.het = line.split(":")[1].strip()
+                    elif label == "unk":
+                        self.unk = line.split(":")[1].strip()
+                    else:
+                        continue
+                elif line[:3] == "Chr":
+                    header_row = line.split("\t")
+                    if header_row[2] == "Mb":
+                        self.mb_exists = True
+                        self.mb_column = 2
+                        self.cm_column = 3
+                    elif header_row[3] == "Mb":
+                        self.mb_exists = True
+                        self.mb_column = 3
+                    elif header_row[2] == "cM":
+                        self.cm_column = 2
+
+                    if self.mb_exists:
+                        self.prgy = header_row[4:]
+                    else:
+                        self.prgy = header_row[3:]
+                    self.nprgy = len(self.prgy)
+                else:
+                    if line.split("\t")[0] != this_chr:
+                        if this_chr != "":
+                            self.chromosomes.append(chr_ob)
+                        this_chr = line.split("\t")[0]
+                        chr_ob = Chr(line.split("\t")[0], self)
+                    chr_ob.add_marker(line.split("\t"))
+
+            self.chromosomes.append(chr_ob)
+
+class Chr(object):
+    def __init__(self, name, geno_ob):
+        self.name = name
+        self.loci = []
+        self.mb_exists = geno_ob.mb_exists
+        self.cm_column = geno_ob.cm_column
+        self.mb_column = geno_ob.mb_column
+        self.geno_ob = geno_ob
+
+    def __iter__(self):
+        return iter(self.loci)
+
+    def __getitem__(self, index):
+        return self.loci[index]
+
+    def __len__(self):
+        return len(self.loci)
+
+    def add_marker(self, marker_row):
+        self.loci.append(Locus(self.geno_ob, marker_row))
+
+class Locus(object):
+    def __init__(self, geno_ob, marker_row = None):
+        self.chr = None
+        self.name = None
+        self.cM = None
+        self.Mb = None
+        self.genotype = []
+        if marker_row:
+            self.chr = marker_row[0]
+            self.name = marker_row[1]
+            try:
+                self.cM = float(marker_row[geno_ob.cm_column])
+            except:
+                self.cM = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else 0
+            try:
+                self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
+            except:
+                self.Mb = self.cM
+
+            geno_table = {
+                geno_ob.mat: -1,
+                geno_ob.pat: 1,
+                geno_ob.het: 0,
+                geno_ob.unk: "U"
+            }
+
+            self.genotype = []
+            if geno_ob.mb_exists:
+                start_pos = 4
+            else:
+                start_pos = 3
+
+            for allele in marker_row[start_pos:]:
+                if allele in geno_table.keys():
+                    self.genotype.append(geno_table[allele])
+                else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown
+                    self.genotype.append("U")
\ No newline at end of file
diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py
index cf16879f..e7c04fef 100644
--- a/wqflask/utility/helper_functions.py
+++ b/wqflask/utility/helper_functions.py
@@ -4,7 +4,7 @@ from base.trait import GeneralTrait
 from base import data_set
 from base.species import TheSpecies
 
-from wqflask import user_manager
+from utility import hmac
 
 from flask import Flask, g
 
@@ -14,7 +14,13 @@ logger = logging.getLogger(__name__ )
 
 def get_species_dataset_trait(self, start_vars):
     #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype"
-    self.dataset = data_set.create_dataset(start_vars['dataset'])
+    if "temp_trait" in start_vars.keys():
+      if start_vars['temp_trait'] == "True":
+        self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group'])
+      else:
+        self.dataset = data_set.create_dataset(start_vars['dataset'])
+    else:
+      self.dataset = data_set.create_dataset(start_vars['dataset'])
     logger.debug("After creating dataset")
     self.species = TheSpecies(dataset=self.dataset)
     logger.debug("After creating species")
@@ -35,11 +41,14 @@ def get_trait_db_obs(self, trait_db_list):
 
     self.trait_list = []
     for trait in trait_db_list:
-        data, _separator, hmac = trait.rpartition(':')
+        data, _separator, hmac_string = trait.rpartition(':')
         data = data.strip()
-        assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?"
+        assert hmac_string==hmac.hmac_creation(data), "Data tampering?"
         trait_name, dataset_name = data.split(":")
-        dataset_ob = data_set.create_dataset(dataset_name)
+        if dataset_name == "Temp":
+            dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2])
+        else:
+            dataset_ob = data_set.create_dataset(dataset_name)
         trait_ob = GeneralTrait(dataset=dataset_ob,
                                name=trait_name,
                                cellid=None)
diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py
new file mode 100644
index 00000000..d8a0eace
--- /dev/null
+++ b/wqflask/utility/hmac.py
@@ -0,0 +1,38 @@
+from __future__ import print_function, division, absolute_import
+
+import hmac
+import hashlib
+
+from wqflask import app
+
+def hmac_creation(stringy):
+    """Helper function to create the actual hmac"""
+
+    secret = app.config['SECRET_HMAC_CODE']
+
+    hmaced = hmac.new(secret, stringy, hashlib.sha1)
+    hm = hmaced.hexdigest()
+    # ZS: Leaving the below comment here to ask Pjotr about
+    # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output."
+    # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html
+    hm = hm[:20]
+    return hm
+
+def data_hmac(stringy):
+    """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with"""
+    return stringy + ":" + hmac_creation(stringy)
+
+def url_for_hmac(endpoint, **values):
+    """Like url_for but adds an hmac at the end to insure the url hasn't been tampered with"""
+
+    url = url_for(endpoint, **values)
+
+    hm = hmac_creation(url)
+    if '?' in url:
+        combiner = "&"
+    else:
+        combiner = "?"
+    return url + combiner + "hm=" + hm
+
+app.jinja_env.globals.update(url_for_hmac=url_for_hmac,
+                             data_hmac=data_hmac)
\ No newline at end of file
diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py
new file mode 100644
index 00000000..0754e16f
--- /dev/null
+++ b/wqflask/utility/redis_tools.py
@@ -0,0 +1,80 @@
+from __future__ import print_function, division, absolute_import
+
+import simplejson as json
+
+import redis # used for collections
+Redis = redis.StrictRedis()
+
+import logging
+
+from flask import (render_template, flash)
+
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+def is_redis_available():
+    try:
+        Redis.ping()
+    except:
+        return False
+    return True
+
+def get_user_id(column_name, column_value):
+    user_list = Redis.hgetall("users")
+    for key in user_list:
+        user_ob = json.loads(user_list[key])
+        if column_name in user_ob and user_ob[column_name] == column_value:
+            return key
+
+    return None
+
+def get_user_by_unique_column(column_name, column_value):
+    item_details = None
+
+    user_list = Redis.hgetall("users")
+    if column_name != "user_id":
+        for key in user_list:
+            user_ob = json.loads(user_list[key])
+            if column_name in user_ob and user_ob[column_name] == column_value:
+                item_details = user_ob
+    else:
+        item_details = json.loads(user_list[column_value])
+
+    return item_details
+
+def set_user_attribute(user_id, column_name, column_value):
+    user_info = json.loads(Redis.hget("users", user_id))
+    user_info[column_name] = column_value
+
+    Redis.hset("users", user_id, json.dumps(user_info))
+
+def get_user_collections(user_id):
+    collections = None
+    collections = Redis.hget("collections", user_id)
+
+    if collections:
+        return json.loads(collections)
+    else:
+        return []
+
+def save_user(user, user_id):
+    Redis.hset("users", user_id, json.dumps(user))
+
+def save_collections(user_id, collections_ob):
+    Redis.hset("collections", user_id, collections_ob)
+
+def save_verification_code(user_email, code):
+    Redis.hset("verification_codes", code, user_email)
+
+def check_verification_code(code):
+    email_address = None
+    user_details = None
+    email_address = Redis.hget("verification_codes", code)
+    return email_address
+
+    if email_address:
+        user_details = get_user_by_unique_column('email_address', email_address)
+        return user_details
+    else:
+        return None
+        flash("Invalid code: Password reset code does not exist or might have expired!", "error")
diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py
index 5a62cc50..817284dd 100644
--- a/wqflask/utility/startup_config.py
+++ b/wqflask/utility/startup_config.py
@@ -33,7 +33,7 @@ def app_config():
         if page.status_code != 200:
             raise Exception("API server not found!")
 
-    import utility.elasticsearch_tools as es
-    es.test_elasticsearch_connection()
+    # import utility.elasticsearch_tools as es
+    # es.test_elasticsearch_connection()
 
     print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL")))
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index f12b0d76..2914d354 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -107,8 +107,8 @@ def js_path(module=None):
         return try_guix
     raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)"
 
-def pylmm_command(guess=None):
-    return assert_bin(get_setting("PYLMM_COMMAND",guess))
+def reaper_command(guess=None):
+    return get_setting("REAPER_COMMAND",guess)
 
 def gemma_command(guess=None):
     return assert_bin(get_setting("GEMMA_COMMAND",guess))
@@ -235,7 +235,10 @@ GN_VERSION         = get_setting('GN_VERSION')
 HOME               = get_setting('HOME')
 SERVER_PORT        = get_setting('SERVER_PORT')
 WEBSERVER_MODE     = get_setting('WEBSERVER_MODE')
+GN2_BASE_URL       = get_setting('GN2_BASE_URL')
+GN2_BRANCH_URL     = get_setting('GN2_BRANCH_URL')
 GN_SERVER_URL      = get_setting('GN_SERVER_URL')
+SERVER_PORT        = get_setting_int('SERVER_PORT')
 SQL_URI            = get_setting('SQL_URI')
 LOG_LEVEL          = get_setting('LOG_LEVEL')
 LOG_LEVEL_DEBUG    = get_setting_int('LOG_LEVEL_DEBUG')
@@ -254,7 +257,6 @@ JS_GN_PATH         = get_setting('JS_GN_PATH')
 
 GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID')
 GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET')
-GITHUB_AUTH_URL = None
 if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET:
     GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \
                       GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET
@@ -264,20 +266,20 @@ ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID')
 ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET')
 ORCID_AUTH_URL = None
 if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET:
-    ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \
+    ORCID_AUTH_URL = "https://orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \
                       ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET
     ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL')
 
-ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
-ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
-import utility.elasticsearch_tools as es
-es.test_elasticsearch_connection()
+# ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
+# ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
+# import utility.elasticsearch_tools as es
+# es.test_elasticsearch_connection()
 
 SMTP_CONNECT = get_setting('SMTP_CONNECT')
 SMTP_USERNAME = get_setting('SMTP_USERNAME')
 SMTP_PASSWORD = get_setting('SMTP_PASSWORD')
 
-PYLMM_COMMAND      = app_set("PYLMM_COMMAND",pylmm_command())
+REAPER_COMMAND     = app_set("REAPER_COMMAND",reaper_command())
 GEMMA_COMMAND      = app_set("GEMMA_COMMAND",gemma_command())
 assert(GEMMA_COMMAND is not None)
 PLINK_COMMAND      = app_set("PLINK_COMMAND",plink_command())
@@ -289,11 +291,14 @@ assert_dir(TEMPDIR)
 JS_GUIX_PATH = get_setting("JS_GUIX_PATH")
 assert_dir(JS_GUIX_PATH)
 assert_dir(JS_GUIX_PATH+'/cytoscape-panzoom')
+
 CSS_PATH = "UNKNOWN"
 # assert_dir(JS_PATH)
-JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher"))
+
+JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("javascript-twitter-post-fetcher"))
 assert_dir(JS_TWITTER_POST_FETCHER_PATH)
 assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js")
+
 JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH",js_path("cytoscape"))
 assert_dir(JS_CYTOSCAPE_PATH)
 assert_file(JS_CYTOSCAPE_PATH+'/cytoscape.min.js')
diff --git a/wqflask/utility/type_checking.py b/wqflask/utility/type_checking.py
index 220e5f62..f15b17e2 100644
--- a/wqflask/utility/type_checking.py
+++ b/wqflask/utility/type_checking.py
@@ -27,7 +27,7 @@ def get_float(vars,name,default=None):
     if name in vars:
         if is_float(vars[name]):
             return float(vars[name])
-    return None
+    return default
 
 def get_int(vars,name,default=None):
     if name in vars:
diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py
index 83fa90b7..53661ae4 100644
--- a/wqflask/utility/webqtlUtil.py
+++ b/wqflask/utility/webqtlUtil.py
@@ -30,8 +30,6 @@ import re
 import math
 from math import *
 
-from htmlgen import HTMLgen2 as HT
-
 from base import webqtlConfig
 
 # NL, 07/27/2010. moved from webqtlForm.py
@@ -54,11 +52,12 @@ ParInfo ={
 'AXB':['ABF1', 'BAF1', 'C57BL/6J', 'A/J'],
 'BXA':['BAF1', 'ABF1', 'C57BL/6J', 'A/J'],
 'LXS':['LSF1', 'SLF1', 'ISS', 'ILS'],
-'HXBBXH':['HSRBNF1', 'BNHSRF1', 'BN', 'HSR'],
+'HXBBXH':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'],
 'BayXSha':['BayXShaF1', 'ShaXBayF1', 'Bay-0','Shahdara'],
 'ColXBur':['ColXBurF1', 'BurXColF1', 'Col-0','Bur-0'],
 'ColXCvi':['ColXCviF1', 'CviXColF1', 'Col-0','Cvi'],
-'SXM':['SMF1', 'MSF1', 'Steptoe','Morex']
+'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'],
+'HRDP':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv']
 }
 
 #########################################