diff options
Diffstat (limited to 'wqflask/utility')
| -rw-r--r-- | wqflask/utility/Plot.py | 3 | ||||
| -rw-r--r-- | wqflask/utility/gen_geno_ob.py | 181 | ||||
| -rw-r--r-- | wqflask/utility/helper_functions.py | 19 | ||||
| -rw-r--r-- | wqflask/utility/hmac.py | 38 | ||||
| -rw-r--r-- | wqflask/utility/redis_tools.py | 80 | ||||
| -rw-r--r-- | wqflask/utility/startup_config.py | 4 | ||||
| -rw-r--r-- | wqflask/utility/tools.py | 25 | ||||
| -rw-r--r-- | wqflask/utility/type_checking.py | 2 | ||||
| -rw-r--r-- | wqflask/utility/webqtlUtil.py | 7 |
9 files changed, 334 insertions, 25 deletions
diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 529cd117..cce8435d 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -98,8 +98,6 @@ def find_outliers(vals): """ - logger.debug("xerxes vals is:", pf(vals)) - if vals: #logger.debug("vals is:", pf(vals)) stats = corestats.Stats(vals) @@ -114,7 +112,6 @@ def find_outliers(vals): upper_bound = None lower_bound = None - logger.debug(pf(locals())) return upper_bound, lower_bound # parameter: data is either object returned by reaper permutation function (called by MarkerRegressionPage.py) diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py new file mode 100644 index 00000000..23b0b650 --- /dev/null +++ b/wqflask/utility/gen_geno_ob.py @@ -0,0 +1,181 @@ +from __future__ import absolute_import, division, print_function + +import utility.logger +logger = utility.logger.getLogger(__name__ ) + +class genotype(object): + """ + Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure + """ + + def __init__(self, filename): + self.group = None + self.type = "riset" + self.prgy = [] + self.nprgy = 0 + self.mat = -1 + self.pat = 1 + self.het = 0 + self.unk = "U" + self.filler = False + self.mb_exists = False + + #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary + self.cm_column = 2 + self.mb_column = 3 + + self.chromosomes = [] + + self.read_file(filename) + + def __iter__(self): + return iter(self.chromosomes) + + def __getitem__(self, index): + return self.chromosomes[index] + + def __len__(self): + return len(self.chromosomes) + + def read_rdata_output(self, qtl_results): + #ZS: This is necessary because R/qtl requires centimorgan marker positions, which it normally gets from the .geno file, but that doesn't exist for HET3-ITP (which only has RData), so it needs to read in the marker cM positions from the results + self.chromosomes = [] #ZS: Overwriting since the .geno file's contents are just placeholders + + this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers + chr_ob = None + for marker in qtl_results: + locus = Locus(self) + if (str(marker['chr']) != this_chr) and this_chr != "X": #ZS: This is really awkward but works as a temporary fix + if this_chr != "": + self.chromosomes.append(chr_ob) + this_chr = str(marker['chr']) + if this_chr == "20": + this_chr = "X" + chr_ob = Chr(this_chr, self) + if 'chr' in marker: + locus.chr = str(marker['chr']) + if 'name' in marker: + locus.name = marker['name'] + if 'Mb' in marker: + locus.Mb = marker['Mb'] + if 'cM' in marker: + locus.cM = marker['cM'] + chr_ob.loci.append(locus) + + self.chromosomes.append(chr_ob) + + return self + + def read_file(self, filename): + with open(filename, 'r') as geno_file: + lines = geno_file.readlines() + + this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers + chr_ob = None + for line in lines: + if line[0] == "#": + continue + elif line[0] == "@": + label = line.split(":")[0][1:] + if label == "name": + self.group = line.split(":")[1].strip() + elif label == "filler": + if line.split(":")[1].strip() == "yes": + self.filler = True + elif label == "type": + self.type = line.split(":")[1].strip() + elif label == "mat": + self.mat = line.split(":")[1].strip() + elif label == "pat": + self.pat = line.split(":")[1].strip() + elif label == "het": + self.het = line.split(":")[1].strip() + elif label == "unk": + self.unk = line.split(":")[1].strip() + else: + continue + elif line[:3] == "Chr": + header_row = line.split("\t") + if header_row[2] == "Mb": + self.mb_exists = True + self.mb_column = 2 + self.cm_column = 3 + elif header_row[3] == "Mb": + self.mb_exists = True + self.mb_column = 3 + elif header_row[2] == "cM": + self.cm_column = 2 + + if self.mb_exists: + self.prgy = header_row[4:] + else: + self.prgy = header_row[3:] + self.nprgy = len(self.prgy) + else: + if line.split("\t")[0] != this_chr: + if this_chr != "": + self.chromosomes.append(chr_ob) + this_chr = line.split("\t")[0] + chr_ob = Chr(line.split("\t")[0], self) + chr_ob.add_marker(line.split("\t")) + + self.chromosomes.append(chr_ob) + +class Chr(object): + def __init__(self, name, geno_ob): + self.name = name + self.loci = [] + self.mb_exists = geno_ob.mb_exists + self.cm_column = geno_ob.cm_column + self.mb_column = geno_ob.mb_column + self.geno_ob = geno_ob + + def __iter__(self): + return iter(self.loci) + + def __getitem__(self, index): + return self.loci[index] + + def __len__(self): + return len(self.loci) + + def add_marker(self, marker_row): + self.loci.append(Locus(self.geno_ob, marker_row)) + +class Locus(object): + def __init__(self, geno_ob, marker_row = None): + self.chr = None + self.name = None + self.cM = None + self.Mb = None + self.genotype = [] + if marker_row: + self.chr = marker_row[0] + self.name = marker_row[1] + try: + self.cM = float(marker_row[geno_ob.cm_column]) + except: + self.cM = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else 0 + try: + self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None + except: + self.Mb = self.cM + + geno_table = { + geno_ob.mat: -1, + geno_ob.pat: 1, + geno_ob.het: 0, + geno_ob.unk: "U" + } + + self.genotype = [] + if geno_ob.mb_exists: + start_pos = 4 + else: + start_pos = 3 + + for allele in marker_row[start_pos:]: + if allele in geno_table.keys(): + self.genotype.append(geno_table[allele]) + else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown + self.genotype.append("U") \ No newline at end of file diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index cf16879f..e7c04fef 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -4,7 +4,7 @@ from base.trait import GeneralTrait from base import data_set from base.species import TheSpecies -from wqflask import user_manager +from utility import hmac from flask import Flask, g @@ -14,7 +14,13 @@ logger = logging.getLogger(__name__ ) def get_species_dataset_trait(self, start_vars): #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype" - self.dataset = data_set.create_dataset(start_vars['dataset']) + if "temp_trait" in start_vars.keys(): + if start_vars['temp_trait'] == "True": + self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) + else: + self.dataset = data_set.create_dataset(start_vars['dataset']) + else: + self.dataset = data_set.create_dataset(start_vars['dataset']) logger.debug("After creating dataset") self.species = TheSpecies(dataset=self.dataset) logger.debug("After creating species") @@ -35,11 +41,14 @@ def get_trait_db_obs(self, trait_db_list): self.trait_list = [] for trait in trait_db_list: - data, _separator, hmac = trait.rpartition(':') + data, _separator, hmac_string = trait.rpartition(':') data = data.strip() - assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?" + assert hmac_string==hmac.hmac_creation(data), "Data tampering?" trait_name, dataset_name = data.split(":") - dataset_ob = data_set.create_dataset(dataset_name) + if dataset_name == "Temp": + dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2]) + else: + dataset_ob = data_set.create_dataset(dataset_name) trait_ob = GeneralTrait(dataset=dataset_ob, name=trait_name, cellid=None) diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py new file mode 100644 index 00000000..d8a0eace --- /dev/null +++ b/wqflask/utility/hmac.py @@ -0,0 +1,38 @@ +from __future__ import print_function, division, absolute_import + +import hmac +import hashlib + +from wqflask import app + +def hmac_creation(stringy): + """Helper function to create the actual hmac""" + + secret = app.config['SECRET_HMAC_CODE'] + + hmaced = hmac.new(secret, stringy, hashlib.sha1) + hm = hmaced.hexdigest() + # ZS: Leaving the below comment here to ask Pjotr about + # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." + # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html + hm = hm[:20] + return hm + +def data_hmac(stringy): + """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with""" + return stringy + ":" + hmac_creation(stringy) + +def url_for_hmac(endpoint, **values): + """Like url_for but adds an hmac at the end to insure the url hasn't been tampered with""" + + url = url_for(endpoint, **values) + + hm = hmac_creation(url) + if '?' in url: + combiner = "&" + else: + combiner = "?" + return url + combiner + "hm=" + hm + +app.jinja_env.globals.update(url_for_hmac=url_for_hmac, + data_hmac=data_hmac) \ No newline at end of file diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py new file mode 100644 index 00000000..0754e16f --- /dev/null +++ b/wqflask/utility/redis_tools.py @@ -0,0 +1,80 @@ +from __future__ import print_function, division, absolute_import + +import simplejson as json + +import redis # used for collections +Redis = redis.StrictRedis() + +import logging + +from flask import (render_template, flash) + +from utility.logger import getLogger +logger = getLogger(__name__) + +def is_redis_available(): + try: + Redis.ping() + except: + return False + return True + +def get_user_id(column_name, column_value): + user_list = Redis.hgetall("users") + for key in user_list: + user_ob = json.loads(user_list[key]) + if column_name in user_ob and user_ob[column_name] == column_value: + return key + + return None + +def get_user_by_unique_column(column_name, column_value): + item_details = None + + user_list = Redis.hgetall("users") + if column_name != "user_id": + for key in user_list: + user_ob = json.loads(user_list[key]) + if column_name in user_ob and user_ob[column_name] == column_value: + item_details = user_ob + else: + item_details = json.loads(user_list[column_value]) + + return item_details + +def set_user_attribute(user_id, column_name, column_value): + user_info = json.loads(Redis.hget("users", user_id)) + user_info[column_name] = column_value + + Redis.hset("users", user_id, json.dumps(user_info)) + +def get_user_collections(user_id): + collections = None + collections = Redis.hget("collections", user_id) + + if collections: + return json.loads(collections) + else: + return [] + +def save_user(user, user_id): + Redis.hset("users", user_id, json.dumps(user)) + +def save_collections(user_id, collections_ob): + Redis.hset("collections", user_id, collections_ob) + +def save_verification_code(user_email, code): + Redis.hset("verification_codes", code, user_email) + +def check_verification_code(code): + email_address = None + user_details = None + email_address = Redis.hget("verification_codes", code) + return email_address + + if email_address: + user_details = get_user_by_unique_column('email_address', email_address) + return user_details + else: + return None + flash("Invalid code: Password reset code does not exist or might have expired!", "error") diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 5a62cc50..817284dd 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -33,7 +33,7 @@ def app_config(): if page.status_code != 200: raise Exception("API server not found!") - import utility.elasticsearch_tools as es - es.test_elasticsearch_connection() + # import utility.elasticsearch_tools as es + # es.test_elasticsearch_connection() print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index f12b0d76..2914d354 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -107,8 +107,8 @@ def js_path(module=None): return try_guix raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)" -def pylmm_command(guess=None): - return assert_bin(get_setting("PYLMM_COMMAND",guess)) +def reaper_command(guess=None): + return get_setting("REAPER_COMMAND",guess) def gemma_command(guess=None): return assert_bin(get_setting("GEMMA_COMMAND",guess)) @@ -235,7 +235,10 @@ GN_VERSION = get_setting('GN_VERSION') HOME = get_setting('HOME') SERVER_PORT = get_setting('SERVER_PORT') WEBSERVER_MODE = get_setting('WEBSERVER_MODE') +GN2_BASE_URL = get_setting('GN2_BASE_URL') +GN2_BRANCH_URL = get_setting('GN2_BRANCH_URL') GN_SERVER_URL = get_setting('GN_SERVER_URL') +SERVER_PORT = get_setting_int('SERVER_PORT') SQL_URI = get_setting('SQL_URI') LOG_LEVEL = get_setting('LOG_LEVEL') LOG_LEVEL_DEBUG = get_setting_int('LOG_LEVEL_DEBUG') @@ -254,7 +257,6 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') -GITHUB_AUTH_URL = None if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET: GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \ GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET @@ -264,20 +266,20 @@ ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = None if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: - ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \ + ORCID_AUTH_URL = "https://orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \ ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') -import utility.elasticsearch_tools as es -es.test_elasticsearch_connection() +# ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') +# ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') +# import utility.elasticsearch_tools as es +# es.test_elasticsearch_connection() SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') SMTP_PASSWORD = get_setting('SMTP_PASSWORD') -PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) +REAPER_COMMAND = app_set("REAPER_COMMAND",reaper_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) assert(GEMMA_COMMAND is not None) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) @@ -289,11 +291,14 @@ assert_dir(TEMPDIR) JS_GUIX_PATH = get_setting("JS_GUIX_PATH") assert_dir(JS_GUIX_PATH) assert_dir(JS_GUIX_PATH+'/cytoscape-panzoom') + CSS_PATH = "UNKNOWN" # assert_dir(JS_PATH) -JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher")) + +JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("javascript-twitter-post-fetcher")) assert_dir(JS_TWITTER_POST_FETCHER_PATH) assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") + JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH",js_path("cytoscape")) assert_dir(JS_CYTOSCAPE_PATH) assert_file(JS_CYTOSCAPE_PATH+'/cytoscape.min.js') diff --git a/wqflask/utility/type_checking.py b/wqflask/utility/type_checking.py index 220e5f62..f15b17e2 100644 --- a/wqflask/utility/type_checking.py +++ b/wqflask/utility/type_checking.py @@ -27,7 +27,7 @@ def get_float(vars,name,default=None): if name in vars: if is_float(vars[name]): return float(vars[name]) - return None + return default def get_int(vars,name,default=None): if name in vars: diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index 83fa90b7..53661ae4 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -30,8 +30,6 @@ import re import math from math import * -from htmlgen import HTMLgen2 as HT - from base import webqtlConfig # NL, 07/27/2010. moved from webqtlForm.py @@ -54,11 +52,12 @@ ParInfo ={ 'AXB':['ABF1', 'BAF1', 'C57BL/6J', 'A/J'], 'BXA':['BAF1', 'ABF1', 'C57BL/6J', 'A/J'], 'LXS':['LSF1', 'SLF1', 'ISS', 'ILS'], -'HXBBXH':['HSRBNF1', 'BNHSRF1', 'BN', 'HSR'], +'HXBBXH':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'], 'BayXSha':['BayXShaF1', 'ShaXBayF1', 'Bay-0','Shahdara'], 'ColXBur':['ColXBurF1', 'BurXColF1', 'Col-0','Bur-0'], 'ColXCvi':['ColXCviF1', 'CviXColF1', 'Col-0','Cvi'], -'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'] +'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'], +'HRDP':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'] } ######################################### |