diff options
95 files changed, 2855 insertions, 9848 deletions
@@ -17,25 +17,35 @@ deploy GN2 and dependencies as a self contained unit on any machine. The database can be run separately as well as the source tree (for developers). See the [installation docs](doc/README.org). -## Test +## Run Once installed GN2 can be run online through a browser interface ```sh -./bin/genenetwork2 +genenetwork2 ``` -(default is http://localhost:5003/). For more examples, including running scripts and a Python REPL -see the startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). +(default is http://localhost:5003/). For full examples (you'll need to +set a number of environment variables), including running scripts and +a Python REPL, see the startup script +[./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). +## Testing -We are building up automated -testing using [mechanize](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) which can be run with +We are building 'Mechanical Rob' automated testing using Python +[requests](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) +which can be run with something like ```sh -./bin/test-website +env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 ``` +The GN2_PROFILE is the Guix profile that contains all +dependencies. The ./bin/genenetwork2 script sets up the environment +and executes test-website.py in a Python interpreter. The -a switch +says to run all tests and the URL points to the running GN2 http +server. + ## Documentation User documentation can be found diff --git a/bin/test-website b/bin/test-website index 5935f016..7fbcfd2f 100755 --- a/bin/test-website +++ b/bin/test-website @@ -2,6 +2,6 @@ if [ -z $GN2_PROFILE ]; then echo "Run request tests with something like" - echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 + echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 exit 1 fi diff --git a/doc/README.org b/doc/README.org index 937a9549..f290480d 100644 --- a/doc/README.org +++ b/doc/README.org @@ -257,6 +257,12 @@ if that works run genenetwork after setting SQL_URI to something like : export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s +* Running ElasticSearch + +In order to start up elasticsearch, use the following command: + +: env JAVA_HOME=/opt/jdk-9.0.4 /opt/elasticsearch-6.2.1/bin/elasticsearch + * Read more If you want to understand the architecture of GN2 read diff --git a/etc/default_settings.py b/etc/default_settings.py index da8cbb7c..3e54ad1f 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -77,7 +77,7 @@ USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls HOME = os.environ['HOME'] # ---- Default locations -GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files +GENENETWORK_FILES = HOME+"/genotype_files" # base dir for all static data files # ---- Path overrides for Genenetwork - the defaults are normally # picked up from Guix or in the HOME directory diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index 64553ed8..715f330c 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -18,6 +18,10 @@ def is_internal_link(link): pattern = re.compile("^/.*") return pattern.match(link) +def is_in_page_link(link): + pattern = re.compile("^#.*") + return pattern.match(link) + def get_links(doc): return filter( lambda x: not ( @@ -40,6 +44,7 @@ def verify_link(link): else: print("ERROR: link `"+link+"` failed with status " , result.status_code) + if DO_FAIL: raise Exception("Failed verify") except ConnectionError as ex: @@ -52,9 +57,10 @@ def check_page(host, start_url): print("Checking links host "+host+" in page `"+start_url+"`") doc = parse(start_url).getroot() links = get_links(doc) + in_page_links = filter(is_in_page_link, links) internal_links = filter(is_internal_link, links) - external_links = filter(lambda x: not is_internal_link(x), links) - # external_links.append("http://somenon-existentsite.brr") + external_links = filter(lambda x: not (is_internal_link(x) or is_in_page_link(x)), links) + for link in internal_links: verify_link(host+link) diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py index fd20df11..8eb19de7 100644 --- a/test/requests/mapping_tests.py +++ b/test/requests/mapping_tests.py @@ -1,17 +1,10 @@ from __future__ import print_function import re +import copy import json import requests from lxml.html import fromstring -def get_data(list_item): - try: - value = list_item[1] - except: - value = None - #print("list_item:", list_item, "==>", value) - return value - def load_data_from_file(): filename = "../test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json" file_handle = open(filename, "r") @@ -19,6 +12,8 @@ def load_data_from_file(): return file_data def check_pylmm_tool_selection(host, data): + print("") + print("pylmm mapping tool selection") data["method"] = "pylmm" page = requests.post(host+"/marker_regression", data=data) doc = fromstring(page.text) @@ -27,10 +22,24 @@ def check_pylmm_tool_selection(host, data): assert form.fields["value:BXD1"] == "15.034" # Check value in the file def check_R_qtl_tool_selection(host, data): - pass + print("") + print("R/qtl mapping tool selection") + headers = {"Content-Type": "application/x-www-form-urlencoded"} + page = requests.post(host+"/marker_regression", data=data, headers=headers) + doc = fromstring(page.text) + form = doc.forms[1] + assert form.fields["dataset"] == "HC_M2_0606_P" + assert form.fields["value:BXD1"] == "15.034" def check_CIM_tool_selection(host, data): - pass + print("") + print("CIM mapping tool selection (using reaper)") + data["method"] = "reaper" + page = requests.post(host+"/marker_regression", data=data) + doc = fromstring(page.text) + form = doc.forms[1] + assert form.fields["dataset"] == "HC_M2_0606_P" + assert form.fields["value:BXD1"] == "15.034" def check_mapping(args_obj, parser): print("") @@ -38,6 +47,6 @@ def check_mapping(args_obj, parser): host = args_obj.host data = load_data_from_file() - check_pylmm_tool_selection(host, data) - check_R_qtl_tool_selection(host, data) - check_CIM_tool_selection(host, data) + check_pylmm_tool_selection(host, copy.deepcopy(data)) + check_R_qtl_tool_selection(host, copy.deepcopy(data)) ## Why does this fail? + check_CIM_tool_selection(host, copy.deepcopy(data)) diff --git a/test/requests/navigation_tests.py b/test/requests/navigation_tests.py new file mode 100644 index 00000000..eda27324 --- /dev/null +++ b/test/requests/navigation_tests.py @@ -0,0 +1,15 @@ +from __future__ import print_function +import re +import requests +from lxml.html import parse + +def check_navigation(args_obj, parser): + print("") + print("Checking navigation.") + + host = args_obj.host + url = host + "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" + print("URL: ", url) + page = requests.get(url) + # Page is built by the javascript, hence using requests fails for this. + # Investigate use of selenium maybe? diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py index abf98fce..50003850 100644 --- a/test/requests/parametrized_test.py +++ b/test/requests/parametrized_test.py @@ -1,5 +1,7 @@ import logging import unittest +from wqflask import app +from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column from elasticsearch import Elasticsearch, TransportError class ParametrizedTest(unittest.TestCase): @@ -10,10 +12,11 @@ class ParametrizedTest(unittest.TestCase): self.es_url = es_url def setUp(self): - self.es = Elasticsearch([self.es_url]) + self.es = get_elasticsearch_connection() self.es_cleanup = [] es_logger = logging.getLogger("elasticsearch") + es_logger.setLevel(app.config.get("LOG_LEVEL")) es_logger.addHandler( logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) es_trace_logger = logging.getLogger("elasticsearch.trace") @@ -21,7 +24,9 @@ class ParametrizedTest(unittest.TestCase): logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) def tearDown(self): + from time import sleep self.es.delete_by_query( index="users" , doc_type="local" , body={"query":{"match":{"email_address":"test@user.com"}}}) + sleep(1) diff --git a/test/requests/run-integration-tests.py b/test/requests/run-integration-tests.py deleted file mode 100644 index 5e816549..00000000 --- a/test/requests/run-integration-tests.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys -from test_login_local import TestLoginLocal -from test_login_orcid import TestLoginOrcid -from test_login_github import TestLoginGithub -from test_registration import TestRegistration -from unittest import TestSuite, TextTestRunner, TestLoader - -test_cases = [ - TestRegistration - , TestLoginLocal - , TestLoginGithub - , TestLoginOrcid -] - -def suite(gn2_url, es_url): - the_suite = TestSuite() - for case in test_cases: - the_suite.addTests(initTest(case, gn2_url, es_url)) - return the_suite - -def initTest(klass, gn2_url, es_url): - loader = TestLoader() - methodNames = loader.getTestCaseNames(klass) - return [klass(mname, gn2_url, es_url) for mname in methodNames] - -def main(gn2_url, es_url): - runner = TextTestRunner() - runner.run(suite(gn2_url, es_url)) - -if __name__ == "__main__": - if len(sys.argv) < 3: - raise Exception("Required arguments missing:\n\tTry running `run-integration-test.py <gn2-url> <es-url>`") - else: - main(sys.argv[1], sys.argv[2]) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 118c9df1..b2e09bc4 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -7,10 +7,20 @@ from __future__ import print_function import argparse from link_checker import check_links from mapping_tests import check_mapping +from navigation_tests import check_navigation from main_web_functionality import check_main_web_functionality import link_checker import sys +# Imports for integration tests +from wqflask import app +from test_login_local import TestLoginLocal +from test_login_orcid import TestLoginOrcid +from test_login_github import TestLoginGithub +from test_registration import TestRegistration +from test_forgot_password import TestForgotPassword +from unittest import TestSuite, TextTestRunner, TestLoader + print("Mechanical Rob firing up...") def run_all(args_obj, parser): @@ -29,6 +39,33 @@ def print_help(args_obj, parser): def dummy(args_obj, parser): print("Not implemented yet.") +def integration_tests(args_obj, parser): + gn2_url = args_obj.host + es_url = app.config.get("ELASTICSEARCH_HOST")+":"+str(app.config.get("ELASTICSEARCH_PORT")) + run_integration_tests(gn2_url, es_url) + +def initTest(klass, gn2_url, es_url): + loader = TestLoader() + methodNames = loader.getTestCaseNames(klass) + return [klass(mname, gn2_url, es_url) for mname in methodNames] + +def integration_suite(gn2_url, es_url): + test_cases = [ + TestRegistration + , TestLoginLocal + , TestLoginGithub + , TestLoginOrcid + , TestForgotPassword + ] + the_suite = TestSuite() + for case in test_cases: + the_suite.addTests(initTest(case, gn2_url, es_url)) + return the_suite + +def run_integration_tests(gn2_url, es_url): + runner = TextTestRunner() + runner.run(integration_suite(gn2_url, es_url)) + desc = """ This is Mechanical-Rob - an automated web server tester for @@ -63,6 +100,11 @@ parser.add_argument("-m", "--mapping", dest="accumulate" , action="store_const", const=check_mapping, default=print_help , help="Checks for mapping.") +parser.add_argument("-i", "--integration-tests", dest="accumulate" + , action="store_const", const=integration_tests, default=print_help + , help="Runs integration tests.") + +# Navigation tests deactivated since system relies on Javascript # parser.add_argument("-n", "--navigation", dest="accumulate" # , action="store_const", const=check_navigation, default=print_help # , help="Checks for navigation.") diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py new file mode 100644 index 00000000..2bf34c5c --- /dev/null +++ b/test/requests/test_forgot_password.py @@ -0,0 +1,52 @@ +import requests +from wqflask import user_manager +from utility.elasticsearch_tools import get_user_by_unique_column +from parameterized import parameterized +from parametrized_test import ParametrizedTest + +passwork_reset_link = '' +forgot_password_page = None + +class TestForgotPassword(ParametrizedTest): + + def setUp(self): + super(TestForgotPassword, self).setUp() + self.forgot_password_url = self.gn2_url+"/n/forgot_password_submit" + def send_email(to_addr, msg, fromaddr="no-reply@genenetwork.org"): + print("CALLING: send_email_mock()") + email_data = { + "to_addr": to_addr + , "msg": msg + , "fromaddr": from_addr} + + data = { + "es_connection": self.es, + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + user_manager.basic_info = lambda : { "basic_info": "basic" } + user_manager.RegisterUser(data) + + def testWithoutEmail(self): + data = {"email_address": ""} + error_notification = '<div class="alert alert-danger">You MUST provide an email</div>' + result = requests.post(self.forgot_password_url, data=data) + self.assertEqual(result.url, self.gn2_url+"/n/forgot_password") + self.assertTrue( + result.content.find(error_notification) >= 0 + , "Error message should be displayed but was not") + + def testWithNonExistingEmail(self): + # Monkey patching doesn't work, so simply test that getting by email + # returns the correct data + user = get_user_by_unique_column(self.es, "email_address", "non-existent@domain.com") + self.assertTrue(user is None, "Should not find non-existent user") + + def testWithExistingEmail(self): + # Monkey patching doesn't work, so simply test that getting by email + # returns the correct data + user = get_user_by_unique_column(self.es, "email_address", "test@user.com") + self.assertTrue(user is not None, "Should find user") diff --git a/wqflask/base/anon_collection.py b/wqflask/base/anon_collection.py deleted file mode 100644 index 8ee73296..00000000 --- a/wqflask/base/anon_collection.py +++ /dev/null @@ -1,21 +0,0 @@ -class AnonCollection(TraitCollection):
-
- def __init__(self, anon_id)
- self.anon_id = anon_id
- self.collection_members = Redis.smembers(self.anon_id)
- print("self.collection_members is:", self.collection_members)
- self.num_members = len(self.collection_members)
-
-
- @app.route("/collections/remove", methods=('POST',))
- def remove_traits(traits_to_remove):
- print("traits_to_remove:", traits_to_remove)
- for trait in traits_to_remove:
- Redis.srem(self.anon_id, trait)
- members_now = self.collection_members - traits_to_remove
- print("members_now:", members_now)
- print("Went from {} to {} members in set.".format(len(self.collection_members), len(members_now)))
-
- # We need to return something so we'll return this...maybe in the future
- # we can use it to check the results
- return str(len(members_now))
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index a4eaaa2e..4a422ee4 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -29,7 +29,6 @@ import json import gzip import cPickle as pickle import itertools -from operator import itemgetter from redis import Redis Redis = Redis() @@ -170,8 +169,27 @@ class Markers(object): """Todo: Build in cacheing so it saves us reading the same file more than once""" def __init__(self, name): json_data_fh = open(locate(name + ".json",'genotype/json')) + try: - markers = json.load(json_data_fh) + markers = [] + with open(locate(name + "_snps.txt", 'r')) as bimbam_fh: + marker = {} + if len(bimbam_fh[0].split(", ")) > 2: + delimiter = ", " + elif len(bimbam_fh[0].split(",")) > 2: + delimiter = "," + elif len(bimbam_fh[0].split("\t")) > 2: + delimiter = "\t" + else: + delimiter = " " + for line in bimbam_fh: + marker['name'] = line.split(delimiter)[0] + marker['Mb'] + marker['chr'] = line.split(delimiter)[2] + marker['cM'] + markers.append(marker) + #try: + # markers = json.load(json_data_fh) except: markers = [] @@ -181,8 +199,6 @@ class Markers(object): marker['Mb'] = float(marker['Mb']) self.markers = markers - #logger.debug("self.markers:", self.markers) - def add_pvalues(self, p_values): logger.debug("length of self.markers:", len(self.markers)) @@ -316,9 +332,6 @@ class DatasetGroup(object): return mapping_id, mapping_names - def get_specified_markers(self, markers = []): - self.markers = HumanMarkers(self.name, markers) - def get_markers(self): logger.debug("self.species is:", self.species) @@ -449,7 +462,6 @@ def datasets(group_name, this_group = None): group_name, webqtlConfig.PUBLICTHRESH, "'" + group_name + "'", webqtlConfig.PUBLICTHRESH)) - #for tissue_name, dataset in itertools.groupby(the_results, itemgetter(0)): for dataset_item in the_results: tissue_name = dataset_item[0] dataset = dataset_item[1] @@ -457,14 +469,10 @@ def datasets(group_name, this_group = None): if tissue_name in ['#PublishFreeze', '#GenoFreeze']: dataset_menu.append(dict(tissue=None, datasets=[(dataset, dataset_short)])) else: - dataset_sub_menu = [item[1:] for item in dataset] - tissue_already_exists = False - tissue_position = None for i, tissue_dict in enumerate(dataset_menu): if tissue_dict['tissue'] == tissue_name: tissue_already_exists = True - tissue_position = i break if tissue_already_exists: @@ -719,20 +727,6 @@ class PhenotypeDataSet(DataSet): # (Urgently?) Need to write this pass - def get_trait_list(self): - query = """ - select PublishXRef.Id - from PublishXRef, PublishFreeze - where PublishFreeze.InbredSetId=PublishXRef.InbredSetId - and PublishFreeze.Id = {} - """.format(escape(str(self.id))) - logger.sql(query) - results = g.db.execute(query).fetchall() - trait_data = {} - for trait in results: - trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) - return trait_data - def get_trait_info(self, trait_list, species = ''): for this_trait in trait_list: @@ -746,7 +740,7 @@ class PhenotypeDataSet(DataSet): #of the post-publication description if this_trait.confidential: this_trait.description_display = "" - continue # for now + continue # for now, because no authorization features if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait( privilege=self.privilege, @@ -770,9 +764,7 @@ class PhenotypeDataSet(DataSet): #LRS and its location this_trait.LRS_score_repr = "N/A" - this_trait.LRS_score_value = 0 this_trait.LRS_location_repr = "N/A" - this_trait.LRS_location_value = 1000000 if this_trait.lrs: query = """ @@ -789,17 +781,7 @@ class PhenotypeDataSet(DataSet): LRS_Chr = result[0] LRS_Mb = result[1] - #XZ: LRS_location_value is used for sorting - try: - LRS_location_value = int(LRS_Chr)*1000 + float(LRS_Mb) - except: - if LRS_Chr.upper() == 'X': - LRS_location_value = 20*1000 + float(LRS_Mb) - else: - LRS_location_value = ord(str(LRS_chr).upper()[0])*1000 + float(LRS_Mb) - this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs - this_trait.LRS_score_value = LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb)) def retrieve_sample_data(self, trait): @@ -861,40 +843,13 @@ class GenotypeDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - def get_trait_list(self): - query = """ - select Geno.Name - from Geno, GenoXRef - where GenoXRef.GenoId = Geno.Id - and GenoFreezeId = {} - """.format(escape(str(self.id))) - logger.sql(query) - results = g.db.execute(query).fetchall() - trait_data = {} - for trait in results: - trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) - return trait_data - def get_trait_info(self, trait_list, species=None): for this_trait in trait_list: if not this_trait.haveinfo: this_trait.retrieveInfo() - #XZ: trait_location_value is used for sorting - trait_location_repr = 'N/A' - trait_location_value = 1000000 - if this_trait.chr and this_trait.mb: - try: - trait_location_value = int(this_trait.chr)*1000 + this_trait.mb - except: - if this_trait.chr.upper() == 'X': - trait_location_value = 20*1000 + this_trait.mb - else: - trait_location_value = ord(str(this_trait.chr).upper()[0])*1000 + this_trait.mb - this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb) ) - this_trait.location_value = trait_location_value def retrieve_sample_data(self, trait): query = """ @@ -989,20 +944,6 @@ class MrnaAssayDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - def get_trait_list_1(self): - query = """ - select ProbeSet.Name - from ProbeSet, ProbeSetXRef - where ProbeSetXRef.ProbeSetId = ProbeSet.Id - and ProbeSetFreezeId = {} - """.format(escape(str(self.id))) - logger.sql(query) - results = g.db.execute(query).fetchall() - trait_data = {} - for trait in results: - trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) - return trait_data - def get_trait_info(self, trait_list=None, species=''): # Note: setting trait_list to [] is probably not a great idea. @@ -1034,27 +975,8 @@ class MrnaAssayDataSet(DataSet): # Save it for the jinja2 template this_trait.description_display = description_display - #XZ: trait_location_value is used for sorting - trait_location_repr = 'N/A' - trait_location_value = 1000000 - if this_trait.chr and this_trait.mb: - #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") - #This is so we can convert the location to a number used for sorting - trait_location_value = self.convert_location_to_value(this_trait.chr, this_trait.mb) - #try: - # trait_location_value = int(this_trait.chr)*1000 + this_trait.mb - #except ValueError: - # if this_trait.chr.upper() == 'X': - # trait_location_value = 20*1000 + this_trait.mb - # else: - # trait_location_value = (ord(str(this_trait.chr).upper()[0])*1000 + - # this_trait.mb) - - #ZS: Put this in function currently called "convert_location_to_value" - this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, - float(this_trait.mb)) - this_trait.location_value = trait_location_value + this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb)) #Get mean expression value query = ( @@ -1076,9 +998,7 @@ class MrnaAssayDataSet(DataSet): #LRS and its location this_trait.LRS_score_repr = 'N/A' - this_trait.LRS_score_value = 0 this_trait.LRS_location_repr = 'N/A' - this_trait.LRS_location_value = 1000000 #Max LRS and its Locus location if this_trait.lrs and this_trait.locus: @@ -1093,40 +1013,10 @@ class MrnaAssayDataSet(DataSet): if result: lrs_chr, lrs_mb = result - #XZ: LRS_location_value is used for sorting - lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb) this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs - this_trait.LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb)) - - def convert_location_to_value(self, chromosome, mb): - try: - location_value = int(chromosome)*1000 + float(mb) - except ValueError: - if chromosome.upper() == 'X': - location_value = 20*1000 + float(mb) - else: - location_value = (ord(str(chromosome).upper()[0])*1000 + - float(mb)) - - return location_value - - def get_sequence(self): - query = """ - SELECT - ProbeSet.BlatSeq - FROM - ProbeSet, ProbeSetFreeze, ProbeSetXRef - WHERE - ProbeSet.Id=ProbeSetXRef.ProbeSetId and - ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and - ProbeSet.Name = %s - ProbeSetFreeze.Name = %s - """ % (escape(self.name), escape(self.dataset.name)) - logger.sql(query) - results = g.db.execute(query).fetchone() - return results[0] + return trait_list def retrieve_sample_data(self, trait): query = """ @@ -1150,7 +1040,6 @@ class MrnaAssayDataSet(DataSet): #logger.debug("RETRIEVED RESULTS HERE:", results) return results - def retrieve_genes(self, column_name): query = """ select ProbeSet.Name, ProbeSet.%s @@ -1204,19 +1093,6 @@ class TempDataSet(DataSet): desc = self.handle_pca(desc) return desc - def get_group(self): - query = """ - SELECT - InbredSet.Name, InbredSet.Id - FROM - InbredSet, Temp - WHERE - Temp.InbredSetId = InbredSet.Id AND - Temp.Name = "%s" - """ % self.name - logger.sql(query) - self.group, self.group_id = g.db.execute(query).fetchone() - def retrieve_sample_data(self, trait): query = """ SELECT diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index eb836e6c..6fec5dcd 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -18,21 +18,11 @@ class MrnaAssayTissueData(object): def __init__(self, gene_symbols=None): self.gene_symbols = gene_symbols - self.have_data = False if self.gene_symbols == None: self.gene_symbols = [] - #print("self.gene_symbols:", self.gene_symbols) - self.data = collections.defaultdict(Bunch) - #self.gene_id_dict ={} - #self.data_id_dict = {} - #self.chr_dict = {} - #self.mb_dict = {} - #self.desc_dict = {} - #self.probe_target_desc_dict = {} - query = '''select t.Symbol, t.GeneId, t.DataId, t.Chr, t.Mb, t.description, t.Probe_Target_Description from ( select Symbol, max(Mean) as maxmean @@ -52,7 +42,6 @@ class MrnaAssayTissueData(object): in_clause = db_tools.create_in_clause(gene_symbols) #ZS: This was in the query, not sure why: http://docs.python.org/2/library/string.html?highlight=lower#string.lower - query += ''' Symbol in {} group by Symbol) as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol and t.Mean = x.maxmean; @@ -67,9 +56,7 @@ class MrnaAssayTissueData(object): for result in results: symbol = result[0] - #if symbol.lower() in [gene_symbol.lower() for gene_symbol in gene_symbols]: if symbol.lower() in lower_symbols: - #gene_symbols.append(symbol) symbol = symbol.lower() self.data[symbol].gene_id = result.GeneId @@ -79,8 +66,6 @@ class MrnaAssayTissueData(object): self.data[symbol].description = result.description self.data[symbol].probe_target_description = result.Probe_Target_Description - print("self.data: ", pf(self.data)) - ########################################################################### #Input: cursor, symbolList (list), dataIdDict(Dict) #output: symbolValuepairDict (dictionary):one dictionary of Symbol and Value Pair, @@ -107,53 +92,4 @@ class MrnaAssayTissueData(object): else: symbol_values_dict[result.Symbol.lower()].append(result.value) - #for symbol in self.data: - # data_id = self.data[symbol].data_id - # symbol_values_dict[symbol] = self.get_tissue_values(data_id) - - - return symbol_values_dict - - - #def get_tissue_values(self, data_id): - # """Gets the tissue values for a particular gene""" - # - # tissue_values=[] - # - # query = """SELECT value, id - # FROM TissueProbeSetData - # WHERE Id IN {}""".format(db_tools.create_in_clause(data_id)) - # - # #try : - # results = g.db.execute(query).fetchall() - # for result in results: - # tissue_values.append(result.value) - # #symbol_values_dict[symbol] = value_list - # #except: - # # symbol_values_pairs[symbol] = None - # - # return tissue_values - -######################################################################################################## -#input: cursor, symbolList (list), dataIdDict(Dict): key is symbol -#output: SymbolValuePairDict(dictionary):one dictionary of Symbol and Value Pair. -# key is symbol, value is one list of expression values of one probeSet. -#function: wrapper function for getSymbolValuePairDict function -# build gene symbol list if necessary, cut it into small lists if necessary, -# then call getSymbolValuePairDict function and merge the results. -######################################################################################################## - -#def get_trait_symbol_and_tissue_values(symbol_list=None): -# tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list) -# -# #symbolList, -# #geneIdDict, -# #dataIdDict, -# #ChrDict, -# #MbDict, -# #descDict, -# #pTargetDescDict = getTissueProbeSetXRefInfo( -# # GeneNameLst=GeneNameLst,TissueProbeSetFreezeId=TissueProbeSetFreezeId) -# -# if len(tissue_data.gene_symbols): -# return get_symbol_values_pairs(tissue_data) + return symbol_values_dict
\ No newline at end of file diff --git a/wqflask/base/species.py b/wqflask/base/species.py index ce763fc3..4ac2213c 100644 --- a/wqflask/base/species.py +++ b/wqflask/base/species.py @@ -18,19 +18,6 @@ class TheSpecies(object): self.dataset = dataset #print("self.dataset is:", pf(self.dataset.__dict__)) self.chromosomes = Chromosomes(self.dataset) - self.genome_mb_length = self.chromosomes.get_genome_mb_length() - - #@property - #def chromosomes(self): - # chromosomes = [("All", -1)] - # - # for counter, genotype in enumerate(self.dataset.group.genotype): - # if len(genotype) > 1: - # chromosomes.append((genotype.name, counter)) - # - # print("chromosomes is: ", pf(chromosomes)) - # - # return chromosomes class IndChromosome(object): def __init__(self, name, length): @@ -42,16 +29,11 @@ class IndChromosome(object): """Chromosome length in megabases""" return self.length / 1000000 - def set_cm_length(self, genofile_chr): - self.cm_length = genofile_chr[-1].cM - genofile_chr[0].cM - - class Chromosomes(object): def __init__(self, dataset): self.dataset = dataset self.chromosomes = collections.OrderedDict() - query = """ Select Chr_Length.Name, Chr_Length.OrderId, Length from Chr_Length, InbredSet @@ -64,64 +46,4 @@ class Chromosomes(object): results = g.db.execute(query).fetchall() for item in results: - self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length) - - self.set_mb_graph_interval() - #self.get_cm_length_list() - - - def set_mb_graph_interval(self): - """Empirical megabase interval""" - - if self.chromosomes: - self.mb_graph_interval = self.get_genome_mb_length()/(len(self.chromosomes)*12) - else: - self.mb_graph_interval = 1 - - #if self.chromosomes: - #assert self.chromosomes, "Have to add some code back in apparently to set it to 1" - #self.mb_graph_interval = self.get_genome_mb_length()/(len(self.chromosomes)*12) - #else: - #self.mb_graph_interval = 1 - - - def get_genome_mb_length(self): - """Gets the sum of each chromosome's length in megabases""" - - return sum([ind_chromosome.mb_length for ind_chromosome in self.chromosomes.values()]) - - - def get_genome_cm_length(self): - """Gets the sum of each chromosome's length in centimorgans""" - - return sum([ind_chromosome.cm_length for ind_chromosome in self.chromosomes.values()]) - - def get_cm_length_list(self): - """Chromosome length in centimorgans - - Calculates the length in centimorgans by subtracting the centimorgan position - of the last marker in a chromosome by the position of the first marker - - """ - - self.dataset.group.read_genotype_file() - - self.cm_length_list = [] - - for chromosome in self.dataset.group.genotype: - self.cm_length_list.append(chromosome[-1].cM - chromosome[0].cM) - - print("self.cm_length_list:", pf(self.cm_length_list)) - - assert len(self.cm_length_list) == len(self.chromosomes), "Uh-oh lengths should be equal!" - for counter, chromosome in enumerate(self.chromosomes.values()): - chromosome.cm_length = self.cm_length_list[counter] - #self.chromosomes[counter].cm_length = item - - for key, value in self.chromosomes.items(): - print("bread - %s: %s" % (key, pf(vars(value)))) - - -# Testing -#if __name__ == '__main__': -# foo = dict(bar=dict(length)) + self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length)
\ No newline at end of file diff --git a/wqflask/base/template.py b/wqflask/base/template.py deleted file mode 100644 index aa8f90dc..00000000 --- a/wqflask/base/template.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# This program is available from Source Forge: at GeneNetwork Project -# (sourceforge.net/projects/genenetwork/). -# -# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) -# at rwilliams@uthsc.edu and xzhou15@uthsc.edu -# -# -# -# This module is used by GeneNetwork project (www.genenetwork.org) -# -# Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 - -template = """ -<?XML VERSION="1.0" ENCODING="UTF-8"> -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> -<HTML> -<HEAD> -<TITLE>%s</TITLE> - -<META http-equiv=Content-Type content="text/html; charset=iso-8859-1"> -<META NAME="keywords" CONTENT="genetics, bioinformatics, genome, phenome, gene expression, complex trait analysis, gene mapping, SNP, quantitative trait locus QTL, expression eQTL, WebQTL, Traitnet, Traitnetwork, personalized medicine"> -<META NAME="description" CONTENT ="GeneNetwork is a free scientific web resource used to study relationships between differences in genes, environmental factors, phenotypes, and disease risk." > -<META NAME="author" CONTENT ="GeneNetwork developers" > -<META NAME="geo.placename" CONTENT ="Memphis, TN" > -<META NAME="geo.region" CONTENT="US-TN"> -%s -<LINK REL="stylesheet" TYPE="text/css" HREF='/css/general.css'> -<LINK REL="stylesheet" TYPE="text/css" HREF='/css/menu.css'> -<link rel="stylesheet" media="all" type="text/css" href="/css/tabbed_pages.css" /> -<LINK REL="apple-touch-icon" href="/images/ipad_icon3.png" /> -<link type="text/css" href='/css/custom-theme/jquery-ui-1.8.12.custom.css' rel='Stylesheet' /> -<link type="text/css" href='/css/tab_style.css' rel='Stylesheet' /> - -<script type="text/javascript" src="/javascript/jquery-1.5.2.min.js"></script> -<SCRIPT SRC="/javascript/webqtl.js"></SCRIPT> -<SCRIPT SRC="/javascript/dhtml.js"></SCRIPT> -<SCRIPT SRC="/javascript/tablesorter.js"></SCRIPT> -<SCRIPT SRC="/javascript/jqueryFunction.js"></SCRIPT> -<script src="/javascript/tabbed_pages.js" type="text/javascript"></script> -<script src="/javascript/jquery-ui-1.8.12.custom.min.js" type="text/javascript"></script> -%s - -<script type="text/javascript"> - var _gaq = _gaq || []; - _gaq.push(['_setAccount', 'UA-3782271-1']); - _gaq.push(['_trackPageview']); - (function() { - var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; - ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; - var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); - })(); -</script> -</HEAD> -<BODY bottommargin="2" leftmargin="2" rightmargin="2" topmargin="2" text=#000000 bgColor=#ffffff %s> -%s -<TABLE cellSpacing=5 cellPadding=4 width="100%%" border=0> - <TBODY> - <!-- Start of header --> - <TR> - %s - </TR> - <!-- End of header --> - - <!-- Start of body --> - <TR> - <TD bgColor=#eeeeee class="solidBorder"> - <Table width= "100%%" cellSpacing=0 cellPadding=5> - <TR> - %s - </TR> - </TABLE> - </TD> - </TR> - <!-- End of body --> - - <!-- Start of footer --> - <TR> - <TD align=center bgColor=#ddddff class="solidBorder"> - <TABLE width="90%%">%s</table> - </td> - </TR> - <!-- End of footer --> -</TABLE> - -<!-- menu script itself. you should not modify this file --> -<script language="JavaScript" src="/javascript/menu_new.js"></script> -<!-- items structure. menu hierarchy and links are stored there --> -<script language="JavaScript" src="/javascript/menu_items.js"></script> -<!-- files with geometry and styles structures --> -<script language="JavaScript" src="/javascript/menu_tpl.js"></script> -<script language="JavaScript"> - <!--// - // Note where menu initialization block is located in HTML document. - // Don't try to position menu locating menu initialization block in - // some table cell or other HTML element. Always put it before </body> - // each menu gets two parameters (see demo files) - // 1. items structure - // 2. geometry structure - new menu (MENU_ITEMS, MENU_POS); - // make sure files containing definitions for these variables are linked to the document - // if you got some javascript error like "MENU_POS is not defined", then you've made syntax - // error in menu_tpl.js file or that file isn't linked properly. - - // also take a look at stylesheets loaded in header in order to set styles - //--> -</script> -</BODY> -</HTML> -""" diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index acc055d8..3daf9ea9 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -18,17 +18,13 @@ import simplejson as json from MySQLdb import escape_string as escape from pprint import pformat as pf -from flask import Flask, g, request +from flask import Flask, g, request, url_for from utility.logger import getLogger logger = getLogger(__name__ ) from wqflask import user_manager -def print_mem(stage=""): - mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - print("{}: {}".format(stage, mem/1024)) - class GeneralTrait(object): """ Trait class defines a trait in webqtl, can be either Microarray, @@ -63,9 +59,7 @@ class GeneralTrait(object): self.symbol = None self.LRS_score_repr = "N/A" - self.LRS_score_value = 0 self.LRS_location_repr = "N/A" - self.LRS_location_value = 1000000 if kw.get('fullname'): name2 = value.split("::") @@ -82,90 +76,6 @@ class GeneralTrait(object): if get_sample_info != False: self = retrieve_sample_data(self, self.dataset) - - def get_name(self): - stringy = "" - if self.dataset and self.name: - stringy = "%s::%s" % (self.dataset, self.name) - if self.cellid: - stringy += "::" + self.cellid - else: - stringy = self.description - return stringy - - - def get_given_name(self): - """ - when user enter a trait or GN generate a trait, user want show the name - not the name that generated by GN randomly, the two follow function are - used to give the real name and the database. displayName() will show the - database also, getGivenName() just show the name. - For other trait, displayName() as same as getName(), getGivenName() as - same as self.name - - Hongqiang 11/29/07 - - """ - stringy = self.name - if self.dataset and self.name: - desc = self.dataset.get_desc() - if desc: - #desc = self.handle_pca(desc) - stringy = desc - return stringy - - - def display_name(self): - stringy = "" - if self.dataset and self.name: - desc = self.dataset.get_desc() - #desc = self.handle_pca(desc) - if desc: - #desc = self.handle_pca(desc) - #stringy = desc - #if desc.__contains__('PCA'): - # desc = desc[desc.rindex(':')+1:].strip() - #else: - # desc = desc[:desc.index('entered')].strip() - #desc = self.handle_pca(desc) - stringy = "%s::%s" % (self.dataset, desc) - else: - stringy = "%s::%s" % (self.dataset, self.name) - if self.cellid: - stringy += "::" + self.cellid - else: - stringy = self.description - - return stringy - - - #def __str__(self): - # #return "%s %s" % (self.getName(), self.group) - # return self.getName() - #__str__ = getName - #__repr__ = __str__ - - def export_data(self, samplelist, the_type="val"): - """ - export data according to samplelist - mostly used in calculating correlation - - """ - result = [] - for sample in samplelist: - if self.data.has_key(sample): - if the_type=='val': - result.append(self.data[sample].val) - elif the_type=='var': - result.append(self.data[sample].var) - elif the_type=='N': - result.append(self.data[sample].N) - else: - raise KeyError, `the_type`+' the_type is incorrect.' - else: - result.append(None) - return result - def export_informative(self, include_variance=0): """ export informative sample @@ -185,19 +95,6 @@ class GeneralTrait(object): sample_aliases.append(sample_data.name2) return samples, vals, the_vars, sample_aliases - - @property - def name_header_fmt(self): - '''Return a human-readable name for use in page header''' - if self.dataset.type == 'ProbeSet': - return self.symbol - elif self.dataset.type == 'Geno': - return self.name - elif self.dataset.type == 'Publish': - return self.post_publication_abbreviation - else: - return "unnamed" - @property def description_fmt(self): '''Return a text formated description''' @@ -252,29 +149,6 @@ class GeneralTrait(object): fmt += (' on the minus strand ') return fmt - -# In ProbeSet, there are maybe several annotations match one sequence -# so we need use sequence(BlatSeq) as the identification, when we update -# one annotation, we update the others who match the sequence also. -# -# Hongqiang Li, 3/3/2008 -def getSequence(trait, dataset_name): - dataset = create_dataset(dataset_name) - - if dataset.type == 'ProbeSet': - results = g.db.execute(''' - SELECT - ProbeSet.BlatSeq - FROM - ProbeSet, ProbeSetFreeze, ProbeSetXRef - WHERE - ProbeSet.Id=ProbeSetXRef.ProbeSetId and - ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and - ProbeSet.Name = %s - ProbeSetFreeze.Name = %s - ''', trait.name, dataset.name).fetchone() - - return results[0] def retrieve_sample_data(trait, dataset, samplelist=None): if samplelist == None: @@ -293,18 +167,6 @@ def retrieve_sample_data(trait, dataset, samplelist=None): if not samplelist or (samplelist and name in samplelist): trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) return trait - -def convert_location_to_value(chromosome, mb): - try: - location_value = int(chromosome)*1000 + float(mb) - except ValueError: - if chromosome.upper() == 'X': - location_value = 20*1000 + float(mb) - else: - location_value = (ord(str(chromosome).upper()[0])*1000 + - float(mb)) - - return location_value @app.route("/trait/get_sample_data") def get_sample_data(): @@ -314,13 +176,23 @@ def get_sample_data(): trait_ob = GeneralTrait(name=trait, dataset_name=dataset) - return json.dumps([trait, {key: value.value for key, value in trait_ob.data.iteritems() }]) - - #jsonable_sample_data = {} - #for sample in trait_ob.data.iteritems(): - # jsonable_sample_data[sample] = trait_ob.data[sample].value - # - #return jsonable_sample_data + trait_dict = {} + trait_dict['name'] = trait + trait_dict['db'] = dataset + trait_dict['type'] = trait_ob.dataset.type + trait_dict['group'] = trait_ob.dataset.group.name + trait_dict['tissue'] = trait_ob.dataset.tissue + trait_dict['species'] = trait_ob.dataset.group.species + trait_dict['url'] = url_for('show_trait_page', trait_id = trait, dataset = dataset) + trait_dict['description'] = trait_ob.description_display + if trait_ob.dataset.type == "ProbeSet": + trait_dict['symbol'] = trait_ob.symbol + trait_dict['location'] = trait_ob.location_repr + elif trait_ob.dataset.type == "Publish": + trait_dict['pubmed_link'] = trait_ob.pubmed_link + trait_dict['pubmed_text'] = trait_ob.pubmed_text + + return json.dumps([trait_dict, {key: value.value for key, value in trait_ob.data.iteritems() }]) def jsonable(trait): """Return a dict suitable for using as json @@ -542,38 +414,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if trait.pubmed_id: trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id - - trait.homologeneid = None if dataset.type == 'ProbeSet' and dataset.group: - if trait.geneid: - #XZ, 05/26/2010: From time to time, this query get error message because some geneid values in database are not number. - #XZ: So I have to test if geneid is number before execute the query. - #XZ: The geneid values in database should be cleaned up. - #try: - # float(self.geneid) - # geneidIsNumber = True - #except ValueError: - # geneidIsNumber = False - #if geneidIsNumber: - query = """ - SELECT - HomologeneId - FROM - Homologene, Species, InbredSet - WHERE - Homologene.GeneId ='%s' AND - InbredSet.Name = '%s' AND - InbredSet.SpeciesId = Species.Id AND - Species.TaxonomyId = Homologene.TaxonomyId - """ % (escape(str(trait.geneid)), escape(dataset.group.name)) - logger.sql(query) - result = g.db.execute(query).fetchone() - #else: - # result = None - - if result: - trait.homologeneid = result[0] - description_string = unicode(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') target_string = unicode(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') @@ -589,46 +430,19 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): # Save it for the jinja2 template trait.description_display = description_display - #XZ: trait_location_value is used for sorting trait.location_repr = 'N/A' - trait.location_value = 1000000 - if trait.chr and trait.mb: - #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") - #This is so we can convert the location to a number used for sorting - trait_location_value = convert_location_to_value(trait.chr, trait.mb) - #try: - # trait_location_value = int(self.chr)*1000 + self.mb - #except ValueError: - # if self.chr.upper() == 'X': - # trait_location_value = 20*1000 + self.mb - # else: - # trait_location_value = (ord(str(self.chr).upper()[0])*1000 + - # self.mb) - - #ZS: Put this in function currently called "convert_location_to_value" trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) - trait.location_value = trait_location_value elif dataset.type == "Geno": trait.location_repr = 'N/A' - trait.location_value = 1000000 - if trait.chr and trait.mb: - #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") - #This is so we can convert the location to a number used for sorting - trait_location_value = convert_location_to_value(trait.chr, trait.mb) - - #ZS: Put this in function currently called "convert_location_to_value" trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) - trait.location_value = trait_location_value if get_qtl_info: #LRS and its location trait.LRS_score_repr = "N/A" - trait.LRS_score_value = 0 trait.LRS_location_repr = "N/A" - trait.LRS_location_value = 1000000 if dataset.type == 'ProbeSet' and not trait.cellid: query = """ SELECT @@ -699,19 +513,9 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.locus = trait.lrs = trait.additive = "" if (dataset.type == 'Publish' or dataset.type == "ProbeSet") and trait.locus_chr != "" and trait.locus_mb != "": - #XZ: LRS_location_value is used for sorting - try: - LRS_location_value = int(trait.locus_chr)*1000 + float(trait.locus_mb) - except: - if trait.locus_chr.upper() == 'X': - LRS_location_value = 20*1000 + float(trait.locus_mb) - else: - LRS_location_value = ord(str(trait.locus_chr).upper()[0])*1000 + float(trait.locus_mb) - trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (trait.locus_chr, float(trait.locus_mb)) if trait.lrs != "": trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs - trait.LRS_score_value = LRS_score_value = trait.lrs else: raise KeyError, `trait.name`+' information is not found in the database.' diff --git a/wqflask/base/trait_collection.py b/wqflask/base/trait_collection.py deleted file mode 100644 index d388a3af..00000000 --- a/wqflask/base/trait_collection.py +++ /dev/null @@ -1,53 +0,0 @@ -class TraitCollection(object):
-
- def __init__(self, is_anon=False):
- self.is_anon = is_anon
-
-
- @app.route("/collections/remove", methods=('POST',))
- def remove_traits():
- if is_anon:
- AnonCollection.remove_traits()
- else:
- UserCollection.remove_traits()
-
- params = request.form
- print("params are:", params)
- uc_id = params['uc_id']
- uc = model.UserCollection.query.get(uc_id)
- traits_to_remove = params.getlist('traits[]')
- print("traits_to_remove are:", traits_to_remove)
- traits_to_remove = process_traits(traits_to_remove)
- print("\n\n after processing, traits_to_remove:", traits_to_remove)
- all_traits = uc.members_as_set()
- print(" all_traits:", all_traits)
- members_now = all_traits - traits_to_remove
- print(" members_now:", members_now)
- print("Went from {} to {} members in set.".format(len(all_traits), len(members_now)))
- uc.members = json.dumps(list(members_now))
- uc.changed_timestamp = datetime.datetime.utcnow()
- db_session.commit()
-
- # We need to return something so we'll return this...maybe in the future
- # we can use it to check the results
- return str(len(members_now))
-
- def __init__(self, anon_id)
- self.anon_key = anon_key
- self.collection_members = Redis.smembers(self.anon_id)
- print("self.collection_members is:", self.collection_members)
- self.num_members = len(self.collection_members)
-
-
- @app.route("/collections/remove", methods=('POST',))
- def remove_traits(traits_to_remove):
- print("traits_to_remove:", traits_to_remove)
- for trait in traits_to_remove:
- Redis.srem(self.anon_id, trait)
- members_now = self.collection_members - traits_to_remove
- print("members_now:", members_now)
- print("Went from {} to {} members in set.".format(len(self.collection_members), len(members_now)))
-
- # We need to return something so we'll return this...maybe in the future
- # we can use it to check the results
- return str(len(members_now))
diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 1e66e957..4708bf0a 100644 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -20,46 +20,20 @@ USERDICT = {'guest':1,'user':2, 'admin':3, 'root':4} #minimum number of informative strains KMININFORMATIVE = 5 -#maximum number of traits for interval mapping -MULTIPLEMAPPINGLIMIT = 11 - -#maximum number of traits for correlation -MAXCORR = 100 - #Daily download limit from one IP DAILYMAXIMUM = 1000 #maximum LRS value MAXLRS = 460.0 -#temporary data life span -MAXLIFE = 86400 - #MINIMUM Database public value PUBLICTHRESH = 0 -#NBCI address -NCBI_LOCUSID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" -UCSC_REFSEQ = "http://genome.cse.ucsc.edu/cgi-bin/hgGene?db=%s&hgg_gene=%s&hgg_chrom=chr%s&hgg_start=%s&hgg_end=%s" -GENBANK_ID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=Nucleotide&cmd=search&doptcmdl=DocSum&term=%s" -OMIM_ID = "http://www.ncbi.nlm.nih.gov/omim/%s" -UNIGEN_ID = "http://www.ncbi.nlm.nih.gov/UniGene/clust.cgi?ORG=%s&CID=%s"; -HOMOLOGENE_ID = "http://www.ncbi.nlm.nih.gov/sites/entrez?Db=homologene&Cmd=DetailsSearch&Term=%s" +#EXTERNAL LINK ADDRESSES PUBMEDLINK_URL = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids=%s&dopt=Abstract" -UCSC_POS = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=mammal&org=%s&db=%s&position=chr%s:%s-%s&pix=800&Submit=submit" UCSC_BLAT = 'http://genome.ucsc.edu/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' UTHSC_BLAT = 'http://ucscbrowser.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' UTHSC_BLAT2 = 'http://ucscbrowserbeta.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' -UCSC_GENOME = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=http://web2qtl.utmem.edu:88/snp/chr%s" -ENSEMBLE_BLAT = 'http://www.ensembl.org/Mus_musculus/featureview?type=AffyProbe&id=%s' -DBSNP = 'http://www.ncbi.nlm.nih.gov/SNP/snp_ref.cgi?type=rs&rs=%s' -UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s&hgt.customText=http://gbic.biol.rug.nl/~ralberts/tracks/%s/%s" -GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit" -ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s" - -# The following paths are no longer in use! -# HTMLPATH is replaced by GENODIR -# IMGDIR is replaced by GENERATED_IMAGE_DIR # Temporary storage (note that this TMPDIR can be set as an # environment variable - use utility.tools.TEMPDIR when you diff --git a/wqflask/base/webqtlFormData.py b/wqflask/base/webqtlFormData.py deleted file mode 100644 index 10251756..00000000 --- a/wqflask/base/webqtlFormData.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# This program is available from Source Forge: at GeneNetwork Project -# (sourceforge.net/projects/genenetwork/). -# -# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) -# at rwilliams@uthsc.edu and xzhou15@uthsc.edu -# -# -# -# This module is used by GeneNetwork project (www.genenetwork.org) -# -# Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 - -#from mod_python import Cookie - -from __future__ import print_function -from pprint import pformat as pf - -import string -import os - -import reaper - -import webqtlConfig -from webqtlCaseData import webqtlCaseData -from utility import webqtlUtil - -class webqtlFormData(object): - 'Represents data from a WebQTL form page, needed to generate the next page' - - attrs = ('formID','group','genotype','samplelist','allsamplelist', 'display_variance' - 'suggestive','significance','submitID','identification', 'enablevariance', - 'nperm','nboot','email','incparentsf1','genotype_1','genotype_2','traitInfo') - - #XZ: Attention! All attribute values must be picklable! - - def __init__(self, - start_vars = None, - req = None, - mod_python_session=None, - FieldStorage_formdata=None): - # Todo: rework this whole thing - print("in webqtlFormData start_vars are:", pf(start_vars)) - for item in webqtlFormData.attrs: - self.__dict__[item] = None - - #ZS: This is only used in DataEditingPage.py (as far as I know) - self.varianceDispName = None - - for item in start_vars: - self.__dict__[item] = start_vars[item] - print(" Now self.dict is:", pf(self.__dict__)) - - #Todo: This can't be good below...rework - try: - self.remote_ip = req.connection.remote_ip - except: - self.remote_ip = '1.2.3.4' - - if req and req.headers_in.has_key('referer'): - self.refURL = req.headers_in['referer'] - else: - self.refURL = None - - # For now let's just comment all this out - Sam - - #self.cookies = cookieData.cookieData(Cookie.get_cookies(req)) #XZ: dictionary type. To hold values transfered from mod_python Cookie. - # - ##XZ: dictionary type. To hold values transfered from mod_python Session object. We assume that it is always picklable. - #self.input_session_data = sessionData.sessionData( mod_python_session ) - # - ##XZ: FieldStorage_formdata may contain item that can't be pickled. Must convert to picklable data. - #self.formdata = cgiData( FieldStorage_formdata ) - # - ##get Form ID - #self.formID = self.formdata.getfirst('FormID') - # - ##get rest of the attributes - #if self.formID: - # for item in self.attrs: - # value = self.formdata.getfirst(item) - # if value != None: - # setattr(self,item,string.strip(value)) - - self.ppolar = None - self.mpolar = None - - print("[yellow] self.group is:", self.group) - if self.group: - #try: - # # NL, 07/27/2010. ParInfo has been moved from webqtlForm.py to webqtlUtil.py; - _f1, _f12, self.mpolar, self.ppolar = webqtlUtil.ParInfo[self.group] - #except: - # f1 = f12 = self.mpolar = self.ppolar = None - - - def set_number(stringy): - return int(stringy) if stringy else 2000 # Rob asked to change the default value to 2000 - - self.nperm = set_number(self.nperm) - self.nboot = set_number(self.nboot) - - - #if self.allsamplelist: - # self.allsamplelist = map(string.strip, string.split(self.allsamplelist)) - print("self.allsamplelist is:", self.allsamplelist) - if self.allsamplelist: - self.allsamplelist = self.allsamplelist.split() - print("now self.allsamplelist is:", self.allsamplelist) - #self.readGenotype() - #self.readData() - - if self.group == 'BXD300': - self.group = 'BXD' - - - def __getitem__(self, key): - print("in __getitem__") - return self.__dict__[key] - - def get(self, key, default=None): - if key in self.__dict__: - return self.__dict__[key] - else: - return default - - def __str__(self): - rstr = '' - for item in self.attrs: - if item != 'genotype': - rstr += '%s:%s\n' % (item,str(getattr(self,item))) - return rstr - - - def readGenotype(self): - '''read genotype from .geno file''' - if self.group == 'BXD300': - self.group = 'BXD' - - assert self.group, "self.group needs to be set" - - #genotype_1 is Dataset Object without parents and f1 - #genotype_2 is Dataset Object with parents and f1 (not for intercross) - - self.genotype_1 = reaper.Dataset() - - full_filename = locate(self.group + '.geno','genotype') - - # reaper barfs on unicode filenames, so here we ensure it's a string - full_filename = str(full_filename) - self.genotype_1.read(full_filename) - - print("Got to after read") - - try: - # NL, 07/27/2010. ParInfo has been moved from webqtlForm.py to webqtlUtil.py; - _f1, _f12, _mat, _pat = webqtlUtil.ParInfo[self.group] - except KeyError: - _f1 = _f12 = _mat = _pat = None - - self.genotype_2 = self.genotype_1 - if self.genotype_1.type == "group" and _mat and _pat: - self.genotype_2 = self.genotype_1.add(Mat=_mat, Pat=_pat) #, F1=_f1) - - #determine default genotype object - if self.incparentsf1 and self.genotype_1.type != "intercross": - self.genotype = self.genotype_2 - else: - self.incparentsf1 = 0 - self.genotype = self.genotype_1 - - self.samplelist = list(self.genotype.prgy) - self.f1list = [] - self.parlist = [] - - if _f1 and _f12: - self.f1list = [_f1, _f12] - if _mat and _pat: - self.parlist = [_mat, _pat] - - - def readData(self, samplelist, incf1=None): - '''read user input data or from trait data and analysis form''' - - if incf1 == None: - incf1 = [] - - if not self.genotype: - self.readGenotype() - if not samplelist: - if incf1: - samplelist = self.f1list + self.samplelist - else: - samplelist = self.samplelist - - #print("before traitfiledata self.traitfile is:", pf(self.traitfile)) - - traitfiledata = getattr(self, "traitfile", None) - traitpastedata = getattr(self, "traitpaste", None) - variancefiledata = getattr(self, "variancefile", None) - variancepastedata = getattr(self, "variancepaste", None) - Nfiledata = getattr(self, "Nfile", None) - - #### Todo: Rewrite below when we get to someone submitting their own trait ##### - - def to_float(item): - try: - return float(item) - except ValueError: - return None - - print("bottle samplelist is:", samplelist) - if traitfiledata: - tt = traitfiledata.split() - values = map(webqtlUtil.StringAsFloat, tt) - elif traitpastedata: - tt = traitpastedata.split() - values = map(webqtlUtil.StringAsFloat, tt) - else: - print("mapping formdataasfloat") - #values = map(self.FormDataAsFloat, samplelist) - values = [to_float(getattr(self, key)) for key in samplelist] - print("rocket values is:", values) - - - if len(values) < len(samplelist): - values += [None] * (len(samplelist) - len(values)) - elif len(values) > len(samplelist): - values = values[:len(samplelist)] - print("now values is:", values) - - - if variancefiledata: - tt = variancefiledata.split() - variances = map(webqtlUtil.StringAsFloat, tt) - elif variancepastedata: - tt = variancepastedata.split() - variances = map(webqtlUtil.StringAsFloat, tt) - else: - variances = map(self.FormVarianceAsFloat, samplelist) - - if len(variances) < len(samplelist): - variances += [None]*(len(samplelist) - len(variances)) - elif len(variances) > len(samplelist): - variances = variances[:len(samplelist)] - - if Nfiledata: - tt = string.split(Nfiledata) - nsamples = map(webqtlUtil.IntAsFloat, tt) - if len(nsamples) < len(samplelist): - nsamples += [None]*(len(samplelist) - len(nsamples)) - else: - nsamples = map(self.FormNAsFloat, samplelist) - - ##values, variances, nsamples is obsolete - self.allTraitData = {} - for i, _sample in enumerate(samplelist): - if values[i] != None: - self.allTraitData[_sample] = webqtlCaseData( - _sample, values[i], variances[i], nsamples[i]) - print("allTraitData is:", pf(self.allTraitData)) - - - - def informativeStrains(self, samplelist=None, include_variances = None): - '''if readData was called, use this to output informative samples (sample with values)''' - - if not samplelist: - samplelist = self.samplelist - - samples = [] - values = [] - variances = [] - - #print("self.allTraitData is:", pf(self.allTraitData)) - - for sample in samplelist: - if sample in self.allTraitData: - _val, _var = self.allTraitData[sample].value, self.allTraitData[sample].variance - if _val != None: - if include_variances: - if _var != None: - samples.append(sample) - values.append(_val) - variances.append(_var) - else: - samples.append(sample) - values.append(_val) - variances.append(None) - - return samples, values, variances, len(samples) - - - - #def FormDataAsFloat(self, key): - # - # #try: - # # return float(self.key) - # #except: - # # return None - - - def FormVarianceAsFloat(self, key): - try: - return float(self.formdata.getfirst('V' + key)) - except: - return None - - def FormNAsFloat(self, key): - try: - return int(self.formdata.getfirst('N' + key)) - except: - return None - - def Sample(self): - 'Create some dummy data for testing' - self.group = 'BXD' - self.incparentsf1 = 'on' - #self.display = 9.2 - #self.significance = 16.1 - self.readGenotype() - self.identification = 'BXD : Coat color example by Lu Lu, et al' - #self.readGenotype() - #self.genotype.ReadMM('AXBXAforQTL') - #self.samplelist = map((lambda x, y='': '%s%s' % (y,x)), self.genotype.prgy) - #self.samplelist.sort() - self.allTraitData = {'BXD29': webqtlCaseData(3), 'BXD28': webqtlCaseData(2), - 'BXD25': webqtlCaseData(2), 'BXD24': webqtlCaseData(2), 'BXD27': webqtlCaseData(2), - 'BXD21': webqtlCaseData(1), 'BXD20': webqtlCaseData(4), 'BXD23': webqtlCaseData(4), - 'BXD22': webqtlCaseData(3), 'BXD14': webqtlCaseData(4), 'BXD15': webqtlCaseData(2), - 'BXD16': webqtlCaseData(3), 'BXD11': webqtlCaseData(4), 'BXD12': webqtlCaseData(3), - 'BXD13': webqtlCaseData(2), 'BXD18': webqtlCaseData(3), 'BXD19': webqtlCaseData(3), - 'BXD38': webqtlCaseData(3), 'BXD39': webqtlCaseData(3), 'BXD36': webqtlCaseData(2), - 'BXD34': webqtlCaseData(4), 'BXD35': webqtlCaseData(4), 'BXD32': webqtlCaseData(4), - 'BXD33': webqtlCaseData(3), 'BXD30': webqtlCaseData(1), 'BXD31': webqtlCaseData(4), - 'DBA/2J': webqtlCaseData(1), 'BXD8': webqtlCaseData(3), 'BXD9': webqtlCaseData(1), - 'BXD6': webqtlCaseData(3), 'BXD5': webqtlCaseData(3), 'BXD2': webqtlCaseData(4), - 'BXD1': webqtlCaseData(1), 'C57BL/6J': webqtlCaseData(4), 'B6D2F1': webqtlCaseData(4), - 'BXD42': webqtlCaseData(4), 'BXD40': webqtlCaseData(3)} diff --git a/wqflask/basicStatistics/BasicStatisticsFunctions.py b/wqflask/basicStatistics/BasicStatisticsFunctions.py deleted file mode 100644 index 1e5646a1..00000000 --- a/wqflask/basicStatistics/BasicStatisticsFunctions.py +++ /dev/null @@ -1,207 +0,0 @@ -from __future__ import print_function - -#import string -from math import * -#import piddle as pid -#import os -import traceback - -from pprint import pformat as pf - -from corestats import Stats - -import reaper -from htmlgen import HTMLgen2 as HT - -#from utility import Plot -from utility import webqtlUtil -from base import webqtlConfig -from db import webqtlDatabaseFunction - -def basicStatsTable(vals, trait_type=None, cellid=None, heritability=None): - print("basicStatsTable called - len of vals", len(vals)) - st = {} # This is the dictionary where we'll put everything for the template - valsOnly = [] - dataXZ = vals[:] - for i in range(len(dataXZ)): - valsOnly.append(dataXZ[i][1]) - - (st['traitmean'], - st['traitmedian'], - st['traitvar'], - st['traitstdev'], - st['traitsem'], - st['N']) = reaper.anova(valsOnly) #ZS: Should convert this from reaper to R in the future - - #tbl = HT.TableLite(cellpadding=20, cellspacing=0) - #dataXZ = vals[:] - dataXZ = sorted(vals, webqtlUtil.cmpOrder) - - print("data for stats is:", pf(dataXZ)) - for num, item in enumerate(dataXZ): - print(" %i - %s" % (num, item)) - print(" length:", len(dataXZ)) - - st['min'] = dataXZ[0][1] - st['max'] = dataXZ[-1][1] - - numbers = [x[1] for x in dataXZ] - stats = Stats(numbers) - - at75 = stats.percentile(75) - at25 = stats.percentile(25) - print("should get a stack") - traceback.print_stack() - print("Interquartile:", at75 - at25) - - #tbl.append(HT.TR(HT.TD("Statistic",align="left", Class="fs14 fwb ffl b1 cw cbrb", width = 180), - # HT.TD("Value", align="right", Class="fs14 fwb ffl b1 cw cbrb", width = 60))) - #tbl.append(HT.TR(HT.TD("N of Samples",align="left", Class="fs13 b1 cbw c222"), - # HT.TD(N,nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - #tbl.append(HT.TR(HT.TD("Mean",align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%2.3f" % traitmean,nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - #tbl.append(HT.TR(HT.TD("Median",align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%2.3f" % traitmedian,nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - ##tbl.append(HT.TR(HT.TD("Variance",align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - ## HT.TD("%2.3f" % traitvar,nowrap="yes",align="left", Class="fs13 b1 cbw c222"))) - #tbl.append(HT.TR(HT.TD("Standard Error (SE)",align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%2.3f" % traitsem,nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - #tbl.append(HT.TR(HT.TD("Standard Deviation (SD)", align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%2.3f" % traitstdev,nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - #tbl.append(HT.TR(HT.TD("Minimum", align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%s" % dataXZ[0][1],nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - #tbl.append(HT.TR(HT.TD("Maximum", align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%s" % dataXZ[-1][1],nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - - - - if (trait_type != None and trait_type == 'ProbeSet'): - #tbl.append(HT.TR(HT.TD("Range (log2)",align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%2.3f" % (dataXZ[-1][1]-dataXZ[0][1]),nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - #tbl.append(HT.TR(HT.TD(HT.Span("Range (fold)"),align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%2.2f" % pow(2.0,(dataXZ[-1][1]-dataXZ[0][1])), nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - #tbl.append(HT.TR(HT.TD(HT.Span(HT.Href(url="/glossary.html#Interquartile", target="_blank", text="Interquartile Range", Class="non_bold")), align="left", Class="fs13 b1 cbw c222",nowrap="yes"), - # HT.TD("%2.2f" % pow(2.0,(dataXZ[int((N-1)*3.0/4.0)][1]-dataXZ[int((N-1)/4.0)][1])), nowrap="yes", Class="fs13 b1 cbw c222"), align="right")) - st['range_log2'] = dataXZ[-1][1]-dataXZ[0][1] - st['range_fold'] = pow(2.0, (dataXZ[-1][1]-dataXZ[0][1])) - st['interquartile'] = pow(2.0, (dataXZ[int((st['N']-1)*3.0/4.0)][1]-dataXZ[int((st['N']-1)/4.0)][1])) - - #XZ, 04/01/2009: don't try to get H2 value for probe. - if not cellid: - if heritability: - # This field needs to still be put into the Jinja2 template - st['heritability'] = heritability - #tbl.append(HT.TR(HT.TD(HT.Span("Heritability"),align="center", Class="fs13 b1 cbw c222",nowrap="yes"),HT.TD("%s" % heritability, nowrap="yes",align="center", Class="fs13 b1 cbw c222"))) - - # Lei Yan - # 2008/12/19 - - return st - -def plotNormalProbability(vals=None, RISet='', title=None, showstrains=0, specialStrains=[None], size=(750,500)): - - dataXZ = vals[:] - dataXZ.sort(webqtlUtil.cmpOrder) - dataLabel = [] - dataX = map(lambda X: X[1], dataXZ) - - showLabel = showstrains - if len(dataXZ) > 50: - showLabel = 0 - for item in dataXZ: - strainName = webqtlUtil.genShortStrainName(RISet=RISet, input_strainName=item[0]) - dataLabel.append(strainName) - - dataY=Plot.U(len(dataX)) - dataZ=map(Plot.inverseCumul,dataY) - c = pid.PILCanvas(size=(750,500)) - Plot.plotXY(c, dataZ, dataX, dataLabel = dataLabel, XLabel='Expected Z score', connectdot=0, YLabel='Trait value', title=title, specialCases=specialStrains, showLabel = showLabel) - - filename= webqtlUtil.genRandStr("nP_") - c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') - - img=HT.Image('/image/'+filename+'.gif',border=0) - - return img - -def plotBoxPlot(vals): - - valsOnly = [] - dataXZ = vals[:] - for i in range(len(dataXZ)): - valsOnly.append(dataXZ[i][1]) - - plotHeight = 320 - plotWidth = 220 - xLeftOffset = 60 - xRightOffset = 40 - yTopOffset = 40 - yBottomOffset = 60 - - canvasHeight = plotHeight + yTopOffset + yBottomOffset - canvasWidth = plotWidth + xLeftOffset + xRightOffset - canvas = pid.PILCanvas(size=(canvasWidth,canvasHeight)) - XXX = [('', valsOnly[:])] - - Plot.plotBoxPlot(canvas, XXX, offset=(xLeftOffset, xRightOffset, yTopOffset, yBottomOffset), XLabel= "Trait") - filename= webqtlUtil.genRandStr("Box_") - canvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') - img=HT.Image('/image/'+filename+'.gif',border=0) - - plotLink = HT.Span("More about ", HT.Href(text="Box Plots", url="http://davidmlane.com/hyperstat/A37797.html", target="_blank", Class="fs13")) - - return img, plotLink - -def plotBarGraph(identification='', RISet='', vals=None, type="name"): - - this_identification = "unnamed trait" - if identification: - this_identification = identification - - if type=="rank": - dataXZ = vals[:] - dataXZ.sort(webqtlUtil.cmpOrder) - title='%s' % this_identification - else: - dataXZ = vals[:] - title='%s' % this_identification - - tvals = [] - tnames = [] - tvars = [] - for i in range(len(dataXZ)): - tvals.append(dataXZ[i][1]) - tnames.append(webqtlUtil.genShortStrainName(RISet=RISet, input_strainName=dataXZ[i][0])) - tvars.append(dataXZ[i][2]) - nnStrain = len(tnames) - - sLabel = 1 - - ###determine bar width and space width - if nnStrain < 20: - sw = 4 - elif nnStrain < 40: - sw = 3 - else: - sw = 2 - - ### 700 is the default plot width minus Xoffsets for 40 strains - defaultWidth = 650 - if nnStrain > 40: - defaultWidth += (nnStrain-40)*10 - defaultOffset = 100 - bw = int(0.5+(defaultWidth - (nnStrain-1.0)*sw)/nnStrain) - if bw < 10: - bw = 10 - - plotWidth = (nnStrain-1)*sw + nnStrain*bw + defaultOffset - plotHeight = 500 - #print [plotWidth, plotHeight, bw, sw, nnStrain] - c = pid.PILCanvas(size=(plotWidth,plotHeight)) - Plot.plotBarText(c, tvals, tnames, variance=tvars, YLabel='Value', title=title, sLabel = sLabel, barSpace = sw) - - filename= webqtlUtil.genRandStr("Bar_") - c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') - img=HT.Image('/image/'+filename+'.gif',border=0) - - return img diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index ba998e91..8a9dc79d 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -30,14 +30,6 @@ logger = getLogger(__name__ ) #output: cursor instance #function: connect to database and return cursor instance ########################################################################### -def getCursor(): - try: - logger.warning("Creating new MySQLdb cursor (this method is OBSOLETE!)") - con = MySQLdb.Connect(db=webqtlConfig.DB_NAME, host=webqtlConfig.MYSQL_SERVER, user=webqtlConfig.DB_USER, passwd=webqtlConfig.DB_PASSWD) - cursor = con.cursor() - return cursor - except: - return None def retrieve_species(group): """Get the species of a group (e.g. returns string "mouse" on "BXD" diff --git a/wqflask/maintenance/convert_dryad_to_bimbam.py b/wqflask/maintenance/convert_dryad_to_bimbam.py new file mode 100644 index 00000000..e833b395 --- /dev/null +++ b/wqflask/maintenance/convert_dryad_to_bimbam.py @@ -0,0 +1,70 @@ +#!/usr/bin/python + +""" +Convert data dryad files to a BIMBAM _geno and _snps file + + +""" + +from __future__ import print_function, division, absolute_import +import sys +sys.path.append("..") + + +def read_dryad_file(filename): + exclude_count = 0 + marker_list = [] + sample_dict = {} + sample_list = [] + geno_rows = [] + with open(filename, 'r') as the_file: + for i, line in enumerate(the_file): + if i > 0: + if line.split(" ")[1] == "no": + sample_name = line.split(" ")[0] + sample_list.append(sample_name) + sample_dict[sample_name] = line.split(" ")[2:] + else: + exclude_count += 1 + else: + marker_list = line.split(" ")[2:] + + for i, marker in enumerate(marker_list): + this_row = [] + this_row.append(marker) + this_row.append("X") + this_row.append("Y") + for sample in sample_list: + this_row.append(sample_dict[sample][i]) + geno_rows.append(this_row) + + print(exclude_count) + + return geno_rows + + #for i, marker in enumerate(marker_list): + # this_row = [] + # this_row.append(marker) + # this_row.append("X") + # this_row.append("Y") + # with open(filename, 'r') as the_file: + # for j, line in enumerate(the_file): + # if j > 0: + # this_row.append(line.split(" ")[i+2]) + # print("row: " + str(i)) + # geno_rows.append(this_row) + # + #return geno_rows + +def write_bimbam_files(geno_rows): + with open('/home/zas1024/cfw_data/CFW_geno.txt', 'w') as geno_fh: + for row in geno_rows: + geno_fh.write(", ".join(row) + "\n") + +def convert_dryad_to_bimbam(filename): + geno_file_rows = read_dryad_file(filename) + write_bimbam_files(geno_file_rows) + +if __name__=="__main__": + input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" + convert_dryad_to_bimbam(input_filename)
\ No newline at end of file diff --git a/wqflask/maintenance/convert_geno_to_bimbam.py b/wqflask/maintenance/convert_geno_to_bimbam.py index 05006d5c..45522705 100644 --- a/wqflask/maintenance/convert_geno_to_bimbam.py +++ b/wqflask/maintenance/convert_geno_to_bimbam.py @@ -17,17 +17,12 @@ import glob import traceback import gzip -#import numpy as np -#from pyLMM import lmm - import simplejson as json from pprint import pformat as pf class EmptyConfigurations(Exception): pass - - class Marker(object): def __init__(self): self.name = None @@ -39,47 +34,34 @@ class Marker(object): class ConvertGenoFile(object): def __init__(self, input_file, output_files): - self.input_file = input_file self.output_files = output_files - + self.mb_exists = False self.cm_exists = False self.markers = [] - + self.latest_row_pos = None self.latest_col_pos = None - + self.latest_row_value = None self.latest_col_value = None - - def convert(self): + def convert(self): self.haplotype_notation = { '@mat': "1", '@pat': "0", '@het': "0.5", '@unk': "NA" } - + self.configurations = {} - #self.skipped_cols = 3 - - #if self.input_file.endswith(".geno.gz"): - # print("self.input_file: ", self.input_file) - # self.input_fh = gzip.open(self.input_file) - #else: self.input_fh = open(self.input_file) - - with open(self.output_files[0], "w") as self.geno_fh: - #if self.file_type == "geno": - self.process_csv() - #elif self.file_type == "snps": - # self.process_snps_file() + self.process_csv() def process_csv(self): - for row_count, row in enumerate(self.process_rows()): + for row in self.process_rows(): row_items = row.split("\t") this_marker = Marker() @@ -102,53 +84,30 @@ class ConvertGenoFile(object): this_marker.genotypes.append(self.configurations[genotype.upper().strip()]) else: this_marker.genotypes.append("NA") - - #print("this_marker is:", pf(this_marker.__dict__)) - #if this_marker.chr == "14": + self.markers.append(this_marker.__dict__) self.write_to_bimbam() - - # with open(self.output_file, 'w') as fh: - # json.dump(self.markers, fh, indent=" ", sort_keys=True) - - # print('configurations:', str(configurations)) - #self.latest_col_pos = item_count + self.skipped_cols - #self.latest_col_value = item - - #if item_count != 0: - # self.output_fh.write(" ") - #self.output_fh.write(self.configurations[item.upper()]) - - #self.output_fh.write("\n") def write_to_bimbam(self): with open(self.output_files[0], "w") as geno_fh: - # geno_fh.write(str(len(self.sample_list)) + "\n") - # geno_fh.write("2\n") - # geno_fh.write("IND") - # for sample in self.sample_list: - # geno_fh.write(" " + sample) - # geno_fh.write("\n") for marker in self.markers: geno_fh.write(marker['name']) geno_fh.write(", X, Y") geno_fh.write(", " + ", ".join(marker['genotypes'])) geno_fh.write("\n") - - #pheno_fh = open(self.output_files[1], 'w') + with open(self.output_files[1], "w") as pheno_fh: for sample in self.sample_list: pheno_fh.write("1\n") - + with open(self.output_files[2], "w") as snp_fh: for marker in self.markers: if self.mb_exists: snp_fh.write(marker['name'] +", " + str(int(float(marker['Mb'])*1000000)) + ", " + marker['chr'] + "\n") else: snp_fh.write(marker['name'] +", " + str(int(float(marker['cM'])*1000000)) + ", " + marker['chr'] + "\n") - - + def get_sample_list(self, row_contents): self.sample_list = [] if self.mb_exists: @@ -164,8 +123,6 @@ class ConvertGenoFile(object): def process_rows(self): for self.latest_row_pos, row in enumerate(self.input_fh): - #if self.input_file.endswith(".geno.gz"): - # print("row: ", row) self.latest_row_value = row # Take care of headers if not row.strip(): @@ -208,10 +165,8 @@ class ConvertGenoFile(object): convertob.convert() except EmptyConfigurations as why: print(" No config info? Continuing...") - #excepted = True continue except Exception as why: - print(" Exception:", why) print(traceback.print_exc()) print(" Found in row %s at tabular column %s" % (convertob.latest_row_pos, @@ -219,12 +174,6 @@ class ConvertGenoFile(object): print(" Column is:", convertob.latest_col_value) print(" Row is:", convertob.latest_row_value) break - - #def process_snps_file(cls, snps_file, new_directory): - # output_file = os.path.join(new_directory, "mouse_families.json") - # print("%s -> %s" % (snps_file, output_file)) - # convertob = ConvertGenoFile(input_file, output_file) - if __name__=="__main__": Old_Geno_Directory = """/home/zas1024/genotype_files/genotype/""" @@ -234,6 +183,4 @@ if __name__=="__main__": #convertob = ConvertGenoFile("/home/zas1024/gene/genotype_files/genotypes/SRxSHRSPF2.geno", "/home/zas1024/gene/genotype_files/new_genotypes/SRxSHRSPF2.json") #convertob.convert() ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory) - #ConvertGenoFiles(Geno_Directory) - - #process_csv(Input_File, Output_File)
\ No newline at end of file + #ConvertGenoFiles(Geno_Directory)
\ No newline at end of file diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index 79242661..18b2dac9 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -63,7 +63,7 @@ from pprint import pformat as pf #conn = Engine.connect() -def parse_db_uri(db_uri): +def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" parsed_uri = urlparse.urlparse(SQL_URI) @@ -78,7 +78,6 @@ def parse_db_uri(db_uri): return db_conn_info - def get_species(): """Build species list""" Cursor.execute("select Name, MenuName from Species where Species.Name != 'macaque monkey' order by OrderId") @@ -265,7 +264,7 @@ def build_datasets(species, group, type_name): def main(): """Generates and outputs (as json file) the data for the main dropdown menus on the home page""" - parse_db_uri(SQL_URI) + parse_db_uri() species = get_species() groups = get_groups(species) @@ -304,6 +303,6 @@ def _test_it(): #print("build_datasets:", pf(datasets)) if __name__ == '__main__': - Conn = MySQLdb.Connect(**parse_db_uri(SQL_URI)) + Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() - main() + main()
\ No newline at end of file diff --git a/wqflask/maintenance/geno_to_json.py b/wqflask/maintenance/geno_to_json.py new file mode 100644 index 00000000..789a1691 --- /dev/null +++ b/wqflask/maintenance/geno_to_json.py @@ -0,0 +1,195 @@ +#!/usr/bin/python + +""" +Convert .geno files to json + +This file goes through all of the genofiles in the genofile directory (.geno) +and converts them to json files that are used when running the marker regression +code + +""" + +from __future__ import print_function, division, absolute_import +import sys +sys.path.append("..") +import os +import glob +import traceback +import gzip + +#import numpy as np +#from pyLMM import lmm + +import simplejson as json + +from pprint import pformat as pf + +class EmptyConfigurations(Exception): pass + + + +class Marker(object): + def __init__(self): + self.name = None + self.chr = None + self.cM = None + self.Mb = None + self.genotypes = [] + +class ConvertGenoFile(object): + + def __init__(self, input_file, output_file): + + self.input_file = input_file + self.output_file = output_file + + self.mb_exists = False + self.cm_exists = False + self.markers = [] + + self.latest_row_pos = None + self.latest_col_pos = None + + self.latest_row_value = None + self.latest_col_value = None + + def convert(self): + + self.haplotype_notation = { + '@mat': "1", + '@pat': "0", + '@het': "0.5", + '@unk': "NA" + } + + self.configurations = {} + #self.skipped_cols = 3 + + #if self.input_file.endswith(".geno.gz"): + # print("self.input_file: ", self.input_file) + # self.input_fh = gzip.open(self.input_file) + #else: + self.input_fh = open(self.input_file) + + with open(self.output_file, "w") as self.output_fh: + #if self.file_type == "geno": + self.process_csv() + #elif self.file_type == "snps": + # self.process_snps_file() + + + def process_csv(self): + for row_count, row in enumerate(self.process_rows()): + row_items = row.split("\t") + + this_marker = Marker() + this_marker.name = row_items[1] + this_marker.chr = row_items[0] + if self.cm_exists and self.mb_exists: + this_marker.cM = row_items[2] + this_marker.Mb = row_items[3] + genotypes = row_items[4:] + elif self.cm_exists: + this_marker.cM = row_items[2] + genotypes = row_items[3:] + elif self.mb_exists: + this_marker.Mb = row_items[2] + genotypes = row_items[3:] + else: + genotypes = row_items[2:] + for item_count, genotype in enumerate(genotypes): + if genotype.upper() in self.configurations: + this_marker.genotypes.append(self.configurations[genotype.upper()]) + else: + this_marker.genotypes.append("NA") + + #print("this_marker is:", pf(this_marker.__dict__)) + #if this_marker.chr == "14": + self.markers.append(this_marker.__dict__) + + with open(self.output_file, 'w') as fh: + json.dump(self.markers, fh, indent=" ", sort_keys=True) + + # print('configurations:', str(configurations)) + #self.latest_col_pos = item_count + self.skipped_cols + #self.latest_col_value = item + + #if item_count != 0: + # self.output_fh.write(" ") + #self.output_fh.write(self.configurations[item.upper()]) + + #self.output_fh.write("\n") + + + def process_rows(self): + for self.latest_row_pos, row in enumerate(self.input_fh): + #if self.input_file.endswith(".geno.gz"): + # print("row: ", row) + self.latest_row_value = row + # Take care of headers + if not row.strip(): + continue + if row.startswith('#'): + continue + if row.startswith('Chr'): + if 'Mb' in row.split(): + self.mb_exists = True + if 'cM' in row.split(): + self.cm_exists = True + continue + if row.startswith('@'): + key, _separater, value = row.partition(':') + key = key.strip() + value = value.strip() + if key in self.haplotype_notation: + self.configurations[value] = self.haplotype_notation[key] + continue + if not len(self.configurations): + raise EmptyConfigurations + yield row + + @classmethod + def process_all(cls, old_directory, new_directory): + os.chdir(old_directory) + for input_file in glob.glob("*"): + if not input_file.endswith(('geno', '.geno.gz')): + continue + group_name = ".".join(input_file.split('.')[:-1]) + output_file = os.path.join(new_directory, group_name + ".json") + print("%s -> %s" % ( + os.path.join(old_directory, input_file), output_file)) + convertob = ConvertGenoFile(input_file, output_file) + try: + convertob.convert() + except EmptyConfigurations as why: + print(" No config info? Continuing...") + #excepted = True + continue + except Exception as why: + + print(" Exception:", why) + print(traceback.print_exc()) + print(" Found in row %s at tabular column %s" % (convertob.latest_row_pos, + convertob.latest_col_pos)) + print(" Column is:", convertob.latest_col_value) + print(" Row is:", convertob.latest_row_value) + break + + #def process_snps_file(cls, snps_file, new_directory): + # output_file = os.path.join(new_directory, "mouse_families.json") + # print("%s -> %s" % (snps_file, output_file)) + # convertob = ConvertGenoFile(input_file, output_file) + + + +if __name__=="__main__": + Old_Geno_Directory = """/home/zas1024/genotype_files/genotype/""" + New_Geno_Directory = """/home/zas1024/genotype_files/genotype/json/""" + #Input_File = """/home/zas1024/gene/genotype_files/genotypes/BXD.geno""" + #Output_File = """/home/zas1024/gene/wqflask/wqflask/pylmm/data/bxd.snps""" + #convertob = ConvertGenoFile("/home/zas1024/gene/genotype_files/genotypes/SRxSHRSPF2.geno", "/home/zas1024/gene/genotype_files/new_genotypes/SRxSHRSPF2.json") + #convertob.convert() + ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory) + #ConvertGenoFiles(Geno_Directory) + + #process_csv(Input_File, Output_File)
\ No newline at end of file diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py index 04e94886..1dc6c46c 100644 --- a/wqflask/maintenance/get_group_samplelists.py +++ b/wqflask/maintenance/get_group_samplelists.py @@ -6,16 +6,6 @@ import gzip from base import webqtlConfig -def process_genofiles(geno_dir=webqtlConfig.GENODIR): - print("Yabba") - #sys.exit("Dabba") - os.chdir(geno_dir) - for geno_file in glob.glob("*"): - if geno_file.lower().endswith(('.geno', '.geno.gz')): - #group_name = genofilename.split('.')[0] - sample_list = get_samplelist(geno_file) - - def get_samplelist(file_type, geno_file): if file_type == "geno": return get_samplelist_from_geno(geno_file) diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py new file mode 100644 index 00000000..41a3aad8 --- /dev/null +++ b/wqflask/maintenance/quantile_normalize.py @@ -0,0 +1,129 @@ +from __future__ import absolute_import, print_function, division + +import sys +sys.path.insert(0,'./') + +from itertools import izip + +import MySQLdb +import urlparse + +import numpy as np +import pandas as pd +from elasticsearch import Elasticsearch, TransportError +from elasticsearch.helpers import bulk + +from flask import Flask, g, request + +from wqflask import app +from utility.elasticsearch_tools import get_elasticsearch_connection +from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI + +def parse_db_uri(): + """Converts a database URI to the db name, host name, user name, and password""" + + parsed_uri = urlparse.urlparse(SQL_URI) + + db_conn_info = dict( + db = parsed_uri.path[1:], + host = parsed_uri.hostname, + user = parsed_uri.username, + passwd = parsed_uri.password) + + print(db_conn_info) + return db_conn_info + +def create_dataframe(input_file): + with open(input_file) as f: + ncols = len(f.readline().split("\t")) + + input_array = np.loadtxt(open(input_file, "rb"), delimiter="\t", skiprows=1, usecols=range(1, ncols)) + return pd.DataFrame(input_array) + +#This function taken from https://github.com/ShawnLYU/Quantile_Normalize +def quantileNormalize(df_input): + df = df_input.copy() + #compute rank + dic = {} + for col in df: + dic.update({col : sorted(df[col])}) + sorted_df = pd.DataFrame(dic) + rank = sorted_df.mean(axis = 1).tolist() + #sort + for col in df: + t = np.searchsorted(np.sort(df[col]), df[col]) + df[col] = [rank[i] for i in t] + return df + +def set_data(dataset_name): + orig_file = "/home/zas1024/cfw_data/" + dataset_name + ".txt" + + sample_list = [] + with open(orig_file, 'r') as orig_fh, open('/home/zas1024/cfw_data/quant_norm.csv', 'r') as quant_fh: + for i, (line1, line2) in enumerate(izip(orig_fh, quant_fh)): + trait_dict = {} + sample_list = [] + if i == 0: + sample_names = line1.split('\t')[1:] + else: + trait_name = line1.split('\t')[0] + for i, sample in enumerate(sample_names): + this_sample = { + "name": sample, + "value": line1.split('\t')[i+1], + "qnorm": line2.split('\t')[i+1] + } + sample_list.append(this_sample) + query = """SELECT Species.SpeciesName, InbredSet.InbredSetName, ProbeSetFreeze.FullName + FROM Species, InbredSet, ProbeSetFreeze, ProbeFreeze, ProbeSetXRef, ProbeSet + WHERE Species.Id = InbredSet.SpeciesId and + InbredSet.Id = ProbeFreeze.InbredSetId and + ProbeFreeze.Id = ProbeSetFreeze.ProbeFreezeId and + ProbeSetFreeze.Name = '%s' and + ProbeSetFreeze.Id = ProbeSetXRef.ProbeSetFreezeId and + ProbeSetXRef.ProbeSetId = ProbeSet.Id and + ProbeSet.Name = '%s'""" % (dataset_name, line1.split('\t')[0]) + Cursor.execute(query) + result_info = Cursor.fetchone() + + yield { + "_index": "traits", + "_type": "trait", + "_source": { + "name": trait_name, + "species": result_info[0], + "group": result_info[1], + "dataset": dataset_name, + "dataset_fullname": result_info[2], + "samples": sample_list, + "transform_types": "qnorm" + } + } + +if __name__ == '__main__': + Conn = MySQLdb.Connect(**parse_db_uri()) + Cursor = Conn.cursor() + + #es = Elasticsearch([{ + # "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT + #}], timeout=60) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + + es = get_elasticsearch_connection(for_user=False) + + #input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" + #input_df = create_dataframe(input_filename) + #output_df = quantileNormalize(input_df) + + #output_df.to_csv('quant_norm.csv', sep='\t') + + #out_filename = sys.argv[1][:-4] + '_quantnorm.txt' + + success, _ = bulk(es, set_data(sys.argv[1])) + + response = es.search( + index = "traits", doc_type = "trait", body = { + "query": { "match": { "name": "ENSMUSG00000028982" } } + } + ) + + print(response)
\ No newline at end of file diff --git a/wqflask/utility/AJAX_table.py b/wqflask/utility/AJAX_table.py deleted file mode 100644 index d70acfcd..00000000 --- a/wqflask/utility/AJAX_table.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# This program is available from Source Forge: at GeneNetwork Project -# (sourceforge.net/projects/genenetwork/). -# -# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) -# at rwilliams@uthsc.edu and xzhou15@uthsc.edu -# -# -# -# This module is used by GeneNetwork project (www.genenetwork.org) -# -# Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 - -import cPickle -import os -import MySQLdb -import time -import pyXLWriter as xl - -from htmlgen import HTMLgen2 as HT - -from base import webqtlConfig -from THCell import THCell -from TDCell import TDCell -import webqtlUtil - - -class AJAX_table: - def __init__(self, fd): - file = fd.formdata.getfirst("file", "") - sort = fd.formdata.getfirst("sort", "") - order = fd.formdata.getfirst("order", "up") - cmd = fd.formdata.getfirst("cmd", "") - tableID = fd.formdata.getfirst("tableID", "") - addIndex = fd.formdata.getfirst("addIndex", "1") - hiddenColumnsString = fd.formdata.getfirst("hiddenColumns", "") - hiddenColumns = hiddenColumnsString.split(',') - - try: - fp = open(os.path.join(webqtlConfig.TMPDIR, file + '.obj'), 'rb') - tblobj = cPickle.load(fp) - fp.close() - - if cmd == 'addCorr': - dbId = int(fd.formdata.getfirst("db")) - dbFullName = fd.formdata.getfirst("dbname") - trait = fd.formdata.getfirst("trait") - form = fd.formdata.getfirst("form") - ids = fd.formdata.getfirst("ids") - vals = fd.formdata.getfirst("vals") - ids = eval(ids) - nnCorr = len(ids) - vals = eval(vals) - - workbook = xl.Writer('%s.xls' % (webqtlConfig.TMPDIR+file)) - worksheet = workbook.add_worksheet() - - logger.warning("Creating new MySQLdb cursor (this method is OBSOLETE!)") - - con = MySQLdb.Connect(db=webqtlConfig.DB_NAME,host=webqtlConfig.MYSQL_SERVER, user=webqtlConfig.DB_USER,passwd=webqtlConfig.DB_PASSWD) - cursor = con.cursor() - - cursor.execute("Select name, ShortName from ProbeSetFreeze where Id = %s", dbId) - dbName, dbShortName = cursor.fetchone() - - tblobj['header'][0].append( - THCell(HT.TD(dbShortName, Class="fs11 ffl b1 cw cbrb"), - text="%s" % dbShortName, idx=tblobj['header'][0][-1].idx + 1), - ) - - headingStyle = workbook.add_format(align = 'center', bold = 1, border = 1, size=13, fg_color = 0x1E, color="white") - for i, item in enumerate(tblobj['header'][0]): - if (i > 0): - worksheet.write([8, i-1], item.text, headingStyle) - worksheet.set_column([i-1, i-1], 2*len(item.text)) - - for i, row in enumerate(tblobj['body']): - ProbeSetId = row[1].text - #XZ, 03/02/2009: Xiaodong changed Data to ProbeSetData - cursor.execute(""" - Select ProbeSetData.StrainId, ProbeSetData.Value - From ProbeSetData, ProbeSetXRef, ProbeSet - where ProbeSetXRef.ProbeSetFreezeId = %d AND - ProbeSetXRef.DataId = ProbeSetData.Id AND - ProbeSetXRef.ProbeSetId = ProbeSet.Id AND - ProbeSet.Name = '%s' - """ % (dbId, ProbeSetId)) - results = cursor.fetchall() - vdict = {} - for item in results: - vdict[item[0]] = item[1] - newvals = [] - for id in ids: - if vdict.has_key(id): - newvals.append(vdict[id]) - else: - newvals.append(None) - corr,nOverlap= webqtlUtil.calCorrelation(newvals,vals,nnCorr) - repr = '%0.4f' % corr - row.append( - TDCell(HT.TD(HT.Href(text=repr, url="javascript:showCorrPlotThird('%s', '%s', '%s')" % (form, dbName, ProbeSetId), Class="fs11 fwn ffl"), " / ", nOverlap, Class="fs11 fwn ffl b1 c222", align="middle"),repr,abs(corr)) - ) - - last_row=0 - for j, item in enumerate(tblobj['body'][i]): - if (j > 0): - worksheet.write([9+i, j-1], item.text) - last_row = 9+i - last_row += 1 - - titleStyle = workbook.add_format(align = 'left', bold = 0, size=14, border = 1, border_color="gray") - ##Write title Info - # Modified by Hongqiang Li - worksheet.write([0, 0], "Citations: Please see %s/reference.html" % webqtlConfig.PORTADDR, titleStyle) - worksheet.write([1, 0], "Trait : %s" % trait, titleStyle) - worksheet.write([2, 0], "Database : %s" % dbFullName, titleStyle) - worksheet.write([3, 0], "Date : %s" % time.strftime("%B %d, %Y", time.gmtime()), titleStyle) - worksheet.write([4, 0], "Time : %s GMT" % time.strftime("%H:%M ", time.gmtime()), titleStyle) - worksheet.write([5, 0], "Status of data ownership: Possibly unpublished data; please see %s/statusandContact.html for details on sources, ownership, and usage of these data." % webqtlConfig.PORTADDR, titleStyle) - #Write footer info - worksheet.write([1 + last_row, 0], "Funding for The GeneNetwork: NIAAA (U01AA13499, U24AA13513), NIDA, NIMH, and NIAAA (P20-DA21131), NCI MMHCC (U01CA105417), and NCRR (U01NR 105417)", titleStyle) - worksheet.write([2 + last_row, 0], "PLEASE RETAIN DATA SOURCE INFORMATION WHENEVER POSSIBLE", titleStyle) - - cursor.close() - workbook.close() - - objfile = open(os.path.join(webqtlConfig.TMPDIR, file + '.obj'), 'wb') - cPickle.dump(tblobj, objfile) - objfile.close() - else: - pass - - self.value = str(webqtlUtil.genTableObj(tblobj=tblobj, file=file, sortby=(sort, order), tableID = tableID, addIndex = addIndex, hiddenColumns = hiddenColumns)) - - except: - self.value = "<span class='fs16 fwb cr ffl'>The table is no longer available on this server</span>" - - def __str__(self): - return self.value - - def write(self): - return str(self) diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index d60e2bb2..529cd117 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -36,11 +36,9 @@ from numarray import linear_algebra as la from numarray import ones, array, dot, swapaxes import reaper -# sys.path.append("..") Never in a running webserver -from basicStatistics import corestats -import svg import webqtlUtil +import corestats from base import webqtlConfig import utility.logger @@ -83,202 +81,6 @@ def frange(start, end=None, inc=1.0): L[i] = start + i * inc return L - -def gammln(xx): - cof=[76.18009173,-86.50532033,24.01409822,-1.231739516,0.120858003e-2,-0.536382e-5] - x=xx-1.0 - tmp=x+5.5 - tmp -=(x+0.5)*log(tmp) - ser=1.0 - for item in cof: - x+=1.0 - ser+=item/x - - return -tmp+log(2.50662827465*ser) - - -def gser(a,x): - gln=gammln(a) - ITMAX=100 - EPS=3.0e-7 - - if x<=0.0: - gamser=0.0 - return [gamser,gln] - else: - ap=a - sum=1.0/a - dele=sum - for i in range(1,ITMAX+1): - ap+=1.0 - dele*=x/ap - sum+=dele - if abs(dele)<abs(sum)*EPS: - gamser=sum*exp(-x+a*log(x)-gln) - return [gamser,gln] - return None - -def gcf(a,x): - ITMAX=100 - EPS=3.0e-7 - gold=0.0 - fac=1 - b1=1.0 - b0=0.0 - a0=1.0 - gln=gammln(a) - - a1=x - for n in range(1,ITMAX+1): - an=n+0.0 - ana=an-a - a0=(a1+a0*ana)*fac - b0=(b1+b0*ana)*fac - anf=an*fac - a1=x*a0+anf*a1 - b1=x*b0+anf*b1 - if (a1): - fac=1.0/a1 - g=b1*fac - if abs((g-gold)/g)<EPS: - gammcf=exp(-x+a*log(x)-gln)*g - return [gammcf,gln] - gold=g - return None - -def gammp(a,x): - if x<0.0 or a<=0.0: - return None - if x<(a+1.0): - a=gser(a,x)[0] - return a - else: - a=gcf(a,x)[0] - return 1.0-a -def U(n): - x=pow(0.5,1.0/n) - m=[1-x] - for i in range(2,n): - a=(i-0.3175)/(n+0.365) - m.append(a) - m.append(x) - return m - -def erf(x): - if x<0.0: - return -gammp(0.5,x*x) - else: - return gammp(0.5,x*x) - -def erfcc(x): - z=abs(x) - t=1.0/(1.0+0.5*z) - ans=t*exp(-z*z-1.26551223+t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277))))))))) - if x>=0.0: - return ans - else: - return 2.0-ans - -def calMeanVar(data): - n=len(data) - if n<2: - return None - else: - sum=reduce(lambda x,y:x+y,data,0.0) - mean=sum/n - z=data[:] - for i in range(n): - z[i]=z[i]-mean - variance=reduce(lambda x,y:x+y*y,z,0.0) - variance /= n-1 - variance =sqrt(variance) - for i in range(n): - z[i]=z[i]/variance - return z - -def inverseCumul(p): - #Coefficients in rational approximations. - a = [-3.969683028665376e+01,2.209460984245205e+02,-2.759285104469687e+02,1.383577518672690e+02,-3.066479806614716e+01,2.506628277459239e+00] - - b = [-5.447609879822406e+01,1.615858368580409e+02,-1.556989798598866e+02,6.680131188771972e+01,-1.328068155288572e+01] - - c = [-7.784894002430293e-03,-3.223964580411365e-01,-2.400758277161838e+00,-2.549732539343734e+00,4.374664141464968e+00,2.938163982698783e+00] - - d = [7.784695709041462e-03,3.224671290700398e-01,2.445134137142996e+00,3.754408661907416e+00] - - #Define break-points. - - p_low = 0.02425 - p_high = 1 - p_low - - #Rational approximation for lower region. - - if p > 0 and p < p_low: - q = sqrt(-2*log(p)) - x = (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) / ((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) - - - #Rational approximation for central region. - - elif p>= p_low and p <= p_high: - q = p - 0.5 - r = q*q - x = (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5])*q /(((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1) - - #Rational approximation for upper region. - - elif p>p_high and p < 1: - q = sqrt(-2*log(1-p)) - x = -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) - - else: - return None - - if p>0 and p < 1: - e = 0.5 * erfcc(-x/sqrt(2)) - p - u = e * sqrt(2*pi) * exp(x*x/2) - x = x - u/(1 + x*u/2) - return x - else: - return None - -def gmean(lst): - N = len(lst) - if N == 0: - return 0 - else: - return (reduce(lambda x,y: x+y, lst, 0.0))/N - -def gmedian(lst2): - lst = lst2[:] - N = len(lst) - if N == 0: - return 0 - else: - lst.sort() - if N % 2 == 0: - return (lst[N/2]+lst[(N-2)/2])/2.0 - else: - return lst[(N-1)/2] - -def gpercentile(lst2, np): - """Obsolete - use percentile in corestats instead""" - lst = lst2[:] - N = len(lst) - if N == 0 or np > 100 or np < 0: - return None - else: - lst.sort() - pNadd1 = (np/100.0)*N - k = int(pNadd1) - d = pNadd1 - k - if k == 0: - return lst[0] - elif k >= N-1: - return lst[N-1] - else: - return lst[k-1] + d*(lst[k] - lst[k-1]) - def find_outliers(vals): """Calculates the upper and lower bounds of a set of sample/case values @@ -315,165 +117,6 @@ def find_outliers(vals): logger.debug(pf(locals())) return upper_bound, lower_bound - -def plotBoxPlot(canvas, data, offset= (40, 40, 40, 40), XLabel="Category", YLabel="Value"): - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - iValues = [] - for item in data: - for item2 in item[1]: - try: - iValues.append(item2[1]) - except: - iValues.append(item2) - - #draw frame - max_Y = max(iValues) - min_Y = min(iValues) - scaleY = detScale(min_Y, max_Y) - Yll = scaleY[0] - Yur = scaleY[1] - nStep = scaleY[2] - stepY = (Yur - Yll)/nStep - stepYPixel = plotHeight/(nStep) - canvas.drawRect(plotWidth+xLeftOffset, plotHeight + yTopOffset, xLeftOffset, yTopOffset) - - ##draw Y Scale - YYY = Yll - YCoord = plotHeight + yTopOffset - scaleFont=pid.Font(ttf="cour",size=11,bold=1) - for i in range(nStep+1): - strY = cformat(d=YYY, rank=0) - YCoord = max(YCoord, yTopOffset) - canvas.drawLine(xLeftOffset,YCoord,xLeftOffset-5,YCoord) - canvas.drawString(strY, xLeftOffset -30,YCoord +5,font=scaleFont) - YYY += stepY - YCoord -= stepYPixel - - ##draw X Scale - stepX = plotWidth/len(data) - XCoord = xLeftOffset + 0.5*stepX - YCoord = plotHeight + yTopOffset - scaleFont = pid.Font(ttf="tahoma",size=12,bold=0) - labelFont = pid.Font(ttf="tahoma",size=13,bold=0) - for item in data: - itemname, itemvalue = item - canvas.drawLine(XCoord, YCoord,XCoord, YCoord+5, color=pid.black) - canvas.drawString(itemname, XCoord - canvas.stringWidth(itemname,font=labelFont)/2.0,\ - YCoord +20,font=labelFont) - - nValue = len(itemvalue) - catValue = [] - for item2 in itemvalue: - try: - tstrain, tvalue = item2 - except: - tvalue = item2 - if nValue <= 4: - canvas.drawCross(XCoord, plotHeight + yTopOffset - (tvalue-Yll)*plotHeight/(Yur - Yll), color=pid.red,size=5) - else: - catValue.append(tvalue) - if catValue != []: - catMean = gmean(catValue) - catMedian = gmedian(catValue) - lowHinge = gpercentile(catValue, 25) - upHinge = gpercentile(catValue, 75) - Hstep = 1.5*(upHinge - lowHinge) - - outlier = [] - extrem = [] - - upperAdj = None - for item in catValue: - if item >= upHinge + 2*Hstep: - extrem.append(item) - elif item >= upHinge + Hstep: - outlier.append(item) - elif item > upHinge and item < upHinge + Hstep: - if upperAdj == None or item > upperAdj: - upperAdj = item - else: - pass - lowerAdj = None - for item in catValue: - if item <= lowHinge - 2*Hstep: - extrem.append(item) - elif item <= lowHinge - Hstep: - outlier.append(item) - if item < lowHinge and item > lowHinge - Hstep: - if lowerAdj == None or item < lowerAdj: - lowerAdj = item - else: - pass - canvas.drawRect(XCoord-20, plotHeight + yTopOffset - (lowHinge-Yll)*plotHeight/(Yur - Yll), \ - XCoord+20, plotHeight + yTopOffset - (upHinge-Yll)*plotHeight/(Yur - Yll)) - canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (catMedian-Yll)*plotHeight/(Yur - Yll), \ - XCoord+20, plotHeight + yTopOffset - (catMedian-Yll)*plotHeight/(Yur - Yll)) - if upperAdj != None: - canvas.drawLine(XCoord, plotHeight + yTopOffset - (upHinge-Yll)*plotHeight/(Yur - Yll), \ - XCoord, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll)) - canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll), \ - XCoord+20, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll)) - if lowerAdj != None: - canvas.drawLine(XCoord, plotHeight + yTopOffset - (lowHinge-Yll)*plotHeight/(Yur - Yll), \ - XCoord, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll)) - canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll), \ - XCoord+20, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll)) - - outlierFont = pid.Font(ttf="cour",size=12,bold=0) - if outlier != []: - for item in outlier: - yc = plotHeight + yTopOffset - (item-Yll)*plotHeight/(Yur - Yll) - #canvas.drawEllipse(XCoord-3, yc-3, XCoord+3, yc+3) - canvas.drawString('o', XCoord-3, yc+5, font=outlierFont, color=pid.orange) - if extrem != []: - for item in extrem: - yc = plotHeight + yTopOffset - (item-Yll)*plotHeight/(Yur - Yll) - #canvas.drawEllipse(XCoord-3, yc-3, XCoord+3, yc+3) - canvas.drawString('*', XCoord-3, yc+6, font=outlierFont, color=pid.red) - - canvas.drawCross(XCoord, plotHeight + yTopOffset - (catMean-Yll)*plotHeight/(Yur - Yll), \ - color=pid.blue,size=3) - #print(catMean, catMedian, cat25per, cat75per) - pass - - XCoord += stepX - - labelFont=pid.Font(ttf="verdana",size=18,bold=0) - canvas.drawString(XLabel, xLeftOffset + (plotWidth -canvas.stringWidth(XLabel,font=labelFont))/2.0, \ - YCoord +40, font=labelFont) - canvas.drawString(YLabel,xLeftOffset-40, YCoord-(plotHeight -canvas.stringWidth(YLabel,font=labelFont))/2.0,\ - font=labelFont, angle =90) - -def plotSecurity(canvas, text="12345"): - if not text: - return - - plotWidth = canvas.size[0] - plotHeight = canvas.size[1] - if plotHeight<=0 or plotWidth<=0: - return - - bgColor = pid.Color(0.6+0.4*random.random(), 0.6+0.4*random.random(), 0.6+0.4*random.random()) - canvas.drawRect(0,0,plotWidth,plotHeight, edgeColor=bgColor, fillColor=bgColor) - - for i in range(30): - randomColor = pid.Color(0.6+0.4*random.random(), 0.6+0.4*random.random(), 0.6+0.4*random.random()) - scaleFont=pid.Font(ttf="cour",size=random.choice(range(20, 50))) - canvas.drawString(random.choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'), - int(random.random()*plotWidth), int(random.random()*plotHeight), font=scaleFont, - color=randomColor, angle=random.choice(range(-45, 50))) - - step = (plotWidth-20)/len(text) - startX = 20 - for item in text: - randomColor = pid.Color(0.6*random.random(),0.6*random.random(), 0.6*random.random()) - scaleFont=pid.Font(ttf="verdana",size=random.choice(range(50, 60)),bold=1) - canvas.drawString(item, startX, plotHeight/2-10, font=scaleFont, - color=randomColor, angle=random.choice(range(-45, 50))) - startX += step - # parameter: data is either object returned by reaper permutation function (called by MarkerRegressionPage.py) # or the first object returned by direct (pair-scan) permu function (called by DirectPlotPage.py) def plotBar(canvas, data, barColor=pid.blue, axesColor=pid.black, labelColor=pid.black, XLabel=None, YLabel=None, title=None, offset= (60, 20, 40, 40), zoom = 1): @@ -561,542 +204,6 @@ def plotBar(canvas, data, barColor=pid.blue, axesColor=pid.black, labelColor=pid canvas.drawString(title,xLeftOffset+(plotWidth-canvas.stringWidth(title,font=labelFont))/2.0, 20,font=labelFont,color=labelColor) -def plotBarText(canvas, data, label, variance=None, barColor=pid.blue, axesColor=pid.black, labelColor=pid.black, XLabel=None, YLabel=None, title=None, sLabel = None, offset= (80, 20, 40, 100), barSpace = 2, zoom = 1): - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - if plotHeight<=0 or plotWidth<=0: - return - - NNN = len(data) - if NNN < 2 or NNN != len(label): - return - if variance and len(variance)!=NNN: - variance = [] - - Y2 = data[:] - if variance: - for i in range(NNN): - if variance[i]: - Y2 += [data[i]-variance[i]] - - #Y axis - YLow, YTop, stepY = detScale(min(Y2), max(Y2)) - YScale = plotHeight/(YTop - YLow) - - if YLow < 0 and YTop > 0: - drawZero = 1 - else: - drawZero = 0 - - #X axis - X = range(NNN) - Xll= 0 - Xur= NNN-1 - - - if drawZero: - YZero = yTopOffset+plotHeight-YScale*(0-YLow) - canvas.drawLine(xLeftOffset, YZero, xLeftOffset+plotWidth, YZero) - else: - YZero = yTopOffset+plotHeight - #draw data - spaceWidth = barSpace - if spaceWidth < 1: - spaceWidth = 1 - barWidth = int((plotWidth - (NNN-1.0)*spaceWidth)/NNN) - - xc= xLeftOffset - scaleFont=pid.Font(ttf="verdana",size=11,bold=0) - for i in range(NNN): - yc = yTopOffset+plotHeight-(data[i]-YLow)*YScale - canvas.drawRect(xc,YZero,xc+barWidth-1, yc, edgeColor=barColor,fillColor=barColor) - if variance and variance[i]: - varlen = variance[i]*YScale - if yc-varlen < yTopOffset: - topYd = yTopOffset - else: - topYd = yc-varlen - canvas.drawLine(xc+barWidth/2-2,yc-varlen,xc+barWidth/2+2,yc-varlen,color=pid.red) - canvas.drawLine(xc+barWidth/2,yc+varlen,xc+barWidth/2,topYd,color=pid.red) - canvas.drawLine(xc+barWidth/2-2,yc+varlen,xc+barWidth/2+2,yc+varlen,color=pid.red) - strX = label[i] - canvas.drawString(strX,xc+barWidth/2.0+2,yTopOffset+plotHeight+2+canvas.stringWidth(strX,font=scaleFont),font=scaleFont,angle=90) - xc += barWidth + spaceWidth - - #draw drawing region - canvas.drawRect(xLeftOffset, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight) - - #draw Y scale - scaleFont=pid.Font(ttf="cour",size=16,bold=1) - y=YLow - for i in range(stepY+1): - yc=yTopOffset+plotHeight-(y-YLow)*YScale - canvas.drawLine(xLeftOffset,yc,xLeftOffset-5,yc, color=axesColor) - strY = cformat(d=y, rank=0) - canvas.drawString(strY,xLeftOffset-canvas.stringWidth(strY,font=scaleFont)-6,yc+5,font=scaleFont) - y+= (YTop - YLow)/stepY - - #draw label - labelFont=pid.Font(ttf="verdana",size=17,bold=0) - if XLabel: - canvas.drawString(XLabel,xLeftOffset+(plotWidth-canvas.stringWidth(XLabel,font=labelFont))/2.0,yTopOffset+plotHeight+65,font=labelFont,color=labelColor) - - if YLabel: - canvas.drawString(YLabel,xLeftOffset-50, yTopOffset+plotHeight-(plotHeight-canvas.stringWidth(YLabel,font=labelFont))/2.0,font=labelFont,color=labelColor,angle=90) - - labelFont=pid.Font(ttf="verdana",size=18,bold=0) - if title: - canvas.drawString(title,xLeftOffset,yTopOffset-15,font=labelFont,color=labelColor) - - return - -#def plotXY(canvas, dataX, dataY, rank=0, dataLabel=[], plotColor = pid.black, axesColor=pid.black, labelColor=pid.black, lineSize="thin", lineColor=pid.grey, idFont="arial", idColor=pid.blue, idSize="14", symbolColor=pid.black, symbolType="circle", filled="yes", symbolSize="tiny", XLabel=None, YLabel=None, title=None, fitcurve=None, connectdot=1, displayR=None, loadingPlot = 0, offset= (80, 20, 40, 60), zoom = 1, specialCases=[], showLabel = 1, bufferSpace = 15): -# 'displayR : correlation scatter plot, loadings : loading plot' -# -# dataXRanked, dataYRanked = webqtlUtil.calRank(dataX, dataY, len(dataX)) -# -# #get ID font size -# idFontSize = int(idSize) -# -# #If filled is yes, set fill color -# if filled == "yes": -# fillColor = symbolColor -# else: -# fillColor = None -# -# if symbolSize == "large": -# sizeModifier = 7 -# fontModifier = 12 -# elif symbolSize == "medium": -# sizeModifier = 5 -# fontModifier = 8 -# elif symbolSize == "small": -# sizeModifier = 3 -# fontModifier = 3 -# else: -# sizeModifier = 1 -# fontModifier = -1 -# -# if rank == 0: # Pearson correlation -# bufferSpace = 0 -# dataXPrimary = dataX -# dataYPrimary = dataY -# dataXAlt = dataXRanked #Values used just for printing the other corr type to the graph image -# dataYAlt = dataYRanked #Values used just for printing the other corr type to the graph image -# else: # Spearman correlation: Switching Ranked and Unranked X and Y values -# dataXPrimary = dataXRanked -# dataYPrimary = dataYRanked -# dataXAlt = dataX #Values used just for printing the other corr type to the graph image -# dataYAlt = dataY #Values used just for printing the other corr type to the graph image -# -# xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset -# plotWidth = canvas.size[0] - xLeftOffset - xRightOffset -# plotHeight = canvas.size[1] - yTopOffset - yBottomOffset -# if plotHeight<=0 or plotWidth<=0: -# return -# if len(dataXPrimary) < 1 or len(dataXPrimary) != len(dataYPrimary) or (dataLabel and len(dataXPrimary) != len(dataLabel)): -# return -# -# max_X=max(dataXPrimary) -# min_X=min(dataXPrimary) -# max_Y=max(dataYPrimary) -# min_Y=min(dataYPrimary) -# -# #for some reason I forgot why I need to do this -# if loadingPlot: -# min_X = min(-0.1,min_X) -# max_X = max(0.1,max_X) -# min_Y = min(-0.1,min_Y) -# max_Y = max(0.1,max_Y) -# -# xLow, xTop, stepX=detScale(min_X,max_X) -# yLow, yTop, stepY=detScale(min_Y,max_Y) -# xScale = plotWidth/(xTop-xLow) -# yScale = plotHeight/(yTop-yLow) -# -# #draw drawing region -# canvas.drawRect(xLeftOffset-bufferSpace, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight+bufferSpace) -# canvas.drawRect(xLeftOffset-bufferSpace+1, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight+bufferSpace-1) -# -# #calculate data points -# data = map(lambda X, Y: (X, Y), dataXPrimary, dataYPrimary) -# xCoord = map(lambda X, Y: ((X-xLow)*xScale + xLeftOffset, yTopOffset+plotHeight-(Y-yLow)*yScale), dataXPrimary, dataYPrimary) -# -# labelFont=pid.Font(ttf=idFont,size=idFontSize,bold=0) -# -# if loadingPlot: -# xZero = -xLow*xScale+xLeftOffset -# yZero = yTopOffset+plotHeight+yLow*yScale -# for point in xCoord: -# canvas.drawLine(xZero,yZero,point[0],point[1],color=pid.red) -# else: -# if connectdot: -# canvas.drawPolygon(xCoord,edgeColor=plotColor,closed=0) -# else: -# pass -# -# symbolFont = pid.Font(ttf="fnt_bs", size=12+fontModifier,bold=0) -# -# for i, item in enumerate(xCoord): -# if dataLabel and dataLabel[i] in specialCases: -# canvas.drawRect(item[0]-3, item[1]-3, item[0]+3, item[1]+3, edgeColor=pid.green) -# #canvas.drawCross(item[0],item[1],color=pid.blue,size=5) -# else: -# if symbolType == "vertRect": -# canvas.drawRect(x1=item[0]-sizeModifier+2,y1=item[1]-sizeModifier-2, x2=item[0]+sizeModifier-1,y2=item[1]+sizeModifier+2, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor) -# elif (symbolType == "circle" and filled != "yes"): -# canvas.drawString(":", item[0]-canvas.stringWidth(":",font=symbolFont)/2+1,item[1]+2,color=symbolColor, font=symbolFont) -# elif (symbolType == "circle" and filled == "yes"): -# canvas.drawString("5", item[0]-canvas.stringWidth("5",font=symbolFont)/2+1,item[1]+2,color=symbolColor, font=symbolFont) -# elif symbolType == "horiRect": -# canvas.drawRect(x1=item[0]-sizeModifier-1,y1=item[1]-sizeModifier+3, x2=item[0]+sizeModifier+3,y2=item[1]+sizeModifier-2, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor) -# elif (symbolType == "square"): -# canvas.drawRect(x1=item[0]-sizeModifier+1,y1=item[1]-sizeModifier-4, x2=item[0]+sizeModifier+2,y2=item[1]+sizeModifier-3, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor) -# elif (symbolType == "diamond" and filled != "yes"): -# canvas.drawString(",", item[0]-canvas.stringWidth(",",font=symbolFont)/2+2, item[1]+6, font=symbolFont, color=symbolColor) -# elif (symbolType == "diamond" and filled == "yes"): -# canvas.drawString("D", item[0]-canvas.stringWidth("D",font=symbolFont)/2+2, item[1]+6, font=symbolFont, color=symbolColor) -# elif symbolType == "4-star": -# canvas.drawString("l", item[0]-canvas.stringWidth("l",font=symbolFont)/2+1, item[1]+3, font=symbolFont, color=symbolColor) -# elif symbolType == "3-star": -# canvas.drawString("k", item[0]-canvas.stringWidth("k",font=symbolFont)/2+1, item[1]+3, font=symbolFont, color=symbolColor) -# else: -# canvas.drawCross(item[0],item[1]-2,color=symbolColor, size=sizeModifier+2) -# -# if showLabel and dataLabel: -# if (symbolType == "vertRect" or symbolType == "diamond"): -# labelGap = 15 -# elif (symbolType == "4-star" or symbolType == "3-star"): -# labelGap = 12 -# else: -# labelGap = 11 -# canvas.drawString(dataLabel[i], item[0]- canvas.stringWidth(dataLabel[i], -# font=labelFont)/2 + 1, item[1]+(labelGap+sizeModifier+(idFontSize-12)), font=labelFont, color=idColor) -# -# #draw scale -# scaleFont=pid.Font(ttf="cour",size=16,bold=1) -# -# -# x=xLow -# for i in range(stepX+1): -# xc=xLeftOffset+(x-xLow)*xScale -# if ((x == 0) & (rank == 1)): -# pass -# else: -# canvas.drawLine(xc,yTopOffset+plotHeight + bufferSpace,xc,yTopOffset+plotHeight+5 + bufferSpace, color=axesColor) -# strX = cformat(d=x, rank=rank) -# if ((strX == "0") & (rank == 1)): -# pass -# else: -# canvas.drawString(strX,xc-canvas.stringWidth(strX,font=scaleFont)/2,yTopOffset+plotHeight+20 + bufferSpace,font=scaleFont) -# x+= (xTop - xLow)/stepX -# -# y=yLow -# for i in range(stepY+1): -# yc=yTopOffset+plotHeight-(y-yLow)*yScale -# if ((y == 0) & (rank == 1)): -# pass -# else: -# canvas.drawLine(xLeftOffset - bufferSpace,yc,xLeftOffset-5 - bufferSpace,yc, color=axesColor) -# strY = cformat(d=y, rank=rank) -# if ((strY == "0") & (rank == 1)): -# pass -# else: -# canvas.drawString(strY,xLeftOffset-canvas.stringWidth(strY,font=scaleFont)- 10 - bufferSpace,yc+4,font=scaleFont) -# y+= (yTop - yLow)/stepY -# -# #draw label -# -# labelFont=pid.Font(ttf="verdana",size=canvas.size[0]/45,bold=0) -# titleFont=pid.Font(ttf="verdana",size=canvas.size[0]/40,bold=0) -# -# if (rank == 1 and not title): -# canvas.drawString("Spearman Rank Correlation", xLeftOffset-canvas.size[0]*.025+(plotWidth-canvas.stringWidth("Spearman Rank Correlation",font=titleFont))/2.0, -# 25,font=titleFont,color=labelColor) -# elif (rank == 0 and not title): -# canvas.drawString("Pearson Correlation", xLeftOffset-canvas.size[0]*.025+(plotWidth-canvas.stringWidth("Pearson Correlation",font=titleFont))/2.0, -# 25,font=titleFont,color=labelColor) -# -# if XLabel: -# canvas.drawString(XLabel,xLeftOffset+(plotWidth-canvas.stringWidth(XLabel,font=labelFont))/2.0, -# yTopOffset+plotHeight+yBottomOffset-25,font=labelFont,color=labelColor) -# -# if YLabel: -# canvas.drawString(YLabel, xLeftOffset-65, yTopOffset+plotHeight- (plotHeight-canvas.stringWidth(YLabel,font=labelFont))/2.0, -# font=labelFont,color=labelColor,angle=90) -# -# labelFont=pid.Font(ttf="verdana",size=20,bold=0) -# if title: -# canvas.drawString(title,xLeftOffset+(plotWidth-canvas.stringWidth(title,font=labelFont))/2.0, -# 20,font=labelFont,color=labelColor) -# -# if fitcurve: -# import sys -# sys.argv = [ "mod_python" ] -# #from numarray import linear_algebra as la -# #from numarray import ones, array, dot, swapaxes -# fitYY = array(dataYPrimary) -# fitXX = array([ones(len(dataXPrimary)),dataXPrimary]) -# AA = dot(fitXX,swapaxes(fitXX,0,1)) -# BB = dot(fitXX,fitYY) -# bb = la.linear_least_squares(AA,BB)[0] -# -# xc1 = xLeftOffset -# yc1 = yTopOffset+plotHeight-(bb[0]+bb[1]*xLow-yLow)*yScale -# if yc1 > yTopOffset+plotHeight: -# yc1 = yTopOffset+plotHeight -# xc1 = (yLow-bb[0])/bb[1] -# xc1=(xc1-xLow)*xScale+xLeftOffset -# elif yc1 < yTopOffset: -# yc1 = yTopOffset -# xc1 = (yTop-bb[0])/bb[1] -# xc1=(xc1-xLow)*xScale+xLeftOffset -# else: -# pass -# -# xc2 = xLeftOffset + plotWidth -# yc2 = yTopOffset+plotHeight-(bb[0]+bb[1]*xTop-yLow)*yScale -# if yc2 > yTopOffset+plotHeight: -# yc2 = yTopOffset+plotHeight -# xc2 = (yLow-bb[0])/bb[1] -# xc2=(xc2-xLow)*xScale+xLeftOffset -# elif yc2 < yTopOffset: -# yc2 = yTopOffset -# xc2 = (yTop-bb[0])/bb[1] -# xc2=(xc2-xLow)*xScale+xLeftOffset -# else: -# pass -# -# canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace,xc2,yc2,color=lineColor) -# if lineSize == "medium": -# canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace+1,xc2,yc2+1,color=lineColor) -# if lineSize == "thick": -# canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace+1,xc2,yc2+1,color=lineColor) -# canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace-1,xc2,yc2-1,color=lineColor) -# -# -# if displayR: -# labelFont=pid.Font(ttf="trebuc",size=canvas.size[0]/60,bold=0) -# NNN = len(dataX) -# corr = webqtlUtil.calCorrelation(dataXPrimary,dataYPrimary,NNN)[0] -# -# if NNN < 3: -# corrPValue = 1.0 -# else: -# if abs(corr) >= 1.0: -# corrPValue = 0.0 -# else: -# ZValue = 0.5*log((1.0+corr)/(1.0-corr)) -# ZValue = ZValue*sqrt(NNN-3) -# corrPValue = 2.0*(1.0 - reaper.normp(abs(ZValue))) -# -# NStr = "N = %d" % NNN -# strLenN = canvas.stringWidth(NStr,font=labelFont) -# -# if rank == 1: -# if corrPValue < 0.0000000000000001: -# corrStr = "Rho = %1.3f P < 1.00 E-16" % (corr) -# else: -# corrStr = "Rho = %1.3f P = %3.2E" % (corr, corrPValue) -# else: -# if corrPValue < 0.0000000000000001: -# corrStr = "r = %1.3f P < 1.00 E-16" % (corr) -# else: -# corrStr = "r = %1.3f P = %3.2E" % (corr, corrPValue) -# strLen = canvas.stringWidth(corrStr,font=labelFont) -# -# canvas.drawString(NStr,xLeftOffset,yTopOffset-10,font=labelFont,color=labelColor) -# canvas.drawString(corrStr,xLeftOffset+plotWidth-strLen,yTopOffset-10,font=labelFont,color=labelColor) -# -# return xCoord - -def plotXYSVG(drawSpace, dataX, dataY, rank=0, dataLabel=[], plotColor = "black", axesColor="black", labelColor="black", symbolColor="red", XLabel=None, YLabel=None, title=None, fitcurve=None, connectdot=1, displayR=None, loadingPlot = 0, offset= (80, 20, 40, 60), zoom = 1, specialCases=[], showLabel = 1): - 'displayR : correlation scatter plot, loadings : loading plot' - - dataXRanked, dataYRanked = webqtlUtil.calRank(dataX, dataY, len(dataX)) - - # Switching Ranked and Unranked X and Y values if a Spearman Rank Correlation - if rank == 0: - dataXPrimary = dataX - dataYPrimary = dataY - dataXAlt = dataXRanked - dataYAlt = dataYRanked - - else: - dataXPrimary = dataXRanked - dataYPrimary = dataYRanked - dataXAlt = dataX - dataYAlt = dataY - - - - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = drawSpace.attributes['width'] - xLeftOffset - xRightOffset - plotHeight = drawSpace.attributes['height'] - yTopOffset - yBottomOffset - if plotHeight<=0 or plotWidth<=0: - return - if len(dataXPrimary) < 1 or len(dataXPrimary) != len(dataYPrimary) or (dataLabel and len(dataXPrimary) != len(dataLabel)): - return - - max_X=max(dataXPrimary) - min_X=min(dataXPrimary) - max_Y=max(dataYPrimary) - min_Y=min(dataYPrimary) - - #for some reason I forgot why I need to do this - if loadingPlot: - min_X = min(-0.1,min_X) - max_X = max(0.1,max_X) - min_Y = min(-0.1,min_Y) - max_Y = max(0.1,max_Y) - - xLow, xTop, stepX=detScale(min_X,max_X) - yLow, yTop, stepY=detScale(min_Y,max_Y) - xScale = plotWidth/(xTop-xLow) - yScale = plotHeight/(yTop-yLow) - - #draw drawing region - r = svg.rect(xLeftOffset, yTopOffset, plotWidth, plotHeight, 'none', axesColor, 1) - drawSpace.addElement(r) - - #calculate data points - data = map(lambda X, Y: (X, Y), dataXPrimary, dataYPrimary) - xCoord = map(lambda X, Y: ((X-xLow)*xScale + xLeftOffset, yTopOffset+plotHeight-(Y-yLow)*yScale), dataXPrimary, dataYPrimary) - labelFontF = "verdana" - labelFontS = 11 - - if loadingPlot: - xZero = -xLow*xScale+xLeftOffset - yZero = yTopOffset+plotHeight+yLow*yScale - for point in xCoord: - drawSpace.addElement(svg.line(xZero,yZero,point[0],point[1], "red", 1)) - else: - if connectdot: - pass - #drawSpace.drawPolygon(xCoord,edgeColor=plotColor,closed=0) - else: - pass - - for i, item in enumerate(xCoord): - if dataLabel and dataLabel[i] in specialCases: - drawSpace.addElement(svg.rect(item[0]-3, item[1]-3, 6, 6, "none", "green", 0.5)) - #drawSpace.drawCross(item[0],item[1],color=pid.blue,size=5) - else: - drawSpace.addElement(svg.line(item[0],item[1]+5,item[0],item[1]-5,symbolColor,1)) - drawSpace.addElement(svg.line(item[0]+5,item[1],item[0]-5,item[1],symbolColor,1)) - if showLabel and dataLabel: - pass - drawSpace.addElement(svg.text(item[0], item[1]+14, dataLabel[i], labelFontS, - labelFontF, text_anchor="middle", style="stroke:blue;stroke-width:0.5;")) - #canvas.drawString(, item[0]- canvas.stringWidth(dataLabel[i], - # font=labelFont)/2, item[1]+14, font=labelFont, color=pid.blue) - - #draw scale - #scaleFont=pid.Font(ttf="cour",size=14,bold=1) - x=xLow - for i in range(stepX+1): - xc=xLeftOffset+(x-xLow)*xScale - drawSpace.addElement(svg.line(xc,yTopOffset+plotHeight,xc,yTopOffset+plotHeight+5, axesColor, 1)) - strX = cformat(d=x, rank=rank) - drawSpace.addElement(svg.text(xc,yTopOffset+plotHeight+20,strX,13, "courier", text_anchor="middle")) - x+= (xTop - xLow)/stepX - - y=yLow - for i in range(stepY+1): - yc=yTopOffset+plotHeight-(y-yLow)*yScale - drawSpace.addElement(svg.line(xLeftOffset,yc,xLeftOffset-5,yc, axesColor, 1)) - strY = cformat(d=y, rank=rank) - drawSpace.addElement(svg.text(xLeftOffset-10,yc+5,strY,13, "courier", text_anchor="end")) - y+= (yTop - yLow)/stepY - - #draw label - labelFontF = "verdana" - labelFontS = 17 - if XLabel: - drawSpace.addElement(svg.text(xLeftOffset+plotWidth/2.0, - yTopOffset+plotHeight+yBottomOffset-10,XLabel, - labelFontS, labelFontF, text_anchor="middle")) - - if YLabel: - drawSpace.addElement(svg.text(xLeftOffset-50, - yTopOffset+plotHeight/2,YLabel, - labelFontS, labelFontF, text_anchor="middle", style="writing-mode:tb-rl", transform="rotate(270 %d %d)" % (xLeftOffset-50, yTopOffset+plotHeight/2))) - #drawSpace.drawString(YLabel, xLeftOffset-50, yTopOffset+plotHeight- (plotHeight-drawSpace.stringWidth(YLabel,font=labelFont))/2.0, - # font=labelFont,color=labelColor,angle=90) - - - if fitcurve: - sys.argv = [ "mod_python" ] - #from numarray import linear_algebra as la - #from numarray import ones, array, dot, swapaxes - fitYY = array(dataYPrimary) - fitXX = array([ones(len(dataXPrimary)),dataXPrimary]) - AA = dot(fitXX,swapaxes(fitXX,0,1)) - BB = dot(fitXX,fitYY) - bb = la.linear_least_squares(AA,BB)[0] - - xc1 = xLeftOffset - yc1 = yTopOffset+plotHeight-(bb[0]+bb[1]*xLow-yLow)*yScale - if yc1 > yTopOffset+plotHeight: - yc1 = yTopOffset+plotHeight - xc1 = (yLow-bb[0])/bb[1] - xc1=(xc1-xLow)*xScale+xLeftOffset - elif yc1 < yTopOffset: - yc1 = yTopOffset - xc1 = (yTop-bb[0])/bb[1] - xc1=(xc1-xLow)*xScale+xLeftOffset - else: - pass - - xc2 = xLeftOffset + plotWidth - yc2 = yTopOffset+plotHeight-(bb[0]+bb[1]*xTop-yLow)*yScale - if yc2 > yTopOffset+plotHeight: - yc2 = yTopOffset+plotHeight - xc2 = (yLow-bb[0])/bb[1] - xc2=(xc2-xLow)*xScale+xLeftOffset - elif yc2 < yTopOffset: - yc2 = yTopOffset - xc2 = (yTop-bb[0])/bb[1] - xc2=(xc2-xLow)*xScale+xLeftOffset - else: - pass - - drawSpace.addElement(svg.line(xc1,yc1,xc2,yc2,"green", 1)) - - if displayR: - labelFontF = "trebuc" - labelFontS = 14 - NNN = len(dataX) - - corr = webqtlUtil.calCorrelation(dataXPrimary,dataYPrimary,NNN)[0] - - if NNN < 3: - corrPValue = 1.0 - else: - if abs(corr) >= 1.0: - corrPValue = 0.0 - else: - ZValue = 0.5*log((1.0+corr)/(1.0-corr)) - ZValue = ZValue*sqrt(NNN-3) - corrPValue = 2.0*(1.0 - reaper.normp(abs(ZValue))) - - NStr = "N of Cases=%d" % NNN - - if rank == 1: - corrStr = "Spearman's r=%1.3f P=%3.2E" % (corr, corrPValue) - else: - corrStr = "Pearson's r=%1.3f P=%3.2E" % (corr, corrPValue) - - drawSpace.addElement(svg.text(xLeftOffset,yTopOffset-10,NStr, - labelFontS, labelFontF, text_anchor="start")) - drawSpace.addElement(svg.text(xLeftOffset+plotWidth,yTopOffset-25,corrStr, - labelFontS, labelFontF, text_anchor="end")) - """ - """ - return - - # This function determines the scale of the plot def detScaleOld(min,max): if min>=max: @@ -1114,7 +221,7 @@ def detScaleOld(min,max): high=c*ceil(max/c) return [low,high,round((high-low)/c)] -def detScale(min=0,max=0,bufferSpace=3): +def detScale(min=0,max=0): if min>=max: return None @@ -1151,57 +258,9 @@ def detScale(min=0,max=0,bufferSpace=3): return [low,high,n] - - -def colorSpectrumOld(n): - if n == 1: - return [pid.Color(1,0,0)] - elif n == 2: - return [pid.Color(1,0,0),pid.Color(0,0,1)] - elif n == 3: - return [pid.Color(1,0,0),pid.Color(0,1,0),pid.Color(0,0,1)] - else: - step = 2.0/(n-1) - red = 1.0 - green = 0.0 - blue = 0.0 - colors = [pid.Color(red,green,blue)] - i = 1 - greenpeak = 0 - while i < n: - if red >= step: - red -= step - green += step - if green >= 1.0: - greenpeak = 1 - blue += green -1.0 - green = 1.0 - else: - red = 0.0 - if greenpeak: - green -= step - blue += step - else: - green += step - if green >= 1.0: - greenpeak = 1 - blue += green -1.0 - green = 2.0 -green - elif green < 0.0: - green = 0.0 - else: - pass - colors.append(pid.Color(red,green,blue)) - i += 1 - return colors - - - - def bluefunc(x): return 1.0 / (1.0 + exp(-10*(x-0.6))) - def redfunc(x): return 1.0 / (1.0 + exp(10*(x-0.5))) @@ -1230,52 +289,10 @@ def colorSpectrum(n=100): out2.append(out[-1]) return out2 - -def colorSpectrumSVG(n=100): - multiple = 10 - if n == 1: - return ["rgb(255,0,0)"] - elif n == 2: - return ["rgb(255,0,0)","rgb(0,0,255)"] - elif n == 3: - return ["rgb(255,0,0)","rgb(0,255,0)","rgb(0,0,255)"] - N = n*multiple - out = [None]*N; - for i in range(N): - x = float(i)/N - out[i] = "rgb(%d, %d, %d)" % (redfunc(x)*255, greenfunc(x)*255, bluefunc(x)*255); - out2 = [out[0]] - step = N/float(n-1) - j = 0 - for i in range(n-2): - j += step - out2.append(out[int(j)]) - out2.append(out[-1]) - return out2 - - -def BWSpectrum(n=100): - multiple = 10 - if n == 1: - return [pid.Color(0,0,0)] - elif n == 2: - return [pid.Color(0,0,0),pid.Color(1,1,1)] - elif n == 3: - return [pid.Color(0,0,0),pid.Color(0.5,0.5,0.5),pid.Color(1,1,1)] - - step = 1.0/n - x = 0.0 - out = [] - for i in range(n): - out.append(pid.Color(x,x,x)); - x += step - return out - - def _test(): import doctest doctest.testmod() if __name__=="__main__": - _test() + _test()
\ No newline at end of file diff --git a/wqflask/utility/after.py b/wqflask/utility/after.py index a3bb85e9..b628a0a4 100644 --- a/wqflask/utility/after.py +++ b/wqflask/utility/after.py @@ -13,10 +13,4 @@ def after_this_request(f): if not hasattr(g, 'after_request_callbacks'): g.after_request_callbacks = [] g.after_request_callbacks.append(f) - return f - -@app.after_request -def call_after_request_callbacks(response): - for callback in getattr(g, 'after_request_callbacks', ()): - callback(response) - return response + return f
\ No newline at end of file diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py index 8c97370d..8f1c916b 100644 --- a/wqflask/utility/benchmark.py +++ b/wqflask/utility/benchmark.py @@ -11,7 +11,7 @@ logger = getLogger(__name__ ) class Bench(object): entries = collections.OrderedDict() - def __init__(self, name=None, write_output=True): + def __init__(self, name=None, write_output=LOG_BENCH): self.name = name self.write_output = write_output diff --git a/wqflask/basicStatistics/corestats.py b/wqflask/utility/corestats.py index eba84c52..67ca3ad3 100644 --- a/wqflask/basicStatistics/corestats.py +++ b/wqflask/utility/corestats.py @@ -13,11 +13,9 @@ # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. - - import sys - +#ZS: Should switch to using some third party library for this; maybe scipy has an equivalent class Stats: def __init__(self, sequence): @@ -25,47 +23,33 @@ class Stats: # convert all items to floats for numerical processing self.sequence = [float(item) for item in sequence] - def sum(self): if len(self.sequence) < 1: return None else: return sum(self.sequence) - def count(self): return len(self.sequence) - def min(self): if len(self.sequence) < 1: return None else: return min(self.sequence) - def max(self): if len(self.sequence) < 1: return None else: return max(self.sequence) - def avg(self): if len(self.sequence) < 1: return None else: return sum(self.sequence) / len(self.sequence) - - def median(self): - if len(self.sequence) < 1: - return None - else: - self.sequence.sort() - return self.sequence[len(self.sequence) // 2] - - def stdev(self): if len(self.sequence) < 1: return None @@ -75,7 +59,6 @@ class Stats: stdev = (sdsq / (len(self.sequence) - 1)) ** .5 return stdev - def percentile(self, percentile): if len(self.sequence) < 1: value = None @@ -88,9 +71,6 @@ class Stats: value = self.sequence[element_idx] return value - - - # Sample script using this class: # ------------------------------------------- # #!/usr/bin/env python @@ -100,4 +80,4 @@ class Stats: # stats = corestats.Stats(sequence) # print stats.avg() # print stats.percentile(90) -# ------------------------------------------- +# -------------------------------------------
\ No newline at end of file diff --git a/wqflask/utility/corr_result_helpers.py b/wqflask/utility/corr_result_helpers.py index ef644d85..b543c589 100644 --- a/wqflask/utility/corr_result_helpers.py +++ b/wqflask/utility/corr_result_helpers.py @@ -15,7 +15,7 @@ def normalize_values(a_values, b_values): a_new = [] b_new = [] for counter in range(min_length): - if a_values[counter] and b_values[counter]: + if (a_values[counter] or a_values[counter] == 0) and (b_values[counter] or b_values[counter] == 0): a_new.append(a_values[counter]) b_new.append(b_values[counter]) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 1dba357d..15cdd0bc 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,3 +1,44 @@ +# Elasticsearch support +# +# Some helpful commands to view the database: +# +# You can test the server being up with +# +# curl -H 'Content-Type: application/json' http://localhost:9200 +# +# List all indices +# +# curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v' +# +# To see the users index 'table' +# +# curl http://localhost:9200/users +# +# To list all user ids +# +# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' +# { +# "query" : { +# "match_all" : {} +# }, +# "stored_fields": [] +# }' +# +# To view a record +# +# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' +# { +# "query" : { +# "match" : { "email_address": "pjotr2017@thebird.nl"} +# } +# }' +# +# +# To delete the users index and data (dangerous!) +# +# curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users' + + from elasticsearch import Elasticsearch, TransportError import logging @@ -7,11 +48,11 @@ logger = getLogger(__name__) from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT def test_elasticsearch_connection(): - es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+ELASTICSEARCH_PORT+'/'], verify_certs=True) + es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+str(ELASTICSEARCH_PORT)+'/'], verify_certs=True) if not es.ping(): logger.warning("Elasticsearch is DOWN") -def get_elasticsearch_connection(): +def get_elasticsearch_connection(for_user=True): """Return a connection to ES. Returns None on failure""" logger.info("get_elasticsearch_connection") es = None @@ -22,17 +63,30 @@ def get_elasticsearch_connection(): es = Elasticsearch([{ "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT - }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + }], timeout=30, retry_on_timeout=True) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + + if for_user: + setup_users_index(es) es_logger = logging.getLogger("elasticsearch") es_logger.setLevel(logging.INFO) es_logger.addHandler(logging.NullHandler()) - except: - logger.error("Failed to get elasticsearch connection") + except Exception as e: + logger.error("Failed to get elasticsearch connection", e) es = None return es +def setup_users_index(es_connection): + if es_connection: + index_settings = { + "properties": { + "email_address": { + "type": "keyword"}}} + + es_connection.indices.create(index='users', ignore=400) + es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local") + def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py index 7149e560..af306731 100644 --- a/wqflask/utility/genofile_parser.py +++ b/wqflask/utility/genofile_parser.py @@ -72,7 +72,7 @@ class ConvertGenoFile(object): yield row def process_csv(self): - for row_count, row in enumerate(self.process_rows()): + for row in self.process_rows(): row_items = row.split("\t") this_marker = Marker() diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py index 128706df..510b1041 100644 --- a/wqflask/utility/logger.py +++ b/wqflask/utility/logger.py @@ -33,7 +33,7 @@ from pprint import pformat as pf from inspect import stack import datetime -from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL, LOG_FORMAT +from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL class GNLogger: """A logger class with some additional functionality, such as diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 512bc9e6..db13b9d1 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -1029,7 +1029,7 @@ class drawing: try: xv.feed(svg) except: - raise "SVG is not well formed, see messages above" + raise Exception("SVG is not well formed, see messages above") else: print "SVG well formed" if __name__=='__main__': diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index 4fc978f5..83fa90b7 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -34,9 +34,6 @@ from htmlgen import HTMLgen2 as HT from base import webqtlConfig - - - # NL, 07/27/2010. moved from webqtlForm.py #Dict of Parents and F1 information, In the order of [F1, Mat, Pat] ParInfo ={ @@ -64,173 +61,10 @@ ParInfo ={ 'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'] } - -# NL, 07/27/2010. moved from template.py -IMGSTEP1 = HT.Image('/images/step1.gif', alt='STEP 1',border=0) #XZ, Only be used in inputPage.py -IMGSTEP2 = HT.Image('/images/step2.gif', alt='STEP 2',border=0) #XZ, Only be used in inputPage.py -IMGSTEP3 = HT.Image('/images/step3.gif', alt='STEP 3',border=0) #XZ, Only be used in inputPage.py -IMGNEXT = HT.Image('/images/arrowdown.gif', alt='NEXT',border=0) #XZ, Only be used in inputPage.py - -IMGASC = HT.Image("/images/sortup.gif", border=0) -IMGASCON = HT.Image("/images/sortupon.gif", border=0) -IMGDESC = HT.Image("/images/sortdown.gif", border=0) -IMGDESCON = HT.Image("/images/sortdownon.gif", border=0) - -""" -IMGASC = HT.Image("/images/sortup_icon.gif", border=0) -IMGASCON = HT.Image("/images/sortupon.gif", border=0) -IMGDESC = HT.Image("/images/sortdown_icon.gif", border=0) -IMGDESCON = HT.Image("/images/sortdownon.gif", border=0) -IMG_UNSORTED = HT.Image("/images/unsorted_icon.gif", border=0) -""" - -PROGRESSBAR = HT.Image('/images/waitAnima2.gif', alt='checkblue',align="middle",border=0) - ######################################### # Accessory Functions ######################################### -def inverseCumul(p): - #Coefficients in rational approximations. - a = [-3.969683028665376e+01,2.209460984245205e+02,-2.759285104469687e+02,1.383577518672690e+02,-3.066479806614716e+01,2.506628277459239e+00] - - b = [-5.447609879822406e+01,1.615858368580409e+02,-1.556989798598866e+02,6.680131188771972e+01,-1.328068155288572e+01] - - c = [-7.784894002430293e-03,-3.223964580411365e-01,-2.400758277161838e+00,-2.549732539343734e+00,4.374664141464968e+00,2.938163982698783e+00] - - d = [7.784695709041462e-03,3.224671290700398e-01,2.445134137142996e+00,3.754408661907416e+00] - - #Define break-points. - - p_low = 0.02425 - p_high = 1 - p_low - - #Rational approximation for lower region. - - if p > 0 and p < p_low: - q = sqrt(-2*log(p)) - x = (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) / ((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) - - - #Rational approximation for central region. - - elif p>= p_low and p <= p_high: - q = p - 0.5 - r = q*q - x = (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5])*q /(((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1) - - #Rational approximation for upper region. - - elif p>p_high and p < 1: - q = sqrt(-2*log(1-p)) - x = -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) - - else: - return None - - if p>0 and p < 1: - e = 0.5 * erfcc(-x/sqrt(2)) - p - u = e * sqrt(2*pi) * exp(x*x/2) - x = x - u/(1 + x*u/2) - return x - else: - return None - -def erfcc(x): - z=abs(x) - t=1.0/(1.0+0.5*z) - ans=t*exp(-z*z-1.26551223+t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277))))))))) - if x>=0.0: - return ans - else: - return 2.0-ans - -def U(n): - x=pow(0.5,1.0/n) - m=[1-x] - for i in range(2,n): - a=(i-0.3175)/(n+0.365) - m.append(a) - m.append(x) - return m - -def decodeEscape(str): - a = str - pattern = re.compile('(%[0-9A-Fa-f][0-9A-Fa-f])') - match = pattern.findall(a) - matched = [] - for item in match: - if item not in matched: - a = a.replace(item, '%c' % eval("0x"+item[-2:])) - matched.append(item) - return a - -def exportData(hddn, tdata, NP = None): - for key in tdata.keys(): - _val, _var, _N = tdata[key].val, tdata[key].var, tdata[key].N - if _val != None: - hddn[key] = _val - if _var != None: - hddn['V'+key] = _var - if NP and _N != None: - hddn['N'+key] = _N - -def genShortStrainName(RISet='', input_strainName=''): - #aliasStrainDict = {'C57BL/6J':'B6','DBA/2J':'D2'} - strainName = input_strainName - if RISet != 'AXBXA': - if RISet == 'BXD300': - this_RISet = 'BXD' - elif RISet == 'BDF2-2005': - this_RISet = 'CASE05_' - else: - this_RISet = RISet - strainName = string.replace(strainName,this_RISet,'') - strainName = string.replace(strainName,'CASE','') - try: - strainName = "%02d" % int(strainName) - except: - pass - else: - strainName = string.replace(strainName,'AXB','A') - strainName = string.replace(strainName,'BXA','B') - try: - strainName = strainName[0] + "%02d" % int(strainName[1:]) - except: - pass - return strainName - -def toInt(in_str): - "Converts an arbitrary string to an unsigned integer" - start = -1 - end = -1 - for i, char in enumerate(in_str): - if char >= '0' and char <= '9': - if start < 0: - start = i - end = i+1 - else: - if start >= 0: - break - if start < end: - return int(in_str[start:end]) - else: - return -1 - -def transpose(m): - 'transpose a matrix' - n = len(m) - return [[m[j][i] for i in range(len(m[0])) for j in range(n)][k*n:k*n+n] for k in range(len(m[0]))] - -def asymTranspose(m): - 'transpose a matrix' - t = max(map(len, m)) - n = len(m) - m2 = [["-"]]*n - for i in range(n): - m2[i] = m[i] + [""]*(t- len(m[i])) - return [[m2[j][i] for i in range(len(m2[0])) for j in range(n)][k*n:k*n+n] for k in range(len(m2[0]))] - def genRandStr(prefix = "", length=8, chars=string.letters+string.digits): from random import choice _str = prefix[:] @@ -238,84 +72,6 @@ def genRandStr(prefix = "", length=8, chars=string.letters+string.digits): _str += choice(chars) return _str -def generate_session(): - import sha - return sha.new(str(time.time())).hexdigest() - -def cvt2Dict(x): - tmp = {} - for key in x.keys(): - tmp[key] = x[key] - return tmp - -def dump_session(session_obj, filename): - "It seems mod python can only cPickle most basic data type" - import cPickle - session_file = open(filename, 'wb') - #try: - # pass - #except: - # pass - cPickle.dump(session_obj, session_file) - session_file.close() - -def StringAsFloat(str): - 'Converts string to float but catches any exception and returns None' - try: - return float(str) - except: - return None - -def IntAsFloat(str): - 'Converts string to Int but catches any exception and returns None' - try: - return int(str) - except: - return None - -def FloatAsFloat(flt): - 'Converts float to string but catches any exception and returns None' - try: - return float("%2.3f" % flt) - except: - return None - -def RemoveZero(flt): - 'Converts string to float but catches any exception and returns None' - try: - if abs(flt) < 1e-6: - return None - else: - return flt - except: - return None - - -def SciFloat(d): - 'Converts string to float but catches any exception and returns None' - - try: - if abs(d) <= 1.0e-4: - return "%1.2e" % d - else: - return "%1.5f" % d - except: - return None - -###To be removed -def FloatList2String(lst): - 'Converts float list to string but catches any exception and returns None' - tt='' - try: - for item in lst: - if item == None: - tt += 'X ' - else: - tt += '%f ' % item - return tt - except: - return "" - def ListNotNull(lst): '''Obsolete - Use built in function any (or all or whatever) @@ -328,427 +84,6 @@ def ListNotNull(lst): return 1 return None -###To be removed -def FileDataProcess(str): - 'Remove the description text from the input file if theres any' - i=0 - while i<len(str): - if str[i]<'\x7f' and str[i]>'\x20': - break - else: - i+=1 - str=str[i:] - str=string.join(string.split(str,'\000'),'') - i=string.find(str,"*****") - if i>-1: - return str[i+5:] - else: - return str - -def rank(a,lst,offset=0): - """Calculate the integer rank of a number in an array, can be used to calculate p-value""" - n = len(lst) - if n == 2: - if a <lst[0]: - return offset - elif a > lst[1]: - return offset + 2 - else: - return offset +1 - elif n == 1: - if a <lst[0]: - return offset - else: - return offset +1 - elif n== 0: - return offset - else: - mid = n/2 - if a < lst[mid]: - return rank(a,lst[:mid-1],offset) - else: - return rank(a,lst[mid:],offset+mid) - -def cmpScanResult(A,B): - try: - if A.LRS > B.LRS: - return 1 - elif A.LRS == B.LRS: - return 0 - else: - return -1 - except: - return 0 - - -def cmpScanResult2(A,B): - try: - if A.LRS < B.LRS: - return 1 - elif A.LRS == B.LRS: - return 0 - else: - return -1 - except: - return 0 - -def cmpOrder(A,B): - try: - if A[1] < B[1]: - return -1 - elif A[1] == B[1]: - return 0 - else: - return 1 - except: - return 0 - -def cmpOrder2(A,B): - try: - if A[-1] < B[-1]: - return -1 - elif A[-1] == B[-1]: - return 0 - else: - return 1 - except: - return 0 - - - - -def calRank(xVals, yVals, N): ### Zach Sloan, February 4 2010 - """ - Returns a ranked set of X and Y values. These are used when generating - a Spearman scatterplot. Bear in mind that this sets values equal to each - other as the same rank. - """ - XX = [] - YY = [] - X = [0]*len(xVals) - Y = [0]*len(yVals) - j = 0 - - for i in range(len(xVals)): - - if xVals[i] != None and yVals[i] != None: - XX.append((j, xVals[i])) - YY.append((j, yVals[i])) - j = j + 1 - - NN = len(XX) - - XX.sort(cmpOrder2) - YY.sort(cmpOrder2) - - j = 1 - rank = 0.0 - - while j < NN: - - if XX[j][1] != XX[j-1][1]: - X[XX[j-1][0]] = j - j = j+1 - - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (XX[jt][1] != XX[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - X[XX[ji][0]] = rank - if (jt == NN-1): - if (XX[jt][1] == XX[j-1][1]): - X[XX[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if X[XX[NN-1][0]] == 0: - X[XX[NN-1][0]] = NN - - j = 1 - rank = 0.0 - - while j < NN: - - if YY[j][1] != YY[j-1][1]: - Y[YY[j-1][0]] = j - j = j+1 - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (YY[jt][1] != YY[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - Y[YY[ji][0]] = rank - if (jt == NN-1): - if (YY[jt][1] == YY[j-1][1]): - Y[YY[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if Y[YY[NN-1][0]] == 0: - Y[YY[NN-1][0]] = NN - - return (X,Y) - -def calCorrelationRank(xVals,yVals,N): - """ - Calculated Spearman Ranked Correlation. The algorithm works - by setting all tied ranks to the average of those ranks (for - example, if ranks 5-10 all have the same value, each will be set - to rank 7.5). - """ - - XX = [] - YY = [] - j = 0 - - for i in range(len(xVals)): - if (xVals[i]!= None and yVals[i]!= None) and (xVals[i] != "None" and yVals[i] != "None"): - XX.append((j,xVals[i])) - YY.append((j,yVals[i])) - j = j+1 - - NN = len(XX) - if NN <6: - return (0.0,NN) - XX.sort(cmpOrder2) - YY.sort(cmpOrder2) - X = [0]*NN - Y = [0]*NN - - j = 1 - rank = 0.0 - t = 0.0 - sx = 0.0 - - while j < NN: - - if XX[j][1] != XX[j-1][1]: - X[XX[j-1][0]] = j - j = j+1 - - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (XX[jt][1] != XX[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - X[XX[ji][0]] = rank - t = jt-j - sx = sx + (t*t*t-t) - if (jt == NN-1): - if (XX[jt][1] == XX[j-1][1]): - X[XX[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if X[XX[NN-1][0]] == 0: - X[XX[NN-1][0]] = NN - - j = 1 - rank = 0.0 - t = 0.0 - sy = 0.0 - - while j < NN: - - if YY[j][1] != YY[j-1][1]: - Y[YY[j-1][0]] = j - j = j+1 - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (YY[jt][1] != YY[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - Y[YY[ji][0]] = rank - t = jt - j - sy = sy + (t*t*t-t) - if (jt == NN-1): - if (YY[jt][1] == YY[j-1][1]): - Y[YY[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if Y[YY[NN-1][0]] == 0: - Y[YY[NN-1][0]] = NN - - D = 0.0 - - for i in range(NN): - D += (X[i]-Y[i])*(X[i]-Y[i]) - - fac = (1.0 -sx/(NN*NN*NN-NN))*(1.0-sy/(NN*NN*NN-NN)) - - return ((1-(6.0/(NN*NN*NN-NN))*(D+(sx+sy)/12.0))/math.sqrt(fac),NN) - - -def calCorrelationRankText(dbdata,userdata,N): ### dcrowell = David Crowell, July 2008 - """Calculates correlation ranks with data formatted from the text file. - dbdata, userdata are lists of strings. N is an int. Returns a float. - Used by correlationPage""" - XX = [] - YY = [] - j = 0 - for i in range(N): - if (dbdata[i]!= None and userdata[i]!=None) and (dbdata[i]!= 'None' and userdata[i]!='None'): - XX.append((j,float(dbdata[i]))) - YY.append((j,float(userdata[i]))) - j += 1 - NN = len(XX) - if NN <6: - return (0.0,NN) - XX.sort(cmpOrder2) - YY.sort(cmpOrder2) - X = [0]*NN - Y = [0]*NN - - j = 1 - rank = 0.0 - t = 0.0 - sx = 0.0 - - while j < NN: - - if XX[j][1] != XX[j-1][1]: - X[XX[j-1][0]] = j - j = j+1 - - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (XX[jt][1] != XX[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - X[XX[ji][0]] = rank - t = jt-j - sx = sx + (t*t*t-t) - if (jt == NN-1): - if (XX[jt][1] == XX[j-1][1]): - X[XX[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if X[XX[NN-1][0]] == 0: - X[XX[NN-1][0]] = NN - - j = 1 - rank = 0.0 - t = 0.0 - sy = 0.0 - - while j < NN: - - if YY[j][1] != YY[j-1][1]: - Y[YY[j-1][0]] = j - j = j+1 - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (YY[jt][1] != YY[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - Y[YY[ji][0]] = rank - t = jt - j - sy = sy + (t*t*t-t) - if (jt == NN-1): - if (YY[jt][1] == YY[j-1][1]): - Y[YY[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if Y[YY[NN-1][0]] == 0: - Y[YY[NN-1][0]] = NN - - D = 0.0 - - for i in range(NN): - D += (X[i]-Y[i])*(X[i]-Y[i]) - - fac = (1.0 -sx/(NN*NN*NN-NN))*(1.0-sy/(NN*NN*NN-NN)) - - return ((1-(6.0/(NN*NN*NN-NN))*(D+(sx+sy)/12.0))/math.sqrt(fac),NN) - - - -def calCorrelation(dbdata,userdata,N): - X = [] - Y = [] - for i in range(N): - if dbdata[i]!= None and userdata[i]!= None: - X.append(dbdata[i]) - Y.append(userdata[i]) - NN = len(X) - if NN <6: - return (0.0,NN) - sx = reduce(lambda x,y:x+y,X,0.0) - sy = reduce(lambda x,y:x+y,Y,0.0) - meanx = sx/NN - meany = sy/NN - xyd = 0.0 - sxd = 0.0 - syd = 0.0 - for i in range(NN): - xyd += (X[i] - meanx)*(Y[i]-meany) - sxd += (X[i] - meanx)*(X[i] - meanx) - syd += (Y[i] - meany)*(Y[i] - meany) - try: - corr = xyd/(sqrt(sxd)*sqrt(syd)) - except: - corr = 0 - return (corr,NN) - -def calCorrelationText(dbdata,userdata,N): ### dcrowell July 2008 - """Calculates correlation coefficients with values formatted from text files. dbdata, userdata are lists of strings. N is an int. Returns a float - Used by correlationPage""" - X = [] - Y = [] - for i in range(N): - #if (dbdata[i]!= None and userdata[i]!= None) and (dbdata[i]!= 'None' and userdata[i]!= 'None'): - # X.append(float(dbdata[i])) - # Y.append(float(userdata[i])) - if dbdata[i] == None or dbdata[i] == 'None' or userdata[i] == None or userdata[i] == 'None': - continue - else: - X.append(float(dbdata[i])) - Y.append(float(userdata[i])) - NN = len(X) - if NN <6: - return (0.0,NN) - sx = sum(X) - sy = sum(Y) - meanx = sx/float(NN) - meany = sy/float(NN) - xyd = 0.0 - sxd = 0.0 - syd = 0.0 - for i in range(NN): - x1 = X[i]-meanx - y1 = Y[i]-meany - xyd += x1*y1 - sxd += x1**2 - syd += y1**2 - try: - corr = xyd/(sqrt(sxd)*sqrt(syd)) - except: - corr = 0 - return (corr,NN) - - def readLineCSV(line): ### dcrowell July 2008 """Parses a CSV string of text and returns a list containing each element as a string. Used by correlationPage""" @@ -757,45 +92,6 @@ def readLineCSV(line): ### dcrowell July 2008 returnList[0]=returnList[0][1:] return returnList - -def cmpCorr(A,B): - try: - if abs(A[1]) < abs(B[1]): - return 1 - elif abs(A[1]) == abs(B[1]): - return 0 - else: - return -1 - except: - return 0 - -def cmpLitCorr(A,B): - try: - if abs(A[3]) < abs(B[3]): return 1 - elif abs(A[3]) == abs(B[3]): - if abs(A[1]) < abs(B[1]): return 1 - elif abs(A[1]) == abs(B[1]): return 0 - else: return -1 - else: return -1 - except: - return 0 - -def cmpPValue(A,B): - try: - if A.corrPValue < B.corrPValue: - return -1 - elif A.corrPValue == B.corrPValue: - if abs(A.corr) > abs(B.corr): - return -1 - elif abs(A.corr) < abs(B.corr): - return 1 - else: - return 0 - else: - return 1 - except: - return 0 - def cmpEigenValue(A,B): try: if A[0] > B[0]: @@ -807,80 +103,6 @@ def cmpEigenValue(A,B): except: return 0 - -def cmpLRSFull(A,B): - try: - if A[0] < B[0]: - return -1 - elif A[0] == B[0]: - return 0 - else: - return 1 - except: - return 0 - -def cmpLRSInteract(A,B): - try: - if A[1] < B[1]: - return -1 - elif A[1] == B[1]: - return 0 - else: - return 1 - except: - return 0 - - -def cmpPos(A,B): - try: - try: - AChr = int(A.chr) - except: - AChr = 20 - try: - BChr = int(B.chr) - except: - BChr = 20 - if AChr > BChr: - return 1 - elif AChr == BChr: - if A.mb > B.mb: - return 1 - if A.mb == B.mb: - return 0 - else: - return -1 - else: - return -1 - except: - return 0 - -def cmpGenoPos(A,B): - try: - A1 = A.chr - B1 = B.chr - try: - A1 = int(A1) - except: - A1 = 25 - try: - B1 = int(B1) - except: - B1 = 25 - if A1 > B1: - return 1 - elif A1 == B1: - if A.mb > B.mb: - return 1 - if A.mb == B.mb: - return 0 - else: - return -1 - else: - return -1 - except: - return 0 - def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users): access_to_confidential_phenotype_trait = 0 if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']: @@ -889,142 +111,4 @@ def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users) AuthorizedUsersList=map(string.strip, string.split(authorized_users, ',')) if AuthorizedUsersList.__contains__(userName): access_to_confidential_phenotype_trait = 1 - return access_to_confidential_phenotype_trait - - -class VisualizeException(Exception): - def __init__(self, message): - self.message = message - def __str__(self): - return self.message - -# safeConvert : (string -> A) -> A -> A -# to convert a string to type A, using the supplied default value -# if the given conversion function doesn't work -def safeConvert(f, value, default): - try: - return f(value) - except: - return default - -# safeFloat : string -> float -> float -# to convert a string to a float safely -def safeFloat(value, default): - return safeConvert(float, value, default) - -# safeInt: string -> int -> int -# to convert a string to an int safely -def safeInt(value, default): - return safeConvert(int, value, default) - -# safeString : string -> (arrayof string) -> string -> string -# if a string is not in a list of strings to pick a default value -# for that string -def safeString(value, validChoices, default): - if value in validChoices: - return value - else: - return default - -# yesNoToInt: string -> int -# map "yes" -> 1 and "no" -> 0 -def yesNoToInt(value): - if value == "yes": - return 1 - elif value == "no": - return 0 - else: - return None - -# IntToYesNo: int -> string -# map 1 -> "yes" and 0 -> "no" -def intToYesNo(value): - if value == 1: - return "yes" - elif value == 0: - return "no" - else: - return None - -def formatField(name): - name = name.replace("_", " ") - name = name.title() - #name = name.replace("Mb Mm6", "Mb"); - return name.replace("Id", "ID") - -#XZ, 03/27/2009: This function is very specific. -#It is used by AJAX_table.py, correlationPage.py and dataPage.py - - -def genTableObj(tblobj=None, file="", sortby = ("", ""), tableID = "sortable", addIndex = "1", hiddenColumns=[]): - header = tblobj['header'] - body = tblobj['body'] - field, order = sortby - - #ZAS 9/12/2011 - The hiddenColumns array needs to be converted into a string so they can be placed into the javascript of each up/down button - hiddenColumnsString = ",".join(hiddenColumns) - - tbl = HT.TableLite(Class="collap b2", cellspacing=1, cellpadding=5) - - hiddenColumnIdx = [] #indices of columns to hide - idx = -1 - last_idx = 0 #ZS: This is the index of the last item in the regular table header (without any extra parameters). It is used to determine the index of each extra parameter. - for row in header: - hr = HT.TR() - for i, item in enumerate(row): - if (item.text == '') or (item.text not in hiddenColumns): - if item.sort and item.text: - down = HT.Href("javascript:xmlhttpPost('%smain.py?FormID=AJAX_table', '%s', 'sort=%s&order=down&file=%s&tableID=%s&addIndex=%s&hiddenColumns=%s')" % (webqtlConfig.CGIDIR, tableID, item.text, file, tableID, addIndex, hiddenColumnsString),IMGDESC) - up = HT.Href("javascript:xmlhttpPost('%smain.py?FormID=AJAX_table', '%s', 'sort=%s&order=up&file=%s&tableID=%s&addIndex=%s&hiddenColumns=%s')" % (webqtlConfig.CGIDIR, tableID, item.text, file, tableID, addIndex, hiddenColumnsString),IMGASC) - if item.text == field: - idx = item.idx - last_idx = idx - if order == 'up': - up = IMGASCON - elif order == 'down': - down = IMGDESCON - item.html.append(HT.Div(up, down, style="float: bottom;")) - hr.append(item.html) - else: - hiddenColumnIdx.append(i) - tbl.append(hr) - - for i, row in enumerate(body): - for j, item in enumerate(row): - if order == 'down': - if (item.val == '' or item.val == 'x' or item.val == 'None'): - item.val = 0 - if order == 'up': - if (item.val == '' or item.val == 'x' or item.val == 'None'): - item.val = 'zzzzz' - - if idx >= 0: - if order == 'down': - body.sort(lambda A, B: cmp(B[idx].val, A[idx].val), key=natsort_key) - elif order == 'up': - body.sort(lambda A, B: cmp(A[idx].val, B[idx].val), key=natsort_key) - else: - pass - - for i, row in enumerate(body): - hr = HT.TR(Id = row[0].text) - for j, item in enumerate(row): - if (j not in hiddenColumnIdx): - if j == 0: - if addIndex == "1": - item.html.contents = [i+1] + item.html.contents - hr.append(item.html) - tbl.append(hr) - - return tbl - -def natsort_key(string): - r = [] - for c in string: - try: - c = int(c) - try: r[-1] = r[-1] * 10 + c - except: r.append(c) - except: - r.append(c) - return r + return access_to_confidential_phenotype_trait
\ No newline at end of file diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index ffc698de..ecc02c06 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -40,13 +40,6 @@ import logging from utility.logger import getLogger logger = getLogger(__name__) -def get_collection(): - if g.user_session.logged_in: - return UserCollection() - else: - return AnonCollection() - #else: - # CauseError class AnonCollection(object): """User is not logged in""" @@ -95,7 +88,6 @@ class AnonCollection(object): def add_traits(self, params): #assert collection_name == "Default", "Unexpected collection name for anonymous user" self.traits = list(process_traits(params['traits'])) - #len_before = len(Redis.smembers(self.key)) existing_collections = Redis.get(self.key) logger.debug("existing_collections:", existing_collections) if existing_collections != None and existing_collections != "None": @@ -130,10 +122,6 @@ class AnonCollection(object): collections_list.append(collection_dict) Redis.set(self.key, json.dumps(collections_list)) - #Redis.sadd(self.key, *list(traits)) - #Redis.expire(self.key, 60 * 60 * 24 * 5) - #len_now = len(Redis.smembers(self.key)) - #report_change(len_before, len_now) def remove_traits(self, params): traits_to_remove = [(":").join(trait.split(":")[:2]) for trait in params.getlist('traits[]')] @@ -149,54 +137,12 @@ class AnonCollection(object): collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members']) collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') len_now = collections_list[collection_position]['num_members'] - #print("before in redis:", json.loads(Redis.get(self.key))) Redis.set(self.key, json.dumps(collections_list)) - #print("currently in redis:", json.loads(Redis.get(self.key))) # We need to return something so we'll return this...maybe in the future # we can use it to check the results return str(len_now) - -class UserCollection(object): - """User is logged in""" - - def add_traits(self, params, collection_name): - logger.debug("---> params are:", params.keys()) - logger.debug(" type(params):", type(params)) - if collection_name=="Default": - uc = g.user_session.user_ob.get_collection_by_name("Default") - # Doesn't exist so we'll create it - if not uc: - return create_new("Default") - else: - uc = model.UserCollection.query.get(params['existing_collection'].split(":")[0]) - members = list(uc.members_as_set()) #set(json.loads(uc.members)) - len_before = len(members) - - traits = process_traits(params['traits']) - - members_now = members - for trait in traits: - if trait in members: - continue - else: - members_now.append(trait) - - #members_now = list(members | traits) - len_now = len(members_now) - uc.members = json.dumps(members_now) - - uc.changed_timestamp = datetime.datetime.utcnow() - - db_session.commit() - - logger.debug("added to existing, now set is:" + str(uc.members)) - report_change(len_before, len_now) - - # Probably have to change that - return redirect(url_for('view_collection', uc_id=uc.id)) - def process_traits(unprocessed_traits): #print("unprocessed_traits are:", unprocessed_traits) if isinstance(unprocessed_traits, basestring): @@ -227,9 +173,8 @@ def collections_add(): traits=request.args['traits'] if g.user_session.logged_in: - logger.debug("user_session",g.user_session) - user_collections = g.user_session.user_ob.user_collections - logger.debug("user_collections are:", user_collections) + user_collections = g.user_session.user_collections + #logger.debug("user_collections are:", user_collections) return render_template("collections/add.html", traits = traits, collections = user_collections, @@ -256,9 +201,12 @@ def collections_new(): return create_new(collection_name) elif "add_to_existing" in params: logger.debug("in add to existing") + collection_id = params['existing_collection'].split(":")[0] collection_name = params['existing_collection'].split(":")[1] if g.user_session.logged_in: - return UserCollection().add_traits(params, collection_name) + traits = list(process_traits(params['traits'])) + g.user_session.add_traits_to_collection(collection_id, traits) + return redirect(url_for('view_collection', uc_id=collection_id)) else: ac = AnonCollection(collection_name) ac.add_traits(params) @@ -273,16 +221,10 @@ def create_new(collection_name): traits = process_traits(unprocessed_traits) if g.user_session.logged_in: - uc = model.UserCollection() - uc.name = collection_name - logger.debug("user_session:", g.user_session.__dict__) - uc.user = g.user_session.user_id - uc.members = json.dumps(list(traits)) - db_session.add(uc) - db_session.commit() - return redirect(url_for('view_collection', uc_id=uc.id)) + uc_id = g.user_session.add_collection(collection_name, traits) + + return redirect(url_for('view_collection', uc_id=uc_id)) else: - current_collections = user_manager.AnonUser().get_collections() ac = AnonCollection(collection_name) ac.changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') ac.add_traits(params) @@ -291,17 +233,17 @@ def create_new(collection_name): @app.route("/collections/list") def list_collections(): params = request.args - logger.debug("PARAMS:", params) + #logger.debug("PARAMS:", params) if g.user_session.logged_in: - user_collections = list(g.user_session.user_ob.user_collections) - logger.debug("user_collections are:", user_collections) + user_collections = list(g.user_session.user_collections) + #logger.debug("user_collections are:", user_collections) return render_template("collections/list.html", params = params, collections = user_collections, ) else: anon_collections = user_manager.AnonUser().get_collections() - logger.debug("anon_collections are:", anon_collections) + #logger.debug("anon_collections are:", anon_collections) return render_template("collections/list.html", params = params, collections = anon_collections) @@ -310,20 +252,14 @@ def list_collections(): @app.route("/collections/remove", methods=('POST',)) def remove_traits(): params = request.form - logger.debug("params are:", params) if "uc_id" in params: uc_id = params['uc_id'] - uc = model.UserCollection.query.get(uc_id) traits_to_remove = params.getlist('traits[]') traits_to_remove = process_traits(traits_to_remove) logger.debug("\n\n after processing, traits_to_remove:", traits_to_remove) - all_traits = uc.members_as_set() - members_now = all_traits - traits_to_remove - logger.debug(" members_now:", members_now) - uc.members = json.dumps(list(members_now)) - uc.changed_timestamp = datetime.datetime.utcnow() - db_session.commit() + + members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove) else: collection_name = params['collection_name'] members_now = AnonCollection(collection_name).remove_traits(params) @@ -337,22 +273,13 @@ def remove_traits(): @app.route("/collections/delete", methods=('POST',)) def delete_collection(): params = request.form - logger.debug("params:", params) if g.user_session.logged_in: uc_id = params['uc_id'] if len(uc_id.split(":")) > 1: for this_uc_id in uc_id.split(":"): - uc = model.UserCollection.query.get(this_uc_id) - collection_name = uc.name - db_session.delete(uc) - db_session.commit() + collection_name = g.user_session.delete_collection(this_uc_id) else: - uc = model.UserCollection.query.get(uc_id) - # Todo: For now having the id is good enough since it's so unique - # But might want to check ownership in the future - collection_name = uc.name - db_session.delete(uc) - db_session.commit() + collection_name = g.user_session.delete_collection(uc_id) else: if "collection_name" in params: collection_name = params['collection_name'] @@ -360,7 +287,10 @@ def delete_collection(): for this_collection in params['uc_id'].split(":"): user_manager.AnonUser().delete_collection(this_collection) - flash("We've deleted the collection: {}.".format(collection_name), "alert-info") + if len(uc_id.split(":")) > 1: + flash("We've deleted the selected collections.", "alert-info") + else: + flash("We've deleted the collection: {}.".format(collection_name), "alert-info") return redirect(url_for('list_collections')) @@ -368,12 +298,11 @@ def delete_collection(): @app.route("/collections/view") def view_collection(): params = request.args - logger.debug("PARAMS in view collection:", params) - if "uc_id" in params: + if g.user_session.logged_in and "uc_id" in params: uc_id = params['uc_id'] - uc = model.UserCollection.query.get(uc_id) - traits = json.loads(uc.members) + uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next() + traits = uc["members"] else: user_collections = json.loads(Redis.get(user_manager.AnonUser().key)) this_collection = {} @@ -381,10 +310,8 @@ def view_collection(): if collection['id'] == params['collection_id']: this_collection = collection break - #this_collection = user_collections[params['collection_id']] - traits = this_collection['members'] - logger.debug("in view_collection traits are:", traits) + traits = this_collection['members'] trait_obs = [] json_version = [] @@ -410,9 +337,8 @@ def view_collection(): collection_info = dict(trait_obs=trait_obs, collection_name=this_collection['name']) if "json" in params: - logger.debug("json_version:", json_version) return json.dumps(json_version) else: return render_template("collections/view.html", **collection_info - ) + )
\ No newline at end of file diff --git a/wqflask/basicStatistics/__init__.py b/wqflask/wqflask/comparison_bar_chart/__init__.py index e69de29b..e69de29b 100644 --- a/wqflask/basicStatistics/__init__.py +++ b/wqflask/wqflask/comparison_bar_chart/__init__.py diff --git a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py new file mode 100644 index 00000000..b9e6f450 --- /dev/null +++ b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py @@ -0,0 +1,118 @@ +## Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# This program is available from Source Forge: at GeneNetwork Project +# (sourceforge.net/projects/genenetwork/). +# +# Contact Dr. Robert W. Williams at rwilliams@uthsc.edu +# +# +# This module is used by GeneNetwork project (www.genenetwork.org) + +from __future__ import absolute_import, print_function, division + +import sys +# sys.path.append(".") Never do this in a webserver! + +import string +import cPickle +import os +import time +import pp +import math +import collections +import resource + + +from pprint import pformat as pf + +from htmlgen import HTMLgen2 as HT +import reaper + +from base.trait import GeneralTrait +from base import data_set +from utility import webqtlUtil, helper_functions, corr_result_helpers +from db import webqtlDatabaseFunction +import utility.webqtlUtil #this is for parallel computing only. +from wqflask.correlation import correlation_functions +from utility.benchmark import Bench + +from MySQLdb import escape_string as escape + +from pprint import pformat as pf + +from flask import Flask, g + + +class ComparisonBarChart(object): + + def __init__(self, start_vars): + trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] + + helper_functions.get_trait_db_obs(self, trait_db_list) + + self.all_sample_list = [] + self.traits = [] + self.insufficient_shared_samples = False + this_group = self.trait_list[0][1].group.name #ZS: Getting initial group name before verifying all traits are in the same group in the following loop + for trait_db in self.trait_list: + + if trait_db[1].group.name != this_group: + self.insufficient_shared_samples = True + break + else: + this_group = trait_db[1].group.name + this_trait = trait_db[0] + self.traits.append(this_trait) + + this_sample_data = this_trait.data + + for sample in this_sample_data: + if sample not in self.all_sample_list: + self.all_sample_list.append(sample) + + if self.insufficient_shared_samples: + pass + else: + self.sample_data = [] + for trait_db in self.trait_list: + this_trait = trait_db[0] + this_sample_data = this_trait.data + + this_trait_vals = [] + for sample in self.all_sample_list: + if sample in this_sample_data: + this_trait_vals.append(this_sample_data[sample].value) + else: + this_trait_vals.append('') + self.sample_data.append(this_trait_vals) + + self.js_data = dict(traits = [trait.name for trait in self.traits], + samples = self.all_sample_list, + sample_data = self.sample_data,) + + def get_trait_db_obs(self, trait_db_list): + + self.trait_list = [] + for i, trait_db in enumerate(trait_db_list): + if i == (len(trait_db_list) - 1): + break + trait_name, dataset_name = trait_db.split(":") + #print("dataset_name:", dataset_name) + dataset_ob = data_set.create_dataset(dataset_name) + trait_ob = GeneralTrait(dataset=dataset_ob, + name=trait_name, + cellid=None) + self.trait_list.append((trait_ob, dataset_ob)) + + #print("trait_list:", self.trait_list) + diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index a08cd759..831baf7e 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -6,50 +6,19 @@ from utility import corr_result_helpers from scipy import stats import numpy as np +import utility.logger +logger = utility.logger.getLogger(__name__ ) + class CorrScatterPlot(object): """Page that displays a correlation scatterplot with a line fitted to it""" def __init__(self, params): self.data_set_1 = data_set.create_dataset(params['dataset_1']) self.data_set_2 = data_set.create_dataset(params['dataset_2']) + #self.data_set_3 = data_set.create_dataset(params['dataset_3']) self.trait_1 = GeneralTrait(name=params['trait_1'], dataset=self.data_set_1) self.trait_2 = GeneralTrait(name=params['trait_2'], dataset=self.data_set_2) - - try: - width = int(params['width']) - except: - width = 800 - self.width = width - - try: - height = int(params['height']) - except: - height = 600 - self.height = height - - try: - circle_color = params['circle_color'] - except: - circle_color = '#3D85C6' - self.circle_color = circle_color - - try: - circle_radius = int(params['circle_radius']) - except: - circle_radius = 5 - self.circle_radius = circle_radius - - try: - line_color = params['line_color'] - except: - line_color = '#FF0000' - self.line_color = line_color - - try: - line_width = int(params['line_width']) - except: - line_width = 1 - self.line_width = line_width + #self.trait_3 = GeneralTrait(name=params['trait_3'], dataset=self.data_set_3) samples_1, samples_2, num_overlap = corr_result_helpers.normalize_values_with_samples(self.trait_1.data, self.trait_2.data) @@ -75,6 +44,10 @@ class CorrScatterPlot(object): self.rdata.append(ry.tolist()) srslope, srintercept, srr_value, srp_value, srstd_err = stats.linregress(rx, ry) + #vals_3 = [] + #for sample in self.trait_3.data: + # vals_3.append(self.trait_3.data[sample].value) + self.js_data = dict( data = self.data, rdata = self.rdata, @@ -86,22 +59,18 @@ class CorrScatterPlot(object): num_overlap = num_overlap, vals_1 = vals_1, vals_2 = vals_2, - + slope = slope, intercept = intercept, r_value = r_value, p_value = p_value, - + srslope = srslope, srintercept = srintercept, srr_value = srr_value, - srp_value = srp_value, - - width = width, - height = height, - circle_color = circle_color, - circle_radius = circle_radius, - line_color = line_color, - line_width = line_width + srp_value = srp_value + + #trait3 = self.trait_3.data, + #vals_3 = vals_3 ) self.jsdata = self.js_data diff --git a/wqflask/wqflask/correlation/correlation_functions.py b/wqflask/wqflask/correlation/correlation_functions.py index 80a0818c..06dec795 100644 --- a/wqflask/wqflask/correlation/correlation_functions.py +++ b/wqflask/wqflask/correlation/correlation_functions.py @@ -28,524 +28,12 @@ from __future__ import absolute_import, print_function, division import math import rpy2.robjects -import pp import string -from utility import webqtlUtil from base.mrna_assay_tissue_data import MrnaAssayTissueData -from base.trait import GeneralTrait -from db import webqtlDatabaseFunction from flask import Flask, g -#XZ: The input 'controls' is String. It contains the full name of control traits. -#XZ: The input variable 'strainlst' is List. It contains the strain names of primary trait. -#XZ: The returned tcstrains is the list of list [[],[]...]. So are tcvals and tcvars. The last returned parameter is list of numbers. -#XZ, 03/29/2010: For each returned control trait, there is no None value in it. -def controlStrains(controls, strainlst): - - controls = controls.split(',') - - cvals = {} - for oneTraitName in controls: - oneTrait = webqtlTrait(fullname=oneTraitName, cursor=webqtlDatabaseFunction.getCursor() ) - oneTrait.retrieveData() - cvals[oneTraitName] = oneTrait.data - - tcstrains = [] - tcvals = [] - tcvars = [] - - for oneTraitName in controls: - strains = [] - vals = [] - vars = [] - - for _strain in strainlst: - if cvals[oneTraitName].has_key(_strain): - _val = cvals[oneTraitName][_strain].val - if _val != None: - strains.append(_strain) - vals.append(_val) - vars.append(None) - - tcstrains.append(strains) - tcvals.append(vals) - tcvars.append(vars) - - return tcstrains, tcvals, tcvars, [len(x) for x in tcstrains] - - - -#XZ, 03/29/2010: After execution of functon "controlStrains" and "fixStrains", primary trait and control traits have the same strains and in the same order. There is no 'None' value in them. -def fixStrains(_strains,_controlstrains,_vals,_controlvals,_vars,_controlvars): - """Corrects strains, vals, and vars so that all contrain only those strains common - to the reference trait and all control traits.""" - - def dictify(strains,vals,vars): - subdict = {} - for i in xrange(len(strains)): - subdict[strains[i]] = (vals[i],vars[i]) - return subdict - - #XZ: The 'dicts' is a list of dictionary. The first element is the dictionary of reference trait. The rest elements are for control traits. - dicts = [] - dicts.append(dictify(_strains,_vals,_vars)) - - nCstrains = len(_controlstrains) - for i in xrange(nCstrains): - dicts.append(dictify(_controlstrains[i],_controlvals[i],_controlvars[i])) - - _newstrains = [] - _vals = [] - _vars = [] - _controlvals = [[] for x in xrange(nCstrains)] - _controlvars = [[] for x in xrange(nCstrains)] - - for strain in _strains: - inall = True - for d in dicts: - if strain not in d: - inall = False - break - if inall: - _newstrains.append(strain) - _vals.append(dicts[0][strain][0]) - _vars.append(dicts[0][strain][1]) - for i in xrange(nCstrains): - _controlvals[i].append(dicts[i+1][strain][0]) - _controlvars[i].append(dicts[i+1][strain][1]) - - return _newstrains, _vals, _controlvals, _vars, _controlvars - - -#XZ, 6/15/2010: If there is no identical control traits, the returned list is empty. -#else, the returned list has two elements of control trait name. -def findIdenticalControlTraits ( controlVals, controlNames ): - nameOfIdenticalTraits = [] - - controlTraitNumber = len(controlVals) - - if controlTraitNumber > 1: - - #XZ: reset the precision of values and convert to string type - for oneTraitVal in controlVals: - for oneStrainVal in oneTraitVal: - oneStrainVal = '%.3f' % oneStrainVal - - for i, oneTraitVal in enumerate( controlVals ): - for j in range(i+1, controlTraitNumber): - if oneTraitVal == controlVals[j]: - nameOfIdenticalTraits.append(controlNames[i]) - nameOfIdenticalTraits.append(controlNames[j]) - - return nameOfIdenticalTraits - -#XZ, 6/15/2010: If there is no identical control traits, the returned list is empty. -#else, the returned list has two elements of control trait name. -#primaryVal is of list type. It contains value of primary trait. -#primaryName is of string type. -#controlVals is of list type. Each element is list too. Each element contain value of one control trait. -#controlNames is of list type. -def findIdenticalTraits (primaryVal, primaryName, controlVals, controlNames ): - nameOfIdenticalTraits = [] - - #XZ: reset the precision of values and convert to string type - for oneStrainVal in primaryVal: - oneStrainVal = '%.3f' % oneStrainVal - - for oneTraitVal in controlVals: - for oneStrainVal in oneTraitVal: - oneStrainVal = '%.3f' % oneStrainVal - - controlTraitNumber = len(controlVals) - - if controlTraitNumber > 1: - for i, oneTraitVal in enumerate( controlVals ): - for j in range(i+1, controlTraitNumber): - if oneTraitVal == controlVals[j]: - nameOfIdenticalTraits.append(controlNames[i]) - nameOfIdenticalTraits.append(controlNames[j]) - break - - if len(nameOfIdenticalTraits) == 0: - for i, oneTraitVal in enumerate( controlVals ): - if primaryVal == oneTraitVal: - nameOfIdenticalTraits.append(primaryName) - nameOfIdenticalTraits.append(controlNames[i]) - break - - return nameOfIdenticalTraits - - - -#XZ, 03/29/2010: The strains in primaryVal, controlVals, targetVals must be of the same number and in same order. -#XZ: No value in primaryVal and controlVals could be None. - -def determinePartialsByR (primaryVal, controlVals, targetVals, targetNames, method='p'): - - def compute_partial ( primaryVal, controlVals, targetVals, targetNames, method ): - - rpy2.robjects.r(""" -pcor.test <- function(x,y,z,use="mat",method="p",na.rm=T){ - # The partial correlation coefficient between x and y given z - # - # pcor.test is free and comes with ABSOLUTELY NO WARRANTY. - # - # x and y should be vectors - # - # z can be either a vector or a matrix - # - # use: There are two methods to calculate the partial correlation coefficient. - # One is by using variance-covariance matrix ("mat") and the other is by using recursive formula ("rec"). - # Default is "mat". - # - # method: There are three ways to calculate the correlation coefficient, - # which are Pearson's ("p"), Spearman's ("s"), and Kendall's ("k") methods. - # The last two methods which are Spearman's and Kendall's coefficient are based on the non-parametric analysis. - # Default is "p". - # - # na.rm: If na.rm is T, then all the missing samples are deleted from the whole dataset, which is (x,y,z). - # If not, the missing samples will be removed just when the correlation coefficient is calculated. - # However, the number of samples for the p-value is the number of samples after removing - # all the missing samples from the whole dataset. - # Default is "T". - - x <- c(x) - y <- c(y) - z <- as.data.frame(z) - - if(use == "mat"){ - p.use <- "Var-Cov matrix" - pcor = pcor.mat(x,y,z,method=method,na.rm=na.rm) - }else if(use == "rec"){ - p.use <- "Recursive formula" - pcor = pcor.rec(x,y,z,method=method,na.rm=na.rm) - }else{ - stop("use should be either rec or mat!\n") - } - - # print the method - if(gregexpr("p",method)[[1]][1] == 1){ - p.method <- "Pearson" - }else if(gregexpr("s",method)[[1]][1] == 1){ - p.method <- "Spearman" - }else if(gregexpr("k",method)[[1]][1] == 1){ - p.method <- "Kendall" - }else{ - stop("method should be pearson or spearman or kendall!\n") - } - - # sample number - n <- dim(na.omit(data.frame(x,y,z)))[1] - - # given variables' number - gn <- dim(z)[2] - - # p-value - if(p.method == "Kendall"){ - statistic <- pcor/sqrt(2*(2*(n-gn)+5)/(9*(n-gn)*(n-1-gn))) - p.value <- 2*pnorm(-abs(statistic)) - - }else{ - statistic <- pcor*sqrt((n-2-gn)/(1-pcor^2)) - p.value <- 2*pnorm(-abs(statistic)) - } - - data.frame(estimate=pcor,p.value=p.value,statistic=statistic,n=n,gn=gn,Method=p.method,Use=p.use) -} - -# By using var-cov matrix -pcor.mat <- function(x,y,z,method="p",na.rm=T){ - - x <- c(x) - y <- c(y) - z <- as.data.frame(z) - - if(dim(z)[2] == 0){ - stop("There should be given data\n") - } - - data <- data.frame(x,y,z) - - if(na.rm == T){ - data = na.omit(data) - } - - xdata <- na.omit(data.frame(data[,c(1,2)])) - Sxx <- cov(xdata,xdata,m=method) - - xzdata <- na.omit(data) - xdata <- data.frame(xzdata[,c(1,2)]) - zdata <- data.frame(xzdata[,-c(1,2)]) - Sxz <- cov(xdata,zdata,m=method) - - zdata <- na.omit(data.frame(data[,-c(1,2)])) - Szz <- cov(zdata,zdata,m=method) - - # is Szz positive definite? - zz.ev <- eigen(Szz)$values - if(min(zz.ev)[1]<0){ - stop("\'Szz\' is not positive definite!\n") - } - - # partial correlation - Sxx.z <- Sxx - Sxz %*% solve(Szz) %*% t(Sxz) - - rxx.z <- cov2cor(Sxx.z)[1,2] - - rxx.z -} - -# By using recursive formula -pcor.rec <- function(x,y,z,method="p",na.rm=T){ - # - - x <- c(x) - y <- c(y) - z <- as.data.frame(z) - - if(dim(z)[2] == 0){ - stop("There should be given data\n") - } - - data <- data.frame(x,y,z) - - if(na.rm == T){ - data = na.omit(data) - } - - # recursive formula - if(dim(z)[2] == 1){ - tdata <- na.omit(data.frame(data[,1],data[,2])) - rxy <- cor(tdata[,1],tdata[,2],m=method) - - tdata <- na.omit(data.frame(data[,1],data[,-c(1,2)])) - rxz <- cor(tdata[,1],tdata[,2],m=method) - - tdata <- na.omit(data.frame(data[,2],data[,-c(1,2)])) - ryz <- cor(tdata[,1],tdata[,2],m=method) - - rxy.z <- (rxy - rxz*ryz)/( sqrt(1-rxz^2)*sqrt(1-ryz^2) ) - - return(rxy.z) - }else{ - x <- c(data[,1]) - y <- c(data[,2]) - z0 <- c(data[,3]) - zc <- as.data.frame(data[,-c(1,2,3)]) - - rxy.zc <- pcor.rec(x,y,zc,method=method,na.rm=na.rm) - rxz0.zc <- pcor.rec(x,z0,zc,method=method,na.rm=na.rm) - ryz0.zc <- pcor.rec(y,z0,zc,method=method,na.rm=na.rm) - - rxy.z <- (rxy.zc - rxz0.zc*ryz0.zc)/( sqrt(1-rxz0.zc^2)*sqrt(1-ryz0.zc^2) ) - return(rxy.z) - } -} -""") - - R_pcorr_function = rpy2.robjects.r['pcor.test'] - R_corr_test = rpy2.robjects.r['cor.test'] - - primary = rpy2.robjects.FloatVector(range(len(primaryVal))) - for i in range(len(primaryVal)): - primary[i] = primaryVal[i] - - control = rpy2.robjects.r.matrix(rpy2.robjects.FloatVector( range(len(controlVals)*len(controlVals[0])) ), ncol=len(controlVals)) - for i in range(len(controlVals)): - for j in range(len(controlVals[0])): - control[i*len(controlVals[0]) + j] = controlVals[i][j] - - allcorrelations = [] - - for targetIndex, oneTargetVals in enumerate(targetVals): - - this_primary = None - this_control = None - this_target = None - - if None in oneTargetVals: - - goodIndex = [] - for i in range(len(oneTargetVals)): - if oneTargetVals[i] != None: - goodIndex.append(i) - - this_primary = rpy2.robjects.FloatVector(range(len(goodIndex))) - for i in range(len(goodIndex)): - this_primary[i] = primaryVal[goodIndex[i]] - - this_control = rpy2.robjects.r.matrix(rpy2.robjects.FloatVector( range(len(controlVals)*len(goodIndex)) ), ncol=len(controlVals)) - for i in range(len(controlVals)): - for j in range(len(goodIndex)): - this_control[i*len(goodIndex) + j] = controlVals[i][goodIndex[j]] - - this_target = rpy2.robjects.FloatVector(range(len(goodIndex))) - for i in range(len(goodIndex)): - this_target[i] = oneTargetVals[goodIndex[i]] - - else: - this_primary = primary - this_control = control - this_target = rpy2.robjects.FloatVector(range(len(oneTargetVals))) - for i in range(len(oneTargetVals)): - this_target[i] = oneTargetVals[i] - - one_name = targetNames[targetIndex] - one_N = len(this_primary) - - #calculate partial correlation - one_pc_coefficient = 'NA' - one_pc_p = 1 - - try: - if method == 's': - result = R_pcorr_function(this_primary, this_target, this_control, method='s') - else: - result = R_pcorr_function(this_primary, this_target, this_control) - - #XZ: In very few cases, the returned coefficient is nan. - #XZ: One way to detect nan is to compare the number to itself. NaN is always != NaN - if result[0][0] == result[0][0]: - one_pc_coefficient = result[0][0] - #XZ: when the coefficient value is 1 (primary trait and target trait are the same), - #XZ: occationally, the returned p value is nan instead of 0. - if result[1][0] == result[1][0]: - one_pc_p = result[1][0] - elif abs(one_pc_coefficient - 1) < 0.0000001: - one_pc_p = 0 - except: - pass - - #calculate zero order correlation - one_corr_coefficient = 0 - one_corr_p = 1 - - try: - if method == 's': - R_result = R_corr_test(this_primary, this_target, method='spearman') - else: - R_result = R_corr_test(this_primary, this_target) - - one_corr_coefficient = R_result[3][0] - one_corr_p = R_result[2][0] - except: - pass - - traitinfo = [ one_name, one_N, one_pc_coefficient, one_pc_p, one_corr_coefficient, one_corr_p ] - - allcorrelations.append(traitinfo) - - return allcorrelations - #End of function compute_partial - - - allcorrelations = [] - - target_trait_number = len(targetVals) - - if target_trait_number < 1000: - allcorrelations = compute_partial ( primaryVal, controlVals, targetVals, targetNames, method ) - else: - step = 1000 - job_number = math.ceil( float(target_trait_number)/step ) - - job_targetVals_lists = [] - job_targetNames_lists = [] - - for job_index in range( int(job_number) ): - starti = job_index*step - endi = min((job_index+1)*step, target_trait_number) - - one_job_targetVals_list = [] - one_job_targetNames_list = [] - - for i in range( starti, endi ): - one_job_targetVals_list.append( targetVals[i] ) - one_job_targetNames_list.append( targetNames[i] ) - - job_targetVals_lists.append( one_job_targetVals_list ) - job_targetNames_lists.append( one_job_targetNames_list ) - - ppservers = () - # Creates jobserver with automatically detected number of workers - job_server = pp.Server(ppservers=ppservers) - - jobs = [] - results = [] - - for i, one_job_targetVals_list in enumerate( job_targetVals_lists ): - one_job_targetNames_list = job_targetNames_lists[i] - #pay attention to modules from outside - jobs.append( job_server.submit(func=compute_partial, args=( primaryVal, controlVals, one_job_targetVals_list, one_job_targetNames_list, method), depfuncs=(), modules=("rpy2.robjects",)) ) - - for one_job in jobs: - one_result = one_job() - results.append( one_result ) - - for one_result in results: - for one_traitinfo in one_result: - allcorrelations.append( one_traitinfo ) - - return allcorrelations - - - -#XZ, April 30, 2010: The input primaryTrait and targetTrait are instance of webqtlTrait -#XZ: The primaryTrait and targetTrait should have executed retrieveData function -def calZeroOrderCorr(primaryTrait, targetTrait, method='pearson'): - - #primaryTrait.retrieveData() - - #there is no None value in primary_val - primary_strain, primary_val, primary_var = primaryTrait.exportInformative() - - #targetTrait.retrieveData() - - #there might be None value in target_val - target_val = targetTrait.exportData(primary_strain, type="val") - - R_primary = rpy2.robjects.FloatVector(range(len(primary_val))) - for i in range(len(primary_val)): - R_primary[i] = primary_val[i] - - N = len(target_val) - - if None in target_val: - goodIndex = [] - for i in range(len(target_val)): - if target_val[i] != None: - goodIndex.append(i) - - N = len(goodIndex) - - R_primary = rpy2.robjects.FloatVector(range(len(goodIndex))) - for i in range(len(goodIndex)): - R_primary[i] = primary_val[goodIndex[i]] - - R_target = rpy2.robjects.FloatVector(range(len(goodIndex))) - for i in range(len(goodIndex)): - R_target[i] = target_val[goodIndex[i]] - - else: - R_target = rpy2.robjects.FloatVector(range(len(target_val))) - for i in range(len(target_val)): - R_target[i] = target_val[i] - - R_corr_test = rpy2.robjects.r['cor.test'] - - if method == 'spearman': - R_result = R_corr_test(R_primary, R_target, method='spearman') - else: - R_result = R_corr_test(R_primary, R_target) - - corr_result = [] - corr_result.append( R_result[3][0] ) - corr_result.append( N ) - corr_result.append( R_result[2][0] ) - - return corr_result ##################################################################################### #Input: primaryValue(list): one list of expression values of one probeSet, @@ -585,170 +73,6 @@ def cal_zero_order_corr_for_tiss (primaryValue=[], targetValue=[], method='pears return corr_result - - -def batchCalTissueCorr(primaryTraitValue=[], SymbolValueDict={}, method='pearson'): - - def cal_tissue_corr(primaryTraitValue, oneSymbolValueDict, method ): - - oneSymbolCorrDict = {} - oneSymbolPvalueDict = {} - - R_corr_test = rpy2.robjects.r['cor.test'] - - R_primary = rpy2.robjects.FloatVector(range(len(primaryTraitValue))) - - for i in range(len(primaryTraitValue)): - R_primary[i] = primaryTraitValue[i] - - for (oneTraitSymbol, oneTraitValue) in oneSymbolValueDict.iteritems(): - R_target = rpy2.robjects.FloatVector(range(len(oneTraitValue))) - for i in range(len(oneTraitValue)): - R_target[i] = oneTraitValue[i] - - if method =='spearman': - R_result = R_corr_test(R_primary, R_target, method='spearman') - else: - R_result = R_corr_test(R_primary, R_target) - - oneSymbolCorrDict[oneTraitSymbol] = R_result[3][0] - oneSymbolPvalueDict[oneTraitSymbol] = R_result[2][0] - - return(oneSymbolCorrDict, oneSymbolPvalueDict) - - - - symbolCorrDict = {} - symbolPvalueDict = {} - - items_number = len(SymbolValueDict) - - if items_number <= 1000: - symbolCorrDict, symbolPvalueDict = cal_tissue_corr(primaryTraitValue, SymbolValueDict, method) - else: - items_list = SymbolValueDict.items() - - step = 1000 - job_number = math.ceil( float(items_number)/step ) - - job_oneSymbolValueDict_list = [] - - for job_index in range( int(job_number) ): - starti = job_index*step - endi = min((job_index+1)*step, items_number) - - oneSymbolValueDict = {} - - for i in range( starti, endi ): - one_item = items_list[i] - one_symbol = one_item[0] - one_value = one_item[1] - oneSymbolValueDict[one_symbol] = one_value - - job_oneSymbolValueDict_list.append( oneSymbolValueDict ) - - - ppservers = () - # Creates jobserver with automatically detected number of workers - job_server = pp.Server(ppservers=ppservers) - - jobs = [] - results = [] - - for i, oneSymbolValueDict in enumerate( job_oneSymbolValueDict_list ): - - #pay attention to modules from outside - jobs.append( job_server.submit(func=cal_tissue_corr, args=(primaryTraitValue, oneSymbolValueDict, method), depfuncs=(), modules=("rpy2.robjects",)) ) - - for one_job in jobs: - one_result = one_job() - results.append( one_result ) - - for one_result in results: - oneSymbolCorrDict, oneSymbolPvalueDict = one_result - symbolCorrDict.update( oneSymbolCorrDict ) - symbolPvalueDict.update( oneSymbolPvalueDict ) - - return (symbolCorrDict, symbolPvalueDict) - -########################################################################### -#Input: cursor, GeneNameLst (list), TissueProbeSetFreezeId -#output: geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict (Dict) -#function: get multi dicts for short and long label functions, and for getSymbolValuePairDict and -# getGeneSymbolTissueValueDict to build dict to get CorrPvArray -#Note: If there are multiple probesets for one gene, select the one with highest mean. -########################################################################### -def getTissueProbeSetXRefInfo(GeneNameLst=[],TissueProbeSetFreezeId=0): - Symbols ="" - symbolList =[] - geneIdDict ={} - dataIdDict = {} - ChrDict = {} - MbDict = {} - descDict = {} - pTargetDescDict = {} - - count = len(GeneNameLst) - - # Added by NL 01/06/2011 - # Note that:inner join is necessary in this query to get distinct record in one symbol group with highest mean value - # Duo to the limit size of TissueProbeSetFreezeId table in DB, performance of inner join is acceptable. - if count==0: - query=''' - select t.Symbol,t.GeneId, t.DataId,t.Chr, t.Mb,t.description,t.Probe_Target_Description - from ( - select Symbol, max(Mean) as maxmean - from TissueProbeSetXRef - where TissueProbeSetFreezeId=%s and Symbol!='' and Symbol Is Not Null group by Symbol) - as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol and t.Mean = x.maxmean; - '''%TissueProbeSetFreezeId - - else: - for i, item in enumerate(GeneNameLst): - - if i == count-1: - Symbols += "'%s'" %item - else: - Symbols += "'%s'," %item - - Symbols = "("+ Symbols+")" - query=''' - select t.Symbol,t.GeneId, t.DataId,t.Chr, t.Mb,t.description,t.Probe_Target_Description - from ( - select Symbol, max(Mean) as maxmean - from TissueProbeSetXRef - where TissueProbeSetFreezeId=%s and Symbol in %s group by Symbol) - as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol and t.Mean = x.maxmean; - '''% (TissueProbeSetFreezeId,Symbols) - - try: - cursor.execute(query) - results =cursor.fetchall() - resultCount = len(results) - # Key in all dicts is the lower-cased symbol - for i, item in enumerate(results): - symbol = item[0] - symbolList.append(symbol) - - key =symbol.lower() - geneIdDict[key]=item[1] - dataIdDict[key]=item[2] - ChrDict[key]=item[3] - MbDict[key]=item[4] - descDict[key]=item[5] - pTargetDescDict[key]=item[6] - - except: - symbolList = None - geneIdDict=None - dataIdDict=None - ChrDict=None - MbDict=None - descDict=None - pTargetDescDict=None - - return symbolList,geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict - ########################################################################### #Input: cursor, symbolList (list), dataIdDict(Dict) #output: symbolValuepairDict (dictionary):one dictionary of Symbol and Value Pair, @@ -757,7 +81,6 @@ def getTissueProbeSetXRefInfo(GeneNameLst=[],TissueProbeSetFreezeId=0): #Attention! All keys are lower case! ########################################################################### def get_symbol_value_pairs(tissue_data): - id_list = [tissue_data[symbol.lower()].data_id for item in tissue_data] symbol_value_pairs = {} @@ -775,23 +98,6 @@ def get_symbol_value_pairs(tissue_data): except: symbol_value_pairs[symbol] = None - #for symbol in symbol_list: - # if tissue_data.has_key(symbol): - # data_id = tissue_data[symbol].data_id - # - # query = """select value, id - # from TissueProbeSetData - # where Id={}""".format(escape(data_id)) - # try : - # results = g.db.execute(query).fetchall() - # for item in results: - # item = item[0] - # value_list.append(item) - # symbol_value_pairs[symbol] = value_list - # value_list=[] - # except: - # symbol_value_pairs[symbol] = None - return symbol_value_pairs @@ -808,150 +114,4 @@ def get_trait_symbol_and_tissue_values(symbol_list=None): tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list) if len(tissue_data.gene_symbols): - return tissue_data.get_symbol_values_pairs() - - #symbolList, - #geneIdDict, - #dataIdDict, - #ChrDict, - #MbDict, - #descDict, - #pTargetDescDict = getTissueProbeSetXRefInfo( - # GeneNameLst=GeneNameLst,TissueProbeSetFreezeId=TissueProbeSetFreezeId) - - #limit_num=1000 - #count = len(symbol_list) - # - #symbol_value_pairs = {} - # - #if count !=0 and count <= limit_num: - # symbol_value_pairs = getSymbolValuePairDict(cursor=cursor,symbolList=symbol_list,dataIdDict=dataIdDict) - # - #elif count > limit_num: - # n = count/limit_num - # start = 0 - # stop = 0 - # - # for i in range(n): - # stop =limit_num*(i+1) - # gList1 = symbolList[start:stop] - # PairDict1 = getSymbolValuePairDict(cursor=cursor,symbolList=gList1,dataIdDict=dataIdDict) - # start =limit_num*(i+1) - # - # SymbolValuePairDict.update(PairDict1) - # - # if stop < count: - # stop = count - # gList2 = symbolList[start:stop] - # PairDict2 = getSymbolValuePairDict(cursor=cursor,symbolList=gList2,dataIdDict=dataIdDict) - # SymbolValuePairDict.update(PairDict2) - # - #return SymbolValuePairDict - -######################################################################################################## -#input: cursor, GeneNameLst (list), TissueProbeSetFreezeId(int) -#output: SymbolValuePairDict(dictionary):one dictionary of Symbol and Value Pair. -# key is symbol, value is one list of expression values of one probeSet. -#function: wrapper function of getGeneSymbolTissueValueDict function -# for CorrelationPage.py -######################################################################################################## - -#def get_trait_symbol_and_tissue_values(cursor=None,GeneNameLst=[],TissueProbeSetFreezeId=0): -# SymbolValuePairDict={} -# -# symbolList,geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict = getTissueProbeSetXRefInfo( -# cursor=cursor,GeneNameLst=GeneNameLst,TissueProbeSetFreezeId=TissueProbeSetFreezeId) -# -# if symbolList: -# SymbolValuePairDict = get_gene_symbol_and_tissue_values(symbolList=symbolList, -# dataIdDict=dataIdDict) -# -# return SymbolValuePairDict - -######################################################################################################## -#Input: cursor(cursor): MySQL connnection cursor; -# priGeneSymbolList(list): one list of gene symbol; -# symbolValuepairDict(dictionary): one dictionary of Symbol and Value Pair, -# key is symbol, value is one list of expression values of one probeSet; -#Output: corrArray(array): array of Correlation Value, -# pvArray(array): array of PValue; -#Function: build corrArray, pvArray for display by calling calculation function:calZeroOrderCorrForTiss -######################################################################################################## - -def getCorrPvArray(cursor=None,priGeneSymbolList=[],symbolValuepairDict={}): - # setting initial value for corrArray, pvArray equal to 0 - Num = len(priGeneSymbolList) - - corrArray = [([0] * (Num))[:] for i in range(Num)] - pvArray = [([0] * (Num))[:] for i in range(Num)] - i = 0 - for pkey in priGeneSymbolList: - j = 0 - pkey = pkey.strip().lower()# key in symbolValuepairDict is low case - if symbolValuepairDict.has_key(pkey): - priValue = symbolValuepairDict[pkey] - for tkey in priGeneSymbolList: - tkey = tkey.strip().lower()# key in symbolValuepairDict is low case - if priValue and symbolValuepairDict.has_key(tkey): - tarValue = symbolValuepairDict[tkey] - - if tarValue: - if i>j: - # corrArray stores Pearson Correlation values - # pvArray stores Pearson P-Values - pcorr_result =calZeroOrderCorrForTiss(primaryValue=priValue,targetValue=tarValue) - corrArray[i][j] =pcorr_result[0] - pvArray[i][j] =pcorr_result[2] - elif i<j: - # corrArray stores Spearman Correlation values - # pvArray stores Spearman P-Values - scorr_result =calZeroOrderCorrForTiss(primaryValue=priValue,targetValue=tarValue,method='spearman') - corrArray[i][j] =scorr_result[0] - pvArray[i][j] =scorr_result[2] - else: - # on the diagonal line, correlation value is 1, P-Values is 0 - corrArray[i][j] =1 - pvArray[i][j] =0 - j+=1 - else: - corrArray[i][j] = None - pvArray[i][j] = None - j+=1 - else: - corrArray[i][j] = None - pvArray[i][j] = None - j+=1 - else: - corrArray[i][j] = None - pvArray[i][j] = None - - i+=1 - - return corrArray, pvArray - -######################################################################################################## -#Input: cursor(cursor): MySQL connnection cursor; -# primaryTraitSymbol(string): one gene symbol; -# TissueProbeSetFreezeId (int): Id of related TissueProbeSetFreeze -# method: '0' default value, Pearson Correlation; '1', Spearman Correlation -#Output: symbolCorrDict(Dict): Dict of Correlation Value, key is symbol -# symbolPvalueDict(Dict): Dict of PValue,key is symbol ; -#Function: build symbolCorrDict, symbolPvalueDict for display by calling calculation function:calZeroOrderCorrForTiss -######################################################################################################## -def calculateCorrOfAllTissueTrait(cursor=None, primaryTraitSymbol=None, TissueProbeSetFreezeId=None,method='0'): - - symbolCorrDict = {} - symbolPvalueDict = {} - - primaryTraitSymbolValueDict = getGeneSymbolTissueValueDictForTrait(cursor=cursor, GeneNameLst=[primaryTraitSymbol], TissueProbeSetFreezeId=TissueProbeSetFreezeId) - primaryTraitValue = primaryTraitSymbolValueDict.values()[0] - - SymbolValueDict = getGeneSymbolTissueValueDictForTrait(cursor=cursor, GeneNameLst=[], TissueProbeSetFreezeId=TissueProbeSetFreezeId) - - if method =='1': - symbolCorrDict, symbolPvalueDict = batchCalTissueCorr(primaryTraitValue,SymbolValueDict,method='spearman') - else: - symbolCorrDict, symbolPvalueDict = batchCalTissueCorr(primaryTraitValue,SymbolValueDict) - - - return (symbolCorrDict, symbolPvalueDict) + return tissue_data.get_symbol_values_pairs()
\ No newline at end of file diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 73072423..9f3f7982 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -21,9 +21,7 @@ from __future__ import absolute_import, print_function, division import sys -# sys.path.append(".") Never in a running webserver -import gc import string import cPickle import os @@ -64,8 +62,6 @@ from flask import Flask, g import utility.logger logger = utility.logger.getLogger(__name__ ) -METHOD_SAMPLE_PEARSON = "1" -METHOD_SAMPLE_RANK = "2" METHOD_LIT = "3" METHOD_TISSUE_PEARSON = "4" METHOD_TISSUE_RANK = "5" @@ -74,26 +70,7 @@ TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] TISSUE_MOUSE_DB = 1 -def print_mem(stage=""): - mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - #print("{}: {}".format(stage, mem/1024)) - -class AuthException(Exception): - pass - class CorrelationResults(object): - - corr_min_informative = 4 - - #PAGE_HEADING = "Correlation Table" - #CORRELATION_METHODS = {"1" : "Genetic Correlation (Pearson's r)", - # "2" : "Genetic Correlation (Spearman's rho)", - # "3" : "SGO Literature Correlation", - # "4" : "Tissue Correlation (Pearson's r)", - # "5" : "Tissue Correlation (Spearman's rho)"} - # - #RANK_ORDERS = {"1": 0, "2": 1, "3": 0, "4": 0, "5": 1} - def __init__(self, start_vars): # get trait list from db (database name) # calculate correlation with Base vector and targets @@ -167,16 +144,12 @@ class CorrelationResults(object): self.process_samples(start_vars, self.this_trait.data.keys(), primary_samples) self.target_dataset = data_set.create_dataset(start_vars['corr_dataset']) - # print("self.sample_data.keys: %s" % self.sample_data.keys) self.target_dataset.get_trait_data(self.sample_data.keys()) self.correlation_results = [] self.correlation_data = {} - db_filename = self.getFileName(target_db_name = self.target_dataset.name) - cache_available = db_filename in os.listdir(webqtlConfig.GENERATED_TEXT_DIR) - if self.corr_type == "tissue": self.trait_symbol_dict = self.dataset.retrieve_genes("Symbol") @@ -196,24 +169,6 @@ class CorrelationResults(object): self.get_sample_r_and_p_values(trait, self.target_dataset.trait_data[trait]) elif self.corr_type == "sample": - #ZS: Commented out since parallel correlation has issues with gunicorn - # if self.dataset.type == "ProbeSet" and cache_available: - # dataset_file = open(webqtlConfig.GENERATED_TEXT_DIR+db_filename,'r') - - ##XZ, 01/08/2009: read the first line - # line = dataset_file.readline() - # dataset_strains = webqtlUtil.readLineCSV(line)[1:] - - # self.this_trait_vals = [] - # for item in dataset_strains: - # if item in self.sample_data: - # self.this_trait_vals.append(self.sample_data[item]) - # else: - # self.this_trait_vals.append("None") - # num_overlap = len(self.this_trait_vals) - # logger.debug("DOING PARALLEL") - # self.do_parallel_correlation(db_filename, num_overlap) - # else: for trait, values in self.target_dataset.trait_data.iteritems(): self.get_sample_r_and_p_values(trait, values) @@ -256,9 +211,6 @@ class CorrelationResults(object): trait_object.sample_p, trait_object.num_overlap) = self.correlation_data[trait] - #Get symbol for trait and call function that gets each tissue value from the database (tables TissueProbeSetXRef, - #TissueProbeSetData, etc) and calculates the correlation (cal_zero_order_corr_for_tissue in correlation_functions) - # Set some sane defaults trait_object.tissue_corr = 0 trait_object.tissue_pvalue = 0 @@ -274,9 +226,6 @@ class CorrelationResults(object): trait_object.sample_p, trait_object.num_overlap) = self.correlation_data[trait] - #Get symbol for trait and call function that gets each tissue value from the database (tables TissueProbeSetXRef, - #TissueProbeSetData, etc) and calculates the correlation (cal_zero_order_corr_for_tissue in correlation_functions) - # Set some sane defaults trait_object.tissue_corr = 0 trait_object.tissue_pvalue = 0 @@ -296,60 +245,8 @@ class CorrelationResults(object): if self.corr_type != "tissue" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": self.do_tissue_correlation_for_trait_list() - #print("self.correlation_results: ", pf(self.correlation_results)) - self.json_results = generate_corr_json(self.correlation_results, self.this_trait, self.dataset, self.target_dataset) - #XZ, 09/18/2008: get all information about the user selected database. - #target_db_name = fd.corr_dataset - #self.target_db_name = start_vars['corr_dataset'] - - # Zach said this is ok - # Auth if needed - #try: - # auth_user_for_db(self.db, self.cursor, self.target_db_name, self.privilege, self.userName) - #except AuthException as e: - # detail = [e.message] - # return self.error(detail) - - #XZ, 09/18/2008: filter out the strains that have no value. - #self.sample_names, vals, vars, N = fd.informativeStrains(sample_list) - - #print("samplenames is:", pf(self.sample_names)) - #CF - If less than a minimum number of strains/cases in common, don't calculate anything - #if len(self.sample_names) < self.corr_min_informative: - # detail = ['Fewer than %d strain data were entered for %s data set. No calculation of correlation has been attempted.' % (self.corr_min_informative, fd.RISet)] - # self.error(heading=None, detail=detail) - - #correlation_method = self.CORRELATION_METHODS[self.method] - #rankOrder = self.RANK_ORDERS[self.method] - - # CF - Number of results returned - # Todo: Get rid of self.returnNumber - - #self.record_count = 0 - - #myTrait = get_custom_trait(fd, self.cursor) - - - # We will not get Literature Correlations if there is no GeneId because there is nothing - # to look against - #self.geneid = self.this_trait.geneid - - # We will not get Tissue Correlations if there is no gene symbol because there is nothing to look against - #self.trait_symbol = myTrait.symbol - - - #XZ, 12/12/2008: if the species is rat or human, translate the geneid to mouse geneid - #self.input_trait_mouse_gene_id = self.translateToMouseGeneID(self.dataset.group.species, self.geneid) - - #XZ: As of Nov/13/2010, this dataset is 'UTHSC Illumina V6.2 RankInv B6 D2 average CNS GI average (May 08)' - #self.tissue_probeset_freeze_id = 1 - - #traitList = self.correlate() - - #print("Done doing correlation calculation") - ############################################################################################################################################ def get_formatted_corr_type(self): @@ -375,13 +272,6 @@ class CorrelationResults(object): if self.this_trait.symbol.lower() in primary_trait_tissue_vals_dict: primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower()] - - #gene_symbol_list = [] - # - #for trait in self.correlation_results: - # if hasattr(trait, 'symbol'): - # gene_symbol_list.append(trait.symbol) - gene_symbol_list = [trait.symbol for trait in self.correlation_results if trait.symbol] corr_result_tissue_vals_dict= correlation_functions.get_trait_symbol_and_tissue_values( @@ -398,17 +288,6 @@ class CorrelationResults(object): trait.tissue_corr = result[0] trait.tissue_pvalue = result[2] - # else: - # trait.tissue_corr = None - # trait.tissue_pvalue = None - #else: - # for trait in self.correlation_results: - # trait.tissue_corr = None - # trait.tissue_pvalue = None - - #return self.correlation_results - - def do_tissue_correlation_for_all_traits(self, tissue_dataset_id=1): #Gets tissue expression values for the primary trait primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( @@ -429,7 +308,6 @@ class CorrelationResults(object): for trait, symbol in self.trait_symbol_dict.iteritems(): if symbol and symbol.lower() in corr_result_tissue_vals_dict: this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower()] - #print("this_trait_tissue_values: ", pf(this_trait_tissue_values)) result = correlation_functions.cal_zero_order_corr_for_tiss(primary_trait_tissue_values, this_trait_tissue_values, @@ -442,7 +320,6 @@ class CorrelationResults(object): return tissue_corr_data - def do_lit_correlation_for_trait_list(self): input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), self.this_trait.geneid) @@ -581,234 +458,6 @@ class CorrelationResults(object): if num_overlap > 5: self.correlation_data[trait] = [sample_r, sample_p, num_overlap] - - """ - correlations = [] - - #XZ: Use the fast method only for probeset dataset, and this dataset must have been created. - #XZ: Otherwise, use original method - #print("Entering correlation") - - #db_filename = self.getFileName(target_db_name=self.target_db_name) - # - #cache_available = db_filename in os.listdir(webqtlConfig.GENERATED_TEXT_DIR) - - # If the cache file exists, do a cached correlation for probeset data - if self.dataset.type == "ProbeSet": -# if self.method in [METHOD_SAMPLE_PEARSON, METHOD_SAMPLE_RANK] and cache_available: -# traits = do_parallel_correlation() -# -# else: - - traits = self.get_traits(self.vals) - - for trait in traits: - trait.calculate_correlation(vals, self.method) - - self.record_count = len(traits) #ZS: This isn't a good way to get this value, so I need to change it later - - #XZ, 3/31/2010: Theoretically, we should create one function 'comTissueCorr' - #to compare each trait by their tissue corr p values. - #But because the tissue corr p values are generated by permutation test, - #the top ones always have p value 0. So comparing p values actually does nothing. - #In addition, for the tissue data in our database, the N is always the same. - #So it's safe to compare with tissue corr statistic value. - #That's the same as literature corr. - #if self.method in [METHOD_LIT, METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] and self.gene_id: - # traits.sort(webqtlUtil.cmpLitCorr) - #else: - #if self.method in TISSUE_METHODS: - # sort(traits, key=lambda A: math.fabs(A.tissue_corr)) - #elif self.method == METHOD_LIT: - # traits.sort(traits, key=lambda A: math.fabs(A.lit_corr)) - #else: - traits = sortTraitCorrelations(traits, self.method) - - # Strip to the top N correlations - traits = traits[:min(self.returnNumber, len(traits))] - - addLiteratureCorr = False - addTissueCorr = False - - trait_list = [] - for trait in traits: - db_trait = webqtlTrait(db=self.db, name=trait.name, cursor=self.cursor) - db_trait.retrieveInfo( QTL='Yes' ) - - db_trait.Name = trait.name - db_trait.corr = trait.correlation - db_trait.nOverlap = trait.overlap - db_trait.corrPValue = trait.p_value - - # NL, 07/19/2010 - # js function changed, add a new parameter rankOrder for js function 'showTissueCorrPlot' - db_trait.RANK_ORDER = self.RANK_ORDERS[self.method] - - #XZ, 26/09/2008: Method is 4 or 5. Have fetched tissue corr, but no literature correlation yet. - if self.method in TISSUE_METHODS: - db_trait.tissueCorr = trait.tissue_corr - db_trait.tissuePValue = trait.p_tissue - addTissueCorr = True - - - #XZ, 26/09/2008: Method is 3, Have fetched literature corr, but no tissue corr yet. - elif self.method == METHOD_LIT: - db_trait.LCorr = trait.lit_corr - db_trait.mouse_geneid = self.translateToMouseGeneID(self.species, db_trait.geneid) - addLiteratureCorr = True - - #XZ, 26/09/2008: Method is 1 or 2. Have NOT fetched literature corr and tissue corr yet. - # Phenotype data will not have geneid, and neither will some probes - # we need to handle this because we will get an attribute error - else: - if self.input_trait_mouse_gene_id and self.db.type=="ProbeSet": - addLiteratureCorr = True - if self.trait_symbol and self.db.type=="ProbeSet": - addTissueCorr = True - - trait_list.append(db_trait) - - if addLiteratureCorr: - trait_list = self.getLiteratureCorrelationByList(self.input_trait_mouse_gene_id, - self.species, trait_list) - if addTissueCorr: - trait_list = self.getTissueCorrelationByList( - primaryTraitSymbol = self.trait_symbol, - traitList = trait_list, - TissueProbeSetFreezeId = TISSUE_MOUSE_DB, - method=self.method) - - return trait_list - """ - - - def do_tissue_corr_for_all_traits_2(self): - """Comments Possibly Out of Date!!!!! - - Uses get_temp_tissue_corr_table to generate table of tissue correlations - - This function then gathers that data and pairs it with the TraitID string. - Takes as its arguments a formdata instance, and a dataset instance. - Returns a dictionary of 'TraitID':(tissueCorr, tissuePValue) - for the requested correlation - - Used when the user selects the tissue correlation method; i.e. not for the - column that is appended to all probeset trait correlation tables - - """ - - # table name string - temp_table = self.get_temp_tissue_corr_table(tissue_probesetfreeze_id=TISSUE_MOUSE_DB, - method=method) - - query = """SELECT ProbeSet.Name, {}.Correlation, {}.PValue - FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) - LEFT JOIN {} ON {}.Symbol=ProbeSet.Symbol - WHERE ProbeSetFreeze.Name = '{}' - and ProbeSetFreeze.Id=ProbeSetXRef.ProbeSetFreezeId - and ProbeSet.Id = ProbeSetXRef.ProbeSetId - and ProbeSet.Symbol IS NOT NULL - and {}.Correlation IS NOT NULL""".format(dataset.mescape( - temp_table, temp_table, temp_table, temp_table, - self.dataset.name, temp_table)) - - results = g.db.execute(query).fetchall() - - tissue_corr_dict = {} - - for entry in results: - trait_name, tissue_corr, tissue_pvalue = entry - tissue_corr_dict[trait_name] = (tissue_corr, tissue_pvalue) - #symbolList, - #geneIdDict, - #dataIdDict, - #ChrDict, - #MbDict, - #descDict, - #pTargetDescDict = getTissueProbeSetXRefInfo( - # GeneNameLst=GeneNameLst,TissueProbeSetFreezeId=TissueProbeSetFreezeId) - - g.db.execute('DROP TEMPORARY TABLE {}'.format(escape(temp_table))) - - return tissue_corr_dict - - - #XZ, 09/23/2008: In tissue correlation tables, there is no record of GeneId1 == GeneId2 - #XZ, 09/24/2008: Note that the correlation value can be negative. - def get_temp_tissue_corr_table(self, - tissue_probesetfreeze_id=0, - method="", - return_number=0): - - - def cmp_tisscorr_absolute_value(A, B): - try: - if abs(A[1]) < abs(B[1]): return 1 - elif abs(A[1]) == abs(B[1]): - return 0 - else: return -1 - except: - return 0 - - symbol_corr_dict, symbol_pvalue_dict = self.calculate_corr_for_all_tissues( - tissue_dataset_id=TISSUE_MOUSE_DB) - - symbol_corr_list = symbol_corr_dict.items() - - symbol_corr_list.sort(cmp_tisscorr_absolute_value) - symbol_corr_list = symbol_corr_list[0 : 2*return_number] - - tmp_table_name = webqtlUtil.genRandStr(prefix="TOPTISSUE") - - q1 = 'CREATE TEMPORARY TABLE %s (Symbol varchar(100) PRIMARY KEY, Correlation float, PValue float)' % tmp_table_name - self.cursor.execute(q1) - - for one_pair in symbol_corr_list: - one_symbol = one_pair[0] - one_corr = one_pair[1] - one_p_value = symbol_pvalue_dict[one_symbol] - - self.cursor.execute( "INSERT INTO %s (Symbol, Correlation, PValue) VALUES ('%s',%f,%f)" % (tmpTableName, one_symbol, float(one_corr), float(one_p_value)) ) - - return tmp_table_name - - - def calculate_corr_for_all_tissues(self, tissue_dataset_id=None): - - symbol_corr_dict = {} - symbol_pvalue_dict = {} - - primary_trait_symbol_value_dict = correlation_functions.make_gene_tissue_value_dict( - GeneNameLst=[self.this_trait.symbol], - TissueProbeSetFreezeId=tissue_dataset_id) - primary_trait_value = primary_trait_symbol_value_dict.values()[0] - - symbol_value_dict = correlation_functions.make_gene_tissue_value_dict( - gene_name_list=[], - tissue_dataset_id=tissue_dataset_id) - - symbol_corr_dict, symbol_pvalue_dict = correlation_functions.batch_cal_tissue_corr( - primaryTraitValue, - SymbolValueDict, - method=self.corr_method) - #else: - # symbol_corr_dict, symbol_pvalue_dict = correlation_functions.batch_cal_tissue_corr( - # primaryTraitValue, - # SymbolValueDict) - - return (symbolCorrDict, symbolPvalueDict) - - ##XZ, 12/16/2008: the input geneid is of mouse type - #def checkSymbolForTissueCorr(self, tissueProbeSetFreezeId=0, symbol=""): - # q = "SELECT 1 FROM TissueProbeSetXRef WHERE TissueProbeSetFreezeId=%s and Symbol='%s' LIMIT 1" % (tissueProbeSetFreezeId,symbol) - # self.cursor.execute(q) - # try: - # x = self.cursor.fetchone() - # if x: return True - # else: raise - # except: return False - - def process_samples(self, start_vars, sample_names, excluded_samples=None): if not excluded_samples: excluded_samples = () @@ -821,581 +470,6 @@ class CorrelationResults(object): if not value.strip().lower() == 'x': self.sample_data[str(sample)] = float(value) - ##XZ, 12/16/2008: the input geneid is of mouse type - #def checkForLitInfo(self,geneId): - # q = 'SELECT 1 FROM LCorrRamin3 WHERE GeneId1=%s LIMIT 1' % geneId - # self.cursor.execute(q) - # try: - # x = self.cursor.fetchone() - # if x: return True - # else: raise - # except: return False - - - - def fetchAllDatabaseData(self, species, GeneId, GeneSymbol, strains, db, method, returnNumber, tissueProbeSetFreezeId): - - StrainIds = [] - for item in strains: - self.cursor.execute('''SELECT Strain.Id FROM Strain, Species WHERE Strain.Name="%s" and Strain.SpeciesId=Species.Id and Species.name = "%s" ''' % (item, species)) - Id = self.cursor.fetchone()[0] - StrainIds.append('%d' % Id) - - # break it into smaller chunks so we don't overload the MySql server - nnn = len(StrainIds) / 25 - if len(StrainIds) % 25: - nnn += 1 - oridata = [] - - #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId - tempTable = None - if GeneId and db.type == "ProbeSet": - if method == "3": - tempTable = self.getTempLiteratureTable(species=species, input_species_geneid=GeneId, returnNumber=returnNumber) - - if method == "4" or method == "5": - tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol, TissueProbeSetFreezeId=TISSUE_MOUSE_DB, method=method, returnNumber=returnNumber) - - for step in range(nnn): - temp = [] - StrainIdstep = StrainIds[step*25:min(len(StrainIds), (step+1)*25)] - for item in StrainIdstep: temp.append('T%s.value' % item) - - if db.type == "Publish": - query = "SELECT PublishXRef.Id, " - dataStartPos = 1 - query += string.join(temp,', ') - query += ' FROM (PublishXRef, PublishFreeze)' - #XZ, 03/04/2009: Xiaodong changed Data to PublishData - for item in StrainIdstep: - query += 'left join PublishData as T%s on T%s.Id = PublishXRef.DataId and T%s.StrainId=%s\n' %(item,item,item,item) - query += "WHERE PublishXRef.InbredSetId = PublishFreeze.InbredSetId and PublishFreeze.Name = '%s'" % (db.name, ) - #XZ, 09/20/2008: extract literature correlation value together with gene expression values. - #XZ, 09/20/2008: notice the difference between the code in next block. - #elif tempTable: - # # we can get a little performance out of selecting our LitCorr here - # # but also we need to do this because we are unconcerned with probes that have no geneId associated with them - # # as we would not have litCorr data. - # - # if method == "3": - # query = "SELECT %s.Name, %s.value," % (db.type,tempTable) - # dataStartPos = 2 - # if method == "4" or method == "5": - # query = "SELECT %s.Name, %s.Correlation, %s.PValue," % (db.type,tempTable, tempTable) - # dataStartPos = 3 - # - # query += string.join(temp,', ') - # query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) - # if method == "3": - # query += ' LEFT JOIN %s ON %s.GeneId2=ProbeSet.GeneId ' % (tempTable,tempTable) - # if method == "4" or method == "5": - # query += ' LEFT JOIN %s ON %s.Symbol=ProbeSet.Symbol ' % (tempTable,tempTable) - # #XZ, 03/04/2009: Xiaodong changed Data to %sData and changed parameters from %(item,item, db.type,item,item) to %(db.type, item,item, db.type,item,item) - # for item in StrainIdstep: - # query += 'left join %sData as T%s on T%s.Id = %sXRef.DataId and T%s.StrainId=%s\n' %(db.type, item,item, db.type,item,item) - # - # if method == "3": - # query += "WHERE ProbeSet.GeneId IS NOT NULL AND %s.value IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable,db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) - # if method == "4" or method == "5": - # query += "WHERE ProbeSet.Symbol IS NOT NULL AND %s.Correlation IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable,db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) - else: - query = "SELECT %s.Name," % db.type - dataStartPos = 1 - query += string.join(temp,', ') - query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) - #XZ, 03/04/2009: Xiaodong changed Data to %sData and changed parameters from %(item,item, db.type,item,item) to %(db.type, item,item, db.type,item,item) - for item in StrainIdstep: - query += 'left join %sData as T%s on T%s.Id = %sXRef.DataId and T%s.StrainId=%s\n' %(db.type, item,item, db.type,item,item) - query += "WHERE %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) - - self.cursor.execute(query) - results = self.cursor.fetchall() - oridata.append(results) - - datasize = len(oridata[0]) - traits = [] - # put all of the separate data together into a huge list of lists - for j in range(datasize): - traitdata = list(oridata[0][j]) - for i in range(1,nnn): - traitdata += list(oridata[i][j][dataStartPos:]) - - trait = Trait(traitdata[0], traitdata[dataStartPos:]) - - if method == METHOD_LIT: - trait.lit_corr = traitdata[1] - - if method in TISSUE_METHODS: - trait.tissue_corr = traitdata[1] - trait.p_tissue = traitdata[2] - - traits.append(trait) - - if tempTable: - self.cursor.execute( 'DROP TEMPORARY TABLE %s' % tempTable ) - - return traits - - - - - # XZ, 09/20/2008: This function creates TEMPORARY TABLE tmpTableName_2 and return its name. - # XZ, 09/20/2008: It stores top literature correlation values associated with the input geneId. - # XZ, 09/20/2008: Attention: In each row, the input geneId is always in column GeneId1. - #XZ, 12/16/2008: the input geneid can be of mouse, rat or human type - def getTempLiteratureTable(self, species, input_species_geneid, returnNumber): - # according to mysql the TEMPORARY TABLE name should not have to be unique because - # it is only available to the current connection. This program will be invoked via command line, but if it - # were to be invoked over mod_python this could cuase problems. mod_python will keep the connection alive - # in its executing threads ( i think) so there is a potential for the table not being dropped between users. - #XZ, 01/29/2009: To prevent the potential risk, I generate random table names and drop the tables after use them. - - - # the 'input_species_geneid' could be rat or human geneid, need to translate it to mouse geneid - translated_mouse_geneid = self.translateToMouseGeneID (species, input_species_geneid) - - tmpTableName_1 = webqtlUtil.genRandStr(prefix="LITERATURE") - - q1 = 'CREATE TEMPORARY TABLE %s (GeneId1 int(12) unsigned, GeneId2 int(12) unsigned PRIMARY KEY, value double)' % tmpTableName_1 - q2 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId1,GeneId2,value FROM LCorrRamin3 WHERE GeneId1=%s' % (tmpTableName_1, translated_mouse_geneid) - q3 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId2,GeneId1,value FROM LCorrRamin3 WHERE GeneId2=%s AND GeneId1!=%s' % (tmpTableName_1, translated_mouse_geneid,translated_mouse_geneid) - for x in [q1,q2,q3]: self.cursor.execute(x) - - #XZ, 09/23/2008: Just use the top records insteard of using all records - tmpTableName_2 = webqtlUtil.genRandStr(prefix="TOPLITERATURE") - - q1 = 'CREATE TEMPORARY TABLE %s (GeneId1 int(12) unsigned, GeneId2 int(12) unsigned PRIMARY KEY, value double)' % tmpTableName_2 - self.cursor.execute(q1) - q2 = 'SELECT GeneId1, GeneId2, value FROM %s ORDER BY value DESC' % tmpTableName_1 - self.cursor.execute(q2) - result = self.cursor.fetchall() - - counter = 0 #this is to count how many records being inserted into table - for one_row in result: - mouse_geneid1, mouse_geneid2, lit_corr_alue = one_row - - #mouse_geneid1 has been tested before, now should test if mouse_geneid2 has corresponding geneid in other species - translated_species_geneid = 0 - if species == 'mouse': - translated_species_geneid = mouse_geneid2 - elif species == 'rat': - self.cursor.execute( "SELECT rat FROM GeneIDXRef WHERE mouse=%d" % int(mouse_geneid2) ) - record = self.cursor.fetchone() - if record: - translated_species_geneid = record[0] - elif species == 'human': - self.cursor.execute( "SELECT human FROM GeneIDXRef WHERE mouse=%d" % int(mouse_geneid2) ) - record = self.cursor.fetchone() - if record: - translated_species_geneid = record[0] - - if translated_species_geneid: - self.cursor.execute( 'INSERT INTO %s (GeneId1, GeneId2, value) VALUES (%d,%d,%f)' % (tmpTableName_2, int(input_species_geneid),int(translated_species_geneid), float(lit_corr_alue)) ) - counter = counter + 1 - - #pay attention to the number - if (counter > 2*returnNumber): - break - - self.cursor.execute('DROP TEMPORARY TABLE %s' % tmpTableName_1) - - return tmpTableName_2 - - - - #XZ, 01/09/2009: This function was created by David Crowell. Xiaodong cleaned up and modified it. - def fetchLitCorrelations(self, species, GeneId, db, returnNumber): ### Used to generate Lit Correlations when calculations are done from text file. dcrowell August 2008 - """Uses getTempLiteratureTable to generate table of literatire correlations. This function then gathers that data and - pairs it with the TraitID string. Takes as its arguments a formdata instance, and a database instance. - Returns a dictionary of 'TraitID':'LitCorr' for the requested correlation""" - - tempTable = self.getTempLiteratureTable(species=species, input_species_geneid=GeneId, returnNumber=returnNumber) - - query = "SELECT %s.Name, %s.value" % (db.type,tempTable) - query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) - query += ' LEFT JOIN %s ON %s.GeneId2=ProbeSet.GeneId ' % (tempTable,tempTable) - query += "WHERE ProbeSet.GeneId IS NOT NULL AND %s.value IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable, db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) - - self.cursor.execute(query) - results = self.cursor.fetchall() - - litCorrDict = {} - - for entry in results: - traitName,litcorr = entry - litCorrDict[traitName] = litcorr - - self.cursor.execute('DROP TEMPORARY TABLE %s' % tempTable) - - return litCorrDict - - - def get_traits(self, vals): - - #Todo: Redo cached stuff using memcached - if False: - lit_corrs = {} - tissue_corrs = {} - use_lit = False - if self.method == METHOD_LIT: - lit_corrs = self.fetchLitCorrelations(species=self.species, GeneId=self.gene_id, db=self.db, returnNumber=self.returnNumber) - use_lit = True - - use_tissue_corr = False - if self.method in TISSUE_METHODS: - tissue_corrs = self.fetch_tissue_correlations(method=self.method, return_number = self.return_number) - use_tissue_corr = True - - DatabaseFileName = self.getFileName( target_db_name=self.target_db_name ) - datasetFile = open(webqtlConfig.CACHEDIR+DatabaseFileName,'r') - - #XZ, 01/08/2009: read the first line - line = datasetFile.readline() - cached_sample_names = webqtlUtil.readLineCSV(line)[1:] - - #XZ, 01/08/2009: This step is critical. It is necessary for this new method. - #XZ: The original function fetchAllDatabaseData uses all strains stored in variable _strains to - #XZ: retrieve the values of each strain from database in real time. - #XZ: The new method uses all strains stored in variable dataset_strains to create a new variable - #XZ: _newvals. _newvals has the same length as dataset_strains. The items in _newvals is in - #XZ: the same order of items in dataset_strains. The value of each item in _newvals is either - #XZ: the value of correspinding strain in _vals or 'None'. - new_vals = [] - for name in cached_sample_names: - if name in self.sample_names: - new_vals.append(float(vals[self.sample_names.index(name)])) - else: - new_vals.append('None') - - nnCorr = len(new_vals) - - #XZ, 01/14/2009: If literature corr or tissue corr is selected, - #XZ: there is no need to use parallel computing. - - traits = [] - data_start = 1 - for line in datasetFile: - raw_trait = webqtlUtil.readLineCSV(line) - trait = Trait.from_csv(raw_trait, data_start) - trait.lit_corr = lit_corrs.get(trait.name) - trait.tissue_corr, trait.p_tissue = tissue_corrs.get(trait.name, (None, None)) - traits.append(trait) - - return traits, new_vals - - else: - traits = self.fetchAllDatabaseData(species=self.dataset.species, - GeneId=self.gene_id, - GeneSymbol=self.trait.symbol, - strains=self.sample_names, - db=self.db, - method=self.method, - returnNumber=self.returnNumber, - tissueProbeSetFreezeId= self.tissue_probeset_freeze_id) - totalTraits = len(traits) #XZ, 09/18/2008: total trait number - - return traits - - def calculate_corr_for_all_tissues(self, tissue_dataset_id=None): - - symbol_corr_dict = {} - symbol_pvalue_dict = {} - - primary_trait_symbol_value_dict = correlation_functions.make_gene_tissue_value_dict( - GeneNameLst=[self.this_trait.symbol], - TissueProbeSetFreezeId=tissue_dataset_id) - primary_trait_value = primary_trait_symbol_value_dict.values()[0] - - symbol_value_dict = correlation_functions.make_gene_tissue_value_dict( - gene_name_list=[], - tissue_dataset_id=tissue_dataset_id) - - symbol_corr_dict, symbol_pvalue_dict = correlation_functions.batch_cal_tissue_corr( - primaryTraitValue, - SymbolValueDict, - method=self.corr_method) - #else: - # symbol_corr_dict, symbol_pvalue_dict = correlation_functions.batch_cal_tissue_corr( - # primaryTraitValue, - # SymbolValueDict) - - return (symbolCorrDict, symbolPvalueDict) - - - def correlate(self): - self.correlation_data = collections.defaultdict(list) - for trait, values in self.target_dataset.trait_data.iteritems(): - values_1 = [] - values_2 = [] - for index,sample in enumerate(self.target_dataset.samplelist): - target_value = values[index] - if sample in self.sample_data.keys(): - this_value = self.sample_data[sample] - values_1.append(this_value) - values_2.append(target_value) - correlation = calCorrelation(values_1, values_2) - self.correlation_data[trait] = correlation - - def getFileName(self, target_db_name): ### dcrowell August 2008 - """Returns the name of the reference database file with which correlations are calculated. - Takes argument cursor which is a cursor object of any instance of a subclass of templatePage - Used by correlationPage""" - - dataset_id = str(self.target_dataset.id) - dataset_fullname = self.target_dataset.fullname.replace(' ','_') - dataset_fullname = dataset_fullname.replace('/','_') - - FileName = 'ProbeSetFreezeId_' + dataset_id + '_FullName_' + dataset_fullname + '.txt' - - return FileName - - def do_parallel_correlation(self, db_filename, num_overlap): - - #XZ, 01/14/2009: This method is for parallel computing only. - #XZ: It is supposed to be called when "Genetic Correlation, Pearson's r" (method 1) - #XZ: or "Genetic Correlation, Spearman's rho" (method 2) is selected - def compute_corr(input_nnCorr, input_trait, input_list, corr_method): - - import math - import reaper - - def cmpOrder2(A,B): - try: - if A[-1] < B[-1]: - return -1 - elif A[-1] == B[-1]: - return 0 - else: - return 1 - except: - return 0 - - def calCorrelation(dbdata,userdata,N): - X = [] - Y = [] - for i in range(N): - if (dbdata[i] != None and userdata[i] != None) and (dbdata[i] != "None" and userdata[i] != "None"): - X.append(float(dbdata[i])) - Y.append(float(userdata[i])) - NN = len(X) - if NN <6: - return (0.0,NN) - sx = reduce(lambda x,y:x+y,X,0.0) - sy = reduce(lambda x,y:x+y,Y,0.0) - meanx = sx/NN - meany = sy/NN - xyd = 0.0 - sxd = 0.0 - syd = 0.0 - for i in range(NN): - xyd += (X[i] - meanx)*(Y[i]-meany) - sxd += (X[i] - meanx)*(X[i] - meanx) - syd += (Y[i] - meany)*(Y[i] - meany) - try: - corr = xyd/(math.sqrt(sxd)*math.sqrt(syd)) - except: - corr = 0 - return (corr,NN) - - def calCorrelationRank(xVals,yVals,N): - """ - Calculated Spearman Ranked Correlation. The algorithm works - by setting all tied ranks to the average of those ranks (for - example, if ranks 5-10 all have the same value, each will be set - to rank 7.5). - """ - - XX = [] - YY = [] - j = 0 - - for i in range(len(xVals)): - if (xVals[i]!= None and yVals[i]!= None) and (xVals[i] != "None" and yVals[i] != "None"): - XX.append((j,float(xVals[i]))) - YY.append((j,float(yVals[i]))) - j = j+1 - - NN = len(XX) - if NN <6: - return (0.0,NN) - XX.sort(cmpOrder2) - YY.sort(cmpOrder2) - X = [0]*NN - Y = [0]*NN - - j = 1 - rank = 0.0 - t = 0.0 - sx = 0.0 - - while j < NN: - - if XX[j][1] != XX[j-1][1]: - X[XX[j-1][0]] = j - j = j+1 - - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (XX[jt][1] != XX[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - X[XX[ji][0]] = rank - t = jt-j - sx = sx + (t*t*t-t) - if (jt == NN-1): - if (XX[jt][1] == XX[j-1][1]): - X[XX[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if X[XX[NN-1][0]] == 0: - X[XX[NN-1][0]] = NN - - j = 1 - rank = 0.0 - t = 0.0 - sy = 0.0 - - while j < NN: - - if YY[j][1] != YY[j-1][1]: - Y[YY[j-1][0]] = j - j = j+1 - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (YY[jt][1] != YY[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - Y[YY[ji][0]] = rank - t = jt - j - sy = sy + (t*t*t-t) - if (jt == NN-1): - if (YY[jt][1] == YY[j-1][1]): - Y[YY[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if Y[YY[NN-1][0]] == 0: - Y[YY[NN-1][0]] = NN - - D = 0.0 - - for i in range(NN): - D += (X[i]-Y[i])*(X[i]-Y[i]) - - fac = (1.0 -sx/(NN*NN*NN-NN))*(1.0-sy/(NN*NN*NN-NN)) - - return ((1-(6.0/(NN*NN*NN-NN))*(D+(sx+sy)/12.0))/math.sqrt(fac),NN) - - # allcorrelations = [] - - correlation_data = {} - for i, line in enumerate(input_list): - if i == 0: - continue - tokens = line.split('","') - tokens[-1] = tokens[-1][:-2] #remove the last " - tokens[0] = tokens[0][1:] #remove the first " - - traitdataName = tokens[0] - database_trait = tokens[1:] - - #print("database_trait:", database_trait) - - #ZS: 2015 could add biweight correlation, see http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3465711/ - # if corr_method == 'pearson': - # sample_r, sample_p = scipy.stats.pearsonr(input_trait, database_trait) - # else: - # sample_r, sample_p = scipy.stats.spearmanr(input_trait, database_trait) - - if corr_method == "pearson": #XZ: Pearson's r - sample_r, nOverlap = calCorrelation(input_trait, database_trait, input_nnCorr) - else: #XZ: Spearman's rho - sample_r, nOverlap = calCorrelationRank(input_trait, database_trait, input_nnCorr) - - #XZ: calculate corrPValue - if nOverlap < 3: - sample_p = 1.0 - else: - if abs(sample_r) >= 1.0: - sample_p = 0.0 - else: - z_value = 0.5*math.log((1.0+sample_r)/(1.0-sample_r)) - z_value = z_value*math.sqrt(nOverlap-3) - sample_p = 2.0*(1.0 - reaper.normp(abs(z_value))) - - correlation_data[traitdataName] = [sample_r, sample_p, nOverlap] - - # traitinfo = [traitdataName, sample_r, nOverlap] - # allcorrelations.append(traitinfo) - - return correlation_data - # return allcorrelations - - - datasetFile = open(webqtlConfig.GENERATED_TEXT_DIR+db_filename,'r') - - print("Invoking parallel computing") - input_line_list = datasetFile.readlines() - print("Read lines from the file") - all_line_number = len(input_line_list) - - step = 1000 - job_number = math.ceil( float(all_line_number)/step ) - - print("JOB NUMBER", job_number) - - job_input_lists = [] - - print("Configuring jobs") - - for job_index in range( int(job_number) ): - starti = job_index*step - endi = min((job_index+1)*step, all_line_number) - - one_job_input_list = [] - - for i in range( starti, endi ): - one_job_input_list.append( input_line_list[i] ) - - job_input_lists.append( one_job_input_list ) - - print("Creating pp servers") - - ppservers = () - # Creates jobserver with automatically detected number of workers - job_server = pp.Server(ppservers=ppservers) - - print("Done creating servers") - - jobs = [] - results = [] - - print("Starting parallel computation, submitting jobs") - for one_job_input_list in job_input_lists: #pay attention to modules from outside - jobs.append( job_server.submit(func=compute_corr, args=(num_overlap, self.this_trait_vals, one_job_input_list, self.corr_method), depfuncs=(), modules=("webqtlUtil",)) ) - print("Done submitting jobs") - - for one_job in jobs: - one_result = one_job() - self.correlation_data.update(one_result) - # one_result = one_job() - # results.append( one_result ) - - #print("CORRELATION DATA:", self.correlation_data) - - # print("Acquiring results") - - # for one_result in results: - # for one_traitinfo in one_result: - # allcorrelations.append( one_traitinfo ) - def generate_corr_json(corr_results, this_trait, dataset, target_dataset): results_list = [] for i, trait in enumerate(corr_results): diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 077386a3..4bb4d65d 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -23,7 +23,6 @@ from __future__ import absolute_import, print_function, division import sys # sys.path.append(".") Never do this in a webserver! -import gc import string import cPickle import os diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 9515d23a..6fda02fd 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -2,7 +2,6 @@ # Author / Maintainer: Danny Arends <Danny.Arends@gmail.com> import sys from numpy import * -import scipy as sp # SciPy import rpy2.robjects as ro # R Objects import rpy2.rinterface as ri @@ -24,60 +23,38 @@ from utility import helper_functions from utility.tools import locate from rpy2.robjects.packages import importr -utils = importr("utils") + +import utility.logger +logger = utility.logger.getLogger(__name__ ) ## Get pointers to some common R functions r_library = ro.r["library"] # Map the library function r_options = ro.r["options"] # Map the options function -r_read_csv = ro.r["read.csv"] # Map the read.csv function -r_dim = ro.r["dim"] # Map the dim function -r_c = ro.r["c"] # Map the c function r_t = ro.r["t"] # Map the t function -r_cat = ro.r["cat"] # Map the cat function -r_paste = ro.r["paste"] # Map the paste function -r_unlist = ro.r["unlist"] # Map the unlist function -r_head = ro.r["head"] # Map the unlist function -r_unique = ro.r["unique"] # Map the unique function -r_length = ro.r["length"] # Map the length function r_unlist = ro.r["unlist"] # Map the unlist function r_list = ro.r.list # Map the list function -r_matrix = ro.r.matrix # Map the matrix function -r_seq = ro.r["seq"] # Map the seq function -r_table = ro.r["table"] # Map the table function -r_names = ro.r["names"] # Map the names function -r_sink = ro.r["sink"] # Map the sink function -r_is_NA = ro.r["is.na"] # Map the is.na function -r_file = ro.r["file"] # Map the file function r_png = ro.r["png"] # Map the png function for plotting r_dev_off = ro.r["dev.off"] # Map the dev.off function -r_save_image = ro.r["save.image"] # Map the save.image function -r_class = ro.r["class"] # Map the class function -r_save = ro.r["save"] # Map the save function r_write_table = ro.r["write.table"] # Map the write.table function -r_read_table = ro.r["read.table"] # Map the read.table function -r_as_data_frame = ro.r["as.data.frame"] # Map the write.table function r_data_frame = ro.r["data.frame"] # Map the write.table function r_as_numeric = ro.r["as.numeric"] # Map the write.table function class CTL(object): def __init__(self): - print("Initialization of CTL") + logger.info("Initialization of CTL") #log = r_file("/tmp/genenetwork_ctl.log", open = "wt") - #r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file + #r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file #r_sink(log, type = "message") - r_library("ctl") # Load CTL - Should only be done once, since it is quite expensive + r_library("ctl") # Load CTL - Should only be done once, since it is quite expensive r_options(stringsAsFactors = False) - print("Initialization of CTL done, package loaded in R session") + logger.info("Initialization of CTL done, package loaded in R session") self.r_CTLscan = ro.r["CTLscan"] # Map the CTLscan function self.r_CTLsignificant = ro.r["CTLsignificant"] # Map the CTLsignificant function self.r_lineplot = ro.r["ctl.lineplot"] # Map the ctl.lineplot function - self.r_CTLsignificant = ro.r["CTLsignificant"] # Map the CTLsignificant function - self.r_CTLnetwork = ro.r["CTLnetwork"] # Map the CTLnetwork function - self.r_CTLprofiles = ro.r["CTLprofiles"] # Map the CTLprofiles function self.r_plotCTLobject = ro.r["plot.CTLobject"] # Map the CTLsignificant function self.nodes_list = [] self.edges_list = [] - print("Obtained pointers to CTL functions") + logger.info("Obtained pointers to CTL functions") def addNode(self, gt): node_dict = { 'data' : {'id' : str(gt.name) + ":" + str(gt.dataset.name), @@ -100,20 +77,20 @@ class CTL(object): self.edges_list.append(edge_dict) def run_analysis(self, requestform): - print("Starting CTL analysis on dataset") + logger.info("Starting CTL analysis on dataset") self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] self.trait_db_list = [x for x in self.trait_db_list if x] - print("strategy:", requestform.get("strategy")) + logger.debug("strategy:", requestform.get("strategy")) strategy = requestform.get("strategy") - print("nperm:", requestform.get("nperm")) + logger.debug("nperm:", requestform.get("nperm")) nperm = int(requestform.get("nperm")) - print("parametric:", requestform.get("parametric")) + logger.debug("parametric:", requestform.get("parametric")) parametric = bool(requestform.get("parametric")) - print("significance:", requestform.get("significance")) + logger.debug("significance:", requestform.get("significance")) significance = float(requestform.get("significance")) # Get the name of the .geno file belonging to the first phenotype @@ -123,7 +100,7 @@ class CTL(object): genofilelocation = locate(dataset.group.name + ".geno", "genotype") parser = genofile_parser.ConvertGenoFile(genofilelocation) parser.process_csv() - print(dataset.group) + logger.debug("dataset group: ", dataset.group) # Create a genotype matrix individuals = parser.individuals markers = [] @@ -133,14 +110,14 @@ class CTL(object): markers.append(marker["genotypes"]) genotypes = list(itertools.chain(*markers)) - print(len(genotypes) / len(individuals), "==", len(parser.markers)) + logger.debug(len(genotypes) / len(individuals), "==", len(parser.markers)) rGeno = r_t(ro.r.matrix(r_unlist(genotypes), nrow=len(markernames), ncol=len(individuals), dimnames = r_list(markernames, individuals), byrow=True)) # Create a phenotype matrix traits = [] for trait in self.trait_db_list: - print("retrieving data for", trait) + logger.debug("retrieving data for", trait) if trait != "": ts = trait.split(':') gt = TRAIT.GeneralTrait(name = ts[0], dataset_name = ts[1]) @@ -153,7 +130,7 @@ class CTL(object): rPheno = r_t(ro.r.matrix(r_as_numeric(r_unlist(traits)), nrow=len(self.trait_db_list), ncol=len(individuals), dimnames = r_list(self.trait_db_list, individuals), byrow=True)) - print(rPheno) + logger.debug(rPheno) # Use a data frame to store the objects rPheno = r_data_frame(rPheno, check_names = False) @@ -196,10 +173,9 @@ class CTL(object): sys.stdout.flush() # Create the interactive graph for cytoscape visualization (Nodes and Edges) - print(type(significant)) if not type(significant) == ri.RNULLType: for x in range(len(significant[0])): - print(significant[0][x], significant[1][x], significant[2][x]) # Debug to console + logger.debug(significant[0][x], significant[1][x], significant[2][x]) # Debug to console tsS = significant[0][x].split(':') # Source tsT = significant[2][x].split(':') # Target gtS = TRAIT.GeneralTrait(name = tsS[0], dataset_name = tsS[1]) # Retrieve Source info from the DB @@ -214,7 +190,6 @@ class CTL(object): self.elements = json.dumps(self.nodes_list + self.edges_list) def loadImage(self, path, name): - print("pre-loading imgage results:", self.results[path]) imgfile = open(self.results[path], 'rb') imgdata = imgfile.read() imgB64 = imgdata.encode("base64") @@ -229,7 +204,7 @@ class CTL(object): n = n + 1 def process_results(self, results): - print("Processing CTL output") + logger.info("Processing CTL output") template_vars = {} template_vars["results"] = self.results template_vars["elements"] = self.elements diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 17625474..c7dbc972 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -79,38 +79,6 @@ class DoSearch(object): else: return None -class QuickMrnaAssaySearch(DoSearch): - """A general search for mRNA assays""" - - DoSearch.search_types['quick_mrna_assay'] = "QuickMrnaAssaySearch" - - base_query = """SELECT ProbeSet.Name as ProbeSet_Name, - ProbeSet.Symbol as ProbeSet_Symbol, - ProbeSet.description as ProbeSet_Description, - ProbeSet.Chr_num as ProbeSet_Chr_Num, - ProbeSet.Mb as ProbeSet_Mb, - ProbeSet.name_num as ProbeSet_name_num - FROM ProbeSet """ - - header_fields = ['Index', - 'Record', - 'Symbol', - 'Location'] - - def run(self): - """Generates and runs a search for assays across all mRNA expression datasets""" - - logger.debug("Running ProbeSetSearch") - query = self.base_query + """WHERE (MATCH (ProbeSet.Name, - ProbeSet.description, - ProbeSet.symbol, - ProbeSet.alias) - AGAINST ('%s' IN BOOLEAN MODE)) - """ % (escape(self.search_term[0])) - - return self.execute(query) - - class MrnaAssaySearch(DoSearch): """A search within an expression dataset, including mRNA, protein, SNP, but not phenotype or metabolites""" @@ -311,54 +279,6 @@ class PhenotypeSearch(DoSearch): return self.execute(query) -class QuickPhenotypeSearch(PhenotypeSearch): - """A search across all phenotype datasets""" - - DoSearch.search_types['quick_phenotype'] = "QuickPhenotypeSearch" - - base_query = """SELECT Species.Name as Species_Name, - PublishFreeze.FullName as Dataset_Name, - PublishFreeze.Name, - PublishXRef.Id, - PublishFreeze.createtime as thistable, - Publication.PubMed_ID as Publication_PubMed_ID, - Phenotype.Post_publication_description as Phenotype_Name - FROM Phenotype, - PublishFreeze, - Publication, - PublishXRef, - InbredSet, - Species """ - - search_fields = ('Phenotype.Post_publication_description', - 'Phenotype.Pre_publication_description', - 'Phenotype.Pre_publication_abbreviation', - 'Phenotype.Post_publication_abbreviation', - 'Phenotype.Lab_code', - 'Publication.PubMed_ID', - 'Publication.Abstract', - 'Publication.Title', - 'Publication.Authors') - - def compile_final_query(self, where_clause = ''): - """Generates the final query string""" - - query = (self.base_query + - """WHERE %s - PublishXRef.PhenotypeId = Phenotype.Id and - PublishXRef.PublicationId = Publication.Id and - PublishXRef.InbredSetId = InbredSet.Id and - InbredSet.SpeciesId = Species.Id""" % where_clause) - - return query - - def run(self): - """Generates and runs a search across all phenotype datasets""" - - query = self.compile_final_query(where_clause = self.get_where_clause()) - - return self.execute(query) - class GenotypeSearch(DoSearch): """A search within a genotype dataset""" @@ -767,14 +687,6 @@ class MeanSearch(MrnaAssaySearch): return where_clause - def get_final_query(self): - self.where_clause = self.get_where_clause() - logger.debug("where_clause is:", pf(self.where_clause)) - - self.query = self.compile_final_query(where_clause = self.where_clause) - - return self.query - def run(self): self.where_clause = self.get_where_clause() logger.debug("where_clause is:", pf(self.where_clause)) @@ -948,7 +860,6 @@ if __name__ == "__main__": from base import webqtlConfig from base.data_set import create_dataset - from base.templatePage import templatePage from utility import webqtlUtil from db import webqtlDatabaseFunction @@ -961,32 +872,6 @@ if __name__ == "__main__": dataset_name = "HC_M2_0606_P" dataset = create_dataset(db_conn, dataset_name) - #cursor.execute(""" - # SELECT ProbeSet.Name as TNAME, 0 as thistable, - # ProbeSetXRef.Mean as TMEAN, ProbeSetXRef.LRS as TLRS, - # ProbeSetXRef.PVALUE as TPVALUE, ProbeSet.Chr_num as TCHR_NUM, - # ProbeSet.Mb as TMB, ProbeSet.Symbol as TSYMBOL, - # ProbeSet.name_num as TNAME_NUM - # FROM ProbeSetXRef, ProbeSet, Geno - # WHERE ProbeSetXRef.LRS > 99.0 and - # ABS(ProbeSet.Mb-Geno.Mb) < 5 and - # ProbeSetXRef.Locus = Geno.name and - # Geno.SpeciesId = 1 and - # ProbeSet.Chr = Geno.Chr and - # ProbeSet.Id = ProbeSetXRef.ProbeSetId and - # ProbeSetXRef.ProbeSetFreezeId = 112""") - - #logger.debug(pf(cursor.fetchall())) - #results = ProbeSetSearch("shh", None, dataset, cursor, db_conn).run() results = PvalueSearch(['0.005'], '<', dataset, cursor, db_conn).run() - #results = RifSearch("diabetes", dataset, cursor, db_conn).run() - #results = WikiSearch("nicotine", dataset, cursor, db_conn).run() - #results = CisLrsSearch(['99'], '>', dataset, cursor, db_conn).run() # cisLRS > 99 - #results = LrsSearch('99', '>', dataset, cursor, db_conn).run() - #results = TransLrsSearch(['9', '999', '10'], dataset, cursor, db_conn).run() - #results = PhenotypeSearch("brain", dataset, cursor, db_conn).run() - #results = GenotypeSearch("rs13475699", dataset, cursor, db_conn).run() - #results = GoSearch("0045202", dataset, cursor, db_conn).run() - - logger.debug("results are:", pf(results)) + db_conn.close() diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index f8fce929..ab4c0d7c 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -1,6 +1,5 @@ from __future__ import print_function, division -import operator import csv import xlsxwriter import StringIO diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index fe1e17d2..37eb46cb 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -5,6 +5,8 @@ from base.data_set import create_dataset from base.trait import GeneralTrait from db import webqtlDatabaseFunction +from base import webqtlConfig + from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string from utility.benchmark import Bench @@ -57,10 +59,29 @@ class GSearch(object): self.trait_list = [] with Bench("Creating trait objects"): for line in re: - dataset = create_dataset(line[3], "ProbeSet", get_samplelist=False) - trait_id = line[4] + this_trait = {} + this_trait['name'] = line[4] + this_trait['dataset'] = line[3] + this_trait['species'] = line[0] + this_trait['group'] = line[1] + this_trait['tissue'] = line[2] + this_trait['symbol'] = line[5] + this_trait['description'] = line[6] + this_trait['location_repr'] = 'N/A' + if (line[7] != "NULL" and line[7] != "") and (line[8] != 0): + this_trait['location_repr'] = 'Chr%s: %.6f' % (line[7], float(line[8])) + this_trait['mean'] = line[9] + this_trait['LRS_score_repr'] = "N/A" + if line[10] != "" and line[10] != None: + this_trait['LRS_score_repr'] = '%3.1f' % line[10] + this_trait['additive'] = "N/A" + if line[13] != "": + this_trait['additive'] = line[13] + + #dataset = create_dataset(line[3], "ProbeSet", get_samplelist=False) + #trait_id = line[4] #with Bench("Building trait object"): - this_trait = GeneralTrait(dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) + #this_trait = GeneralTrait(dataset=dataset, name=trait_id, get_qtl_info=False, get_sample_info=False) self.trait_list.append(this_trait) elif self.type == "phenotype": @@ -73,8 +94,8 @@ class GSearch(object): Phenotype.`Post_publication_description`, Publication.`Authors`, Publication.`Year`, + Publication.`PubMed_ID`, PublishXRef.`LRS`, - PublishXRef.`Locus`, PublishXRef.`additive` FROM Species,InbredSet,PublishFreeze,PublishXRef,Phenotype,Publication WHERE PublishXRef.`InbredSetId`=InbredSet.`Id` @@ -100,7 +121,30 @@ class GSearch(object): self.trait_list = [] with Bench("Creating trait objects"): for line in re: - dataset = create_dataset(line[2], "Publish") - trait_id = line[3] - this_trait = GeneralTrait(dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) + this_trait = {} + this_trait['name'] = line[3] + this_trait['dataset'] = line[2] + this_trait['species'] = line[0] + this_trait['group'] = line[1] + this_trait['description'] = line[4] + this_trait['authors'] = line[5] + this_trait['year'] = line[6] + if this_trait['year'].isdigit(): + this_trait['pubmed_text'] = this_trait['year'] + else: + this_trait['pubmed_text'] = "N/A" + if line[7] != "" and line[7] != None: + this_trait['pubmed_link'] = webqtlConfig.PUBMEDLINK_URL % line[7] + else: + this_trait['pubmed_link'] = "N/A" + this_trait['LRS_score_repr'] = "N/A" + if line[8] != "" and line[8] != None: + this_trait['LRS_score_repr'] = '%3.1f' % line[8] + this_trait['additive'] = "N/A" + if line[9] != "": + this_trait['additive'] = line[9] + + #dataset = create_dataset(line[2], "Publish") + #trait_id = line[3] + #this_trait = GeneralTrait(dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) self.trait_list.append(this_trait) diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 56ff11cd..af75d441 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -3,7 +3,6 @@ from __future__ import absolute_import, print_function, division import sys # sys.path.append(".") Never in a running webserver -import gc import string import cPickle import os @@ -16,7 +15,6 @@ import resource import scipy import numpy as np -from scipy import linalg from pprint import pformat as pf @@ -82,21 +80,16 @@ class Heatmap(object): this_trait = trait_db[0] this_sample_data = this_trait.data - #self.sample_data[this_trait.name] = [] this_trait_vals = [] for sample in self.all_sample_list: if sample in this_sample_data: this_trait_vals.append(this_sample_data[sample].value) - #self.sample_data[this_trait.name].append(this_sample_data[sample].value) else: this_trait_vals.append('') - #self.sample_data[this_trait.name].append('') self.sample_data.append(this_trait_vals) self.gen_reaper_results() - #self.gen_pylmm_results() - #chrnames = [] lodnames = [] chr_pos = [] pos = [] @@ -106,9 +99,6 @@ class Heatmap(object): lodnames.append(trait) for marker in self.dataset.group.markers.markers: - #if marker['chr'] not in chrnames: - # chr_ob = [marker['chr'], "filler"] - # chrnames.append(chr_ob) chr_pos.append(marker['chr']) pos.append(marker['Mb']) markernames.append(marker['name']) @@ -126,9 +116,6 @@ class Heatmap(object): json_data = self.json_data ) - print("self.js_data:", self.js_data) - - def gen_reaper_results(self): self.trait_results = {} for trait_db in self.trait_list: @@ -145,172 +132,14 @@ class Heatmap(object): trimmed_samples.append(samples[i]) trimmed_values.append(values[i]) - self.lrs_array = genotype.permutation(strains = trimmed_samples, - trait = trimmed_values, - nperm= self.num_permutations) - - #self.suggestive = self.lrs_array[int(self.num_permutations*0.37-1)] - #self.significant = self.lrs_array[int(self.num_permutations*0.95-1)] - reaper_results = genotype.regression(strains = trimmed_samples, trait = trimmed_values) - lrs_values = [float(qtl.lrs) for qtl in reaper_results] - print("lrs_values:", lrs_values) - #self.dataset.group.markers.add_pvalues(p_values) self.trait_results[this_trait.name] = [] for qtl in reaper_results: if qtl.additive > 0: self.trait_results[this_trait.name].append(-float(qtl.lrs)) else: - self.trait_results[this_trait.name].append(float(qtl.lrs)) - #for lrs in lrs_values: - # if - # self.trait_results[this_trait.name].append(lrs) - - - #this_db_samples = self.dataset.group.samplelist - #this_sample_data = this_trait.data - ##print("this_sample_data", this_sample_data) - #this_trait_vals = [] - #for index, sample in enumerate(this_db_samples): - # if sample in this_sample_data: - # sample_value = this_sample_data[sample].value - # this_trait_vals.append(sample_value) - # else: - # this_trait_vals.append("x") - - #pheno_vector = np.array([val == "x" and np.nan or float(val) for val in this_trait_vals]) - - #key = "pylmm:input:" + str(self.temp_uuid) - #print("key is:", pf(key)) - - #genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers] - - #no_val_samples = self.identify_empty_samples(this_trait_vals) - #trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples) - - #genotype_matrix = np.array(trimmed_genotype_data).T - - #print("genotype_matrix:", str(genotype_matrix.tolist())) - #print("pheno_vector:", str(pheno_vector.tolist())) - - #params = dict(pheno_vector = pheno_vector.tolist(), - # genotype_matrix = genotype_matrix.tolist(), - # restricted_max_likelihood = True, - # refit = False, - # temp_uuid = str(self.temp_uuid), - # - # # meta data - # timestamp = datetime.datetime.now().isoformat(), - # ) - # - #json_params = json.dumps(params) - ##print("json_params:", json_params) - #Redis.set(key, json_params) - #Redis.expire(key, 60*60) - #print("before printing command") - # - #command = 'python lmm.py --key {} --species {}'.format(key, - # "other") - #print("command is:", command) - #print("after printing command") - # - #os.system(command) - # - #json_results = Redis.blpop("pylmm:results:" + str(self.temp_uuid), 45*60) - - def gen_pylmm_results(self): - # This function is NOT used. If it is, we should use a shared function with marker_regression.py - self.trait_results = {} - for trait_db in self.trait_list: - this_trait = trait_db[0] - #this_db = trait_db[1] - self.dataset.group.get_markers() - - this_db_samples = self.dataset.group.samplelist - this_sample_data = this_trait.data - #print("this_sample_data", this_sample_data) - this_trait_vals = [] - for index, sample in enumerate(this_db_samples): - if sample in this_sample_data: - sample_value = this_sample_data[sample].value - this_trait_vals.append(sample_value) - else: - this_trait_vals.append("x") - - pheno_vector = np.array([val == "x" and np.nan or float(val) for val in this_trait_vals]) - - key = "pylmm:input:" + str(self.temp_uuid) - #print("key is:", pf(key)) - - genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers] - - no_val_samples = self.identify_empty_samples(this_trait_vals) - trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples) - - genotype_matrix = np.array(trimmed_genotype_data).T - - #print("genotype_matrix:", str(genotype_matrix.tolist())) - #print("pheno_vector:", str(pheno_vector.tolist())) - - params = dict(pheno_vector = pheno_vector.tolist(), - genotype_matrix = genotype_matrix.tolist(), - restricted_max_likelihood = True, - refit = False, - temp_uuid = str(self.temp_uuid), - - # meta data - timestamp = datetime.datetime.now().isoformat(), - ) - - json_params = json.dumps(params) - #print("json_params:", json_params) - Redis.set(key, json_params) - Redis.expire(key, 60*60) - print("before printing command") - - command = PYLMM_COMMAND+' --key {} --species {}'.format(key, - "other") - print("command is:", command) - print("after printing command") - - os.system(command) - - json_results = Redis.blpop("pylmm:results:" + str(self.temp_uuid), 45*60) - results = json.loads(json_results[1]) - p_values = [float(result) for result in results['p_values']] - #print("p_values:", p_values) - self.dataset.group.markers.add_pvalues(p_values) - - self.trait_results[this_trait.name] = [] - for marker in self.dataset.group.markers.markers: - self.trait_results[this_trait.name].append(marker['lod_score']) - - - def identify_empty_samples(self, values): - no_val_samples = [] - for sample_count, val in enumerate(values): - if val == "x": - no_val_samples.append(sample_count) - return no_val_samples - - def trim_genotypes(self, genotype_data, no_value_samples): - trimmed_genotype_data = [] - for marker in genotype_data: - new_genotypes = [] - for item_count, genotype in enumerate(marker): - if item_count in no_value_samples: - continue - try: - genotype = float(genotype) - except ValueError: - genotype = np.nan - pass - new_genotypes.append(genotype) - trimmed_genotype_data.append(new_genotypes) - return trimmed_genotype_data - - + self.trait_results[this_trait.name].append(float(qtl.lrs))
\ No newline at end of file diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index fda7773f..2c60dd70 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -6,7 +6,7 @@ from flask import Flask, g #Just return a list of dictionaries #each dictionary contains sub-dictionary -def loadGenes(chrName, diffCol, startMb, endMb, webqtlDb =None, species='mouse'): +def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): fetchFields = ['SpeciesId', 'Id', 'GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'TxEnd', 'Strand', 'GeneID', 'NM_ID', 'kgID', 'GenBankID', 'UnigenID', 'ProteinID', 'AlignID', 'exonCount', 'exonStarts', 'exonEnds', 'cdsStart', 'cdsEnd'] diff --git a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py deleted file mode 100644 index deb47ad0..00000000 --- a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py +++ /dev/null @@ -1,1648 +0,0 @@ -# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# This program is available from Source Forge: at GeneNetwork Project -# (sourceforge.net/projects/genenetwork/). -# -# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) -# at rwilliams@uthsc.edu and xzhou15@uthsc.edu -# -# This module is used by GeneNetwork project (www.genenetwork.org) -# -# Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 - -import time -import string -import math -from math import * -import piddle as pid -import sys,os -import httplib, urllib - -from htmlgen import HTMLgen2 as HT -from utility import Plot -from intervalAnalyst import GeneUtil -from base.webqtlTrait import webqtlTrait -from base.templatePage import templatePage -from utility import webqtlUtil -from base import webqtlConfig -from db import webqtlDatabaseFunction -from base.GeneralObject import GeneralObject - -import reaper -import cPickle -from utility.THCell import THCell -from utility.TDCell import TDCell - -class MarkerRegressionPage(templatePage): - - def __init__(self, fd): - - templatePage.__init__(self, fd) - - if not self.openMysql(): - return - - self.initializeParameters(fd) - - filename= webqtlUtil.genRandStr("Itvl_") - ChrList,ChrNameOrderIdDict,ChrOrderIdNameDict,ChrLengthMbList= self.getChrNameOrderIdLength(RISet=fd.RISet) - - if self.mappingMethodId == '4': # For PLINK - - traitInfoList = string.split(string.strip(fd.identification),':') - probesetName = string.strip(traitInfoList[-1]) - plinkOutputFileName= webqtlUtil.genRandStr("%s_%s_"%(fd.RISet,probesetName)) - - # get related values from fd.allTraitData; the format of 'allTraitValueDict'is {strainName1: value=-0.2...} - fd.readData() - allTraitValueDict = fd.allTraitData - - #automatically generate pheno txt file for PLINK - self.genPhenoTxtFileForPlink(phenoFileName=plinkOutputFileName,RISetName=fd.RISet,probesetName=probesetName, valueDict=allTraitValueDict) - # os.system full path is required for input and output files; specify missing value is -9999 - plink_command = '%splink/plink --noweb --ped %splink/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %splink/%s.map --pheno %s/%s.txt --pheno-name %s --missing-phenotype -9999 --out %s%s --assoc ' % (webqtlConfig.GENODIR, webqtlConfig.GENODIR, fd.RISet, webqtlConfig.GENODIR, fd.RISet, webqtlConfig.TMPDIR, plinkOutputFileName, probesetName, webqtlConfig.TMPDIR, plinkOutputFileName) - - os.system(plink_command) - - if fd.identification: - heading2 = HT.Paragraph('Trait ID: %s' % fd.identification) - heading2.__setattr__("class","subtitle") - self.dict['title'] = '%s: Genome Association' % fd.identification - else: - heading2 = "" - self.dict['title'] = 'Genome Association' - - if fd.traitInfo: - symbol,chromosome,MB = string.split(fd.traitInfo,'\t') - heading3 = HT.Paragraph('[ ',HT.Strong(HT.Italic('%s' % symbol,id="green")),' on Chr %s @ %s Mb ]' % (chromosome,MB)) - else: - heading3 = "" - - heading = HT.Paragraph('Trait Data Entered for %s Set' % fd.RISet) - heading.__setattr__("class","title") - - # header info part:Trait Data Entered for HLC Set & Trait ID: - headerdiv = HT.TR(HT.TD(heading, heading2,heading3, width='45%',valign='top', align='left', bgColor='#eeeeee')) - - self.ChrList=ChrList # get chr name from '1' to 'X' - self.ChrLengthMbList = ChrLengthMbList - - # build plink result dict based on chr, key is chr name, value is in list type including Snpname, bp and pvalue info - plinkResultDict={} - count,minPvalue,plinkResultDict =self.getPlinkResultDict(outputFileName=plinkOutputFileName,thresholdPvalue=self.pValue,ChrOrderIdNameDict=ChrOrderIdNameDict) - - # if can not find results which are matched with assigned p-value, system info will show up - if count >0: - - #for genome association report table - reportTable="" - # sortable table object - resultstable,tblobj,bottomInfo = self.GenReportForPLINK(ChrNameOrderIdDict=ChrNameOrderIdDict, RISet=fd.RISet,plinkResultDict=plinkResultDict,thresholdPvalue=self.pValue,chrList=self.ChrList) - - # creat object for result table for sort function - objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') - cPickle.dump(tblobj, objfile) - objfile.close() - - sortby = ("Index", "up") - reportTable =HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "0"), Id="sortable") - - descriptionTable = HT.TableLite(border=0, cellpadding=0, cellspacing=0) - descriptionTable.append(HT.TR(HT.TD(reportTable, colspan=3))) - descriptionTable.append(HT.TR(HT.TD(HT.BR(),HT.BR()))) - descriptionTable.append(bottomInfo) - - # get each chr's length - self.ChrLengthMbList = map(lambda x: x/1000000.0, self.ChrLengthMbList) # change unit from bp to mb - self.ChrLengthMbSum = reduce(lambda x, y:x+y, self.ChrLengthMbList, 0.0)# get total length of all chrs - if self.ChrLengthMbList: - self.GraphInterval = self.ChrLengthMbSum/(len(self.ChrLengthMbList)*12) #Empirical Mb interval - else: - self.GraphInterval = 1 - - # for human data, there's no CM value - self.ChrLengthCMList = [] - self.ChrLengthCMSum = 0 - - # begin: common part with human data - intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight)) - gifmap = self.plotIntMappingForPLINK(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, plinkResultDict=plinkResultDict) - - intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png') - intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap') - - TD_LR = HT.TR(HT.TD(HT.Blockquote(gifmap,intImg, HT.P()), bgColor='#eeeeee', height = 200)) - self.dict['body'] = str(headerdiv)+str(TD_LR)+str(resultstable)+str(HT.TR(HT.TD(descriptionTable))) - - else: - heading = "Genome Association" - detail = ['There is no association with marker that meets this criteria. Please provide a less stringend threshold. The minimun p-value is %s.'%minPvalue] - self.error(heading=heading,detail=detail) - return - - elif self.mappingMethodId == '1': # QTLreaper result - if not fd.genotype: - fd.readData() - - fd.parentsf14regression = fd.formdata.getvalue('parentsf14regression') - weightedRegression = fd.formdata.getvalue('applyVarianceSE') - - if fd.parentsf14regression and fd.genotype_2: - _genotype = fd.genotype_2 - else: - _genotype = fd.genotype_1 - - _strains, _vals, _vars, N = fd.informativeStrains(_genotype.prgy, weightedRegression) - - if fd.identification: - heading2 = HT.Paragraph('Trait ID: %s' % fd.identification) - heading2.__setattr__("class","subtitle") - self.dict['title'] = '%s: Genome Association' % fd.identification - else: - heading2 = "" - self.dict['title'] = 'Genome Association' - - if fd.traitInfo: - symbol,chromosome,MB = string.split(fd.traitInfo,'\t') - heading3 = HT.Paragraph('[ ',HT.Strong(HT.Italic('%s' % symbol,id="green")),' on Chr %s @ %s Mb ]' % (chromosome,MB)) - else: - heading3 = "" - - if N < webqtlConfig.KMININFORMATIVE: - heading = "Genome Association" - detail = ['Fewer than %d strain data were entered for %s data set. No mapping attempted.' % (webqtlConfig.KMININFORMATIVE, fd.RISet)] - self.error(heading=heading,detail=detail) - return - else: - heading = HT.Paragraph('Trait Data Entered for %s Set' % fd.RISet) - heading.__setattr__("class","title") - - datadiv = HT.TD(heading, heading2,heading3, width='45%',valign='top', align='left', bgColor='#eeeeee') - resultstable,tblobj,bottomInfo = self.GenReport(ChrNameOrderIdDict,fd, _genotype, _strains, _vals, _vars) - #resultstable = self.GenReport(fd, _genotype, _strains, _vals, _vars) - - # creat object for result table for sort function - objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') - cPickle.dump(tblobj, objfile) - objfile.close() - - sortby = ("Index", "up") - reportTable =HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "0"), Id="sortable") - - descriptionTable = HT.TableLite(border=0, cellpadding=0, cellspacing=0) - descriptionTable.append(HT.TR(HT.TD(reportTable, colspan=3))) - descriptionTable.append(HT.TR(HT.TD(HT.BR(),HT.BR()))) - descriptionTable.append(bottomInfo) - - self.traitList=_vals - - ##########################plot####################### - - ################################################################ - # Generate Chr list and Retrieve Length Information - ################################################################ - self.genotype= _genotype - self.ChrList = [("All", -1)] - - for i, indChr in enumerate(self.genotype): - self.ChrList.append((indChr.name, i)) - - self.cursor.execute(""" - Select - Length from Chr_Length, InbredSet - where - Chr_Length.SpeciesId = InbredSet.SpeciesId AND - InbredSet.Name = '%s' AND - Chr_Length.Name in (%s) - Order by - OrderId - """ % (fd.RISet, string.join(map(lambda X: "'%s'" % X[0], self.ChrList[1:]), ", "))) - - self.ChrLengthMbList = self.cursor.fetchall() - self.ChrLengthMbList = map(lambda x: x[0]/1000000.0, self.ChrLengthMbList) - self.ChrLengthMbSum = reduce(lambda x, y:x+y, self.ChrLengthMbList, 0.0) - if self.ChrLengthMbList: - self.MbGraphInterval = self.ChrLengthMbSum/(len(self.ChrLengthMbList)*12) #Empirical Mb interval - else: - self.MbGraphInterval = 1 - - self.ChrLengthCMList = [] - for i, _chr in enumerate(self.genotype): - self.ChrLengthCMList.append(_chr[-1].cM - _chr[0].cM) - self.ChrLengthCMSum = reduce(lambda x, y:x+y, self.ChrLengthCMList, 0.0)# used for calculate plot scale - - self.GraphInterval = self.MbGraphInterval #Mb - - # begin: common part with human data - intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight)) - gifmap = self.plotIntMapping(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= "") - filename= webqtlUtil.genRandStr("Itvl_") - intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png') - intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap') - - ################################################################ - # footnote goes here - ################################################################ - btminfo = HT.Paragraph(Id="smallsize") #Small('More information about this graph is available here.') - - if (self.additiveChecked): - btminfo.append(HT.BR(), 'A positive additive coefficient (', HT.Font('green', color='green'), ' line) indicates that %s alleles increase trait values. In contrast, a negative additive coefficient (' % fd.ppolar, HT.Font('red', color='red'), ' line) indicates that %s alleles increase trait values.' % fd.mpolar) - - - TD_LR = HT.TR(HT.TD(HT.Blockquote(gifmap,intImg, HT.P()), bgColor='#eeeeee', height = 200)) - - self.dict['body'] = str(datadiv)+str(TD_LR)+str(resultstable)+str(HT.TR(HT.TD(descriptionTable))) - - # end: common part with human data - - else: - pass - - - # add by NL 10-2-2011 - def initializeParameters(self, fd): - """ - Initializes all of the MarkerRegressionPage class parameters, - acquiring most values from the formdata (fd) - """ - ################################### - # manhattam plot parameters - ################################### - - self.graphHeight = 600 - self.graphWidth = 1280 - self.plotScale = 'physic' - self.selectedChr = -1 - self.GRAPH_BACK_DARK_COLOR = pid.HexColor(0xF1F1F9) - self.GRAPH_BACK_LIGHT_COLOR = pid.HexColor(0xFBFBFF) - self.LRS_COLOR = pid.HexColor(0x0000FF) - self.LRS_LOD ='LRS' - self.lrsMax = float(fd.formdata.getvalue('lrsMax', 0)) - self.startMb = fd.formdata.getvalue('startMb', "-1") - self.endMb = fd.formdata.getvalue('endMb', "-1") - self.mappingMethodId = fd.formdata.getvalue('mappingMethodId', "0") - self.permChecked=True - self.multipleInterval=False - self.SIGNIFICANT_WIDTH = 5 - self.SUGGESTIVE_WIDTH = 5 - self.SIGNIFICANT_COLOR = pid.HexColor(0xEBC7C7) - self.SUGGESTIVE_COLOR = pid.gainsboro - self.colorCollection = [self.LRS_COLOR] - self.additiveChecked= True - self.ADDITIVE_COLOR_POSITIVE = pid.green - self.legendChecked =False - self.pValue=float(fd.formdata.getvalue('pValue',-1)) - - # allow user to input p-value greater than 1, - # in this case, the value will be treated as -lgP value. so the input value needs to be transferred to power of 10 format - if self.pValue >1: - self.pValue =10**-(self.pValue) - - try: - self.startMb = float(self.startMb) - self.endMb = float(self.endMb) - if self.startMb > self.endMb: - temp = self.startMb - self.startMb = self.endMb - self.endMb = temp - #minimal distance 10bp - if self.endMb - self.startMb < 0.00001: - self.endMb = self.startMb + 0.00001 - except: - self.startMb = self.endMb = -1 - - def GenReportForPLINK(self, ChrNameOrderIdDict={},RISet='',plinkResultDict= {},thresholdPvalue=-1,chrList=[]): - - 'Create an HTML division which reports any loci which are significantly associated with the submitted trait data.' - ######################################### - # Genome Association report - ######################################### - locusFormName = webqtlUtil.genRandStr("fm_") - locusForm = HT.Form(cgi = os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), \ - enctype='multipart/form-data', name=locusFormName, submit=HT.Input(type='hidden')) - hddn = {'FormID':'showDatabase','ProbeSetID':'_','database':RISet+"Geno",'CellID':'_', \ - 'RISet':RISet, 'incparentsf1':'on'} - for key in hddn.keys(): - locusForm.append(HT.Input(name=key, value=hddn[key], type='hidden')) - - regressionHeading = HT.Paragraph('Genome Association Report') - regressionHeading.__setattr__("class","title") - - filename= webqtlUtil.genRandStr("GenomeAsscociation_") - fpText = open('%s.txt' % (webqtlConfig.TMPDIR+filename), 'wb') - fpText.write('The loci meet the criteria of P-Value <= %3.6f.\n'%thresholdPvalue) - pValueInfo =HT.Paragraph('The loci meet the criteria of P-Value <= %3.6f.\n'%thresholdPvalue) - - textUrl = HT.Href(text = 'Download', url= '/tmp/'+filename+'.txt', target = "_blank", Class='fs12 fwn') - bottomInfo = HT.TR(HT.TD(HT.Paragraph(textUrl, ' result in tab-delimited text format.', HT.BR(), HT.BR(),Class="fs12 fwn"), colspan=3)) - - tblobj={} # build dict for genTableObj function; keys include header and body - tblobj_header = [] # value of key 'header' - tblobj_body=[] # value of key 'body' - reportHeaderRow=[] # header row list for tblobj_header (html part) - headerList=['Index','SNP Name','Chr','Mb','-log(P)'] - headerStyle="fs14 fwb ffl b1 cw cbrb" # style of the header - cellColorStyle = "fs13 b1 fwn c222" # style of the cells - - if headerList: - for ncol, item in enumerate(headerList): - reportHeaderRow.append(THCell(HT.TD(item, Class=headerStyle, valign='bottom',nowrap='ON'),text=item, idx=ncol)) - #download file for table headers' names - fpText.write('SNP_Name\tChromosome\tMb\t-log(P)\n') - - tblobj_header.append(reportHeaderRow) - tblobj['header']=tblobj_header - - index=1 - for chr in chrList: - - if plinkResultDict.has_key(chr): - if chr in ChrNameOrderIdDict.keys(): - chrOrderId =ChrNameOrderIdDict[chr] - else: - chrOrderId=chr - - valueList=plinkResultDict[chr] - - for value in valueList: - reportBodyRow=[] # row list for tblobj_body (html part) - snpName=value[0] - bp=value[1] - mb=int(bp)/1000000.0 - - try: - pValue =float(value[2]) - except: - pValue =1 - formattedPvalue = -math.log10(pValue) - - formattedPvalue = webqtlUtil.SciFloat(formattedPvalue) - dbSnprs=snpName.replace('rs','') - SnpHref = HT.Href(text=snpName, url="http://www.ncbi.nlm.nih.gov/projects/SNP/snp_ref.cgi?rs=%s"%dbSnprs, target="_blank") - - selectCheck=HT.Input(type="checkbox", Class="checkbox", name="index",value=index, onClick="highlight(this)") - reportBodyRow.append(TDCell(HT.TD(str(index),selectCheck, align='right',Class=cellColorStyle,nowrap='ON'),str(index),index)) - reportBodyRow.append(TDCell(HT.TD(SnpHref, Class=cellColorStyle,nowrap='ON'),snpName, snpName)) - reportBodyRow.append(TDCell(HT.TD(chr, Class=cellColorStyle, align="center",nowrap='ON'),chr, chrOrderId)) - reportBodyRow.append(TDCell(HT.TD('%3.6f'%mb, Class=cellColorStyle, align="center",nowrap='ON'),mb, mb)) - reportBodyRow.append(TDCell(HT.TD(formattedPvalue, Class=cellColorStyle, align="center",nowrap='ON'),formattedPvalue, float(formattedPvalue))) - - fpText.write('%s\t%s\t%3.6f\t%s\n' % (snpName, str(chr), mb, formattedPvalue)) - index+=1 - - tblobj_body.append(reportBodyRow) - - tblobj['body']=tblobj_body - rv=HT.TR(HT.TD(regressionHeading,pValueInfo, locusForm, HT.P(), width='55%',valign='top', align='left',bgColor='#eeeeee')) - - return rv, tblobj,bottomInfo - - - def GenReport(self, ChrNameOrderIdDict,fd, _genotype, _strains, _vals, _vars= []): - 'Create an HTML division which reports any loci which are significantly associated with the submitted trait data.' - #calculate QTL for each trait - self.qtlresults = [] - if webqtlUtil.ListNotNull(_vars): - qtlresults = _genotype.regression(strains = _strains, trait = _vals, variance = _vars) - LRSArray = _genotype.permutation(strains = _strains, trait = _vals, variance = _vars, nperm=fd.nperm) - else: - qtlresults = _genotype.regression(strains = _strains, trait = _vals) - LRSArray = _genotype.permutation(strains = _strains, trait = _vals,nperm=fd.nperm) - - self.qtlresults.append(qtlresults) - - filename= webqtlUtil.genRandStr("GenomeAsscociation_") - - # set suggestive, significant and highly significant LRS - if fd.suggestive == None: - fd.suggestive = LRSArray[int(fd.nperm*0.37-1)] - else: - fd.suggestive = float(fd.suggestive) - if fd.significance == None: - fd.significance = LRSArray[int(fd.nperm*0.95-1)] - else: - fd.significance = float(fd.significance) - - self.significance =fd.significance - self.suggestive = fd.suggestive - self.highlysignificant = LRSArray[int(fd.nperm*0.99-1)] - _dispAllLRS = 0 - if fd.formdata.getvalue('displayAllLRS'): - _dispAllLRS = 1 - qtlresults2 = [] - if _dispAllLRS: - filtered = qtlresults[:] - else: - filtered = filter(lambda x, y=fd.suggestive: x.lrs > y, qtlresults) - if len(filtered) == 0: - qtlresults2 = qtlresults[:] - qtlresults2.sort() - filtered = qtlresults2[-10:] - - ######################################### - # Permutation Graph - ######################################### - myCanvas = pid.PILCanvas(size=(400,300)) - #plotBar(myCanvas,10,10,390,290,LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test',identification=fd.identification) - Plot.plotBar(myCanvas, LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test') - filename= webqtlUtil.genRandStr("Reg_") - myCanvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') - img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test') - - if fd.suggestive == None: - fd.suggestive = LRSArray[int(fd.nperm*0.37-1)] - else: - fd.suggestive = float(fd.suggestive) - if fd.significance == None: - fd.significance = LRSArray[int(fd.nperm*0.95-1)] - else: - fd.significance = float(fd.significance) - - permutationHeading = HT.Paragraph('Histogram of Permutation Test') - permutationHeading.__setattr__("class","title") - - permutation = HT.TableLite() - permutation.append(HT.TR(HT.TD(img))) - - - ######################################### - # Genome Association report - ######################################### - locusFormName = webqtlUtil.genRandStr("fm_") - locusForm = HT.Form(cgi = os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), \ - enctype='multipart/form-data', name=locusFormName, submit=HT.Input(type='hidden')) - hddn = {'FormID':'showDatabase','ProbeSetID':'_','database':fd.RISet+"Geno",'CellID':'_', \ - 'RISet':fd.RISet, 'incparentsf1':'on'} - for key in hddn.keys(): - locusForm.append(HT.Input(name=key, value=hddn[key], type='hidden')) - - regressionHeading = HT.Paragraph('Genome Association Report') - regressionHeading.__setattr__("class","title") - # report is the info part above report table - if qtlresults2 != []: - report = HT.Blockquote(HT.Font('No association ',color="#FF0000"),HT.Font('with a likelihood ratio statistic greater than %3.1f was found. Here are the top 10 LRSs.' % fd.suggestive,color="#000000")) - else: - report = HT.Blockquote('The following loci in the %s data set have associations with the above trait data.\n' % fd.RISet, HT.P()) - report.__setattr__("class","normalsize") - - fpText = open('%s.txt' % (webqtlConfig.TMPDIR+filename), 'wb') - fpText.write('Suggestive LRS =%3.2f\n'%self.suggestive) - fpText.write('Significant LRS =%3.2f\n'%self.significance) - fpText.write('Highly Significant LRS =%3.2f\n'%self.highlysignificant) - LRSInfo =HT.Paragraph(' Suggestive LRS =%3.2f\n'%fd.suggestive, HT.BR(), ' Significant LRS =%3.2f\n'%fd.significance,HT.BR(),' Highly Significant LRS =%3.2f\n' % self.highlysignificant) - - textUrl = HT.Href(text = 'Download', url= '/tmp/'+filename+'.txt', target = "_blank", Class='fs12 fwn') - - bottomInfo = HT.TR(HT.TD(HT.Paragraph(textUrl, ' result in tab-delimited text format.', HT.BR(), HT.BR(),'LRS values marked with',HT.Font(' * ',color="red"), 'are greater than the significance threshold (specified by you or by permutation test). ' , HT.BR(), HT.BR(), HT.Strong('Additive Effect'), ' is half the difference in the mean phenotype of all cases that are homozygous for one parental allel at this marker minus the mean of all cases that are homozygous for the other parental allele at this marker. ','In the case of %s strains, for example,' % fd.RISet,' A positive additive effect indicates that %s alleles increase trait values. Negative additive effect indicates that %s alleles increase trait values.'% (fd.ppolar,fd.mpolar),Class="fs12 fwn"))) - - tblobj={} # build dict for genTableObj function; keys include header and body - tblobj_header = [] # value of key 'header' - tblobj_body=[] # value of key 'body' - reportHeaderRow=[] # header row list for tblobj_header (html part) - headerStyle="fs14 fwb ffl b1 cw cbrb" # style of the header - cellColorStyle = "fs13 b1 fwn c222" # style of the cells - - headerList=['Index','LRS','Chr','Mb','Locus','Additive Effect'] - for ncol, item in enumerate(headerList): - reportHeaderRow.append(THCell(HT.TD(item, Class=headerStyle, valign='bottom',nowrap='ON'),text=item, idx=ncol)) - - if fd.genotype.type == 'intercross': - ncol =len(headerList) - reportHeaderRow.append(THCell(HT.TD('Dominance Effect', Class=headerStyle, valign='bottom',nowrap='ON'),text='Dominance Effect', idx=ncol)) - - #download file for table headers' names - fpText.write('LRS\tChromosome\tMb\tLocus\tAdditive Effect\tDominance Effect\n') - - index=1 - for ii in filtered: - #add by NL 06-20-2011: set LRS to 460 when LRS is infinite, - if ii.lrs==float('inf') or ii.lrs>webqtlConfig.MAXLRS: - LRS=webqtlConfig.MAXLRS #maximum LRS value - else: - LRS=ii.lrs - - if LRS > fd.significance: - lrs = HT.TD(HT.Font('%3.3f*' % LRS, color='#FF0000'),Class=cellColorStyle) - else: - lrs = HT.TD('%3.3f' % LRS,Class=cellColorStyle) - - if ii.locus.chr in ChrNameOrderIdDict.keys(): - chrOrderId =ChrNameOrderIdDict[ii.locus.chr] - else: - chrOrderId=ii.locus.chr - - reportBodyRow=[] # row list for tblobj_body (html part) - selectCheck=HT.Input(type="checkbox", Class="checkbox", name="index",value=index, onClick="highlight(this)") - reportBodyRow.append(TDCell(HT.TD(str(index),selectCheck, align='right',Class=cellColorStyle,nowrap='ON'),str(index),index)) - reportBodyRow.append(TDCell(lrs,LRS, LRS)) - reportBodyRow.append(TDCell(HT.TD(ii.locus.chr, Class=cellColorStyle, align="center",nowrap='ON'),ii.locus.chr, chrOrderId)) - reportBodyRow.append(TDCell(HT.TD('%3.6f'%ii.locus.Mb, Class=cellColorStyle, align="center",nowrap='ON'),ii.locus.Mb, ii.locus.Mb)) - reportBodyRow.append(TDCell(HT.TD(HT.Href(text=ii.locus.name, url = "javascript:showTrait('%s','%s');" % (locusFormName, ii.locus.name), Class='normalsize'), Class=cellColorStyle, align="center",nowrap='ON'),ii.locus.name, ii.locus.name)) - reportBodyRow.append(TDCell(HT.TD('%3.3f' % ii.additive, Class=cellColorStyle, align="center",nowrap='ON'),ii.additive, ii.additive)) - reportBodyRow.append(TDCell(HT.TD('%3.3f' % ii.dominance, Class=cellColorStyle, align="center",nowrap='ON'),ii.dominance, ii.dominance)) - - fpText.write('%2.3f\t%s\t%3.6f\t%s\t%2.3f\t%2.3f\n' % (LRS, ii.locus.chr, ii.locus.Mb, ii.locus.name, ii.additive, ii.dominance)) - index+=1 - tblobj_body.append(reportBodyRow) - else: - #download file for table headers' names - fpText.write('LRS\tChromosome\tMb\tLocus\tAdditive Effect\n') - - index=1 - for ii in filtered: - #add by NL 06-20-2011: set LRS to 460 when LRS is infinite, - if ii.lrs==float('inf') or ii.lrs>webqtlConfig.MAXLRS: - LRS=webqtlConfig.MAXLRS #maximum LRS value - else: - LRS=ii.lrs - - if LRS > fd.significance: - lrs = HT.TD(HT.Font('%3.3f*' % LRS, color='#FF0000'),Class=cellColorStyle) - else: - lrs = HT.TD('%3.3f' % LRS,Class=cellColorStyle) - - if ii.locus.chr in ChrNameOrderIdDict.keys(): - chrOrderId =ChrNameOrderIdDict[ii.locus.chr] - else: - chrOrderId=ii.locus.chr - - reportBodyRow=[] # row list for tblobj_body (html part) - selectCheck=HT.Input(type="checkbox", Class="checkbox", name="index",value=index, onClick="highlight(this)") - reportBodyRow.append(TDCell(HT.TD(str(index),selectCheck, align='right',Class=cellColorStyle,nowrap='ON'),str(index),index)) - reportBodyRow.append(TDCell(lrs,LRS, LRS)) - reportBodyRow.append(TDCell(HT.TD(ii.locus.chr, Class=cellColorStyle, align="center",nowrap='ON'),ii.locus.chr, chrOrderId)) - reportBodyRow.append(TDCell(HT.TD('%3.6f'%ii.locus.Mb, Class=cellColorStyle, align="center",nowrap='ON'),ii.locus.Mb, ii.locus.Mb)) - reportBodyRow.append(TDCell(HT.TD(HT.Href(text=ii.locus.name, url = "javascript:showTrait('%s','%s');" % (locusFormName, ii.locus.name), Class='normalsize'), Class=cellColorStyle, align="center",nowrap='ON'),ii.locus.name, ii.locus.name)) - reportBodyRow.append(TDCell(HT.TD('%3.3f' % ii.additive, Class=cellColorStyle, align="center",nowrap='ON'),ii.additive, ii.additive)) - - fpText.write('%2.3f\t%s\t%3.6f\t%s\t%2.3f\n' % (LRS, ii.locus.chr, ii.locus.Mb, ii.locus.name, ii.additive)) - index+=1 - tblobj_body.append(reportBodyRow) - - tblobj_header.append(reportHeaderRow) - tblobj['header']=tblobj_header - tblobj['body']=tblobj_body - - rv=HT.TD(regressionHeading,LRSInfo,report, locusForm, HT.P(),width='55%',valign='top', align='left', bgColor='#eeeeee') - if fd.genotype.type == 'intercross': - bottomInfo.append(HT.BR(), HT.BR(), HT.Strong('Dominance Effect'),' is the difference between the mean trait value of cases heterozygous at a marker and the average mean for the two groups homozygous at this marker: e.g., BD - (BB+DD)/2]. A positive dominance effect indicates that the average phenotype of BD heterozygotes exceeds the mean of BB and DD homozygotes. No dominance deviation can be computed for a set of recombinant inbred strains or for a backcross.') - return rv,tblobj,bottomInfo - - return rv,tblobj,bottomInfo - - def plotIntMappingForPLINK(self, fd, canvas, offset= (80, 120, 20, 80), zoom = 1, startMb = None, endMb = None, showLocusForm = "",plinkResultDict={}): - #calculating margins - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - xLeftOffset = int(xLeftOffset*fontZoom) - xRightOffset = int(xRightOffset*fontZoom) - yBottomOffset = int(yBottomOffset*fontZoom) - - cWidth = canvas.size[0] - cHeight = canvas.size[1] - plotWidth = cWidth - xLeftOffset - xRightOffset - plotHeight = cHeight - yTopOffset - yBottomOffset - startPixelX = xLeftOffset - endPixelX = (xLeftOffset + plotWidth) - - #Drawing Area Height - drawAreaHeight = plotHeight - if self.plotScale == 'physic' and self.selectedChr > -1: # for single chr - drawAreaHeight -= self.ENSEMBL_BAND_HEIGHT + self.UCSC_BAND_HEIGHT+ self.WEBQTL_BAND_HEIGHT + 3*self.BAND_SPACING+ 10*zoom - if self.geneChecked: - drawAreaHeight -= self.NUM_GENE_ROWS*self.EACH_GENE_HEIGHT + 3*self.BAND_SPACING + 10*zoom - else: - if self.selectedChr > -1: - drawAreaHeight -= 20 - else:# for all chrs - drawAreaHeight -= 30 - - #Image map - gifmap = HT.Map(name='WebQTLImageMap') - - newoffset = (xLeftOffset, xRightOffset, yTopOffset, yBottomOffset) - # Draw the alternating-color background first and get plotXScale - plotXScale = self.drawGraphBackgroundForPLINK(canvas, gifmap, offset=newoffset, zoom= zoom, startMb=startMb, endMb = endMb,plinkResultDict=plinkResultDict) - - # Draw X axis - self.drawXAxisForPLINK(fd, canvas, drawAreaHeight, gifmap, plotXScale, showLocusForm, offset=newoffset, zoom= zoom, startMb=startMb, endMb = endMb) - # Draw manhattam plot - self.drawManhattanPlotForPLINK(canvas, drawAreaHeight, gifmap, plotXScale, offset=newoffset, zoom= zoom, startMb=startMb, endMb = endMb,plinkResultDict=plinkResultDict,thresholdPvalue=self.pValue) - - return gifmap - - - def plotIntMapping(self, fd, canvas, offset= (80, 120, 20, 80), zoom = 1, startMb = None, endMb = None, showLocusForm = ""): - #calculating margins - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - xLeftOffset = int(xLeftOffset*fontZoom) - xRightOffset = int(xRightOffset*fontZoom) - yBottomOffset = int(yBottomOffset*fontZoom) - - cWidth = canvas.size[0] - cHeight = canvas.size[1] - plotWidth = cWidth - xLeftOffset - xRightOffset - plotHeight = cHeight - yTopOffset - yBottomOffset - startPixelX = xLeftOffset - endPixelX = (xLeftOffset + plotWidth) - - #Drawing Area Height - drawAreaHeight = plotHeight - if self.plotScale == 'physic' and self.selectedChr > -1: # for single chr - drawAreaHeight -= self.ENSEMBL_BAND_HEIGHT + self.UCSC_BAND_HEIGHT+ self.WEBQTL_BAND_HEIGHT + 3*self.BAND_SPACING+ 10*zoom - if self.geneChecked: - drawAreaHeight -= self.NUM_GENE_ROWS*self.EACH_GENE_HEIGHT + 3*self.BAND_SPACING + 10*zoom - else:# for all chrs - if self.selectedChr > -1: - drawAreaHeight -= 20 - else: - drawAreaHeight -= 30 - - #Image map - gifmap = HT.Map(name='WebQTLImageMap') - - newoffset = (xLeftOffset, xRightOffset, yTopOffset, yBottomOffset) - # Draw the alternating-color background first and get plotXScale - plotXScale = self.drawGraphBackground(canvas, gifmap, offset=newoffset, zoom= zoom, startMb=startMb, endMb = endMb) - - # Draw X axis - self.drawXAxis(fd, canvas, drawAreaHeight, gifmap, plotXScale, showLocusForm, offset=newoffset, zoom= zoom, startMb=startMb, endMb = endMb) - # Draw QTL curve - self.drawQTL(canvas, drawAreaHeight, gifmap, plotXScale, offset=newoffset, zoom= zoom, startMb=startMb, endMb = endMb) - - #draw legend - if self.multipleInterval: - self.drawMultiTraitName(fd, canvas, gifmap, showLocusForm, offset=newoffset) - elif self.legendChecked: - self.drawLegendPanel(fd, canvas, offset=newoffset) - else: - pass - - #draw position, no need to use a separate function - if fd.genotype.Mbmap: - self.drawProbeSetPosition(canvas, plotXScale, offset=newoffset) - - return gifmap - - - # functions for manhattam plot of markers - def drawManhattanPlotForPLINK(self, canvas, drawAreaHeight, gifmap, plotXScale, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None,plinkResultDict={},thresholdPvalue=-1): - - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - # INTERCROSS = (self.genotype.type=="intercross") - INTERCROSS ='' #?????? - - ChrLengthDistList = self.ChrLengthMbList - drawRegionDistance = self.ChrLengthMbSum - GraphInterval=self.GraphInterval - pvalueHeightThresh = drawAreaHeight - 80 #ZS: Otherwise the plot gets very close to the chromosome labels - - #draw the pvalue scale - #We first determine whether or not we are using a sliding scale. - #If so, we need to compute the maximum pvalue value to determine where the max y-value should be, and call this pvalueMax. - #pvalueTop is then defined to be above the pvalueMax by enough to add one additional pvalueScale increment. - #if we are using a set-scale, then we set pvalueTop to be the user's value, and pvalueMax doesn't matter. - - # for human data we use p value instead of lrs - pValueList=[] - for key in plinkResultDict: - valueList = plinkResultDict[key] - for item in valueList: - pValue = item[-1] - pValueList.append(pValue) - - formattedPValueList=[] - for pValue in pValueList: - try: - pValue=float(pValue) - except: - pValue =1 - formattedpValue = -math.log10(pValue) - formattedPValueList.append(formattedpValue) - - #sliding scale - pvalueMax = max(formattedPValueList) - #pvalueMax =pvalueMax +1 - # no permutation result for plink func: GenReport() - pvalueMin = int(-math.log10(thresholdPvalue)) - - if pvalueMax> 100: - pvalueScale = 20.0 - elif pvalueMax > 20: - pvalueScale = 5.0 - elif pvalueMax > 7.5: - pvalueScale = 2.5 - else: - pvalueScale = 1.0 - - # the base line for x-axis is -log(thresholdPvalue) - pvalueAxisList = Plot.frange(pvalueMin, pvalueMax, pvalueScale) - #make sure the user's value appears on the y-axis - #ZS: There is no way to do this without making the position of the points not directly proportional to a given distance on the y-axis - #tempPvalueMax=round(pvalueMax) - tempPvalueMax = pvalueAxisList[len(pvalueAxisList)-1] + pvalueScale - pvalueAxisList.append(tempPvalueMax) - - #ZS: I don't understand this; the if statement will be true for any number that isn't exactly X.5. - #if abs(tempPvalueMax-pvalueMax) <0.5: - # tempPvalueMax=tempPvalueMax+1 - # pvalueAxisList.append(tempPvalueMax) - - #draw the "pvalue" string to the left of the axis - pvalueScaleFont=pid.Font(ttf="verdana", size=14*fontZoom, bold=0) - pvalueLODFont=pid.Font(ttf="verdana", size=14*zoom*1.5, bold=0) - yZero = yTopOffset + plotHeight - - #yAxis label display area - yAxis_label ='-log(P)' - canvas.drawString(yAxis_label, xLeftOffset - canvas.stringWidth("999.99", font=pvalueScaleFont) - 10*zoom, \ - yZero - 150, font=pvalueLODFont, color=pid.black, angle=90) - - for i,item in enumerate(pvalueAxisList): - ypvalue = yZero - (float(i)/float(len(pvalueAxisList) - 1)) * pvalueHeightThresh - canvas.drawLine(xLeftOffset, ypvalue, xLeftOffset - 4, ypvalue, color=self.LRS_COLOR, width=1*zoom) - scaleStr = "%2.1f" % item - #added by NL 6-24-2011:Y-axis scale display - canvas.drawString(scaleStr, xLeftOffset-4-canvas.stringWidth(scaleStr, font=pvalueScaleFont)-5, ypvalue+3, font=pvalueScaleFont, color=self.LRS_COLOR) - - ChrList=self.ChrList - startPosX = xLeftOffset - - for i, chr in enumerate(ChrList): - - if plinkResultDict.has_key(chr): - plinkresultList = plinkResultDict[chr] - - m = 0 - #add by NL 06-24-2011: for mahanttam plot - symbolFont = pid.Font(ttf="fnt_bs", size=5,bold=0) - # color for point in each chr - chrCount=len(ChrList) - chrColorDict =self.getColorForMarker(chrCount=chrCount,flag=1) - for j, item in enumerate(plinkresultList): - try : - mb=float(item[1])/1000000.0 - except: - mb=0 - - try : - pvalue =float(item[-1]) - except: - pvalue =1 - - try: - snpName = item[0] - except: - snpName='' - - formattedPvalue = -math.log10(pvalue) - - Xc = startPosX + (mb-startMb)*plotXScale - Yc = yZero - (formattedPvalue-pvalueMin)*pvalueHeightThresh/(tempPvalueMax - pvalueMin) - canvas.drawString("5", Xc-canvas.stringWidth("5",font=symbolFont)/2+1,Yc+2,color=chrColorDict[i], font=symbolFont) - m += 1 - - startPosX += (ChrLengthDistList[i]+GraphInterval)*plotXScale - - canvas.drawLine(xLeftOffset, yZero, xLeftOffset, yTopOffset, color=self.LRS_COLOR, width=1*zoom) #the blue line running up the y axis - - def drawQTL(self, canvas, drawAreaHeight, gifmap, plotXScale, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): - - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - INTERCROSS = (self.genotype.type=="intercross") - - ChrLengthDistList = self.ChrLengthMbList - GraphInterval=self.GraphInterval - LRSHeightThresh = drawAreaHeight - AdditiveHeightThresh = drawAreaHeight/2 - DominanceHeightThresh = drawAreaHeight/2 - - #draw the LRS scale - #We first determine whether or not we are using a sliding scale. - #If so, we need to compute the maximum LRS value to determine where the max y-value should be, and call this LRSMax. - #LRSTop is then defined to be above the LRSMax by enough to add one additional LRSScale increment. - #if we are using a set-scale, then we set LRSTop to be the user's value, and LRSMax doesn't matter. - - if self.LRS_LOD == 'LOD': - lodm = self.LODFACTOR - else: - lodm = 1.0 - - if self.lrsMax <= 0: #sliding scale - LRSMax = max(map(max, self.qtlresults)).lrs - #genotype trait will give infinite LRS - LRSMax = min(LRSMax, webqtlConfig.MAXLRS) - LRSMax = max(self.significance, LRSMax) - else: - LRSMax = self.lrsMax*lodm - - if LRSMax/lodm > 100: - LRSScale = 20.0 - elif LRSMax/lodm > 20: - LRSScale = 5.0 - elif LRSMax/lodm > 7.5: - LRSScale = 2.5 - else: - LRSScale = 1.0 - - LRSAxisList = Plot.frange(LRSScale, LRSMax/lodm, LRSScale) - #make sure the user's value appears on the y-axis - #update by NL 6-21-2011: round the LOD value to 100 when LRSMax is equal to 460 - LRSAxisList.append(round(LRSMax/lodm)) - - #draw the "LRS" or "LOD" string to the left of the axis - LRSScaleFont=pid.Font(ttf="verdana", size=14*fontZoom, bold=0) - LRSLODFont=pid.Font(ttf="verdana", size=14*zoom*1.5, bold=0) - yZero = yTopOffset + plotHeight - - #yAxis label display area - canvas.drawString(self.LRS_LOD, xLeftOffset - canvas.stringWidth("999.99", font=LRSScaleFont) - 10*zoom, \ - yZero - 150, font=LRSLODFont, color=pid.black, angle=90) - - for item in LRSAxisList: - yLRS = yZero - (item*lodm/LRSMax) * LRSHeightThresh - canvas.drawLine(xLeftOffset, yLRS, xLeftOffset - 4, yLRS, color=self.LRS_COLOR, width=1*zoom) - scaleStr = "%2.1f" % item - #added by NL 6-24-2011:Y-axis scale display - canvas.drawString(scaleStr, xLeftOffset-4-canvas.stringWidth(scaleStr, font=LRSScaleFont)-5, yLRS+3, font=LRSScaleFont, color=self.LRS_COLOR) - - - #"Significant" and "Suggestive" Drawing Routine - # ======= Draw the thick lines for "Significant" and "Suggestive" ===== (crowell: I tried to make the SNPs draw over these lines, but piddle wouldn't have it...) - if self.permChecked and not self.multipleInterval: - significantY = yZero - self.significance*LRSHeightThresh/LRSMax - suggestiveY = yZero - self.suggestive*LRSHeightThresh/LRSMax - - - startPosX = xLeftOffset - for i, _chr in enumerate(self.genotype): - rightEdge = int(startPosX + self.ChrLengthDistList[i]*plotXScale - self.SUGGESTIVE_WIDTH/1.5) - #added by NL 6-24-2011:draw suggestive line (grey one) - canvas.drawLine(startPosX+self.SUGGESTIVE_WIDTH/1.5, suggestiveY, rightEdge, suggestiveY, color=self.SUGGESTIVE_COLOR, - width=self.SUGGESTIVE_WIDTH*zoom, clipX=(xLeftOffset, xLeftOffset + plotWidth-2)) - #added by NL 6-24-2011:draw significant line (pink one) - canvas.drawLine(startPosX+self.SUGGESTIVE_WIDTH/1.5, significantY, rightEdge, significantY, color=self.SIGNIFICANT_COLOR, - width=self.SIGNIFICANT_WIDTH*zoom, clipX=(xLeftOffset, xLeftOffset + plotWidth-2)) - sugg_coords = "%d, %d, %d, %d" % (startPosX, suggestiveY-2, rightEdge + 2*zoom, suggestiveY+2) - sig_coords = "%d, %d, %d, %d" % (startPosX, significantY-2, rightEdge + 2*zoom, significantY+2) - if self.LRS_LOD == 'LRS': - sugg_title = "Suggestive LRS = %0.2f" % self.suggestive - sig_title = "Significant LRS = %0.2f" % self.significance - else: - sugg_title = "Suggestive LOD = %0.2f" % (self.suggestive/4.61) - sig_title = "Significant LOD = %0.2f" % (self.significance/4.61) - Areas1 = HT.Area(shape='rect',coords=sugg_coords,title=sugg_title) - Areas2 = HT.Area(shape='rect',coords=sig_coords,title=sig_title) - gifmap.areas.append(Areas1) - gifmap.areas.append(Areas2) - - startPosX += (self.ChrLengthDistList[i]+self.GraphInterval)*plotXScale - - - if self.multipleInterval: - lrsEdgeWidth = 1 - else: - additiveMax = max(map(lambda X : abs(X.additive), self.qtlresults[0])) - if INTERCROSS: - dominanceMax = max(map(lambda X : abs(X.dominance), self.qtlresults[0])) - else: - dominanceMax = -1 - lrsEdgeWidth = 2 - for i, qtlresult in enumerate(self.qtlresults): - m = 0 - startPosX = xLeftOffset - thisLRSColor = self.colorCollection[i] - - #add by NL 06-24-2011: for mahanttam plot - symbolFont = pid.Font(ttf="fnt_bs", size=5,bold=0) - - for j, _chr in enumerate(self.genotype): - chrCount=len(self.genotype) - chrColorDict =self.getColorForMarker(chrCount=chrCount,flag=1) - LRSCoordXY = [] - AdditiveCoordXY = [] - DominanceCoordXY = [] - for k, _locus in enumerate(_chr): - if self.plotScale == 'physic': - Xc = startPosX + (_locus.Mb-startMb)*plotXScale - else: - Xc = startPosX + (_locus.cM-_chr[0].cM)*plotXScale - # updated by NL 06-18-2011: - # fix the over limit LRS graph issue since genotype trait may give infinite LRS; - # for any lrs is over than 460(LRS max in this system), it will be reset to 460 - if qtlresult[m].lrs> 460 or qtlresult[m].lrs=='inf': - Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRSMax - else: - Yc = yZero - qtlresult[m].lrs*LRSHeightThresh/LRSMax - - LRSCoordXY.append((Xc, Yc)) - #add by NL 06-24-2011: for mahanttam plot - #self.significance/4.61 consider chr and LOD - # significantY = yZero - self.significance*LRSHeightThresh/LRSMax - # if Yc >significantY: - # canvas.drawString(":", Xc-canvas.stringWidth(":",font=symbolFont)/2+1,Yc+2,color=pid.black, font=symbolFont) - # else: - # canvas.drawString(":", Xc-canvas.stringWidth(":",font=symbolFont)/2+1,Yc+2,color=pid.black, font=symbolFont) - - # add by NL 06-27-2011: eliminate imputed value when locus name is equal to '-' - if (qtlresult[m].locus.name) and (qtlresult[m].locus.name!=' - '): - canvas.drawString("5", Xc-canvas.stringWidth("5",font=symbolFont)/2+1,Yc+2,color=chrColorDict[j], font=symbolFont) - - if not self.multipleInterval and self.additiveChecked: - Yc = yZero - qtlresult[m].additive*AdditiveHeightThresh/additiveMax - AdditiveCoordXY.append((Xc, Yc)) - if not self.multipleInterval and INTERCROSS and self.additiveChecked: - Yc = yZero - qtlresult[m].dominance*DominanceHeightThresh/dominanceMax - DominanceCoordXY.append((Xc, Yc)) - m += 1 - - startPosX += (ChrLengthDistList[j]+GraphInterval)*plotXScale - - - ###draw additive scale - if not self.multipleInterval and self.additiveChecked: - additiveScaleFont=pid.Font(ttf="verdana",size=12*fontZoom,bold=0) - additiveScale = Plot.detScaleOld(0,additiveMax) - additiveStep = (additiveScale[1]-additiveScale[0])/additiveScale[2] - additiveAxisList = Plot.frange(0, additiveScale[1], additiveStep) - maxAdd = additiveScale[1] - addPlotScale = AdditiveHeightThresh/additiveMax - - additiveAxisList.append(additiveScale[1]) - for item in additiveAxisList: - additiveY = yZero - item*addPlotScale - canvas.drawLine(xLeftOffset + plotWidth,additiveY,xLeftOffset+4+ plotWidth,additiveY,color=self.ADDITIVE_COLOR_POSITIVE, width=1*zoom) - scaleStr = "%2.3f" % item - canvas.drawString(scaleStr,xLeftOffset + plotWidth +6,additiveY+5,font=additiveScaleFont,color=self.ADDITIVE_COLOR_POSITIVE) - - canvas.drawLine(xLeftOffset+plotWidth,additiveY,xLeftOffset+plotWidth,yZero,color=self.ADDITIVE_COLOR_POSITIVE, width=1*zoom) - - canvas.drawLine(xLeftOffset, yZero, xLeftOffset, yTopOffset, color=self.LRS_COLOR, width=1*zoom) #the blue line running up the y axis - - def drawGraphBackgroundForPLINK(self, canvas, gifmap, offset= (80, 120, 80, 50), zoom = 1, startMb = None, endMb = None,plinkResultDict={} ): - - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - #calculate plot scale - #XZ: all of these global variables should be passed from function signiture - ChrLengthDistList = self.ChrLengthMbList - drawRegionDistance = self.ChrLengthMbSum - GraphInterval=self.GraphInterval - ChrList =self.ChrList - - #multiple chromosome view - plotXScale = plotWidth / ((len(ChrList)-1)*GraphInterval + drawRegionDistance) - - startPosX = xLeftOffset - chrLabelFont=pid.Font(ttf="verdana",size=24*fontZoom,bold=0) - - for i, _chr in enumerate(ChrList): - - if (i % 2 == 0): - theBackColor = self.GRAPH_BACK_DARK_COLOR - else: - theBackColor = self.GRAPH_BACK_LIGHT_COLOR - # NL:resize chr width for drawing - if float(ChrLengthDistList[i])<90: - ChrLengthDistList[i]=90 - #draw the shaded boxes and the sig/sug thick lines - canvas.drawRect(startPosX, yTopOffset, startPosX + ChrLengthDistList[i]*plotXScale, \ - yTopOffset+plotHeight, edgeColor=pid.gainsboro,fillColor=theBackColor) - - chrNameWidth = canvas.stringWidth(_chr, font=chrLabelFont) - chrStartPix = startPosX + (ChrLengthDistList[i]*plotXScale -chrNameWidth)/2 - chrEndPix = startPosX + (ChrLengthDistList[i]*plotXScale +chrNameWidth)/2 - - canvas.drawString(_chr, chrStartPix, yTopOffset +20,font = chrLabelFont,color=pid.dimgray) - COORDS = "%d,%d,%d,%d" %(chrStartPix, yTopOffset, chrEndPix,yTopOffset +20) - - #add by NL 09-03-2010 - HREF = "javascript:changeView(%d,%s);" % (i,ChrLengthDistList) - Areas = HT.Area(shape='rect',coords=COORDS,href=HREF) - gifmap.areas.append(Areas) - startPosX += (ChrLengthDistList[i]+GraphInterval)*plotXScale - - return plotXScale - - - def drawGraphBackground(self, canvas, gifmap, offset= (80, 120, 80, 50), zoom = 1, startMb = None, endMb = None): - ##conditions - ##multiple Chromosome view - ##single Chromosome Physical - ##single Chromosome Genetic - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - #calculate plot scale - if self.plotScale != 'physic': - self.ChrLengthDistList = self.ChrLengthCMList - drawRegionDistance = self.ChrLengthCMSum - else: - self.ChrLengthDistList = self.ChrLengthMbList - drawRegionDistance = self.ChrLengthMbSum - - if self.selectedChr > -1: #single chromosome view - spacingAmt = plotWidth/13.5 - i = 0 - for startPix in Plot.frange(xLeftOffset, xLeftOffset+plotWidth, spacingAmt): - if (i % 2 == 0): - theBackColor = self.GRAPH_BACK_DARK_COLOR - else: - theBackColor = self.GRAPH_BACK_LIGHT_COLOR - i += 1 - canvas.drawRect(startPix, yTopOffset, min(startPix+spacingAmt, xLeftOffset+plotWidth), \ - yTopOffset+plotHeight, edgeColor=theBackColor, fillColor=theBackColor) - - drawRegionDistance = self.ChrLengthDistList[self.selectedChr] - self.ChrLengthDistList = [drawRegionDistance] - if self.plotScale == 'physic': - plotXScale = plotWidth / (endMb-startMb) - else: - plotXScale = plotWidth / drawRegionDistance - - else: #multiple chromosome view - plotXScale = plotWidth / ((len(self.genotype)-1)*self.GraphInterval + drawRegionDistance) - - startPosX = xLeftOffset - chrLabelFont=pid.Font(ttf="verdana",size=24*fontZoom,bold=0) - - for i, _chr in enumerate(self.genotype): - - if (i % 2 == 0): - theBackColor = self.GRAPH_BACK_DARK_COLOR - else: - theBackColor = self.GRAPH_BACK_LIGHT_COLOR - - #draw the shaded boxes and the sig/sug thick lines - canvas.drawRect(startPosX, yTopOffset, startPosX + self.ChrLengthDistList[i]*plotXScale, \ - yTopOffset+plotHeight, edgeColor=pid.gainsboro,fillColor=theBackColor) - - chrNameWidth = canvas.stringWidth(_chr.name, font=chrLabelFont) - chrStartPix = startPosX + (self.ChrLengthDistList[i]*plotXScale -chrNameWidth)/2 - chrEndPix = startPosX + (self.ChrLengthDistList[i]*plotXScale +chrNameWidth)/2 - - canvas.drawString(_chr.name, chrStartPix, yTopOffset +20,font = chrLabelFont,color=pid.dimgray) - COORDS = "%d,%d,%d,%d" %(chrStartPix, yTopOffset, chrEndPix,yTopOffset +20) - - #add by NL 09-03-2010 - HREF = "javascript:changeView(%d,%s);" % (i,self.ChrLengthMbList) - Areas = HT.Area(shape='rect',coords=COORDS,href=HREF) - gifmap.areas.append(Areas) - startPosX += (self.ChrLengthDistList[i]+self.GraphInterval)*plotXScale - - return plotXScale - - # XZ: The only difference of function drawXAxisForPLINK and function drawXAxis are the function name and the self.plotScale condition. - def drawXAxisForPLINK(self, fd, canvas, drawAreaHeight, gifmap, plotXScale, showLocusForm, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - yZero = canvas.size[1] - yBottomOffset - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - #Parameters - ChrLengthDistList = self.ChrLengthMbList - GraphInterval=self.GraphInterval - - NUM_MINOR_TICKS = 5 # Number of minor ticks between major ticks - X_MAJOR_TICK_THICKNESS = 2 - X_MINOR_TICK_THICKNESS = 1 - X_AXIS_THICKNESS = 1*zoom - - # ======= Alex: Draw the X-axis labels (megabase location) - MBLabelFont = pid.Font(ttf="verdana", size=12*fontZoom, bold=0) - xMajorTickHeight = 15 # How high the tick extends below the axis - xMinorTickHeight = 5*zoom - xAxisTickMarkColor = pid.black - xAxisLabelColor = pid.black - fontHeight = 12*fontZoom # How tall the font that we're using is - spacingFromLabelToAxis = 20 - spacingFromLineToLabel = 3 - - if self.plotScale == 'physic': - strYLoc = yZero + spacingFromLabelToAxis + canvas.fontHeight(MBLabelFont) - ###Physical single chromosome view - if self.selectedChr > -1: - graphMbWidth = endMb - startMb - XScale = Plot.detScale(startMb, endMb) - XStart, XEnd, XStep = XScale - if XStep < 8: - XStep *= 2 - spacingAmtX = spacingAmt = (XEnd-XStart)/XStep - - j = 0 - while abs(spacingAmtX -int(spacingAmtX)) >= spacingAmtX/100.0 and j < 6: - j += 1 - spacingAmtX *= 10 - - formatStr = '%%2.%df' % j - - for counter, _Mb in enumerate(Plot.frange(XStart, XEnd, spacingAmt / NUM_MINOR_TICKS)): - if _Mb < startMb or _Mb > endMb: - continue - Xc = xLeftOffset + plotXScale*(_Mb - startMb) - if counter % NUM_MINOR_TICKS == 0: # Draw a MAJOR mark, not just a minor tick mark - canvas.drawLine(Xc, yZero, Xc, yZero+xMajorTickHeight, color=xAxisTickMarkColor, width=X_MAJOR_TICK_THICKNESS) # Draw the MAJOR tick mark - labelStr = str(formatStr % _Mb) # What Mbase location to put on the label - strWidth = canvas.stringWidth(labelStr, font=MBLabelFont) - drawStringXc = (Xc - (strWidth / 2.0)) - canvas.drawString(labelStr, drawStringXc, strYLoc, font=MBLabelFont, color=xAxisLabelColor, angle=0) - else: - canvas.drawLine(Xc, yZero, Xc, yZero+xMinorTickHeight, color=xAxisTickMarkColor, width=X_MINOR_TICK_THICKNESS) # Draw the MINOR tick mark - # end else - - ###Physical genome wide view - else: - distScale = 0 - startPosX = xLeftOffset - for i, distLen in enumerate(ChrLengthDistList): - if distScale == 0: #universal scale in whole genome mapping - if distLen > 75: - distScale = 25 - elif distLen > 30: - distScale = 10 - else: - distScale = 5 - for tickdists in range(distScale, ceil(distLen), distScale): - canvas.drawLine(startPosX + tickdists*plotXScale, yZero, startPosX + tickdists*plotXScale, yZero + 7, color=pid.black, width=1*zoom) - canvas.drawString(str(tickdists), startPosX+tickdists*plotXScale, yZero + 10*zoom, color=pid.black, font=MBLabelFont, angle=270) - startPosX += (ChrLengthDistList[i]+GraphInterval)*plotXScale - - megabaseLabelFont = pid.Font(ttf="verdana", size=14*zoom*1.5, bold=0) - canvas.drawString("Megabases", xLeftOffset + (plotWidth -canvas.stringWidth("Megabases", font=megabaseLabelFont))/2, - strYLoc + canvas.fontHeight(MBLabelFont) + 5*zoom, font=megabaseLabelFont, color=pid.black) - pass - - canvas.drawLine(xLeftOffset, yZero, xLeftOffset+plotWidth, yZero, color=pid.black, width=X_AXIS_THICKNESS) # Draw the X axis itself - - def drawXAxis(self, fd, canvas, drawAreaHeight, gifmap, plotXScale, showLocusForm, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - yZero = canvas.size[1] - yBottomOffset - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - #Parameters - NUM_MINOR_TICKS = 5 # Number of minor ticks between major ticks - X_MAJOR_TICK_THICKNESS = 2 - X_MINOR_TICK_THICKNESS = 1 - X_AXIS_THICKNESS = 1*zoom - - # ======= Alex: Draw the X-axis labels (megabase location) - MBLabelFont = pid.Font(ttf="verdana", size=12*fontZoom, bold=0) - xMajorTickHeight = 15 # How high the tick extends below the axis - xMinorTickHeight = 5*zoom - xAxisTickMarkColor = pid.black - xAxisLabelColor = pid.black - fontHeight = 12*fontZoom # How tall the font that we're using is - spacingFromLabelToAxis = 20 - spacingFromLineToLabel = 3 - - if self.plotScale == 'physic': - strYLoc = yZero + spacingFromLabelToAxis + canvas.fontHeight(MBLabelFont) - ###Physical single chromosome view - if self.selectedChr > -1: - graphMbWidth = endMb - startMb - XScale = Plot.detScale(startMb, endMb) - XStart, XEnd, XStep = XScale - if XStep < 8: - XStep *= 2 - spacingAmtX = spacingAmt = (XEnd-XStart)/XStep - - j = 0 - while abs(spacingAmtX -int(spacingAmtX)) >= spacingAmtX/100.0 and j < 6: - j += 1 - spacingAmtX *= 10 - - formatStr = '%%2.%df' % j - - for counter, _Mb in enumerate(Plot.frange(XStart, XEnd, spacingAmt / NUM_MINOR_TICKS)): - if _Mb < startMb or _Mb > endMb: - continue - Xc = xLeftOffset + plotXScale*(_Mb - startMb) - if counter % NUM_MINOR_TICKS == 0: # Draw a MAJOR mark, not just a minor tick mark - canvas.drawLine(Xc, yZero, Xc, yZero+xMajorTickHeight, color=xAxisTickMarkColor, width=X_MAJOR_TICK_THICKNESS) # Draw the MAJOR tick mark - labelStr = str(formatStr % _Mb) # What Mbase location to put on the label - strWidth = canvas.stringWidth(labelStr, font=MBLabelFont) - drawStringXc = (Xc - (strWidth / 2.0)) - canvas.drawString(labelStr, drawStringXc, strYLoc, font=MBLabelFont, color=xAxisLabelColor, angle=0) - else: - canvas.drawLine(Xc, yZero, Xc, yZero+xMinorTickHeight, color=xAxisTickMarkColor, width=X_MINOR_TICK_THICKNESS) # Draw the MINOR tick mark - # end else - - ###Physical genome wide view - else: - distScale = 0 - startPosX = xLeftOffset - for i, distLen in enumerate(self.ChrLengthDistList): - if distScale == 0: #universal scale in whole genome mapping - if distLen > 75: - distScale = 25 - elif distLen > 30: - distScale = 10 - else: - distScale = 5 - for tickdists in range(distScale, ceil(distLen), distScale): - canvas.drawLine(startPosX + tickdists*plotXScale, yZero, startPosX + tickdists*plotXScale, yZero + 7, color=pid.black, width=1*zoom) - canvas.drawString(str(tickdists), startPosX+tickdists*plotXScale, yZero + 10*zoom, color=pid.black, font=MBLabelFont, angle=270) - startPosX += (self.ChrLengthDistList[i]+self.GraphInterval)*plotXScale - - megabaseLabelFont = pid.Font(ttf="verdana", size=14*zoom*1.5, bold=0) - canvas.drawString("Megabases", xLeftOffset + (plotWidth -canvas.stringWidth("Megabases", font=megabaseLabelFont))/2, - strYLoc + canvas.fontHeight(MBLabelFont) + 5*zoom, font=megabaseLabelFont, color=pid.black) - pass - else: - ChrAInfo = [] - preLpos = -1 - distinctCount = 0.0 - if len(self.genotype) > 1: - for i, _chr in enumerate(self.genotype): - thisChr = [] - Locus0CM = _chr[0].cM - nLoci = len(_chr) - if nLoci <= 8: - for _locus in _chr: - if _locus.name != ' - ': - if _locus.cM != preLpos: - distinctCount += 1 - preLpos = _locus.cM - thisChr.append([_locus.name, _locus.cM-Locus0CM]) - else: - for j in (0, nLoci/4, nLoci/2, nLoci*3/4, -1): - while _chr[j].name == ' - ': - j += 1 - if _chr[j].cM != preLpos: - distinctCount += 1 - preLpos = _chr[j].cM - thisChr.append([_chr[j].name, _chr[j].cM-Locus0CM]) - ChrAInfo.append(thisChr) - else: - for i, _chr in enumerate(self.genotype): - thisChr = [] - Locus0CM = _chr[0].cM - for _locus in _chr: - if _locus.name != ' - ': - if _locus.cM != preLpos: - distinctCount += 1 - preLpos = _locus.cM - thisChr.append([_locus.name, _locus.cM-Locus0CM]) - ChrAInfo.append(thisChr) - - stepA = (plotWidth+0.0)/distinctCount - - LRectWidth = 10 - LRectHeight = 3 - offsetA = -stepA - lineColor = pid.lightblue - startPosX = xLeftOffset - for j, ChrInfo in enumerate(ChrAInfo): - preLpos = -1 - for i, item in enumerate(ChrInfo): - Lname,Lpos = item - if Lpos != preLpos: - offsetA += stepA - differ = 1 - else: - differ = 0 - preLpos = Lpos - Lpos *= plotXScale - if self.selectedChr > -1: - Zorder = i % 5 - else: - Zorder = 0 - if differ: - canvas.drawLine(startPosX+Lpos,yZero,xLeftOffset+offsetA,\ - yZero+25, color=lineColor) - canvas.drawLine(xLeftOffset+offsetA,yZero+25,xLeftOffset+offsetA,\ - yZero+40+Zorder*(LRectWidth+3),color=lineColor) - rectColor = pid.orange - else: - canvas.drawLine(xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3)-3,\ - xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3),color=lineColor) - rectColor = pid.deeppink - canvas.drawRect(xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3),\ - xLeftOffset+offsetA-LRectHeight,yZero+40+Zorder*(LRectWidth+3)+LRectWidth,\ - edgeColor=rectColor,fillColor=rectColor,edgeWidth = 0) - COORDS="%d,%d,%d,%d"%(xLeftOffset+offsetA-LRectHeight, yZero+40+Zorder*(LRectWidth+3),\ - xLeftOffset+offsetA,yZero+40+Zorder*(LRectWidth+3)+LRectWidth) - HREF="javascript:showDatabase3('%s','%s','%s','');" % (showLocusForm,fd.RISet+"Geno", Lname) - Areas=HT.Area(shape='rect',coords=COORDS,href=HREF, title="Locus : " + Lname) - gifmap.areas.append(Areas) - ##piddle bug - if j == 0: - canvas.drawLine(startPosX,yZero,startPosX,yZero+40, color=lineColor) - startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale - - canvas.drawLine(xLeftOffset, yZero, xLeftOffset+plotWidth, yZero, color=pid.black, width=X_AXIS_THICKNESS) # Draw the X axis itself - - def getColorForMarker(self, chrCount,flag):# no change is needed - chrColorDict={} - for i in range(chrCount): - if flag==1: # display blue and lightblue intercross - chrColorDict[i]=pid.black - elif flag==0: - if (i%2==0): - chrColorDict[i]=pid.blue - else: - chrColorDict[i]=pid.lightblue - else:#display different color for different chr - if i in [0,8,16]: - chrColorDict[i]=pid.black - elif i in [1,9,17]: - chrColorDict[i]=pid.red - elif i in [2,10,18]: - chrColorDict[i]=pid.lightgreen - elif i in [3,11,19]: - chrColorDict[i]=pid.blue - elif i in [4,12]: - chrColorDict[i]=pid.lightblue - elif i in [5,13]: - chrColorDict[i]=pid.hotpink - elif i in [6,14]: - chrColorDict[i]=pid.gold - elif i in [7,15]: - chrColorDict[i]=pid.grey - - return chrColorDict - - - def drawProbeSetPosition(self, canvas, plotXScale, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): - if len(self.traitList) != 1: - return - - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - yZero = canvas.size[1] - yBottomOffset - fontZoom = zoom - if zoom == 2: - fontZoom = 1.5 - - try: - Chr = self.traitList[0].chr # self.traitListChr =self.traitList[0].chr=_vals need to change to chrList and mbList - Mb = self.traitList[0].mb # self.traitListMb =self.traitList[0].mb=_vals - except: - return - - if self.plotScale == 'physic': - if self.selectedChr > -1: - if self.genotype[0].name != Chr or Mb < self.startMb or Mb > self.endMb: - return - else: - locPixel = xLeftOffset + (Mb-self.startMb)*plotXScale - else: - locPixel = xLeftOffset - for i, _chr in enumerate(self.genotype): - if _chr.name != Chr: - locPixel += (self.ChrLengthDistList[i] + self.GraphInterval)*plotXScale - else: - locPixel += Mb*plotXScale - break - else: - if self.selectedChr > -1: - if self.genotype[0].name != Chr: - return - else: - for i, _locus in enumerate(self.genotype[0]): - #the trait's position is on the left of the first genotype - if i==0 and _locus.Mb >= Mb: - locPixel=-1 - break - - #the trait's position is between two traits - if i > 0 and self.genotype[0][i-1].Mb < Mb and _locus.Mb >= Mb: - locPixel = xLeftOffset + plotXScale*(self.genotype[0][i-1].cM+(_locus.cM-self.genotype[0][i-1].cM)*(Mb -self.genotype[0][i-1].Mb)/(_locus.Mb-self.genotype[0][i-1].Mb)) - break - - #the trait's position is on the right of the last genotype - if i==len(self.genotype[0]) and Mb>=_locus.Mb: - locPixel = -1 - else: - locPixel = xLeftOffset - for i, _chr in enumerate(self.genotype): - if _chr.name != Chr: - locPixel += (self.ChrLengthDistList[i] + self.GraphInterval)*plotXScale - else: - locPixel += (Mb*(_chr[-1].cM-_chr[0].cM)/self.ChrLengthCMList[i])*plotXScale - break - if locPixel >= 0: - traitPixel = ((locPixel, yZero), (locPixel-6, yZero+12), (locPixel+6, yZero+12)) - canvas.drawPolygon(traitPixel, edgeColor=pid.black, fillColor=self.TRANSCRIPT_LOCATION_COLOR, closed=1) - - if self.legendChecked: - startPosY = 15 - nCol = 2 - smallLabelFont = pid.Font(ttf="trebuc", size=12, bold=1) - leftOffset = xLeftOffset+(nCol-1)*200 - canvas.drawPolygon(((leftOffset+6, startPosY-6), (leftOffset, startPosY+6), (leftOffset+12, startPosY+6)), edgeColor=pid.black, fillColor=self.TRANSCRIPT_LOCATION_COLOR, closed=1) - canvas.drawString("Sequence Site", (leftOffset+15), (startPosY+5), smallLabelFont, self.TOP_RIGHT_INFO_COLOR) - - # build dict based on plink result, key is chr, value is list of [snp,BP,pValue] - def getPlinkResultDict(self,outputFileName='',thresholdPvalue=-1,ChrOrderIdNameDict={}): - - ChrList =self.ChrList - plinkResultDict={} - - plinkResultfp = open("%s%s.qassoc"% (webqtlConfig.TMPDIR, outputFileName), "rb") - - headerLine=plinkResultfp.readline()# read header line - line = plinkResultfp.readline() - - valueList=[] # initialize value list, this list will include snp, bp and pvalue info - pValueList=[] - count=0 - - while line: - #convert line from str to list - lineList=self.buildLineList(line=line) - - # only keep the records whose chromosome name is in db - if ChrOrderIdNameDict.has_key(int(lineList[0])) and lineList[-1] and lineList[-1].strip()!='NA': - - chrName=ChrOrderIdNameDict[int(lineList[0])] - snp = lineList[1] - BP = lineList[2] - pValue = float(lineList[-1]) - pValueList.append(pValue) - - if plinkResultDict.has_key(chrName): - valueList=plinkResultDict[chrName] - - # pvalue range is [0,1] - if thresholdPvalue >=0 and thresholdPvalue<=1: - if pValue < thresholdPvalue: - valueList.append((snp,BP,pValue)) - count+=1 - - plinkResultDict[chrName]=valueList - valueList=[] - else: - if thresholdPvalue>=0 and thresholdPvalue<=1: - if pValue < thresholdPvalue: - valueList.append((snp,BP,pValue)) - count+=1 - - if valueList: - plinkResultDict[chrName]=valueList - - valueList=[] - - - line =plinkResultfp.readline() - else: - line=plinkResultfp.readline() - - if pValueList: - minPvalue= min(pValueList) - else: - minPvalue=0 - - return count,minPvalue,plinkResultDict - - - ###################################################### - # input: line: str,one line read from file - # function: convert line from str to list; - # output: lineList list - ####################################################### - def buildLineList(self,line=None): - - lineList = string.split(string.strip(line),' ')# irregular number of whitespaces between columns - lineList =[ item for item in lineList if item <>''] - lineList = map(string.strip, lineList) - - return lineList - - #added by NL: automatically generate pheno txt file for PLINK based on strainList passed from dataEditing page - def genPhenoTxtFileForPlink(self,phenoFileName='', RISetName='', probesetName='', valueDict={}): - pedFileStrainList=self.getStrainNameFromPedFile(RISetName=RISetName) - outputFile = open("%s%s.txt"%(webqtlConfig.TMPDIR,phenoFileName),"wb") - headerLine = 'FID\tIID\t%s\n'%probesetName - outputFile.write(headerLine) - - newValueList=[] - - #if valueDict does not include some strain, value will be set to -9999 as missing value - for item in pedFileStrainList: - try: - value=valueDict[item] - value=str(value).replace('value=','') - value=value.strip() - except: - value=-9999 - - newValueList.append(value) - - - newLine='' - for i, strain in enumerate(pedFileStrainList): - j=i+1 - value=newValueList[i] - newLine+='%s\t%s\t%s\n'%(strain, strain, value) - - if j%1000==0: - outputFile.write(newLine) - newLine='' - - if newLine: - outputFile.write(newLine) - - outputFile.close() - - # get strain name from ped file in order - def getStrainNameFromPedFile(self, RISetName=''): - pedFileopen= open("%splink/%s.ped"%(webqtlConfig.GENODIR, RISetName),"r") - line =pedFileopen.readline() - strainNameList=[] - - while line: - lineList=string.split(string.strip(line),'\t') - lineList=map(string.strip,lineList) - - strainName=lineList[0] - strainNameList.append(strainName) - - line =pedFileopen.readline() - - return strainNameList - - ################################################################ - # Generate Chr list, Chr OrderId and Retrieve Length Information - ################################################################ - def getChrNameOrderIdLength(self,RISet=''): - - try: - query = """ - Select - Chr_Length.Name,Chr_Length.OrderId,Length from Chr_Length, InbredSet - where - Chr_Length.SpeciesId = InbredSet.SpeciesId AND - InbredSet.Name = '%s' - Order by OrderId - """ % (RISet) - self.cursor.execute(query) - - results =self.cursor.fetchall() - ChrList=[] - ChrLengthMbList=[] - ChrNameOrderIdDict={} - ChrOrderIdNameDict={} - - for item in results: - ChrList.append(item[0]) - ChrNameOrderIdDict[item[0]]=item[1] # key is chr name, value is orderId - ChrOrderIdNameDict[item[1]]=item[0] # key is orderId, value is chr name - ChrLengthMbList.append(item[2]) - - except: - ChrList=[] - ChrNameOrderIdDict={} - ChrLengthMbList=[] - - return ChrList,ChrNameOrderIdDict,ChrOrderIdNameDict,ChrLengthMbList diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index f28cc63e..3d04e592 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -9,7 +9,7 @@ from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMP import utility.logger logger = utility.logger.getLogger(__name__ ) -def run_gemma(this_dataset, samples, vals, covariates, method, use_loco): +def run_gemma(this_dataset, samples, vals, covariates, use_loco): """Generates p-values for each marker using GEMMA""" if this_dataset.group.genofile != None: @@ -17,7 +17,7 @@ def run_gemma(this_dataset, samples, vals, covariates, method, use_loco): else: genofile_name = this_dataset.group.name - gen_pheno_txt_file(this_dataset, genofile_name, vals, method) + gen_pheno_txt_file(this_dataset, genofile_name, vals) if not os.path.isfile("{}{}_output.assoc.txt".format(webqtlConfig.GENERATED_IMAGE_DIR, genofile_name)): open("{}{}_output.assoc.txt".format(webqtlConfig.GENERATED_IMAGE_DIR, genofile_name), "w+") @@ -33,73 +33,58 @@ def run_gemma(this_dataset, samples, vals, covariates, method, use_loco): if covariates != "": gen_covariates_file(this_dataset, covariates) - if method == "gemma": - gemma_command = GEMMA_COMMAND + ' -bfile %s/%s -k %s/%s.cXX.txt -lmm 1 -maf 0.1' % (flat_files('mapping'), - this_dataset.group.name, - flat_files('mapping'), - this_dataset.group.name) + if use_loco == "True": + k_output_filename = this_dataset.group.name + "_K_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) + generate_k_command = GEMMA_WRAPPER_COMMAND + ' --json --loco ' + chr_list_string + ' -- -g %s/%s_geno.txt -p %s/%s_pheno.txt -a %s/%s_snps.txt -gk -debug > %s/gn2/%s.json' % (flat_files('genotype/bimbam'), + genofile_name, + flat_files('genotype/bimbam'), + genofile_name, + flat_files('genotype/bimbam'), + genofile_name, + TEMPDIR, + k_output_filename) + logger.debug("k_command:" + generate_k_command) + os.system(generate_k_command) + + gemma_command = GEMMA_WRAPPER_COMMAND + ' --json --loco --input %s/gn2/%s.json -- -g %s/%s_geno.txt -p %s/%s_pheno.txt' % (TEMPDIR, + k_output_filename, + flat_files('genotype/bimbam'), + genofile_name, + flat_files('genotype/bimbam'), + genofile_name) + + gwa_output_filename = this_dataset.group.name + "_GWA_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) if covariates != "": - gemma_command += ' -c %s/%s_covariates.txt -outdir %s -o %s_output' % (flat_files('mapping'), - this_dataset.group.name, - webqtlConfig.GENERATED_IMAGE_DIR, - this_dataset.group.name) + gemma_command += ' -c %s/%s_covariates.txt -a %s/%s_snps.txt -lmm 2 -maf 0.1 -debug > %s/gn2/%s.json' % (flat_files('mapping'), + this_dataset.group.name, + flat_files('genotype/bimbam'), + genofile_name, + TEMPDIR, + gwa_output_filename) else: - #gemma_command = GEMMA_COMMAND + ' -bfile %s/%s -k %s/%s.sXX.txt -lmm 1 -maf 0.1 -o %s_output' % (flat_files('mapping'), - gemma_command += ' -outdir %s -o %s_output' % (webqtlConfig.GENERATED_IMAGE_DIR, - this_dataset.group.name) + gemma_command += ' -a %s/%s_snps.txt -lmm 2 -maf 0.1 -debug > %s/gn2/%s.json' % (flat_files('genotype/bimbam'), + genofile_name, + TEMPDIR, + gwa_output_filename) + else: - if use_loco == "True": - k_output_filename = this_dataset.group.name + "_K_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) - generate_k_command = GEMMA_WRAPPER_COMMAND + ' --json --loco ' + chr_list_string + ' -- -g %s/%s_geno.txt -p %s/%s_pheno.txt -a %s/%s_snps.txt -gk -debug > %s/gn2/%s.json' % (flat_files('genotype/bimbam'), - genofile_name, - flat_files('genotype/bimbam'), - genofile_name, - flat_files('genotype/bimbam'), - genofile_name, - TEMPDIR, - k_output_filename) - logger.debug("k_command:" + generate_k_command) - os.system(generate_k_command) - - gemma_command = GEMMA_WRAPPER_COMMAND + ' --json --loco --input %s/gn2/%s.json -- -g %s/%s_geno.txt -p %s/%s_pheno.txt' % (TEMPDIR, - k_output_filename, - flat_files('genotype/bimbam'), - genofile_name, - flat_files('genotype/bimbam'), - genofile_name) - - gwa_output_filename = this_dataset.group.name + "_GWA_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) - if covariates != "": - gemma_command += ' -c %s/%s_covariates.txt -a %s/%s_snps.txt -lmm 1 -maf 0.1 -debug > %s/gn2/%s.json' % (flat_files('mapping'), - this_dataset.group.name, - flat_files('genotype/bimbam'), - genofile_name, - TEMPDIR, - gwa_output_filename) - else: - gemma_command += ' -a %s/%s_snps.txt -lmm 1 -maf 0.1 -debug > %s/gn2/%s.json' % (flat_files('genotype/bimbam'), - genofile_name, - TEMPDIR, - gwa_output_filename) + gemma_command = GEMMA_COMMAND + ' -g %s/%s_geno.txt -p %s/%s_pheno.txt -a %s/%s_snps.txt -k %s/%s.cXX.txt -lmm 2 -maf 0.1' % (flat_files('genotype/bimbam'), + genofile_name, + flat_files('genotype/bimbam'), + genofile_name, + flat_files('genotype/bimbam'), + genofile_name, + flat_files('genotype/bimbam'), + genofile_name) + if covariates != "": + gemma_command += ' -c %s/%s_covariates.txt -outdir %s -debug -o %s_output' % (flat_files('mapping'), + this_dataset.group.name, + webqtlConfig.GENERATED_IMAGE_DIR, + genofile_name) else: - gemma_command = GEMMA_COMMAND + ' -g %s/%s_geno.txt -p %s/%s_pheno.txt -a %s/%s_snps.txt -k %s/%s.cXX.txt -lmm 1 -maf 0.1' % (flat_files('genotype/bimbam'), - genofile_name, - flat_files('genotype/bimbam'), - genofile_name, - flat_files('genotype/bimbam'), - genofile_name, - flat_files('genotype/bimbam'), - genofile_name) - - if covariates != "": - gemma_command += ' -c %s/%s_covariates.txt -outdir %s -debug -o %s_output' % (flat_files('mapping'), - this_dataset.group.name, - webqtlConfig.GENERATED_IMAGE_DIR, - genofile_name) - else: - gemma_command += ' -outdir %s -debug -o %s_output' % (webqtlConfig.GENERATED_IMAGE_DIR, - genofile_name) + gemma_command += ' -outdir %s -debug -o %s_output' % (webqtlConfig.GENERATED_IMAGE_DIR, + genofile_name) logger.debug("gemma_command:" + gemma_command) os.system(gemma_command) @@ -111,31 +96,16 @@ def run_gemma(this_dataset, samples, vals, covariates, method, use_loco): return marker_obs -def gen_pheno_txt_file(this_dataset, genofile_name, vals, method): +def gen_pheno_txt_file(this_dataset, genofile_name, vals): """Generates phenotype file for GEMMA""" - if method == "gemma": - current_file_data = [] - with open("{}/{}.fam".format(flat_files('mapping'), this_dataset.group.name), "r") as outfile: - for i, line in enumerate(outfile): - split_line = line.split() - current_file_data.append(split_line) - - with open("{}/{}.fam".format(flat_files('mapping'), this_dataset.group.name), "w") as outfile: - for i, line in enumerate(current_file_data): - if vals[i] == "x": - this_val = -9 - else: - this_val = vals[i] - outfile.write("0" + " " + line[1] + " " + line[2] + " " + line[3] + " " + line[4] + " " + str(this_val) + "\n") - else: - current_file_data = [] - with open("{}/{}_pheno.txt".format(flat_files('genotype/bimbam'), genofile_name), "w") as outfile: - for value in vals: - if value == "x": - outfile.write("NA\n") - else: - outfile.write(value + "\n") + current_file_data = [] + with open("{}/{}_pheno.txt".format(flat_files('genotype/bimbam'), genofile_name), "w") as outfile: + for value in vals: + if value == "x": + outfile.write("NA\n") + else: + outfile.write(value + "\n") def gen_covariates_file(this_dataset, covariates): covariate_list = covariates.split(",") @@ -172,28 +142,23 @@ def parse_gemma_output(genofile_name): included_markers = [] p_values = [] marker_obs = [] - previous_chr = 0 - #with open("/home/zas1024/gene/wqflask/output/{}_output.assoc.txt".format(this_dataset.group.name)) as output_file: with open("{}{}_output.assoc.txt".format(webqtlConfig.GENERATED_IMAGE_DIR, genofile_name)) as output_file: for line in output_file: - if line.startswith("chr"): + if line.startswith("chr\t"): continue else: marker = {} marker['name'] = line.split("\t")[1] if line.split("\t")[0] != "X" and line.split("\t")[0] != "X/Y": - marker['chr'] = int(line.split("\t")[0]) + if "chr" in line.split("\t")[0]: + marker['chr'] = int(line.split("\t")[0][3:]) + else: + marker['chr'] = int(line.split("\t")[0]) else: marker['chr'] = line.split("\t")[0] - # try: - # marker['chr'] = int(line.split("\t")[0]) - # except: - # marker['chr'] = previous_chr + 1 - # if marker['chr'] != previous_chr: - # previous_chr = marker['chr'] marker['Mb'] = float(line.split("\t")[2]) / 1000000 - marker['p_value'] = float(line.split("\t")[11]) + marker['p_value'] = float(line.split("\t")[9]) if math.isnan(marker['p_value']) or (marker['p_value'] <= 0): marker['lod_score'] = 0 #marker['lrs_value'] = 0 @@ -203,7 +168,7 @@ def parse_gemma_output(genofile_name): marker_obs.append(marker) included_markers.append(line.split("\t")[1]) - p_values.append(float(line.split("\t")[11])) + p_values.append(float(line.split("\t")[9])) return marker_obs @@ -223,20 +188,26 @@ def parse_loco_output(this_dataset, gwa_output_filename): previous_chr = 0 for this_file in output_filelist: - #with open("/home/zas1024/gene/wqflask/output/{}_output.assoc.txt".format(this_dataset.group.name)) as output_file: with open(this_file) as output_file: for line in output_file: - if line.startswith("chr"): + if line.startswith("chr\t"): continue else: marker = {} marker['name'] = line.split("\t")[1] if line.split("\t")[0] != "X" and line.split("\t")[0] != "X/Y": - marker['chr'] = int(line.split("\t")[0]) + if "chr" in line.split("\t")[0]: + marker['chr'] = int(line.split("\t")[0][3:]) + else: + marker['chr'] = int(line.split("\t")[0]) + if marker['chr'] > previous_chr: + previous_chr = marker['chr'] + elif marker['chr'] < previous_chr: + break else: marker['chr'] = line.split("\t")[0] marker['Mb'] = float(line.split("\t")[2]) / 1000000 - marker['p_value'] = float(line.split("\t")[11]) + marker['p_value'] = float(line.split("\t")[9]) if math.isnan(marker['p_value']) or (marker['p_value'] <= 0): marker['lod_score'] = 0 #marker['lrs_value'] = 0 @@ -246,6 +217,6 @@ def parse_loco_output(this_dataset, gwa_output_filename): marker_obs.append(marker) included_markers.append(line.split("\t")[1]) - p_values.append(float(line.split("\t")[11])) + p_values.append(float(line.split("\t")[9])) return marker_obs diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 3ec61e55..033fa2a4 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -16,7 +16,6 @@ import uuid import rpy2.robjects as ro import numpy as np -from scipy import linalg import cPickle as pickle import itertools @@ -84,7 +83,6 @@ class MarkerRegression(object): self.geno_db_exists = start_vars['geno_db_exists'] else: try: - geno_dataset = data_set.create_dataset(self.dataset.group.name + "Geno") self.geno_db_exists = "True" except: self.geno_db_exists = "False" @@ -176,11 +174,11 @@ class MarkerRegression(object): self.genofile_string = start_vars['genofile'] self.dataset.group.genofile = self.genofile_string.split(":")[0] self.dataset.group.get_markers() - if self.mapping_method == "gemma" or self.mapping_method == "gemma_bimbam": + if self.mapping_method == "gemma": self.score_type = "-log(p)" self.manhattan_plot = True with Bench("Running GEMMA"): - marker_obs = gemma_mapping.run_gemma(self.dataset, self.samples, self.vals, self.covariates, self.mapping_method, self.use_loco) + marker_obs = gemma_mapping.run_gemma(self.dataset, self.samples, self.vals, self.covariates, self.use_loco) results = marker_obs elif self.mapping_method == "rqtl_plink": results = self.run_rqtl_plink() @@ -278,7 +276,6 @@ class MarkerRegression(object): ) else: - self.cutoff = 2 self.qtl_results = [] highest_chr = 1 #This is needed in order to convert the highest chr to X/Y for marker in results: @@ -288,9 +285,15 @@ class MarkerRegression(object): if ('lod_score' in marker.keys()) or ('lrs_value' in marker.keys()): self.qtl_results.append(marker) - export_mapping_results(self.dataset, self.this_trait, self.qtl_results, self.mapping_results_path, self.mapping_scale, self.score_type) + with Bench("Exporting Results"): + export_mapping_results(self.dataset, self.this_trait, self.qtl_results, self.mapping_results_path, self.mapping_scale, self.score_type) - self.trimmed_markers = trim_markers_for_table(results) + with Bench("Trimming Markers for Figure"): + if len(self.qtl_results) > 30000: + self.qtl_results = trim_markers_for_figure(self.qtl_results) + + with Bench("Trimming Markers for Table"): + self.trimmed_markers = trim_markers_for_table(results) if self.mapping_method != "gemma": self.json_data['chr'] = [] @@ -324,8 +327,6 @@ class MarkerRegression(object): self.json_data['chrnames'].append([self.species.chromosomes.chromosomes[key].name, self.species.chromosomes.chromosomes[key].mb_length]) chromosome_mb_lengths[key] = self.species.chromosomes.chromosomes[key].mb_length - # logger.debug("json_data:", self.json_data) - self.js_data = dict( result_score_type = self.score_type, json_data = self.json_data, @@ -426,7 +427,6 @@ class MarkerRegression(object): if self.dataset.group.species == "human": p_values, t_stats = self.gen_human_results(pheno_vector, key, temp_uuid) - #p_values = self.trim_results(p_values) else: logger.debug("NOW CWD IS:", os.getcwd()) @@ -478,8 +478,6 @@ class MarkerRegression(object): json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60) results = json.loads(json_results[1]) p_values = [float(result) for result in results['p_values']] - #logger.debug("p_values:", p_values[:10]) - #p_values = self.trim_results(p_values) t_stats = results['t_stats'] #t_stats, p_values = lmm.run( @@ -493,19 +491,8 @@ class MarkerRegression(object): self.dataset.group.markers.add_pvalues(p_values) - #self.get_lod_score_cutoff() - return self.dataset.group.markers.markers - def trim_results(self, p_values): - logger.debug("len_p_values:", len(p_values)) - if len(p_values) > 500: - p_values.sort(reverse=True) - trimmed_values = p_values[:500] - - return trimmed_values - - #def gen_human_results(self, pheno_vector, tempdata): def gen_human_results(self, pheno_vector, key, temp_uuid): file_base = locate(self.dataset.group.name,"mapping") @@ -562,18 +549,6 @@ class MarkerRegression(object): return p_values, t_stats - def get_lod_score_cutoff(self): - logger.debug("INSIDE GET LOD CUTOFF") - high_qtl_count = 0 - for marker in self.dataset.group.markers.markers: - if marker['lod_score'] > 1: - high_qtl_count += 1 - - if high_qtl_count > 1000: - return 1 - else: - return 0 - def identify_empty_samples(self): no_val_samples = [] for sample_count, val in enumerate(self.vals): @@ -597,28 +572,6 @@ class MarkerRegression(object): trimmed_genotype_data.append(new_genotypes) return trimmed_genotype_data -def create_snp_iterator_file(group): - """ - This function is only called by main below - """ - raise Exception("Paths are undefined here") - plink_file_base = os.path.join(TMPDIR, group) - plink_input = input.plink(plink_file_base, type='b') - - data = dict(plink_input = list(plink_input), - numSNPs = plink_input.numSNPs) - - #input_dict = {} - # - #input_dict['plink_input'] = list(plink_input) - #input_dict['numSNPs'] = plink_input.numSNPs - # - - snp_file_base = os.path.join(webqtlConfig.SNP_PATH, group + ".snps.gz") - - with gzip.open(snp_file_base, "wb") as fh: - pickle.dump(data, fh, pickle.HIGHEST_PROTOCOL) - def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, score_type): with open(results_path, "w+") as output_file: output_file.write("Population: " + dataset.group.species.title() + " " + dataset.group.name + "\n") @@ -638,7 +591,6 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, output_file.write(",Dominance") output_file.write("\n") for i, marker in enumerate(markers): - logger.debug("THE MARKER:", marker) output_file.write(marker['name'] + "," + str(marker['chr']) + "," + str(marker['Mb']) + ",") if "lod_score" in marker.keys(): output_file.write(str(marker['lod_score'])) @@ -651,9 +603,50 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, if i < (len(markers) - 1): output_file.write("\n") -def trim_markers_for_table(markers): - num_markers = len(markers) +def trim_markers_for_figure(markers): + if 'lod_score' in markers[0].keys(): + score_type = 'lod_score' + else: + score_type = 'lrs_value' + + filtered_markers = [] + low_counter = 0 + med_counter = 0 + high_counter = 0 + for marker in markers: + if score_type == 'lod_score': + if marker[score_type] < 1: + if low_counter % 20 == 0: + filtered_markers.append(marker) + low_counter += 1 + elif 1 <= marker[score_type] < 2: + if med_counter % 10 == 0: + filtered_markers.append(marker) + med_counter += 1 + elif 2 <= marker[score_type] <= 3: + if high_counter % 2 == 0: + filtered_markers.append(marker) + high_counter += 1 + else: + filtered_markers.append(marker) + else: + if marker[score_type] < 4.16: + if low_counter % 20 == 0: + filtered_markers.append(marker) + low_counter += 1 + elif 4.16 <= marker[score_type] < (2*4.16): + if med_counter % 10 == 0: + filtered_markers.append(marker) + med_counter += 1 + elif (2*4.16) <= marker[score_type] <= (3*4.16): + if high_counter % 2 == 0: + filtered_markers.append(marker) + high_counter += 1 + else: + filtered_markers.append(marker) + return filtered_markers +def trim_markers_for_table(markers): if 'lod_score' in markers[0].keys(): sorted_markers = sorted(markers, key=lambda k: k['lod_score'], reverse=True) else: @@ -664,8 +657,4 @@ def trim_markers_for_table(markers): trimmed_sorted_markers = sorted_markers[:200] return trimmed_sorted_markers else: - return sorted_markers - - -if __name__ == '__main__': - import cPickle as pickle + return sorted_markers
\ No newline at end of file diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 66884b0c..94145bb3 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -30,7 +30,7 @@ from math import * import piddle as pid import sys,os import cPickle -import httplib, urllib +import httplib from flask import Flask, g @@ -41,6 +41,7 @@ from base.GeneralObject import GeneralObject from utility import webqtlUtil from utility import helper_functions from utility import Plot +from utility.benchmark import Bench from wqflask.interval_analyst import GeneUtil from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR, GENERATED_IMAGE_DIR @@ -52,7 +53,6 @@ logger = utility.logger.getLogger(__name__ ) ######################################### class MarkerRegression(object): cMGraphInterval = 5 - maxBootStrap = 50 GRAPH_MIN_WIDTH = 900 GRAPH_MAX_WIDTH = 10000 # Don't set this too high GRAPH_DEFAULT_WIDTH = 1280 @@ -68,12 +68,10 @@ class MarkerRegression(object): # ** GENES ********** BAND_SPACING = 4 - #ENSEMBL_BAND_Y = UCSC_BAND_Y + UCSC_BAND_HEIGHT + BAND_SPACING - UCSC_BAND_HEIGHT = 10 - ENSEMBL_BAND_HEIGHT = 10 - WEBQTL_BAND_HEIGHT = 10 + BAND_HEIGHT = 10 + BAND_HEIGHT = 10 + BAND_HEIGHT = 10 - #GENE_START_Y = ENSEMBL_BAND_Y + ENSEMBL_BAND_HEIGHT + BAND_SPACING NUM_GENE_ROWS = 10 EACH_GENE_HEIGHT = 6 # number of pixels tall, for each gene to display EACH_GENE_ARROW_WIDTH = 5 @@ -81,8 +79,6 @@ class MarkerRegression(object): DRAW_DETAIL_MB = 4 DRAW_UTR_LABELS_MB = 4 - MIN_PIXELS_BETWEEN_LABELS = 50 - qmarkImg = HT.Image('/images/qmarkBoxBlue.gif', width=10, height=13, border=0, alt='Glossary') # Note that "qmark.gif" is a similar, smaller, rounded-edges question mark. It doesn't look # like the ones on the image, though, which is why we don't use it here. @@ -93,11 +89,8 @@ class MarkerRegression(object): NR_INDIVIDUALS = 0 ## END HaplotypeAnalyst - ALEX_DEBUG_BOOL_COLORIZE_GENES = 1 # 0=don't colorize, 1=colorize ALEX_DEBUG_BOOL_PRINT_GENE_LIST = 1 - kWIDTH_DEFAULT=1 - kONE_MILLION = 1000000 LODFACTOR = 4.61 @@ -105,18 +98,14 @@ class MarkerRegression(object): SNP_COLOR = pid.orange # Color for the SNP "seismograph" TRANSCRIPT_LOCATION_COLOR = pid.mediumpurple - GENE_FILL_COLOR = pid.HexColor(0x6666FF) - GENE_OUTLINE_COLOR = pid.HexColor(0x000077) BOOTSTRAP_BOX_COLOR = pid.yellow LRS_COLOR = pid.HexColor(0x0000FF) - LRS_LINE_WIDTH = 2 SIGNIFICANT_COLOR = pid.HexColor(0xEBC7C7) SUGGESTIVE_COLOR = pid.gainsboro SIGNIFICANT_WIDTH = 5 SUGGESTIVE_WIDTH = 5 ADDITIVE_COLOR_POSITIVE = pid.green ADDITIVE_COLOR_NEGATIVE = pid.orange - ADDITIVE_COLOR = ADDITIVE_COLOR_POSITIVE DOMINANCE_COLOR_POSITIVE = pid.darkviolet DOMINANCE_COLOR_NEGATIVE = pid.red @@ -127,20 +116,16 @@ class MarkerRegression(object): HAPLOTYPE_RECOMBINATION = pid.darkgray ## END HaplotypeAnalyst - QMARK_EDGE_COLOR = pid.HexColor(0x718118) - QMARK_FILL_COLOR = pid.HexColor(0xDEE3BB) - TOP_RIGHT_INFO_COLOR = pid.black - X_AXIS_LABEL_COLOR = pid.black #HexColor(0x505050) - - MINI_VIEW_MAGNIFIED_REGION_COLOR = pid.HexColor(0xCC0000) - MINI_VIEW_OUTSIDE_REGION_COLOR = pid.HexColor(0xEEEEEE) - MINI_VIEW_BORDER_COLOR = pid.black CLICKABLE_WEBQTL_REGION_COLOR = pid.HexColor(0xF5D3D3) CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR = pid.HexColor(0xFCE9E9) CLICKABLE_WEBQTL_TEXT_COLOR = pid.HexColor(0x912828) + CLICKABLE_PHENOGEN_REGION_COLOR = pid.HexColor(0xA2FB94) + CLICKABLE_PHENOGEN_REGION_OUTLINE_COLOR = pid.HexColor(0xCEFEC7) + CLICKABLE_PHENOGEN_TEXT_COLOR = pid.HexColor(0x1FD504) + CLICKABLE_UCSC_REGION_COLOR = pid.HexColor(0xDDDDEE) CLICKABLE_UCSC_REGION_OUTLINE_COLOR = pid.HexColor(0xEDEDFF) CLICKABLE_UCSC_TEXT_COLOR = pid.HexColor(0x333366) @@ -154,18 +139,9 @@ class MarkerRegression(object): HELP_PAGE_REF = '/glossary.html' - DRAW_UTR_LABELS=0 - def __init__(self, start_vars): - - #templatePage.__init__(self, fd) - - #if not self.openMysql(): - # return logger.info("Running qtlreaper") - #helper_functions.get_species_dataset_trait(self, start_vars) - self.temp_uuid = start_vars['temp_uuid'] self.dataset = start_vars['dataset'] @@ -190,21 +166,6 @@ class MarkerRegression(object): self.js_data = start_vars['js_data'] self.trimmed_markers = start_vars['trimmed_markers'] #Top markers to display in table - #ZS: Think I can just get all this from dataset object now - #RISet and Species - #if not fd.genotype: - # fd.readGenotype() - # - #fd.parentsf14regression = fd.formdata.getvalue('parentsf14regression') - # - #if ((fd.parentsf14regression == 'on') and fd.genotype_2): - # fd.genotype = fd.genotype_2 - #else: - # fd.genotype = fd.genotype_1 - #fd.strainlist = list(fd.genotype.prgy) - # - #self.species = webqtlDatabaseFunction.retrieveSpecies(cursor=self.cursor, RISet=fd.RISet) - if self.dataset.group.species == "rat": self._ucscDb = "rn3" elif self.dataset.group.species == "mouse": @@ -212,7 +173,6 @@ class MarkerRegression(object): else: self._ucscDb = "" - ##################################### # Options ##################################### @@ -265,17 +225,7 @@ class MarkerRegression(object): if 'use_loco' in start_vars.keys(): self.use_loco = start_vars['use_loco'] - #try: self.selectedChr = int(start_vars['selected_chr']) - #except: - # self.selectedChr = -1 - - #whether include parents and F1 for InbredSet - #fd.parentsf14regression = fd.formdata.getvalue('parentsf14regression') - #if ((fd.parentsf14regression == 'on') and fd.genotype_2): - # fd.genotype = fd.genotype_2 - #else: - # fd.genotype = fd.genotype_1 self.strainlist = self.dataset.group.samplelist self.genotype = self.dataset.group.read_genotype_file() @@ -295,7 +245,6 @@ class MarkerRegression(object): self.graphWidth = self.MULT_GRAPH_DEFAULT_WIDTH ## BEGIN HaplotypeAnalyst - #self.haplotypeAnalystChecked = fd.formdata.getvalue('haplotypeAnalystCheck') if 'haplotypeAnalystCheck' in start_vars.keys(): self.haplotypeAnalystChecked = start_vars['haplotypeAnalystCheck'] else: @@ -308,7 +257,6 @@ class MarkerRegression(object): self.LRS_LOD = start_vars['LRSCheck'] else: self.LRS_LOD = start_vars['score_type'] - self.cutoff = start_vars['cutoff'] self.intervalAnalystChecked = True self.draw2X = False if 'additiveCheck' in start_vars.keys(): @@ -340,34 +288,8 @@ class MarkerRegression(object): except: self.lrsMax = 0 - #self.additiveChecked = fd.formdata.getvalue('additiveCheck') - #self.dominanceChecked = fd.formdata.getvalue('dominanceCheck') - #self.LRS_LOD = fd.formdata.getvalue('LRSCheck', 'LRS') - #self.intervalAnalystChecked = fd.formdata.getvalue('intervalAnalystCheck') - #self.legendChecked = fd.formdata.getvalue('viewLegend') - #self.geneChecked = fd.formdata.getvalue('showGenes') - #self.SNPChecked = fd.formdata.getvalue('showSNP') - #self.draw2X = fd.formdata.getvalue('draw2X') - #self.lrsMax = float(fd.formdata.getvalue('lrsMax', 0)) - #self.startMb = fd.formdata.getvalue('startMb', "-1") - #self.endMb = fd.formdata.getvalue('endMb', "-1") - - #try: - # self.startMb = float(self.startMb) - # self.endMb = float(self.endMb) - # if self.startMb > self.endMb: - # temp = self.startMb - # self.startMb = self.endMb - # self.endMb = temp - # #minimal distance 10bp - # if self.endMb - self.startMb < 0.00001: - # self.endMb = self.startMb + 0.00001 - #except: - # self.startMb = self.endMb = -1 - #Trait Infos self.identification = "" - #self.identification = fd.formdata.getvalue('identification', "") ################################################################ # Generate Chr list and Retrieve Length Information @@ -406,51 +328,10 @@ class MarkerRegression(object): else: self.GraphInterval = self.cMGraphInterval #cM - ################################################################ - # Get Trait Values and Infomation - ################################################################ - ##input from search page or selection page - #self.searchResult = fd.formdata.getvalue('searchResult') - ##convert single selection into a list - #if type("1") == type(self.searchResult): - # self.searchResult = string.split(self.searchResult,'\t') - # - #self.traitList = [] - #if self.searchResult and len(self.searchResult) > webqtlConfig.MULTIPLEMAPPINGLIMIT: - # heading = 'Multiple Interval Mapping' - # detail = ['In order to get clear result, do not select more than %d traits for \ - # Multiple Interval Mapping analysis.' % webqtlConfig.MULTIPLEMAPPINGLIMIT] - # self.error(heading=heading,detail=detail) - # return - #elif self.searchResult: - # self.dataSource = 'selectionPage' - # for item in self.searchResult: - # thisTrait = webqtlTrait(fullname=item, cursor=self.cursor) - # thisTrait.retrieveInfo() - # thisTrait.retrieveData(fd.strainlist) - # self.traitList.append(thisTrait) - #else: - - #input from data editing page - #fd.readData() - #if not fd.allTraitData: - # heading = "Mapping" - # detail = ['No trait data was selected for %s data set. No mapping attempted.' % fd.RISet] - # self.error(heading=heading,detail=detail) - # return - - self.dataSource = 'editingPage' self.traitList = [] thisTrait = start_vars['this_trait'] - #fullname = fd.formdata.getvalue('fullname', '') - #if fullname: - # thisTrait = webqtlTrait(fullname=fullname, data=fd.allTraitData, cursor=self.cursor) - # thisTrait.retrieveInfo() - #else: - # thisTrait = webqtlTrait(data=fd.allTraitData) self.traitList.append(thisTrait) - ## BEGIN HaplotypeAnalyst ## count the amount of individuals to be plotted, and increase self.graphHeight if self.haplotypeAnalystChecked and self.selectedChr > -1: @@ -468,8 +349,6 @@ class MarkerRegression(object): self.NR_INDIVIDUALS = self.NR_INDIVIDUALS + 1 # default: self.graphHeight = self.graphHeight + 2 * (self.NR_INDIVIDUALS+10) * self.EACH_GENE_HEIGHT -## for paper: - # #self.graphHeight = self.graphHeight + 1 * self.NR_INDIVIDUALS * self.EACH_GENE_HEIGHT - 180 ## END HaplotypeAnalyst ################################################################ @@ -477,12 +356,6 @@ class MarkerRegression(object): ################################################################ self.multipleInterval = len(self.traitList) > 1 self.qtlresults = start_vars['qtl_results'] - #errorMessage = self.calculateAllResult(fd) - #if errorMessage: - # heading = "Mapping" - # detail = ['%s' % errorMessage] - # self.error(heading=heading,detail=detail) - # return if self.multipleInterval: self.colorCollection = Plot.colorSpectrum(len(self.qtlresults)) @@ -511,9 +384,6 @@ class MarkerRegression(object): for i, strain in enumerate(self.diffCol): self.diffCol[i] = g.db.execute("select Id from Strain where Symbol = %s", strain).fetchone()[0] - #self.cursor.execute("select Id from Strain where Symbol = %s", strain) - #self.diffCol[i] = self.cursor.fetchone()[0] - #print self.diffCol ################################################################ # GeneCollection goes here @@ -536,13 +406,13 @@ class MarkerRegression(object): chrName = "X" else: chrName = self.selectedChr - self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, webqtldatabase, "mouse") + self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, "mouse") elif self.dataset.group.species == "rat": if self.selectedChr == 21: chrName = "X" else: chrName = self.selectedChr - self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, webqtldatabase, "rat") + self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, "rat") if self.geneCol and self.intervalAnalystChecked: ####################################################################### @@ -551,29 +421,17 @@ class MarkerRegression(object): #through set GENEID is None # ####################################################################### - #GENEID = fd.formdata.getvalue('GeneId') or None GENEID = None - geneTableContainer = HT.Div(Id="sortable") #Div to hold table self.geneTable(self.geneCol, GENEID) - #geneTable = self.geneTable(self.geneCol, GENEID) - #geneTableContainer.append(geneTable) - - #mainfmName = webqtlUtil.genRandStr("fm_") - #tableForm = HT.Form(cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name=mainfmName, submit=HT.Input(type='hidden')) - #tableForm.append(HT.Input(name='FormID', value='', type='hidden')) - #tableForm.append(geneTableContainer) - ################################################################ # Plots goes here ################################################################ - #if self.plotScale != 'physic' or self.multipleInterval: - # showLocusForm = webqtlUtil.genRandStr("fm_") - #else: showLocusForm = "" intCanvas = pid.PILCanvas(size=(self.graphWidth, self.graphHeight)) - gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) + with Bench("Drawing Plot"): + gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) self.gifmap = gifmap.__str__() @@ -586,23 +444,6 @@ class MarkerRegression(object): intCanvasX2 = pid.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2)) gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2) intCanvasX2.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename+"X2"), format='png') - #DLintImgX2=HT.Href(text='Download',url = '/image/'+self.filename+'X2.png', Class='smallsize', target='_blank') - - #textUrl = self.writeQTL2Text(fd, self.filename) - - ################################################################ - # Info tables goes here - ################################################################ - #traitInfoTD = self.traitInfoTD(fd) - - #if self.draw2X: - # traitInfoTD.append(HT.P(), DLintImgX2, ' a higher resolution 2X image. ') - #else: - # traitInfoTD.append(HT.P()) - #if textUrl: - # traitInfoTD.append(HT.BR(), textUrl, ' results in tab-delimited text format.') - #traitRemapTD = self.traitRemapTD(self.cursor, fd) - #topTable = HT.TableLite(HT.TR(traitInfoTD, HT.TD(" ", width=25), traitRemapTD), border=0, cellspacing=0, cellpadding=0) ################################################################ # Outputs goes here @@ -620,158 +461,15 @@ class MarkerRegression(object): if (self.permChecked and self.nperm > 0) and not (self.multipleInterval and 0 < self.nperm): self.perm_filename = self.drawPermutationHistogram() - #perm_text_file = self.permutationTextFile() ################################################################ # footnote goes here ################################################################ btminfo = HT.Paragraph(Id="smallsize") #Small('More information about this graph is available here.') - #if (self.additiveChecked): - # btminfo.append(HT.BR(), 'A positive additive coefficient (', HT.Font('green', color='green'), ' line) indicates that %s alleles increase trait values. In contrast, a negative additive coefficient (' % fd.ppolar, HT.Font('red', color='red'), ' line) indicates that %s alleles increase trait values.' % fd.mpolar) - if self.traitList and self.traitList[0].dataset and self.traitList[0].dataset.type == 'Geno': btminfo.append(HT.BR(), 'Mapping using genotype data as a trait will result in infinity LRS at one locus. In order to display the result properly, all LRSs higher than 100 are capped at 100.') - #if self.permChecked and not self.multipleInterval and 0 < self.nperm: - # TD_LR = HT.TD(HT.Blockquote(gifmap, showLocusForm, HT.P(), btminfo, HT.P(), perm_histogram, HT.P(), perm_text_file), bgColor='#eeeeee', height = 200) - # #TD_LR = HT.TD(HT.Blockquote(topTable), HT.Blockquote(gifmap, showLocusForm, HT.P(), btminfo, HT.P(), perm_histogram, HT.P(), perm_text_file), bgColor='#eeeeee', height = 200) - #else: - TD_LR = HT.TD(HT.Blockquote(gifmap, showLocusForm, HT.P(), btminfo), bgColor='#eeeeee', height = 200) - #TD_LR = HT.TD(HT.Blockquote(topTable), HT.Blockquote(gifmap, showLocusForm, HT.P(), btminfo, HT.P(), perm_histogram, HT.P(), perm_text_file), bgColor='#eeeeee', height = 200) - - - if geneTable: - iaForm = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, "main.py?FormID=intervalAnalyst"), enctype='multipart/form-data', - name="iaForm", submit=HT.Input(type='hidden')) - hddn = {'chromosome':self.genotype[0].name, 'species':self.species,'startMb':self.startMb,'endMb':self.endMb} - if self.diffCol: - hddn['s1'] = self.diffCol[0] - hddn['s2'] = self.diffCol[1] - for key in hddn.keys(): - iaForm.append(HT.Input(name=key, value=hddn[key], type='hidden')) - iaForm.append(HT.Paragraph("Interval Analyst : Chr %s from %2.6f to %2.6f Mb" % (self.genotype[0].name, self.startMb, self.endMb), - HT.Input(name='customize', value='Customize', onClick= "formInNewWindow(this.form);", type='button', Class="button"), Class="subtitle")) - TD_LR.append(HT.Blockquote(iaForm)) - # optionsTable - selectall = HT.Href(url="#redirect", onClick="checkAll(document.getElementsByName('%s')[0]);" % mainfmName) - selectall_img = HT.Image("/images/select_all2_final.jpg", name="selectall", alt="Select All", title="Select All", style="border:none;") - selectall.append(selectall_img) - reset = HT.Href(url="#redirect", onClick="checkNone(document.getElementsByName('%s')[0]); return false;" % mainfmName) - reset_img = HT.Image("/images/select_none2_final.jpg", alt="Select None", title="Select None", style="border:none;") - reset.append(reset_img) - selectinvert = HT.Href(url="#redirect", onClick = "checkInvert(document.getElementsByName('%s')[0]);" % mainfmName) - selectinvert_img = HT.Image("/images/invert_selection2_final.jpg", name="selectinvert", alt="Invert Selection", title="Invert Selection", style="border:none;") - selectinvert.append(selectinvert_img) - addselect = HT.Href(url="#redirect", onClick="addRmvSelection('%s', document.getElementsByName('%s')[0], 'addToSelection');" % (RISet, mainfmName)) - addselect_img = HT.Image("/images/add_collection1_final.jpg", name="addselect", alt="Add To Collection", title="Add To Collection", style="border:none;") - addselect.append(addselect_img) - geneweaver = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'ODEIM');" % mainfmName) - geneweaver_img = HT.Image("/images/ODE_logo_final.jpg", name="GeneWeaver", alt="Gene Weaver", title="Gene Weaver", style="border:none") - geneweaver.append(geneweaver_img) - optionsTable = HT.TableLite() - optionsTable.append(HT.TR( - HT.TD(selectall, width="77", style="text-align:center"), - HT.TD(reset, width="77", style="text-align:center"), - HT.TD(selectinvert, width="77", style="text-align:center"), - HT.TD(geneweaver, width="77", style="text-align:center"), - )) - optionsTable.append(HT.TR( - HT.TD("Select", style="text-align:center"), - HT.TD("Deselect", style="text-align:center"), - HT.TD("Invert", style="text-align:center"), - HT.TD("Gene Weaver", style="text-align:center"), - )) - TD_LR.append(HT.Blockquote(optionsTable)) - # geneTableContainer - TD_LR.append(HT.Blockquote(tableForm)) - - self.body = TD_LR - - #self.dict['body'] = TD_LR - #self.dict['title'] = "Mapping" - - - def writeQTL2Text(self, filename): - if self.multipleInterval: - return "" - #_dominance = (self.genotype.type == 'intercross') - _Mb = self.genotype.Mbmap - - ###Write to text file - fpText = open(os.path.join(webqtlConfig.TMPDIR, filename) + '.txt','wb') - - fpText.write("Source: WebQTL, The GeneNetwork (%s)\n" % webqtlConfig.PORTADDR) - # - fpText.write("Site: GN\n") - fpText.write("Page: Map Viewer\n") - fpText.write(time.strftime("Date and Time (US Center): %b %d, %Y at %I.%M %p\n", time.localtime())) - fpText.write("Trait ID: %s\n" % self.this_trait.name) - fpText.write("Suggestive LRS = %0.2f\n" % self.suggestive) - fpText.write("Significant LRS = %0.2f\n" % self.significant) - """ - if self.this_trait.symbol and self.this_trait.chr and self.this_trait.mb: - writeSymbol, writeChromosome, writeMb = self.this_trait.symbol, self.this_trait.chr, self.this_trait.mb - else: - writeSymbol, writeChromosome, writeMb = (" ", " ", " ") - fpText.write("Gene Symbol: %s\n" % writeSymbol) - fpText.write("Location: Chr %s @ %s Mb\n" % (writeChromosome, writeMb)) - #selectedChr = self.indexToChrName(int(fd.formdata.getvalue('chromosomes', -1))) - #fpText.write("Chromosome: %s\n" % selectedChr) - fpText.write("Region: %0.6f-%0.6f Mb\n\n" % (self.startMb, self.endMb)) - """ - - if hasattr(self, 'LRSArray'): - if _dominance: - fpText.write('Chr\tLocus\tcM\tMb\tLRS\tP-value\tAdditive\tDominance\n') - else: - fpText.write('Chr\tLocus\tcM\tMb\tLRS\tP-value\tAdditive\n') - else: - if _dominance: - fpText.write('Chr\tLocus\tcM\tMb\tLRS\tAdditive\tDominance\n') - else: - fpText.write('Chr\tLocus\tcM\tMb\tLRS\tAdditive\n') - - i = 0 - for marker in self.qtlresults: - if _Mb: - locusMb = '%2.3f' % marker['Mb'] - else: - locusMb = 'N/A' - - if hasattr(self, 'LRSArray'): - if start_vars['score_type'] == "LRS": - lrs_lod = marker['lrs_value'] - else: - lrs_lod = marker['lod_score'] - - P_value = self.calculatePValue(lrs_lod, self.perm_output) - - #if _dominance: - # fpText.write("%s\t%s\t%2.3f\t%s\t%2.3f\t%2.3f\t%2.3f\t%2.3f\n" %(qtlresult.locus.chr, \ - # qtlresult.locus.name, qtlresult.locus.cM, locusMb , qtlresult.lrs, P_value, qtlresult.additive, qtlresult.dominance)) - #else: - if P_value: - fpText.write("%s\t%s\t%2.3f\t%s\t%2.3f\t%2.3f\n" %(marker['chr'], \ - marker['name'], marker['cM'], locusMb, lrs_lod, P_value)) - else: - fpText.write("%s\t%s\t%2.3f\t%s\t%2.3f\t%s\n" %(marker['chr'], \ - marker['name'], marker['cM'], locusMb , lrs_lod, '-')) - else: - #if _dominance: - # fpText.write("%s\t%s\t%2.3f\t%s\t%2.3f\t%2.3f\t%2.3f\n" %(qtlresult.locus.chr, \ - # qtlresult.locus.name, qtlresult.locus.cM, locusMb , qtlresult.lrs, qtlresult.additive, qtlresult.dominance)) - #else: - fpText.write("%s\t%s\t%2.3f\t%s\t%2.3f\n" %(marker['chr'], \ - marker['name'], marker['cM'], locusMb , lrs_lod)) - - i += 1 - - fpText.close() - textUrl = '/tmp/'+filename+'.txt' - #textUrl = HT.Href(text = 'Download', url= '/tmp/'+filename+'.txt', target = "_blank", Class='smallsize') - return textUrl - def plotIntMapping(self, canvas, offset= (80, 120, 20, 100), zoom = 1, startMb = None, endMb = None, showLocusForm = ""): #calculating margins xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset @@ -798,13 +496,14 @@ class MarkerRegression(object): cHeight = canvas.size[1] plotWidth = cWidth - xLeftOffset - xRightOffset plotHeight = cHeight - yTopOffset - yBottomOffset - startPixelX = xLeftOffset - endPixelX = (xLeftOffset + plotWidth) #Drawing Area Height drawAreaHeight = plotHeight if self.plotScale == 'physic' and self.selectedChr > -1: - drawAreaHeight -= self.ENSEMBL_BAND_HEIGHT + self.UCSC_BAND_HEIGHT+ self.WEBQTL_BAND_HEIGHT + 3*self.BAND_SPACING+ 10*zoom + if self.dataset.group.species == "mouse" or self.dataset.group.species == "rat": + drawAreaHeight -= 4*self.BAND_HEIGHT + 4*self.BAND_SPACING+ 10*zoom + else: + drawAreaHeight -= 3*self.BAND_HEIGHT + 3*self.BAND_SPACING+ 10*zoom if self.geneChecked: drawAreaHeight -= self.NUM_GENE_ROWS*self.EACH_GENE_HEIGHT + 3*self.BAND_SPACING + 10*zoom else: @@ -823,7 +522,6 @@ class MarkerRegression(object): #Image map gifmap = HT.Map(name = "WebQTLImageMap") - #gifmap = None newoffset = (xLeftOffset, xRightOffset, yTopOffset, yBottomOffset) # Draw the alternating-color background first and get plotXScale @@ -1002,15 +700,6 @@ class MarkerRegression(object): this_chr = str(self.ChrList[self.selectedChr][0]) else: this_chr = str(self.ChrList[self.selectedChr][1]+1) - # for i, qtlresult in enumerate(self.qtlresults): - # if Chr == this_chr: - # if Mb < self.startMb or Mb > self.endMb: - # return - # else: - # locPixel = xLeftOffset + (Mb-self.startMb)*plotXScale - # break - # elif self.selectedChr == -1: - # if str(qtlresult['chr']) != Chr: if self.plotScale == 'physic': if self.selectedChr > -1: @@ -1068,7 +757,6 @@ class MarkerRegression(object): canvas.drawPolygon(((leftOffset+6, startPosY-6), (leftOffset, startPosY+6), (leftOffset+12, startPosY+6)), edgeColor=pid.black, fillColor=self.TRANSCRIPT_LOCATION_COLOR, closed=1) canvas.drawString("Sequence Site", (leftOffset+15), (startPosY+5), smallLabelFont, self.TOP_RIGHT_INFO_COLOR) - def drawSNPTrackNew(self, canvas, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): if self.plotScale != 'physic' or self.selectedChr == -1 or not self.diffCol: return @@ -1108,7 +796,7 @@ class MarkerRegression(object): snpDensity = float(SNPCounts[i-xLeftOffset]*SNP_HEIGHT_MODIFIER/maxCount) canvas.drawLine(i, drawSNPLocationY+(snpDensity)*zoom, i, drawSNPLocationY-(snpDensity)*zoom, color=self.SNP_COLOR, width=1) - def drawMultiTraitName(self, fd, canvas, gifmap, showLocusForm, offset= (40, 120, 80, 10), zoom = 1, locLocation= None): + def drawMultiTraitName(self, fd, canvas, gifmap, showLocusForm, offset= (40, 120, 80, 10), zoom = 1): nameWidths = [] yPaddingTop = 10 colorFont=pid.Font(ttf="trebuc",size=12,bold=1) @@ -1136,14 +824,12 @@ class MarkerRegression(object): canvas.drawRect(rightShift,yPaddingTop+kstep*15, rectWidth+rightShift,yPaddingTop+10+kstep*15, fillColor=thisLRSColor) canvas.drawString(name,rectWidth+2+rightShift,yPaddingTop+10+kstep*15,font=colorFont,color=pid.black) if thisTrait.db: - COORDS = "%d,%d,%d,%d" %(rectWidth+2+rightShift,yPaddingTop+kstep*15,rectWidth+2+rightShift+nameWidth,yPaddingTop+10+kstep*15,) HREF= "javascript:showDatabase3('%s','%s','%s','');" % (showLocusForm, thisTrait.db.name, thisTrait.name) Areas = HT.Area(shape='rect',coords=COORDS,href=HREF) gifmap.areas.append(Areas) - - def drawLegendPanel(self, canvas, offset= (40, 120, 80, 10), zoom = 1, locLocation= None): + def drawLegendPanel(self, canvas, offset= (40, 120, 80, 10), zoom = 1): xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset plotWidth = canvas.size[0] - xLeftOffset - xRightOffset plotHeight = canvas.size[1] - yTopOffset - yBottomOffset @@ -1187,31 +873,43 @@ class MarkerRegression(object): startPosX = xLeftOffset canvas.drawLine(startPosX, startPosY, startPosX + 32, startPosY, color=self.SIGNIFICANT_COLOR, width=self.SIGNIFICANT_WIDTH) canvas.drawLine(startPosX, startPosY + stepPosY, startPosX + 32, startPosY + stepPosY, color=self.SUGGESTIVE_COLOR, width=self.SUGGESTIVE_WIDTH) - lod = 1 - if self.LRS_LOD == 'LOD': - lod = self.LODFACTOR canvas.drawString('Significant %s = %2.2f' % (self.LRS_LOD, self.significant),xLeftOffset+42,startPosY +5,font=labelFont,color=pid.black) canvas.drawString('Suggestive %s = %2.2f' % (self.LRS_LOD, self.suggestive),xLeftOffset+42,startPosY + 5 +stepPosY,font=labelFont,color=pid.black) - labelFont=pid.Font(ttf="verdana",size=12*fontZoom) + labelFont = pid.Font(ttf="verdana",size=12*fontZoom) labelColor = pid.black if self.selectedChr == -1: string1 = 'Mapping for Dataset: %s, mapping on All Chromosomes' % self.dataset.group.name else: string1 = 'Mapping for Dataset: %s, mapping on Chromosome %s' % (self.dataset.group.name, self.ChrList[self.selectedChr][0]) - if self.controlLocus and self.doControl != "false": - string2 = 'Using %s as control' % self.controlLocus + + string3 = '' + if self.mapping_method == "gemma" or self.mapping_method == "gemma_bimbam": + if self.use_loco == "True": + string2 = 'Using GEMMA mapping method with LOCO and ' + else: + string2 = 'Using GEMMA mapping method with ' + if self.covariates != "": + string2 += 'the cofactors below:' + cofactor_names = ", ".join([covar.split(":")[0] for covar in self.covariates.split(",")]) + string3 = cofactor_names + else: + string2 += 'no cofactors' + elif self.mapping_method == "rqtl_plink" or self.mapping_method == "rqtl_geno": + string2 = 'Using R/qtl mapping method with ' + if self.controlLocus and self.doControl != "false": + string2 += '%s as control' % self.controlLocus + else: + string2 += 'no control for other QTLs' + elif self.mapping_method == "plink": + string2 = 'Using PLINK mapping method with no control for other QTLs' else: - if self.mapping_method == "gemma" or self.mapping_method == "gemma_bimbam": - string2 = 'Using GEMMA mapping method with no control for other QTLs.' - if self.covariates != "": - string3 = 'Using following traits as covariates: ' + self.covariates - elif self.mapping_method == "rqtl_plink" or self.mapping_method == "rqtl_geno": - string2 = 'Using R/qtl mapping method with no control for other QTLs.' - elif self.mapping_method == "plink": - string2 = 'Using PLINK mapping method with no control for other QTLs.' + string2 = 'Using Haldane mapping function with ' + if self.controlLocus and self.doControl != "false": + string2 += '%s as control' % self.controlLocus else: - string2 = 'Using Haldane mapping function with no control for other QTLs' + string2 += 'no control for other QTLs' + if self.this_trait.name: identification = "Trait ID: %s : %s" % (self.dataset.fullname, self.this_trait.name) d = 4+ max(canvas.stringWidth(identification, font=labelFont), canvas.stringWidth(string1, font=labelFont), canvas.stringWidth(string2, font=labelFont)) @@ -1220,6 +918,8 @@ class MarkerRegression(object): d = 4+ max(canvas.stringWidth(string1, font=labelFont), canvas.stringWidth(string2, font=labelFont)) canvas.drawString(string1,canvas.size[0] - xRightOffset-d,35*fontZoom,font=labelFont,color=labelColor) canvas.drawString(string2,canvas.size[0] - xRightOffset-d,50*fontZoom,font=labelFont,color=labelColor) + if string3 != '': + canvas.drawString(string3,canvas.size[0] - xRightOffset-d,65*fontZoom,font=labelFont,color=labelColor) def drawGeneBand(self, canvas, gifmap, plotXScale, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): @@ -1236,9 +936,6 @@ class MarkerRegression(object): yPaddingTop = yTopOffset - displayStartInBases = startMb*self.kONE_MILLION - displayEndInBases = endMb*self.kONE_MILLION - for gIndex, theGO in enumerate(self.geneCol): geneNCBILink = 'http://www.ncbi.nlm.nih.gov/gene?term=%s' if self.dataset.group.species == "mouse": @@ -1253,7 +950,6 @@ class MarkerRegression(object): cdsStart = theGO['cdsStart'] cdsEnd = theGO['cdsEnd'] accession = theGO['NM_ID'] - geneId = theGO['GeneID'] geneSymbol = theGO["GeneSymbol"] strand = theGO["Strand"] exonCount = theGO["exonCount"] @@ -1271,10 +967,7 @@ class MarkerRegression(object): geneStartPix = xLeftOffset; # clip the first in-range gene #color the gene based on SNP density - - #found earlier, needs to be recomputed as snps are added - #always apply colors now, even if SNP Track not checked - Zach 11/24/2010 densities=[1.0000000000000001e-05, 0.094094033555233408, 0.3306166377816987, 0.88246026851027781, 2.6690084029581951, 4.1, 61.0] @@ -1307,7 +1000,6 @@ class MarkerRegression(object): txEnd = theGO["TxEnd"] cdsStart = theGO["TxStart"] cdsEnd = theGO["TxEnd"] - geneId = theGO["GeneID"] geneSymbol = theGO["GeneSymbol"] strand = theGO["Strand"] exonCount = 0 @@ -1336,11 +1028,10 @@ class MarkerRegression(object): #Draw Genes geneYLocation = yPaddingTop + (gIndex % self.NUM_GENE_ROWS) * self.EACH_GENE_HEIGHT*zoom - - if 1:#drawClickableRegions: - geneYLocation += self.UCSC_BAND_HEIGHT + self.BAND_SPACING + self.ENSEMBL_BAND_HEIGHT + self.BAND_SPACING + self.WEBQTL_BAND_HEIGHT + self.BAND_SPACING + if self.dataset.group.species == "mouse" or self.dataset.group.species == "rat": + geneYLocation += 4*self.BAND_HEIGHT + 4*self.BAND_SPACING else: - geneYLocation += self.BAND_SPACING + geneYLocation += 3*self.BAND_HEIGHT + 3*self.BAND_SPACING #draw the detail view if self.endMb - self.startMb <= self.DRAW_DETAIL_MB and geneEndPix - geneStartPix > self.EACH_GENE_ARROW_SPACING * 3: @@ -1348,7 +1039,6 @@ class MarkerRegression(object): arrowColor = pid.Color(0.7, 0.7, 0.7) #draw the line that runs the entire length of the gene - #canvas.drawString(str(geneStartPix), 300, 400) canvas.drawLine(geneStartPix, geneYLocation + self.EACH_GENE_HEIGHT/2*zoom, geneEndPix, geneYLocation + self.EACH_GENE_HEIGHT/2*zoom, color=outlineColor, width=1) #draw the arrows @@ -1382,10 +1072,6 @@ class MarkerRegression(object): #draw gray blocks for 3' and 5' UTR blocks if cdsStart and cdsEnd: - logger.debug("txStart:", txStart) - logger.debug("cdsStart:", cdsStart) - logger.debug("txEnd:", txEnd) - logger.debug("cdsEnd:", cdsEnd) utrStartPix = (txStart-startMb)*plotXScale + xLeftOffset utrEndPix = (cdsStart-startMb)*plotXScale + xLeftOffset if (utrStartPix < xLeftOffset): @@ -1398,7 +1084,6 @@ class MarkerRegression(object): utrStartPix = xLeftOffset + plotWidth #canvas.drawRect(utrStartPix, geneYLocation, utrEndPix, (geneYLocation+self.EACH_GENE_HEIGHT*zoom), edgeColor=utrColor, fillColor =utrColor) - #if self.DRAW_UTR_LABELS and self.endMb - self.startMb <= self.DRAW_UTR_LABELS_MB: if self.endMb - self.startMb <= self.DRAW_UTR_LABELS_MB: if strand == "-": labelText = "3'" @@ -1420,7 +1105,6 @@ class MarkerRegression(object): utrStartPix = xLeftOffset + plotWidth #canvas.drawRect(utrStartPix, geneYLocation, utrEndPix, (geneYLocation+self.EACH_GENE_HEIGHT*zoom), edgeColor=utrColor, fillColor =utrColor) - #if self.DRAW_UTR_LABELS and self.endMb - self.startMb <= self.DRAW_UTR_LABELS_MB: if self.endMb - self.startMb <= self.DRAW_UTR_LABELS_MB: if strand == "-": labelText = "5'" @@ -1441,8 +1125,6 @@ class MarkerRegression(object): if self.plotScale != 'physic' or self.selectedChr == -1 or not self.geneCol: return - fpText = open(os.path.join(webqtlConfig.TMPDIR, "hallo") + '.txt','wb') - clickableRegionLabelFont=pid.Font(ttf="verdana", size=9, bold=0) xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset @@ -1450,13 +1132,9 @@ class MarkerRegression(object): plotHeight = canvas.size[1] - yTopOffset - yBottomOffset yZero = canvas.size[1] - yBottomOffset fontZoom = zoom - widthMultiplier = 1 yPaddingTop = yTopOffset - exprdrawn = 0 - - #thisTrait = self.traitList[0] thisTrait = self.this_trait _strains, _vals, _vars, _aliases = thisTrait.export_informative() @@ -1466,7 +1144,6 @@ class MarkerRegression(object): temp = GeneralObject(name=_strains[ii], value=_val) smd.append(temp) - smd.sort(lambda A, B: cmp(A.value, B.value)) smd.reverse() @@ -1491,13 +1168,9 @@ class MarkerRegression(object): drawit = 0; if drawit == 1: - if self.genotype[0][i].name != " - " : - plotRight = geneEndPix + 4 - - #### end find out PlotRight firstGene = 1 @@ -1552,11 +1225,10 @@ class MarkerRegression(object): #Draw Genes geneYLocation = yPaddingTop + self.NUM_GENE_ROWS * (self.EACH_GENE_HEIGHT)*zoom - - if 1:#drawClickableRegions: - geneYLocation += self.UCSC_BAND_HEIGHT + self.BAND_SPACING + self.ENSEMBL_BAND_HEIGHT + self.BAND_SPACING + self.WEBQTL_BAND_HEIGHT + self.BAND_SPACING + if self.dataset.group.species == "mouse" or self.dataset.group.species == "rat": + geneYLocation += 4*self.BAND_HEIGHT + 4*self.BAND_SPACING else: - geneYLocation += self.BAND_SPACING + geneYLocation += 3*self.BAND_HEIGHT + 3*self.BAND_SPACING if self.genotype[0][i].name != " - " : @@ -1643,8 +1315,6 @@ class MarkerRegression(object): canvas.drawString("%s" % (samplelist[j]), (xLeftOffset + plotWidth + 10) , geneYLocation+8+2*ind*self.EACH_GENE_HEIGHT*zoom, font=pid.Font(ttf="verdana", size=12, bold=0), color=pid.black) canvas.drawString("%2.2f" % (expr), (xLeftOffset + plotWidth + 60) , geneYLocation+8+2*ind*self.EACH_GENE_HEIGHT*zoom, font=pid.Font(ttf="verdana", size=12, bold=0), color=pid.black) - fpText.close() - ## END HaplotypeAnalyst def drawClickBand(self, canvas, gifmap, plotXScale, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): @@ -1676,57 +1346,76 @@ class MarkerRegression(object): i = 0 paddingTop = yTopOffset - ucscPaddingTop = paddingTop + self.WEBQTL_BAND_HEIGHT + self.BAND_SPACING - ensemblPaddingTop = ucscPaddingTop + self.UCSC_BAND_HEIGHT + self.BAND_SPACING + if self.dataset.group.species == "mouse" or self.dataset.group.species == "rat": + phenogenPaddingTop = paddingTop + (self.BAND_HEIGHT + self.BAND_SPACING) + ucscPaddingTop = paddingTop + 2*(self.BAND_HEIGHT + self.BAND_SPACING) + ensemblPaddingTop = paddingTop + 3*(self.BAND_HEIGHT + self.BAND_SPACING) + else: + ucscPaddingTop = paddingTop + (self.BAND_HEIGHT + self.BAND_SPACING) + ensemblPaddingTop = paddingTop + 2*(self.BAND_HEIGHT + self.BAND_SPACING) if zoom == 1: for pixel in range(xLeftOffset, xLeftOffset + plotWidth, pixelStep): calBase = self.kONE_MILLION*(startMb + (endMb-startMb)*(pixel-xLeftOffset-0.0)/plotWidth) + if pixel == (xLeftOffset + pixelStep*2): + logger.debug("CALBASE:", calBase) + logger.debug("FLANKING:", flankingWidthInBases) xBrowse1 = pixel xBrowse2 = min(xLeftOffset + plotWidth, (pixel + pixelStep - 1)) - WEBQTL_COORDS = "%d, %d, %d, %d" % (xBrowse1, paddingTop, xBrowse2, (paddingTop+self.WEBQTL_BAND_HEIGHT)) - bandWidth = xBrowse2 - xBrowse1 + WEBQTL_COORDS = "%d, %d, %d, %d" % (xBrowse1, paddingTop, xBrowse2, (paddingTop+self.BAND_HEIGHT)) WEBQTL_HREF = "javascript:rangeView('%s', %f, %f)" % (self.selectedChr - 1, max(0, (calBase-webqtlZoomWidth))/1000000.0, (calBase+webqtlZoomWidth)/1000000.0) WEBQTL_TITLE = "Click to view this section of the genome in WebQTL" gifmap.areas.append(HT.Area(shape='rect',coords=WEBQTL_COORDS,href=WEBQTL_HREF, title=WEBQTL_TITLE)) - canvas.drawRect(xBrowse1, paddingTop, xBrowse2, (paddingTop + self.WEBQTL_BAND_HEIGHT), edgeColor=self.CLICKABLE_WEBQTL_REGION_COLOR, fillColor=self.CLICKABLE_WEBQTL_REGION_COLOR) - canvas.drawLine(xBrowse1, paddingTop, xBrowse1, (paddingTop + self.WEBQTL_BAND_HEIGHT), color=self.CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR) + canvas.drawRect(xBrowse1, paddingTop, xBrowse2, (paddingTop + self.BAND_HEIGHT), edgeColor=self.CLICKABLE_WEBQTL_REGION_COLOR, fillColor=self.CLICKABLE_WEBQTL_REGION_COLOR) + canvas.drawLine(xBrowse1, paddingTop, xBrowse1, (paddingTop + self.BAND_HEIGHT), color=self.CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR) + + if self.dataset.group.species == "mouse" or self.dataset.group.species == "rat": + PHENOGEN_COORDS = "%d, %d, %d, %d" % (xBrowse1, phenogenPaddingTop, xBrowse2, (phenogenPaddingTop+self.BAND_HEIGHT)) + if self.dataset.group.species == "mouse": + PHENOGEN_HREF = "https://phenogen.ucdenver.edu/PhenoGen/gene.jsp?speciesCB=Mm&auto=Y&geneTxt=chr%s:%d-%d&genomeVer=mm10" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) + else: + PHENOGEN_HREF = "https://phenogen.ucdenver.edu/PhenoGen/gene.jsp?speciesCB=Mm&auto=Y&geneTxt=chr%s:%d-%d&genomeVer=mm10" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) + PHENOGEN_TITLE = "Click to view this section of the genome in PhenoGen" + gifmap.areas.append(HT.Area(shape='rect',coords=PHENOGEN_COORDS,href=PHENOGEN_HREF, title=PHENOGEN_TITLE)) + canvas.drawRect(xBrowse1, phenogenPaddingTop, xBrowse2, (phenogenPaddingTop+self.BAND_HEIGHT), edgeColor=self.CLICKABLE_PHENOGEN_REGION_COLOR, fillColor=self.CLICKABLE_PHENOGEN_REGION_COLOR) + canvas.drawLine(xBrowse1, phenogenPaddingTop, xBrowse1, (phenogenPaddingTop+self.BAND_HEIGHT), color=self.CLICKABLE_PHENOGEN_REGION_OUTLINE_COLOR) - UCSC_COORDS = "%d, %d, %d, %d" %(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT)) + UCSC_COORDS = "%d, %d, %d, %d" %(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.BAND_HEIGHT)) if self.dataset.group.species == "mouse": UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=%s/snp/chr%s" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases, webqtlConfig.PORTADDR, self.selectedChr) else: UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) UCSC_TITLE = "Click to view this section of the genome in the UCSC Genome Browser" gifmap.areas.append(HT.Area(shape='rect',coords=UCSC_COORDS,href=UCSC_HREF, title=UCSC_TITLE)) - canvas.drawRect(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), edgeColor=self.CLICKABLE_UCSC_REGION_COLOR, fillColor=self.CLICKABLE_UCSC_REGION_COLOR) - canvas.drawLine(xBrowse1, ucscPaddingTop, xBrowse1, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), color=self.CLICKABLE_UCSC_REGION_OUTLINE_COLOR) + canvas.drawRect(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.BAND_HEIGHT), edgeColor=self.CLICKABLE_UCSC_REGION_COLOR, fillColor=self.CLICKABLE_UCSC_REGION_COLOR) + canvas.drawLine(xBrowse1, ucscPaddingTop, xBrowse1, (ucscPaddingTop+self.BAND_HEIGHT), color=self.CLICKABLE_UCSC_REGION_OUTLINE_COLOR) - ENSEMBL_COORDS = "%d, %d, %d, %d" %(xBrowse1, ensemblPaddingTop, xBrowse2, (ensemblPaddingTop+self.ENSEMBL_BAND_HEIGHT)) + ENSEMBL_COORDS = "%d, %d, %d, %d" %(xBrowse1, ensemblPaddingTop, xBrowse2, (ensemblPaddingTop+self.BAND_HEIGHT)) if self.dataset.group.species == "mouse": ENSEMBL_HREF = "http://www.ensembl.org/Mus_musculus/contigview?highlight=&chr=%s&vc_start=%d&vc_end=%d&x=35&y=12" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) else: ENSEMBL_HREF = "http://www.ensembl.org/Rattus_norvegicus/contigview?chr=%s&start=%d&end=%d" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) ENSEMBL_TITLE = "Click to view this section of the genome in the Ensembl Genome Browser" gifmap.areas.append(HT.Area(shape='rect',coords=ENSEMBL_COORDS,href=ENSEMBL_HREF, title=ENSEMBL_TITLE)) - canvas.drawRect(xBrowse1, ensemblPaddingTop, xBrowse2, (ensemblPaddingTop+self.ENSEMBL_BAND_HEIGHT), edgeColor=self.CLICKABLE_ENSEMBL_REGION_COLOR, fillColor=self.CLICKABLE_ENSEMBL_REGION_COLOR) - canvas.drawLine(xBrowse1, ensemblPaddingTop, xBrowse1, (ensemblPaddingTop+self.ENSEMBL_BAND_HEIGHT), color=self.CLICKABLE_ENSEMBL_REGION_OUTLINE_COLOR) + canvas.drawRect(xBrowse1, ensemblPaddingTop, xBrowse2, (ensemblPaddingTop+self.BAND_HEIGHT), edgeColor=self.CLICKABLE_ENSEMBL_REGION_COLOR, fillColor=self.CLICKABLE_ENSEMBL_REGION_COLOR) + canvas.drawLine(xBrowse1, ensemblPaddingTop, xBrowse1, (ensemblPaddingTop+self.BAND_HEIGHT), color=self.CLICKABLE_ENSEMBL_REGION_OUTLINE_COLOR) # end for - canvas.drawString("Click to view the corresponding section of the genome in an 8x expanded WebQTL map", (xLeftOffset + 10), paddingTop + self.WEBQTL_BAND_HEIGHT/2, font=clickableRegionLabelFont, color=self.CLICKABLE_WEBQTL_TEXT_COLOR) - canvas.drawString("Click to view the corresponding section of the genome in the UCSC Genome Browser", (xLeftOffset + 10), ucscPaddingTop + self.UCSC_BAND_HEIGHT/2, font=clickableRegionLabelFont, color=self.CLICKABLE_UCSC_TEXT_COLOR) - canvas.drawString("Click to view the corresponding section of the genome in the Ensembl Genome Browser", (xLeftOffset+10), ensemblPaddingTop + self.ENSEMBL_BAND_HEIGHT/2, font=clickableRegionLabelFont, color=self.CLICKABLE_ENSEMBL_TEXT_COLOR) + canvas.drawString("Click to view the corresponding section of the genome in an 8x expanded WebQTL map", (xLeftOffset + 10), paddingTop + self.BAND_HEIGHT/2, font=clickableRegionLabelFont, color=self.CLICKABLE_WEBQTL_TEXT_COLOR) + if self.dataset.group.species == "mouse" or self.dataset.group.species == "rat": + canvas.drawString("Click to view the corresponding section of the genome in PhenoGen", (xLeftOffset + 10), phenogenPaddingTop + self.BAND_HEIGHT/2, font=clickableRegionLabelFont, color=self.CLICKABLE_PHENOGEN_TEXT_COLOR) + canvas.drawString("Click to view the corresponding section of the genome in the UCSC Genome Browser", (xLeftOffset + 10), ucscPaddingTop + self.BAND_HEIGHT/2, font=clickableRegionLabelFont, color=self.CLICKABLE_UCSC_TEXT_COLOR) + canvas.drawString("Click to view the corresponding section of the genome in the Ensembl Genome Browser", (xLeftOffset+10), ensemblPaddingTop + self.BAND_HEIGHT/2, font=clickableRegionLabelFont, color=self.CLICKABLE_ENSEMBL_TEXT_COLOR) #draw the gray text chrFont = pid.Font(ttf="verdana", size=26*zoom, bold=1) traitFont = pid.Font(ttf="verdana", size=14, bold=0) chrX = xLeftOffset + plotWidth - 2 - canvas.stringWidth("Chr %s" % self.ChrList[self.selectedChr][0], font=chrFont) canvas.drawString("Chr %s" % self.ChrList[self.selectedChr][0], chrX, ensemblPaddingTop-5, font=chrFont, color=pid.gray) - traitX = chrX - 28 - canvas.stringWidth("database", font=traitFont) # end of drawBrowserClickableRegions else: #draw the gray text @@ -1734,7 +1423,6 @@ class MarkerRegression(object): traitFont = pid.Font(ttf="verdana", size=14, bold=0) chrX = xLeftOffset + (plotWidth - canvas.stringWidth("Chr %s" % currentChromosome, font=chrFont))/2 canvas.drawString("Chr %s" % currentChromosome, chrX, 32, font=chrFont, color=pid.gray) - traitX = chrX - 28 - canvas.stringWidth("database", font=traitFont) # end of drawBrowserClickableRegions pass @@ -1761,13 +1449,11 @@ class MarkerRegression(object): xAxisLabelColor = pid.black fontHeight = 12*fontZoom # How tall the font that we're using is spacingFromLabelToAxis = 5 - spacingFromLineToLabel = 3 if self.plotScale == 'physic': strYLoc = yZero + spacingFromLabelToAxis + canvas.fontHeight(MBLabelFont) ###Physical single chromosome view if self.selectedChr > -1: - graphMbWidth = endMb - startMb XScale = Plot.detScale(startMb, endMb) XStart, XEnd, XStep = XScale if XStep < 8: @@ -1793,7 +1479,6 @@ class MarkerRegression(object): canvas.drawString(labelStr, drawStringXc, strYLoc, font=MBLabelFont, color=xAxisLabelColor, angle=0) else: canvas.drawLine(Xc, yZero, Xc, yZero+xMinorTickHeight, color=xAxisTickMarkColor, width=X_MINOR_TICK_THICKNESS) # Draw the MINOR tick mark - # end else ###Physical genome wide view else: @@ -1823,7 +1508,6 @@ class MarkerRegression(object): preLpos = -1 distinctCount = 0.0 - #if len(self.genotype) > 1: if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes for i, _chr in enumerate(self.genotype): thisChr = [] @@ -1932,12 +1616,6 @@ class MarkerRegression(object): #LRSTop is then defined to be above the LRS_LOD_Max by enough to add one additional LRSScale increment. #if we are using a set-scale, then we set LRSTop to be the user's value, and LRS_LOD_Max doesn't matter. - #ZS: I'm not sure what this if statement is supposed to do. It appears to work correctly for both LOD and LRS if I just set lodm to 1.0 - # if self.LRS_LOD == 'LRS': - # lodm = self.LODFACTOR - # else: - # lodm = 1.0 - #ZS: This is a mess, but I don't know a better way to account for different mapping methods returning results in different formats + the option to change between LRS and LOD if self.lrsMax <= 0: #sliding scale if "lrs_value" in self.qtlresults[0]: @@ -2054,10 +1732,6 @@ class MarkerRegression(object): else: if self.additiveChecked: additiveMax = max(map(lambda X : abs(X['additive']), self.qtlresults)) - #if INTERCROSS: - # dominanceMax = max(map(lambda X : abs(X.dominance), self.qtlresults[0])) - #else: - # dominanceMax = -1 lrsEdgeWidth = 2 if zoom == 2: @@ -2076,10 +1750,9 @@ class MarkerRegression(object): startPosX = xLeftOffset for i, qtlresult in enumerate(self.qtlresults): m = 0 - #startPosX = xLeftOffset thisLRSColor = self.colorCollection[0] - if qtlresult['chr'] != previous_chr and self.selectedChr == -1: + if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) @@ -2117,32 +1790,18 @@ class MarkerRegression(object): AdditiveCoordXY = [] previous_chr = qtlresult['chr'] previous_chr_as_int += 1 - newStartPosX = (self.ChrLengthDistList[previous_chr_as_int - 1]+self.GraphInterval)*plotXScale if newStartPosX != oldStartPosX: startPosX += newStartPosX oldStartPosX = newStartPosX - #startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale - - #for j, _chr in enumerate(self.genotype): #ZS: This is beause the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used if self.plotScale == "physic": this_chr = str(self.ChrList[self.selectedChr][0]) else: this_chr = str(self.ChrList[self.selectedChr][1]+1) if self.selectedChr == -1 or str(qtlresult['chr']) == this_chr: - #AdditiveCoordXY = [] - #DominanceCoordXY = [] - #for k, _locus in enumerate(_chr): Xc = startPosX + (qtlresult['Mb']-startMb)*plotXScale - #if self.plotScale == 'physic': - #Xc = startPosX + (_locus.Mb-startMb)*plotXScale - #Xc = startPosX + (qtlresult['Mb']-startMb)*plotXScale - #else: - #Xc = startPosX + (_locus.cM-_chr[0].cM)*plotXScale - #Xc = startPosX + (qtlresult['cM']-qtlresult[0]['cM'])*plotXScale - # updated by NL 06-18-2011: # fix the over limit LRS graph issue since genotype trait may give infinite LRS; # for any lrs is over than 460(LRS max in this system), it will be reset to 460 @@ -2165,11 +1824,6 @@ class MarkerRegression(object): Yc = yZero - qtlresult['lod_score']*self.LODFACTOR*LRSHeightThresh/LRS_LOD_Max else: Yc = yZero - qtlresult['lod_score']*LRSHeightThresh/LRS_LOD_Max - #if qtlresult['lrs_value'] > 460 or qtlresult['lrs_value']=='inf': - #if self.qtlresults[j]['lrs_value'] > 460 or self.qtlresults[j]['lrs_value']=='inf': - # Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRS_LOD_Max - #else: - # Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/LRS_LOD_Max if self.manhattan_plot == True: point_color = pid.black @@ -2182,11 +1836,8 @@ class MarkerRegression(object): additiveMax = 0.000001 Yc = yZero - qtlresult['additive']*AdditiveHeightThresh/additiveMax AdditiveCoordXY.append((Xc, Yc)) - # if not self.multipleInterval and INTERCROSS and self.additiveChecked: - # Yc = yZero - qtlresult['dominance']*DominanceHeightThresh/dominanceMax - # DominanceCoordXY.append((Xc, Yc)) + m += 1 - #canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) @@ -2258,7 +1909,6 @@ class MarkerRegression(object): additiveScale = Plot.detScaleOld(0,additiveMax) additiveStep = (additiveScale[1]-additiveScale[0])/additiveScale[2] additiveAxisList = Plot.frange(0, additiveScale[1], additiveStep) - maxAdd = additiveScale[1] addPlotScale = AdditiveHeightThresh/additiveMax additiveAxisList.append(additiveScale[1]) @@ -2350,298 +2000,6 @@ class MarkerRegression(object): return plotXScale - def calculateAllResult(self, fd): - - weightedRegression = fd.formdata.getvalue('applyVarianceSE') - - self.genotype = self.genotype.addinterval() - resultSlice = [] - controlGeno = [] - - if self.multipleInterval: - self.suggestive = 0 - self.significant = 0 - if self.selectedChr > -1: - self.genotype.chromosome = [self.genotype[self.selectedChr]] - else: - #single interval mapping - try: - self.suggestive = float(fd.formdata.getvalue('permSuggestive')) - self.significant = float(fd.formdata.getvalue('permSignificance')) - except: - self.suggestive = None - self.significant = None - - _strains, _vals, _vars = self.traitList[0].exportInformative(weightedRegression) - - if webqtlUtil.ListNotNull(_vars): - pass - else: - weightedRegression = 0 - _strains, _vals, _vars = self.traitList[0].exportInformative() - - ##locate genotype of control Locus - if self.controlLocus: - controlGeno2 = [] - _FIND = 0 - for _chr in self.genotype: - for _locus in _chr: - if _locus.name == self.controlLocus: - controlGeno2 = _locus.genotype - _FIND = 1 - break - if _FIND: - break - if controlGeno2: - _prgy = list(self.genotype.prgy) - for _strain in _strains: - _idx = _prgy.index(_strain) - controlGeno.append(controlGeno2[_idx]) - else: - return "The control marker you selected is not in the genofile." - - if weightedRegression: - self.perm_output = self.genotype.permutation(strains = _strains, trait = _vals, - variance = _vars, nperm=self.nperm) - else: - self.perm_output = self.genotype.permutation(strains = _strains, trait = _vals, - nperm=self.nperm) - - if self.significant and self.suggestive: - pass - else: - if self.nperm < 100: - self.suggestive = 0 - self.significant = 0 - else: - self.suggestive = self.perm_output[int(self.nperm*0.37-1)] - self.significant = self.perm_output[int(self.nperm*0.95-1)] - - #calculating bootstrap - #from now on, genotype could only contain a single chromosome - #permutation need to be performed genome wide, this is not the case for bootstrap - - #due to the design of qtlreaper, composite regression need to be performed genome wide - if not self.controlLocus and self.selectedChr > -1: - self.genotype.chromosome = [self.genotype[self.selectedChr]] - elif self.selectedChr > -1: #self.controlLocus and self.selectedChr > -1 - lociPerChr = map(len, self.genotype) - resultSlice = reduce(lambda X, Y: X+Y, lociPerChr[:self.selectedChr], 0) - resultSlice = [resultSlice,resultSlice+lociPerChr[self.selectedChr]] - else: - pass - - #calculate QTL for each trait - self.qtlresults = [] - - for thisTrait in self.traitList: - _strains, _vals, _vars = thisTrait.exportInformative(weightedRegression) - if self.controlLocus: - if weightedRegression: - qtlresult = self.genotype.regression(strains = _strains, trait = _vals, - variance = _vars, control = self.controlLocus) - else: - qtlresult = self.genotype.regression(strains = _strains, trait = _vals, - control = self.controlLocus) - if resultSlice: - qtlresult = qtlresult[resultSlice[0]:resultSlice[1]] - else: - if weightedRegression: - qtlresult = self.genotype.regression(strains = _strains, trait = _vals, - variance = _vars) - else: - qtlresult = self.genotype.regression(strains = _strains, trait = _vals) - - self.qtlresults.append(qtlresult) - - if not self.multipleInterval: - if self.controlLocus and self.selectedChr > -1: - self.genotype.chromosome = [self.genotype[self.selectedChr]] - - if self.bootChecked: - if controlGeno: - self.bootResult = self.genotype.bootstrap(strains = _strains, trait = _vals, - control = controlGeno, nboot=fd.nboot) - elif weightedRegression: - self.bootResult = self.genotype.bootstrap(strains = _strains, trait = _vals, - variance = _vars, nboot=fd.nboot) - else: - self.bootResult = self.genotype.bootstrap(strains = _strains, trait = _vals, - nboot=fd.nboot) - else: - self.bootResult = [] - - def calculatePValue (self, query_LRS, permutation_LRS_array): - query_index = len(permutation_LRS_array) - for i, one_permutation_LRS in enumerate(permutation_LRS_array): - if one_permutation_LRS >= query_LRS: - query_index = i - break - try: - P_value = float(len(permutation_LRS_array) - query_index) / len(permutation_LRS_array) - except: - P_value = '' - return P_value - - def helpButton(self, anchor): - return HT.Href(self.HELP_PAGE_REF + '#%s' % anchor, self.qmarkImg, target=self.HELP_WINDOW_NAME) - - - def traitRemapTD(self, cursor, fd): - chrList = HT.Select(name="chromosomes", data=self.ChrList, selected=[self.selectedChr], - onChange="chrLength(this.form.chromosomes.value, this.form.scale.value, this.form, self.ChrLengthMbList);") - - physicOnly = HT.Span(' *', Class="cr") - - showSNPCheck = HT.Input(type='checkbox', Class='checkbox', name='showSNP', value='ON', checked=self.SNPChecked) - showSNPText = HT.Span('SNP Track ', self.helpButton("snpSeismograph"), Class="fs12 fwn") - - showGenesCheck = HT.Input(type='checkbox', Class='checkbox', name='showGenes', value='ON', checked=self.geneChecked) - showGenesText = HT.Span('Gene Track', Class="fs12 fwn") - - showIntervalAnalystCheck = HT.Input(type='checkbox', Class='checkbox', name='intervalAnalystCheck', value='ON', checked=self.intervalAnalystChecked) - showIntervalAnalystText = HT.Span('Interval Analyst', Class="fs12 fwn") -## BEGIN HaplotypeAnalyst - - showHaplotypeAnalystCheck = HT.Input(type='checkbox', Class='checkbox', name='haplotypeAnalystCheck', value='ON', checked=self.haplotypeAnalystChecked) - showHaplotypeAnalystText = HT.Span('Haplotype Analyst', Class="fs12 fwn") -## END HaplotypeAnalyst - - leftBox = HT.Input(type="text", name="startMb", size=10) - rightBox = HT.Input(type="text", name="endMb", size=10) - if self.selectedChr > -1 and self.plotScale=='physic': - leftBox.value = self.startMb - rightBox.value = self.endMb - - scaleBox = HT.Select(name="scale", onChange="chrLength(this.form.chromosomes.value, this.form.scale.value, this.form, self.ChrLengthMbList);") - scaleBox.append(("Genetic", "morgan")) - if fd.genotype.Mbmap: - scaleBox.append(("Physical", "physic")) - scaleBox.selected.append(self.plotScale) - - permBox = HT.Input(type="checkbox", name="permCheck", value='ON', checked=self.permChecked, Class="checkbox") - permText = HT.Span("Permutation Test ", self.helpButton("Permutation"), Class="fs12 fwn") - bootBox = HT.Input(type="checkbox", name="bootCheck", value='ON', checked=self.bootChecked, Class="checkbox") - bootText = HT.Span("Bootstrap Test ", self.helpButton("bootstrap"), Class="fs12 fwn") - additiveBox = HT.Input(type="checkbox", name="additiveCheck", value='ON', checked=self.additiveChecked, Class="checkbox") - additiveText = HT.Span("Allele Effects ", self.helpButton("additive"), Class="fs12 fwn") - dominanceBox = HT.Input(type="checkbox", name="dominanceCheck", value='ON', checked=self.dominanceChecked, Class="checkbox") - dominanceText = HT.Span("Dominance Effects ", self.helpButton("Dominance"), Class="fs12 fwn") - - lrsRadio = HT.Input(type="radio", name="LRSCheck", value='LRS', checked = (self.LRS_LOD == "LRS")) - lodRadio = HT.Input(type="radio", name="LRSCheck", value='LOD', checked = (self.LRS_LOD != "LRS")) - lrsMaxBox = HT.Input(type="text", name="lrsMax", value=self.lrsMax, size=3) - widthBox = HT.Input(type="text", name="graphWidth", size=5, value=str(self.graphWidth)) - legendBox = HT.Input(type="checkbox", name="viewLegend", value='ON', checked=self.legendChecked, Class="checkbox") - legendText = HT.Span("Legend", Class="fs12 fwn") - - draw2XBox = HT.Input(type="checkbox", name="draw2X", value='ON', Class="checkbox") - draw2XText = HT.Span("2X Plot", Class="fs12 fwn") - - regraphButton = HT.Input(type="button", Class="button", onClick="javascript:databaseFunc(this.form,'showIntMap');", value="Remap") - controlsForm = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype="multipart/form-data", name="changeViewForm", submit=HT.Input(type='hidden')) - controlsTable = HT.TableLite(border=0) - innerControlsTable = HT.TableLite(border=0) - if self.selectedChr == -1: - minimumGraphWidth = self.MULT_GRAPH_MIN_WIDTH - else: - minimumGraphWidth = self.GRAPH_MIN_WIDTH - innerControlsTable.append( - HT.TR(HT.TD("Chr: ", Class="fs12 fwb ffl"),HT.TD(chrList, scaleBox, regraphButton)), - HT.TR(HT.TD("View: ", Class="fs12 fwb ffl"),HT.TD(leftBox, " to ", rightBox, "Mb", physicOnly, NOWRAP="on")), - HT.TR(HT.TD("Units: ", Class="fs12 fwb ffl"), HT.TD(lrsRadio, "LRS ", lodRadio, "LOD ", self.helpButton("LOD"))), - HT.TR(HT.TD(" ", Class="fs12 fwb ffl"), HT.TD(lrsMaxBox, "units on Y-axis (0 for default)", Class="fs11 fwn")), - HT.TR(HT.TD("Width: ", Class="fs12 fwb ffl"), HT.TD(widthBox, "pixels (minimum=%d)" % minimumGraphWidth, Class="fs11 fwn ")) - ) - #whether SNP - # comment this, because this will make caculation very slow. - #cursor.execute("Select Species.Id from SnpAll, Species where SnpAll.SpeciesId = Species.Id and Species.Name = %s limit 1", self.species) - #SNPorNot = cursor.fetchall() - SNPorNot = True - #Whether Gene - cursor.execute("Select Species.Id from GeneList, Species where GeneList.SpeciesId = Species.Id and Species.Name = %s limit 1", self.species) - GeneorNot = cursor.fetchall() - - if self.multipleInterval: - optionPanel = HT.TD(valign="top", NOWRAP="on") - else: - optionPanel = HT.TD(permBox, permText, HT.BR(), bootBox, bootText, HT.BR(), additiveBox, additiveText, HT.BR(), valign="top", NOWRAP="on") - #whether dominance - if self.genotype.type == 'intercross': - optionPanel.append(dominanceBox, dominanceText, HT.BR()) - if SNPorNot: - optionPanel.append(showSNPCheck, showSNPText, physicOnly, HT.BR()) - if GeneorNot: - optionPanel.append(showGenesCheck, showGenesText, physicOnly, HT.BR(), - showIntervalAnalystCheck, showIntervalAnalystText, physicOnly, HT.BR()) -## BEGIN HaplotypeAnalyst - optionPanel.append(showHaplotypeAnalystCheck, showHaplotypeAnalystText, physicOnly, HT.BR()) -## END HaplotypeAnalyst - optionPanel.append(legendBox, legendText, HT.BR(),draw2XBox, draw2XText) - controlsTable.append( - HT.TR(HT.TD(innerControlsTable, valign="top"), - HT.TD(" ", width=15), optionPanel), - HT.TR(HT.TD(physicOnly, " only apply to single chromosome physical mapping", align="Center", colspan=3, Class="fs11 fwn")) - ) - controlsForm.append(controlsTable) - - controlsForm.append(HT.Input(name="permSuggestive", value=self.suggestive, type="hidden")) - controlsForm.append(HT.Input(name="permSignificance", value=self.significant, type="hidden")) - -## BEGIN HaplotypeAnalyst #### haplotypeAnalystCheck added below -## END HaplotypeAnalyst - - for key in fd.formdata.keys(): - if key == "searchResult" and type([]) == type(fd.formdata.getvalue(key)): - controlsForm.append(HT.Input(name=key, value=string.join(fd.formdata.getvalue(key), "\t"), type="hidden")) - elif key not in ("endMb", "startMb", "chromosomes", "scale", "permCheck", "bootCheck", "additiveCheck", "dominanceCheck", - "LRSCheck", "intervalAnalystCheck", "haplotypeAnalystCheck", "lrsMax", "graphWidth", "viewLegend", 'showGenes', 'showSNP', 'draw2X', - 'permSuggestive', "permSignificance"): - controlsForm.append(HT.Input(name=key, value=fd.formdata.getvalue(key), type="hidden")) - else: - pass - - # updated by NL, move function changeView(i) to webqtl.js and change it to function changeView(i, Chr_Mb_list) - # move function chrLength(a, b, c) to webqtl.js and change it to function chrLength(a, b, c, Chr_Mb_list) - self.dict['js1'] = '<script src="/javascript/sorttable.js"></script>' - return HT.TD(controlsForm, Class="doubleBorder", width=400) - - def traitInfoTD(self, fd): - if self.selectedChr == -1: - intMapHeading = HT.Paragraph('Map Viewer: Whole Genome', Class="title") - else: - intMapHeading = HT.Paragraph('Map Viewer: Chr %s' % self.genotype[0].name, Class="title") - - heading2 = HT.Paragraph(HT.Strong('Population: '), "%s %s" % (self.species.title(), fd.RISet) , HT.BR()) - #Trait is from an database - if self.traitList and self.traitList[0] and self.traitList[0].db: - #single trait - if len(self.traitList) == 1: - thisTrait = self.traitList[0] - trait_url = HT.Href(text=thisTrait.name, url = os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE) + \ - "?FormID=showDatabase&incparentsf1=1&database=%s&ProbeSetID=%s" % (thisTrait.db.name, thisTrait.name), \ - target='_blank', Class="normalsize") - heading2.append(HT.Strong("Database: "), HT.Href(text=thisTrait.db.fullname, url = webqtlConfig.INFOPAGEHREF % thisTrait.db.name ,\ - target='_blank',Class="normalsize"),HT.BR()) - if thisTrait.db.type == 'ProbeSet': - heading2.append(HT.Strong('Trait ID: '), trait_url, HT.BR(), - HT.Strong("Gene Symbol: "), HT.Italic('%s' % thisTrait.symbol,id="green"),HT.BR()) - if thisTrait.chr and thisTrait.mb: - heading2.append(HT.Strong("Location: "), 'Chr %s @ %s Mb' % (thisTrait.chr, thisTrait.mb)) - elif thisTrait.db.type == 'Geno': - heading2.append(HT.Strong('Locus : '), trait_url, HT.BR()) - if thisTrait.chr and thisTrait.mb: - heading2.append(HT.Strong("Location: "), 'Chr %s @ %s Mb' % (thisTrait.chr, thisTrait.mb)) - elif thisTrait.db.type == 'Publish': - heading2.append(HT.Strong('Record ID: '), trait_url, HT.BR()) - else: - pass - else: - heading2.append(HT.Strong("Traits: "), "Multiple Traits") - else: - heading2.append(HT.Strong("Trait Name: "), fd.identification) - return HT.TD(intMapHeading, heading2, valign="top") - def drawPermutationHistogram(self): ######################################### # Permutation Graph @@ -2660,78 +2018,18 @@ class MarkerRegression(object): return filename - # img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test') - - # self.suggestive = self.perm_output[int(self.nperm*0.37-1)] - # self.significant = self.perm_output[int(self.nperm*0.95-1)] - # self.highlysignificant = self.perm_output[int(self.nperm*0.99-1)] - - # permutationHeading = HT.Paragraph('Histogram of Permutation Test') - # permutationHeading.__setattr__("class","title") - - # permutation = HT.TableLite() - # permutation.append(HT.TR(HT.TD(img)), - # HT.TR(HT.TD('')), - # HT.TR(HT.TD('Total of %d permutations'%self.nperm))) - - # return permutation - - def permutationTextFile(self): - filename= webqtlUtil.genRandStr("Reg_") - fpText = open('%s.txt' % (webqtlConfig.TMPDIR+filename), 'wb') - fpText.write('Suggestive LRS (p = 0.63) = %3.2f\n'%self.suggestive) - fpText.write('Significant LRS (p = 0.05) = %3.2f\n'%self.significant) - fpText.write('Highly Significant LRS (p = 0.01) = %3.2f\n\n'%self.highlysignificant) - fpText.write('%s Permutations\n\n' % str(len(self.perm_output))) - LRSInfo =HT.Paragraph(' Suggestive LRS = %3.2f\n'%self.suggestive, - HT.BR(), - ' Significant LRS =%3.2f\n'%self.significant, - HT.BR(), - ' Highly Significant LRS =%3.2f\n' % self.highlysignificant) - - for lrs_value in self.perm_output: - fpText.write(str(lrs_value) + "\n") - - textUrl = HT.Href(text = 'Download Permutation Results', url= '/tmp/'+filename+'.txt', target = "_blank", Class='fs12 fwn') - - return textUrl - def geneTable(self, geneCol, refGene=None): - #SNPLink = 0 #Not sure what this is used for - if self.dataset.group.species == 'mouse' or self.dataset.group.species == 'rat': - #gene_tblobj = {} - self.gene_table_header = self.getGeneTableHeader(refGene=None) + self.gene_table_header = self.getGeneTableHeaderList(refGene=None) self.gene_table_body = self.getGeneTableBody(geneCol, refGene=None) - #gene_tblobj["header"] = self.getGeneTableHeader(refGene=None) - #gene_tblobj["body"] = self.getGeneTableBody(geneCol, refGene=None) - - #sortby = self.getSortByValue() - - #filename= webqtlUtil.genRandStr("Mapping_") - - #objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') - #cPickle.dump(gene_tblobj, objfile) - #objfile.close() - - #gene_table = webqtlUtil.genTableObj(tblobj=gene_tblobj, file=filename, sortby=sortby, tableID="sortable", addIndex="0") else: self.gene_table_header = None self.gene_table_body = None - #gene_table = "" - #return gene_table - - def getGeneTableHeader(self, refGene=None): - - gene_tblobj_header = [] + def getGeneTableHeaderList(self, refGene=None): gene_table_header_list = [] - - col_class = "fs14 fwb ffl b1 cw cbrb" - if self.dataset.group.species == "mouse": - if refGene: gene_table_header_list = ["Index", "Symbol", @@ -2744,25 +2042,6 @@ class MarkerRegression(object): "Mb Start (hg19)", "Literature Correlation", "Gene Description"] - #"PolymiRTS Database" + HT.Href(url='http://compbio.uthsc.edu/miRSNP/', text='>>', target="_blank").__str__(), - #"Gene Weaver Info Content" + HT.Href(url='http://geneweaver.org/', text='>>', target="_blank").__str__()] - - # gene_tblobj_header = [[THCell(HT.TD('Index', HT.BR(), HT.BR(), align='left', width=50, Class=col_class), text="index", idx=0), - # THCell(HT.TD('Symbol', HT.BR(), HT.BR(), align='left', width=100, Class=col_class), text="symbol", idx=1), - # THCell(HT.TD('Mb Start',HT.BR(),'(mm9)', align='left', width=100, Class=col_class), text="mb_start_mm9", idx=2), - # THCell(HT.TD('Length (Kb)', HT.BR(), HT.BR(), align='left', width=100, Class=col_class), text="length", idx=3), - # THCell(HT.TD('SNP',HT.BR(),'Count', align='left', width=47, Class=col_class), text="snp_count", idx=4), - # THCell(HT.TD('SNP',HT.BR(),'Density', align='left', width=78, Class=col_class), text="snp_density", idx=5), - # THCell(HT.TD('Avg',HT.BR(),'Expr', HT.BR(), HT.BR(), align='left', width=44, Class=col_class), sort=0, idx=6), - # THCell(HT.TD('Human',HT.BR(),'Chr', align='left', width=60, Class=col_class), text="human_chr", idx=7), - # THCell(HT.TD('Mb Start',HT.BR(),'(hg19)', align='left', width=100, Class=col_class), text="mb_start_hg19", idx=8), - # THCell(HT.TD('Literature',HT.BR(),'Correlation', align='left', width=100, Class=col_class), text="lit_corr", idx=9), - # THCell(HT.TD('Gene Description', HT.BR(), HT.BR(), align='left', width=290, Class=col_class), text="description", idx=10), - # THCell(HT.TD('PolymiRTS',HT.BR(),'Database', HT.BR(), HT.Href(url='http://compbio.uthsc.edu/miRSNP/', text='>>', target="_blank", Class="normalsize"), - # align='left', width=100, Class=col_class), sort=0, idx=11), - # THCell(HT.TD('Gene Weaver', HT.BR(), 'Info Content', HT.BR(), HT.Href(url='http://geneweaver.org/', text='>>', target="_blank", Class="normalsize"), - # align='left', width=110, Class=col_class), sort=0, idx=12), - # ]] else: gene_table_header_list = ["", "Index", @@ -2775,27 +2054,7 @@ class MarkerRegression(object): "Human Chr", "Mb Start (hg19)", "Gene Description"] - #"PolymiRTS Database" + HT.Href(url='http://compbio.uthsc.edu/miRSNP/', text='>>', target="_blank").__str__(), - #"Gene Weaver Info Content" + HT.Href(url='http://geneweaver.org/', text='>>', target="_blank").__str__()] - - # gene_tblobj_header = [[THCell(HT.TD('Index', HT.BR(), HT.BR(), align='left', width=50, Class=col_class), text="index", idx=0), - # THCell(HT.TD('Symbol', HT.BR(), HT.BR(), align='left', width=100, Class=col_class), text="symbol", idx=1), - # THCell(HT.TD('Mb Start',HT.BR(),'(mm9)', align='left', width=100, Class=col_class), text="mb_start_mm9", idx=2), - # THCell(HT.TD('Length (Kb)', HT.BR(), HT.BR(), align='left', width=100, Class=col_class), text="length", idx=3), - # THCell(HT.TD('SNP',HT.BR(),'Count', align='left', width=47, Class=col_class), text="snp_count", idx=4), - # THCell(HT.TD('SNP',HT.BR(),'Density', align='left', width=78, Class=col_class), text="snp_density", idx=5), - # THCell(HT.TD('Avg',HT.BR(),'Expr', HT.BR(), HT.BR(), align='left', width=44, Class=col_class), sort=0, idx=6), - # THCell(HT.TD('Human',HT.BR(),'Chr', align='left', width=60, Class=col_class), text="human_chr", idx=7), - # THCell(HT.TD('Mb Start',HT.BR(),'(hg19)', align='left', width=100, Class=col_class), text="mb_start_hg19", idx=8), - # THCell(HT.TD('Gene Description', HT.BR(), HT.BR(), align='left', width=290, Class=col_class), text="description", idx=9), - # THCell(HT.TD('PolymiRTS',HT.BR(),'Database', HT.BR(), HT.Href(url='http://compbio.uthsc.edu/miRSNP/', text='>>', target="_blank", Class="normalsize"), - # align='left', width=100, Class=col_class), sort=0, idx=10), - # THCell(HT.TD('Gene Weaver', HT.BR(), 'Info Content', HT.BR(), HT.Href(url='http://geneweaver.org/', text='>>', target="_blank", Class="normalsize"), - # align='left', width=110, Class=col_class), sort=0, idx=11), - # ]] - elif self.dataset.group.species == "rat": - gene_table_header_list = ["", "Index", "Symbol", @@ -2808,52 +2067,13 @@ class MarkerRegression(object): "Mb Start (hg19)", "Gene Description"] - # gene_tblobj_header = [[THCell(HT.TD('Index', HT.BR(), HT.BR(), align='left', width=50, Class=col_class), text="index", idx=0), - # THCell(HT.TD('Symbol', HT.BR(), HT.BR(), align='left', width=100, Class=col_class), text="symbol", idx=1), - # THCell(HT.TD('Mb Start',HT.BR(),'(rn3)', align='left', width=100, Class=col_class), text="mb_start_rn3", idx=2), - # THCell(HT.TD('Length (Kb)', HT.BR(), HT.BR(), align='left', width=100, Class=col_class), text="length", idx=3), - # THCell(HT.TD('Avg',HT.BR(),'Expr', HT.BR(), HT.BR(), align='left', width=44, Class=col_class), sort=0, idx=4), - # THCell(HT.TD('Mouse',HT.BR(),'Chr', align='left', width=60, Class=col_class), text="mouse_chr", idx=5), - # THCell(HT.TD('Mb Start',HT.BR(),'(mm9)', align='left', width=100, Class=col_class), text="mb_start_mm9", idx=6), - # THCell(HT.TD('Human',HT.BR(),'Chr', align='left', width=60, Class=col_class), text="human_chr", idx=7), - # THCell(HT.TD('Mb Start',HT.BR(),'(hg19)', align='left', width=100, Class=col_class), text="mb_start_hg19", idx=8), - # THCell(HT.TD('Gene Description', HT.BR(), HT.BR(), align='left', Class=col_class), text="description", idx=9)]] - - else: - pass - return gene_table_header_list - #return gene_tblobj_header def getGeneTableBody(self, geneCol, refGene=None): - - tblobj_body = [] #contains table rows - className = "fs13 b1 c222" - gene_table_body = [] tableIterationsCnt = 0 if self.dataset.group.species == "mouse": - - # polymiRTS - # http://lily.uthsc.edu:8080/20090422_UTHSC_cuiyan/PolymiRTS_CLS?chrom=2&chrom_from=115&chrom_to=125 - #XZ: We can NOT assume their web service is always on. We must put this block of code in try except. - try: - conn = httplib.HTTPConnection("lily.uthsc.edu:8080") - conn.request("GET", "/20090422_UTHSC_cuiyan/PolymiRTS_CLS?chrom=%s&chrom_from=%s&chrom_to=%s" % (self.genotype[0].name, self.startMb, self.endMb)) - response = conn.getresponse() - data = response.read() - data = data.split() - conn.close() - dic = {} - index = 0 - for i in data: - if index%3==0: - dic[data[index]] = HT.Href(url=data[index+2], text=data[index+1], target="_blank", Class="normalsize") - index = index+1 - except Exception: - dic={} - for gIndex, theGO in enumerate(geneCol): tableIterationsCnt = tableIterationsCnt + 1 @@ -2867,7 +2087,6 @@ class MarkerRegression(object): txEnd = theGO["TxEnd"] theGO["snpDensity"] = theGO["snpCount"]/geneLength if self.ALEX_DEBUG_BOOL_PRINT_GENE_LIST: - #accessionString = 'http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?CMD=Display&DB=gene&term=%s' % theGO["NM_ID"] geneIdString = 'http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s' % theGO["GeneID"] allProbeString = '%s?cmd=sch&gene=%s&alias=1' % (os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), theGO["GeneSymbol"]) @@ -2880,10 +2099,6 @@ class MarkerRegression(object): mouseStartString = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=vertebrate&org=Mouse&db=mm9&position=chr" + theGO["Chromosome"] + "%3A" + str(int(theGO["TxStart"] * 1000000.0)) + "-" + str(int(theGO["TxEnd"]*1000000.0)) +"&pix=620&Submit=submit" - #Used for sorting - mouseStartValue = int(theGO["TxStart"]) - - #the chromosomes for human 1 are 1qXX.XX if theGO['humanGene']: if theGO['humanGene']["TxStart"] == '': @@ -2892,20 +2107,11 @@ class MarkerRegression(object): humanStartDisplay = "%0.6f" % theGO['humanGene']["TxStart"] humanChr = theGO['humanGene']["Chromosome"] - if humanChr.find('q'): - humanChrSort = humanChr[:humanChr.find("q")].join(humanChr[(humanChr.find("q")+1):]) #value used when sorting table - elif humanChr.find('p'): - humanChrSort = humanChr[:humanChr.find("p")].join(humanChr[(humanChr.find("p")+1):]) #value used when sorting table - else: - humanChrSort = humanChr humanTxStart = theGO['humanGene']["TxStart"] - #Used for sorting - humanStartValue = float(theGO['humanGene']["TxStart"]) - humanStartString = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=vertebrate&org=Human&db=hg17&position=chr%s:%d-%d" % (humanChr, int(1000000*theGO['humanGene']["TxStart"]), int(1000000*theGO['humanGene']["TxEnd"])) else: - humanStartString = humanChr = humanChrSort = humanStartDisplay = humanStartValue = "--" + humanStartString = humanChr = humanStartDisplay = "--" geneDescription = theGO["GeneDescription"] if len(geneDescription) > 26: @@ -2923,22 +2129,12 @@ class MarkerRegression(object): else: avgExpr = "%0.6f" % avgExpr - # polymiRTS - polymiRTS = ' ' - if dic.has_key(theGO["GeneID"]): - polymiRTS = dic[theGO["GeneID"]] - # If we have a referenceGene then we will show the Literature Correlation if theGO["Chromosome"] == "X": chr_as_int = 19 else: chr_as_int = int(theGO["Chromosome"]) - 1 if refGene: - try: - literatureCorrelation = self.getLiteratureCorrelation(self.cursor,refGene,theGO['GeneID']) - except: - literatureCorrelation = "N/A" - literatureCorrelationString = str(self.getLiteratureCorrelation(self.cursor,refGene,theGO['GeneID']) or "N/A") this_row = [selectCheck.__str__(), @@ -2953,25 +2149,7 @@ class MarkerRegression(object): HT.Href(humanStartString, humanStartDisplay, target="_blank").__str__(), literatureCorrelationString, geneDescription] - #polymiRTS, - - - # this_row.append(TDCell(HT.TD(tableIterationsCnt, selectCheck, width=30, align='right', Class=className), tableIterationsCnt, tableIterationsCnt)) - # this_row.append(TDCell(HT.TD(HT.Href(geneIdString, theGO["GeneSymbol"], target="_blank"), " ", probeSetSearch, align='right', Class=className), theGO["GeneSymbol"], theGO["GeneSymbol"])) - # this_row.append(TDCell(HT.TD(HT.Href(mouseStartString, "%0.6f" % txStart, target="_blank"), align='right', Class=className), str(mouseStartValue), mouseStartValue)) - # this_row.append(TDCell(HT.TD(HT.Href("javascript:centerIntervalMapOnRange2('%s', " % theGO["Chromosome"]+str(txStart-tenPercentLength) + ", " + str(txEnd+tenPercentLength) + ", document.changeViewForm)", "%0.3f" % geneLength), align='right', Class=className), "%0.3f" % geneLength, geneLength)) - # this_row.append(TDCell(HT.TD(snpString, align='right', Class=className), str(theGO["snpCount"]), theGO["snpCount"])) - # this_row.append(TDCell(HT.TD(snpDensityStr, align='right', Class=className), snpDensityStr, theGO["snpDensity"])) - # this_row.append(TDCell(HT.TD(avgExpr, align='right', Class=className), "--", "--")) - # this_row.append(TDCell(HT.TD(humanChr, align='right', Class=className), humanChr, humanChrSort)) - # this_row.append(TDCell(HT.TD(HT.Href(humanStartString, humanStartDisplay, target="_blank"), align='right', Class=className), humanStartDisplay, humanStartValue)) - # this_row.append(TDCell(HT.TD(literatureCorrelationString, align='right', Class=className), literatureCorrelationString, literatureCorrelation)) - # this_row.append(TDCell(HT.TD(geneDescription, align='right', Class=className), geneDescription, geneDescription)) - # this_row.append(TDCell(HT.TD(polymiRTS, align='right', Class=className), "", "")) - # this_row.append(TDCell(HT.TD("", align='right', Class=className), "", "")) - else: - this_row = [selectCheck.__str__(), str(tableIterationsCnt), HT.Href(geneIdString, theGO["GeneSymbol"], target="_blank").__str__() + " " + probeSetSearch.__str__(), @@ -2983,29 +2161,11 @@ class MarkerRegression(object): humanChr, HT.Href(humanStartString, humanStartDisplay, target="_blank").__str__(), geneDescription] - #polymiRTS, - - - # this_row.append(TDCell(HT.TD(tableIterationsCnt, selectCheck, width=30, align='right', Class=className), tableIterationsCnt, tableIterationsCnt)) - # this_row.append(TDCell(HT.TD(HT.Href(geneIdString, theGO["GeneSymbol"], target="_blank"), " ", probeSetSearch, align='right', Class=className), theGO["GeneSymbol"], theGO["GeneSymbol"])) - # this_row.append(TDCell(HT.TD(HT.Href(mouseStartString, "%0.6f" % txStart, target="_blank"), align='right', Class=className), str(mouseStartValue), mouseStartValue)) - # this_row.append(TDCell(HT.TD(HT.Href("javascript:centerIntervalMapOnRange2('%s', " % theGO["Chromosome"]+str(txStart-tenPercentLength) + ", " + str(txEnd+tenPercentLength) + ", document.changeViewForm)", "%0.3f" % geneLength), align='right', Class=className), "%0.3f" % geneLength, geneLength)) - # this_row.append(TDCell(HT.TD(snpString, align='right', Class=className), str(theGO["snpCount"]), theGO["snpCount"])) - # this_row.append(TDCell(HT.TD(snpDensityStr, align='right', Class=className), snpDensityStr, theGO["snpDensity"])) - # this_row.append(TDCell(HT.TD(avgExpr, align='right', Class=className), "--", "--")) - # this_row.append(TDCell(HT.TD(humanChr, align='right', Class=className), humanChr, humanChrSort)) - # this_row.append(TDCell(HT.TD(HT.Href(humanStartString, humanStartDisplay, target="_blank"), align='right', Class=className), humanStartDisplay, humanStartValue)) - # this_row.append(TDCell(HT.TD(geneDescription, align='right', Class=className), geneDescription, geneDescription)) - # this_row.append(TDCell(HT.TD(polymiRTS, align='right', Class=className), "", "")) - # this_row.append(TDCell(HT.TD("", align='right', Class=className), "", "")) gene_table_body.append(this_row) - #tblobj_body.append(this_row) elif self.dataset.group.species == 'rat': - for gIndex, theGO in enumerate(geneCol): - this_row = [] #container for the cells of each row selectCheck = HT.Input(type="checkbox", name="searchResult", Class="checkbox", onClick="highlight(this)").__str__() #checkbox for each row @@ -3022,7 +2182,6 @@ class MarkerRegression(object): chr_as_int = int(theGO["Chromosome"]) - 1 geneLength = (float(theGO["TxEnd"]) - float(theGO["TxStart"])) - #geneLengthURL = "javascript:centerIntervalMapOnRange2('%s', %f, %f, document.changeViewForm)" % (theGO["Chromosome"], float(theGO["TxStart"])-(geneLength*0.1), float(theGO["TxEnd"])+(geneLength*0.1)) geneLengthURL = "javascript:rangeView('%s', %f, %f)" % (theGO["Chromosome"], float(theGO["TxStart"])-(geneLength*0.1), float(theGO["TxEnd"])+(geneLength*0.1)) avgExprVal = [] @@ -3041,15 +2200,9 @@ class MarkerRegression(object): #the chromosomes for human 1 are 1qXX.XX if theGO['humanGene']: humanChr = theGO['humanGene']["Chromosome"] - if 'q' in humanChr: - humanChrSort = humanChr[:humanChr.find("q")].join(humanChr[(humanChr.find("q")+1):]) #value used when sorting table - elif 'p' in humanChr: - humanChrSort = humanChr[:humanChr.find("p")].join(humanChr[(humanChr.find("p")+1):]) #value used when sorting table - else: - humanChrSort = humanChr humanTxStart = theGO['humanGene']["TxStart"] else: - humanChr = humanTxStart = humanChrSort = "" + humanChr = humanTxStart = "" geneDesc = theGO["GeneDescription"] if geneDesc == "---": @@ -3067,26 +2220,9 @@ class MarkerRegression(object): humanTxStart, geneDesc] - - #this_row.append(TDCell(HT.TD(gIndex + 1, selectCheck, align='left', Class=className), str(gIndex+1), gIndex+1)) - #this_row.append(TDCell(HT.TD(webqtlSearch, geneSymbolNCBI, align='left', Class=className), theGO["GeneSymbol"], theGO["GeneSymbol"])) - #this_row.append(TDCell(HT.TD(theGO["TxStart"], align='left', Class=className), theGO["TxStart"], theGO["TxStart"])) - #this_row.append(TDCell(HT.TD(HT.Href(geneLengthURL, "%0.3f" % (geneLength*1000.0)), align='left', Class=className), "%0.3f" % (geneLength*1000.0), (geneLength*1000.0))) - #this_row.append(TDCell(HT.TD(avgExprVal, align='left', Class=className), "", "")) - #this_row.append(TDCell(HT.TD(mouseChr, align='left', Class=className), mouseChr, mouseChr)) - #this_row.append(TDCell(HT.TD(mouseTxStart, align='left', Class=className), mouseTxStart, mouseTxStart)) - #this_row.append(TDCell(HT.TD(humanChr, align='left', Class=className), humanChr, humanChrSort)) - #this_row.append(TDCell(HT.TD(humanTxStart, align='left', Class=className), humanTxStart, humanTxStart)) - #this_row.append(TDCell(HT.TD(geneDesc, align='left', Class=className), geneDesc, geneDesc)) - gene_table_body.append(this_row) - #tblobj_body.append(this_row) - - else: - pass return gene_table_body - #return tblobj_body def getLiteratureCorrelation(cursor,geneId1=None,geneId2=None): if not geneId1 or not geneId2: @@ -3105,10 +2241,4 @@ class MarkerRegression(object): lCorr = lCorr[0] break except: raise #lCorr = None - return lCorr - - def getSortByValue(self): - - sortby = ("", "") - - return sortby + return lCorr
\ No newline at end of file diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index 4de88f00..2f327faf 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -10,9 +10,7 @@ logger = utility.logger.getLogger(__name__ ) def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr("%s_%s_"%(dataset.group.name, this_trait.name)) - gen_pheno_txt_file(dataset, vals) - #gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename = plink_output_filename) plink_command = PLINK_COMMAND + ' --noweb --bfile %s/%s --no-pheno --no-fid --no-parents --no-sex --maf %s --out %s%s --assoc ' % ( flat_files('mapping'), dataset.group.name, maf, TMPDIR, plink_output_filename) @@ -22,12 +20,6 @@ def run_plink(this_trait, dataset, species, vals, maf): count, p_values = parse_plink_output(plink_output_filename, species) - #for marker in self.dataset.group.markers.markers: - # if marker['name'] not in included_markers: - # logger.debug("marker:", marker) - # self.dataset.group.markers.markers.remove(marker) - # #del self.dataset.group.markers.markers[marker] - logger.debug("p_values:", p_values) dataset.group.markers.add_pvalues(p_values) @@ -108,7 +100,6 @@ def parse_plink_output(output_filename, species): result_fp = open("%s%s.qassoc"% (TMPDIR, output_filename), "rb") - header_line = result_fp.readline()# read header line line = result_fp.readline() value_list = [] # initialize value list, this list will include snp, bp and pvalue info @@ -156,11 +147,6 @@ def parse_plink_output(output_filename, species): else: line = result_fp.readline() - #if p_value_list: - # min_p_value = min(p_value_list) - #else: - # min_p_value = 0 - return count, p_value_dict ###################################################### @@ -173,4 +159,4 @@ def build_line_list(line=None): line_list = [item for item in line_list if item <>''] line_list = map(string.strip, line_list) - return line_list + return line_list
\ No newline at end of file diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index 6b58190f..ffbfb5c5 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -26,7 +26,7 @@ def gen_reaper_results(this_trait, dataset, samples_before, trait_vals, json_dat perm_output = genotype.permutation(strains = trimmed_samples, trait = trimmed_values, nperm=num_perm) suggestive = perm_output[int(num_perm*0.37-1)] significant = perm_output[int(num_perm*0.95-1)] - highly_significant = perm_output[int(num_perm*0.99-1)] + #highly_significant = perm_output[int(num_perm*0.99-1)] #ZS: Currently not used, but leaving it here just in case json_data['suggestive'] = suggestive json_data['significant'] = significant diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index f3694f0b..41d67012 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -5,15 +5,16 @@ from base.webqtlConfig import TMPDIR from utility import webqtlUtil from utility.tools import locate, TEMPDIR +import utility.logger +logger = utility.logger.getLogger(__name__ ) + def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan): geno_to_rqtl_function(dataset) ## Get pointers to some common R functions r_library = ro.r["library"] # Map the library function r_c = ro.r["c"] # Map the c function - r_sum = ro.r["sum"] # Map the sum function plot = ro.r["plot"] # Map the plot function - postscript = ro.r["postscript"] # Map the postscript function png = ro.r["png"] # Map the png function dev_off = ro.r["dev.off"] # Map the device off function @@ -23,17 +24,13 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, scanone = ro.r["scanone"] # Map the scanone function scantwo = ro.r["scantwo"] # Map the scantwo function calc_genoprob = ro.r["calc.genoprob"] # Map the calc.genoprob function - read_cross = ro.r["read.cross"] # Map the read.cross function - write_cross = ro.r["write.cross"] # Map the write.cross function GENOtoCSVR = ro.r["GENOtoCSVR"] # Map the local GENOtoCSVR function crossname = dataset.group.name genofilelocation = locate(crossname + ".geno", "genotype") crossfilelocation = TMPDIR + crossname + ".cross" - #print("Conversion of geno to cross at location:", genofilelocation, " to ", crossfilelocation) - - cross_object = GENOtoCSVR(genofilelocation, crossfilelocation) # TODO: Add the SEX if that is available + cross_object = GENOtoCSVR(genofilelocation, crossfilelocation) # TODO: Add the SEX if that is available if manhattan_plot: cross_object = calc_genoprob(cross_object) @@ -42,18 +39,14 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, cross_object = add_phenotype(cross_object, sanitize_rqtl_phenotype(vals)) # Add the phenotype - # for debug: write_cross(cross_object, "csvr", "test.csvr") - # Scan for QTLs - covar = create_covariates(control_marker, cross_object) # Create the additive covariate matrix + covar = create_covariates(control_marker, cross_object) # Create the additive covariate matrix if pair_scan: - if do_control == "true": # If sum(covar) > 0 we have a covariate matrix - print("Using covariate"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method, n_cluster = 16) + if do_control == "true": + logger.info("Using covariate"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method, n_cluster = 16) else: - print("No covariates"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", model=model, method=method, n_cluster = 16) - - #print("Pair scan results:", result_data_frame) + logger.info("No covariates"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", model=model, method=method, n_cluster = 16) pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png" png(file=TEMPDIR+pair_scan_filename) @@ -63,9 +56,9 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, return process_pair_scan_results(result_data_frame) else: if do_control == "true": - print("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method) + logger.info("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method) else: - print("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method) + logger.info("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method) if num_perm > 0 and permCheck == "ON": # Do permutation (if requested by user) if do_control == "true": @@ -79,7 +72,6 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, return process_rqtl_results(result_data_frame) def geno_to_rqtl_function(dataset): # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly - ro.r(""" trim <- function( x ) { gsub("(^[[:space:]]+|[[:space:]]+$)", "", x) } @@ -117,15 +109,13 @@ def add_phenotype(cross, pheno_as_string): def create_covariates(control_marker, cross): ro.globalenv["the_cross"] = cross ro.r('genotypes <- pull.geno(the_cross)') # Get the genotype matrix - userinputS = control_marker.replace(" ", "").split(",") # TODO: sanitize user input, Never Ever trust a user + userinputS = control_marker.replace(" ", "").split(",") # TODO: sanitize user input, Never Ever trust a user covariate_names = ', '.join('"{0}"'.format(w) for w in userinputS) - #print("Marker names of selected covariates:", covariate_names) ro.r('covnames <- c(' + covariate_names + ')') ro.r('covInGeno <- which(covnames %in% colnames(genotypes))') ro.r('covnames <- covnames[covInGeno]') ro.r("cat('covnames (purged): ', covnames,'\n')") ro.r('covariates <- genotypes[,covnames]') # Get the covariate matrix by using the marker name as index to the genotype file - #print("R/qtl matrix of covariates:", ro.r["covariates"]) return ro.r["covariates"] def sanitize_rqtl_phenotype(vals): @@ -149,7 +139,6 @@ def process_pair_scan_results(result): result = result[1] output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)] - #print("R/qtl scantwo output:", output) for i, line in enumerate(result.iter_row()): marker = {} @@ -175,9 +164,7 @@ def process_rqtl_perm_results(num_perm, results): def process_rqtl_results(result): # TODO: how to make this a one liner and not copy the stuff in a loop qtl_results = [] - output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)] - #print("R/qtl scanone output:", output) for i, line in enumerate(result.iter_row()): marker = {} @@ -187,5 +174,4 @@ def process_rqtl_results(result): # TODO: how to make this a one liner an marker['lod_score'] = output[i][2] qtl_results.append(marker) - return qtl_results - + return qtl_results
\ No newline at end of file diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py index 5321e420..38117a8e 100644 --- a/wqflask/wqflask/model.py +++ b/wqflask/wqflask/model.py @@ -6,31 +6,16 @@ import datetime import simplejson as json from flask import request -from flask.ext.sqlalchemy import SQLAlchemy from wqflask import app import sqlalchemy - -from sqlalchemy import (Column, Integer, String, Table, ForeignKey, Unicode, Boolean, DateTime, +from sqlalchemy import (Column, ForeignKey, Unicode, Boolean, DateTime, Text, Index) -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import relationship from wqflask.database import Base, init_db - - -# Define models -#roles_users = Table('roles_users', -# Column('user_id', Integer(), ForeignKey('user.the_id')), -# Column('role_id', Integer(), ForeignKey('role.the_id'))) - -#class Role(Base): -# __tablename__ = "role" -# id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) -# name = Column(Unicode(80), unique=True, nullable=False) -# description = Column(Unicode(255)) - class User(Base): __tablename__ = "user" id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) @@ -133,11 +118,6 @@ class User(Base): except IndexError: return None - - #roles = relationship('Role', secondary=roles_users, - # backref=backref('users', lazy='dynamic')) - - class Login(Base): __tablename__ = "login" id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) @@ -177,22 +157,15 @@ class UserCollection(Base): except: return 0 - #@property - #def display_num_members(self): - # return display_collapsible(self.num_members) - def members_as_set(self): return set(json.loads(self.members)) - def display_collapsible(number): if number: return number else: return "" - def user_uuid(): """Unique cookie for a user""" - user_uuid = request.cookies.get('user_uuid') - + user_uuid = request.cookies.get('user_uuid')
\ No newline at end of file diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index b42904a4..63273a29 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -21,9 +21,7 @@ from __future__ import absolute_import, print_function, division import sys -# sys.path.append(".") Never do this in a webserver! -import gc import string import cPickle import os @@ -95,7 +93,6 @@ class NetworkGraph(object): self.lowest_overlap = 8 #ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning) - self.network_data = {} self.nodes_list = [] self.edges_list = [] for trait_db in self.trait_list: @@ -107,9 +104,9 @@ class NetworkGraph(object): corr_result_row = [] is_spearman = False #ZS: To determine if it's above or below the diagonal - + max_corr = 0 #ZS: Used to determine whether node should be hidden when correlation coefficient slider is used - + for target in self.trait_list: target_trait = target[0] target_db = target[1] @@ -141,7 +138,7 @@ class NetworkGraph(object): continue else: sample_r, sample_p = scipy.stats.spearmanr(this_trait_vals, target_vals) - + if -1 <= sample_r < -0.7: color = "#0000ff" width = 3 @@ -163,10 +160,10 @@ class NetworkGraph(object): else: color = "#000000" width = 0 - + if abs(sample_r) > max_corr: max_corr = abs(sample_r) - + edge_data = {'id' : str(this_trait.name) + '_to_' + str(target_trait.name), 'source' : str(this_trait.name) + ":" + str(this_trait.dataset.name), 'target' : str(target_trait.name) + ":" + str(target_trait.dataset.name), @@ -176,11 +173,11 @@ class NetworkGraph(object): 'overlap' : num_overlap, 'color' : color, 'width' : width } - + edge_dict = { 'data' : edge_data } - + self.edges_list.append(edge_dict) - + if trait_db[1].type == "ProbeSet": node_dict = { 'data' : {'id' : str(this_trait.name) + ":" + str(this_trait.dataset.name), 'label' : this_trait.symbol, @@ -197,19 +194,13 @@ class NetworkGraph(object): 'label' : this_trait.name, 'max_corr' : max_corr } } self.nodes_list.append(node_dict) - - #self.network_data['dataSchema'] = {'nodes' : [{'name' : "label" , 'type' : "string"}], - # 'edges' : [{'name' : "label" , 'type' : "string"}] } - - #self.network_data['data'] = {'nodes' : self.nodes_list, - # 'edges' : self.edges_list } self.elements = json.dumps(self.nodes_list + self.edges_list) - + groups = [] for sample in self.all_sample_list: groups.append(1) - + self.js_data = dict(traits = [trait.name for trait in self.traits], groups = groups, cols = range(len(self.traits)), @@ -217,7 +208,6 @@ class NetworkGraph(object): samples = self.all_sample_list, sample_data = self.sample_data, elements = self.elements,) - # corr_results = [result[1] for result in result_row for result_row in self.corr_results]) def get_trait_db_obs(self, trait_db_list): self.trait_list = [] @@ -229,4 +219,4 @@ class NetworkGraph(object): trait_ob = GeneralTrait(dataset=dataset_ob, name=trait_name, cellid=None) - self.trait_list.append((trait_ob, dataset_ob)) + self.trait_list.append((trait_ob, dataset_ob))
\ No newline at end of file diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 6d84a960..8dbba530 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -10,8 +10,12 @@ import numpy as np from scipy import stats from pprint import pformat as pf +import simplejson as json + import itertools +from utility.elasticsearch_tools import get_elasticsearch_connection + import utility.logger logger = utility.logger.getLogger(__name__ ) @@ -33,6 +37,8 @@ class SampleList(object): self.get_attributes() + #self.sample_qnorm = get_transform_vals(self.dataset, this_trait) + if self.this_trait and self.dataset and self.dataset.type == 'ProbeSet': self.get_extra_attribute_values() @@ -152,36 +158,47 @@ class SampleList(object): return any(sample.variance for sample in self.sample_list) -#def z_score(vals): -# vals_array = np.array(vals) -# mean = np.mean(vals_array) -# stdv = np.std(vals_array) -# -# z_scores = [] -# for val in vals_array: -# z_score = (val - mean)/stdv -# z_scores.append(z_score) -# -# -# -# return z_scores - - -#def z_score(row): -# L = [n for n in row if not np.isnan(n)] -# m = np.mean(L) -# s = np.std(L) -# zL = [1.0 * (n - m) / s for n in L] -# if len(L) == len(row): return zL -# # deal with nan -# retL = list() -# for n in row: -# if np.isnan(n): -# retL.append(nan) -# else: -# retL.append(zL.pop(0)) -# assert len(zL) == 0 -# return retL +def get_transform_vals(dataset, trait): + es = get_elasticsearch_connection(for_user=False) + + logger.info("DATASET NAME:", dataset.name) + + query = '{"bool": {"must": [{"match": {"name": "%s"}}, {"match": {"dataset": "%s"}}]}}' % (trait.name, dataset.name) + + es_body = { + "query": { + "bool": { + "must": [ + { + "match": { + "name": "%s" % (trait.name) + } + }, + { + "match": { + "dataset": "%s" % (dataset.name) + } + } + ] + } + } + } + + response = es.search( index = "traits", doc_type = "trait", body = es_body ) + logger.info("THE RESPONSE:", response) + results = response['hits']['hits'] + + if len(results) > 0: + samples = results[0]['_source']['samples'] + + sample_dict = {} + for sample in samples: + sample_dict[sample['name']] = sample['qnorm'] + + logger.info("SAMPLE DICT:", sample_dict) + return sample_dict + else: + return None def natural_sort_key(x): """Get expected results when using as a key for sort - ints or strings are sorted properly""" diff --git a/wqflask/wqflask/show_trait/export_trait_data.py b/wqflask/wqflask/show_trait/export_trait_data.py index ac3cd366..558372bb 100644 --- a/wqflask/wqflask/show_trait/export_trait_data.py +++ b/wqflask/wqflask/show_trait/export_trait_data.py @@ -1,7 +1,5 @@ from __future__ import print_function, division -import operator - import simplejson as json from pprint import pformat as pf diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 6de5cd5f..7b952af4 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -6,13 +6,14 @@ import datetime import cPickle import uuid import json as json -#import pyXLWriter as xl from collections import OrderedDict import redis Redis = redis.StrictRedis() +import scipy.stats as ss + from flask import Flask, g from htmlgen import HTMLgen2 as HT @@ -24,13 +25,9 @@ from utility import webqtlUtil, Plot, Bunch, helper_functions from base.trait import GeneralTrait from base import data_set from db import webqtlDatabaseFunction -from basicStatistics import BasicStatisticsFunctions from pprint import pformat as pf -from utility.tools import flat_files, flat_file_exists -from utility.tools import get_setting - from utility.logger import getLogger logger = getLogger(__name__ ) @@ -74,14 +71,6 @@ class ShowTrait(object): cellid=None) self.trait_vals = Redis.get(self.trait_id).split() - #self.dataset.group.read_genotype_file() - - #if this_trait: - # if this_trait.dataset and this_trait.dataset.type and this_trait.dataset.type == 'ProbeSet': - # self.cursor.execute("SELECT h2 from ProbeSetXRef WHERE DataId = %d" % - # this_trait.mysqlid) - # heritability = self.cursor.fetchone() - #ZS: Get verify/rna-seq link URLs try: blatsequence = self.this_trait.blatseq @@ -150,6 +139,8 @@ class ShowTrait(object): self.make_sample_lists() + self.qnorm_vals = quantile_normalize_vals(self.sample_groups) + # Todo: Add back in the ones we actually need from below, as we discover we need them hddn = OrderedDict() @@ -193,7 +184,7 @@ class ShowTrait(object): self.sample_group_types['samples_primary'] = self.dataset.group.name sample_lists = [group.sample_list for group in self.sample_groups] - self.get_mapping_methods() + self.genofiles = get_genofiles(self.dataset) self.stats_table_width, self.trait_table_width = get_table_widths(self.sample_groups) @@ -212,28 +203,6 @@ class ShowTrait(object): temp_uuid = self.temp_uuid) self.js_data = js_data - def get_mapping_methods(self): - '''Only display mapping methods when the dataset group's genotype file exists''' - def check_plink_gemma(): - if flat_file_exists("mapping"): - MAPPING_PATH = flat_files("mapping")+"/" - if (os.path.isfile(MAPPING_PATH+self.dataset.group.name+".bed") and - (os.path.isfile(MAPPING_PATH+self.dataset.group.name+".map") or - os.path.isfile(MAPPING_PATH+self.dataset.group.name+".bim"))): - return True - return False - - def check_pylmm_rqtl(): - if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.JSON_GENODIR+self.dataset.group.name+".json") > 0): - return True - else: - return False - - self.genofiles = get_genofiles(self.dataset) - self.use_plink_gemma = check_plink_gemma() - self.use_pylmm_rqtl = check_pylmm_rqtl() - - def build_correlation_tools(self): if self.temp_trait == True: this_group = self.temp_group @@ -246,7 +215,6 @@ class ShowTrait(object): this_group = 'BXD' if this_group: - #dataset_menu = self.dataset.group.datasets() if self.temp_trait == True: dataset_menu = data_set.datasets(this_group) else: @@ -264,7 +232,6 @@ class ShowTrait(object): return_results_menu = return_results_menu, return_results_menu_selected = return_results_menu_selected,) - def make_sample_lists(self): all_samples_ordered = self.dataset.group.all_samples_ordered() @@ -316,12 +283,46 @@ class ShowTrait(object): sample_group_type='primary', header="%s Only" % (self.dataset.group.name)) self.sample_groups = (primary_samples,) - #TODO: Figure out why this if statement is written this way - Zach - #if (other_sample_names or (fd.f1list and this_trait.data.has_key(fd.f1list[0])) - # or (fd.f1list and this_trait.data.has_key(fd.f1list[1]))): - # logger.debug("hjs") self.dataset.group.allsamples = all_samples_ordered +def quantile_normalize_vals(sample_groups): + def normf(trait_vals): + ranked_vals = ss.rankdata(trait_vals) + p_list = [] + for i, val in enumerate(trait_vals): + p_list.append(((i+1) - 0.5)/len(trait_vals)) + + z = ss.norm.ppf(p_list) + normed_vals = [] + for rank in ranked_vals: + normed_vals.append("%0.3f" % z[int(rank)-1]) + + return normed_vals + + qnorm_by_group = [] + for sample_type in sample_groups: + trait_vals = [] + for sample in sample_type.sample_list: + try: + trait_vals.append(float(sample.value)) + except: + continue + + qnorm_vals = normf(trait_vals) + + qnorm_vals_with_x = [] + counter = 0 + for sample in sample_type.sample_list: + if sample.display_value == "x": + qnorm_vals_with_x.append("x") + else: + qnorm_vals_with_x.append(qnorm_vals[counter]) + counter += 1 + + qnorm_by_group.append(qnorm_vals_with_x) + + return qnorm_by_group + def get_nearest_marker(this_trait, this_db): this_chr = this_trait.locus_chr logger.debug("this_chr:", this_chr) @@ -344,7 +345,6 @@ def get_nearest_marker(this_trait, this_db): #return "", "" else: return result[0][0] - #return result[0][0], result[1][0] def get_genofiles(this_dataset): jsonfile = "%s/%s.json" % (webqtlConfig.GENODIR, this_dataset.group.name) diff --git a/wqflask/wqflask/show_trait/show_trait_page.py b/wqflask/wqflask/show_trait/show_trait_page.py deleted file mode 100644 index de818a64..00000000 --- a/wqflask/wqflask/show_trait/show_trait_page.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# This program is available from Source Forge: at GeneNetwork Project -# (sourceforge.net/projects/genenetwork/). -# -# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) -# at rwilliams@uthsc.edu and xzhou15@uthsc.edu -# -# -# -# This module is used by GeneNetwork project (www.genenetwork.org) -# -# Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 - -from __future__ import division, print_function - -from flask import request - -from htmlgen import HTMLgen2 as HT - -from base import webqtlConfig -from utility import webqtlUtil -from base.webqtlTrait import webqtlTrait -from base.templatePage import templatePage -from DataEditingPage import DataEditingPage - - - -class ShowTraitPage(DataEditingPage): - - def __init__(self, fd, traitInfos = None): - self.fd = fd - - # This sets self.cursor - assert self.openMysql(), "No database" - - # When is traitInfos used? - if traitInfos: - database, ProbeSetID, CellID = traitInfos - else: - print("fd is:", fd) - database = fd['database'] - ProbeSetID = fd['ProbeSetID'] - - CellID = fd.get('CellID') - - - thisTrait = webqtlTrait(db=database, name=ProbeSetID, cellid=CellID, cursor=self.cursor) - - if thisTrait.db.type == "ProbeSet": - - self.cursor.execute('''SELECT Id, Name, FullName, confidentiality, AuthorisedUsers - FROM ProbeSetFreeze WHERE Name = "%s"''' % database) - - indId, indName, indFullName, confidential, AuthorisedUsers = self.cursor.fetchall()[0] - - if confidential == 1: - access_to_confidential_dataset = 0 - - #for the dataset that confidentiality is 1 - #1. 'admin' and 'root' can see all of the dataset - #2. 'user' can see the dataset that AuthorisedUsers contains his id(stored in the Id field of User table) - if webqtlConfig.USERDICT[self.privilege] > webqtlConfig.USERDICT['user']: - access_to_confidential_dataset = 1 - else: - AuthorisedUsersList=AuthorisedUsers.split(',') - if AuthorisedUsersList.__contains__(self.userName): - access_to_confidential_dataset = 1 - - if not access_to_confidential_dataset: - #Error, Confidential Database - heading = "Show Database" - detail = ["The %s database you selected is not open to the public \ - at this time, please go back and select other database." % indFullName] - self.error(heading=heading,detail=detail,error="Confidential Database") - return - print("environ:", request.environ) - - # Becuase of proxying remote_addr is probably localhost, so we first try for - # HTTP_X_FORWARDED_FOR - user_ip = request.environ.get('HTTP_X_FORWARDED_FOR') or request.remote_addr # in old app was fd.remote_ip - print("user_ip is:", user_ip) - query = "SELECT count(id) FROM AccessLog WHERE ip_address = %s and \ - UNIX_TIMESTAMP()-UNIX_TIMESTAMP(accesstime)<86400" - self.cursor.execute(query,user_ip) - daycount = self.cursor.fetchall() - if daycount: - daycount = daycount[0][0] - if daycount > webqtlConfig.DAILYMAXIMUM: - heading = "Retrieve Data" - detail = ['For security reasons, the maximum access to a database is \ - %d times per day per ip address. You have reached the limit, please \ - try it again tomorrow.' % webqtlConfig.DAILYMAXIMUM] - self.error(heading=heading,detail=detail) - return - - - if thisTrait.db.type != 'ProbeSet' and thisTrait.cellid: - heading = "Retrieve Data" - detail = ['The Record you requested doesn\'t exist!'] - self.error(heading=heading,detail=detail) - return - - ##identification, etc. - fd.identification = '%s : %s' % (thisTrait.db.shortname,ProbeSetID) - thisTrait.returnURL = webqtlConfig.CGIDIR + webqtlConfig.SCRIPTFILE + '?FormID=showDatabase&database=%s\ - &ProbeSetID=%s&RISet=%s&parentsf1=on' %(database, ProbeSetID, fd['RISet']) - - if CellID: - fd.identification = '%s/%s'%(fd.identification, CellID) - thisTrait.returnURL = '%s&CellID=%s' % (thisTrait.returnURL, CellID) - - thisTrait.retrieveInfo() - thisTrait.retrieveData() - self.updMysql() - self.cursor.execute("insert into AccessLog(accesstime,ip_address) values(Now(),%s)", user_ip) - self.openMysql() - - - ##read genotype file - fd.RISet = thisTrait.riset - fd.readGenotype() - - #if webqtlUtil.ListNotNull(map(lambda x:x.var, thisTrait.data.values())): - if any([x.variance for x in thisTrait.data.values()]): - fd.display_variance = True - fd.formID = 'varianceChoice' - - DataEditingPage.__init__(self, fd, thisTrait) diff --git a/wqflask/wqflask/static/new/css/corr_scatter_plot.css b/wqflask/wqflask/static/new/css/corr_scatter_plot.css index c62d4c9a..a2ebb252 100644 --- a/wqflask/wqflask/static/new/css/corr_scatter_plot.css +++ b/wqflask/wqflask/static/new/css/corr_scatter_plot.css @@ -1,13 +1,41 @@ -.chart { +.nvd3 .nv-axis.nv-x text { + font-size: 16px; + font-weight: normal; + fill: black; +} +.nvd3 .nv-axis.nv-y text { + font-size: 16px; + font-weight: normal; + fill: black; } -.main text { - font: 10px sans-serif; +.nv-x .nv-axis g path.domain { + stroke: black; + stroke-width: 2; } -.axis line, .axis path { - shape-rendering: crispEdges; +.nv-y .nv-axis g path.domain { stroke: black; - fill: none; + stroke-width: 2; +} + +.nvd3 .nv-axis.nv-x path.domain { + stroke-opacity: 1; +} + +.nvd3 .nv-axis.nv-y path.domain { + stroke-opacity: 1; +} + +line.nv-regLine { + stroke: red; + stroke-width: 1; +} + +.nv-axisMin-x, +.nv-axisMax-x, +.nv-axisMin-y, +.nv-axisMax-y { + display: none; } diff --git a/wqflask/wqflask/static/new/javascript/comparison_bar_chart.js b/wqflask/wqflask/static/new/javascript/comparison_bar_chart.js new file mode 100644 index 00000000..5e73807c --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/comparison_bar_chart.js @@ -0,0 +1,25 @@ +generate_traces = function() { + traces = []; + for (i = 0, _len = js_data.traits.length; i < _len; i++) { + this_trace = { + x: js_data.samples, + y: js_data.sample_data[i], + name: js_data.traits[i], + type: 'bar', + bargap: 20 + } + + traces.push(this_trace) + } + + return traces +} + +create_bar_chart = function() { + var data = generate_traces() + var layout = {barmode: 'group', bargap: 5}; + + Plotly.newPlot('comp_bar_chart', data, layout); +} + +create_bar_chart();
\ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json index b7ebb9ed..81185136 100644 --- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json +++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json @@ -1527,6 +1527,36 @@ } }, "mouse": { + "AIL": { + "Hippocampus mRNA": [ + [ + "844", + "UCSD_AIL_HIP_RNA-Seq_0418", + "UCSD AIL Hippocampus (Apr18) RNA-Seq" + ] + ], + "Phenotypes": [ + [ + "None", + "AILPublish", + "AIL Published Phenotypes" + ] + ], + "Prefrontal Cortex mRNA": [ + [ + "846", + "UCSD_AIL_PFC_RNA-Seq_0418", + "UCSD AIL Prefrontal Cortex (Apr18) RNA-Seq" + ] + ], + "Striatum mRNA": [ + [ + "845", + "UCSD_AIL_STR_RNA-Seq_0418", + "UCSD AIL Striatum (Apr18) RNA-Seq" + ] + ] + }, "AKXD": { "Genotypes": [ [ @@ -1721,7 +1751,7 @@ ], "Phenotypes": [ [ - "None", + "651", "B6D2RIPublish", "B6D2RI Published Phenotypes" ] @@ -1936,6 +1966,21 @@ "470", "EPFLADEL1013", "EPFL/LISP BXD CD Brown Adipose Affy Mouse Gene 2.0 ST Exon Level (Oct13) RMA" + ], + [ + "777", + "EL_BXDCDHFDScWAT_0216", + "EPFL/LISP BXD CD+HFD Subcutaneous WAT Affy MTA 1.0 Gene Level (Feb16) RMA" + ], + [ + "778", + "EL_BXDHFDScWAT_0216", + "EPFL/LISP BXD HFD Subcutaneous WAT Affy MTA 1.0 Gene Level (Feb16) RMA" + ], + [ + "779", + "EL_BXDCDScWAT_0216", + "EPFL/LISP BXD CD Subcutaneous WAT Affy MTA 1.0 Gene Level (Feb16) RMA **" ] ], "Adrenal Gland mRNA": [ @@ -2571,6 +2616,16 @@ "EPFL/LISP BXD CD+HFD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA Exon Level" ], [ + "849", + "EPFLMouseLiverCDEx0413", + "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA Exon Level" + ], + [ + "848", + "EPFLMouseLiverHFCEx0413", + "EPFL/LISP BXD HFC Liver Affy Mouse Gene 1.0 ST (Apr13) RMA Exon Level" + ], + [ "700", "UTHSC-VGX_MmBXDHepatocytesRMA1014", "UT-VGX Hepatocytes Affy Mouse Gene 1.0 ST Gene Level (Oct14) RMA" @@ -2738,7 +2793,7 @@ ], "Phenotypes": [ [ - "None", + "602", "BXDPublish", "BXD Published Phenotypes" ] @@ -2968,7 +3023,7 @@ "BXD-Bone": { "Phenotypes": [ [ - "None", + "650", "BXD-BonePublish", "BXD-Bone Published Phenotypes" ] @@ -2985,6 +3040,18 @@ "842", "UTHSC-BXD-Liv-0917", "UTHSC BXD Liver Affy Clariom S GeneLevel Main (Sep17) RMA **" + ], + [ + "850", + "UTHSC-BXD-Liv-0818", + "UTHSC BXD Liver Affy Clariom S GeneLevel Main (Aug18) RMA **" + ] + ], + "Phenotypes": [ + [ + "None", + "BXD-HarvestedPublish", + "BXD-Harvested Published Phenotypes" ] ] }, @@ -3156,6 +3223,13 @@ ] }, "CIE-RMA": { + "Hippocampus mRNA": [ + [ + "831", + "INIA_UTHSC_Hip_AffyMTA1_May17", + "INIA-UTHSC Hippocampus CIE Affy MTA 1.0 GeneLevel (Mar18) RMA" + ] + ], "Midbrain mRNA": [ [ "830", @@ -3318,6 +3392,15 @@ ] ] }, + "D2GM": { + "Retina mRNA": [ + [ + "847", + "JAX_D2GM_RSeq_log2Z_0418", + "JAX Retina (Apr18) RNA-Seq log2-Z" + ] + ] + }, "EMSR": {}, "HS": { "Hippocampus mRNA": [ @@ -3577,7 +3660,7 @@ "HSNIH-Palmer": { "Phenotypes": [ [ - "None", + "652", "HSNIH-PalmerPublish", "HSNIH-Palmer Published Phenotypes" ] @@ -3826,12 +3909,16 @@ ], "mouse": [ [ + "AIL", + "AIL Advanced Intercross Line" + ], + [ "AKXD", - "AKXD" + "AKXD RI Family" ], [ "AXBXA", - "AXB/BXA" + "AXB/BXA RI Family" ], [ "B6BTBRF2", @@ -3866,28 +3953,28 @@ "BHF2 (Apoe Null) UCLA" ], [ - "BXD", - "BXD" - ], - [ "B6D2RI", "BXD Aged" ], [ "BXD-Bone", - "BXD Bone" + "BXD Bone Individual Data" ], [ "BXD-Harvested", "BXD NIA Longevity Study" ], [ + "BXD", + "BXD RI Family" + ], + [ "BXD300", "BXD300" ], [ "BXH", - "BXH" + "BXH RI Family" ], [ "CTB6F2", @@ -3911,7 +3998,11 @@ ], [ "CXB", - "CXB" + "CXB RI Family (strain means)" + ], + [ + "D2GM", + "D2 Glaucoma Model" ], [ "EMSR", @@ -3931,11 +4022,11 @@ ], [ "LXS", - "LXS" + "ILSXISS (LXS) RI Family (strain means)" ], [ "MDP", - "Mouse Diversity Panel" + "Mouse Diversity Panel (strain means)" ], [ "NZBXFVB-N2", @@ -4727,6 +4818,24 @@ ] }, "mouse": { + "AIL": [ + [ + "Phenotypes", + "Phenotypes" + ], + [ + "Hippocampus mRNA", + "Hippocampus mRNA" + ], + [ + "Prefrontal Cortex mRNA", + "Prefrontal Cortex mRNA" + ], + [ + "Striatum mRNA", + "Striatum mRNA" + ] + ], "AKXD": [ [ "Genotypes", @@ -5027,6 +5136,10 @@ ], "BXD-Harvested": [ [ + "Phenotypes", + "Phenotypes" + ], + [ "Liver mRNA", "Liver mRNA" ] @@ -5111,6 +5224,10 @@ "Phenotypes" ], [ + "Hippocampus mRNA", + "Hippocampus mRNA" + ], + [ "Midbrain mRNA", "Midbrain mRNA" ] @@ -5173,6 +5290,12 @@ "Spleen mRNA" ] ], + "D2GM": [ + [ + "Retina mRNA", + "Retina mRNA" + ] + ], "EMSR": [], "HS": [ [ diff --git a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js deleted file mode 100644 index bf0a14e6..00000000 --- a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js +++ /dev/null @@ -1,222 +0,0 @@ -var chart; -var srchart; - -function drawg() { - // - chart = nv.models.scatterChart(); - // - chart.showLegend(false); - chart.duration(300); - chart.color(d3.scale.category10().range()); - chart.pointRange([0, 400]); - chart.pointDomain([0, 10]); - // - chart.xAxis.axisLabel(js_data.trait_1); - chart.xAxis.axisLabelDistance(11); - chart.yAxis.axisLabel(js_data.trait_2); - chart.yAxis.axisLabelDistance(11); - // - xmin = d3.min(js_data.data[0]); - xmax = d3.max(js_data.data[0]); - xrange = xmax - xmin; - ymin = d3.min(js_data.data[1]); - ymax = d3.max(js_data.data[1]); - yrange = ymax - ymin; - chart.xDomain([xmin - xrange/10, xmax + xrange/10]); - chart.yDomain([ymin - yrange/10, ymax + yrange/10]); - chart.xAxis.tickFormat(d3.format(checkformat(xrange))); - chart.yAxis.tickFormat(d3.format(checkformat(yrange))); - // - chart.tooltip.contentGenerator(function (obj) { - return tiptext(obj); - }); -} - -function srdrawg() { - // - srchart = nv.models.scatterChart(); - // - srchart.showLegend(false); - srchart.duration(300); - srchart.color(d3.scale.category10().range()); - srchart.pointRange([0, 400]); - srchart.pointDomain([0, 10]); - // - srchart.xAxis.axisLabel(js_data.trait_1); - srchart.xAxis.axisLabelDistance(11); - srchart.yAxis.axisLabel(js_data.trait_2); - srchart.yAxis.axisLabelDistance(11); - // - xmin = d3.min(js_data.rdata[0]); - xmax = d3.max(js_data.rdata[0]); - xrange = xmax - xmin; - ymin = d3.min(js_data.rdata[1]); - ymax = d3.max(js_data.rdata[1]); - yrange = ymax - ymin; - srchart.xDomain([0, xmax + xrange/10]); - srchart.yDomain([0, ymax + yrange/10]); - srchart.xAxis.tickFormat(d3.format(checkformat(xrange))); - srchart.yAxis.tickFormat(d3.format(checkformat(yrange))); - // - srchart.tooltip.contentGenerator(function (obj) { - return tiptext(obj); - }); -} - -function tiptext(obj) { - return '<b style="font-size: 18px">' + obj.point.name + " (" + obj.point.x + ', ' + obj.point.y + ')</b>'; -} - -function getdata(size, shape) { - var data = []; - data.push({ - values: [], - slope: js_data.slope, - intercept: js_data.intercept - }); - for (j = 0; j < js_data.data[0].length; j++) { - data[0].values.push({ - x: js_data.data[0][j], - y: js_data.data[1][j], - name: js_data.indIDs[j], - size: size, - shape: shape - }); - } - return data; -} - -function srgetdata(size, shape) { - var data = []; - data.push({ - values: [], - slope: js_data.srslope, - intercept: js_data.srintercept - }); - for (j = 0; j < js_data.rdata[0].length; j++) { - data[0].values.push({ - x: js_data.rdata[0][j], - y: js_data.rdata[1][j], - name: js_data.indIDs[j], - size: size, - shape: shape - }); - } - return data; -} - -function checkformat(range) { - cell = range / 10.0; - if (cell >= 1) { - return ",r"; - } else { - cell = -Math.log(cell); - n = cell.toString().split(".")[0].length; - return ",.0" + n + "f"; - } -} - -function chartupdate() { - // - var labelcolor = $("#labelcolor").val(); - $(".nvd3 .nv-axis.nv-x text").css("fill", labelcolor); - $(".nvd3 .nv-axis.nv-y text").css("fill", labelcolor); - // - var labelfont = $("#labelfont").val(); - $(".nvd3 .nv-axis.nv-x text").css("font-size", labelfont); - $(".nvd3 .nv-axis.nv-y text").css("font-size", labelfont); - // - var numbercolor = $("#numbercolor").val(); - $("g.tick text").css("fill", numbercolor); - // - var numberfont = $("#numberfont").val(); - $("g.tick text").css("font-size", numberfont); - // - var axiscolor = $("#axiscolor").val(); - $(".nv-x .nv-axis g path.domain").css("stroke", axiscolor); - $(".nv-y .nv-axis g path.domain").css("stroke", axiscolor); - // - var axiswidth = $("#axiswidth").val(); - $(".nv-x .nv-axis g path.domain").css("stroke-width", axiswidth); - $(".nv-y .nv-axis g path.domain").css("stroke-width", axiswidth); - // - var linecolor = $("#linecolor").val(); - $("line.nv-regLine").css("stroke", linecolor); - // - var linewidth = $("#linewidth").val(); - $("line.nv-regLine").css("stroke-width", linewidth); - // - var markcolor = $("#markcolor").val(); - $(".nvd3 g path").css("fill", markcolor); -} - -function chartupdatewh() { - // - var width = $("#width").val(); - $("#scatterplot2 svg").css("width", width); - $("#srscatterplot2 svg").css("width", width); - // - var height = $("#height").val(); - $("#scatterplot2 svg").css("height", height); - $("#srscatterplot2 svg").css("height", height); - // - window.dispatchEvent(new Event('resize')); -} - -function chartupdatedata() { - // - var size = $("#marksize").val(); - var shape = $("#markshape").val(); - // - d3.select('#scatterplot2 svg').datum(nv.log(getdata(size, shape))).call(chart); - d3.select('#srscatterplot2 svg').datum(nv.log(srgetdata(size, shape))).call(srchart); - nv.utils.windowResize(chart.update); - nv.utils.windowResize(srchart.update); -} - -function savesvg(svgEl, name) { - svgEl.setAttribute("xmlns", "http://www.w3.org/2000/svg"); - var svgData = svgEl.outerHTML; - var preface = '<?xml version="1.0" standalone="no"?>\r\n'; - preface += '<?xml-stylesheet type="text/css" href="http://cdnjs.cloudflare.com/ajax/libs/nvd3/1.8.5/nv.d3.min.css"?>\r\n'; - var svgBlob = new Blob([preface, svgData], {type:"image/svg+xml;charset=utf-8"}); - var svgUrl = URL.createObjectURL(svgBlob); - var downloadLink = document.createElement("a"); - downloadLink.href = svgUrl; - downloadLink.download = name; - document.body.appendChild(downloadLink); - downloadLink.click(); - document.body.removeChild(downloadLink); -} - -function saveassvg_pcs() { - savesvg($("#svg_pcs")[0], "Pearson Correlation Scatterplot.svg"); -} - -function saveassvg_srcs() { - savesvg($("#svg_srcs")[0], "Spearman Rank Correlation Scatterplot.svg"); -} - -drawg(); -srdrawg(); - - - -$(".chartupdate").change(function () { - chartupdate(); -}); - -$(".chartupdatewh").change(function () { - chartupdatewh(); -}); - -$(".chartupdatedata").change(function () { - chartupdatedata(); -}); - -$(document).ready(function(){ - //chartupdate(); -//chartupdatewh(); -chartupdatedata(); -//chartupdate(); -});
\ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot.js b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot.js index 28534709..57dae378 100644 --- a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot.js +++ b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot.js @@ -1,47 +1,398 @@ -// Generated by CoffeeScript 1.8.0 -var data, h, halfh, halfw, indID, intercept, margin, mychart, slope, totalh, totalw, w; +var chart; +var srchart; -h = 600; +var layout = { + height: 700, + width: 800, + margin: { + l: 60, + r: 30, + t: 80, + b: 50 + }, + xaxis: { + title: js_data.trait_1, + zeroline: false, + visible: true, + linecolor: 'black', + linewidth: 1, + }, + yaxis: { + title: js_data.trait_2, + zeroline: false, + visible: true, + linecolor: 'black', + linewidth: 1, + } +} -w = 800; +cofactor1_dict = {} +ranked_cofactor1_dict = {} +cofactor1_values = [] +ranked_cofactor1_values = [] +cofactor2_dict = {} +ranked_cofactor2_dict = {} -margin = { - left: 80, - top: 60, - right: 40, - bottom: 40, - inner: 30 -}; +function drawg() { + x_values = [] + y_values = [] + sample_names = [] + for (j = 0; j < js_data.data[0].length; j++) { + x_values.push(js_data.data[0][j]) + y_values.push(js_data.data[1][j]) + sample_names.push(js_data.indIDs[j]) + } + + var trace = { + x: x_values, + y: y_values, + mode: 'markers', + text: sample_names + } + + Plotly.newPlot('scatterplot2', [trace], layout) + +} + +function srdrawg() { + x_values = [] + y_values = [] + sample_names = [] + for (j = 0; j < js_data.rdata[0].length; j++) { + x_values.push(js_data.rdata[0][j]) + y_values.push(js_data.rdata[1][j]) + sample_names.push(js_data.indIDs[j]) + } + + var trace = { + x: x_values, + y: y_values, + mode: 'markers', + text: sample_names + } + + Plotly.newPlot('srscatterplot2', [trace], layout) +} + +function getdata() { + var data = []; + data.push({ + values: [], + slope: js_data.slope, + intercept: js_data.intercept + }); + + sizemin = 1; + sizemax = 50; + + if ($('input[name=cofactor1_vals]').val()){ + just_vals = [] + val_sample_dict = {} + val_sample_pairs = $('input[name=cofactor1_vals]').val().split(",") + for (i=0; i < val_sample_pairs.length; i++) { + just_vals.push(parseFloat(val_sample_pairs[i].split(":")[1])) + val_sample_dict[val_sample_pairs[i].split(":")[0]] = parseFloat(val_sample_pairs[i].split(":")[1]) + } + + cofactor1_dict = val_sample_dict + cofactor1_values = just_vals + } + + if ($('input[name=cofactor2_vals]').val()){ + vals_3 = []; + samples_3 = []; + val_sample_dict = {} + val_sample_pairs = $('input[name=cofactor2_vals]').val().split(",") + for (i=0; i < val_sample_pairs.length; i++) { + samples_3.push(val_sample_pairs[i].split(":")[0]) + vals_3.push(parseFloat(val_sample_pairs[i].split(":")[1])) + val_sample_dict[val_sample_pairs[i].split(":")[0]] = val_sample_pairs[i].split(":")[1] + } + datamin = d3.min(vals_3); + datamax = d3.max(vals_3); + + cofactor2_dict = val_sample_dict + } + + x_values = [] + y_values = [] + sample_names = [] + sizes = [] + + for (j = 0; j < js_data.data[0].length; j++) { + if ($('input[name=cofactor2_vals]').val()){ + if (samples_3.indexOf(js_data.indIDs[j])) { + datav = vals_3[j] + sizev = map1to2(datamin, datamax, sizemin, sizemax, datav); + } + } else { + datav = 0; + sizev = 10; + } + + x_values.push(js_data.data[0][j]) + y_values.push(js_data.data[1][j]) + sample_names.push(js_data.indIDs[j]) + sizes.push(sizev) + + data[0].values.push({ + type: "normal", + x: js_data.data[0][j], + y: js_data.data[1][j], + name: js_data.indIDs[j], + size: sizev, + v3: datav + }); + } + + point_text = [] + for (j = 0; j < sample_names.length; j++) { + this_text = "" + this_text += sample_names[j] + if (sample_names[j] in cofactor1_dict){ + this_text += "<br>Cofactor 1: " + cofactor1_dict[sample_names[j]] + } + if (sample_names[j] in cofactor2_dict){ + this_text += "<br>Cofactor 2: " + cofactor2_dict[sample_names[j]] + } + point_text.push(this_text) + } + + var trace = { + x: x_values, + y: y_values, + mode: 'markers', + text: point_text, + marker: { + size: sizes + } + } + + return [trace]; +} + +function map1to2 (min1, max1, min2, max2, v1) { + v2 = (v1 - min1) * (max2 - min2) / (max1 - min1) + min2; + return v2; +} + +function srgetdata() { + var data = []; + data.push({ + values: [], + slope: js_data.srslope, + intercept: js_data.srintercept + }); + + sizemin = 1; + sizemax = 50; + + x_values = [] + y_values = [] + sample_names = [] + sizes = [] + + if ($('input[name=ranked_cofactor1_vals]').val()){ + just_vals = [] + val_sample_dict = {} + val_sample_pairs = $('input[name=ranked_cofactor1_vals]').val().split(",") + for (i=0; i < val_sample_pairs.length; i++) { + just_vals.push(parseFloat(val_sample_pairs[i].split(":")[1])) + val_sample_dict[val_sample_pairs[i].split(":")[0]] = parseFloat(val_sample_pairs[i].split(":")[1]) + } + + ranked_cofactor1_dict = val_sample_dict + ranked_cofactor1_values = just_vals + } -halfh = h + margin.top + margin.bottom; + if ($('input[name=ranked_cofactor2_vals]').val()){ + vals_3 = [] + samples_3 = []; + val_sample_dict = {} + val_sample_pairs = $('input[name=ranked_cofactor2_vals]').val().split(",") + for (i=0; i<val_sample_pairs.length; i++){ + samples_3.push(val_sample_pairs[i].split(":")[0]) + vals_3.push(val_sample_pairs[i].split(":")[1]) + val_sample_dict[val_sample_pairs[i].split(":")[0]] = val_sample_pairs[i].split(":")[1] + } + datamin = d3.min(vals_3); + datamax = d3.max(vals_3); -totalh = halfh * 2; + ranked_cofactor2_dict = val_sample_dict + } -halfw = w + margin.left + margin.right; + for (j = 0; j < js_data.rdata[0].length; j++) { + if ($('input[name=ranked_cofactor2_vals]').val()){ + if (samples_3.indexOf(js_data.indIDs[j])) { + datav = vals_3[j] + sizev = map1to2(datamin, datamax, sizemin, sizemax, datav); + } + } else { + sizev = 10; + } -totalw = halfw * 2; + x_values.push(js_data.rdata[0][j]) + y_values.push(js_data.rdata[1][j]) + sample_names.push(js_data.indIDs[j]) + sizes.push(sizev) -mychart = scatterplot().xvar(0).yvar(1).xlab(js_data.trait_1).ylab(js_data.trait_2).height(h).width(w).margin(margin); + data[0].values.push({ + type: "ranked", + x: js_data.rdata[0][j], + y: js_data.rdata[1][j], + name: js_data.indIDs[j], + size: sizev, + }); + } -data = js_data.data; + point_text = [] + for (j = 0; j < sample_names.length; j++) { + this_text = "" + this_text += sample_names[j] + if (sample_names[j] in ranked_cofactor1_dict){ + this_text += "<br>Cofactor 1: " + ranked_cofactor1_dict[sample_names[j]] + } + if (sample_names[j] in ranked_cofactor2_dict){ + this_text += "<br>Cofactor 2: " + ranked_cofactor2_dict[sample_names[j]] + } + point_text.push(this_text) + } -indID = js_data.indIDs; + var trace = { + x: x_values, + y: y_values, + mode: 'markers', + text: point_text, + marker: { + size: sizes + } + } -slope = js_data.slope; + return [trace]; +} -intercept = js_data.intercept; +function chartupdatewh() { + var width = $("#width").val(); + var height = $("#height").val(); + + width_height_update = { + height: height, + width: width + } + + Plotly.newPlot('scatterplot2', getdata(), layout) + Plotly.relayout('scatterplot2', width_height_update) + Plotly.newPlot('srscatterplot2', srgetdata(), layout) + Plotly.relayout('srscatterplot2', width_height_update) +} + +function colorer(d) { + datamin = d3.min(cofactor1_values); + datamax = d3.max(cofactor1_values); + colormin = $("#cocolorfrom").val(); + colormax = $("#cocolorto").val(); + + compute = d3.interpolate(colormin, colormax); + linear = d3.scale.linear().domain([datamin, datamax]).range([0,1]); + + this_sample = d.tx.split("<br>")[0] + + c = compute(linear(cofactor1_dict[this_sample])); + + return c; +} + +function ranked_colorer(d) { + datamin = d3.min(ranked_cofactor1_values); + datamax = d3.max(ranked_cofactor1_values); + colormin = $("#cocolorfrom").val(); + colormax = $("#cocolorto").val(); + + compute = d3.interpolate(colormin, colormax); + linear = d3.scale.linear().domain([datamin, datamax]).range([0,1]); + + this_sample = d.tx.split("<br>")[0] + + c= compute(linear(ranked_cofactor1_dict[this_sample])); + + return c; +} + +function chartupdatedata() { + var size = $("#marksize").val(); + var shape = $("#markshape").val(); + + var pearson_title_update = { + title: "Pearson Correlation Scatterplot" + } + var spearman_title_update = { + title: "Spearman Rank Correlation Scatterplot" + } + + Plotly.newPlot('scatterplot2', getdata(), layout) + Plotly.relayout('scatterplot2', pearson_title_update) + Plotly.newPlot('srscatterplot2', srgetdata(), layout) + Plotly.relayout('srscatterplot2', spearman_title_update) + + if ($('input[name=cofactor1_vals]').val()){ + d3.select('#scatterplot2 svg').selectAll('.point') + .style({ + 'stroke': colorer, + 'fill': colorer + }); + d3.select('#srscatterplot2 svg').selectAll('.point') + .style({ + 'stroke': ranked_colorer, + 'fill': ranked_colorer + }); + } +} + +drawg(); +srdrawg(); + +$(".chartupdatewh").change(function () { + chartupdatewh(); +}); + +$(".chartupdatedata").change(function () { + chartupdatedata(); +}); + +$(".cofactor1_type").change(function () { + console.log("cofactor1 type:", $(".cofactor1_type").val()) + if ($(".cofactor1_type").val() == "color"){ + $(".cofactor2_type").val("size") + } else { + $(".cofactor2_type").val("color") + } +}); + +open_covariate_selection = function() { + return $('#collections_holder').load('/collections/list #collections_list', (function(_this) { + return function() { + $.colorbox({ + inline: true, + href: "#collections_holder", + onComplete: function(){ + $.getScript("/static/new/javascript/get_traits_from_collection.js"); + } + }); + return $('a.collection_name').attr('onClick', 'return false'); + }; + })(this)); +}; -console.log("THE DATA IS:", data); +$(document).ready(function(){ + chartupdatedata(); -d3.select("div#chart1").datum({ - data: data, - indID: indID, - slope: slope, - intercept: intercept -}).call(mychart); + $('#select_cofactor1').click(function () { + $('input[name=selecting_which_cofactor]').val("1"); + open_covariate_selection(); + }); -mychart.pointsSelect().on("mouseover", function(d) { - return d3.select(this).attr("r", mychart.pointsize() * 3); -}).on("mouseout", function(d) { - return d3.select(this).attr("r", mychart.pointsize()); + $('#select_cofactor2').click(function () { + $('input[name=selecting_which_cofactor]').val("2"); + open_covariate_selection(); + }); }); diff --git a/wqflask/wqflask/static/new/javascript/draw_probability_plot.js b/wqflask/wqflask/static/new/javascript/draw_probability_plot.js index f23dad0c..3d756303 100644 --- a/wqflask/wqflask/static/new/javascript/draw_probability_plot.js +++ b/wqflask/wqflask/static/new/javascript/draw_probability_plot.js @@ -26,7 +26,7 @@ redraw_prob_plot = function(samples, sample_group) { var container, h, margin, totalh, totalw, w; h = 600; - w = 600; + w = 500; margin = { left: 60, top: 40, diff --git a/wqflask/wqflask/static/new/javascript/get_traits_from_collection.js b/wqflask/wqflask/static/new/javascript/get_traits_from_collection.js index 4abb0735..8a79627a 100644 --- a/wqflask/wqflask/static/new/javascript/get_traits_from_collection.js +++ b/wqflask/wqflask/static/new/javascript/get_traits_from_collection.js @@ -4,7 +4,7 @@ var add_trait_data, assemble_into_json, back_to_collections, collection_click, c console.log("before get_traits_from_collection"); -collection_list = null; +//collection_list = null; this_trait_data = null; @@ -15,7 +15,6 @@ collection_click = function() { console.log("Clicking on:", $(this)); this_collection_url = $(this).find('.collection_name').prop("href"); this_collection_url += "&json"; - console.log("this_collection_url", this_collection_url); collection_list = $("#collections_holder").html(); return $.ajax({ dataType: "json", @@ -42,7 +41,6 @@ submit_click = function() { success: add_trait_data }); }); - console.log("SELECTED_TRAITS IS:", selected_traits); trait_names = []; samples = $('input[name=allsamples]').val().split(" "); all_vals = []; @@ -117,7 +115,6 @@ trait_click = function() { trait = $(this).parent().find('.trait').text(); dataset = $(this).parent().find('.dataset').text(); this_trait_url = "/trait/get_sample_data?trait=" + trait + "&dataset=" + dataset; - console.log("this_trait_url", this_trait_url); $.ajax({ dataType: "json", url: this_trait_url, @@ -134,28 +131,84 @@ add_trait_data = function(trait_data, textStatus, jqXHR) { return console.log("selected_traits:", selected_traits); }; +populate_cofactor_info = function(trait_info) { + if ($('input[name=selecting_which_cofactor]').val() == "1"){ + $('#cofactor1_trait_link').attr("href", trait_info['url']) + if (trait_info['type'] == "ProbeSet"){ + $('#cofactor1_trait_link').text(trait_info['species'] + " " + trait_info['group'] + " " + trait_info['tissue'] + " " + trait_info['db'] + ": " + trait_info['name']) + $('#cofactor1_description').text("[" + trait_info['symbol'] + " on " + trait_info['location'] + " Mb]\n" + trait_info['description']) + } else { + $('#cofactor1_trait_link').text(trait_info['species'] + " " + trait_info['group'] + " " + trait_info['db'] + ": " + trait_info['name']) + $('#cofactor1_description').html('<a href=\"' + trait_info['pubmed_link'] + '\">PubMed: ' + trait_info['pubmed_text'] + '</a><br>' + trait_info['description']) + } + $('#select_cofactor1').text("Change Cofactor 1"); + $('#cofactor1_info_container').css("display", "inline"); + $('#cofactor2_button').css("display", "inline"); + } else { + $('#cofactor2_trait_link').attr("href", trait_info['url']) + if (trait_info['type'] == "ProbeSet"){ + $('#cofactor2_trait_link').text(trait_info['species'] + " " + trait_info['group'] + " " + trait_info['tissue'] + " " + trait_info['db'] + ": " + trait_info['name']) + $('#cofactor2_description').text("[" + trait_info['symbol'] + " on " + trait_info['location'] + " Mb]\n" + trait_info['description']) + } else { + $('#cofactor2_trait_link').text(trait_info['species'] + " " + trait_info['group'] + " " + trait_info['db'] + ": " + trait_info['name']) + $('#cofactor2_description').html('<a href=\"' + trait_info['pubmed_link'] + '\">PubMed: ' + trait_info['pubmed_text'] + '</a><br>' + trait_info['description']) + } + $('#select_cofactor2').text("Change Cofactor 2"); + $('#cofactor2_info_container').css("display", "inline"); + } +} + get_trait_data = function(trait_data, textStatus, jqXHR) { var sample, samples, this_trait_vals, trait_sample_data, vals, _i, _len; - console.log("trait:", trait_data[0]); trait_sample_data = trait_data[1]; - console.log("trait_sample_data:", trait_sample_data); - samples = $('input[name=allsamples]').val().split(" "); + if ( $('input[name=allsamples]').length ) { + samples = $('input[name=allsamples]').val().split(" "); + } else { + samples = js_data.indIDs + } + sample_vals = []; vals = []; for (_i = 0, _len = samples.length; _i < _len; _i++) { sample = samples[_i]; - if (__indexOf.call(Object.keys(trait_sample_data), sample) >= 0) { - vals.push(parseFloat(trait_sample_data[sample])); + if (sample in trait_sample_data) { + sample_vals.push(sample + ":" + parseFloat(trait_sample_data[sample])) + vals.push(parseFloat(trait_sample_data[sample])) } else { - vals.push(null); + sample_vals.push(null) + vals.push(null) } } - if ($('input[name=samples]').length < 1) { - $('#hidden_inputs').append('<input type="hidden" name="samples" value="[' + samples.toString() + ']" />'); + if ( $('input[name=allsamples]').length ) { + if ($('input[name=samples]').length < 1) { + $('#hidden_inputs').append('<input type="hidden" name="samples" value="[' + samples.toString() + ']" />'); + } + $('#hidden_inputs').append('<input type="hidden" name="vals" value="[' + vals.toString() + ']" />'); + this_trait_vals = get_this_trait_vals(samples); + return color_by_trait(trait_sample_data); + } else{ + populate_cofactor_info(trait_data[0]) + sorted = vals.slice().sort(function(a,b){return a-b}) + ranks = vals.slice().map(function(v){ return sorted.indexOf(v)+1 }); + sample_ranks = [] + for (_i = 0; _i < samples.length; _i++){ + if (samples[_i] in trait_sample_data){ + sample_ranks.push(samples[_i] + ":" + ranks[_i]) + } else { + sample_ranks.push(null) + } + } + + if ($('input[name=selecting_which_cofactor]').val() == "1"){ + $('input[name=cofactor1_vals]').val(sample_vals) + $('input[name=ranked_cofactor1_vals]').val(sample_ranks) + } else { + $('input[name=cofactor2_vals]').val(sample_vals) + $('input[name=ranked_cofactor2_vals]').val(sample_ranks) + } + chartupdatedata(); + chartupdate(); + return false } - $('#hidden_inputs').append('<input type="hidden" name="vals" value="[' + vals.toString() + ']" />'); - this_trait_vals = get_this_trait_vals(samples); - console.log("THE LENGTH IS:", $('input[name=vals]').length); - return color_by_trait(trait_sample_data); }; get_this_trait_vals = function(samples) { @@ -215,14 +268,17 @@ process_traits = function(trait_data, textStatus, jqXHR) { } the_html += "</tbody>"; the_html += "</table>"; + the_html += "<div id=\"collection_list_html\" style=\"display: none;\">" + the_html += collection_list + the_html += "</div>" the_html += "<script type='text/javascript' src='/static/new/javascript/get_traits_from_collection.js'></script>" $("#collections_holder").html(the_html); return $('#collections_holder').colorbox.resize(); }; back_to_collections = function() { - console.log("collection_list:", collection_list); - $("#collections_holder").html(collection_list); + collection_list_html = $('#collection_list_html').html() + $("#collections_holder").html(collection_list_html); $(document).on("click", ".collection_line", collection_click); return $('#collections_holder').colorbox.resize(); }; diff --git a/wqflask/wqflask/static/new/javascript/plotly_probability_plot.js b/wqflask/wqflask/static/new/javascript/plotly_probability_plot.js index 49fd53b1..97d768c8 100644 --- a/wqflask/wqflask/static/new/javascript/plotly_probability_plot.js +++ b/wqflask/wqflask/static/new/javascript/plotly_probability_plot.js @@ -25,7 +25,7 @@ redraw_prob_plot = function(samples, sample_group) { var container, h, margin, totalh, totalw, w; - h = 600; + h = 370; w = 600; margin = { left: 60, @@ -71,11 +71,11 @@ })(); //ZS: 0.1 indicates buffer, increase to increase buffer y_domain = [sorted_values[0] - (sorted_values.slice(-1)[0] - sorted_values[0])*0.1, sorted_values.slice(-1)[0] + (sorted_values.slice(-1)[0] - sorted_values[0])*0.1] - sw_result = ShapiroWilkW(sorted_values); - W = sw_result.w.toFixed(3); - pvalue = sw_result.p.toFixed(3); - pvalue_str = pvalue > 0.05 ? pvalue.toString() : "<span style='color:red'>" + pvalue + "</span>"; - test_str = "Shapiro-Wilk test statistic is " + W + " (p = " + pvalue_str + ")"; + //sw_result = ShapiroWilkW(sorted_values); + //W = sw_result.w.toFixed(3); + //pvalue = sw_result.p.toFixed(3); + //pvalue_str = pvalue > 0.05 ? pvalue.toString() : "<span style='color:red'>" + pvalue + "</span>"; + //test_str = "Shapiro-Wilk test statistic is " + W + " (p = " + pvalue_str + ")"; z_scores = get_z_scores(sorted_values.length); //ZS: 0.1 indicates buffer, increase to increase buffer x_domain = [z_scores[0] - (z_scores.slice(-1)[0] - z_scores[0])*0.1, z_scores.slice(-1)[0] + (z_scores.slice(-1)[0] - z_scores[0])*0.1] @@ -106,7 +106,6 @@ }; }; data = [make_data('samples_primary'), make_data('samples_other'), make_data('samples_all')]; - console.log("THE DATA IS:", data); x_values = {} y_values = {} point_names = {} diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 117d8f12..e9dd3c9d 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -247,6 +247,9 @@ } root.bar_data[0]['x'] = trait_samples Plotly.newPlot('bar_chart', root.bar_data, root.bar_layout); + Plotly.relayout('bar_chart', { + 'yaxis.autorange': true + }); }; redraw_box_plot = function() { @@ -549,6 +552,7 @@ }; $('#block_outliers').click(block_outliers); reset_samples_table = function() { + $('input[name="transform"]').val(""); return $('.trait_value_input').each((function(_this) { return function(_index, element) { console.log("value is:", $(element).val()); @@ -559,6 +563,84 @@ })(this)); }; $('#reset').click(reset_samples_table); + + log_normalize_data = function() { + return $('.trait_value_input').each((function(_this) { + return function(_index, element) { + current_value = parseFloat($(element).data("value")) + 1; + if(isNaN(current_value)) { + return current_value + } else { + $(element).val(Math.log2(current_value).toFixed(3)); + return Math.log2(current_value).toFixed(3) + } + }; + })(this)); + }; + + sqrt_normalize_data = function() { + return $('.trait_value_input').each((function(_this) { + return function(_index, element) { + current_value = parseFloat($(element).data("value")) + 1; + if(isNaN(current_value)) { + return current_value + } else { + $(element).val(Math.sqrt(current_value).toFixed(3)); + return Math.sqrt(current_value).toFixed(3) + } + }; + })(this)); + }; + + qnorm_data = function() { + return $('.trait_value_input').each((function(_this) { + return function(_index, element) { + current_value = $(element).data("value"); + if(isNaN(current_value)) { + return current_value + } else { + $(element).val($(element).data("qnorm")); + return $(element).data("qnorm"); + } + }; + })(this)); + }; + + normalize_data = function() { + if ($('#norm_method option:selected').val() == 'log2'){ + if ($('input[name="transform"]').val() != "log2") { + log_normalize_data() + $('input[name="transform"]').val("log2") + } + } + else if ($('#norm_method option:selected').val() == 'sqrt'){ + if ($('input[name="transform"]').val() != "sqrt") { + sqrt_normalize_data() + $('input[name="transform"]').val("sqrt") + } + } + else if ($('#norm_method option:selected').val() == 'qnorm'){ + if ($('input[name="transform"]').val() != "qnorm") { + qnorm_data() + $('input[name="transform"]').val("qnorm") + } + } + } + + $('#normalize').click(normalize_data); + + switch_qnorm_data = function() { + return $('.trait_value_input').each((function(_this) { + return function(_index, element) { + transform_val = $(element).data('transform') + if (transform_val != "") { + $(element).val(transform_val.toFixed(3)); + } + return transform_val + }; + })(this)); + }; + $('#qnorm').click(switch_qnorm_data); get_sample_table_data = function(table_name) { var samples; samples = []; @@ -722,7 +804,7 @@ box_data = [trace1, trace2, trace3] } else { var box_layout = { - width: 500, + width: 300, height: 500, margin: { l: 50, @@ -789,7 +871,42 @@ type: 'bar' }; root.bar_data = [bar_trace] + + positive_error_vals = [] + negative_error_vals = [] + for (i = 0;i < get_sample_vals(sample_lists[0]).length; i++){ + if (get_sample_errors(sample_lists[0])[i] != undefined) { + positive_error_vals.push(get_sample_vals(sample_lists[0])[i] + get_sample_errors(sample_lists[0])[i]) + negative_error_vals.push(get_sample_vals(sample_lists[0])[i] - get_sample_errors(sample_lists[0])[i]) + } else { + positive_error_vals.push(get_sample_vals(sample_lists[0])[i]) + negative_error_vals.push(get_sample_vals(sample_lists[0])[i]) + } + } + + // Calculate the y axis cutoff to avoid a situation where all bar variation is clustered at the top of the chart + min_y_val = Math.min(...negative_error_vals) + max_y_val = Math.max(...positive_error_vals) + + if (min_y_val == 0) { + range_top = max_y_val + Math.abs(max_y_val)*0.1 + range_bottom = 0; + } else { + range_top = max_y_val + Math.abs(max_y_val)*0.1 + range_bottom = min_y_val - Math.abs(min_y_val)*0.1 + if (min_y_val > 0) { + range_bottom = min_y_val - 0.1*Math.abs(min_y_val) + } else if (min_y_val < 0) { + range_bottom = min_y_val + 0.1*min_y_val + } else { + range_bottom = 0 + } + } + var layout = { + yaxis: { + range: [range_bottom, range_top], + }, width: 1200, height: 500, margin: { @@ -837,6 +954,8 @@ $('#exclude_group').click(edit_data_change); $('#block_outliers').click(edit_data_change); $('#reset').click(edit_data_change); + $('#qnorm').click(edit_data_change); + $('#normalize').click(edit_data_change); return console.log("end"); }); diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index 4d98f5d8..daa5b3f2 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -208,50 +208,18 @@ }; })(this)); - $("#plink_compute").on("click", (function(_this) { - return function() { - var form_data, url; - //$("#static_progress_bar_container").modal(); - url = "/loading"; - $('input[name=method]').val("plink"); - $('input[name=maf]').val($('input[name=maf_plink]').val()); - form_data = $('#trait_data_form').serialize(); - console.log("form_data is:", form_data); - return submit_special(url); - //return do_ajax_post(url, form_data); - }; - })(this)); - $("#gemma_compute").on("click", (function(_this) { return function() { var form_data, url; console.log("RUNNING GEMMA"); - //$("#static_progress_bar_container").modal(); url = "/loading"; $('input[name=method]').val("gemma"); - $('input[name=genofile]').val($('#genofile_gemma').val()); - $('input[name=maf]').val($('input[name=maf_gemma]').val()); - form_data = $('#trait_data_form').serialize(); - console.log("form_data is:", form_data); - return submit_special(url); - //return do_ajax_post(url, form_data); - }; - })(this)); - - $("#gemma_bimbam_compute").on("click", (function(_this) { - return function() { - var form_data, url; - console.log("RUNNING GEMMA"); - //$("#static_progress_bar_container").modal(); - url = "/loading"; - $('input[name=method]').val("gemma_bimbam"); $('input[name=num_perm]').val(0); $('input[name=genofile]').val($('#genofile_gemma').val()); $('input[name=maf]').val($('input[name=maf_gemma]').val()); form_data = $('#trait_data_form').serialize(); console.log("form_data is:", form_data); return submit_special(url); - //return do_ajax_post(url, form_data); }; })(this)); diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 64daa1cc..3df4c81a 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -22,9 +22,9 @@ <body style="width: 100% !important;"> <!-- Navbar ================================================== --> - <div class="navbar navbar-inverse navbar-static-top pull-left" role="navigation"> + <div class="navbar navbar-inverse navbar-static-top pull-left" role="navigation" style="width: 100%;"> - <div class="container-fluid" style="width: 1500px !important;"> + <div class="container-fluid" style="width: 100%;"> <!-- Brand and toggle get grouped for better mobile display --> <div class="navbar-header"> <a class="navbar-brand" href="/">GeneNetwork</a> @@ -44,8 +44,8 @@ </li> <li class=""> <a href="/collections/list">Collections - {% if g.user_session.user_ob %} - <span class="badge badge-info">{{ g.user_session.user_ob.display_num_collections() }}</span> + {% if g.user_session.logged_in %} + <span class="badge badge-info">{{ g.user_session.num_collections }}</span> {% else %} <span class="badge badge-info">{{ g.cookie_session.display_num_collections() }}</span> {% endif %} @@ -71,7 +71,7 @@ </li> <li class=""> {% if g.user_session.logged_in %} - <a id="login_out" title="Signed in as {{ g.user_session.user_ob.name_and_org }}." href="/n/logout">Sign out</a> + <a id="login_out" title="Signed in as {{ g.user_session.user_name }}" href="/n/logout">Sign out</a> {% else %} <a id="login_in" href="/n/login">Sign in</a> {% endif %} diff --git a/wqflask/wqflask/templates/collections/add.html b/wqflask/wqflask/templates/collections/add.html index d45aa015..603947f4 100644 --- a/wqflask/wqflask/templates/collections/add.html +++ b/wqflask/wqflask/templates/collections/add.html @@ -1,36 +1,38 @@ <div id="myModal"> <div class="modal-header"> - <h3>Add to collection</h3> - <p>You have two choices: Create and name a collection, - or add the traits to an existing collection.</p> + <h2>Define or Add to Collection</h2> + <p>You have two choices: Name a new collection + or add to an existing collection.</p> </div> - <div class="modal-body"> + <div class="modal-body" style="margin-left: 20px;"> <form action="/collections/new" data-validate="parsley" id="add_form"> <input type="hidden" name="traits" value="{{ traits }}" /> <fieldset> - <legend>Create a new named collection</legend> - <label>New collection name</label> - <input type="text" name="new_collection" placeholder="Name of new collection..." - data-trigger="change" data-minlength="5" data-maxlength="50"> - <span class="help-block">Type the name of the new collection.</span> + <legend>1. Create a new collection</legend> + <div style="margin-left: 20px;"> + <!--<label>Collection name:</label>--> + <input type="text" name="new_collection" placeholder=" Name of new collection..." + data-trigger="change" data-minlength="5" data-maxlength="50" style="width: 100%"> + <button type="submit" name="create_new" class="btn btn-primary" style="margin-top: 20px;">Create collection</button> {% if uc is not defined %} <span class="help-block" style="color:red;">This collection will be saved to this computer for 5 days.</span> {% endif %} - <button type="submit" name="create_new" class="btn">Create and add traits</button> + </div> </fieldset> <hr /> <fieldset> - <legend>Or add to an existing collection</legend> - <label>Existing collection name</label> - + <legend>2. Add to an existing collection</legend> + <div style="margin-left: 20px;"> + <!--<label>Existing collection name:</label>--> <select name="existing_collection" class="form-control"> + <option selected disabled>Select Collection</option> {% for col in collections %} <option value="{{ col.id }}:{{ col.name }}">{{ col.name }}</option> {% endfor %} </select> <br /> - - <button type="submit" name="add_to_existing" class="btn">Add to existing collection</button> + <button type="submit" name="add_to_existing" class="btn btn-primary">Add to existing collection</button> + </div> </fieldset> </form> </div> diff --git a/wqflask/wqflask/templates/collections/list.html b/wqflask/wqflask/templates/collections/list.html index e7f3229b..64d5a676 100644 --- a/wqflask/wqflask/templates/collections/list.html +++ b/wqflask/wqflask/templates/collections/list.html @@ -3,12 +3,11 @@ {% block css %} <link rel="stylesheet" type="text/css" href="/static/new/packages/DataTables/css/jquery.dataTables.css" /> <link rel="stylesheet" type="text/css" href="/static/packages/DT_bootstrap/DT_bootstrap.css" /> - <link rel="stylesheet" type="text/css" href="/static/packages/TableTools/media/css/TableTools.css" /> <link rel="stylesheet" type="text/css" href="/static/new/packages/DataTables/extensions/buttons.bootstrap.css" /> {% endblock %} {% block content %} <!-- Start of body --> - {% if g.user_session.user_ob %} + {% if g.user_session.logged_in %} {{ header("Your Collections", 'You have {}.'.format(numify(collections|count, "collection", "collections"))) }} {% else %} @@ -19,8 +18,8 @@ <div class="container"> <div class="page-header"> - {% if g.user_session.user_ob %} - <h1>Collections owned by {{ g.user_session.user_ob.full_name }}</h1> + {% if g.user_session.logged_in %} + <h1>Collections owned by {{ g.user_session.user_name }}</h1> {% else %} <h1>Your Collections</h1> {% endif %} @@ -51,19 +50,24 @@ <tbody> {% for uc in collections %} <tr class="collection_line"> - {% if g.user_session.user_ob %} + {% if g.user_session.logged_in %} <td style="padding-left: 8px; padding-right: 0px; padding-top: 4px; align: center;"><INPUT TYPE="checkbox" NAME="collection" class="checkbox trait_checkbox" VALUE="{{ uc.id }}"></td> {% else %} <td style="padding-left: 8px; padding-right: 0px; padding-top: 4px; align: center;"><INPUT TYPE="checkbox" NAME="collection" class="checkbox trait_checkbox" VALUE="{{ uc.name }}"></td> {% endif %} <td align="right">{{ loop.index }} - {% if g.user_session.user_ob %} + {% if g.user_session.logged_in %} <td><a class="collection_name" href="{{ url_for('view_collection', uc_id=uc.id) }}">{{ uc.name }}</a></td> {% else %} <td><a class="collection_name" href="{{ url_for('view_collection', collection_id=uc.id) }}">{{ uc.name }}</a></td> {% endif %} + {% if g.user_session.logged_in %} + <td>{{ uc.created_timestamp }}</td> + <td>{{ uc.changed_timestamp }}</td> + {% else %} <td>{{ timeago(uc.created_timestamp.isoformat() + "Z") }}</td> <td>{{ timeago(uc.changed_timestamp.isoformat() + "Z") }}</td> + {% endif %} <td align="right">{{ uc.num_members }}</td> </tr> {% endfor %} diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index cae03b4a..864299a2 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -29,10 +29,6 @@ {{ this_trait.name }}:{{ this_trait.dataset.name }}, {% endfor %}" > - <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" title="Delete this collection" > - Delete this collection - </button> - <button id="corr_matrix" class="btn btn-primary submit_special" data-url="/corr_matrix" title="Correlation Matrix" > Correlation Matrix @@ -58,6 +54,15 @@ Heatmap </button> + <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" > + Comparison Bar Chart + </button> + + + <button id="delete" class="btn btn-danger submit_special" style="margin-left: 15px;" data-url="/collections/delete" title="Delete this collection" > + Delete Collection + </button> + </form> </div> @@ -66,8 +71,8 @@ <button class="btn btn-default" id="select_all"><span class="glyphicon glyphicon-ok"></span> Select All</button> <button class="btn btn-default" id="deselect_all"><span class="glyphicon glyphicon-remove"></span> Deselect All</button> <button class="btn btn-default" id="invert"><span class="glyphicon glyphicon-resize-vertical"></span> Invert</button> - <button class="btn" id="add" disabled="disabled"><i class="icon-plus-sign"></i> Add Record to Other Collection</button> - <button class="btn" id="remove" disabled="disabled"><i class="icon-minus-sign"></i> Remove Record</button> + <button class="btn btn-danger" id="remove" disabled="disabled"><i class="icon-minus-sign"></i> Delete From Collection</button> + <button class="btn" id="add" disabled="disabled"><i class="icon-plus-sign"></i> Copy Record to Other Collection</button> <br /> <br /> <form id="export_form" method="POST" action="/export_traits_csv"> @@ -246,6 +251,14 @@ url = $(this).data("url") return submit_special(url) }); + $("#comp_bar_chart").on("click", function() { + traits = $("#trait_table input:checked").map(function() { + return $(this).val(); + }).get(); + $("#trait_list").val(traits) + url = $(this).data("url") + return submit_special(url) + }); }); </script> diff --git a/wqflask/wqflask/templates/comparison_bar_chart.html b/wqflask/wqflask/templates/comparison_bar_chart.html new file mode 100644 index 00000000..033ff0a4 --- /dev/null +++ b/wqflask/wqflask/templates/comparison_bar_chart.html @@ -0,0 +1,39 @@ +{% extends "base.html" %} +{% block title %}Comparison Bar Chart{% endblock %} +{% block css %} + <link rel="stylesheet" type="text/css" href="/static/new/css/d3-tip.min.css" /> + <link rel="stylesheet" type="text/css" href="/static/new/css/panelutil.css" /> +{% endblock %} +{% block content %} <!-- Start of body --> + + {{ header("Comparison Bar Chart") }} + + <div class="container"> + <div> + <h3> + The following is a grouped bar chart with the sample values for each selected trait. + </h3> + </div> + <div id="chart_container"> + <div id="comp_bar_chart"></div> + </div> + + </div> + + <!-- End of body --> + +{% endblock %} + +{% block js %} + <script> + js_data = {{ js_data | safe }} + </script> + + <script language="javascript" type="text/javascript" src="http://d3js.org/d3.v3.min.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/js_external/d3-tip.min.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/panelutil.js"></script> + <script language="javascript" type="text/javascript" src="/static/packages/underscore/underscore-min.js"></script> + <script type="text/javascript" src="/static/new/js_external/plotly-latest.min.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/comparison_bar_chart.js"></script> + +{% endblock %}
\ No newline at end of file diff --git a/wqflask/wqflask/templates/corr_scatterplot.html b/wqflask/wqflask/templates/corr_scatterplot.html index 87588534..6451fda5 100644 --- a/wqflask/wqflask/templates/corr_scatterplot.html +++ b/wqflask/wqflask/templates/corr_scatterplot.html @@ -1,222 +1,294 @@ -{% extends "base.html" %}
-
-{% block css %}
- <link rel="stylesheet" type="text/css" href="/static/new/packages/DataTables/css/jquery.dataTables.css" />
- <link rel="stylesheet" type="text/css" href="/static/packages/DT_bootstrap/DT_bootstrap.css" />
- <link rel="stylesheet" type="text/css" href="/static/packages/TableTools/media/css/TableTools.css" />
- <link rel="stylesheet" type="text/css" href="/static/new/css/panelutil.css" />
- <link rel="stylesheet" type="text/css" href="/static/new/css/d3-tip.min.css" />
- <link rel="stylesheet" type="text/css" href="//cdnjs.cloudflare.com/ajax/libs/nvd3/1.8.5/nv.d3.min.css">
- <link rel="stylesheet" type="text/css" href="/static/new/css/corr_scatter_plot2.css" />
-{% endblock %}
-
-{% block content %}
-
-<div class="container-fluid">
-
-<h2>Correlation Scatterplot</h2>
-
-<div style="margin-left: 0px;">
-
- {% if trait_1.dataset.type == "ProbeSet" %}
- <div>
- X axis:
- <a href="{{url_for('show_trait_page', trait_id = trait_1.name, dataset = trait_1.dataset.name)}}">
- {{trait_1.dataset.group.species + " " + trait_1.dataset.group.name + " " + trait_1.dataset.tissue + " " + trait_1.dataset.name + ": " + trait_1.name|string}}
- </a>
- </div>
- <div>
- [{{trait_1.symbol}} on {{trait_1.location_repr}} Mb]
- {{trait_1.description_display}}
- </div>
- {% elif trait_1.dataset.type == "Publish" %}
- <div>
- X axis:
- <a href="{{url_for('show_trait_page', trait_id = trait_1.name, dataset = trait_1.dataset.name)}}">
- {{trait_1.dataset.group.species + " " + trait_1.dataset.group.name + " " + trait_1.dataset.name + ": " + trait_1.name|string}}
- </a>
- </div>
- <div>
- <a href="{{trait_1.pubmed_link}}">PubMed: {{trait_1.pubmed_text}}</a>
- {{trait_1.description_display}}
- </div>
- {% endif %}
-
- <br/>
-
- {% if trait_2.dataset.type == "ProbeSet" %}
- <div>
- Y axis:
- <a href="{{url_for('show_trait_page', trait_id = trait_2.name, dataset = trait_2.dataset.name)}}">
- {{trait_2.dataset.group.species + " " + trait_2.dataset.group.name + " " + trait_2.dataset.tissue + " " + trait_2.dataset.name + ": " + trait_2.name|string}}
- </a>
- </div>
- <div>
- [{{trait_2.symbol}} on {{trait_2.location_repr}} Mb]
- {{trait_2.description_display}}
- </div>
- {% elif trait_2.dataset.type == "Publish" %}
- <div>
- Y axis:
- <a href="{{url_for('show_trait_page', trait_id = trait_2.name, dataset = trait_2.dataset.name)}}">
- {{trait_2.dataset.group.species + " " + trait_2.dataset.group.name + " " + trait_2.dataset.name + ": " + trait_2.name|string}}
- </a>
- </div>
- <div>
- <a href="{{trait_2.pubmed_link}}">PubMed: {{trait_2.pubmed_text}}</a>
- {{trait_2.description_display}}
- </div>
- {% endif %}
-
-</div>
-
-<br>
-
-<table class="table">
- <tr>
- <td style="vertical-align: middle;">Width <input class="chartupdatewh" id="width" type="text" value="1000" style="width: 44px; height: 22px;"> px</td>
- <td style="vertical-align: middle;">Height <input class="chartupdatewh" id="height" type="text" value="800" style="width: 44px; height: 22px;"> px</td>
- </tr>
-</table>
-
-<table class="table">
- <tr>
- <td style="vertical-align: middle;">Mark
- <input class="chartupdate" id="markcolor" type="color" value="#8fbbda">
- <select class="chartupdatedata" id="marksize" style="width: 44px; height: 22px;">
- <option value="0">0</option>
- <option value="1">1</option>
- <option value="2">2</option>
- <option value="3">3</option>
- <option value="4">4</option>
- <option value="5" selected>5</option>
- <option value="6">6</option>
- <option value="7">7</option>
- <option value="8">8</option>
- <option value="9">9</option>
- <option value="10">10</option>
- <option value="15">15</option>
- <option value="20">20</option>
- <option value="25">25</option>
- <option value="30">30</option>
- </select>
- <select class="chartupdatedata" id="markshape" style="width: 100px; height: 22px;">
- <option value="circle" selected>Circle</option>
- <option value="cross">Cross</option>
- <option value="triangle-up">Triangle-up</option>
- <option value="triangle-down">Triangle-down</option>
- <option value="diamond">Diamond</option>
- <option value="square">Square</option>
- </select>
- </td>
- </tr>
-</table>
-
-<table class="table">
- <tr>
- <td style="vertical-align: middle;">Label
- <input class="chartupdate" id="labelcolor" type="color" value="#000000">
- <input class="chartupdate" id="labelfont" type="text" value="16" style="width: 44px; height: 22px;"> px
- </td>
- <td style="vertical-align: middle;">Number
- <input class="chartupdate" id="numbercolor" type="color" value="#000000">
- <input class="chartupdate" id="numberfont" type="text" value="16" style="width: 44px; height: 22px;"> px
- </td>
- <td style="vertical-align: middle;">Axis
- <input class="chartupdate" id="axiscolor" type="color" value="#000000">
- <input class="chartupdate" id="axiswidth" type="text" value="2" style="width: 44px; height: 22px;"> px
- </td>
- <td style="vertical-align: middle;">Line
- <input class="chartupdate" id="linecolor" type="color" value="#8fbbda">
- <input class="chartupdate" id="linewidth" type="text" value="1" style="width: 44px; height: 22px;"> px
- </td>
- </tr>
-</table>
-
-<br>
-
-<ul class="nav nav-tabs">
- <li class="active"><a href="#tp1" data-toggle="tab">Pearson</a></li>
- <li> <a href="#tp2" data-toggle="tab">Spearman Rank</a></li>
-</ul>
-
-<div class="tab-content">
- <div class="tab-pane active" id="tp1">
-
-<br>
-
-<div><a id="a_svg_pcs" href="#" onclick="javascript:saveassvg_pcs();" class="btn btn-primary">Save as SVG</a></div>
-
-<div style="width: 1000px; text-align: center;"><h2>Pearson Correlation Scatterplot</h2></div>
-
-<div id="scatterplot2"><svg id="svg_pcs" style="width: 1000px; height: 800px; margin-left: 10px;"></svg></div>
-
-<br>
-
-<table class="table table-hover table-striped table-bordered" style="width: 200px; margin-left: 80px; text-align: right;">
- <thead>
- <tr><th style="text-align: right;">Statistic</th><th style="text-align: right;">Value</th></tr>
- </thead>
- <tbody>
- <tr><td>Number</td> <td>{{jsdata.num_overlap}}</td></tr>
- <tr><td>Slope</td> <td>{{'%0.3f' % jsdata.slope}}</td></tr>
- <tr><td>Intercept</td> <td>{{'%0.3f' % jsdata.intercept}}</td></tr>
- <tr><td>r value</td> <td>{{'%0.3f' % jsdata.r_value}}</td></tr>
- <tr><td>P value</td> <td>{% if jsdata.p_value < 0.001 %}{{'%0.3e' % jsdata.p_value}}{% else %}{{'%0.3f' % jsdata.p_value}}{% endif %}</td></tr>
- <tr>
- <td style="text-align: left;" colspan="2">
- Regression Line
- <br>
- y = {{'%0.3f' % jsdata.slope}} * x + {{'%0.3f' % jsdata.intercept}}
- </td>
- </tr>
- </tbody>
-</table>
-
- </div>
-
- <div class="tab-pane" id="tp2">
-
-<br>
-
-<div><a id="a_svg_srcs" href="#" onclick="javascript:saveassvg_srcs();" class="btn btn-primary">Save as SVG</a></div>
-
-<div style="width: 1000px; text-align: center;"><h2>Spearman Rank Correlation Scatterplot</h2></div>
-
-<div id="srscatterplot2"><svg id="svg_srcs" style="width: 1000px; height: 800px; margin-left: 10px;"></svg></div>
-
-<br>
-
-<table class="table table-hover table-striped table-bordered" style="width: 200px; margin-left: 80px; text-align: right;">
- <thead>
- <tr><th style="text-align: right;">Statistic</th><th style="text-align: right;">Value</th></tr>
- </thead>
- <tbody>
- <tr><td>Number</td> <td>{{jsdata.num_overlap}}</td></tr>
- <tr><td>Slope</td> <td>{{'%0.3f' % jsdata.srslope}}</td></tr>
- <tr><td>Intercept</td> <td>{{'%0.3f' % jsdata.srintercept}}</td></tr>
- <tr><td>r value</td> <td>{{'%0.3f' % jsdata.srr_value}}</td></tr>
- <tr><td>P value</td> <td>{% if jsdata.srp_value < 0.001 %}{{'%0.3e' % jsdata.srp_value}}{% else %}{{'%0.3f' % jsdata.srp_value}}{% endif %}</td></tr>
- </tbody>
-</table>
-
- </div>
-
-</div>
-
-</div>
-
-{% endblock %}
-
-{% block js %}
- <script>
- js_data = {{ js_data | safe }};
- </script>
- <script language="javascript" type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/d3/3.5.17/d3.min.js"></script>
- <script language="javascript" type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/nvd3/1.8.5/nv.d3.min.js"></script>
- <script language="javascript" type="text/javascript" src="/static/packages/underscore/underscore-min.js"></script>
- <script language="javascript" type="text/javascript" src="/static/new/js_external/d3-tip.min.js"></script>
- <script language="javascript" type="text/javascript" src="/static/new/javascript/colorbrewer.js"></script>
- <script language="javascript" type="text/javascript" src="/static/new/javascript/panelutil.js"></script>
- <script language="javascript" type="text/javascript" src="/static/new/javascript/draw_corr_scatterplot-2.js"></script>
-{% endblock %}
+{% extends "base.html" %} + +{% block css %} + <link rel="stylesheet" type="text/css" href="/static/new/packages/DataTables/css/jquery.dataTables.css" /> + <link rel="stylesheet" type="text/css" href="/static/new/css/panelutil.css" /> + <link rel="stylesheet" type="text/css" href="/static/new/css/d3-tip.min.css" /> + <link rel="stylesheet" type="text/css" href="//cdnjs.cloudflare.com/ajax/libs/nvd3/1.8.5/nv.d3.min.css"> + <link rel="stylesheet" type="text/css" href="/static/new/css/corr_scatter_plot.css" /> +{% endblock %} + +{% block content %} + +<div class="container-fluid"> + + <input type="hidden" name="cofactor1_vals"></input> + <input type="hidden" name="ranked_cofactor1_vals"></input> + <input type="hidden" name="cofactor2_vals"></input> + <input type="hidden" name="ranked_cofactor2_vals"></input> + <input type="hidden" name="selecting_which_cofactor"></input> + + <h2>Correlation Scatterplot</h2> + + <table class="table"> + <tr> + <td style="vertical-align: middle;">Width <input class="chartupdatewh" id="width" type="text" value="800" style="width: 44px; height: 22px;"> px</td> + <td style="vertical-align: middle;">Height <input class="chartupdatewh" id="height" type="text" value="700" style="width: 44px; height: 22px;"> px</td> + </tr> + </table> + + <div> + <button type="button" class="btn btn-default" id="select_cofactor1"> + Select Cofactor 1 + </button> +<!-- + <select class="cofactor1_type"> + <option value="color">Color</option> + <option value="size">Size</option> + </select> +--> + <div id="cofactor2_button" style="margin-left: 10px; display: none;"> + <button type="button" class="btn btn-default" id="select_cofactor2"> + Add a Second Cofactor? + </button> +<!-- + <select class="cofactor2_type"> + <option value="color">Color</option> + <option value="size" selected="selected">Size</option> + </select> +--> + </div> + </div> + <div id="collections_holder_wrapper" style="display:none;"> + <div id="collections_holder"></div> + </div> + <br/> + + <div id="cofactor1_info_container" style="margin-left: 0px; display: none;"> + <div> + <b>Cofactor 1</b>: <a id="cofactor1_trait_link" href="#"></a> + </div> + <div id="cofactor1_description"></div> + <br> + <div style="margin-bottom: 10px;"> + <b>Cofactor 1 Color Range:</b> + <input class="chartupdatedata" id="cocolorfrom" type="color" value="#D9D9D9"> + <input class="chartupdatedata" id="cocolorto" type="color" value="#000000"> + </div> + </div> + + <div id="cofactor2_info_container" style="margin-left: 0px; display: none;"> + <hr> + <div> + <b>Cofactor 2</b>: <a id="cofactor2_trait_link" href="#"></a> + </div> + <div id="cofactor2_description"></div> + <br> + <div style="margin-bottom: 20px;"> + <b>Cofactor 2 corresponds to point radius</b> + </div> + <br> + </div> + + <ul class="nav nav-tabs"> + <li class="active"> + <a href="#tp1" data-toggle="tab">Pearson</a> + </li> + <li> + <a href="#tp2" data-toggle="tab">Spearman Rank</a> + </li> + </ul> + + <div class="tab-content"> + + <div class="tab-pane active" id="tp1"> + <br> + <div id="scatterplot2"></div> + <br> + <div class="row" style="width: 70%;"> + <div class="col-xs-3"> + <table class="table table-hover table-striped table-bordered" style="width: 80%; margin-left: 60px; text-align: right;"> + <thead> + <tr><th style="text-align: right;">Statistic</th><th style="text-align: right;">Value</th></tr> + </thead> + <tbody> + <tr> + <td>Number</td> + <td>{{jsdata.num_overlap}}</td> + </tr> + <tr> + <td>Slope</td> + <td>{{'%0.3f' % jsdata.slope}}</td> + </tr> + <tr> + <td>Intercept</td> + <td>{{'%0.3f' % jsdata.intercept}}</td> + </tr> + <tr> + <td>r value</td> + <td>{{'%0.3f' % jsdata.r_value}}</td> + </tr> + <tr> + <td>P value</td> + <td>{% if jsdata.p_value < 0.001 %}{{'%0.3e' % jsdata.p_value}}{% else %}{{'%0.3f' % jsdata.p_value}}{% endif %}</td> + </tr> + <tr> + <td style="text-align: left;" colspan="2"> + Regression Line + <br> + y = {{'%0.3f' % jsdata.slope}} * x + {{'%0.3f' % jsdata.intercept}} + </td> + </tr> + </tbody> + </table> + </div> + <div class="col-xs-9"> + {% if trait_1.dataset.type == "ProbeSet" %} + <div> + X axis: + <a href="{{url_for('show_trait_page', trait_id = trait_1.name, dataset = trait_1.dataset.name)}}"> + {{trait_1.dataset.group.species + " " + trait_1.dataset.group.name + " " + trait_1.dataset.tissue + " " + trait_1.dataset.name + ": " + trait_1.name|string}} + </a> + </div> + <div> + [{{trait_1.symbol}} on {{trait_1.location_repr}} Mb] + {{trait_1.description_display}} + </div> + {% elif trait_1.dataset.type == "Publish" %} + <div> + X axis: + <a href="{{url_for('show_trait_page', trait_id = trait_1.name, dataset = trait_1.dataset.name)}}"> + {{trait_1.dataset.group.species + " " + trait_1.dataset.group.name + " " + trait_1.dataset.name + ": " + trait_1.name|string}} + </a> + </div> + <div> + <a href="{{trait_1.pubmed_link}}">PubMed: {{trait_1.pubmed_text}}</a> + {{trait_1.description_display}} + </div> + {% endif %} + + <br/> + + {% if trait_2.dataset.type == "ProbeSet" %} + <div> + Y axis: + <a href="{{url_for('show_trait_page', trait_id = trait_2.name, dataset = trait_2.dataset.name)}}"> + {{trait_2.dataset.group.species + " " + trait_2.dataset.group.name + " " + trait_2.dataset.tissue + " " + trait_2.dataset.name + ": " + trait_2.name|string}} + </a> + </div> + <div> + [{{trait_2.symbol}} on {{trait_2.location_repr}} Mb] + {{trait_2.description_display}} + </div> + {% elif trait_2.dataset.type == "Publish" %} + <div> + Y axis: + <a href="{{url_for('show_trait_page', trait_id = trait_2.name, dataset = trait_2.dataset.name)}}"> + {{trait_2.dataset.group.species + " " + trait_2.dataset.group.name + " " + trait_2.dataset.name + ": " + trait_2.name|string}} + </a> + </div> + <div> + <a href="{{trait_2.pubmed_link}}">PubMed: {{trait_2.pubmed_text}}</a> + {{trait_2.description_display}} + </div> + {% endif %} + </div> + </div> + </div> + + <div class="tab-pane" id="tp2"> + <br> + <div id="srscatterplot2"></div> + <br> + <div class="row" style="width: 70%;"> + <div class="col-xs-3"> + <table class="table table-hover table-striped table-bordered" style="width: 80%; margin-left: 60px; text-align: right;"> + <thead> + <tr><th style="text-align: right;">Statistic</th><th style="text-align: right;">Value</th></tr> + </thead> + <tbody> + <tr> + <td>Number</td> + <td>{{jsdata.num_overlap}}</td> + </tr> + <tr> + <td>Slope</td> + <td>{{'%0.3f' % jsdata.srslope}}</td> + </tr> + <tr> + <td>Intercept</td> + <td>{{'%0.3f' % jsdata.srintercept}}</td> + </tr> + <tr> + <td>r value</td> + <td>{{'%0.3f' % jsdata.srr_value}}</td> + </tr> + <tr> + <td>P value</td> + <td>{% if jsdata.srp_value < 0.001 %}{{'%0.3e' % jsdata.srp_value}}{% else %}{{'%0.3f' % jsdata.srp_value}}{% endif %}</td> + </tr> + </tbody> + </table> + </div> + <div class="col-xs-9"> + {% if trait_1.dataset.type == "ProbeSet" %} + <div> + X axis: + <a href="{{url_for('show_trait_page', trait_id = trait_1.name, dataset = trait_1.dataset.name)}}"> + {{trait_1.dataset.group.species + " " + trait_1.dataset.group.name + " " + trait_1.dataset.tissue + " " + trait_1.dataset.name + ": " + trait_1.name|string}} + </a> + </div> + <div> + [{{trait_1.symbol}} on {{trait_1.location_repr}} Mb] + {{trait_1.description_display}} + </div> + {% elif trait_1.dataset.type == "Publish" %} + <div> + X axis: + <a href="{{url_for('show_trait_page', trait_id = trait_1.name, dataset = trait_1.dataset.name)}}"> + {{trait_1.dataset.group.species + " " + trait_1.dataset.group.name + " " + trait_1.dataset.name + ": " + trait_1.name|string}} + </a> + </div> + <div> + <a href="{{trait_1.pubmed_link}}">PubMed: {{trait_1.pubmed_text}}</a> + {{trait_1.description_display}} + </div> + {% endif %} + + <br/> + + {% if trait_2.dataset.type == "ProbeSet" %} + <div> + Y axis: + <a href="{{url_for('show_trait_page', trait_id = trait_2.name, dataset = trait_2.dataset.name)}}"> + {{trait_2.dataset.group.species + " " + trait_2.dataset.group.name + " " + trait_2.dataset.tissue + " " + trait_2.dataset.name + ": " + trait_2.name|string}} + </a> + </div> + <div> + [{{trait_2.symbol}} on {{trait_2.location_repr}} Mb] + {{trait_2.description_display}} + </div> + {% elif trait_2.dataset.type == "Publish" %} + <div> + Y axis: + <a href="{{url_for('show_trait_page', trait_id = trait_2.name, dataset = trait_2.dataset.name)}}"> + {{trait_2.dataset.group.species + " " + trait_2.dataset.group.name + " " + trait_2.dataset.name + ": " + trait_2.name|string}} + </a> + </div> + <div> + <a href="{{trait_2.pubmed_link}}">PubMed: {{trait_2.pubmed_text}}</a> + {{trait_2.description_display}} + </div> + {% endif %} + </div> + </div> + </div> + +</div> + +{% endblock %} + +{% block js %} + <script> + js_data = {{ js_data | safe }}; + </script> + <script language="javascript" type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/d3/3.5.17/d3.min.js"></script> + <script language="javascript" type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/nvd3/1.8.5/nv.d3.min.js"></script> + <script language="javascript" type="text/javascript" src="/static/packages/underscore/underscore-min.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/js_external/d3-tip.min.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/colorbrewer.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/panelutil.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/get_traits_from_collection.js"></script> + <script type="text/javascript" src="/static/new/js_external/plotly-latest.min.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/draw_corr_scatterplot.js"></script> +{% endblock %} diff --git a/wqflask/wqflask/templates/correlation_matrix.html b/wqflask/wqflask/templates/correlation_matrix.html index c4668e05..e4fd3136 100644 --- a/wqflask/wqflask/templates/correlation_matrix.html +++ b/wqflask/wqflask/templates/correlation_matrix.html @@ -1,8 +1,6 @@ {% extends "base.html" %} {% block css %} <link rel="stylesheet" type="text/css" href="/static/new/packages/DataTables/css/jquery.dataTables.css" /> - <link rel="stylesheet" type="text/css" href="/static/packages/DT_bootstrap/DT_bootstrap.css" /> - <link rel="stylesheet" type="text/css" href="/static/packages/TableTools/media/css/TableTools.css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/corr_matrix.css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/panelutil.css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/d3-tip.min.css" /> @@ -107,8 +105,6 @@ <script type="text/javascript" src="/static/new/js_external/d3-tip.min.js"></script> <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/jquery.js"></script> <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/jquery.dataTables.min.js"></script> - <script language="javascript" type="text/javascript" src="/static/packages/DT_bootstrap/DT_bootstrap.js"></script> - <script language="javascript" type="text/javascript" src="/static/packages/TableTools/media/js/TableTools.min.js"></script> <script language="javascript" type="text/javascript" src="/static/packages/underscore/underscore-min.js"></script> <script type="text/javascript" src="/static/new/javascript/panelutil.js"></script> <script language="javascript" type="text/javascript" src="/static/new/js_external/chroma.js"></script> diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index fb4e19a1..05136ad8 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -94,12 +94,14 @@ <th>Sample p(r)</th> <th>Lit r</th> <th>Tissue r</th> + <th>Tissue p(r)</th> {% else %} <th>Sample rho</th> <th>N</th> <th>Sample p(rho)</th> <th>Lit r</th> <th>Tissue rho</th> + <th>Tissue p(rho)</th> {% endif %} {% elif target_dataset.type == "Publish" %} {% if corr_method == 'pearson' %} @@ -156,8 +158,10 @@ {% endif %} {% if trait.tissue_corr == "" or trait.tissue_corr == 0.000 %} <td align="right">--</td> + <td align="right">--</td> {% else %} <td align="right">{{'%0.3f'|format(trait.tissue_corr)}}</td> + <td align="right">{{'%0.3e'|format(trait.tissue_pvalue)}}</td> {% endif %} {% elif target_dataset.type == "Publish" %} <td>{{ trait.description_display }}</td> @@ -319,7 +323,7 @@ title: 'correlation_results', fieldBoundary: '"', exportOptions: { - columns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + columns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] } } ], @@ -342,7 +346,8 @@ { "type": "natural" }, { "type": "scientific" }, { "type": "natural" }, - { "type": "natural" } + { "type": "natural" }, + { "type": "scientific" } ], "createdRow": function ( row, data, index ) { $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); diff --git a/wqflask/wqflask/templates/gsearch_gene.html b/wqflask/wqflask/templates/gsearch_gene.html index 6f2ad0b8..0612bfcc 100644 --- a/wqflask/wqflask/templates/gsearch_gene.html +++ b/wqflask/wqflask/templates/gsearch_gene.html @@ -43,26 +43,24 @@ <th data-export="Location">Location</th> <th data-export="Mean">Mean</th> <th data-export="Max LRS">Max LRS <a href="http://genenetwork.org//glossary.html#LRS" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th> - <th data-export="Max LRS Location">Max LRS Location</th> <th data-export="Additive Effect">Additive Effect <a href="http://genenetwork.org//glossary.html#A" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th> </tr> </thead> <tbody> {% for this_trait in trait_list %} - <tr id="trait:{{ this_trait.name }}:{{ this_trait.dataset.name }}"> - <td align="center" style="padding-right: 0px; padding-left: 5px;"><input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="{{ data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) }}"></td> + <tr id="trait:{{ this_trait.name }}:{{ this_trait.dataset }}"> + <td align="center" style="padding-right: 0px; padding-left: 5px;"><input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="{{ data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset)) }}"></td> <td align="right" data-export="{{ loop.index }}">{{ loop.index }}</td> - <td data-export="{{ this_trait.name }}"><a href="{{ url_for('show_trait_page', trait_id = this_trait.name, dataset = this_trait.dataset.name)}}">{{ this_trait.name }}</a></td> - <td data-export="{{ this_trait.dataset.group.species }}">{{ this_trait.dataset.group.species }}</td> - <td data-export="{{ this_trait.dataset.group.name }}">{{ this_trait.dataset.group.name }}</td> - <td data-export="{{ this_trait.dataset.tissue }}">{{ this_trait.dataset.tissue }}</td> - <td data-export="{{ this_trait.dataset.name }}">{{ this_trait.dataset.name }}</td> + <td data-export="{{ this_trait.name }}"><a href="{{ url_for('show_trait_page', trait_id = this_trait.name, dataset = this_trait.dataset)}}">{{ this_trait.name }}</a></td> + <td data-export="{{ this_trait.species }}">{{ this_trait.species }}</td> + <td data-export="{{ this_trait.group }}">{{ this_trait.group }}</td> + <td data-export="{{ this_trait.tissue }}">{{ this_trait.tissue }}</td> + <td data-export="{{ this_trait.dataset }}">{{ this_trait.dataset }}</td> <td data-export="{{ this_trait.symbol }}">{{ this_trait.symbol }}</td> - <td data-export="{{ this_trait.description_display }}">{{ this_trait.description_display }}</td> + <td data-export="{{ this_trait.description }}">{{ this_trait.description }}</td> <td data-export="{{ this_trait.location_repr }}" align="right">{{ this_trait.location_repr }}</td> <td data-export="{{ '%0.3f' % this_trait.mean|float }}" align="right">{{ '%0.3f' % this_trait.mean|float }}</td> <td data-export="{% if this_trait.LRS_score_repr != "N/A" %}{{ '%0.1f' % this_trait.LRS_score_repr|float }}{% else %}N/A{% endif %}" align="right">{% if this_trait.LRS_score_repr != "N/A" %}{{ '%0.1f' % this_trait.LRS_score_repr|float }}{% else %}N/A{% endif %}</td> - <td data-export="{{ this_trait.LRS_location_repr }}" align="right">{{ this_trait.LRS_location_repr }}</td> <td data-export="{% if this_trait.additive != "" %}{{ '%0.3f' % this_trait.additive|float }}{% else %}N/A{% endif %}" align="right">{% if this_trait.additive != "" %}{{ '%0.3f' % this_trait.additive|float }}{% else %}N/A{% endif %}</td> </tr> {% endfor %} @@ -81,7 +79,6 @@ <th>Location</th> <th>Mean</th> <th data-export="Max LRS">Max LRS <a href="http://genenetwork.org//glossary.html#LRS" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th> - <th data-export="LRS Location">Max LRS Location</th> <th data-export="Additive Effect">Additive Effect <a href="http://genenetwork.org//glossary.html#A" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th> </tr> </tfoot> @@ -142,7 +139,6 @@ { "type": "natural", "width": "7%" }, { "type": "natural", "width": "4%" }, { "type": "natural", "width": "5%" }, - { "type": "natural", "width": "7%" }, { "type": "natural", "width": "5%" } ], "columnDefs": [ diff --git a/wqflask/wqflask/templates/gsearch_pheno.html b/wqflask/wqflask/templates/gsearch_pheno.html index 7a8ca07b..c2cbdadd 100644 --- a/wqflask/wqflask/templates/gsearch_pheno.html +++ b/wqflask/wqflask/templates/gsearch_pheno.html @@ -40,23 +40,21 @@ <th data-export="Authors">Authors</th> <th data-export="Year">Year</th> <th data-export="Max LRS">Max LRS <a href="http://genenetwork.org//glossary.html#LRS" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th> - <th data-export="LRS Location">Max LRS Location</th> <th data-export="Additive Effect">Additive Effect <a href="http://genenetwork.org//glossary.html#A" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th> </tr> </thead> <tbody> {% for this_trait in trait_list %} - <tr id="trait:{{ this_trait.name }}:{{ this_trait.dataset.name }}"> - <td align="center" style="padding-right: 0px; padding-left: 5px;"><input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="{{ data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) }}"></td> + <tr id="trait:{{ this_trait.name }}:{{ this_trait.dataset }}"> + <td align="center" style="padding-right: 0px; padding-left: 5px;"><input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="{{ data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset)) }}"></td> <td align="right" data-export="{{ loop.index }}">{{ loop.index }}</td> - <td data-export="{{ this_trait.dataset.group.species }}">{{ this_trait.dataset.group.species }}</td> - <td data-export="{{ this_trait.dataset.group.name }}">{{ this_trait.dataset.group.name }}</td> - <td data-export="{{ this_trait.name }}"><a href="{{ url_for('show_trait_page', trait_id = this_trait.name, dataset = this_trait.dataset.name)}}">{{ this_trait.name }}</a></td> - <td data-export="{{ this_trait.description_display }}">{{ this_trait.description_display }}</td> + <td data-export="{{ this_trait.species }}">{{ this_trait.species }}</td> + <td data-export="{{ this_trait.group }}">{{ this_trait.group }}</td> + <td data-export="{{ this_trait.name }}"><a href="{{ url_for('show_trait_page', trait_id = this_trait.name, dataset = this_trait.dataset)}}">{{ this_trait.name }}</a></td> + <td data-export="{{ this_trait.description }}">{{ this_trait.description }}</td> <td data-export="{{ this_trait.authors }}">{{ this_trait.authors }}</td> <td data-export="{{ this_trait.pubmed_text }}" data-order="{{ this_trait.pubmed_text }}"><a href="{{ this_trait.pubmed_link }}">{{ this_trait.pubmed_text }}</a></td> <td data-export="{% if this_trait.LRS_score_repr != "N/A" %}{{ '%0.1f' % this_trait.LRS_score_repr|float }}{% else %}N/A{% endif %}" align="right">{% if this_trait.LRS_score_repr != "N/A" %}{{ '%0.1f' % this_trait.LRS_score_repr|float }}{% else %}N/A{% endif %}</td> - <td data-export="{{ this_trait.LRS_location_repr }}" align="right">{{ this_trait.LRS_location_repr }}</td> <td data-export="{% if this_trait.additive != "" %}{{ this_trait.additive }}{% else %}N/A{% endif %}" align="right">{% if this_trait.additive != "" %}{{ this_trait.additive }}{% else %}N/A{% endif %}</td> </tr> {% endfor %} @@ -72,7 +70,6 @@ <th>Authors</th> <th>Year</th> <th data-export="Max LRS">Max LRS <a href="http://genenetwork.org//glossary.html#LRS" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th> - <th data-export="LRS Location">Max LRS Location</th> <th data-export="Additive Effect">Additive Effect <a href="http://genenetwork.org//glossary.html#A" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th> </tr> </tfoot> @@ -135,7 +132,6 @@ { "type": "natural", "width": "25%"}, { "type": "natural" }, { "type": "natural", "width": "8%"}, - { "type": "natural", "width": "12%"}, { "type": "natural" } ], "columnDefs": [ diff --git a/wqflask/wqflask/templates/heatmap.html b/wqflask/wqflask/templates/heatmap.html index 49d7f962..68880829 100644 --- a/wqflask/wqflask/templates/heatmap.html +++ b/wqflask/wqflask/templates/heatmap.html @@ -39,7 +39,6 @@ <script language="javascript" type="text/javascript" src="/static/new/javascript/lodheatmap.js"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/lod_chart.js"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/curvechart.js"></script> -<!-- <script language="javascript" type="text/javascript" src="/static/new/javascript/create_heatmap.js"></script>--> <script language="javascript" type="text/javascript" src="/static/new/javascript/iplotMScanone_noeff.js"></script> <script language="javascript" type="text/javascript" src="/static/packages/underscore/underscore-min.js"></script> diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html index 72ced084..dba3e266 100755 --- a/wqflask/wqflask/templates/index_page_orig.html +++ b/wqflask/wqflask/templates/index_page_orig.html @@ -21,7 +21,7 @@ {{ flash_me() }} - <div class="row" style="width: 1400px !important;"> + <div class="row" style="width: 100%;"> <div class="col-xs-5"> <section id="search"> @@ -34,11 +34,11 @@ <div class="form-group"> <label for="species" class="col-xs-1 control-label" style="width: 65px !important;">Species:</label> - <div class="col-xs-10 controls input-append" style="padding-right: 0px;"> - <div class="col-xs-8"> - <select name="species" id="species" class="form-control span3" style="width: 280px !important;"></select> + <div class="col-xs-10 controls input-append" style="display: flex; padding-left: 20px;"> + <div class="col-8"> + <select name="species" id="species" class="form-control" style="width: 280px !important;"></select> </div> - <div class="col-xs-4"> + <div class="col-4" style="margin-left: 10px;"> <button type="button" id="make_default" class="btn btn-primary form-control">Make Default</button> </div> </div> @@ -46,9 +46,9 @@ <div class="form-group"> <label for="group" class="col-xs-1 control-label" style="width: 65px !important;">Group:</label> - <div class="col-xs-10 controls input-append"> - <div class="col-xs-8"> - <select name="group" id="group" class="form-control span3" style="width: 280px !important;"></select> + <div class="col-xs-10 controls input-append" style="padding-left: 20px;"> + <div class="col-8"> + <select name="group" id="group" class="form-control" style="width: 280px !important;"></select> <i class="icon-question-sign"></i> </div> </div> @@ -56,21 +56,21 @@ <div class="form-group"> <label for="tissue" class="col-xs-1 control-label" style="width: 65px !important;">Type:</label> - <div class="col-xs-10 controls"> - <div class="col-xs-8"> - <select name="type" id="type" class="form-control span3" style="width: 280px !important;"></select> + <div class="col-xs-10 controls" style="padding-left: 20px;"> + <div class="col-8"> + <select name="type" id="type" class="form-control" style="width: 280px !important;"></select> </div> </div> </div> <div class="form-group"> <label for="dataset" class="col-xs-1 control-label" style="width: 65px !important;">Dataset:</label> - <div class="col-xs-10 controls input-append"> - <div class="col-xs-10"> - <select name="dataset" id="dataset" class="form-control span5" style="width: 340px !important;"></select> + <div class="col-xs-10 controls input-append" style="display: flex; padding-left: 20px;"> + <div class="col-9"> + <select name="dataset" id="dataset" class="form-control" style="width: 340px !important;"></select> <i class="icon-question-sign"></i> </div> - <div class="col-xs-2"> + <div class="col-3" style="margin-left: 10px;"> <button type="button" id="dataset_info" class="btn btn-primary form-control" style="width: 75px !important;">Info</button> </div> </div> @@ -85,8 +85,8 @@ <div class="form-group"> <label for="or_search" class="col-xs-1 control-label" style="padding-left: 0px; padding-right: 0px; width: 65px !important;">Get Any:</label> - <div class="col-xs-10 controls"> - <div class="col-xs-8"> + <div class="col-xs-10 controls" style="padding-left: 20px;"> + <div class="col-8"> <textarea onkeydown="pressed(event)" name="search_terms_or" rows="1" class="form-control search-query" style="max-width: 550px; width: 450px !important;" id="or_search"></textarea> </div> </div> @@ -95,8 +95,8 @@ <!-- GET ANY HELP --> <div class="form-group"> <label for="btsearch" class="col-xs-1 control-label" style="width: 65px !important;"></label> - <div class="col-xs-10 controls"> - <div class="col-xs-12 controls"> + <div class="col-xs-10 controls" style="padding-left: 20px;"> + <div class="col-12 controls"> Enter terms, genes, ID numbers in the <b>Search</b> field.<br> Use <b>*</b> or <b>?</b> wildcards (Cyp*a?, synap*).<br> Use <b>quotes</b> for terms such as <i>"tyrosine kinase"</i>. @@ -106,8 +106,8 @@ <div class="form-group"> <label for="and_search" class="col-xs-1 control-label" style="padding-left: 0px; padding-right: 0px; width: 65px !important;">Combined:</label> - <div class="col-xs-10 controls"> - <div class="col-xs-8"> + <div class="col-xs-10 controls" style="padding-left: 20px;"> + <div class="col-8"> <textarea onkeydown="pressed(event)" name="search_terms_and" rows="1" class="form-control search-query" style="max-width: 550px; width: 450px !important;" id="and_search"></textarea> </div> </div> @@ -115,8 +115,8 @@ <div class="form-group"> <label for="btsearch" class="col-xs-1 control-label" style="width: 65px !important;"></label> - <div class="col-xs-10 controls"> - <div class="col-xs-2 controls" style="width: 100px !important;"> + <div class="col-xs-10 controls" style="padding-left: 20px;"> + <div class="col-2 controls" style="width: 100px !important;"> <input id="btsearch" type="submit" class="btn btn-primary form-control" value="Search"> </div> </div> diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 4a857c60..27b20ebf 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -31,16 +31,19 @@ <div> {% if external_login["github"]: %} <a href="{{external_login['github']}}" title="Login with GitHub" class="btn btn-info btn-group">Login with Github</a> + {% else %} + <p>Github login is not available right now</p> {% endif %} {% if external_login["orcid"]: %} <a href="{{external_login['orcid']}}" title="Login with ORCID" class="btn btn-info btn-group">Login with ORCID</a> + {% else %} + <p>ORCID login is not available right now</p> {% endif %} </div> {% else: %} <div class="alert alert-warning"> - <p>You cannot login with external services at this time.<br /> - Please try again later.</p> + <p>Sorry, you cannot login with Github or ORCID at this time.</p> </div> {% endif %} <hr /> diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index 5d128b4a..f67fff90 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -16,16 +16,6 @@ {% endblock %} {% block content %} <!-- Start of body --> -<!-- - {% if dataset.type == 'Temp' %} - {{ header("{}".format("Temporary Trait")) }} - {% elif dataset.type != 'Geno' %} - {{ header("{}".format(this_trait.name_header_fmt), - '{}: {}'.format(this_trait.name, this_trait.description_fmt)) }} - {% else %} - {{ header("{}".format(this_trait.name_header_fmt)) }} - {% endif %} ---> <div class="container"> <h2>Trait Data and Analysis for <b>{{ this_trait.name }}</b></h2> {% if this_trait.dataset.type != 'Publish' %} @@ -45,6 +35,7 @@ <input type="hidden" name="temp_uuid" id="temp_uuid" value="{{ temp_uuid }}"> <input type="hidden" name="genofile" value=""> <input type="hidden" name="covariates" value=""> + <input type="hidden" name="transform" value=""> <div class="container"> <div class="panel-group" id="accordion"> diff --git a/wqflask/wqflask/templates/show_trait_edit_data.html b/wqflask/wqflask/templates/show_trait_edit_data.html index a431821e..ac2f1078 100644 --- a/wqflask/wqflask/templates/show_trait_edit_data.html +++ b/wqflask/wqflask/templates/show_trait_edit_data.html @@ -9,7 +9,7 @@ needed. </p> - <div id="blockMenuSpan" class="input-append"> + <div id="blockMenuSpan" class="input-append" style="margin-bottom: 10px;"> <label for="remove_samples_field">Block samples by index:</label> <input type="text" id="remove_samples_field"> <select id="block_group" size="1"> @@ -25,11 +25,8 @@ <div id="remove_samples_invalid" class="alert alert-error" style="display:none;"> Please check that your input is formatted correctly, e.g. <strong>3, 5-10, 12</strong> </div> - - <br> - {% if sample_groups[0].attributes %} - <div class="input-append"> + <div class="input-append" style="margin-top:10px; margin-bottom:10px;"> <label for="exclude_menu">Block samples by group:</label> <select id="exclude_menu" size=1> {% for attribute in sample_groups[0].attributes %} @@ -42,7 +39,6 @@ <input type="button" id="exclude_group" class="btn" value="Block"> </div> {% endif %} - <br> <div> <input type="button" id="hide_no_value" class="btn btn-default" value="Hide No Value"> <input type="button" id="block_outliers" class="btn btn-default" value="Block Outliers"> @@ -54,6 +50,20 @@ <option value="csv">CSV</option> </select> </span> + <br> + <div style="margin-top:10px;"> + <input type="button" id="normalize" class="btn btn-default" value="Normalize"> + <select id="norm_method" class="select optional span2"> + <option value="log2">Log2</option> + <option value="qnorm">Quantile</option> + <option value="sqrt">Square Root</option> + </select> + </div> + <!-- + {% if sample_groups[0].sample_qnorm is not none %} + <input type="button" id="qnorm" class="btn btn-default" value="Quantile Normalize"> + {% endif %} + --> </div> </fieldset> <br> @@ -75,6 +85,7 @@ <!--<div id="edit_sample_lists">--> {% for sample_type in sample_groups %} + {% set outer_loop = loop %} <div class="sample_group" style="width:{{ trait_table_width }}%;"> <h3>{{ sample_type.header }}</h3> <hr> @@ -111,7 +122,7 @@ {# Todo: Add IDs #} <td class="column_name-Value" align="right"> - <input type="text" data-value="{{ sample.display_value }}" name="{{ 'value:' + sample.name }}" + <input type="text" data-value="{{ sample.display_value }}" data-qnorm="{{ qnorm_vals[outer_loop.index - 1][loop.index - 1] }}" name="{{ 'value:' + sample.name }}" style="text-align:right;" class="trait_value_input edit_sample_value" value="{{ sample.display_value }}" diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 03590c2c..495a266e 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -14,9 +14,11 @@ <li> <a href="#rqtl_geno" data-toggle="tab">R/qtl</a> </li> +<!-- <li> <a href="#pylmm" data-toggle="tab">pyLMM</a> </li> +--> {% endif %} {% for mapping_method in dataset.group.mapping_names %} {% if mapping_method == "GEMMA" %} @@ -35,8 +37,6 @@ </ul> <div class="tab-content"> - {# if use_pylmm_rqtl and not use_plink_gemma and dataset.group.species != "human" #} - {% if dataset.group.mapping_id == "1" %} <div class="tab-pane active" id="gemma"> <div style="padding-top: 20px;" class="form-horizontal"> {% if genofiles and genofiles|length>0 %} @@ -70,15 +70,19 @@ </label> </div> </div> - <!-- - </div> - <div style="padding-top: 5px; padding-bottom: 5px; padding-left: 20px;" class="form-horizontal"> - --> <div class="mapping_method_fields form-group"> <label style="text-align: right;" class="col-xs-3 control-label">Covariates</label> <div style="margin-left:20px;" class="col-xs-7"> - {% if g.user_session.user_ob and (g.user_session.user_ob.display_num_collections() == "") %} + {% if g.user_session.logged_in %} + {% if g.user_session.num_collections < 1 %} No collections available. Please add traits to a collection to use them as covariates. + {% else %} + <div style="margin-bottom: 10px;" class="btn-group" role="group"> + <button type="button" id="select_covariates" class="btn btn-default">Select</button> + <button type="button" id="remove_covariates" class="btn btn-default">Remove</button> + </div> + <textarea rows="3" cols="20" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none;" class="selected_covariates"></textarea> + {% endif %} {% elif g.cookie_session.display_num_collections() == "" %} No collections available. Please add traits to a collection to use them as covariates. {% else %} @@ -93,25 +97,14 @@ <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> <div style="margin-left:20px;" class="col-xs-6"> - <button id="gemma_bimbam_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression"> + <button id="gemma_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression"> Compute </button> </div> </div> </div> -<!-- - <div class="form-group"> - <div class="col-xs-4 controls"> - <label class="col-xs-2 control-label"></label> - <div class="col-xs-4"> - <button id="gemma_bimbam_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression"> - Compute - </button> - </div> - </div> - </div> ---> </div> + {% if dataset.group.mapping_id == "1" %} <div class="tab-pane" id="interval_mapping"> <div style="margin-top: 20px" class="form-horizontal"> {% if genofiles and genofiles|length>0 %} @@ -181,8 +174,6 @@ </div> </div> --> - - <div class="mapping_method_fields form-group"> <label style="text-align: right;" class="col-xs-3 control-label">Marker<br>Regression</label> <div style="margin-left:20px;" class="col-xs-6 controls"> @@ -204,11 +195,9 @@ </button> </div> </div> - <!--<div id="alert_placeholder"></div>--> </div> </div> <div class="tab-pane" id="rqtl_geno"> - <div style="margin-top: 20px" class="form-horizontal"> {% if genofiles and genofiles|length>0 %} <div class="mapping_method_fields form-group"> @@ -337,104 +326,24 @@ </div> </div> {% endif %} - {% for mapping_method in dataset.group.mapping_names %} - {% if mapping_method == "GEMMA" %} - <div class="tab-pane active" id="gemma_plink"> - <div style="padding-top: 10px;" class="form-horizontal"> - {% if genofiles and genofiles|length>0 %} - <div class="mapping_method_fields form-group"> - <label style="text-align:right;" class="col-xs-3 control-label">Genotypes</label> - <div style="margin-left: 20px;" class="col-xs-4 controls"> - <select id="genofile_gemma" class="form-control"> - {% for item in genofiles %} - <option value="{{item['location']}}:{{item['title']}}">{{item['title']}}</option> - {% endfor %} - </select> - </div> - </div> - {% endif %} - <div class="mapping_method_fields form-group"> - <label style="text-align:right;" class="col-xs-3 control-label">MAF</label> - <div style="margin-left: 20px;" class="col-xs-4 controls"> - <input name="maf_gemma" value="0.01" type="text" class="form-control"> - </div> - </div> - <div class="mapping_method_fields form-group"> - <label style="text-align: right;" class="col-xs-3 control-label">Covariates</label> - <div style="margin-left:20px;" class="col-xs-7"> - {% if g.user_session.user_ob and (g.user_session.user_ob.display_num_collections() == "") %} - No collections available. Please add traits to a collection to use them as covariates. - {% elif g.cookie_session.display_num_collections() == "" %} - No collections available. Please add traits to a collection to use them as covariates. - {% else %} - <div style="margin-bottom: 10px;" class="btn-group" role="group"> - <button type="button" id="select_covariates" class="btn btn-default">Select</button> - <button type="button" id="remove_covariates" class="btn btn-default">Remove</button> - </div> - <textarea rows="3" cols="20" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none;" class="selected_covariates"></textarea> - {% endif %} - </div> - </div> - <div class="mapping_method_fields form-group"> - <label class="col-xs-3 control-label"></label> - <div style="margin-left:20px;" class="col-xs-6"> - <button id="gemma_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression"> - Compute - </button> - </div> - </div> - </div> - </div> - {% elif mapping_method == "PLINK" %} - <div class="tab-pane" id="plink"> - <div style="padding: 20px" class="form-horizontal"> - <div class="mapping_method_fields form-group"> - <label style="text-align: right;" class="col-xs-3 control-label">MAF</label> - <div style="margin-left: 20px;" class="col-xs-4 controls"> - <input name="maf_plink" value="0.01" type="text" class="form-control"> - </div> - </div> - </div> - - <div class="form-group"> - <label style="text-align: right;" class="col-xs-3 control-label"></label> - <div style="margin-left:20px;" class="col-xs-6 controls"> - <button id="plink_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression"> - Compute - </button> - </div> - </div> - </div> - {% endif %} - {% endfor %} </div> </div> </div> <div class="col-xs-5"> - {% if dataset.group.mapping_id == "1" %} <dl> <dt>GEMMA</dt> <dd>GEMMA is software implementing the Genome-wide Efficient Mixed Model Association algorithm for a standard linear mixed model for genome-wide association studies (GWAS).</dd> + {% if dataset.group.mapping_id == "1" %} <dt>Interval Mapping</dt> <dd>Interval mapping is a process in which the statistical significance of a hypothetical QTL is evaluated at regular points across a chromosome, even in the absence of explicit genotype data at those points.</dd> <dt>R/qtl</dt> <dd>R/qtl is an extensible, interactive environment for mapping quantitative trait loci (QTL) in experimental crosses.</dd> +<!-- <dt>pyLMM</dt> <dd>pyLMM is a fast and lightweight linear mixed-model (LMM) solver for use in genome-wide association studies (GWAS).</dd> +--> + {% endif %} </dl> - {% else %} - <dl> - {% if mapping_method == "GEMMA" %} - <dt>GEMMA</dt> - <dd> - GEMMA is software implementing the Genome-wide Efficient Mixed Model Association algorithm for a standard linear mixed model for genome-wide association studies (GWAS).<br> - </dd> - {% elif mapping_method == "PLINK" %} - <dt>PLINK</dt> - <dd>PLINK is a free, open-source whole genome association analysis toolset.</dd> - {% endif %} - </dl> - {% endif %} </div> <div id="mapping_result_holder_wrapper" style="display:none;"> <div id="mapping_result_holder"></div> diff --git a/wqflask/wqflask/templates/show_trait_statistics.html b/wqflask/wqflask/templates/show_trait_statistics.html index 09df3245..988c8047 100644 --- a/wqflask/wqflask/templates/show_trait_statistics.html +++ b/wqflask/wqflask/templates/show_trait_statistics.html @@ -13,7 +13,7 @@ <li> <a href="#probability_plot" data-toggle="tab">Probability Plot</a> </li> - {% if g.user_session.user_ob %} + {% if g.user_session.logged_in %} <li> <a href="#scatterplot_matrix" data-toggle="tab">Scatterplot Matrix</a> </li> @@ -72,7 +72,7 @@ <i class="icon-signal"></i> Sort By Value </button> </div> - {% if g.user_session.user_ob %} + {% if g.user_session.logged_in %} <div class="btn-group"> <button type="button" class="btn btn-default" id="color_by_trait"> <i class="icon-tint"></i> Color by Trait @@ -126,7 +126,7 @@ <div id="box_plot"></div> </div> </div> - {% if g.user_session.user_ob %} + {% if g.user_session.logged_in %} <div class="tab-pane" id="scatterplot_matrix"> <div style="margin-bottom:40px;" class="btn-group"> diff --git a/wqflask/wqflask/tracer.py b/wqflask/wqflask/tracer.py deleted file mode 100644 index a1043d28..00000000 --- a/wqflask/wqflask/tracer.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import absolute_import, division, print_function - -print("At top of tracer") - -import sys - -#################################################################################### - -# Originally based on http://stackoverflow.com/a/8315566 -def tracefunc(frame, event, arg, indent=[0]): - - func = dict(funcname = frame.f_code.co_name, - filename = frame.f_code.co_filename, - lineno = frame.f_lineno) - - #These are too common to bother printing... - too_common = ( - '/home/sam/ve27/local/lib/python2.7/site-packages/werkzeug/', - '/home/sam/ve27/local/lib/python2.7/site-packages/jinja2/', - ) - - - if func['filename'].startswith(too_common): - return tracefunc - - info = "{funcname} [{filename}: {lineno}]".format(**func) - - if event == "call": - indent[0] += 2 - #print("-" * indent[0] + "> call function", frame.f_code.co_name) - print("-" * indent[0] + "> call function:", info) - elif event == "return": - print("<" + "-" * indent[0], "exit function:", info) - indent[0] -= 2 - return tracefunc - -def turn_on(): - sys.settrace(tracefunc) - print("Tracing turned on!!!!") -#################################################################################### - diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 5f6c818e..d6f39e83 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -49,23 +49,21 @@ def timestamp(): class AnonUser(object): """Anonymous user handling""" - cookie_name = 'anon_user_v8' + cookie_name = 'anon_user_v1' def __init__(self): self.cookie = request.cookies.get(self.cookie_name) if self.cookie: - logger.debug("already is cookie") + logger.debug("ANON COOKIE ALREADY EXISTS") self.anon_id = verify_cookie(self.cookie) else: - logger.debug("creating new cookie") + logger.debug("CREATING NEW ANON COOKIE") self.anon_id, self.cookie = create_signed_cookie() + res = flask.make_response() + res.set_cookie(self.cookie_name, self.cookie) self.key = "anon_collection:v1:{}".format(self.anon_id) - @after.after_this_request - def set_cookie(response): - response.set_cookie(self.cookie_name, self.cookie) - def add_collection(self, new_collection): collection_dict = dict(name = new_collection.name, created_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), @@ -109,16 +107,11 @@ class AnonUser(object): result = Redis.get(self.key) collections_list = json.loads(result if result else "[]") for collection in collections_list: - uc = model.UserCollection() - uc.name = collection['name'] - collection_exists = g.user_session.user_ob.get_collection_by_name(uc.name) + collection_exists = g.user_session.get_collection_by_name(collection['name']) if collection_exists: continue else: - uc.user = g.user_session.user_id - uc.members = json.dumps(collection['members']) - db_session.add(uc) - db_session.commit() + g.user_session.add_collection(collection['name'], collection['members']) def display_num_collections(self): """ @@ -156,11 +149,12 @@ def create_signed_cookie(): class UserSession(object): """Logged in user handling""" - cookie_name = 'session_id_v2' + cookie_name = 'session_id_v1' def __init__(self): cookie = request.cookies.get(self.cookie_name) if not cookie: + logger.debug("NO USER COOKIE") self.logged_in = False return else: @@ -171,7 +165,6 @@ class UserSession(object): self.session_id = session_id self.record = Redis.hgetall(self.redis_key) - if not self.record: # This will occur, for example, when the browser has been left open over a long # weekend and the site hasn't been visited by the user @@ -203,21 +196,211 @@ class UserSession(object): return '' @property - def user_ob(self): - """Actual sqlalchemy record""" - # Only look it up once if needed, then store it - # raise "OBSOLETE: use ElasticSearch instead" - try: - if LOG_SQL_ALCHEMY: - logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG) + def es_user_id(self): + """User id from ElasticSearch (need to check if this is the same as the id stored in self.records)""" + + es = get_elasticsearch_connection() + user_email = self.record['user_email_address'] + + #ZS: Get user's collections if they exist + response = es.search( + index = "users", doc_type = "local", body = { + "query": { "match": { "email_address": user_email } } + }) + + user_id = response['hits']['hits'][0]['_id'] + return user_id + + @property + def user_name(self): + """Shortcut to the user_name""" + if 'user_name' in self.record: + return self.record['user_name'] + else: + return '' + + @property + def user_collections(self): + """List of user's collections""" + + es = get_elasticsearch_connection() + + user_email = self.record['user_email_address'] + + #ZS: Get user's collections if they exist + response = es.search( + index = "users", doc_type = "local", body = { + "query": { "match": { "email_address": user_email } } + }) + user_info = response['hits']['hits'][0]['_source'] + if 'collections' in user_info.keys(): + if len(user_info['collections']) > 0: + return json.loads(user_info['collections']) + else: + return [] + else: + return [] + + @property + def num_collections(self): + """Number of user's collections""" + + es = get_elasticsearch_connection() + + user_email = self.record['user_email_address'] + + #ZS: Get user's collections if they exist + response = es.search( + index = "users", doc_type = "local", body = { + "query": { "match": { "email_address": user_email } } + }) + + user_info = response['hits']['hits'][0]['_source'] + logger.debug("USER NUM COLL:", user_info) + if 'collections' in user_info.keys(): + if user_info['collections'] != "[]" and len(user_info['collections']) > 0: + collections_json = json.loads(user_info['collections']) + return len(collections_json) + else: + return 0 + else: + return 0 + +### +# ZS: This is currently not used, but I'm leaving it here commented out because the old "set superuser" code (at the bottom of this file) used it +### +# @property +# def user_ob(self): +# """Actual sqlalchemy record""" +# # Only look it up once if needed, then store it +# # raise "OBSOLETE: use ElasticSearch instead" +# try: +# if LOG_SQL_ALCHEMY: +# logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG) +# +# # Already did this before +# return self.db_object +# except AttributeError: +# # Doesn't exist so we'll create it +# self.db_object = model.User.query.get(self.user_id) +# return self.db_object + + def add_collection(self, collection_name, traits): + """Add collection into ElasticSearch""" + + collection_dict = {'id': unicode(uuid.uuid4()), + 'name': collection_name, + 'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), + 'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), + 'num_members': len(traits), + 'members': list(traits) } + + es = get_elasticsearch_connection() + + user_email = self.record['user_email_address'] + response = es.search( + index = "users", doc_type = "local", body = { + "query": { "match": { "email_address": user_email } } + }) + + user_id = response['hits']['hits'][0]['_id'] + user_info = response['hits']['hits'][0]['_source'] + + if 'collections' in user_info.keys(): + if user_info['collections'] != [] and user_info['collections'] != "[]": + current_collections = json.loads(user_info['collections']) + current_collections.append(collection_dict) + self.update_collections(current_collections) + #collections_json = json.dumps(current_collections) + else: + self.update_collections([collection_dict]) + #collections_json = json.dumps([collection_dict]) + else: + self.update_collections([collection_dict]) + #collections_json = json.dumps([collection_dict]) + + return collection_dict['id'] + + def delete_collection(self, collection_id): + """Remove collection with given ID""" + + updated_collections = [] + for collection in self.user_collections: + if collection['id'] == collection_id: + continue + else: + updated_collections.append(collection) + + self.update_collections(updated_collections) + + return collection['name'] + + def add_traits_to_collection(self, collection_id, traits_to_add): + """Add specified traits to a collection""" + + this_collection = self.get_collection_by_id(collection_id) + + updated_collection = this_collection + updated_traits = this_collection['members'] + traits_to_add + + updated_collection['members'] = updated_traits + updated_collection['num_members'] = len(updated_traits) + updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + + updated_collections = [] + for collection in self.user_collections: + if collection['id'] == collection_id: + updated_collections.append(updated_collection) + else: + updated_collections.append(collection) + + self.update_collections(updated_collections) + + def remove_traits_from_collection(self, collection_id, traits_to_remove): + """Remove specified traits from a collection""" + + this_collection = self.get_collection_by_id(collection_id) + + updated_collection = this_collection + updated_traits = [] + for trait in this_collection['members']: + if trait in traits_to_remove: + continue + else: + updated_traits.append(trait) + + updated_collection['members'] = updated_traits + updated_collection['num_members'] = len(updated_traits) + updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + + updated_collections = [] + for collection in self.user_collections: + if collection['id'] == collection_id: + updated_collections.append(updated_collection) + else: + updated_collections.append(collection) + + self.update_collections(updated_collections) + + return updated_traits + + def get_collection_by_id(self, collection_id): + for collection in self.user_collections: + if collection['id'] == collection_id: + return collection + + def get_collection_by_name(self, collection_name): + for collection in self.user_collections: + if collection['name'] == collection_name: + return collection + + return None - # Already did this before - return self.db_object - except AttributeError: - # Doesn't exist so we'll create it - self.db_object = model.User.query.get(self.user_id) - return self.db_object + def update_collections(self, updated_collections): + es = get_elasticsearch_connection() + collection_body = {'doc': {'collections': json.dumps(updated_collections)}} + es.update(index='users', doc_type='local', id=self.es_user_id, refresh='wait_for', body=collection_body) def delete_session(self): # And more importantly delete the redis record @@ -594,6 +777,7 @@ class LoginUser(object): def __init__(self): self.remember_me = False + self.logged_in = False def oauth2_login(self, login_type, user_id): """Login via an OAuth2 provider""" @@ -664,6 +848,8 @@ class LoginUser(object): #g.cookie_session.import_traits_to_user() + self.logged_in = True + return self.actual_login(user, import_collections=import_col) else: @@ -683,6 +869,7 @@ class LoginUser(object): max_age = self.remember_time else: max_age = None + response.set_cookie(UserSession.cookie_name, session_id_signed, max_age=max_age) return response @@ -698,6 +885,7 @@ class LoginUser(object): session = dict(login_time = time.time(), user_id = user.id, + user_name = user.full_name, user_email_address = user.email_address) key = UserSession.cookie_name + ":" + login_rec.session_id @@ -708,6 +896,7 @@ class LoginUser(object): else: expire_time = THREE_DAYS Redis.expire(key, expire_time) + return session_id_signed def unsuccessful_login(self, user): @@ -727,85 +916,89 @@ def logout(): return response -@app.route("/n/forgot_password") +@app.route("/n/forgot_password", methods=['GET']) def forgot_password(): """Entry point for forgotten password""" - return render_template("new_security/forgot_password.html") + print("ARGS: ", request.args) + errors = {"no-email": request.args.get("no-email")} + print("ERRORS: ", errors) + return render_template("new_security/forgot_password.html", errors=errors) @app.route("/n/forgot_password_submit", methods=('POST',)) def forgot_password_submit(): """When a forgotten password form is submitted we get here""" params = request.form email_address = params['email_address'] - logger.debug("Wants to send password E-mail to ",email_address) - es = get_elasticsearch_connection() - user_details = get_user_by_unique_column(es, "email_address", email_address) - if user_details: - ForgotPasswordEmail(user_details["email_address"]) - # try: - # user = model.User.query.filter_by(email_address=email_address).one() - # except orm.exc.NoResultFound: - # flash("Couldn't find a user associated with the email address {}. Sorry.".format( - # email_address)) - # return redirect(url_for("login")) - # ForgotPasswordEmail(user) - return render_template("new_security/forgot_password_step2.html", - subject=ForgotPasswordEmail.subject) + next_page = None + if email_address != "": + logger.debug("Wants to send password E-mail to ",email_address) + es = get_elasticsearch_connection() + user_details = get_user_by_unique_column(es, "email_address", email_address) + if user_details: + ForgotPasswordEmail(user_details["email_address"]) + + return render_template("new_security/forgot_password_step2.html", + subject=ForgotPasswordEmail.subject) + + else: + flash("You MUST provide an email", "alert-danger") + return redirect(url_for("forgot_password")) @app.errorhandler(401) def unauthorized(error): return redirect(url_for('login')) -def super_only(): - try: - superuser = g.user_session.user_ob.superuser - except AttributeError: - superuser = False - if not superuser: - flash("You must be a superuser to access that page.", "alert-error") - abort(401) - - - -@app.route("/manage/users") -def manage_users(): - super_only() - template_vars = UsersManager() - return render_template("admin/user_manager.html", **template_vars.__dict__) - -@app.route("/manage/user") -def manage_user(): - super_only() - template_vars = UserManager(request.args) - return render_template("admin/ind_user_manager.html", **template_vars.__dict__) - -@app.route("/manage/groups") -def manage_groups(): - super_only() - template_vars = GroupsManager(request.args) - return render_template("admin/group_manager.html", **template_vars.__dict__) - -@app.route("/manage/make_superuser") -def make_superuser(): - super_only() - params = request.args - user_id = params['user_id'] - user = model.User.query.get(user_id) - superuser_info = basic_info() - superuser_info['crowned_by'] = g.user_session.user_id - user.superuser = json.dumps(superuser_info, sort_keys=True) - db_session.commit() - flash("We've made {} a superuser!".format(user.name_and_org)) - return redirect(url_for("manage_users")) - -@app.route("/manage/assume_identity") -def assume_identity(): - super_only() - params = request.args - user_id = params['user_id'] - user = model.User.query.get(user_id) - assumed_by = g.user_session.user_id - return LoginUser().actual_login(user, assumed_by=assumed_by) +### +# ZS: The following 6 functions require the old MySQL User accounts; I'm leaving them commented out just in case we decide to reimplement them using ElasticSearch +### +#def super_only(): +# try: +# superuser = g.user_session.user_ob.superuser +# except AttributeError: +# superuser = False +# if not superuser: +# flash("You must be a superuser to access that page.", "alert-error") +# abort(401) + +#@app.route("/manage/users") +#def manage_users(): +# super_only() +# template_vars = UsersManager() +# return render_template("admin/user_manager.html", **template_vars.__dict__) + +#@app.route("/manage/user") +#def manage_user(): +# super_only() +# template_vars = UserManager(request.args) +# return render_template("admin/ind_user_manager.html", **template_vars.__dict__) + +#@app.route("/manage/groups") +#def manage_groups(): +# super_only() +# template_vars = GroupsManager(request.args) +# return render_template("admin/group_manager.html", **template_vars.__dict__) + +#@app.route("/manage/make_superuser") +#def make_superuser(): +# super_only() +# params = request.args +# user_id = params['user_id'] +# user = model.User.query.get(user_id) +# superuser_info = basic_info() +# superuser_info['crowned_by'] = g.user_session.user_id +# user.superuser = json.dumps(superuser_info, sort_keys=True) +# db_session.commit() +# flash("We've made {} a superuser!".format(user.name_and_org)) +# return redirect(url_for("manage_users")) + +#@app.route("/manage/assume_identity") +#def assume_identity(): +# super_only() +# params = request.args +# user_id = params['user_id'] +# user = model.User.query.get(user_id) +# assumed_by = g.user_session.user_id +# return LoginUser().actual_login(user, assumed_by=assumed_by) @app.route("/n/register", methods=('GET', 'POST')) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3c2cca94..ae333e73 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -43,6 +43,7 @@ from base.data_set import DataSet # Used by YAML in marker_regression from wqflask.show_trait import show_trait from wqflask.show_trait import export_trait_data from wqflask.heatmap import heatmap +from wqflask.comparison_bar_chart import comparison_bar_chart from wqflask.marker_regression import marker_regression from wqflask.marker_regression import marker_regression_gn1 from wqflask.network_graph import network_graph @@ -57,7 +58,6 @@ from utility import temp_data from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,JS_TWITTER_POST_FETCHER_PATH,JS_GUIX_PATH, CSS_PATH from utility.helper_functions import get_species_groups -from base import webqtlFormData from base.webqtlConfig import GENERATED_IMAGE_DIR from utility.benchmark import Bench @@ -73,7 +73,6 @@ import utility.logger logger = utility.logger.getLogger(__name__ ) - @app.before_request def connect_db(): db = getattr(g, '_database', None) @@ -90,11 +89,6 @@ def shutdown_session(exception=None): db_session.remove() g.db = None -#@app.before_request -#def trace_it(): -# from wqflask import tracer -# tracer.turn_on() - @app.errorhandler(Exception) def handle_bad_request(e): err_msg = str(e) @@ -163,62 +157,37 @@ def css(filename): def twitter(filename): return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename) -#@app.route("/data_sharing") -#def data_sharing_page(): -# logger.info("In data_sharing") -# fd = webqtlFormData.webqtlFormData(request.args) -# logger.info("1Have fd") -# sharingInfoObject = SharingInfo.SharingInfo(request.args['GN_AccessionId'], None) -# info, htmlfilelist = sharingInfoObject.getBody(infoupdate="") -# logger.info("type(htmlfilelist):", type(htmlfilelist)) -# htmlfilelist = htmlfilelist.encode("utf-8") -# #template_vars = SharingInfo.SharingInfo(request.args['GN_AccessionId'], None) -# logger.info("1 Made it to rendering") -# return render_template("data_sharing.html", -# info=info, -# htmlfilelist=htmlfilelist) - - @app.route("/search", methods=('GET',)) def search_page(): logger.info("in search_page") logger.info(request.url) - if 'info_database' in request.args: - logger.info("Going to sharing_info_page") - template_vars = sharing_info_page() - if template_vars.redirect_url: - logger.info("Going to redirect") - return flask.redirect(template_vars.redirect_url) - else: - return render_template("data_sharing.html", **template_vars.__dict__) + result = None + if USE_REDIS: + with Bench("Trying Redis cache"): + key = "search_results:v1:" + json.dumps(request.args, sort_keys=True) + logger.debug("key is:", pf(key)) + result = Redis.get(key) + if result: + logger.info("Redis cache hit on search results!") + result = pickle.loads(result) else: - result = None - if USE_REDIS: - with Bench("Trying Redis cache"): - key = "search_results:v1:" + json.dumps(request.args, sort_keys=True) - logger.debug("key is:", pf(key)) - result = Redis.get(key) - if result: - logger.info("Redis cache hit on search results!") - result = pickle.loads(result) - else: - logger.info("Skipping Redis cache (USE_REDIS=False)") + logger.info("Skipping Redis cache (USE_REDIS=False)") - logger.info("request.args is", request.args) - the_search = search_results.SearchResultPage(request.args) - result = the_search.__dict__ - valid_search = result['search_term_exists'] + logger.info("request.args is", request.args) + the_search = search_results.SearchResultPage(request.args) + result = the_search.__dict__ + valid_search = result['search_term_exists'] - logger.debugf("result", result) + logger.debugf("result", result) - if USE_REDIS and valid_search: - Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) - Redis.expire(key, 60*60) + if USE_REDIS and valid_search: + Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) + Redis.expire(key, 60*60) - if valid_search: - return render_template("search_result_page.html", **result) - else: - return render_template("search_error.html") + if valid_search: + return render_template("search_result_page.html", **result) + else: + return render_template("search_error.html") @app.route("/gsearch", methods=('GET',)) def gsearchact(): @@ -496,6 +465,27 @@ def heatmap_page(): return rendered_template +@app.route("/comparison_bar_chart", methods=('POST',)) +def comp_bar_chart_page(): + logger.info("In comp bar chart, request.form is:", pf(request.form)) + logger.info(request.url) + + start_vars = request.form + + traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] + if traits[0] != "": + template_vars = comparison_bar_chart.ComparisonBarChart(request.form) + template_vars.js_data = json.dumps(template_vars.js_data, + default=json_default_handler, + indent=" ") + + result = template_vars.__dict__ + rendered_template = render_template("comparison_bar_chart.html", **result) + else: + rendered_template = render_template("empty_collection.html", **{'tool':'Comparison Bar Chart'}) + + return rendered_template + @app.route("/mapping_results_container") def mapping_results_container_page(): return render_template("mapping_results_container.html") @@ -741,7 +731,6 @@ def network_graph_page(): def corr_compute_page(): logger.info("In corr_compute, request.form is:", pf(request.form)) logger.info(request.url) - #fd = webqtlFormData.webqtlFormData(request.form) template_vars = show_corr_results.CorrelationResults(request.form) return render_template("correlation_page.html", **template_vars.__dict__) @@ -777,15 +766,6 @@ def submit_bnw(): template_vars = get_bnw_input(request.form) return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) -# Todo: Can we simplify this? -Sam -def sharing_info_page(): - """Info page displayed when the user clicks the "Info" button next to the dataset selection""" - logger.info("In sharing_info_page") - logger.info(request.url) - fd = webqtlFormData.webqtlFormData(request.args) - template_vars = SharingInfoPage.SharingInfoPage(fd) - return template_vars - # Take this out or secure it before putting into production @app.route("/get_temp_data") def get_temp_data(): |