diff options
| author | zsloan | 2022-09-06 13:32:28 -0500 |
|---|---|---|
| committer | GitHub | 2022-09-06 13:32:28 -0500 |
| commit | d8bc7067b717e0b680d98b7cfcbc26c758a109bf (patch) | |
| tree | 571677761b3bb464b55bc19f208f135214321ad2 | |
| parent | e0626f40d8fe4fa83daba52b82c1b459b34b1849 (diff) | |
| parent | 363237f11b9eb14f52c4f0c43a931c99c827c496 (diff) | |
| download | genenetwork2-d8bc7067b717e0b680d98b7cfcbc26c758a109bf.tar.gz | |
Merge branch 'testing' into feature/generalize_tables
81 files changed, 1496 insertions, 1455 deletions
@@ -84,7 +84,7 @@ use `GN3_PYTHONPATH` environment that gets injected in the ./bin/genenetwork2 startup. A continuously deployed instance of genenetwork2 is available at -[https://gn2dev.genenetwork.org/](https://gn2dev.genenetwork.org/). This +[https://cd.genenetwork.org/](https://cd.genenetwork.org/). This instance is redeployed on every commit provided that the [continuous integration tests](https://ci.genenetwork.org/jobs/genenetwork2) pass. @@ -173,6 +173,25 @@ check the logs. If there is ERROR 1054 (42S22): Unknown column 'InbredSet.Family' in 'field list' it may be you are trying the small database. +### Run Scripts + +As part of the profiling effort, some scripts are added to run specific parts of the system under a profiler without running the entire web-server - as such, to run the script, you could do something like: + +``` +env HOME=/home/frederick \ + GN2_PROFILE=~/opt/gn2-latest \ + GN3_DEV_REPO_PATH=~/genenetwork/genenetwork3 \ + SQL_URI="mysql://username:password@host-ip:host-port/db_webqtl" \ + SERVER_PORT=5001 \ + bin/genenetwork2 ../gn2_settings.py \ + -cli python3 -m scripts.profile_corrs \ + ../performance_$(date +"%Y%m%dT%H:%M:%S").profile +``` + +and you can find the performance metrics at the file specified, in this case, a file starting with `performance_` with the date and time of the run, and ending with `.profile`. + +Please replace the environment variables in the sample command above with the appropriate values for your environment. + ## Documentation User documentation can be found diff --git a/containers/db-container.scm b/containers/db-container.scm new file mode 100644 index 00000000..848b6a2e --- /dev/null +++ b/containers/db-container.scm @@ -0,0 +1,38 @@ +;;; This file describes a Guix system container to run the database +;;; services required by genenetwork2 locally on your own +;;; machine. This is to allow a purely offline development setup for +;;; hacking on genenetwork2. To build and run the container, use +;;; db-container.sh. + +(use-modules (gnu) + (gnu packages databases) + (gnu services databases)) + +(define %mariadb-state-directory + "/var/lib/mysql") + +(define set-permissions-gexp + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + ;; Set ownership of mariadb state directory. + (let ((user (getpw "mysql"))) + (for-each (lambda (file) + (chown file (passwd:uid user) (passwd:gid user))) + (find-files #$%mariadb-state-directory #:directories? #t)))))) + +(operating-system + (host-name "genenetwork2") + (timezone "Etc/UTC") + (locale "en_US.utf8") + (bootloader (bootloader-configuration + (bootloader grub-bootloader) + (targets (list "does-not-matter")))) + (file-systems %base-file-systems) + (services (cons* (service mysql-service-type) + (service redis-service-type) + (simple-service 'set-permissions + activation-service-type + set-permissions-gexp) + %base-services))) diff --git a/containers/db-container.sh b/containers/db-container.sh new file mode 100755 index 00000000..bbe0247a --- /dev/null +++ b/containers/db-container.sh @@ -0,0 +1,12 @@ +#! /bin/sh -e + +# Find path to containers directory. +containers=$(dirname $0) + +# Create mariadb state directory if it does not exist. +mkdir -p $containers/mariadb-state + +# Build the container. +guix system container --network \ + --share=$containers/mariadb-state=/var/lib/mysql \ + $containers/db-container.scm diff --git a/etc/default_settings.py b/etc/default_settings.py index a0d01de0..ff84f021 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -33,6 +33,9 @@ REDIS_URL = "redis://:@localhost:6379/0" # gn2-proxy GN2_PROXY = "http://localhost:8080" +# GN PROXY +GN_PROXY_URL="https://genenetwork.org/gn3-proxy/" + # ---- MySQL SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py new file mode 100644 index 00000000..abb5794a --- /dev/null +++ b/test/requests/correlation_tests.py @@ -0,0 +1,70 @@ +import requests +from lxml.html import parse +from link_checker import check_page + +def sample_vals(): + return '{"C57BL/6J":"10.835","DBA/2J":"11.142","B6D2F1":"11.126","D2B6F1":"11.143","BXD1":"10.811","BXD2":"11.503","BXD5":"10.766","BXD6":"10.986","BXD8":"11.050","BXD9":"10.822","BXD11":"10.670","BXD12":"10.946","BXD13":"10.890","BXD14":"x","BXD15":"10.884","BXD16":"11.222","BXD18":"x","BXD19":"10.968","BXD20":"10.962","BXD21":"10.906","BXD22":"11.080","BXD23":"11.046","BXD24":"11.146","BXD24a":"x","BXD25":"x","BXD27":"11.078","BXD28":"11.034","BXD29":"10.808","BXD30":"x","BXD31":"11.087","BXD32":"11.029","BXD33":"10.662","BXD34":"11.482","BXD35":"x","BXD36":"x","BXD37":"x","BXD38":"10.836","BXD39":"10.926","BXD40":"10.638","BXD41":"x","BXD42":"10.974","BXD43":"10.828","BXD44":"10.900","BXD45":"11.358","BXD48":"11.042","BXD48a":"10.975","BXD49":"x","BXD50":"11.228","BXD51":"11.126","BXD52":"x","BXD53":"x","BXD54":"x","BXD55":"11.580","BXD56":"x","BXD59":"x","BXD60":"10.829","BXD61":"11.152","BXD62":"11.156","BXD63":"10.942","BXD64":"10.506","BXD65":"11.126","BXD65a":"11.272","BXD65b":"11.157","BXD66":"11.071","BXD67":"11.080","BXD68":"10.997","BXD69":"11.096","BXD70":"11.152","BXD71":"x","BXD72":"x","BXD73":"11.262","BXD73a":"11.444","BXD73b":"x","BXD74":"10.974","BXD75":"11.150","BXD76":"10.920","BXD77":"10.928","BXD78":"x","BXD79":"11.371","BXD81":"x","BXD83":"10.946","BXD84":"11.181","BXD85":"10.992","BXD86":"10.770","BXD87":"11.200","BXD88":"x","BXD89":"10.930","BXD90":"11.183","BXD91":"x","BXD93":"11.056","BXD94":"10.737","BXD95":"x","BXD98":"10.986","BXD99":"10.892","BXD100":"x","BXD101":"x","BXD102":"x","BXD104":"x","BXD105":"x","BXD106":"x","BXD107":"x","BXD108":"x","BXD109":"x","BXD110":"x","BXD111":"x","BXD112":"x","BXD113":"x","BXD114":"x","BXD115":"x","BXD116":"x","BXD117":"x","BXD119":"x","BXD120":"x","BXD121":"x","BXD122":"x","BXD123":"x","BXD124":"x","BXD125":"x","BXD126":"x","BXD127":"x","BXD128":"x","BXD128a":"x","BXD130":"x","BXD131":"x","BXD132":"x","BXD133":"x","BXD134":"x","BXD135":"x","BXD136":"x","BXD137":"x","BXD138":"x","BXD139":"x","BXD141":"x","BXD142":"x","BXD144":"x","BXD145":"x","BXD146":"x","BXD147":"x","BXD148":"x","BXD149":"x","BXD150":"x","BXD151":"x","BXD152":"x","BXD153":"x","BXD154":"x","BXD155":"x","BXD156":"x","BXD157":"x","BXD160":"x","BXD161":"x","BXD162":"x","BXD165":"x","BXD168":"x","BXD169":"x","BXD170":"x","BXD171":"x","BXD172":"x","BXD173":"x","BXD174":"x","BXD175":"x","BXD176":"x","BXD177":"x","BXD178":"x","BXD180":"x","BXD181":"x","BXD183":"x","BXD184":"x","BXD186":"x","BXD187":"x","BXD188":"x","BXD189":"x","BXD190":"x","BXD191":"x","BXD192":"x","BXD193":"x","BXD194":"x","BXD195":"x","BXD196":"x","BXD197":"x","BXD198":"x","BXD199":"x","BXD200":"x","BXD201":"x","BXD202":"x","BXD203":"x","BXD204":"x","BXD205":"x","BXD206":"x","BXD207":"x","BXD208":"x","BXD209":"x","BXD210":"x","BXD211":"x","BXD212":"x","BXD213":"x","BXD214":"x","BXD215":"x","BXD216":"x","BXD217":"x","BXD218":"x","BXD219":"x","BXD220":"x"}' + +def do_request(url, data): + return requests.post( + url, + data={ + "dataset": "HC_M2_0606_P", + "trait_id": "1435464_at", + "corr_dataset": "HC_M2_0606_P", + "corr_sample_method": "pearson", + "corr_return_results": "100", + "corr_samples_group": "samples_primary", + "sample_vals": sample_vals(), + **data, + }) + +def check_sample_correlations(baseurl): + data = { + "corr_type": "sample", + "corr_sample_method": "pearson", + "location_type": "gene", + "corr_return_results": "200" + } + top_n_message = "The top 200 correlations ranked by the Genetic Correlation" + result = do_request(f"{baseurl}/corr_compute", data) + assert result.status_code == 200 + assert (result.text.find("Values of record 1435464_at") >= 0), result.text + assert (result.text.find(top_n_message) >= 0), result.text + +def check_tissue_correlations(baseurl): + data = { + "corr_type": "tissue", + "location_type": "gene", + } + top_n_message = "The top 100 correlations ranked by the Tissue Correlation" + result = do_request(f"{baseurl}/corr_compute", data) + assert result.status_code == 200 + assert (result.text.find("Values of record 1435464_at") >= 0), result.text + assert (result.text.find(top_n_message) >= 0), result.text + +def check_lit_correlations(baseurl): + data = { + "corr_type": "lit", + "corr_return_results": "200" + } + top_n_message = "The top 200 correlations ranked by the Literature Correlation" + result = do_request(f"{baseurl}/corr_compute", data) + assert result.status_code == 200 + assert (result.text.find("Values of record 1435464_at") >= 0), result.text + assert (result.text.find(top_n_message) >= 0), result.text + +def check_correlations(args_obj, parser): + print("") + print("Checking the correlations...") + corr_type_fns = { + "sample": check_sample_correlations, + "tissue": check_tissue_correlations, + "lit": check_lit_correlations + } + host = args_obj.host + for corr_type, corr_type_fn in corr_type_fns.items(): + print(f"\tChecking {corr_type} correlations...", end="") + corr_type_fn(host) + print(" ok") + print("OK") diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index feae6526..86222254 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -1,6 +1,6 @@ import re import requests -from lxml.html import parse +from lxml.html import document_fromstring from requests.exceptions import ConnectionError DO_FAIL=False # fail on error @@ -66,7 +66,8 @@ def verify_static_file(link): def check_page(host, start_url): print("") print("Checking links host "+host+" in page `"+start_url+"`") - doc = parse(start_url).getroot() + req = requests.get(start_url) + doc = document_fromstring(start_url) links = get_links(doc) in_page_links = list(filter(is_in_page_link, links)) internal_links = list(filter(is_internal_link, links)) diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py index 28033ad5..79f37bb4 100644 --- a/test/requests/main_web_functionality.py +++ b/test/requests/main_web_functionality.py @@ -1,10 +1,11 @@ import requests -from lxml.html import parse +from lxml.html import document_fromstring from link_checker import check_page def check_home(url): - doc = parse(url).getroot() + results = requests.get(url) + doc = document_fromstring(results.text) search_button = doc.cssselect("#btsearch") assert(search_button[0].value == "Search") print("OK") @@ -18,7 +19,7 @@ def check_search_page(host): search_terms_or="", search_terms_and="MEAN=(15 16) LRS=(23 46)") result = requests.get(host+"/search", params=data) - found = result.text.find("records were found") + found = result.text.find("records found") assert(found >= 0) assert(result.status_code == 200) print("OK") @@ -27,7 +28,8 @@ def check_search_page(host): def check_traits_page(host, traits_url): - doc = parse(host+traits_url).getroot() + results = requests.get(host+traits_url) + doc = document_fromstring(results.text) traits_form = doc.forms[1] assert(traits_form.fields["corr_dataset"] == "HC_M2_0606_P") print("OK") diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 71055fca..c0b36182 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -9,6 +9,7 @@ from link_checker import check_links from link_checker import check_packaged_js_files from mapping_tests import check_mapping from navigation_tests import check_navigation +from correlation_tests import check_correlations from main_web_functionality import check_main_web_functionality import link_checker import sys @@ -18,16 +19,30 @@ from unittest import TestSuite, TextTestRunner, TestLoader print("Mechanical Rob firing up...") +def host_is_online(host): + import time + import requests + for count in range(1, 5): + try: + time.sleep(count) + requests.get(host) + return True + except Exception as cre: + print(f"Retrying in {count + 1} seconds ...") + + return False def run_all(args_obj, parser): print("") print("Running all tests.") print(args_obj) + assert host_is_online(args_obj.host), f"Could not connect to {host}" link_checker.DO_FAIL = args_obj.fail check_main_web_functionality(args_obj, parser) check_links(args_obj, parser) check_packaged_js_files(args_obj, parser) check_mapping(args_obj, parser) + check_correlations(args_obj, parser) # TODO: Add other functions as they are created. diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index a7811a30..f66bf186 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -21,10 +21,7 @@ from dataclasses import dataclass from dataclasses import field from dataclasses import InitVar from typing import Optional, Dict, List -from db.call import fetchall, fetchone, fetch1 -from utility.logger import getLogger from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL -from db.gn_server import menu_main from pprint import pformat as pf from utility.db_tools import escape from utility.db_tools import mescape @@ -33,7 +30,6 @@ from maintenance import get_group_samplelists from utility.tools import locate, locate_ignore_error, flat_files from utility import gen_geno_ob from utility import chunks -from utility.benchmark import Bench from utility import webqtlUtil from db import webqtlDatabaseFunction from base import species @@ -42,6 +38,7 @@ from flask import Flask, g from base.webqtlConfig import TMPDIR from urllib.parse import urlparse from utility.tools import SQL_URI +from wqflask.database import database_connection import os import math import string @@ -60,8 +57,6 @@ from redis import Redis r = Redis() -logger = getLogger(__name__) - # Used by create_database to instantiate objects # Each subclass will add to this DS_NAME_MAP = {} @@ -200,26 +195,23 @@ def create_datasets_list(): result = r.get(key) if result: - logger.debug("Redis cache hit") datasets = pickle.loads(result) if result is None: datasets = list() - with Bench("Creating DataSets object"): - type_dict = {'Publish': 'PublishFreeze', - 'ProbeSet': 'ProbeSetFreeze', - 'Geno': 'GenoFreeze'} - - for dataset_type in type_dict: - query = "SELECT Name FROM {}".format(type_dict[dataset_type]) - for result in fetchall(query): - # The query at the beginning of this function isn't - # necessary here, but still would rather just reuse - # it logger.debug("type: {}\tname: - # {}".format(dataset_type, result.Name)) - dataset = create_dataset(result.Name, dataset_type) - datasets.append(dataset) - + type_dict = {'Publish': 'PublishFreeze', + 'ProbeSet': 'ProbeSetFreeze', + 'Geno': 'GenoFreeze'} + + for dataset_type in type_dict: + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute("SELECT Name FROM %s", + (type_dict[dataset_type],)) + results = cursor.fetchall(query) + if results: + for result in results: + datasets.append( + create_dataset(result.Name, dataset_type)) if USE_REDIS: r.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL)) r.expire(key, 60 * 60) @@ -259,9 +251,6 @@ class Markers: self.markers = markers def add_pvalues(self, p_values): - logger.debug("length of self.markers:", len(self.markers)) - logger.debug("length of p_values:", len(p_values)) - if isinstance(p_values, list): # THIS IS only needed for the case when we are limiting the number of p-values calculated # if len(self.markers) > len(p_values): @@ -332,12 +321,16 @@ class DatasetGroup: def __init__(self, dataset, name=None): """This sets self.group and self.group_id""" - if name == None: - self.name, self.id, self.genetic_type, self.code = fetchone( - dataset.query_for_group) - else: - self.name, self.id, self.genetic_type, self.code = fetchone( - "SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode FROM InbredSet where Name='%s'" % name) + query = """SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, + InbredSet.InbredSetCode FROM InbredSet where Name=%s + """ + if not name: + query, name = dataset.query_for_group, dataset.name + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute(query, (name,)) + results = cursor.fetchone() + if results: + self.name, self.id, self.genetic_type, self.code = results if self.name == 'BXD300': self.name = "BXD" @@ -633,39 +626,33 @@ class DataSet: """ try: + query = "" + _vars = None + query_args = (self.name, self.name, self.name) if self.type == "ProbeSet": - query_args = tuple(escape(x) for x in ( - self.name, - self.name, - self.name)) - - self.id, self.name, self.fullname, self.shortname, self.data_scale, self.tissue = fetch1(""" - SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, ProbeSetFreeze.DataScale, Tissue.Name - FROM ProbeSetFreeze, ProbeFreeze, Tissue - WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id - AND ProbeFreeze.TissueId = Tissue.Id - AND (ProbeSetFreeze.Name = '%s' OR ProbeSetFreeze.FullName = '%s' OR ProbeSetFreeze.ShortName = '%s') - """ % (query_args), "/dataset/" + self.name + ".json", - lambda r: (r["id"], r["name"], r["full_name"], - r["short_name"], r["data_scale"], r["tissue"]) - ) + query = ( + "SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, " + "ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, " + "ProbeSetFreeze.DataScale, Tissue.Name " + "FROM ProbeSetFreeze, ProbeFreeze, Tissue " + "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " + "AND ProbeFreeze.TissueId = Tissue.Id " + "AND (ProbeSetFreeze.Name = %s OR ProbeSetFreeze.FullName = %s OR ProbeSetFreeze.ShortName = %s)") else: - query_args = tuple(escape(x) for x in ( - (self.type + "Freeze"), - self.name, - self.name, - self.name)) - + query = ( + "SELECT Id, Name, FullName, ShortName " + f"FROM {self.type}Freeze " + "WHERE (Name = %s OR FullName = %s OR ShortName = %s)") self.tissue = "N/A" - self.id, self.name, self.fullname, self.shortname = fetchone(""" - SELECT Id, Name, FullName, ShortName - FROM %s - WHERE (Name = '%s' OR FullName = '%s' OR ShortName = '%s') - """ % (query_args)) - + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute(query, query_args) + _vars = cursor.fetchone() + if self.type == "ProbeSet": + (self.id, self.name, self.fullname, self.shortname, + self.data_scale, self.tissue) = _vars + else: + self.id, self.name, self.fullname, self.shortname = _vars except TypeError: - logger.debug( - "Dataset {} is not yet available in GeneNetwork.".format(self.name)) pass def chunk_dataset(self, dataset, n): @@ -863,16 +850,8 @@ class PhenotypeDataSet(DataSet): 'Additive Effect'] self.type = 'Publish' - - self.query_for_group = ''' - SELECT - InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode - FROM - InbredSet, PublishFreeze - WHERE - PublishFreeze.InbredSetId = InbredSet.Id AND - PublishFreeze.Name = "%s" - ''' % escape(self.name) + self.query_for_group = """ +SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode FROM InbredSet, PublishFreeze WHERE PublishFreeze.InbredSetId = InbredSet.Id AND PublishFreeze.Name = %s""" def check_confidentiality(self): # (Urgently?) Need to write this @@ -979,16 +958,10 @@ class GenotypeDataSet(DataSet): # Todo: Obsolete or rename this field self.type = 'Geno' - - self.query_for_group = ''' - SELECT - InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode - FROM - InbredSet, GenoFreeze - WHERE - GenoFreeze.InbredSetId = InbredSet.Id AND - GenoFreeze.Name = "%s" - ''' % escape(self.name) + self.query_for_group = """ +SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode +FROM InbredSet, GenoFreeze WHERE GenoFreeze.InbredSetId = InbredSet.Id AND +GenoFreeze.Name = %s""" def check_confidentiality(self): return geno_mrna_confidentiality(self) @@ -1022,6 +995,14 @@ class GenotypeDataSet(DataSet): results = g.db.execute(query, (webqtlDatabaseFunction.retrieve_species_id(self.group.name), trait, self.name)).fetchall() + + if self.group.name in webqtlUtil.ParInfo: + f1_1, f1_2, ref, nonref = webqtlUtil.ParInfo[self.group.name] + results.append([f1_1, 0, None, "N/A", f1_1]) + results.append([f1_2, 0, None, "N/A", f1_2]) + results.append([ref, -1, None, "N/A", ref]) + results.append([nonref, 1, None, "N/A", nonref]) + return results @@ -1079,17 +1060,10 @@ class MrnaAssayDataSet(DataSet): # Todo: Obsolete or rename this field self.type = 'ProbeSet' - - self.query_for_group = ''' - SELECT - InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode - FROM - InbredSet, ProbeSetFreeze, ProbeFreeze - WHERE - ProbeFreeze.InbredSetId = InbredSet.Id AND - ProbeFreeze.Id = ProbeSetFreeze.ProbeFreezeId AND - ProbeSetFreeze.Name = "%s" - ''' % escape(self.name) + self.query_for_group = """ +SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode +FROM InbredSet, ProbeSetFreeze, ProbeFreeze WHERE ProbeFreeze.InbredSetId = InbredSet.Id AND +ProbeFreeze.Id = ProbeSetFreeze.ProbeFreezeId AND ProbeSetFreeze.Name = %s""" def check_confidentiality(self): return geno_mrna_confidentiality(self) diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index 8f8e2b0a..d7e747aa 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -9,10 +9,6 @@ from utility.db_tools import escape from gn3.db_utils import database_connector -from utility.logger import getLogger -logger = getLogger(__name__) - - class MrnaAssayTissueData: def __init__(self, gene_symbols=None): diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py index 25b6cb8a..dd6fad04 100644 --- a/wqflask/base/webqtlCaseData.py +++ b/wqflask/base/webqtlCaseData.py @@ -21,9 +21,6 @@ # Created by GeneNetwork Core Team 2010/08/10 -from utility.logger import getLogger -logger = getLogger(__name__) - import utility.tools utility.tools.show_settings() diff --git a/wqflask/db/call.py b/wqflask/db/call.py deleted file mode 100644 index 1fe0772b..00000000 --- a/wqflask/db/call.py +++ /dev/null @@ -1,79 +0,0 @@ -# Module for calling the backend - -from flask import g - -import string -try: # Python2 support - import urllib.request - import urllib.error - import urllib.parse -except: - import urllib2 -import json -from utility.tools import USE_GN_SERVER, LOG_SQL, GN_SERVER_URL -from utility.benchmark import Bench - -from utility.logger import getLogger -logger = getLogger(__name__) - -# from inspect import stack - - -def fetch1(query, path=None, func=None): - """Fetch one result as a Tuple using either a SQL query or the URI -path to GN_SERVER (when USE_GN_SERVER is True). Apply func to -GN_SERVER result when set (which should return a Tuple) - - """ - if USE_GN_SERVER and path: - result = gn_server(path) - if func != None: - res2 = func(result) - else: - res2 = result, - if LOG_SQL: - logger.debug("Replaced SQL call", query) - logger.debug(path, res2) - return res2 - else: - return fetchone(query) - - -def fetchone(query): - """Return tuple containing one row by calling SQL directly (the -original fetchone, but with logging) - - """ - with Bench("SQL", LOG_SQL): - def helper(query): - res = g.db.execute(query) - return res.fetchone() - return logger.sql(query, helper) - - -def fetchall(query): - """Return row iterator by calling SQL directly (the -original fetchall, but with logging) - - """ - with Bench("SQL", LOG_SQL): - def helper(query): - res = g.db.execute(query) - return res.fetchall() - return logger.sql(query, helper) - - -def gn_server(path): - """Return JSON record by calling GN_SERVER - - """ - with Bench("GN_SERVER", LOG_SQL): - res = () - try: - res = urllib.request.urlopen(GN_SERVER_URL + path) - except: - res = urllib2.urlopen(GN_SERVER_URL + path) - rest = res.read() - res2 = json.loads(rest) - logger.debug(res2) - return res2 diff --git a/wqflask/db/gn_server.py b/wqflask/db/gn_server.py deleted file mode 100644 index f9b01658..00000000 --- a/wqflask/db/gn_server.py +++ /dev/null @@ -1,10 +0,0 @@ -# Backend query functions (logic) - -from db.call import gn_server - -from utility.logger import getLogger -logger = getLogger(__name__) - - -def menu_main(): - return gn_server("/int/menu/main.json") diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index 9ec650a4..122c546f 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -20,19 +20,23 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from db.call import fetch1 +from wqflask.database import database_connection def retrieve_species(group): """Get the species of a group (e.g. returns string "mouse" on "BXD" """ - result = fetch1("select Species.Name from Species, InbredSet where InbredSet.Name = '%s' and InbredSet.SpeciesId = Species.Id" % ( - group), "/cross/" + group + ".json", lambda r: (r["species"],))[0] + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute( + "SELECT Species.Name FROM Species, InbredSet WHERE InbredSet.Name = %s AND InbredSet.SpeciesId = Species.Id", + (group,)) + return cursor.fetchone()[0] return result def retrieve_species_id(group): - result = fetch1("select SpeciesId from InbredSet where Name = '%s'" % ( - group), "/cross/" + group + ".json", lambda r: (r["species_id"],))[0] - return result + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute("SELECT SpeciesId FROM InbredSet WHERE Name = %s", + (group,)) + return cursor.fetchone()[0] diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index 22d73ba3..0d9372ff 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -33,8 +33,6 @@ Redis = get_redis_conn() import urllib.parse from wqflask.database import database_connection -from utility.logger import getLogger -logger = getLogger(__name__) def parse_db_uri(): diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 8198b921..fee29be1 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -23,9 +23,7 @@ app_config() werkzeug_logger = logging.getLogger('werkzeug') if WEBSERVER_MODE == 'DEBUG': - from flask_debugtoolbar import DebugToolbarExtension app.debug = True - toolbar = DebugToolbarExtension(app) app.run(host='0.0.0.0', port=SERVER_PORT, debug=True, diff --git a/wqflask/scripts/__init__.py b/wqflask/scripts/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wqflask/scripts/__init__.py diff --git a/wqflask/scripts/profile_corrs.py b/wqflask/scripts/profile_corrs.py new file mode 100644 index 00000000..7c43b8c7 --- /dev/null +++ b/wqflask/scripts/profile_corrs.py @@ -0,0 +1,78 @@ +import io +import sys +import pstats +import cProfile + +import sqlalchemy +from flask import g, request + +from utility.startup_config import app_config + +from wqflask import app +from wqflask.user_session import UserSession +from wqflask.views import connect_db, shutdown_session +from wqflask.correlation.correlation_gn3_api import compute_correlation + +def sample_vals(): + return '{"C57BL/6J":"10.835","DBA/2J":"11.142","B6D2F1":"11.126","D2B6F1":"11.143","BXD1":"10.811","BXD2":"11.503","BXD5":"10.766","BXD6":"10.986","BXD8":"11.050","BXD9":"10.822","BXD11":"10.670","BXD12":"10.946","BXD13":"10.890","BXD14":"x","BXD15":"10.884","BXD16":"11.222","BXD18":"x","BXD19":"10.968","BXD20":"10.962","BXD21":"10.906","BXD22":"11.080","BXD23":"11.046","BXD24":"11.146","BXD24a":"x","BXD25":"x","BXD27":"11.078","BXD28":"11.034","BXD29":"10.808","BXD30":"x","BXD31":"11.087","BXD32":"11.029","BXD33":"10.662","BXD34":"11.482","BXD35":"x","BXD36":"x","BXD37":"x","BXD38":"10.836","BXD39":"10.926","BXD40":"10.638","BXD41":"x","BXD42":"10.974","BXD43":"10.828","BXD44":"10.900","BXD45":"11.358","BXD48":"11.042","BXD48a":"10.975","BXD49":"x","BXD50":"11.228","BXD51":"11.126","BXD52":"x","BXD53":"x","BXD54":"x","BXD55":"11.580","BXD56":"x","BXD59":"x","BXD60":"10.829","BXD61":"11.152","BXD62":"11.156","BXD63":"10.942","BXD64":"10.506","BXD65":"11.126","BXD65a":"11.272","BXD65b":"11.157","BXD66":"11.071","BXD67":"11.080","BXD68":"10.997","BXD69":"11.096","BXD70":"11.152","BXD71":"x","BXD72":"x","BXD73":"11.262","BXD73a":"11.444","BXD73b":"x","BXD74":"10.974","BXD75":"11.150","BXD76":"10.920","BXD77":"10.928","BXD78":"x","BXD79":"11.371","BXD81":"x","BXD83":"10.946","BXD84":"11.181","BXD85":"10.992","BXD86":"10.770","BXD87":"11.200","BXD88":"x","BXD89":"10.930","BXD90":"11.183","BXD91":"x","BXD93":"11.056","BXD94":"10.737","BXD95":"x","BXD98":"10.986","BXD99":"10.892","BXD100":"x","BXD101":"x","BXD102":"x","BXD104":"x","BXD105":"x","BXD106":"x","BXD107":"x","BXD108":"x","BXD109":"x","BXD110":"x","BXD111":"x","BXD112":"x","BXD113":"x","BXD114":"x","BXD115":"x","BXD116":"x","BXD117":"x","BXD119":"x","BXD120":"x","BXD121":"x","BXD122":"x","BXD123":"x","BXD124":"x","BXD125":"x","BXD126":"x","BXD127":"x","BXD128":"x","BXD128a":"x","BXD130":"x","BXD131":"x","BXD132":"x","BXD133":"x","BXD134":"x","BXD135":"x","BXD136":"x","BXD137":"x","BXD138":"x","BXD139":"x","BXD141":"x","BXD142":"x","BXD144":"x","BXD145":"x","BXD146":"x","BXD147":"x","BXD148":"x","BXD149":"x","BXD150":"x","BXD151":"x","BXD152":"x","BXD153":"x","BXD154":"x","BXD155":"x","BXD156":"x","BXD157":"x","BXD160":"x","BXD161":"x","BXD162":"x","BXD165":"x","BXD168":"x","BXD169":"x","BXD170":"x","BXD171":"x","BXD172":"x","BXD173":"x","BXD174":"x","BXD175":"x","BXD176":"x","BXD177":"x","BXD178":"x","BXD180":"x","BXD181":"x","BXD183":"x","BXD184":"x","BXD186":"x","BXD187":"x","BXD188":"x","BXD189":"x","BXD190":"x","BXD191":"x","BXD192":"x","BXD193":"x","BXD194":"x","BXD195":"x","BXD196":"x","BXD197":"x","BXD198":"x","BXD199":"x","BXD200":"x","BXD201":"x","BXD202":"x","BXD203":"x","BXD204":"x","BXD205":"x","BXD206":"x","BXD207":"x","BXD208":"x","BXD209":"x","BXD210":"x","BXD211":"x","BXD212":"x","BXD213":"x","BXD214":"x","BXD215":"x","BXD216":"x","BXD217":"x","BXD218":"x","BXD219":"x","BXD220":"x"}' + +def simulated_form(corr_type: str = "sample"): + assert corr_type in ("sample", "tissue", "lit") + return { + "dataset": "HC_M2_0606_P", + "trait_id": "1435464_at", + "corr_dataset": "HC_M2_0606_P", + "corr_sample_method": "pearson", + "corr_return_results": "100", + "corr_samples_group": "samples_primary", + "sample_vals": sample_vals(), + "corr_type": corr_type, + "corr_sample_method": "pearson", + "location_type": "gene", + "corr_return_results": "20000" + } + +def profile_corrs(): + "Profile tho correlations" + profiler = cProfile.Profile() + # compute_correlation(simulated_form(), compute_all=True) + profiler.enable() + correlation_results = compute_correlation(request.form, compute_all=True) + # correlation_results = set_template_vars(request.form, correlation_results) + profiler.disable() + + return profiler + +def dump_stats(profiler): + iostr = io.StringIO() + sort_by = pstats.SortKey.CUMULATIVE + ps = pstats.Stats(profiler, stream=iostr).sort_stats(sort_by) + ps.print_stats() + + cli_args = sys.argv + if len(cli_args) > 1: + with open(cli_args[1], "w+", encoding="utf-8") as output_file: + print(iostr.getvalue(), file=output_file) + + return 0 + + print(iostr.getvalue()) + return 0 + + +if __name__ == "__main__": + def main(): + "Entry point for profiler script" + return dump_stats(profile_corrs()) + + app_config() + with app.app_context(): + with app.test_request_context("/corr_compute", data=simulated_form()): + g.db = sqlalchemy.create_engine( ## Setup global db connection + app.config.get('SQL_URI'), encoding="latin1") + g.user_session = UserSession() + # g.user_session.user_id = 0 + main() + ## dispose global db connection + g.db.dispose() + g.db.close() diff --git a/wqflask/tests/unit/wqflask/api/test_mapping.py b/wqflask/tests/unit/wqflask/api/test_mapping.py index 159c982b..006e9826 100644 --- a/wqflask/tests/unit/wqflask/api/test_mapping.py +++ b/wqflask/tests/unit/wqflask/api/test_mapping.py @@ -25,7 +25,9 @@ class TestMapping(unittest.TestCase): "maf": 0.01, "use_loco": True, "num_perm": 0, - "perm_check": False + "perm_check": False, + "transform": False, + "genofile": False } results = initialize_parameters( @@ -40,8 +42,9 @@ class TestMapping(unittest.TestCase): "pair_scan": "true", "interval_mapping": "true", "use_loco": "true", - "num_perm": "14" - + "num_perm": "14", + "transform": "qnorm", + "genofile": "BXD.8.geno" } results_2 = initialize_parameters( @@ -53,7 +56,9 @@ class TestMapping(unittest.TestCase): "maf": 0.01, "use_loco": True, "num_perm": 14, - "perm_check": "ON" + "perm_check": "ON", + "transform": "qnorm", + "genofile": "BXD.8.geno" } self.assertEqual(results_2, expected_results) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py index 611c0d70..53d96d7f 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py @@ -165,12 +165,12 @@ X\tM5\t12\tQ\tE\tMMB\tR\t21.1\tW\t0.65\t0.6""" this_dataset={}, gwa_output_filename=".xw/") expected_results = [ {'name': 'M1', 'chr': 'X/Y', 'Mb': 2.8457155e-05, 'p_value': 0.85, - 'additive': -23.3, 'lod_score': 0.07058107428570727}, + 'additive': -11.65, 'lod_score': 0.07058107428570727}, {'name': 'M2', 'chr': 4, 'Mb': 1.2e-05, 'p_value': 0.5, - 'additive': -24.0, 'lod_score': 0.3010299956639812}, + 'additive': -12.0, 'lod_score': 0.3010299956639812}, {'name': 'M4', 'chr': 'Y', 'Mb': 1.2e-05, 'p_value': 0.7, - 'additive': -11.6, 'lod_score': 0.1549019599857432}, - {'name': 'M5', 'chr': 'X', 'Mb': 1.2e-05, 'p_value': 0.6, 'additive': -21.1, 'lod_score': 0.22184874961635637}] + 'additive': -5.8, 'lod_score': 0.1549019599857432}, + {'name': 'M5', 'chr': 'X', 'Mb': 1.2e-05, 'p_value': 0.6, 'additive': -10.55, 'lod_score': 0.22184874961635637}] self.assertEqual(expected_results, results) @mock.patch("wqflask.marker_regression.gemma_mapping.TEMPDIR", "/home/tmp") diff --git a/wqflask/tests/unit/wqflask/show_trait/test_export_trait_data.py b/wqflask/tests/unit/wqflask/show_trait/test_export_trait_data.py index 41761944..fbe39f13 100644 --- a/wqflask/tests/unit/wqflask/show_trait/test_export_trait_data.py +++ b/wqflask/tests/unit/wqflask/show_trait/test_export_trait_data.py @@ -1,3 +1,4 @@ +import datetime import unittest from unittest import mock from wqflask.show_trait.export_trait_data import dict_to_sorted_list @@ -13,116 +14,53 @@ class AttributesSetter: class TestExportTraits(unittest.TestCase): - """Test methods related to converting dict to sortedlist""" - @mock.patch("wqflask.show_trait.export_trait_data.create_trait") - @mock.patch("wqflask.show_trait.export_trait_data.data_set") - def test_get_export_metadata_no_publish(self, mock_dataset, mock_trait): - """test for exporting metadata with no publish""" - mock_dataset_attributes = AttributesSetter( - {"type": "no_publish", "dataset_name": "Temp", "name": "Temp"}) - - mock_nested_attributes = AttributesSetter({"name": "name"}) - mock_dataset_attributes.group = mock_nested_attributes - mock_dataset.create_dataset.return_value = mock_dataset_attributes - mock_trait.return_value = AttributesSetter({"symbol": "", "description_display": "Description", - "title": "research1", "journal": "", "authors": ""}) - - results = get_export_metadata("random_id", "Temp") - expected = [["Record ID: random_id"], - ["Trait URL: http://genenetwork.org/show_trait?trait_id=random_id&dataset=Temp"], - ["Dataset: Temp"], - ["Group: name"], []] - - mock_dataset.create_dataset.assert_called_with("Temp") - mock_trait.assert_called_with( - dataset=mock_dataset_attributes, name="random_id", cellid=None, get_qtl_info=False) - self.assertEqual(results, expected) + """Test methods for exporting traits and metadata""" + @mock.patch("wqflask.show_trait.export_trait_data.datetime") @mock.patch("wqflask.show_trait.export_trait_data.create_trait") @mock.patch("wqflask.show_trait.export_trait_data.data_set") - def test_get_export_metadata_with_publish(self, data_mock, trait_mock): + def test_get_export_metadata(self, data_mock, trait_mock, date_mock): """test for exporting metadata with dataset.type=Publish""" - mock_dataset_attributes = AttributesSetter({"type": "Publish", "dataset_name": "Temp", - "name": "Temp", "description_display": "Description goes here"}) - - mock_nested_attributes = AttributesSetter({"name": "name"}) - mock_dataset_attributes.group = mock_nested_attributes - data_mock.create_dataset.return_value = mock_dataset_attributes - trait_instance = AttributesSetter({"symbol": "", "description_display": "Description", - "title": "research1", "journal": "", "authors": ""}) - trait_mock.return_value = trait_instance + mock_dataset = AttributesSetter({"type": "Publish", + "name": "HC_M2_0606_P", + "dataset_name": "HC_M2_0606_P"}) - results = get_export_metadata( - "29ae0615-0d77-4814-97c7-c9e91f6bfd7b", "Temp") + mock_dataset.group = AttributesSetter({"name": "C"}) + data_mock.create_dataset.return_value = mock_dataset - expected = [['Phenotype ID: 29ae0615-0d77-4814-97c7-c9e91f6bfd7b'], - ['Phenotype URL: http://genenetwork.org/show_trait?trait_id=29ae0615-0d77-4814-97c7-c9e91f6bfd7b&dataset=Temp'], [ - 'Group: name'], ['Phenotype: Description'], - ['Authors: N/A'], ['Title: research1'], - ['Journal: N/A'], ['Dataset Link: http://gn1.genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=Temp'], []] + trait_data = { + "symbol": "Nr3c1", + "description_display": "nuclear receptor subfamily 3,group C, member 1 (glucocorticoid receptor); distal 3' UTR", + "title": "Trait_1 title", - self.assertEqual(results, expected) + "authors": "XL_1", + "journal": "" - @mock.patch("wqflask.show_trait.export_trait_data.dict_to_sorted_list") - @mock.patch("wqflask.show_trait.export_trait_data.get_export_metadata") - def test_export_sample_table(self, exp_metadata, dict_list): - """test for exporting sample table""" - targs_obj = { - "export_data": """{ - "primary_samples": [ - { - "other": "germanotta", - "name": "Sauroniops", - "se":{ - "name":"S2" - }, - "num_cases":{ - "k1":"value" - - } - } - ], - "other_samples": [ - { - "se": 1, - "num_cases": 4, - "value": 6, - "name": 3 - } - ] - }""", - "trait_display_name": "Hair_color", - "trait_id": "23177fdc-312e-4084-ad0c-f3eae785fff5", - "dataset": { - } } - exp_metadata.return_value = [ - ["Phenotype ID:0a2be192-57f5-400b-bbbd-0cf50135995f"], ['Group:gp1'], - ["Phenotype:p1"], [ - "Authors:N/A"], - ["Title:research1"], - ["Journal:N/A"], - ["Dataset Link: http://gn1.genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=name1"], []] - expected = ('Hair_color', - [['Phenotype ID:0a2be192-57f5-400b-bbbd-0cf50135995f'], - ['Group:gp1'], - ['Phenotype:p1'], - ['Authors:N/A'], - ['Title:research1'], - ['Journal:N/A'], - ['Dataset Link: ' - 'http://gn1.genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=name1'], - [], - ['Name', 'Value', 'SE', 'N'], - ['Sauroniops', 'germanotta'], - [3, 6, 1, 4]]) - - dict_list.side_effect = [['Sauroniops', 'germanotta'], [3, 6, 1, 4]] - - self.assertEqual(export_sample_table(targs_obj), expected) - exp_metadata.assert_called_with( - "23177fdc-312e-4084-ad0c-f3eae785fff5", {}) - self.assertEqual(dict_list.call_count, 2) + + date_mock.datetime.now.return_value = datetime.datetime( + 2022, 8, 8, 19, 2, 31, 628813) + trait_mock.return_value = AttributesSetter(trait_data) + + results = get_export_metadata({ + "trait_id": "1460303_at", + "trait_display_name": "1460303_at", + "dataset": "HC_M2_0606_P" + }) + + expected = [["Phenotype ID:", "1460303_at"], + ["Phenotype URL: ", "http://genenetwork.org/show_trait?trait_id=1460303_at&dataset=HC_M2_0606_P"], + ["Group: ", "C"], + ["Phenotype: ", + 'nuclear receptor subfamily 3","group C"," member 1 (glucocorticoid receptor); distal 3\' UTR'], + ["Authors: ", "XL_1"], + ["Title: ", "Trait_1 title"], + ["Journal: ", "N/A"], + ["Dataset Link: ", "http://gn1.genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=HC_M2_0606_P"], + ["Export Date: ", "August 08, 2022"], + ["Export Time: ", "19:02 GMT"]] + + self.assertEqual(results, expected) def test_dict_to_sortedlist(self): """test for conversion of dict to sorted list""" diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index d4256a46..df7156b4 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -33,8 +33,7 @@ from math import * import utility.corestats as corestats from base import webqtlConfig from utility.pillow_utils import draw_rotated_text -import utility.logger -logger = utility.logger.getLogger(__name__) + # ---- Define common colours ---- # BLUE = ImageColor.getrgb("blue") @@ -105,7 +104,6 @@ def find_outliers(vals): """ if vals: - #logger.debug("vals is:", pf(vals)) stats = corestats.Stats(vals) low_hinge = stats.percentile(25) up_hinge = stats.percentile(75) diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py deleted file mode 100644 index 6ece2f21..00000000 --- a/wqflask/utility/benchmark.py +++ /dev/null @@ -1,52 +0,0 @@ -import collections -import inspect -import time -from utility.tools import LOG_BENCH - -from utility.logger import getLogger -logger = getLogger(__name__) - - -class Bench: - entries = collections.OrderedDict() - - def __init__(self, name=None, write_output=LOG_BENCH): - self.name = name - self.write_output = write_output - - def __enter__(self): - if self.write_output: - if self.name: - logger.debug("Starting benchmark: %s" % (self.name)) - else: - logger.debug("Starting benchmark at: %s [%i]" % ( - inspect.stack()[1][3], inspect.stack()[1][2])) - self.start_time = time.time() - - def __exit__(self, type, value, traceback): - time_taken = time.time() - self.start_time - if self.write_output: - if self.name: - name = self.name - else: - name = "That" - - logger.info(" %s took: %f seconds" % (name, (time_taken))) - - if self.name: - Bench.entries[self.name] = Bench.entries.get( - self.name, 0) + time_taken - - @classmethod - def report(cls): - total_time = sum( - (time_taken for time_taken in list(cls.entries.values()))) - print("\nTiming report\n") - for name, time_taken in list(cls.entries.items()): - percent = int(round((time_taken / total_time) * 100)) - print("[{}%] {}: {}".format(percent, name, time_taken)) - print() - - def reset(cls): - """Reset the entries""" - cls.entries = collections.OrderedDict() diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index e619b7b6..c7a1ea59 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -1,7 +1,3 @@ -import utility.logger -logger = utility.logger.getLogger(__name__) - - class genotype: """ Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 27dd0729..4229a91f 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -6,9 +6,6 @@ from utility import hmac from flask import g -import logging -logger = logging.getLogger(__name__) - def get_species_dataset_trait(self, start_vars): if "temp_trait" in list(start_vars.keys()): diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py index d706e32a..65f2f1ee 100644 --- a/wqflask/utility/logger.py +++ b/wqflask/utility/logger.py @@ -27,7 +27,6 @@ # We'll add more overrides soon. import logging -import string from inspect import isfunction from pprint import pformat as pf from inspect import stack diff --git a/wqflask/utility/pillow_utils.py b/wqflask/utility/pillow_utils.py index 5713e155..e302df18 100644 --- a/wqflask/utility/pillow_utils.py +++ b/wqflask/utility/pillow_utils.py @@ -2,9 +2,6 @@ from PIL import Image, ImageColor, ImageDraw, ImageFont from utility.tools import TEMPDIR -import utility.logger -logger = utility.logger.getLogger(__name__) - BLACK = ImageColor.getrgb("black") WHITE = ImageColor.getrgb("white") diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index 641d973e..945efbbd 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -5,8 +5,6 @@ import datetime import redis # used for collections from utility.hmac import hmac_creation -from utility.logger import getLogger -logger = getLogger(__name__) def get_redis_conn(): diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 778fb64d..59923fa1 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -23,12 +23,9 @@ def app_config(): if mode in ["DEV", "DEBUG"]: app.config['TEMPLATES_AUTO_RELOAD'] = True if mode == "DEBUG": - from flask_debugtoolbar import DebugToolbarExtension app.debug = True - toolbar = DebugToolbarExtension(app) print("==========================================") - show_settings() port = get_setting_int("SERVER_PORT") diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index db0b4320..19bf686b 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -50,7 +50,7 @@ def get_setting(command_id, guess=None): app_set(command_id, command) return command else: - return None + return app.config.get(command_id) # ---- Check whether environment exists # print("Looking for "+command_id+"\n") diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index 5d239343..45d5739e 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -1,8 +1,5 @@ from gn3.db.species import get_all_species -import utility.logger -logger = utility.logger.getLogger(__name__) - def gen_dropdown_json(conn): """Generates and outputs (as json file) the data for the main dropdown menus on the home page @@ -31,7 +28,6 @@ def get_groups(species, conn): "InbredSet.FullName) ASC, IFNULL(InbredSet.Family, " "InbredSet.FullName) ASC, InbredSet.FullName ASC, " "InbredSet.MenuOrderId ASC").format(species_name) - # logger.debug(query) cursor.execute(query) results = cursor.fetchall() for result in results: diff --git a/wqflask/wqflask/api/mapping.py b/wqflask/wqflask/api/mapping.py index d1cc368e..5eacc83a 100644 --- a/wqflask/wqflask/api/mapping.py +++ b/wqflask/wqflask/api/mapping.py @@ -1,19 +1,12 @@ -import string - from base import data_set -from base import webqtlConfig from base.trait import create_trait, retrieve_sample_data -from utility import helper_functions -from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping - -import utility.logger -logger = utility.logger.getLogger(__name__) - +from wqflask.marker_regression import gemma_mapping, rqtl_mapping +from wqflask.show_trait.show_trait import normf def do_mapping_for_api(start_vars): - assert('db' in start_vars) - assert('trait_id' in start_vars) + if ('db' not in start_vars) or ("trait_id" not in start_vars): + raise ValueError("Mapping: db and trait_id are not in start_vars") dataset = data_set.create_dataset(dataset_name=start_vars['db']) dataset.group.get_markers() @@ -23,33 +16,67 @@ def do_mapping_for_api(start_vars): samples = [] vals = [] - for sample in dataset.group.samplelist: - in_trait_data = False - for item in this_trait.data: - if this_trait.data[item].name == sample: - value = str(this_trait.data[item].value) - samples.append(item) - vals.append(value) - in_trait_data = True - break - if not in_trait_data: - vals.append("x") - mapping_params = initialize_parameters(start_vars, dataset, this_trait) - # ZS: It seems to take an empty string as default. This should probably be changed. + genofile_samplelist = [] + if mapping_params.get('genofile'): + dataset.group.genofile = mapping_params['genofile'] + genofile_samplelist = get_genofile_samplelist(dataset) + + if (len(genofile_samplelist) > 0): + samplelist = genofile_samplelist + for sample in samplelist: + in_trait_data = False + for item in this_trait.data: + if this_trait.data[item].name == sample: + value = str(this_trait.data[item].value) + samples.append(item) + vals.append(value) + in_trait_data = True + break + if not in_trait_data: + vals.append("x") + else: + samplelist = dataset.group.samplelist + for sample in samplelist: + in_trait_data = False + for item in this_trait.data: + if this_trait.data[item].name == sample: + value = str(this_trait.data[item].value) + samples.append(item) + vals.append(value) + in_trait_data = True + break + if not in_trait_data: + vals.append("x") + + if mapping_params.get('transform') == "qnorm": + vals_minus_x = [float(val) for val in vals if val != "x"] + qnorm_vals = normf(vals_minus_x) + qnorm_vals_with_x = [] + counter = 0 + for val in vals: + if val == "x": + qnorm_vals_with_x.append("x") + else: + qnorm_vals_with_x.append(qnorm_vals[counter]) + counter += 1 + + vals = qnorm_vals_with_x + + # It seems to take an empty string as default. This should probably be changed. covariates = "" - if mapping_params['mapping_method'] == "gemma": + if mapping_params.get('mapping_method') == "gemma": header_row = ["name", "chr", "Mb", "lod_score", "p_value"] - # ZS: gemma_mapping returns both results and the filename for LOCO, so need to only grab the former for api - if mapping_params['use_loco'] == "True": + # gemma_mapping returns both results and the filename for LOCO, so need to only grab the former for api + if mapping_params.get('use_loco') == "True": result_markers = gemma_mapping.run_gemma( this_trait, dataset, samples, vals, covariates, mapping_params['use_loco'], mapping_params['maf'])[0] else: result_markers = gemma_mapping.run_gemma( this_trait, dataset, samples, vals, covariates, mapping_params['use_loco'], mapping_params['maf']) - elif mapping_params['mapping_method'] == "rqtl": + elif mapping_params.get('mapping_method') == "rqtl": header_row = ["name", "chr", "cM", "lod_score"] if mapping_params['num_perm'] > 0: _sperm_output, _suggestive, _significant, result_markers = rqtl_mapping.run_rqtl(this_trait.name, vals, samples, dataset, None, "Mb", mapping_params['rqtl_model'], @@ -62,10 +89,10 @@ def do_mapping_for_api(start_vars): mapping_params['do_control'], mapping_params['control_marker'], mapping_params['manhattan_plot'], None) - if mapping_params['limit_to']: + if mapping_params.get('limit_to'): result_markers = result_markers[:mapping_params['limit_to']] - if mapping_params['format'] == "csv": + if mapping_params.get('format') == "csv": output_rows = [] output_rows.append(header_row) for marker in result_markers: @@ -138,4 +165,22 @@ def initialize_parameters(start_vars, dataset, this_trait): except: mapping_params['perm_check'] = False + mapping_params['transform'] = False + if 'transform' in start_vars: + mapping_params['transform'] = start_vars['transform'] + + mapping_params['genofile'] = False + if 'genofile' in start_vars: + mapping_params['genofile'] = start_vars['genofile'] + return mapping_params + +def get_genofile_samplelist(dataset): + genofile_samplelist = [] + + genofile_json = dataset.group.get_genofiles() + for genofile in genofile_json: + if genofile['location'] == dataset.group.genofile and 'sample_list' in genofile: + genofile_samplelist = genofile['sample_list'] + + return genofile_samplelist diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index 3d33cc87..95cd2953 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -25,8 +25,6 @@ from utility.tools import flat_files from wqflask.database import database_connection -import utility.logger -logger = utility.logger.getLogger(__name__) version = "pre1" @@ -275,8 +273,6 @@ def get_dataset_info(dataset_name, group_name=None, file_format="json"): InbredSet.Name = "{0}" AND PublishXRef.Id = "{1}" """.format(group_name, dataset_name) - logger.debug("QUERY:", pheno_query) - pheno_results = g.db.execute(pheno_query) dataset = pheno_results.fetchone() diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index b46e1859..09ab2b60 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -21,9 +21,7 @@ from base.trait import retrieve_trait_info from base.trait import jsonable from base.data_set import create_dataset -from utility.logger import getLogger -logger = getLogger(__name__) Redis = get_redis_conn() @@ -48,8 +46,6 @@ def report_change(len_before, len_now): if new_length: flash("We've added {} to your collection.".format( numify(new_length, 'new trait', 'new traits'))) - else: - logger.debug("No new traits were added.") @app.route("/collections/store_trait_list", methods=('POST',)) diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index cafb9265..5df28c45 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -8,9 +8,6 @@ from utility import corr_result_helpers from scipy import stats import numpy as np -import utility.logger -logger = utility.logger.getLogger(__name__) - class CorrScatterPlot: """Page that displays a correlation scatterplot with a line fitted to it""" diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index caa9b619..1a375501 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -11,12 +11,12 @@ from base import data_set from base.trait import create_trait from base.trait import retrieve_sample_data -from gn3.computations.correlations import compute_all_sample_correlation -from gn3.computations.correlations import fast_compute_all_sample_correlation +from gn3.db_utils import database_connector +from gn3.commands import run_sample_corr_cmd from gn3.computations.correlations import map_shared_keys_to_values from gn3.computations.correlations import compute_all_lit_correlation from gn3.computations.correlations import compute_tissue_correlation -from gn3.db_utils import database_connector +from gn3.computations.correlations import fast_compute_all_sample_correlation def create_target_this_trait(start_vars): @@ -108,9 +108,9 @@ def sample_for_trait_lists(corr_results, target_dataset, (this_trait_data, target_dataset) = fetch_sample_data( start_vars, this_trait, this_dataset, target_dataset) - correlation_results = compute_all_sample_correlation(corr_method="pearson", - this_trait=this_trait_data, - target_dataset=target_dataset) + correlation_results = run_sample_corr_cmd( + corr_method="pearson", this_trait=this_trait_data, + target_dataset=target_dataset) return correlation_results @@ -194,78 +194,13 @@ def compute_correlation(start_vars, method="pearson", compute_all=False): method -- Correlation method to be used (pearson, spearman, or bicor) compute_all -- Include sample, tissue, and literature correlations (when applicable) """ - # pylint: disable-msg=too-many-locals + from wqflask.correlation.rust_correlation import compute_correlation_rust corr_type = start_vars['corr_type'] - - (this_dataset, this_trait, target_dataset, - sample_data) = create_target_this_trait(start_vars) - - target_dataset_type = target_dataset.type - this_dataset_type = this_dataset.type - method = start_vars['corr_sample_method'] corr_return_results = int(start_vars.get("corr_return_results", 100)) - corr_input_data = {} - - if corr_type == "sample": - (this_trait_data, target_dataset_data) = fetch_sample_data( - start_vars, this_trait, this_dataset, target_dataset) - - correlation_results = compute_all_sample_correlation( - corr_method=method, this_trait=this_trait_data, target_dataset=target_dataset_data) - - elif corr_type == "tissue": - trait_symbol_dict = this_dataset.retrieve_genes("Symbol") - tissue_input = get_tissue_correlation_input( - this_trait, trait_symbol_dict) - - if tissue_input is not None: - (primary_tissue_data, target_tissue_data) = tissue_input - - corr_input_data = { - "primary_tissue": primary_tissue_data, - "target_tissues_dict": target_tissue_data - } - correlation_results = compute_tissue_correlation( - primary_tissue_dict=corr_input_data["primary_tissue"], - target_tissues_data=corr_input_data[ - "target_tissues_dict"], - corr_method=method - - ) - else: - return {"correlation_results": [], - "this_trait": this_trait.name, - "target_dataset": start_vars['corr_dataset'], - "return_results": corr_return_results} - - elif corr_type == "lit": - (this_trait_geneid, geneid_dict, species) = do_lit_correlation( - this_trait, this_dataset) - - conn = database_connector() - with conn: - correlation_results = compute_all_lit_correlation( - conn=conn, trait_lists=list(geneid_dict.items()), - species=species, gene_id=this_trait_geneid) - - correlation_results = correlation_results[0:corr_return_results] - - if (compute_all): - correlation_results = compute_corr_for_top_results(start_vars, - correlation_results, - this_trait, - this_dataset, - target_dataset, - corr_type) - - correlation_data = {"correlation_results": correlation_results, - "this_trait": this_trait.name, - "target_dataset": start_vars['corr_dataset'], - "return_results": corr_return_results} - - return correlation_data + return compute_correlation_rust( + start_vars, corr_type, method, corr_return_results, compute_all) def compute_corr_for_top_results(start_vars, @@ -275,12 +210,10 @@ def compute_corr_for_top_results(start_vars, target_dataset, corr_type): if corr_type != "tissue" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": - tissue_result = tissue_for_trait_lists( correlation_results, this_dataset, this_trait) if tissue_result: - correlation_results = merge_correlation_results( correlation_results, tissue_result) diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py new file mode 100644 index 00000000..cfa360d0 --- /dev/null +++ b/wqflask/wqflask/correlation/rust_correlation.py @@ -0,0 +1,329 @@ +"""module contains integration code for rust-gn3""" +import json +from functools import reduce +from utility.db_tools import mescape +from utility.db_tools import create_in_clause +from wqflask.correlation.correlation_functions\ + import get_trait_symbol_and_tissue_values +from wqflask.correlation.correlation_gn3_api import create_target_this_trait +from wqflask.correlation.correlation_gn3_api import lit_for_trait_list +from wqflask.correlation.correlation_gn3_api import do_lit_correlation +from gn3.computations.correlations import compute_all_lit_correlation +from gn3.computations.rust_correlation import run_correlation +from gn3.computations.rust_correlation import get_sample_corr_data +from gn3.computations.rust_correlation import parse_tissue_corr_data +from gn3.db_utils import database_connector + + +def query_probes_metadata(dataset, trait_list): + """query traits metadata in bulk for probeset""" + + with database_connector() as conn: + with conn.cursor() as cursor: + + query = """ + SELECT ProbeSet.Name,ProbeSet.Chr,ProbeSet.Mb, + ProbeSet.Symbol,ProbeSetXRef.mean,ProbeSet.description, + ProbeSetXRef.additive,ProbeSetXRef.LRS,Geno.Chr, Geno.Mb + FROM ProbeSet INNER JOIN ProbeSetXRef + ON ProbeSet.Id=ProbeSetXRef.ProbeSetId + INNER JOIN Geno + ON ProbeSetXRef.Locus = Geno.Name + INNER JOIN Species + ON Geno.SpeciesId = Species.Id + WHERE ProbeSet.Name in ({}) AND + Species.Name = %s AND + ProbeSetXRef.ProbeSetFreezeId IN ( + SELECT ProbeSetFreeze.Id + FROM ProbeSetFreeze WHERE ProbeSetFreeze.Name = %s) + """.format(", ".join(["%s"] * len(trait_list))) + + cursor.execute(query, + (tuple(trait_list) + + (dataset.group.species,) + (dataset.name,)) + ) + + return cursor.fetchall() + + +def get_metadata(dataset, traits): + """Retrieve the metadata""" + def __location__(probe_chr, probe_mb): + if probe_mb: + return f"Chr{probe_chr}: {probe_mb:.6f}" + return f"Chr{probe_chr}: ???" + + return {trait_name: { + "name": trait_name, + "view": True, + "symbol": symbol, + "dataset": dataset.name, + "dataset_name": dataset.shortname, + "mean": mean, + "description": description, + "additive": additive, + "lrs_score": f"{lrs:3.1f}", + "location": __location__(probe_chr, probe_mb), + "lrs_location": f"Chr{chr_score}: {mb:.6f}" + + } for trait_name, probe_chr, probe_mb, symbol, mean, description, + additive, lrs, chr_score, mb + in query_probes_metadata(dataset, traits)} + + +def chunk_dataset(dataset, steps, name): + + results = [] + + query = """ + SELECT ProbeSetXRef.DataId,ProbeSet.Name + FROM ProbeSet, ProbeSetXRef, ProbeSetFreeze + WHERE ProbeSetFreeze.Name = '{}' AND + ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND + ProbeSetXRef.ProbeSetId = ProbeSet.Id + """.format(name) + + with database_connector() as conn: + with conn.cursor() as curr: + curr.execute(query) + traits_name_dict = dict(curr.fetchall()) + + for i in range(0, len(dataset), steps): + matrix = list(dataset[i:i + steps]) + trait_name = traits_name_dict[matrix[0][0]] + + strains = [trait_name] + [str(value) + for (trait_name, strain, value) in matrix] + results.append(",".join(strains)) + + return results + + +def compute_top_n_sample(start_vars, dataset, trait_list): + """check if dataset is of type probeset""" + + if dataset.type.lower() != "probeset": + return {} + + def __fetch_sample_ids__(samples_vals, samples_group): + all_samples = json.loads(samples_vals) + sample_data = get_sample_corr_data( + sample_type=samples_group, all_samples=all_samples, + dataset_samples=dataset.group.all_samples_ordered()) + + with database_connector() as conn: + with conn.cursor() as curr: + curr.execute( + """ + SELECT Strain.Name, Strain.Id FROM Strain, Species + WHERE Strain.Name IN {} + and Strain.SpeciesId=Species.Id + and Species.name = '{}' + """.format(create_in_clause(list(sample_data.keys())), + *mescape(dataset.group.species))) + return (sample_data, dict(curr.fetchall())) + + (sample_data, sample_ids) = __fetch_sample_ids__( + start_vars["sample_vals"], start_vars["corr_samples_group"]) + + with database_connector() as conn: + with conn.cursor() as curr: + # fetching strain data in bulk + curr.execute( + """ + SELECT * from ProbeSetData + where StrainID in {} + and id in (SELECT ProbeSetXRef.DataId + FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) + WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id + and ProbeSetFreeze.Name = '{}' + and ProbeSet.Name in {} + and ProbeSet.Id = ProbeSetXRef.ProbeSetId) + """.format( + create_in_clause(list(sample_ids.values())), + dataset.name, + create_in_clause(trait_list))) + + corr_data = chunk_dataset( + list(curr.fetchall()), len(sample_ids.values()), dataset.name) + + return run_correlation( + corr_data, list(sample_data.values()), "pearson", ",") + + +def compute_top_n_lit(corr_results, this_dataset, this_trait) -> dict: + (this_trait_geneid, geneid_dict, species) = do_lit_correlation( + this_trait, this_dataset) + + geneid_dict = {trait_name: geneid for (trait_name, geneid) + in geneid_dict.items() if + corr_results.get(trait_name)} + with database_connector() as conn: + return reduce( + lambda acc, corr: {**acc, **corr}, + compute_all_lit_correlation( + conn=conn, trait_lists=list(geneid_dict.items()), + species=species, gene_id=this_trait_geneid), + {}) + + return {} + + +def compute_top_n_tissue(this_dataset, this_trait, traits, method): + + # refactor lots of rpt + + trait_symbol_dict = dict({ + trait_name: symbol + for (trait_name, symbol) + in this_dataset.retrieve_genes("Symbol").items() + if traits.get(trait_name)}) + + corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values( + symbol_list=list(trait_symbol_dict.values())) + + data = parse_tissue_corr_data(symbol_name=this_trait.symbol, + symbol_dict=get_trait_symbol_and_tissue_values( + symbol_list=[this_trait.symbol]), + dataset_symbols=trait_symbol_dict, + dataset_vals=corr_result_tissue_vals_dict) + + if data: + return run_correlation( + data[1], data[0], method, ",", "tissue") + + return {} + + +def merge_results(dict_a: dict, dict_b: dict, dict_c: dict) -> list[dict]: + """code to merge diff corr into individual dicts + a""" + + def __merge__(trait_name, trait_corrs): + return { + trait_name: { + **trait_corrs, + **dict_b.get(trait_name, {}), + **dict_c.get(trait_name, {}) + } + } + return [__merge__(tname, tcorrs) for tname, tcorrs in dict_a.items()] + + +def __compute_sample_corr__( + start_vars: dict, corr_type: str, method: str, n_top: int, + target_trait_info: tuple): + """Compute the sample correlations""" + (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info + all_samples = json.loads(start_vars["sample_vals"]) + sample_data = get_sample_corr_data( + sample_type=start_vars["corr_samples_group"], all_samples=all_samples, + dataset_samples=this_dataset.group.all_samples_ordered()) + target_dataset.get_trait_data(list(sample_data.keys())) + + target_data = [] + for (key, val) in target_dataset.trait_data.items(): + lts = [key] + [str(x) for x in val] + r = ",".join(lts) + target_data.append(r) + + return run_correlation( + target_data, list(sample_data.values()), method, ",", corr_type, + n_top) + + +def __compute_tissue_corr__( + start_vars: dict, corr_type: str, method: str, n_top: int, + target_trait_info: tuple): + """Compute the tissue correlations""" + (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info + trait_symbol_dict = this_dataset.retrieve_genes("Symbol") + corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values( + symbol_list=list(trait_symbol_dict.values())) + + data = parse_tissue_corr_data( + symbol_name=this_trait.symbol, + symbol_dict=get_trait_symbol_and_tissue_values( + symbol_list=[this_trait.symbol]), + dataset_symbols=trait_symbol_dict, + dataset_vals=corr_result_tissue_vals_dict) + + if data: + return run_correlation(data[1], data[0], method, ",", "tissue") + return {} + + +def __compute_lit_corr__( + start_vars: dict, corr_type: str, method: str, n_top: int, + target_trait_info: tuple): + """Compute the literature correlations""" + (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info + target_dataset_type = target_dataset.type + this_dataset_type = this_dataset.type + (this_trait_geneid, geneid_dict, species) = do_lit_correlation( + this_trait, target_dataset) + + with database_connector() as conn: + return reduce( + lambda acc, lit: {**acc, **lit}, + compute_all_lit_correlation( + conn=conn, trait_lists=list(geneid_dict.items()), + species=species, gene_id=this_trait_geneid)[:n_top], + {}) + return {} + + +def compute_correlation_rust( + start_vars: dict, corr_type: str, method: str = "pearson", + n_top: int = 500, should_compute_all: bool = False): + """function to compute correlation""" + target_trait_info = create_target_this_trait(start_vars) + (this_dataset, this_trait, target_dataset, sample_data) = ( + target_trait_info) + + # Replace this with `match ...` once we hit Python 3.10 + corr_type_fns = { + "sample": __compute_sample_corr__, + "tissue": __compute_tissue_corr__, + "lit": __compute_lit_corr__ + } + results = corr_type_fns[corr_type]( + start_vars, corr_type, method, n_top, target_trait_info) + + # END: Replace this with `match ...` once we hit Python 3.10 + + top_a = top_b = {} + + if should_compute_all: + + if corr_type == "sample": + + top_a = compute_top_n_tissue( + this_dataset, this_trait, results, method) + + top_b = compute_top_n_lit(results, this_dataset, this_trait) + + elif corr_type == "lit": + + # currently fails for lit + + top_a = compute_top_n_sample( + start_vars, target_dataset, list(results.keys())) + top_b = compute_top_n_tissue( + this_dataset, this_trait, results, method) + + else: + + top_a = compute_top_n_sample( + start_vars, target_dataset, list(results.keys())) + + top_b = compute_top_n_lit(results, this_dataset, this_trait) + + return { + "correlation_results": merge_results( + results, top_a, top_b), + "this_trait": this_trait.name, + "target_dataset": start_vars['corr_dataset'], + "traits_metadata": get_metadata(target_dataset, list(results.keys())), + "return_results": n_top + } diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 1c391386..4910b0e0 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -93,8 +93,7 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe new_traits_metadata = {} - (file_path, dataset_metadata) = fetch_all_cached_metadata( - target_dataset['name']) + dataset_metadata = correlation_data["traits_metadata"] for i, trait_dict in enumerate(corr_results): trait_name = list(trait_dict.keys())[0] @@ -121,9 +120,9 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe results_dict['dataset'] = target_dataset['name'] results_dict['hmac'] = hmac.data_hmac( '{}:{}'.format(target_trait['name'], target_dataset['name'])) - results_dict['sample_r'] = f"{float(trait['corr_coefficient']):.3f}" - results_dict['num_overlap'] = trait['num_overlap'] - results_dict['sample_p'] = f"{float(trait['p_value']):.3e}" + results_dict['sample_r'] = f"{float(trait.get('corr_coefficient',0.0)):.3f}" + results_dict['num_overlap'] = trait.get('num_overlap',0) + results_dict['sample_p'] = f"{float(trait.get('p_value',0)):.3e}" if target_dataset['type'] == "ProbeSet": results_dict['symbol'] = target_trait['symbol'] results_dict['description'] = "N/A" @@ -187,10 +186,6 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe results_list.append(results_dict) - cache_new_traits_metadata(dataset_metadata, - new_traits_metadata, - file_path) - return json.dumps(results_list) diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index cf37b9e9..17c174b1 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -77,7 +77,7 @@ class CorrelationMatrix: self.sample_data.append(this_trait_vals) # Shouldn't do PCA if there are more traits than observations/samples - if len(this_trait_vals) < len(self.trait_list): + if len(this_trait_vals) < len(self.trait_list) or len(self.trait_list) < 3: self.do_PCA = False # ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning) @@ -85,6 +85,7 @@ class CorrelationMatrix: self.corr_results = [] self.pca_corr_results = [] + self.scree_data = [] self.shared_samples_list = self.all_sample_list for trait_db in self.trait_list: this_trait = trait_db[0] diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index bb928ec5..96a47eb8 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -24,8 +24,6 @@ from utility.tools import locate, GN2_BRANCH_URL from rpy2.robjects.packages import importr -import utility.logger -logger = utility.logger.getLogger(__name__) # Get pointers to some common R functions r_library = ro.r["library"] # Map the library function @@ -42,14 +40,9 @@ r_as_numeric = ro.r["as.numeric"] # Map the write.table function class CTL: def __init__(self): - logger.info("Initialization of CTL") - #log = r_file("/tmp/genenetwork_ctl.log", open = "wt") - # r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file - #r_sink(log, type = "message") # Load CTL - Should only be done once, since it is quite expensive r_library("ctl") r_options(stringsAsFactors=False) - logger.info("Initialization of CTL done, package loaded in R session") # Map the CTLscan function self.r_CTLscan = ro.r["CTLscan"] # Map the CTLsignificant function @@ -60,7 +53,6 @@ class CTL: self.r_plotCTLobject = ro.r["plot.CTLobject"] self.nodes_list = [] self.edges_list = [] - logger.info("Obtained pointers to CTL functions") self.gn2_url = GN2_BRANCH_URL @@ -85,21 +77,12 @@ class CTL: self.edges_list.append(edge_dict) def run_analysis(self, requestform): - logger.info("Starting CTL analysis on dataset") self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] self.trait_db_list = [x for x in self.trait_db_list if x] - - logger.debug("strategy:", requestform.get("strategy")) strategy = requestform.get("strategy") - - logger.debug("nperm:", requestform.get("nperm")) nperm = int(requestform.get("nperm")) - - logger.debug("parametric:", requestform.get("parametric")) parametric = bool(requestform.get("parametric")) - - logger.debug("significance:", requestform.get("significance")) significance = float(requestform.get("significance")) # Get the name of the .geno file belonging to the first phenotype @@ -109,7 +92,6 @@ class CTL: genofilelocation = locate(dataset.group.name + ".geno", "genotype") parser = genofile_parser.ConvertGenoFile(genofilelocation) parser.process_csv() - logger.debug("dataset group: ", dataset.group) # Create a genotype matrix individuals = parser.individuals markers = [] @@ -119,8 +101,6 @@ class CTL: markers.append(marker["genotypes"]) genotypes = list(itertools.chain(*markers)) - logger.debug(len(genotypes) / len(individuals), - "==", len(parser.markers)) rGeno = r_t(ro.r.matrix(r_unlist(genotypes), nrow=len(markernames), ncol=len( individuals), dimnames=r_list(markernames, individuals), byrow=True)) @@ -128,7 +108,6 @@ class CTL: # Create a phenotype matrix traits = [] for trait in self.trait_db_list: - logger.debug("retrieving data for", trait) if trait != "": ts = trait.split(':') gt = create_trait(name=ts[0], dataset_name=ts[1]) @@ -142,8 +121,6 @@ class CTL: rPheno = r_t(ro.r.matrix(r_as_numeric(r_unlist(traits)), nrow=len(self.trait_db_list), ncol=len( individuals), dimnames=r_list(self.trait_db_list, individuals), byrow=True)) - logger.debug(rPheno) - # Use a data frame to store the objects rPheno = r_data_frame(rPheno, check_names=False) rGeno = r_data_frame(rGeno, check_names=False) @@ -195,8 +172,6 @@ class CTL: # Create the interactive graph for cytoscape visualization (Nodes and Edges) if not isinstance(significant, ri.RNULLType): for x in range(len(significant[0])): - logger.debug(significant[0][x], significant[1] - [x], significant[2][x]) # Debug to console # Source tsS = significant[0][x].split(':') # Target @@ -231,7 +206,6 @@ class CTL: n = n + 1 def process_results(self, results): - logger.info("Processing CTL output") template_vars = {} template_vars["results"] = self.results template_vars["elements"] = self.elements diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index f0ce0a77..ec616d07 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -1,16 +1,13 @@ # Module to initialize sqlalchemy with flask import os import sys -from string import Template from typing import Tuple from urllib.parse import urlparse import importlib +import contextlib import MySQLdb -from sqlalchemy import create_engine -from sqlalchemy.orm import scoped_session, sessionmaker -from sqlalchemy.ext.declarative import declarative_base def read_from_pyfile(pyfile, setting): orig_sys_path = sys.path[:] @@ -19,6 +16,7 @@ def read_from_pyfile(pyfile, setting): sys.path = orig_sys_path[:] return module.__dict__.get(setting) + def sql_uri(): """Read the SQL_URI from the environment or settings file.""" return os.environ.get( @@ -37,8 +35,26 @@ def parse_db_url(sql_uri: str) -> Tuple: parsed_db.hostname, parsed_db.username, parsed_db.password, parsed_db.path[1:], parsed_db.port) + +@contextlib.contextmanager def database_connection(): - """Returns a database connection""" + """Provide a context manager for opening, closing, and rolling + back - if supported - a database connection. Should an error occur, + and if the table supports transactions, the connection will be + rolled back. + + """ host, user, passwd, db_name, port = parse_db_url(sql_uri()) - return MySQLdb.connect( - db=db_name, user=user, passwd=passwd, host=host, port=port) + connection = MySQLdb.connect( + db=db_name, user=user, passwd=passwd or '', host=host, port=port, + autocommit=False # Required for roll-backs + ) + try: + yield connection + except Exception: + connection.rollback() + raise + else: + connection.commit() + finally: + connection.close() diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index 938c453e..f52c30e4 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -6,9 +6,6 @@ import re from flask import Flask, g -from utility.logger import getLogger -logger = getLogger(__name__) - class InfoPage: def __init__(self, start_vars): diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index b0756361..97143486 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -13,10 +13,6 @@ import sys from db import webqtlDatabaseFunction from utility.tools import GN2_BASE_URL -import logging -from utility.logger import getLogger -logger = getLogger(__name__) - class DoSearch: """Parent class containing parameters/functions used for all searches""" @@ -41,7 +37,6 @@ class DoSearch: def execute(self, query): """Executes query and returns results""" query = self.normalize_spaces(query) - logger.sql(query) results = g.db.execute(query, no_parameters=True).fetchall() return results @@ -55,7 +50,6 @@ class DoSearch: def mescape(self, *items): """Multiple escape""" escaped = [escape(str(item)) for item in items] - logger.debug("escaped is:", escaped) return tuple(escaped) def normalize_spaces(self, stringy): @@ -69,8 +63,6 @@ class DoSearch: if 'key' in search_type and search_type['key'] != None: search_type_string += '_' + search_type['key'] - logger.debug("search_type_string is:", search_type_string) - if search_type_string in cls.search_types: return cls.search_types[search_type_string] else: @@ -177,8 +169,6 @@ class MrnaAssaySearch(DoSearch): def run_combined(self, from_clause='', where_clause=''): """Generates and runs a combined search of an mRNA expression dataset""" - - logger.debug("Running ProbeSetSearch") #query = self.base_query + from_clause + " WHERE " + where_clause from_clause = self.normalize_spaces(from_clause) @@ -197,11 +187,8 @@ class MrnaAssaySearch(DoSearch): def run(self): """Generates and runs a simple search of an mRNA expression dataset""" - - logger.debug("Running ProbeSetSearch") where_clause = self.get_where_clause() query = self.base_query + "WHERE " + where_clause + "ORDER BY ProbeSet.symbol ASC" - return self.execute(query) @@ -310,9 +297,6 @@ class PhenotypeSearch(DoSearch): def run_combined(self, from_clause, where_clause): """Generates and runs a combined search of an phenotype dataset""" - - logger.debug("Running PhenotypeSearch") - from_clause = self.normalize_spaces(from_clause) query = (self.base_query + @@ -370,7 +354,6 @@ class GenotypeSearch(DoSearch): where_clause.append('''%s REGEXP "%s"''' % ("%s.%s" % self.mescape(self.dataset.type, field), self.search_term)) - logger.debug("hello ;where_clause is:", pf(where_clause)) where_clause = "(%s) " % ' OR '.join(where_clause) return where_clause @@ -559,7 +542,6 @@ class LrsSearch(DoSearch): self.species_id) else: # Deal with >, <, >=, and <= - logger.debug("self.search_term is:", self.search_term) lrs_val = self.search_term[0] if self.search_type == "LOD": lrs_val = lrs_val * 4.61 @@ -794,7 +776,6 @@ class MeanSearch(MrnaAssaySearch): def run(self): self.where_clause = self.get_where_clause() - logger.debug("where_clause is:", pf(self.where_clause)) self.query = self.compile_final_query(where_clause=self.where_clause) @@ -824,9 +805,6 @@ class RangeSearch(MrnaAssaySearch): FROM ProbeSetData WHERE ProbeSetData.Id = ProbeSetXRef.dataId) > %s """ % (escape(self.search_term[0])) - - logger.debug("where_clause is:", pf(where_clause)) - return where_clause def run(self): @@ -932,11 +910,7 @@ class PvalueSearch(MrnaAssaySearch): self.search_operator, self.search_term[0]) - logger.debug("where_clause is:", pf(self.where_clause)) - self.query = self.compile_final_query(where_clause=self.where_clause) - - logger.sql(self.query) return self.execute(self.query) diff --git a/wqflask/wqflask/docs.py b/wqflask/wqflask/docs.py index 0a1a597d..9d58162e 100644 --- a/wqflask/wqflask/docs.py +++ b/wqflask/wqflask/docs.py @@ -2,9 +2,6 @@ import codecs from flask import g -from utility.logger import getLogger -logger = getLogger(__name__) - class Docs: diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 0c80e9a4..4b37c7f7 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -14,8 +14,6 @@ from gn3.computations.gemma import generate_hash_of_string from base.trait import create_trait, retrieve_trait_info -from utility.logger import getLogger -logger = getLogger(__name__) def export_traits(targs, export_type): if export_type == "collection": diff --git a/wqflask/wqflask/external_tools/send_to_bnw.py b/wqflask/wqflask/external_tools/send_to_bnw.py index c1b14ede..dfb59c63 100644 --- a/wqflask/wqflask/external_tools/send_to_bnw.py +++ b/wqflask/wqflask/external_tools/send_to_bnw.py @@ -21,9 +21,6 @@ from base.trait import GeneralTrait from utility import helper_functions, corr_result_helpers -import utility.logger -logger = utility.logger.getLogger(__name__) - class SendToBNW: def __init__(self, start_vars): diff --git a/wqflask/wqflask/external_tools/send_to_geneweaver.py b/wqflask/wqflask/external_tools/send_to_geneweaver.py index 9a4f7150..a8066b43 100644 --- a/wqflask/wqflask/external_tools/send_to_geneweaver.py +++ b/wqflask/wqflask/external_tools/send_to_geneweaver.py @@ -26,9 +26,6 @@ from base.trait import GeneralTrait, retrieve_trait_info from base.species import TheSpecies from utility import helper_functions, corr_result_helpers -import utility.logger -logger = utility.logger.getLogger(__name__) - class SendToGeneWeaver: def __init__(self, start_vars): diff --git a/wqflask/wqflask/external_tools/send_to_webgestalt.py b/wqflask/wqflask/external_tools/send_to_webgestalt.py index 6e74f4fe..4de684b0 100644 --- a/wqflask/wqflask/external_tools/send_to_webgestalt.py +++ b/wqflask/wqflask/external_tools/send_to_webgestalt.py @@ -26,9 +26,6 @@ from base.trait import GeneralTrait, retrieve_trait_info from base.species import TheSpecies from utility import helper_functions, corr_result_helpers -import utility.logger -logger = utility.logger.getLogger(__name__) - class SendToWebGestalt: def __init__(self, start_vars): diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 9df7b2e1..56877524 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -11,13 +11,9 @@ from base import webqtlConfig from utility import hmac -from utility.benchmark import Bench from utility.authentication_tools import check_resource_availability from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string -from utility.logger import getLogger -logger = getLogger(__name__) - class GSearch: @@ -64,75 +60,72 @@ class GSearch: ORDER BY species_name, inbredset_name, tissue_name, probesetfreeze_name, probeset_name LIMIT 6000 """ % (self.terms) - with Bench("Running query"): - logger.sql(sql) - re = g.db.execute(sql).fetchall() + re = g.db.execute(sql).fetchall() trait_list = [] dataset_to_permissions = {} - with Bench("Creating trait objects"): - for i, line in enumerate(re): - trait_dict = {} - trait_dict['index'] = i + 1 - trait_dict['name'] = line[5] - trait_dict['dataset'] = line[3] - trait_dict['dataset_fullname'] = line[4] - trait_dict['hmac'] = hmac.data_hmac( - '{}:{}'.format(line[5], line[3])) - trait_dict['species'] = line[0] - trait_dict['group'] = line[1] - trait_dict['tissue'] = line[2] - trait_dict['symbol'] = "N/A" - if line[6]: - trait_dict['symbol'] = line[6] - trait_dict['description'] = "N/A" - if line[7]: - trait_dict['description'] = line[7].decode( - 'utf-8', 'replace') - trait_dict['location_repr'] = "N/A" - if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): - trait_dict['location_repr'] = 'Chr%s: %.6f' % ( - line[8], float(line[9])) - - trait_dict['LRS_score_repr'] = "N/A" - trait_dict['additive'] = "N/A" - trait_dict['mean'] = "N/A" - - if line[11] != "" and line[11] != None: - trait_dict['LRS_score_repr'] = f"{line[11]:.3f}" - if line[14] != "" and line[14] != None: - trait_dict['additive'] = f"{line[14]:.3f}" - if line[10] != "" and line[10] != None: - trait_dict['mean'] = f"{line[10]:.3f}" - - locus_chr = line[16] - locus_mb = line[17] - - max_lrs_text = "N/A" - if locus_chr and locus_mb: - max_lrs_text = f"Chr{locus_chr}: {locus_mb}" - trait_dict['max_lrs_text'] = max_lrs_text - - trait_dict['additive'] = "N/A" - if line[14] != "" and line[14] != None: - trait_dict['additive'] = '%.3f' % line[14] - trait_dict['dataset_id'] = line[15] - - dataset_ob = SimpleNamespace( - id=trait_dict["dataset_id"], type="ProbeSet", name=trait_dict["dataset"], species=trait_dict["species"]) - if dataset_ob.id not in dataset_to_permissions: - permissions = check_resource_availability(dataset_ob) - dataset_to_permissions[dataset_ob.id] = permissions - else: - pemissions = dataset_to_permissions[dataset_ob.id] - if type(permissions['data']) is list: - if "view" not in permissions['data']: - continue - else: - if permissions['data'] == 'no-access': - continue - - trait_list.append(trait_dict) + for i, line in enumerate(re): + this_trait = {} + this_trait['index'] = i + 1 + this_trait['name'] = line[5] + this_trait['dataset'] = line[3] + this_trait['dataset_fullname'] = line[4] + this_trait['hmac'] = hmac.data_hmac( + '{}:{}'.format(line[5], line[3])) + this_trait['species'] = line[0] + this_trait['group'] = line[1] + this_trait['tissue'] = line[2] + this_trait['symbol'] = "N/A" + if line[6]: + this_trait['symbol'] = line[6] + this_trait['description'] = "N/A" + if line[7]: + this_trait['description'] = line[7].decode( + 'utf-8', 'replace') + this_trait['location_repr'] = "N/A" + if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): + this_trait['location_repr'] = 'Chr%s: %.6f' % ( + line[8], float(line[9])) + + this_trait['LRS_score_repr'] = "N/A" + this_trait['additive'] = "N/A" + this_trait['mean'] = "N/A" + + if line[11] != "" and line[11] != None: + this_trait['LRS_score_repr'] = f"{line[11]:.3f}" + if line[14] != "" and line[14] != None: + this_trait['additive'] = f"{line[14]:.3f}" + if line[10] != "" and line[10] != None: + this_trait['mean'] = f"{line[10]:.3f}" + + locus_chr = line[16] + locus_mb = line[17] + + max_lrs_text = "N/A" + if locus_chr and locus_mb: + max_lrs_text = f"Chr{locus_chr}: {locus_mb}" + this_trait['max_lrs_text'] = max_lrs_text + + this_trait['additive'] = "N/A" + if line[14] != "" and line[14] != None: + this_trait['additive'] = '%.3f' % line[14] + this_trait['dataset_id'] = line[15] + + dataset_ob = SimpleNamespace( + id=this_trait["dataset_id"], type="ProbeSet", name=this_trait["dataset"], species=this_trait["species"]) + if dataset_ob.id not in dataset_to_permissions: + permissions = check_resource_availability(dataset_ob) + dataset_to_permissions[dataset_ob.id] = permissions + else: + pemissions = dataset_to_permissions[dataset_ob.id] + if type(permissions['data']) is list: + if "view" not in permissions['data']: + continue + else: + if permissions['data'] == 'no-access': + continue + + trait_list.append(this_trait) self.trait_count = len(trait_list) self.trait_list = trait_list @@ -210,86 +203,84 @@ class GSearch: ORDER BY Species.`Name`, InbredSet.`Name`, PublishXRef.`Id` LIMIT 6000 """.format(group_clause, search_term) - logger.sql(sql) re = g.db.execute(sql).fetchall() trait_list = [] - with Bench("Creating trait objects"): - for i, line in enumerate(re): - trait_dict = {} - trait_dict['index'] = i + 1 - trait_dict['name'] = str(line[4]) - if len(str(line[12])) == 3: - trait_dict['display_name'] = str( - line[12]) + "_" + trait_dict['name'] - else: - trait_dict['display_name'] = trait_dict['name'] - trait_dict['dataset'] = line[2] - trait_dict['dataset_fullname'] = line[3] - trait_dict['hmac'] = hmac.data_hmac( - '{}:{}'.format(line[4], line[2])) - trait_dict['species'] = line[0] - trait_dict['group'] = line[1] - if line[9] != None and line[6] != None: - trait_dict['description'] = line[6].decode( - 'utf-8', 'replace') - elif line[5] != None: - trait_dict['description'] = line[5].decode( - 'utf-8', 'replace') - else: - trait_dict['description'] = "N/A" - trait_dict['dataset_id'] = line[14] - - trait_dict['LRS_score_repr'] = "N/A" - trait_dict['additive'] = "N/A" - trait_dict['mean'] = "N/A" - - if line[10] != "" and line[10] != None: - trait_dict['LRS_score_repr'] = f"{line[10]:.3f}" - # Some Max LRS values in the DB are wrongly listed as 0.000, but shouldn't be displayed - if trait_dict['LRS_score_repr'] == "0.000": - trait_dict['LRS_score_repr'] = "N/A" - if line[11] != "" and line[11] != None: - trait_dict['additive'] = f"{line[11]:.3f}" - if line[13] != "" and line[13] != None: - trait_dict['mean'] = f"{line[13]:.3f}" - - locus_chr = line[15] - locus_mb = line[16] - - max_lrs_text = "N/A" - if locus_chr and locus_mb: - max_lrs_text = f"Chr{locus_chr}: {locus_mb}" - trait_dict['max_lrs_text'] = max_lrs_text - - trait_dict['authors'] = line[7] - - trait_dict['authors'] = line[7] - trait_dict['authors_display'] = trait_dict['authors'] - author_list = trait_dict['authors'].split(",") - if len(author_list) >= 2: - trait_dict['authors_display'] = (",").join(author_list[:2]) + ", et al." - - trait_dict['year'] = line[8] - trait_dict['pubmed_text'] = "N/A" - trait_dict['pubmed_link'] = "N/A" - if trait_dict['year'].isdigit(): - trait_dict['pubmed_text'] = trait_dict['year'] - if line[9] != "" and line[9] != None: - trait_dict['pubmed_link'] = webqtlConfig.PUBMEDLINK_URL % line[8] - if line[12]: - trait_dict['display_name'] = line[12] + \ - "_" + str(trait_dict['name']) - - dataset_ob = SimpleNamespace(id=trait_dict["dataset_id"], type="Publish", species=trait_dict["species"]) - permissions = check_resource_availability(dataset_ob, trait_dict['name']) - if type(permissions['data']) is list: - if "view" not in permissions['data']: - continue - else: - if permissions['data'] == 'no-access': - continue - - trait_list.append(trait_dict) + for i, line in enumerate(re): + trait_dict = {} + trait_dict['index'] = i + 1 + trait_dict['name'] = str(line[4]) + if len(str(line[12])) == 3: + trait_dict['display_name'] = str( + line[12]) + "_" + trait_dict['name'] + else: + trait_dict['display_name'] = trait_dict['name'] + trait_dict['dataset'] = line[2] + trait_dict['dataset_fullname'] = line[3] + trait_dict['hmac'] = hmac.data_hmac( + '{}:{}'.format(line[4], line[2])) + trait_dict['species'] = line[0] + trait_dict['group'] = line[1] + if line[9] != None and line[6] != None: + trait_dict['description'] = line[6].decode( + 'utf-8', 'replace') + elif line[5] != None: + trait_dict['description'] = line[5].decode( + 'utf-8', 'replace') + else: + trait_dict['description'] = "N/A" + trait_dict['dataset_id'] = line[14] + + trait_dict['LRS_score_repr'] = "N/A" + trait_dict['additive'] = "N/A" + trait_dict['mean'] = "N/A" + + if line[10] != "" and line[10] != None: + trait_dict['LRS_score_repr'] = f"{line[10]:.3f}" + # Some Max LRS values in the DB are wrongly listed as 0.000, but shouldn't be displayed + if trait_dict['LRS_score_repr'] == "0.000": + trait_dict['LRS_score_repr'] = "N/A" + if line[11] != "" and line[11] != None: + trait_dict['additive'] = f"{line[11]:.3f}" + if line[13] != "" and line[13] != None: + trait_dict['mean'] = f"{line[13]:.3f}" + + locus_chr = line[15] + locus_mb = line[16] + + max_lrs_text = "N/A" + if locus_chr and locus_mb: + max_lrs_text = f"Chr{locus_chr}: {locus_mb}" + trait_dict['max_lrs_text'] = max_lrs_text + + trait_dict['authors'] = line[7] + + trait_dict['authors'] = line[7] + trait_dict['authors_display'] = trait_dict['authors'] + author_list = trait_dict['authors'].split(",") + if len(author_list) >= 2: + trait_dict['authors_display'] = (",").join(author_list[:2]) + ", et al." + + trait_dict['year'] = line[8] + trait_dict['pubmed_text'] = "N/A" + trait_dict['pubmed_link'] = "N/A" + if trait_dict['year'].isdigit(): + trait_dict['pubmed_text'] = trait_dict['year'] + if line[9] != "" and line[9] != None: + trait_dict['pubmed_link'] = webqtlConfig.PUBMEDLINK_URL % line[8] + if line[12]: + trait_dict['display_name'] = line[12] + \ + "_" + str(trait_dict['name']) + + dataset_ob = SimpleNamespace(id=trait_dict["dataset_id"], type="Publish", species=trait_dict["species"]) + permissions = check_resource_availability(dataset_ob, trait_dict['name']) + if type(permissions['data']) is list: + if "view" not in permissions['data']: + continue + else: + if permissions['data'] == 'no-access': + continue + + trait_list.append(trait_dict) self.trait_count = len(trait_list) self.trait_list = trait_list diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 001bab3b..1c8a4ff6 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -8,12 +8,9 @@ from utility import helper_functions from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR from redis import Redis from flask import Flask, g -from utility.logger import getLogger Redis = Redis() -logger = getLogger(__name__) - class Heatmap: diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 8dffd21d..ef0c357e 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -42,17 +42,15 @@ from base import webqtlConfig from base.GeneralObject import GeneralObject from utility import webqtlUtil from utility import Plot -from utility.benchmark import Bench from wqflask.interval_analyst import GeneUtil from base.webqtlConfig import GENERATED_IMAGE_DIR from utility.pillow_utils import draw_rotated_text, draw_open_polygon -import utility.logger + try: # Only import this for Python3 from functools import reduce except: pass -logger = utility.logger.getLogger(__name__) RED = ImageColor.getrgb("red") BLUE = ImageColor.getrgb("blue") @@ -247,8 +245,6 @@ class DisplayMappingResults: HELP_PAGE_REF = '/glossary.html' def __init__(self, start_vars): - logger.info("Running qtlreaper") - self.temp_uuid = start_vars['temp_uuid'] self.dataset = start_vars['dataset'] @@ -590,9 +586,8 @@ class DisplayMappingResults: ################################################################ showLocusForm = "" intCanvas = Image.new("RGBA", size=(self.graphWidth, self.graphHeight)) - with Bench("Drawing Plot"): - gifmap = self.plotIntMapping( - intCanvas, startMb=self.startMb, endMb=self.endMb, showLocusForm=showLocusForm) + gifmap = self.plotIntMapping( + intCanvas, startMb=self.startMb, endMb=self.endMb, showLocusForm=showLocusForm) self.gifmap = gifmap.__str__() @@ -3012,7 +3007,7 @@ class DisplayMappingResults: else: snpString = 0 - mouseStartString = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=vertebrate&org=Mouse&db=mm9&position=chr" + \ + mouseStartString = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=vertebrate&org=Mouse&db=mm10&position=chr" + \ theGO["Chromosome"] + "%3A" + str(int(theGO["TxStart"] * 1000000.0)) + "-" + str( int(theGO["TxEnd"] * 1000000.0)) + "&pix=620&Submit=submit" diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index ec12f9b7..646728ba 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -13,8 +13,6 @@ from utility.tools import TEMPDIR from utility.tools import WEBSERVER_MODE from gn3.computations.gemma import generate_hash_of_string -import utility.logger -logger = utility.logger.getLogger(__name__) GEMMAOPTS = "-debug" if WEBSERVER_MODE == 'PROD': @@ -220,7 +218,7 @@ def parse_loco_output(this_dataset, gwa_output_filename, loco="True"): marker['chr'] = line.split("\t")[0] marker['Mb'] = float(line.split("\t")[2]) / 1000000 marker['p_value'] = float(line.split("\t")[10]) - marker['additive'] = -(float(line.split("\t")[7])) + marker['additive'] = -(float(line.split("\t")[7])/2) if math.isnan(marker['p_value']) or (marker['p_value'] <= 0): marker['lod_score'] = 0 else: diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index 2fa80841..75ee189e 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -5,9 +5,6 @@ from base.webqtlConfig import TMPDIR from utility import webqtlUtil from utility.tools import flat_files, PLINK_COMMAND -import utility.logger -logger = utility.logger.getLogger(__name__) - def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr( @@ -15,13 +12,11 @@ def run_plink(this_trait, dataset, species, vals, maf): gen_pheno_txt_file(dataset, vals) plink_command = f"{PLINK_COMMAND} --noweb --bfile {flat_files('mapping')}/{dataset.group.name} --no-pheno --no-fid --no-parents --no-sex --maf {maf} --out { TMPDIR}{plink_output_filename} --assoc " - logger.debug("plink_command:", plink_command) os.system(plink_command) count, p_values = parse_plink_output(plink_output_filename, species) - logger.debug("p_values:", p_values) dataset.group.markers.add_pvalues(p_values) return dataset.group.markers.markers diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index c4b495d7..4d5db2ee 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -10,9 +10,6 @@ from base.trait import GeneralTrait from base.data_set import create_dataset from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR -import utility.logger -logger = utility.logger.getLogger(__name__) - def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boot_check, num_bootstrap, do_control, control_marker, manhattan_plot, first_run=True, output_files=None): """Generates p-values for each marker using qtlreaper""" @@ -67,8 +64,6 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo opt_list), webqtlConfig.GENERATED_IMAGE_DIR, output_filename)) - - logger.debug("reaper_command:" + reaper_command) os.system(reaper_command) else: output_filename, permu_filename, bootstrap_filename = output_files diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 3bf06ea6..7d112c68 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -17,9 +17,6 @@ from base.webqtlConfig import TMPDIR from base.trait import create_trait from utility.tools import locate, GN3_LOCAL_URL -import utility.logger -logger = utility.logger.getLogger(__name__) - def run_rqtl(trait_name, vals, samples, dataset, pair_scan, mapping_scale, model, method, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, cofactors): """Run R/qtl by making a request to the GN3 endpoint and reading in the output file(s)""" diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index af229f5c..d9f33f8f 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -30,7 +30,6 @@ from base import data_set from base import species from base import webqtlConfig from utility import webqtlUtil, helper_functions, hmac, Plot, Bunch, temp_data -from utility.benchmark import Bench from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping from wqflask.show_trait.SampleList import SampleList @@ -38,9 +37,6 @@ from utility.tools import locate, locate_ignore_error, GEMMA_COMMAND, PLINK_COMM from utility.external import shell from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR -import utility.logger -logger = utility.logger.getLogger(__name__) - class RunMapping: @@ -204,13 +200,12 @@ class RunMapping: self.first_run = False self.score_type = "-logP" self.manhattan_plot = True - with Bench("Running GEMMA"): - if self.use_loco == "True": - marker_obs, self.output_files = gemma_mapping.run_gemma( - self.this_trait, self.dataset, self.samples, self.vals, self.covariates, self.use_loco, self.maf, self.first_run, self.output_files) - else: - marker_obs, self.output_files = gemma_mapping.run_gemma( - self.this_trait, self.dataset, self.samples, self.vals, self.covariates, self.use_loco, self.maf, self.first_run, self.output_files) + if self.use_loco == "True": + marker_obs, self.output_files = gemma_mapping.run_gemma( + self.this_trait, self.dataset, self.samples, self.vals, self.covariates, self.use_loco, self.maf, self.first_run, self.output_files) + else: + marker_obs, self.output_files = gemma_mapping.run_gemma( + self.this_trait, self.dataset, self.samples, self.vals, self.covariates, self.use_loco, self.maf, self.first_run, self.output_files) results = marker_obs elif self.mapping_method == "rqtl_plink": results = self.run_rqtl_plink() @@ -270,7 +265,6 @@ class RunMapping: self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] - logger.info("Running qtlreaper") self.first_run = True self.output_files = None @@ -300,8 +294,6 @@ class RunMapping: results = plink_mapping.run_plink( self.this_trait, self.dataset, self.species, self.vals, self.maf) #results = self.run_plink() - else: - logger.debug("RUNNING NOTHING") self.no_results = False if len(results) == 0: @@ -376,33 +368,29 @@ class RunMapping: self.qtl_results.append(marker) total_markers = len(self.qtl_results) + export_mapping_results(self.dataset, self.this_trait, self.qtl_results, self.mapping_results_path, + self.mapping_method, self.mapping_scale, self.score_type, + self.transform, self.covariates, self.n_samples, self.vals_hash) + + if len(self.qtl_results) > 30000: + self.qtl_results = trim_markers_for_figure( + self.qtl_results) + self.results_for_browser = trim_markers_for_figure( + self.results_for_browser) + filtered_annotations = [] + for marker in self.results_for_browser: + for annot_marker in self.annotations_for_browser: + if annot_marker['rs'] == marker['rs']: + filtered_annotations.append(annot_marker) + break + self.annotations_for_browser = filtered_annotations + browser_files = write_input_for_browser( + self.dataset, self.results_for_browser, self.annotations_for_browser) + else: + browser_files = write_input_for_browser( + self.dataset, self.results_for_browser, self.annotations_for_browser) - with Bench("Exporting Results"): - export_mapping_results(self.dataset, self.this_trait, self.qtl_results, self.mapping_results_path, - self.mapping_method, self.mapping_scale, self.score_type, - self.transform, self.covariates, self.n_samples, self.vals_hash) - - with Bench("Trimming Markers for Figure"): - if len(self.qtl_results) > 30000: - self.qtl_results = trim_markers_for_figure( - self.qtl_results) - self.results_for_browser = trim_markers_for_figure( - self.results_for_browser) - filtered_annotations = [] - for marker in self.results_for_browser: - for annot_marker in self.annotations_for_browser: - if annot_marker['rs'] == marker['rs']: - filtered_annotations.append(annot_marker) - break - self.annotations_for_browser = filtered_annotations - browser_files = write_input_for_browser( - self.dataset, self.results_for_browser, self.annotations_for_browser) - else: - browser_files = write_input_for_browser( - self.dataset, self.results_for_browser, self.annotations_for_browser) - - with Bench("Trimming Markers for Table"): - self.trimmed_markers = trim_markers_for_table(results) + self.trimmed_markers = trim_markers_for_table(results) chr_lengths = get_chr_lengths( self.mapping_scale, self.mapping_method, self.dataset, self.qtl_results) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 5a5ee4ac..0840d4dd 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -62,13 +62,16 @@ def _get_diffs( ): def __get_file_metadata(file_name: str) -> Dict: author, resource_id, time_stamp, *_ = file_name.split(".") - + try: + author = json.loads(redis_conn.hget("users", author)).get( + "full_name" + ) + except (AttributeError, TypeError): + author = author return { "resource_id": resource_id, "file_name": file_name, - "author": json.loads(redis_conn.hget("users", author)).get( - "full_name" - ), + "author": author, "time_stamp": time_stamp, "roles": get_highest_user_access_role( resource_id=resource_id, @@ -124,40 +127,7 @@ def edit_probeset(conn, name): columns=list(probeset_mapping.values()), where=Probeset(name=name), ) - json_data = fetchall( - conn, "metadata_audit", where=MetadataAudit(dataset_id=probeset_.id_) - ) - Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) - Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) - diff_data = [] - for data in json_data: - json_ = json.loads(data.json_data) - timestamp = json_.get("timestamp") - author = json_.get("author") - for key, value in json_.items(): - if isinstance(value, dict): - for field, data_ in value.items(): - diff_data.append( - Diff( - author=author, - diff=Edit( - field, - data_.get("old"), - data_.get("new"), - "\n".join( - difflib.ndiff( - [data_.get("old")], [data_.get("new")] - ) - ), - ), - timestamp=timestamp, - ) - ) - diff_data_ = None - if len(diff_data) > 0: - diff_data_ = groupby(diff_data, lambda x: x.timestamp) return { - "diff": diff_data_, "probeset": probeset_, } @@ -227,8 +197,7 @@ def update_phenotype(dataset_id: str, name: str): diff_data = {} with database_connection() as conn: headers = ["Strain Name", "Value", "SE", "Count"] + list( - get_case_attributes(conn).keys() - ) + map(lambda x: x[1], get_case_attributes(conn))) diff_data = remove_insignificant_edits( diff_data=csv_diff( base_csv=( @@ -552,21 +521,31 @@ def show_diff(name): @metadata_edit.route("/<dataset_id>/traits/<name>/history") -def show_history(dataset_id: str, name: str): +@metadata_edit.route("/probeset/<name>") +def show_history(dataset_id: str = "", name: str = ""): diff_data_ = None with database_connection() as conn: - publish_xref = fetchone( - conn=conn, - table="PublishXRef", - where=PublishXRef(id_=name, inbred_set_id=dataset_id), - ) - - json_data = fetchall( - conn, - "metadata_audit", - where=MetadataAudit(dataset_id=publish_xref.id_), - ) - + json_data = None + if dataset_id: # This is a published phenotype + json_data = fetchall( + conn, + "metadata_audit", + where=MetadataAudit(dataset_id=fetchone( + conn=conn, + table="PublishXRef", + where=PublishXRef(id_=name, inbred_set_id=dataset_id), + ).id_), + ) + else: # This is a probeset + json_data = fetchall( + conn, "metadata_audit", + where=MetadataAudit(dataset_id=fetchone( + conn=conn, + table="ProbeSet", + columns=list(probeset_mapping.values()), + where=Probeset(name=name), + ).id_) + ) Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) diff_data = [] diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index 7a808ac9..ddf48d90 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -21,9 +21,6 @@ import re from pprint import pformat as pf -from utility.logger import getLogger -logger = getLogger(__name__) - def parse(pstring): """ @@ -45,7 +42,6 @@ def parse(pstring): for item in pstring: splat = re.split(separators, item) - logger.debug("splat is:", splat) # splat is an array of 1 if no match, otherwise more than 1 if len(splat) > 1: @@ -73,7 +69,6 @@ def parse(pstring): search_term=[item]) items.append(term) - logger.debug("* items are:", pf(items) + "\n") return(items) diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py index ce5e46e0..358a9d3b 100644 --- a/wqflask/wqflask/partial_correlations_views.py +++ b/wqflask/wqflask/partial_correlations_views.py @@ -139,6 +139,7 @@ def target_db_error(args, with_target_db: bool): def method_error(args): methods = ( + "pearson's r", "spearman's rho", "genetic correlation, pearson's r", "genetic correlation, spearman's rho", "sgo literature correlation", diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index eb8879dd..a835f631 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -25,8 +25,6 @@ from utility.authentication_tools import check_resource_availability from utility.tools import GN2_BASE_URL from utility.type_checking import is_str -from utility.logger import getLogger -logger = getLogger(__name__) class SearchResultPage: #maxReturn = 3000 @@ -56,7 +54,6 @@ class SearchResultPage: rx = re.compile( r'.*\W(href|http|sql|select|update)\W.*', re.IGNORECASE) if rx.match(search): - logger.debug("Regex failed search") self.search_term_exists = False return else: @@ -192,6 +189,8 @@ class SearchResultPage: trait_dict['additive'] = "N/A" if not result[8] else f"{result[8]:.3f}" + trait_dict['trait_info_str'] = trait_info_str(trait_dict, self.dataset.type) + # Convert any bytes in dict to a normal utf-8 string for key in trait_dict.keys(): if isinstance(trait_dict[key], bytes): @@ -336,6 +335,49 @@ class SearchResultPage: else: return None +def trait_info_str(trait, dataset_type): + """Provide a string representation for given trait""" + def __trait_desc(trt): + if dataset_type == "Geno": + return f"Marker: {trait['display_name']}" + return trait['description'] or "N/A" + + def __symbol(trt): + if dataset_type == "ProbeSet": + return (trait['symbol'] or "N/A")[:20] + + def __lrs(trt): + if dataset_type == "Geno": + return 0 + else: + if trait['lod_score'] != "N/A": + return ( + f"{float(trait['lod_score']):0.3f}" if float(trait['lod_score']) > 0 + else f"{trait['lod_score']}") + else: + return "N/A" + + def __lrs_location(trt): + if 'lrs_location' in trait: + return trait['lrs_location'] + else: + return "N/A" + + def __location(trt): + if 'location' in trait: + return trait['location'] + else: + return None + + def __mean(trt): + if 'mean' in trait: + return trait['mean'] + else: + return 0 + + return "{}|||{}|||{}|||{}|||{}|||{}|||{}|||{}".format( + trait['display_name'], trait['dataset'], __trait_desc(trait), __symbol(trait), + __location(trait), __mean(trait), __lrs(trait), __lrs_location(trait)) def get_GO_symbols(a_search): query = """SELECT genes diff --git a/wqflask/wqflask/show_trait/export_trait_data.py b/wqflask/wqflask/show_trait/export_trait_data.py index 7fabc3f6..019d4d73 100644 --- a/wqflask/wqflask/show_trait/export_trait_data.py +++ b/wqflask/wqflask/show_trait/export_trait_data.py @@ -1,3 +1,4 @@ +import datetime import simplejson as json from pprint import pformat as pf @@ -5,32 +6,43 @@ from functools import cmp_to_key from base.trait import create_trait from base import data_set - def export_sample_table(targs): sample_data = json.loads(targs['export_data']) trait_name = targs['trait_display_name'] - meta_data = get_export_metadata(targs['trait_id'], targs['dataset']) + meta_data = get_export_metadata(targs) final_sample_data = meta_data - column_headers = ["Name", "Value"] + column_headers = ["Index", "Name", "Value"] + attr_pos = 2 if any(sample["se"] for sample in sample_data['primary_samples']): column_headers.append("SE") + attr_pos = 3 if any(sample["num_cases"] for sample in sample_data['primary_samples']): column_headers.append("N") + attr_pos = 4 + + for key in sample_data["primary_samples"][0].keys(): + if key not in ["name", "value", "se", "num_cases"]: + column_headers.append(key) final_sample_data.append(column_headers) for sample_group in ['primary_samples', 'other_samples']: - for row in sample_data[sample_group]: - sorted_row = dict_to_sorted_list(row) + for i, row in enumerate(sample_data[sample_group]): + sorted_row = [i + 1] + dict_to_sorted_list(row)[:attr_pos] + for attr in sample_data['attributes']: + sorted_row.append(row[attr]) final_sample_data.append(sorted_row) return trait_name, final_sample_data -def get_export_metadata(trait_id, dataset_name): +def get_export_metadata(trait_metadata): + + trait_id, display_name, dataset_name = trait_metadata['trait_id'], trait_metadata['trait_display_name'], trait_metadata['dataset'] + dataset = data_set.create_dataset(dataset_name) this_trait = create_trait(dataset=dataset, name=trait_id, @@ -39,30 +51,34 @@ def get_export_metadata(trait_id, dataset_name): metadata = [] if dataset.type == "Publish": - metadata.append(["Phenotype ID: " + trait_id]) - metadata.append(["Phenotype URL: " + "http://genenetwork.org/show_trait?trait_id=" + \ + metadata.append(["Phenotype ID:", display_name]) + metadata.append(["Phenotype URL: ", "http://genenetwork.org/show_trait?trait_id=" + \ trait_id + "&dataset=" + dataset_name]) - metadata.append(["Group: " + dataset.group.name]) + metadata.append(["Group: ", dataset.group.name]) metadata.append( - ["Phenotype: " + this_trait.description_display.replace(",", "\",\"")]) + ["Phenotype: ", this_trait.description_display.replace(",", "\",\"")]) metadata.append( - ["Authors: " + (this_trait.authors if this_trait.authors else "N/A")]) + ["Authors: ", (this_trait.authors if this_trait.authors else "N/A")]) metadata.append( - ["Title: " + (this_trait.title if this_trait.title else "N/A")]) + ["Title: ", (this_trait.title if this_trait.title else "N/A")]) metadata.append( - ["Journal: " + (this_trait.journal if this_trait.journal else "N/A")]) + ["Journal: ", (this_trait.journal if this_trait.journal else "N/A")]) + metadata.append( - ["Dataset Link: http://gn1.genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=" + dataset.name]) + ["Dataset Link: ", "http://gn1.genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=" + dataset.name]) else: - metadata.append(["Record ID: " + trait_id]) - metadata.append(["Trait URL: " + "http://genenetwork.org/show_trait?trait_id=" + \ + metadata.append(["Record ID: ", trait_id]) + metadata.append(["Trait URL: ", "http://genenetwork.org/show_trait?trait_id=" + \ trait_id + "&dataset=" + dataset_name]) if this_trait.symbol: - metadata.append(["Symbol: " + this_trait.symbol]) - metadata.append(["Dataset: " + dataset.name]) - metadata.append(["Group: " + dataset.group.name]) + metadata.append(["Symbol: ", this_trait.symbol]) + metadata.append(["Dataset: ", dataset.name]) + metadata.append(["Group: ", dataset.group.name]) + metadata.append( + ["Export Date: ", datetime.datetime.now().strftime("%B %d, %Y")]) + metadata.append( + ["Export Time: ", datetime.datetime.now().strftime("%H:%M GMT")]) - metadata.append([]) return metadata diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index d9821d9c..c1d9ad84 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -550,21 +550,21 @@ def get_max_digits(trait_vals): return max_digits -def quantile_normalize_vals(sample_groups, trait_vals): - def normf(trait_vals): - ranked_vals = ss.rankdata(trait_vals) - p_list = [] - for i, val in enumerate(trait_vals): - p_list.append(((i + 1) - 0.5) / len(trait_vals)) +def normf(trait_vals): + ranked_vals = ss.rankdata(trait_vals) + p_list = [] + for i, val in enumerate(trait_vals): + p_list.append(((i + 1) - 0.5) / len(trait_vals)) - z = ss.norm.ppf(p_list) + z = ss.norm.ppf(p_list) - normed_vals = [] - for rank in ranked_vals: - normed_vals.append("%0.3f" % z[int(rank) - 1]) + normed_vals = [] + for rank in ranked_vals: + normed_vals.append("%0.3f" % z[int(rank) - 1]) - return normed_vals + return normed_vals +def quantile_normalize_vals(sample_groups, trait_vals): qnorm_by_group = [] for i, sample_type in enumerate(sample_groups): qnorm_vals = normf(trait_vals[i]) diff --git a/wqflask/wqflask/static/new/javascript/scatterplot.js b/wqflask/wqflask/static/new/javascript/scatterplot.js index d7f2a4a5..3fea0503 100644 --- a/wqflask/wqflask/static/new/javascript/scatterplot.js +++ b/wqflask/wqflask/static/new/javascript/scatterplot.js @@ -67,15 +67,14 @@ scatterplot = function() { x = data.data[xvar]; y = data.data[yvar]; } - console.log("x:", x); - console.log("y:", y); + indID = (_ref = data != null ? data.indID : void 0) != null ? _ref : null; indID = indID != null ? indID : (function() { _results = []; for (var _i = 1, _ref1 = x.length; 1 <= _ref1 ? _i <= _ref1 : _i >= _ref1; 1 <= _ref1 ? _i++ : _i--){ _results.push(_i); } return _results; }).apply(this); - console.log("indID:", indID); + group = (_ref2 = data != null ? data.group : void 0) != null ? _ref2 : (function() { var _j, _len, _results1; _results1 = []; diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 1b79e261..e9ea5afb 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -1113,7 +1113,7 @@ switchQNormData = function() { }; $('#qnorm').click(switchQNormData); -getSampleTableData = function(table_name) { +getSampleTableData = function(table_name, attributes_as_list) { var samples; samples = []; @@ -1123,20 +1123,29 @@ getSampleTableData = function(table_name) { if ($('#' + table_name).length){ tableApi = $('#' + table_name).DataTable(); sample_vals = []; + attr_col = 4 name_nodes = tableApi.column(2).nodes().to$(); val_nodes = tableApi.column(3).nodes().to$(); if (js_data.se_exists){ var_nodes = tableApi.column(5).nodes().to$(); + attr_col = 6 if (js_data.has_num_cases) { n_nodes = tableApi.column(6).nodes().to$(); + attr_col = 7 } } else { if (js_data.has_num_cases){ n_nodes = tableApi.column(4).nodes().to$(); + attr_col = 5 } } + attribute_nodes = [] + for (_i = 0; _i < attributes_as_list.length; _i++){ + attribute_nodes.push(table_api.column(attr_col + _i).nodes().to$()) + } + for (_j = 0; _j < val_nodes.length; _j++){ sample_val = val_nodes[_j].childNodes[0].value sample_name = $.trim(name_nodes[_j].childNodes[0].textContent) @@ -1164,12 +1173,18 @@ getSampleTableData = function(table_name) { sample_n = null; } } + row_dict = { name: sample_name, value: sample_val, se: sample_var, num_cases: sample_n } + + for (_k = 0; _k < attribute_nodes.length; _k++){ + row_dict[attributes_as_list[_k]] = attribute_nodes[_k][_j].textContent; + } + samples.push(row_dict) } } @@ -1179,9 +1194,15 @@ getSampleTableData = function(table_name) { }; exportSampleTableData = function() { var format, json_sample_data, sample_data; + + var attributes_as_list = Object.keys(js_data.attributes).map(function(key) { + return js_data.attributes[key].name; + }); + sample_data = {}; - sample_data.primary_samples = getSampleTableData('samples_primary'); - sample_data.other_samples = getSampleTableData('samples_other'); + sample_data.primary_samples = getSampleTableData('samples_primary', attributes_as_list); + sample_data.other_samples = getSampleTableData('samples_other', attributes_as_list); + sample_data.attributes = attributes_as_list; json_sample_data = JSON.stringify(sample_data); $('input[name=export_data]').val(json_sample_data); format = $('input[name=export_format]').val(); diff --git a/wqflask/wqflask/submit_bnw.py b/wqflask/wqflask/submit_bnw.py deleted file mode 100644 index b21a88cc..00000000 --- a/wqflask/wqflask/submit_bnw.py +++ /dev/null @@ -1,10 +0,0 @@ -from base.trait import GeneralTrait -from base import data_set -from utility import helper_functions - -import utility.logger -logger = utility.logger.getLogger(__name__) - - -def get_bnw_input(start_vars): - logger.debug("BNW VARS:", start_vars) diff --git a/wqflask/wqflask/templates/bnw_page.html b/wqflask/wqflask/templates/bnw_page.html index 317b4bd7..a3e090a5 100644 --- a/wqflask/wqflask/templates/bnw_page.html +++ b/wqflask/wqflask/templates/bnw_page.html @@ -1,5 +1,5 @@ <title>Opening BNW</title> -<form method="post" action="https://bnw.genenetwork.org/sourcecodes/bn_genenet.php" name="bnwform" id="bnwform"> +<form method="post" action="http://bnw.genenetwork.org/sourcecodes/bn_genenet.php" name="bnwform" id="bnwform"> <input type="hidden" name="My_Genenet" value="{{ form_value }}"> </form> <script type="text/javascript"> diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index a9c84cb0..4b63453e 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -20,7 +20,7 @@ <h3>This collection has {{ '{}'.format(numify(trait_obs|count, "record", "records")) }}</h3> <div class="tool-button-container"> - <form id="collection_form" action="/delete" method="post" target="_blank"> + <form id="collection_form" action="/delete" method="post"> <input type="hidden" name="uc_id" id="uc_id" value="{{ uc.id }}" /> <input type="hidden" name="collection_name" id="collection_name" value="{{ uc.name }}" /> <input type="hidden" name="tool_used" value="" /> @@ -309,13 +309,14 @@ submit_special = function(url) { $("#collection_form").attr("action", url); - e.stopImmediatePropagation(); - return $("#collection_form").submit(); + $("#collection_form").attr('target', '_blank').submit(); + return false; }; $("#delete").on("click", function() { url = $(this).data("url") - return submit_special(url) + $("#collection_form").attr("action", url); + return $("#collection_form").removeAttr('target').submit(); }); $("#remove").on("click", function() { @@ -324,8 +325,8 @@ return $(this).val(); }).get(); $("#trait_list").val(traits) - - return submit_special(url) + $("#collection_form").attr("action", url); + return $("#collection_form").removeAttr('target').submit(); }); $("#change_collection_name").on("click", function() { diff --git a/wqflask/wqflask/templates/correlation_matrix.html b/wqflask/wqflask/templates/correlation_matrix.html index e90accf2..d392be34 100644 --- a/wqflask/wqflask/templates/correlation_matrix.html +++ b/wqflask/wqflask/templates/correlation_matrix.html @@ -11,7 +11,7 @@ <div class="container" width="100%"> <h1>Correlation Matrix</h1> -<div style="width: 100%; max-width: 850px;">Lower left cells list Pearson product-moment correlations; upper right cells list Spearman rank order correlations. Each cell also contains the n of cases. Values ranging from 0.4 to 1.0 range from orange to white, while values ranging from -0.4 to -1.0 range from dark blue to white. Select any cell to generate a scatter plot. Select trait labels for more information.</div> +<div style="width: 100%; max-width: 850px;">Select any cell in the matrix to generate a <b>scatter plot.</b><br><b>Lower left</b> cells list Pearson product-moment correlations; <b>upper right</b> cells list Spearman rank order correlations. Each cell also contains the <b><i>n</i> of cases</b> in parenthesis. Values ranging from 0.4 to 1.0 range from orange to white, while values ranging from –0.4 to –1.0 range from dark blue to white.</div> <hr style="height: 1px; background-color: #A9A9A9;"> {% if lowest_overlap < 8 %} <div style="margin-bottom: 10px;"><i><font style="color: red;">Caution</font>: This matrix of correlations contains some cells with small sample sizes of fewer than 8.</i></div> @@ -37,7 +37,7 @@ <tr> <td align="center"><input type="checkbox" class="checkbox" style="margin-left: 3px; margin-right: 1px;"></td> <td align="right" style="padding-right: 4px;" > - <a href="/show_trait?trait_id={{ trait.name }}&dataset={{ trait.dataset.name }}"><font style="font-size: 14px; font-style: Bold;"><b>Trait {{ loop.index }}: {{ trait.dataset.name }} {{ trait.name }}</b></font></a> + <a href="/show_trait?trait_id={{ trait.name }}&dataset={{ trait.dataset.name }}"><font style="font-size: 14px; font-style: Bold;"><b>{{ loop.index }}: {{ trait.dataset.name }} {{ trait.name }}</b></font></a> <div class="shortName">{% if trait.dataset.type == "ProbeSet" %}Gene Symbol: {{ trait.symbol }}{% elif trait.dataset.type == "Publish" %}Trait Symbol: {{ trait.post_publication_abbreviation }}{% elif trait.dataset.type == "Geno" %}Genotype{% endif %} </div> <div class="verboseName" style="display: none;"> {% if trait.dataset.type == "ProbeSet" %} @@ -56,7 +56,7 @@ {% if result[1] == 0 %} <td nowrap="ON" align="middle" bgcolor="#eeeeee" style="padding: 3px; line-height: 1.1;">N/A</td> {% else %} - <td nowrap="ON" align="middle" class="corr_cell" style="padding: 3px; line-height: 1.1;"><a href="/corr_scatter_plot?method={% if loop.index > outer_loop %}spearman{% else %}pearson{% endif %}&dataset_1={% if trait.dataset.name == 'Temp' %}Temp_{{ trait.dataset.group.name }}{% else %}{{ trait.dataset.name }}{% endif %}&dataset_2={% if result[0].dataset.name == 'Temp' %}Temp_{{ result[0].dataset.group.name }}{% else %}{{ result[0].dataset.name }}{% endif %}&trait_1={{ trait.name }}&trait_2={{ result[0].name }}"><font style="font-size: 12px; color: #3071a9; font-weight: bold;" ><span class="corr_value">{{ '%0.2f' % result[1] }}</span><br>{{ result[2] }}</font></a></td> + <td nowrap="ON" align="middle" class="corr_cell" style="padding: 3px; line-height: 1.1;"><a href="/corr_scatter_plot?method={% if loop.index > outer_loop %}spearman{% else %}pearson{% endif %}&dataset_1={% if trait.dataset.name == 'Temp' %}Temp_{{ trait.dataset.group.name }}{% else %}{{ trait.dataset.name }}{% endif %}&dataset_2={% if result[0].dataset.name == 'Temp' %}Temp_{{ result[0].dataset.group.name }}{% else %}{{ result[0].dataset.name }}{% endif %}&trait_1={{ trait.name }}&trait_2={{ result[0].name }}"><span class="corr_value" style="font-size: 14px; color: #3071a9; font-weight: bold;">{{ '%0.2f' % result[1] }}</span><br><span class="corr_value" style="font-size: 12px; color: #3071a9; font-weight: bold;">({{ result[2] }})</span></a></td> {% endif %} {% endif %} {% endfor %} @@ -72,12 +72,11 @@ </form> <br> -<div style="margin:20x;"> +{% if pca_works == "True" %} +<div> {% include 'pca_scree_plot.html' %} - </div> -{% if pca_works == "True" %} <h2>PCA Traits</h2> <div style="margin-bottom: 20px; overflow:hidden; width: 450px;"> <table id="pca_table" class="table-hover table-striped cell-border" id='trait_table' style="float: left;"> diff --git a/wqflask/wqflask/templates/edit_history.html b/wqflask/wqflask/templates/edit_history.html index f181fd87..7538324b 100644 --- a/wqflask/wqflask/templates/edit_history.html +++ b/wqflask/wqflask/templates/edit_history.html @@ -9,7 +9,7 @@ <!-- Start of body --> <div class="container"> - <h1>Edit history: {{}}</h1> + <h1>History</h1> {% if diff %} <div class="row"> <table id="history" class="table-responsive table-hover table-striped cell-border"> diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index a0b7a6eb..feeadadb 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -17,53 +17,6 @@ <small><a href="{{url_for('metadata_edit.show_history', dataset_id=dataset_id, name=name)}}" target="_blank">[View History]</a></small> </div> - {% if diff %} - - <div class="container text-left"> - <details class="col-md-12 col-lg-20 col-lg-22"> - <summary> - <h2>Update History</h2> - </summary> - <table class="table"> - <tbody> - <tr> - <th>Timestamp</th> - <th>Editor</th> - <th>Field</th> - <th>Diff</th> - </tr> - {% set ns = namespace(display_cell=True) %} - - {% for timestamp, group in diff %} - {% set ns.display_cell = True %} - {% for i in group %} - <tr> - {% if ns.display_cell and i.timestamp == timestamp %} - - {% set author = i.author %} - {% set timestamp_ = i.timestamp %} - - {% else %} - - {% set author = "" %} - {% set timestamp_ = "" %} - - {% endif %} - <td>{{ timestamp_ }}</td> - <td>{{ author }}</td> - <td>{{ i.diff.field }}</td> - <td><pre>{{ i.diff.diff }}</pre></td> - {% set ns.display_cell = False %} - </tr> - {% endfor %} - {% endfor %} - </tbody> - </table> - </details> - - </div> - - {% endif %} <form id="edit-form" class="container form-horizontal" method="post" action="/datasets/{{dataset_id}}/traits/{{ publish_xref.id_ }}?resource-id={{ resource_id }}" enctype='multipart/form-data'> <div class="form-group"> <div class="controls left-block col-sm-8 col-lg-8" style="width: max-content;"> diff --git a/wqflask/wqflask/templates/edit_probeset.html b/wqflask/wqflask/templates/edit_probeset.html index ab91b701..6c068e63 100644 --- a/wqflask/wqflask/templates/edit_probeset.html +++ b/wqflask/wqflask/templates/edit_probeset.html @@ -2,226 +2,187 @@ {% block title %}Trait Submission{% endblock %} {% block content %} <!-- Start of body --> -Edit Trait for Probeset -Submit Trait | Reset - -{% if diff %} - <div class="container"> - <details class="col-sm-12 col-md-10 col-lg-12"> - <summary> - <h2>Update History</h2> - </summary> - <table class="table"> - <tbody> - <tr> - <th>Timestamp</th> - <th>Editor</th> - <th>Field</th> - <th>Diff</th> - </tr> - {% set ns = namespace(display_cell=True) %} - - {% for timestamp, group in diff %} - {% set ns.display_cell = True %} - {% for i in group %} - <tr> - {% if ns.display_cell and i.timestamp == timestamp %} - - {% set author = i.author %} - {% set timestamp_ = i.timestamp %} - - {% else %} - - {% set author = "" %} - {% set timestamp_ = "" %} - - {% endif %} - <td>{{ timestamp_ }}</td> - <td>{{ author }}</td> - <td>{{ i.diff.field }}</td> - <td><pre>{{ i.diff.diff }}</pre></td> - {% set ns.display_cell = False %} - </tr> - {% endfor %} - {% endfor %} - </tbody> - </table> - </details> - + <div class="page-header text-left"> + <h1>Probeset Editing Form: {{ probeset.name }}</h1> + <small><a href="{{url_for('metadata_edit.show_history', name=name)}}" target="_blank">[View History]</a></small> + </div> + <form id="edit-form" class="container form-horizontal" method="post" action="/datasets/traits/{{ name }}?resource-id={{ resource_id }}" enctype='multipart/form-data'> + <div class="form-group"> + <div class="controls left-block col-sm-8 col-lg-8" style="width: max-content;"> + <input name="id" class="changed" type="hidden" value="{{ probeset.id_ }}"/> + <input name="old_id_" class="changed" type="hidden" value="{{ probeset.id_ }}"/> + <input name="probeset_name" class="changed" type="hidden" value="{{ probeset.name }}"/> + <input type="submit" style="width: 125px; margin-right: 25px;" class="btn btn-success form-control col-xs-2 changed" value="Submit Change"> + </div> + </div> + <div class="form-group"> + <label for="symbol" class="col-sm-3 col-lg-2 control-label text-left">Symbol</label> + <div class="col-sm-4"> + <textarea name="symbol" class="form-control" rows="1">{{ probeset.symbol |default('', true) }}</textarea> + <input name="old_symbol" class="changed" type="hidden" value="{{ probeset.symbol |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="description" class="col-sm-3 col-lg-2 control-label text-left">Description</label> + <div class="col-sm-4"> + <textarea name="description" class="form-control" rows="3">{{ probeset.description |default('', true) }}</textarea> + <input name="old_description" class="changed" type="hidden" value="{{ probeset.description |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_target_description" class="col-sm-3 col-lg-2 control-label text-left">Probe Target Description</label> + <div class="col-sm-4"> + <textarea name="probe_target_description" class="form-control" rows="4">{{ probeset.probe_target_description |default('', true) }}</textarea> + <input name="old_probe_target_description" class="changed" type="hidden" value="{{ probeset.probe_target_description |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="chr" class="col-sm-3 col-lg-2 control-label text-left">Chr</label> + <div class="col-sm-4"> + <textarea name="chr" class="form-control" rows="1">{{ probeset.chr_ |default('', true) }}</textarea> + <input name="old_chr_" class="changed" type="hidden" value="{{ probeset.chr_ |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="mb" class="col-sm-3 col-lg-2 control-label text-left">Mb</label> + <div class="col-sm-4"> + <textarea name="mb" class="form-control" rows="1">{{ probeset.mb |default('', true) }}</textarea> + <input name="old_mb" class="changed" type="hidden" value="{{ probeset.mb |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="alias" class="col-sm-3 col-lg-2 control-label text-left"> + Alias: + </label> + <div class="col-sm-4"> + <textarea name="alias" class="form-control" rows="1">{{ probeset.alias |default('', true) }}</textarea> + <input name="old_alias" class="changed" type="hidden" value="{{ probeset.alias |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="geneid" class="col-sm-3 col-lg-2 control-label text-left"> + Gene Id: + </label> + <div class="col-sm-4"> + <textarea name="geneid" class="form-control" rows="1">{{ probeset.geneid |default('', true) }}</textarea> + <input name="old_geneid" class="changed" type="hidden" value="{{ probeset.geneid |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="homologeneid" class="col-sm-3 col-lg-2 control-label text-left"> + Homolegene Id: + </label> + <div class="col-sm-4"> + <textarea name="homologeneid" class="form-control" rows="1">{{ probeset.homologeneid |default('', true) }}</textarea> + <input name="old_homologeneid" class="changed" type="hidden" value="{{ probeset.homologeneid |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="unigeneid" class="col-sm-3 col-lg-2 control-label text-left"> + Unigene Id: + </label> + <div class="col-sm-4"> + <textarea name="unigeneid" class="form-control" rows="1">{{ probeset.unigeneid |default('', true) }}</textarea> + <input name="old_unigeneid" class="changed" type="hidden" value="{{ probeset.unigeneid |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="omim" class="col-sm-3 col-lg-2 control-label text-left">OMIM</label> + <div class="col-sm-4"> + <textarea name="omim" class="form-control" rows="1">{{ probeset.omim |default('', true) }}</textarea> + <input name="old_omim" class="changed" type="hidden" value="{{ probeset.omim |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="refseq_transcriptid" class="col-sm-3 col-lg-2 control-label text-left"> + Refseq TranscriptId: + </label> + <div class="col-sm-4"> + <textarea name="refseq_transcriptid" class="form-control" rows="1">{{ probeset.refseq_transcriptid |default('', true) }}</textarea> + <input name="old_refseq_transcriptid" class="changed" type="hidden" value="{{ probeset.refseq_transcriptid |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="blatseq" class="col-sm-3 col-lg-2 control-label text-left">BlatSeq</label> + <div class="col-sm-8"> + <textarea name="blatseq" class="form-control" rows="6">{{ probeset.blatseq |default('', true) }}</textarea> + <input name="old_blatseq" class="changed" type="hidden" value="{{ probeset.blatseq |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="targetseq" class="col-sm-3 col-lg-2 control-label text-left">TargetSeq</label> + <div class="col-sm-8"> + <textarea name="targetseq" class="form-control" rows="6">{{ probeset.targetseq |default('', true) }}</textarea> + <input name="old_targetseq" class="changed" type="hidden" value="{{ probeset.targetseq |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="strand_probe" class="col-sm-3 col-lg-2 control-label text-left">Strand Probe</label> + <div class="col-sm-2"> + <textarea name="strand_probe" class="form-control" rows="1">{{ probeset.strand_probe |default('', true) }}</textarea> + <input name="old_strand_probe" class="changed" type="hidden" value="{{ probeset.strand_probe |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_target_region" class="col-sm-3 col-lg-2 control-label text-left">Probe Set Target Region</label> + <div class="col-sm-8"> + <textarea name="probe_set_target_region" class="form-control" rows="1">{{ probeset.probe_set_target_region |default('', true) }}</textarea> + <input name="old_probe_set_target_region" class="changed" type="hidden" value="{{ probeset.probe_set_target_region_ |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_specificity" class="col-sm-3 col-lg-2 control-label text-left">Probeset Specificity</label> + <div class="col-sm-8"> + <textarea name="probe_set_specificity" class="form-control" rows="1">{{ probeset.probe_set_specificity |default('', true) }}</textarea> + <input name="old_probe_set_specificity" class="changed" type="hidden" value="{{ probeset.probe_set_specificity |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_blat_score" class="col-sm-3 col-lg-2 control-label text-left">Probeset Blat Score</label> + <div class="col-sm-8"> + <textarea name="probe_set_blat_score" class="form-control" rows="1">{{ probeset.probe_set_blat_score |default('', true) }}</textarea> + <input name="old_probe_set_blat_score" class="changed" type="hidden" value="{{ probeset.probe_set_blat_score |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_blat_mb_start" class="col-sm-3 col-lg-2 control-label text-left"> + Probeset Blat Mb Start</label> + <div class="col-sm-8"> + <textarea name="probe_set_blat_mb_start" class="form-control" rows="1">{{ probeset.probe_set_blat_mb_start |default('', true) }}</textarea> + <input name="old_probe_set_blat_mb_start" class="changed" type="hidden" value="{{ probeset.probe_set_blat_mb_start |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_blat_mb_end" class="col-sm-3 col-lg-2 control-label text-left">Probeset Blat Mb End</label> + <div class="col-sm-8"> + <textarea name="probe_set_blat_mb_end" class="form-control" rows="6">{{ probeset.probe_set_blat_mb_end |default('', true) }}</textarea> + <input name="old_probe_set_blat_mb_end" class="changed" type="hidden" value="{{ probeset.probe_set_blat_mb_end |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_strand" class="col-sm-3 col-lg-2 control-label text-left">Probeset Strand</label> + <div class="col-sm-8"> + <textarea name="probe_set_strand" class="form-control" rows="6">{{ probeset.probe_set_strand |default('', true) }}</textarea> + <input name="old_probe_set_strand" class="changed" type="hidden" value="{{ probeset.probe_set_strand |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_note_by_rw" class="col-sm-3 col-lg-2 control-label text-left">Probeset Strand</label> + <div class="col-sm-8"> + <textarea name="probe_set_note_by_rw" class="form-control" rows="6">{{ probeset.probe_set_note_by_rw |default('', true) }}</textarea> + <input name="old_probe_set_note_by_rw" class="changed" type="hidden" value="{{ probeset.probe_set_note_by_rw |default('', true) }}"/> + </div> + </div> + <div class="controls left-block col-sm-8 col-lg-8" style="width: max-content;"> + <input name="id" class="changed" type="hidden" value="{{ probeset.id_ }}"/> + <input name="old_id_" class="changed" type="hidden" value="{{ probeset.id_ }}"/> + <input name="probeset_name" class="changed" type="hidden" value="{{ probeset.name }}"/> + <input type="submit" style="width: 125px; margin-right: 25px;" class="btn btn-success form-control col-xs-2 changed" value="Submit Change"> + </div> + </form> </div> -{% endif %} - -<form id="edit-form" class="form-horizontal" method="post" action="/datasets/traits/{{ name }}?resource-id={{ resource_id }}"> - <h2 class="text-center">Probeset Information:</h2> - <div class="form-group"> - <label for="symbol" class="col-sm-2 control-label">Symbol:</label> - <div class="col-sm-4"> - <textarea name="symbol" class="form-control" rows="1">{{ probeset.symbol |default('', true) }}</textarea> - <input name="old_symbol" class="changed" type="hidden" value="{{ probeset.symbol |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="description" class="col-sm-2 control-label">Description:</label> - <div class="col-sm-5"> - <textarea name="description" class="form-control" rows="3">{{ probeset.description |default('', true) }}</textarea> - <input name="old_description" class="changed" type="hidden" value="{{ probeset.description |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="probe_target_description" class="col-sm-2 control-label">Probe Target Description:</label> - <div class="col-sm-4"> - <textarea name="probe_target_description" class="form-control" rows="4">{{ probeset.probe_target_description |default('', true) }}</textarea> - <input name="old_probe_target_description" class="changed" type="hidden" value="{{ probeset.probe_target_description |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="chr" class="col-sm-2 control-label">Chr:</label> - <div class="col-sm-4"> - <textarea name="chr" class="form-control" rows="1">{{ probeset.chr_ |default('', true) }}</textarea> - <input name="old_chr_" class="changed" type="hidden" value="{{ probeset.chr_ |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="mb" class="col-sm-2 control-label">Mb:</label> - <div class="col-sm-4"> - <textarea name="mb" class="form-control" rows="1">{{ probeset.mb |default('', true) }}</textarea> - <input name="old_mb" class="changed" type="hidden" value="{{ probeset.mb |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="alias" class="col-sm-2 control-label"> - Alias: - </label> - <div class="col-sm-4"> - <textarea name="alias" class="form-control" rows="1">{{ probeset.alias |default('', true) }}</textarea> - <input name="old_alias" class="changed" type="hidden" value="{{ probeset.alias |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="geneid" class="col-sm-2 control-label"> - Gene Id: - </label> - <div class="col-sm-4"> - <textarea name="geneid" class="form-control" rows="1">{{ probeset.geneid |default('', true) }}</textarea> - <input name="old_geneid" class="changed" type="hidden" value="{{ probeset.geneid |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="homologeneid" class="col-sm-2 control-label"> - Homolegene Id: - </label> - <div class="col-sm-4"> - <textarea name="homologeneid" class="form-control" rows="1">{{ probeset.homologeneid |default('', true) }}</textarea> - <input name="old_homologeneid" class="changed" type="hidden" value="{{ probeset.homologeneid |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="unigeneid" class="col-sm-2 control-label"> - Unigene Id: - </label> - <div class="col-sm-4"> - <textarea name="unigeneid" class="form-control" rows="1">{{ probeset.unigeneid |default('', true) }}</textarea> - <input name="old_unigeneid" class="changed" type="hidden" value="{{ probeset.unigeneid |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="omim" class="col-sm-2 control-label">OMIM:</label> - <div class="col-sm-4"> - <textarea name="omim" class="form-control" rows="1">{{ probeset.omim |default('', true) }}</textarea> - <input name="old_omim" class="changed" type="hidden" value="{{ probeset.omim |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="refseq_transcriptid" class="col-sm-2 control-label"> - Refseq TranscriptId: - </label> - <div class="col-sm-4"> - <textarea name="refseq_transcriptid" class="form-control" rows="1">{{ probeset.refseq_transcriptid |default('', true) }}</textarea> - <input name="old_refseq_transcriptid" class="changed" type="hidden" value="{{ probeset.refseq_transcriptid |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="blatseq" class="col-sm-2 control-label">BlatSeq:</label> - <div class="col-sm-8"> - <textarea name="blatseq" class="form-control" rows="6">{{ probeset.blatseq |default('', true) }}</textarea> - <input name="old_blatseq" class="changed" type="hidden" value="{{ probeset.blatseq |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="targetseq" class="col-sm-2 control-label">TargetSeq:</label> - <div class="col-sm-8"> - <textarea name="targetseq" class="form-control" rows="6">{{ probeset.targetseq |default('', true) }}</textarea> - <input name="old_targetseq" class="changed" type="hidden" value="{{ probeset.targetseq |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="strand_probe" class="col-sm-2 control-label">Strand Probe:</label> - <div class="col-sm-2"> - <textarea name="strand_probe" class="form-control" rows="1">{{ probeset.strand_probe |default('', true) }}</textarea> - <input name="old_strand_probe" class="changed" type="hidden" value="{{ probeset.strand_probe |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="probe_set_target_region" class="col-sm-2 control-label">Probe Set Target Region:</label> - <div class="col-sm-8"> - <textarea name="probe_set_target_region" class="form-control" rows="1">{{ probeset.probe_set_target_region |default('', true) }}</textarea> - <input name="old_probe_set_target_region" class="changed" type="hidden" value="{{ probeset.probe_set_target_region_ |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="probe_set_specificity" class="col-sm-2 control-label">Probeset Specificity:</label> - <div class="col-sm-8"> - <textarea name="probe_set_specificity" class="form-control" rows="1">{{ probeset.probe_set_specificity |default('', true) }}</textarea> - <input name="old_probe_set_specificity" class="changed" type="hidden" value="{{ probeset.probe_set_specificity |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="probe_set_blat_score" class="col-sm-2 control-label">Probeset Blat Score:</label> - <div class="col-sm-8"> - <textarea name="probe_set_blat_score" class="form-control" rows="1">{{ probeset.probe_set_blat_score |default('', true) }}</textarea> - <input name="old_probe_set_blat_score" class="changed" type="hidden" value="{{ probeset.probe_set_blat_score |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="probe_set_blat_mb_start" class="col-sm-2 control-label"> - Probeset Blat Mb Start:</label> - <div class="col-sm-8"> - <textarea name="probe_set_blat_mb_start" class="form-control" rows="1">{{ probeset.probe_set_blat_mb_start |default('', true) }}</textarea> - <input name="old_probe_set_blat_mb_start" class="changed" type="hidden" value="{{ probeset.probe_set_blat_mb_start |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="probe_set_blat_mb_end" class="col-sm-2 control-label">Probeset Blat Mb End:</label> - <div class="col-sm-8"> - <textarea name="probe_set_blat_mb_end" class="form-control" rows="6">{{ probeset.probe_set_blat_mb_end |default('', true) }}</textarea> - <input name="old_probe_set_blat_mb_end" class="changed" type="hidden" value="{{ probeset.probe_set_blat_mb_end |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="probe_set_strand" class="col-sm-2 control-label">Probeset Strand:</label> - <div class="col-sm-8"> - <textarea name="probe_set_strand" class="form-control" rows="6">{{ probeset.probe_set_strand |default('', true) }}</textarea> - <input name="old_probe_set_strand" class="changed" type="hidden" value="{{ probeset.probe_set_strand |default('', true) }}"/> - </div> - </div> - <div class="form-group"> - <label for="probe_set_note_by_rw" class="col-sm-2 control-label">Probeset Strand:</label> - <div class="col-sm-8"> - <textarea name="probe_set_note_by_rw" class="form-control" rows="6">{{ probeset.probe_set_note_by_rw |default('', true) }}</textarea> - <input name="old_probe_set_note_by_rw" class="changed" type="hidden" value="{{ probeset.probe_set_note_by_rw |default('', true) }}"/> - </div> - </div> - <div class="controls" style="display:block; margin-left: 40%; margin-right: 20%;"> - <input name="id" class="changed" type="hidden" value="{{ probeset.id_ }}"/> - <input name="old_id_" class="changed" type="hidden" value="{{ probeset.id_ }}"/> - <input name="probeset_name" class="changed" type="hidden" value="{{ probeset.name }}"/> - <input type="submit" style="width: 125px; margin-right: 25px;" class="btn btn-primary form-control col-xs-2 changed" value="Submit Change"> - <input type="reset" style="width: 110px;" class="btn btn-primary form-control col-xs-2 changed" onClick="window.location.reload();" value="Reset"> - </div> -</form> - {%endblock%} {% block js %} diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index af0e2fa2..2b72d6b2 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -59,7 +59,7 @@ <div class="col-xs-4" style="margin-right:50px; min-width: 530px; max-width: 550px;"> <section id="search"> <div class="page-header"> - <h2>Select and search</h2> + <h2>Select and Search</h2> </div> <form method="get" action="/search" target="_blank" name="SEARCHFORM", data-gn_server_url="{{gn_server_url}}"> @@ -148,7 +148,7 @@ <input id="btsearch" type="submit" class="btn btn-primary form-control" value="Search"> </div> <div class="col-2 controls" style="padding-left: 20px;"> - <button type="button" id="make_default" class="btn btn-primary form-control">Make Default</button> + <button type="button" id="make_default" class="btn btn-primary form-control">Save Settings</button> </div> </div> </div> @@ -164,7 +164,7 @@ </section> <section id="advanced"> <div class="page-header"> - <h2>Advanced commands</h2> + <h2>Advanced Commands</h2> </div> <p>You can also use advanced commands. Copy these simple examples into the Get Any field for single term searches and Combined for searches with multiple terms:</p> diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html index fc60aa3e..fe7f8cd4 100644 --- a/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html +++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html @@ -29,6 +29,8 @@ {%endwith%} <input type="hidden" value="{{trait_list_str}}" name="trait_list"> + <h1>Partial Correlation</h1> + <div>Please select one primary trait, one to three control traits, and at least one target trait.</div> <br /> <table id="pcorrs_traits_table" class="table-hover table-striped cell-border dataTable" @@ -106,16 +108,8 @@ <label for="corr-method-input" class="form-label">Compute</label> <select id="corr-method-input" required="required" name="method" class="form-control"> - <option value="Genetic Correlation, Pearson's r"> - Genetic Correlation, Pearson's r</option> - <option value="Genetic Correlation, Spearman's rho"> - Genetic Correlation, Spearman's rho</option> - <option value="SGO Literature Correlation"> - SGO Literature Correlation</option> - <option value="Tissue Correlation, Pearson's r"> - Tissue Correlation, Pearson's r</option> - <option value="Tissue Correlation, Spearman's rho"> - Tissue Correlation, Spearman's rho</option> + <option value="Pearson's r">Pearson's r</option> + <option value="Spearman's rho">Spearman's rho</option> </select> </div> diff --git a/wqflask/wqflask/templates/pca_scree_plot.html b/wqflask/wqflask/templates/pca_scree_plot.html index a9a6e761..74eb2c15 100644 --- a/wqflask/wqflask/templates/pca_scree_plot.html +++ b/wqflask/wqflask/templates/pca_scree_plot.html @@ -9,7 +9,8 @@ <body> <div> - + <h2>Scree Plot</h2> + <div style="padding-bottom: 10px;">Review more on <a href="https://en.wikipedia.org/wiki/Scree_plot">scree plots</a>.</div> <div id="scree_plot" style="width:700px;height:600px;"></div> </div> </body> @@ -21,19 +22,9 @@ let { x_coord, y_coord } = js_data["scree_data"] const layout = { - - title: { - text: "<b>PCA Scree Plot</b>", - font: { - "size": 24, - "family": "Arial", - "color": "#000000" - } - }, - yaxis: { title: { - text: "Percent of total variance %", + text: "% of Variance", font: { "size": 18, "color": "" @@ -44,7 +35,7 @@ const layout = { xaxis: { title: { - text: "PCA components", + text: "Principal Components", font: { "size": 18, "color": "" diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index bd832420..9cbff200 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -11,49 +11,54 @@ {% endblock %} {% block content %} <!-- Start of body --> - <div style="padding-left: 10px;"> + <div style="margin-left: 20px;"> <input type="hidden" name="uc_id" id="uc_id" value="{{ uc_id }}"> <div style="padding-top: 10px; padding-bottom: 10px; font-size: 16px;"> <!-- Need to customize text more for other types of searches --> - <p><b>Search Results:</b> We searched <a href="http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&{% if dataset.accession_id != 'None' %}GN_AccessionId={{ dataset.accession_id }}{% else %}InfoPageName={{ dataset.name }}{% endif %}">{{ dataset.fullname }}</a> - to find all records + <p>We searched <a href="http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&{% if dataset.accession_id != 'None' %}GN_AccessionId={{ dataset.accession_id }}{% else %}InfoPageName={{ dataset.name }}{% endif %}">{{ dataset.fullname }}</a> + <br> + to find all records {% if go_term is not none %} - with <u>Gene Ontology ID</u> <strong>GO:{{ go_term }}</strong>. + with <span style="font-family: Courier New"><b>Gene Ontology ID</b></span> <strong>GO:{{ go_term }}</strong>. {% else %} {% for word in search_terms %} {% if word.key|lower == "rif" %} - with <u>GeneRIF</u> containing <strong>{{ word.search_term[0] }}</strong>{% if loop.last %}.{% else %} and {% endif %} + with <span style="font-family: Courier New"><b>GeneRIF</b></span> containing <strong>{{ word.search_term[0] }}</strong>{% if loop.last %}{% else %} and {% endif %} {% elif word.key|lower == "go" %} - with <u>Gene Ontology ID</u> <strong>{{ word.search_term[0] }}</strong>{% if loop.last %}.{% else %} and {% endif %} + with <span style="font-family: Courier New"><b>Gene Ontology ID</b></span> <strong>{{ word.search_term[0] }}</strong>{% if loop.last %}{% else %} and {% endif %} {% elif word.key|lower == "wiki" %} - with <u>GeneWiki</u> containing <strong>{{ word.search_term[0] }}</strong>{% if loop.last %}.{% else %} and {% endif %} + with <span style="font-family: Courier New"><b>GeneWiki</b></span> containing <strong>{{ word.search_term[0] }}</strong>{% if loop.last %}{% else %} and {% endif %} {% elif word.key|lower == "mean" %} - with <u>MEAN</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong>{% if loop.last %}.{% else %} and {% endif %} + with <span style="font-family: Courier New"><b>mean</b></span> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong>{% if loop.last %}{% else %} and {% endif %} {% elif word.key|lower == "range" %} - with <u>RANGE</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong>{% if loop.last %}.{% else %} and {% endif %} + with <span style="font-family: Courier New"><b>RANGE</b></span> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong>{% if loop.last %}{% else %} and {% endif %} {% elif word.key|lower == "lrs" or word.key|lower == "lod" or word.key|lower == "translrs" or word.key|lower == "cislrs" or word.key|lower == "translod" or word.key|lower == "cislod" %} {% if word.search_term|length == 1 %} - with {% if word.key|lower == "translrs" %}trans{% elif word.key|lower == "cislrs" %}cis{% endif %}LRS {% if word.separator == ">" %} greater than {% elif word.separator == "<" %} less than {% elif word.separator == ">=" %} greater than or equal to {% elif word.separator == "<=" %} less than or equal to {% endif %} <strong>{{ word.search_term[0] }}</strong>{% if loop.last %}.{% else %} and {% endif %} + with {% if word.key|lower == "translrs" %}trans{% elif word.key|lower == "cislrs" %}cis{% endif %}LRS {% if word.separator == ">" %} greater than {% elif word.separator == "<" %} less than {% elif word.separator == ">=" %} greater than or equal to {% elif word.separator == "<=" %} less than or equal to {% endif %} <strong>{{ word.search_term[0] }}</strong>{% if loop.last %}{% else %} and {% endif %} {% elif word.search_term|length == 2 %} - with <u>{{ word.key|upper }}</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong>{% if loop.last %}.{% else %} and {% endif %} + with <span style="font-family: Courier New"><b>{{ word.key|upper }}</b></span> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong>{% if loop.last %}{% else %} and {% endif %} {% elif word.search_term|length == 3 %} - with <u>{{ word.key|upper }}</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[2] }}</strong>{% if loop.last %}.{% else %} and {% endif %} + with <span style="font-family: Courier New"><b>{{ word.key|upper }}</b></span> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[2] }}</strong>{% if loop.last %}{% else %} and {% endif %} {% elif word.search_term|length == 4 %} - with <u>{{ word.key|upper }}</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[3] }}</strong> with an exclusion zone of <strong>{{ word.search_term[2] }}</strong> Mb + with <span style="font-family: Courier New"><b>{{ word.key|upper }}</b></span> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[3] }}</strong> with an exclusion zone of <strong>{{ word.search_term[2] }}</strong> Mb {% elif word.search_term|length == 5 %} - with <u>{{ word.key|upper }}</u> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[2] }}</strong> between <strong>{{ word.search_term[3] }}</strong> and <strong>{{ word.search_term[4] }}</strong> Mb{% if loop.last %}.{% else %} and {% endif %} + with <span style="font-family: Courier New"><b>{{ word.key|upper }}</b></span> between <strong>{{ word.search_term[0] }}</strong> and <strong>{{ word.search_term[1] }}</strong> on chromosome <strong>{{ word.search_term[2] }}</strong> between <strong>{{ word.search_term[3] }}</strong> and <strong>{{ word.search_term[4] }}</strong> Mb{% if loop.last %}{% else %} and {% endif %} {% endif %} {% elif word.key|lower == "position" or word.key|lower == "mb" %} - with <u>target genes</u> on chromosome <strong>{% if (word.search_term[0]|lower).split('chr')|length > 1 %}{{ (word.search_term[0]|lower).split('chr')[1] }}{% else %}{{ word.search_term[0] }}{% endif %}</strong> between <strong>{{ word.search_term[1] }}</strong> and <strong>{{ word.search_term[2] }}</strong> Mb{% if loop.last %}.{% else %} and {% endif %} + with <u>target genes</u> on chromosome <strong>{% if (word.search_term[0]|lower).split('chr')|length > 1 %}{{ (word.search_term[0]|lower).split('chr')[1] }}{% else %}{{ word.search_term[0] }}{% endif %}</strong> between <strong>{{ word.search_term[1] }}</strong> and <strong>{{ word.search_term[2] }}</strong> Mb{% if loop.last %}{% else %} and {% endif %} {% else %} - {% if word.search_term[0] == "*" %} in the dataset.{% else %}{% if loop.first %}that match:<br>{% endif %}<b>{{ word.search_term[0] }}</b>{% if loop.last %}{% else %} and {% endif %}{% endif %} + {% if word.search_term[0] == "*" %} in the dataset.{% else %}{% if loop.first %}that match: {% endif %}<b>{{ word.search_term[0] }}</b>{% if loop.last %}{% else %} and {% endif %}{% endif %} {% endif %} {% endfor %} {% endif %} <br> - A total of {{ results|count }} records were found. + {% if results|count > 0 %} + <b>{{ results|count }}</b> record{% if results|count > 1 %}s{% else %}{% endif %} found + {% else %} + No (<b>0</b>) records found for this search. Modify your search, check target dataset, or use <b>Search All</b> above. + {% endif %} </p> {% if results|count > 0 %} @@ -69,7 +74,7 @@ {% if too_many_results %} <p>Your search generated over 50000 results. Please modify your search to generate 50000 or fewer matches.</p> {% else %} - <div style="min-width: 950px;"> + <div style="min-width: 1050px;"> <form id="trait_submission_form" target="_blank" action="/corr_matrix" method="post"> <input type="hidden" name="tool_used" value="" /> <input type="hidden" name="form_url" value="" /> @@ -142,7 +147,7 @@ {% endif %} {% else %} <br> - <button type="button" onclick="window.location.href='/'">Return To Index Page</button> + <button type="button" onclick="window.location.href='/'">Return To Search</button> {% endif %} </div> <div id="myModal"></div> @@ -164,6 +169,7 @@ <script language="javascript" type="text/javascript" src="/static/new/javascript/search_results.js"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/table_functions.js"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/create_datatable.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/partial_correlations.js"></script> <script type='text/javascript'> var traitsJson = {{ trait_list|safe }}; @@ -188,13 +194,52 @@ var widthChange = 0; //ZS: For storing the change in width so overall table width can be adjusted by that amount columnDefs = [ - { - 'data': null, - 'width': "5px", - 'orderDataType': "dom-checkbox", - 'targets': 0, - 'render': function(data) { - return '<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + data.hmac + '">' + { + 'data': null, + 'width': "5px", + 'orderDataType': "dom-checkbox", + 'targets': 0, + 'render': function(data) { + return '<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + data.hmac + '" data-trait-info="' + data.trait_info_str + '">' + } + }, + { + 'title': "Index", + 'type': "natural", + 'width': "35px", + "searchable": false, + "orderable": false, + 'targets': 1, + 'data': "index" + } + {% if dataset.type == 'ProbeSet' %}, + { + 'title': "Record", + 'type': "natural-minus-na", + 'data': null, + 'width': "{{ max_widths.display_name * 8 }}px", + 'targets': 2, + 'render': function(data) { + return '<a target="_blank" href="/show_trait?trait_id=' + data.display_name + '&dataset=' + data.dataset + '">' + data.display_name + '</a>' + } + }, + { + 'title': "Symbol", + 'type': "natural", + 'width': "{{ max_widths.symbol * 8 }}px", + 'targets': 3, + 'data': "symbol" + }, + { + 'title': "Description", + 'type': "natural", + 'data': null, + 'targets': 4, + 'render': function(data) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return escape(data.description) } }, { diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html index 2f98f14e..69fb5d3b 100644 --- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html +++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html @@ -119,6 +119,10 @@ <div class="col-xs-3 controls"> <input type="button" class="btn corr_compute submit_special btn-success" data-url="/corr_compute" title="Compute Correlation" value="Compute"> </div> + + <div class="col-xs-3 controls"> + <input type="button" class="btn test_corr_compute submit_special btn-success" data-url="/test_corr_compute" title="Compute Correlation" value="Test Compute"> + </div> </div> </div> </div> diff --git a/wqflask/wqflask/templates/tool_buttons.html b/wqflask/wqflask/templates/tool_buttons.html index c3927495..c6d1476c 100644 --- a/wqflask/wqflask/templates/tool_buttons.html +++ b/wqflask/wqflask/templates/tool_buttons.html @@ -30,10 +30,6 @@ MultiMap </button> -<button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" > - Comparison Bar Chart -</button> - <button id="partial-correlations" class="btn btn-primary submit_special" data-url="{{url_for('partial_correlations')}}" diff --git a/wqflask/wqflask/update_search_results.py b/wqflask/wqflask/update_search_results.py index 2e467dc8..f132e2c6 100644 --- a/wqflask/wqflask/update_search_results.py +++ b/wqflask/wqflask/update_search_results.py @@ -5,11 +5,6 @@ from base.data_set import create_dataset from base.trait import GeneralTrait from db import webqtlDatabaseFunction -from utility.benchmark import Bench - -from utility.logger import getLogger -logger = getLogger(__name__) - class GSearch: @@ -46,19 +41,15 @@ class GSearch: ORDER BY species_name, inbredset_name, tissue_name, probesetfreeze_name, probeset_name LIMIT 6000 """ % (self.terms) - with Bench("Running query"): - logger.sql(sql) - re = g.db.execute(sql).fetchall() + re = g.db.execute(sql).fetchall() self.trait_list = [] - with Bench("Creating trait objects"): - for line in re: - dataset = create_dataset( - line[3], "ProbeSet", get_samplelist=False) - trait_id = line[4] - # with Bench("Building trait object"): - this_trait = GeneralTrait( - dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) - self.trait_list.append(this_trait) + for line in re: + dataset = create_dataset( + line[3], "ProbeSet", get_samplelist=False) + trait_id = line[4] + this_trait = GeneralTrait( + dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) + self.trait_list.append(this_trait) elif self.type == "phenotype": sql = """ @@ -92,16 +83,14 @@ class GSearch: ORDER BY Species.`Name`, InbredSet.`Name`, PublishXRef.`Id` LIMIT 6000 """ % (self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms) - logger.sql(sql) re = g.db.execute(sql).fetchall() self.trait_list = [] - with Bench("Creating trait objects"): - for line in re: - dataset = create_dataset(line[2], "Publish") - trait_id = line[3] - this_trait = GeneralTrait( - dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) - self.trait_list.append(this_trait) + for line in re: + dataset = create_dataset(line[2], "Publish") + trait_id = line[3] + this_trait = GeneralTrait( + dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) + self.trait_list.append(this_trait) self.results = self.convert_to_json() diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index 45a12f77..ae61edb0 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -2,7 +2,6 @@ import os import hashlib import datetime import time -import logging import uuid import hmac import base64 @@ -21,9 +20,6 @@ from utility import hmac from utility.redis_tools import is_redis_available, get_redis_conn, get_user_id, get_user_by_unique_column, set_user_attribute, save_user, save_verification_code, check_verification_code, get_user_collections, save_collections Redis = get_redis_conn() -from utility.logger import getLogger -logger = getLogger(__name__) - from smtplib import SMTP from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_ALCHEMY, GN2_BRANCH_URL @@ -129,7 +125,6 @@ def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): server.login(SMTP_USERNAME, SMTP_PASSWORD) server.sendmail(fromaddr, toaddr, msg) server.quit() - logger.info("Successfully sent email to " + toaddr) def send_verification_email(user_details, template_name="email/user_verification.txt", key_prefix="verification_code", subject="GeneNetwork e-mail verification"): @@ -346,7 +341,6 @@ def get_github_user_details(access_token): @app.route("/n/logout") def logout(): - logger.debug("Logging out...") UserSession().delete_session() flash("You are now logged out. We hope you come back soon!") response = make_response(redirect(url_for('index_page'))) @@ -404,7 +398,6 @@ def forgot_password_submit(): email_address = params['email_address'] next_page = None if email_address != "": - logger.debug("Wants to send password E-mail to ", email_address) user_details = get_user_by_unique_column( "email_address", email_address) if user_details: @@ -425,8 +418,6 @@ def forgot_password_submit(): @app.route("/n/password_reset", methods=['GET']) def password_reset(): """Entry point after user clicks link in E-mail""" - logger.debug("in password_reset request.url is:", request.url) - verification_code = request.args.get('code') hmac = request.args.get('hm') @@ -446,8 +437,6 @@ def password_reset(): @app.route("/n/password_reset_step2", methods=('POST',)) def password_reset_step2(): """Handle confirmation E-mail for password reset""" - logger.debug("in password_reset request.url is:", request.url) - errors = [] user_email = request.form['user_encode'] user_id = get_user_id("email_address", user_email) @@ -515,7 +504,6 @@ def register(): params = params.to_dict(flat=True) if params: - logger.debug("Attempting to register the user...") errors = register_user(params) if len(errors) == 0: diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 00b268a7..1ed5b802 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -13,8 +13,6 @@ from utility import hmac from utility.redis_tools import get_redis_conn, get_user_id, get_user_by_unique_column, set_user_attribute, get_user_collections, save_collections Redis = get_redis_conn() -from utility.logger import getLogger -logger = getLogger(__name__) THREE_DAYS = 60 * 60 * 24 * 3 THIRTY_DAYS = 60 * 60 * 24 * 30 @@ -54,7 +52,6 @@ def create_signed_cookie(): the_uuid = str(uuid.uuid4()) signature = hmac.hmac_creation(the_uuid) uuid_signed = the_uuid + ":" + signature - logger.debug("uuid_signed:", uuid_signed) return the_uuid, uuid_signed diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index a371d983..acd65587 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -64,6 +64,7 @@ from wqflask.marker_regression import display_mapping_results from wqflask.network_graph import network_graph from wqflask.correlation.show_corr_results import set_template_vars from wqflask.correlation.correlation_gn3_api import compute_correlation +from wqflask.correlation.rust_correlation import compute_correlation_rust from wqflask.correlation_matrix import show_corr_matrix from wqflask.correlation import corr_scatter_plot # from wqflask.wgcna import wgcna_analysis @@ -94,37 +95,34 @@ from utility.redis_tools import get_redis_conn from base.webqtlConfig import GENERATED_IMAGE_DIR, DEFAULT_PRIVILEGES -from utility.benchmark import Bench from pprint import pformat as pf -import utility.logger - Redis = get_redis_conn() -logger = utility.logger.getLogger(__name__) - @app.before_request def connect_db(): db = getattr(g, '_database', None) if request.endpoint not in ("static", "js") and db is None: - logger.debug( - f"Creating a database connection\n" - f"\t\tfor request: {request.endpoint}") - g.db = g._database = sqlalchemy.create_engine( - SQL_URI, encoding="latin1") + try: + g.db = sqlalchemy.create_engine( + SQL_URI, encoding="latin1") + except Exception: # Capture everything + app.logger.error(f"DATABASE: Error creating connection for: {request.endpoint}") @app.teardown_appcontext def shutdown_session(exception=None): db = getattr(g, '_database', None) if db is not None: - logger.debug(f"Removing the session") - g.db.dispose() - g.db = None - logger.debug(f"g.db: {g.db}\n\tg._database: {g._database}") + try: + g.db.dispose() + except Exception: # Capture Everything + app.logger.error(f"DATABASE: Error disposing: {g.db=}") + finally: # Reset regardless of what happens + g.db = None @app.errorhandler(Exception) @@ -158,8 +156,6 @@ def handle_generic_exceptions(e): stack={formatted_lines}, error_image=animation, version=GN_VERSION)) - - # logger.error("Set cookie %s with %s" % (err_msg, animation)) resp.set_cookie(err_msg[:32], animation) return resp @@ -171,8 +167,6 @@ def no_access_page(): @app.route("/") def index_page(): - logger.info("Sending index_page") - logger.info(request.url) params = request.args if 'import_collections' in params: import_collections = params['import_collections'] @@ -184,11 +178,7 @@ def index_page(): @app.route("/tmp/<img_path>") def tmp_page(img_path): - logger.info("In tmp_page") - logger.info("img_path:", img_path) - logger.info(request.url) initial_start_vars = request.form - logger.info("initial_start_vars:", initial_start_vars) imgfile = open(GENERATED_IMAGE_DIR + img_path, 'rb') imgdata = imgfile.read() imgB64 = base64.b64encode(imgdata) @@ -224,22 +214,13 @@ def twitter(filename): @app.route("/search", methods=('GET',)) def search_page(): - logger.info("in search_page") - logger.info(request.url) result = None if USE_REDIS: - with Bench("Trying Redis cache"): - key = "search_results:v1:" + \ - json.dumps(request.args, sort_keys=True) - logger.debug("key is:", pf(key)) - result = Redis.get(key) - if result: - logger.info("Redis cache hit on search results!") - result = pickle.loads(result) - else: - logger.info("Skipping Redis cache (USE_REDIS=False)") - - logger.info("request.args is", request.args) + key = "search_results:v1:" + \ + json.dumps(request.args, sort_keys=True) + result = Redis.get(key) + if result: + result = pickle.loads(result) the_search = SearchResultPage(request.args) result = the_search.__dict__ valid_search = result['search_term_exists'] @@ -256,15 +237,7 @@ def search_page(): @app.route("/search_table", methods=('GET',)) def search_page_table(): - logger.info("in search_page table") - logger.info(request.url) - - logger.info("request.args is", request.args) the_search = search_results.SearchResultPage(request.args) - - logger.info(type(the_search.trait_list)) - logger.info(the_search.trait_list) - current_page = server_side.ServerSideTable( len(the_search.trait_list), the_search.trait_list, @@ -277,7 +250,6 @@ def search_page_table(): @app.route("/gsearch", methods=('GET',)) def gsearchact(): - logger.info(request.url) result = GSearch(request.args).__dict__ type = request.args['type'] if type == "gene": @@ -288,8 +260,6 @@ def gsearchact(): @app.route("/gsearch_table", methods=('GET',)) def gsearchtable(): - logger.info(request.url) - gsearch_table_data = GSearch(request.args) current_page = server_side.ServerSideTable( gsearch_table_data.trait_count, @@ -303,15 +273,12 @@ def gsearchtable(): @app.route("/gsearch_updating", methods=('POST',)) def gsearch_updating(): - logger.info("REQUEST ARGS:", request.values) - logger.info(request.url) result = UpdateGSearch(request.args).__dict__ return result['results'] @app.route("/docedit") def docedit(): - logger.info(request.url) try: if g.user_session.record['user_email_address'] == "zachary.a.sloan@gmail.com" or g.user_session.record['user_email_address'] == "labwilliams@gmail.com": doc = Docs(request.args['entry'], request.args) @@ -324,13 +291,11 @@ def docedit(): @app.route('/generated/<filename>') def generated_file(filename): - logger.info(request.url) return send_from_directory(GENERATED_IMAGE_DIR, filename) @app.route("/help") def help(): - logger.info(request.url) doc = Docs("help", request.args) return render_template("docs.html", **doc.__dict__) @@ -338,8 +303,6 @@ def help(): @app.route("/wgcna_setup", methods=('POST',)) def wcgna_setup(): # We are going to get additional user input for the analysis - logger.info("In wgcna, request.form is:", request.form) - logger.info(request.url) # Display them using the template return render_template("wgcna_setup.html", **request.form) @@ -350,19 +313,16 @@ def wcgna_results(): results = run_wgcna(dict(request.form)) return render_template("gn3_wgcna_results.html", **results) + @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): # We are going to get additional user input for the analysis - logger.info("In ctl, request.form is:", request.form) - logger.info(request.url) # Display them using the template return render_template("ctl_setup.html", **request.form) - @app.route("/ctl_results", methods=["POST"]) def ctl_results(): - ctl_results = run_ctl(request.form) return render_template("gn3_ctl_results.html", **ctl_results) @@ -371,10 +331,11 @@ def ctl_results(): def fetch_network_files(file_name, file_type): file_path = f"{file_name}.{file_type}" - file_path = os.path.join("/tmp/",file_path) + file_path = os.path.join("/tmp/", file_path) return send_file(file_path) + @app.route("/intro") def intro(): doc = Docs("intro", request.args) @@ -400,7 +361,6 @@ def update_page(): @app.route("/submit_trait") def submit_trait_form(): - logger.info(request.url) species_and_groups = get_species_groups() return render_template( "submit_trait.html", @@ -411,7 +371,6 @@ def submit_trait_form(): @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): - logger.info(request.url) doc = Docs("links") return render_template("links.html", **doc.__dict__) @@ -422,9 +381,6 @@ def export_trait_excel(): trait_name, sample_data = export_trait_data.export_sample_table( request.form) app.logger.info(request.url) - logger.info("sample_data - type: %s -- size: %s" % - (type(sample_data), len(sample_data))) - buff = io.BytesIO() workbook = xlsxwriter.Workbook(buff, {'in_memory': True}) worksheet = workbook.add_worksheet() @@ -443,15 +399,9 @@ def export_trait_excel(): @app.route('/export_trait_csv', methods=('POST',)) def export_trait_csv(): """CSV file consisting of the sample data from the trait data and analysis page""" - logger.info("In export_trait_csv") - logger.info("request.form:", request.form) - logger.info(request.url) trait_name, sample_data = export_trait_data.export_sample_table( request.form) - logger.info("sample_data - type: %s -- size: %s" % - (type(sample_data), len(sample_data))) - buff = io.StringIO() writer = csv.writer(buff) for row in sample_data: @@ -467,9 +417,6 @@ def export_trait_csv(): @app.route('/export_traits_csv', methods=('POST',)) def export_traits_csv(): """CSV file consisting of the traits from the search result page""" - logger.info("In export_traits_csv") - logger.info("request.form:", request.form) - logger.info(request.url) file_list = export_traits(request.form, "metadata") if len(file_list) > 1: @@ -498,10 +445,10 @@ def export_collection_csv(): mimetype='text/csv', headers={"Content-Disposition": "attachment;filename=" + out_file[0] + ".csv"}) + @app.route('/export_perm_data', methods=('POST',)) def export_perm_data(): """CSV file consisting of the permutation data for the mapping results""" - logger.info(request.url) perm_info = json.loads(request.form['perm_info']) now = datetime.datetime.now() @@ -572,9 +519,6 @@ def show_trait_page(): @app.route("/heatmap", methods=('POST',)) def heatmap_page(): - logger.info("In heatmap, request.form is:", pf(request.form)) - logger.info(request.url) - start_vars = request.form temp_uuid = uuid.uuid4() @@ -583,18 +527,12 @@ def heatmap_page(): version = "v5" key = "heatmap:{}:".format( version) + json.dumps(start_vars, sort_keys=True) - logger.info("key is:", pf(key)) - with Bench("Loading cache"): - result = Redis.get(key) + result = Redis.get(key) if result: - logger.info("Cache hit!!!") - with Bench("Loading results"): - result = pickle.loads(result) + result = pickle.loads(result) else: - logger.info("Cache miss!!!") - template_vars = heatmap.Heatmap(request.form, temp_uuid) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, @@ -602,17 +540,10 @@ def heatmap_page(): result = template_vars.__dict__ - for item in list(template_vars.__dict__.keys()): - logger.info( - " ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) - pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) - logger.info("pickled result length:", len(pickled_result)) Redis.set(key, pickled_result) Redis.expire(key, 60 * 60) - - with Bench("Rendering template"): - rendered_template = render_template("heatmap.html", **result) + rendered_template = render_template("heatmap.html", **result) else: rendered_template = render_template( @@ -623,9 +554,6 @@ def heatmap_page(): @app.route("/bnw_page", methods=('POST',)) def bnw_page(): - logger.info("In run BNW, request.form is:", pf(request.form)) - logger.info(request.url) - start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -643,9 +571,6 @@ def bnw_page(): @app.route("/webgestalt_page", methods=('POST',)) def webgestalt_page(): - logger.info("In run WebGestalt, request.form is:", pf(request.form)) - logger.info(request.url) - start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -663,9 +588,6 @@ def webgestalt_page(): @app.route("/geneweaver_page", methods=('POST',)) def geneweaver_page(): - logger.info("In run WebGestalt, request.form is:", pf(request.form)) - logger.info(request.url) - start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -683,9 +605,6 @@ def geneweaver_page(): @app.route("/comparison_bar_chart", methods=('POST',)) def comp_bar_chart_page(): - logger.info("In comp bar chart, request.form is:", pf(request.form)) - logger.info(request.url) - start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -712,7 +631,6 @@ def mapping_results_container_page(): @app.route("/loading", methods=('POST',)) def loading_page(): - # logger.info(request.url) initial_start_vars = request.form start_vars_container = {} n_samples = 0 # ZS: So it can be displayed on loading page @@ -734,7 +652,8 @@ def loading_page(): dataset = create_dataset(start_vars['dataset']) start_vars['trait_name'] = start_vars['trait_id'] if dataset.type == "Publish": - start_vars['trait_name'] = dataset.group.code + "_" + start_vars['trait_name'] + start_vars['trait_name'] = dataset.group.code + \ + "_" + start_vars['trait_name'] samples = dataset.group.samplelist if 'genofile' in start_vars: if start_vars['genofile'] != "": @@ -751,9 +670,11 @@ def loading_page(): n_samples += 1 start_vars['n_samples'] = n_samples - start_vars['vals_hash'] = generate_hash_of_string(str(sample_vals_dict)) - if start_vars['dataset'] != "Temp": # Currently can't get diff for temp traits - start_vars['vals_diff'] = get_diff_of_vals(sample_vals_dict, str(start_vars['trait_id'] + ":" + str(start_vars['dataset']))) + start_vars['vals_hash'] = generate_hash_of_string( + str(sample_vals_dict)) + if start_vars['dataset'] != "Temp": # Currently can't get diff for temp traits + start_vars['vals_diff'] = get_diff_of_vals(sample_vals_dict, str( + start_vars['trait_id'] + ":" + str(start_vars['dataset']))) start_vars['wanted_inputs'] = initial_start_vars['wanted_inputs'] @@ -769,7 +690,6 @@ def loading_page(): @app.route("/run_mapping", methods=('POST',)) def mapping_results_page(): initial_start_vars = request.form - logger.info(request.url) temp_uuid = initial_start_vars['temp_uuid'] wanted = ( 'trait_id', @@ -836,55 +756,42 @@ def mapping_results_page(): version = "v3" key = "mapping_results:{}:".format( version) + json.dumps(start_vars, sort_keys=True) - with Bench("Loading cache"): - result = None # Just for testing - #result = Redis.get(key) - - #logger.info("************************ Starting result *****************") - #logger.info("result is [{}]: {}".format(type(result), result)) - #logger.info("************************ Ending result ********************") + result = None # Just for testing if result: - logger.info("Cache hit!!!") - with Bench("Loading results"): - result = pickle.loads(result) + result = pickle.loads(result) else: - logger.info("Cache miss!!!") - with Bench("Total time in RunMapping"): - try: - template_vars = run_mapping.RunMapping(start_vars, temp_uuid) - if template_vars.no_results: - rendered_template = render_template("mapping_error.html") - return rendered_template - except: + try: + template_vars = run_mapping.RunMapping(start_vars, temp_uuid) + if template_vars.no_results: rendered_template = render_template("mapping_error.html") return rendered_template + except: + rendered_template = render_template("mapping_error.html") + return rendered_template - if not template_vars.pair_scan: - template_vars.js_data = json.dumps(template_vars.js_data, - default=json_default_handler, - indent=" ") + if not template_vars.pair_scan: + template_vars.js_data = json.dumps(template_vars.js_data, + default=json_default_handler, + indent=" ") - result = template_vars.__dict__ + result = template_vars.__dict__ - if result['pair_scan']: - with Bench("Rendering template"): - rendered_template = render_template( - "pair_scan_results.html", **result) - else: - gn1_template_vars = display_mapping_results.DisplayMappingResults( - result).__dict__ + if result['pair_scan']: + rendered_template = render_template( + "pair_scan_results.html", **result) + else: + gn1_template_vars = display_mapping_results.DisplayMappingResults( + result).__dict__ - rendered_template = render_template( - "mapping_results.html", **gn1_template_vars) + rendered_template = render_template( + "mapping_results.html", **gn1_template_vars) return rendered_template @app.route("/export_mapping_results", methods=('POST',)) def export_mapping_results(): - logger.info("request.form:", request.form) - logger.info(request.url) file_path = request.form.get("results_path") results_csv = open(file_path, "r").read() response = Response(results_csv, @@ -908,8 +815,6 @@ def export_corr_matrix(): @app.route("/export", methods=('POST',)) def export(): - logger.info("request.form:", request.form) - logger.info(request.url) svg_xml = request.form.get("data", "Invalid data") filename = request.form.get("filename", "manhattan_plot_snp") response = Response(svg_xml, mimetype="image/svg+xml") @@ -920,10 +825,7 @@ def export(): @app.route("/export_pdf", methods=('POST',)) def export_pdf(): import cairosvg - logger.info("request.form:", request.form) - logger.info(request.url) svg_xml = request.form.get("data", "Invalid data") - logger.info("svg_xml:", svg_xml) filename = request.form.get("filename", "interval_map_pdf") pdf_file = cairosvg.svg2pdf(bytestring=svg_xml) response = Response(pdf_file, mimetype="application/pdf") @@ -933,8 +835,6 @@ def export_pdf(): @app.route("/network_graph", methods=('POST',)) def network_graph_page(): - logger.info("In network_graph, request.form is:", pf(request.form)) - logger.info(request.url) start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] if traits[0] != "": @@ -951,21 +851,29 @@ def network_graph_page(): @app.route("/corr_compute", methods=('POST',)) def corr_compute_page(): correlation_results = compute_correlation(request.form, compute_all=True) + correlation_results = set_template_vars(request.form, correlation_results) return render_template("correlation_page.html", **correlation_results) @app.route("/test_corr_compute", methods=["POST"]) def test_corr_compute_page(): - correlation_data = compute_correlation(request.form, compute_all=True) - return render_template("test_correlation_page.html", **correlation_data) + + start_vars = request.form + + correlation_results = compute_correlation_rust(start_vars, + start_vars["corr_type"], + start_vars['corr_sample_method'], + int(start_vars.get("corr_return_results", 500)),True) + + correlation_results = set_template_vars(request.form, correlation_results) + + return render_template("correlation_page.html", **correlation_results) + # return render_template("test_correlation_page.html", **correlation_data) @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): - logger.info("In corr_matrix, request.form is:", pf(request.form)) - logger.info(request.url) - start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] if len(traits) > 1: @@ -981,7 +889,6 @@ def corr_matrix_page(): @app.route("/corr_scatter_plot") def corr_scatter_plot_page(): - logger.info(request.url) template_vars = corr_scatter_plot.CorrScatterPlot(request.args) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, @@ -991,7 +898,6 @@ def corr_scatter_plot_page(): @app.route("/snp_browser", methods=('GET',)) def snp_browser_page(): - logger.info(request.url) template_vars = snp_browser.SnpBrowser(request.args) return render_template("snp_browser.html", **template_vars.__dict__) @@ -1006,7 +912,6 @@ def db_info_page(): @app.route("/snp_browser_table", methods=('GET',)) def snp_browser_table(): - logger.info(request.url) snp_table_data = snp_browser.SnpBrowser(request.args) current_page = server_side.ServerSideTable( snp_table_data.rows_count, @@ -1021,20 +926,17 @@ def snp_browser_table(): @app.route("/tutorial/WebQTLTour", methods=('GET',)) def tutorial_page(): # ZS: Currently just links to GN1 - logger.info(request.url) return redirect("http://gn1.genenetwork.org/tutorial/WebQTLTour/") @app.route("/tutorial/security", methods=('GET',)) def security_tutorial_page(): # ZS: Currently just links to GN1 - logger.info(request.url) return render_template("admin/security_help.html") @app.route("/submit_bnw", methods=('POST',)) def submit_bnw(): - logger.info(request.url) return render_template("empty_collection.html", **{'tool': 'Correlation Matrix'}) # Take this out or secure it before putting into production @@ -1042,7 +944,6 @@ def submit_bnw(): @app.route("/get_temp_data") def get_temp_data(): - logger.info(request.url) temp_uuid = request.args['key'] return flask.jsonify(temp_data.TempData(temp_uuid).get_all()) @@ -1097,9 +998,8 @@ def display_diffs_users(): author = g.user_session.record.get(b'user_name').decode("utf-8") if os.path.exists(DIFF_DIR): files = os.listdir(DIFF_DIR) - files = filter(lambda x: not(x.endswith((".approved", ".rejected"))) \ + files = filter(lambda x: not(x.endswith((".approved", ".rejected"))) and author in x, files) return render_template("display_files_user.html", files=files) - |