From 1e403623792daca83875b5fefd5193455facb88e Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 12 Jul 2022 19:06:07 +0000 Subject: Include genotypes for F1s/reference/non-reference strains when fetching sample data for genotype traits --- wqflask/base/data_set.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index a7811a30..211c6752 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1022,6 +1022,14 @@ class GenotypeDataSet(DataSet): results = g.db.execute(query, (webqtlDatabaseFunction.retrieve_species_id(self.group.name), trait, self.name)).fetchall() + + if self.group.name in webqtlUtil.ParInfo: + f1_1, f1_2, ref, nonref = webqtlUtil.ParInfo[self.group.name] + results.append([f1_1, 0, None, "N/A", f1_1]) + results.append([f1_2, 0, None, "N/A", f1_2]) + results.append([ref, -1, None, "N/A", ref]) + results.append([nonref, 1, None, "N/A", nonref]) + return results -- cgit v1.2.3 From 8c06a9534d3340ff0287b5c70f63b2000b6eb612 Mon Sep 17 00:00:00 2001 From: Munyoki Kilyungi Date: Mon, 29 Aug 2022 12:11:41 +0300 Subject: Remove usage of "logger" and un-necessary comments wrt the same Logging is used to introspect variables or notify the commencement of a given operation. Logging should only be used to log errors. Also, most of the logging is either "logger.debug" or "logger.info"; and this won't show up in production/testing since we need a logging level above "WARNING" for them to show up. * wqflask/base/data_set.py (create_datasets_list): Remove logger. (Markers.add_pvalues): Ditto. (DataSet.retrieve_other_names): Ditto. * wqflask/base/mrna_assay_tissue_data.py: Ditto. * wqflask/base/webqtlCaseData.py: Ditto. * wqflask/db/call.py (fetch1): Ditto. (gn_server): Ditto. * wqflask/db/gn_server.py: Ditto. * wqflask/maintenance/set_resource_defaults.py: Ditto. * wqflask/utility/Plot.py (find_outliers): Ditto. * wqflask/utility/gen_geno_ob.py: Ditto. * wqflask/utility/helper_functions.py: Ditto. * wqflask/utility/pillow_utils.py: Ditto. * wqflask/utility/redis_tools.py: Ditto. * wqflask/wqflask/api/gen_menu.py (get_groups): Ditto. * wqflask/wqflask/api/mapping.py: Ditto. * wqflask/wqflask/api/router.py (get_dataset_info): Ditto. * wqflask/wqflask/collect.py (report_change): Ditto. * wqflask/wqflask/correlation/corr_scatter_plot.py: Ditto. * wqflask/wqflask/ctl/ctl_analysis.py (CTL): Ditto. (CTL.__init__): Ditto. (CTL.run_analysis): Ditto. (CTL.process_results): Ditto. * wqflask/wqflask/db_info.py: Ditto. * wqflask/wqflask/do_search.py (DoSearch.execute): Ditto. (DoSearch.mescape): Ditto. (DoSearch.get_search): Ditto. (MrnaAssaySearch.run_combined): Ditto. (MrnaAssaySearch.run): Ditto. (PhenotypeSearch.run_combined): Ditto. (GenotypeSearch.get_where_clause): Ditto. (LrsSearch.get_where_clause): Ditto. (MeanSearch.run): Ditto. (RangeSearch.get_where_clause): Ditto. (PvalueSearch.run): Ditto. * wqflask/wqflask/docs.py: Ditto. * wqflask/wqflask/export_traits.py: Ditto. * wqflask/wqflask/external_tools/send_to_bnw.py: Ditto. * wqflask/wqflask/external_tools/send_to_geneweaver.py: Ditto. * wqflask/wqflask/external_tools/send_to_webgestalt.py: Ditto. * wqflask/wqflask/gsearch.py (GSearch.__init__): Ditto. * wqflask/wqflask/heatmap/heatmap.py: Ditto. * wqflask/wqflask/marker_regression/display_mapping_results.py (DisplayMappingResults): Ditto. * wqflask/wqflask/marker_regression/gemma_mapping.py: Ditto. * wqflask/wqflask/marker_regression/plink_mapping.py (run_plink): Ditto. * wqflask/wqflask/marker_regression/qtlreaper_mapping.py (run_reaper): Ditto. * wqflask/wqflask/marker_regression/rqtl_mapping.py: Ditto. * wqflask/wqflask/marker_regression/run_mapping.py (RunMapping.__init__): Ditto. * wqflask/wqflask/parser.py (parse): Ditto. * wqflask/wqflask/search_results.py (SearchResultPage.__init__): Ditto. * wqflask/wqflask/update_search_results.py (GSearch.__init__): Ditto. * wqflask/wqflask/user_login.py (send_email): Ditto. (logout): Ditto. (forgot_password_submit): Ditto. (password_reset): Ditto. (password_reset_step2): Ditto. (register): Ditto. * wqflask/wqflask/user_session.py (create_signed_cookie): Ditto. --- wqflask/base/data_set.py | 13 ----------- wqflask/base/mrna_assay_tissue_data.py | 4 ---- wqflask/base/webqtlCaseData.py | 3 --- wqflask/db/call.py | 4 ---- wqflask/db/gn_server.py | 3 --- wqflask/maintenance/set_resource_defaults.py | 2 -- wqflask/utility/Plot.py | 4 +--- wqflask/utility/gen_geno_ob.py | 4 ---- wqflask/utility/helper_functions.py | 3 --- wqflask/utility/pillow_utils.py | 3 --- wqflask/utility/redis_tools.py | 2 -- wqflask/wqflask/api/gen_menu.py | 4 ---- wqflask/wqflask/api/router.py | 4 ---- wqflask/wqflask/collect.py | 4 ---- wqflask/wqflask/correlation/corr_scatter_plot.py | 3 --- wqflask/wqflask/ctl/ctl_analysis.py | 26 ---------------------- wqflask/wqflask/db_info.py | 3 --- wqflask/wqflask/do_search.py | 26 ---------------------- wqflask/wqflask/docs.py | 3 --- wqflask/wqflask/export_traits.py | 2 -- wqflask/wqflask/external_tools/send_to_bnw.py | 3 --- .../wqflask/external_tools/send_to_geneweaver.py | 3 --- .../wqflask/external_tools/send_to_webgestalt.py | 3 --- wqflask/wqflask/gsearch.py | 5 ----- wqflask/wqflask/heatmap/heatmap.py | 3 --- .../marker_regression/display_mapping_results.py | 5 +---- wqflask/wqflask/marker_regression/gemma_mapping.py | 2 -- wqflask/wqflask/marker_regression/plink_mapping.py | 5 ----- .../wqflask/marker_regression/qtlreaper_mapping.py | 5 ----- wqflask/wqflask/marker_regression/rqtl_mapping.py | 3 --- wqflask/wqflask/marker_regression/run_mapping.py | 6 ----- wqflask/wqflask/parser.py | 5 ----- wqflask/wqflask/search_results.py | 3 --- wqflask/wqflask/update_search_results.py | 5 ----- wqflask/wqflask/user_login.py | 12 ---------- wqflask/wqflask/user_session.py | 3 --- 36 files changed, 2 insertions(+), 189 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 211c6752..8206b67c 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -22,7 +22,6 @@ from dataclasses import field from dataclasses import InitVar from typing import Optional, Dict, List from db.call import fetchall, fetchone, fetch1 -from utility.logger import getLogger from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL from db.gn_server import menu_main from pprint import pformat as pf @@ -60,8 +59,6 @@ from redis import Redis r = Redis() -logger = getLogger(__name__) - # Used by create_database to instantiate objects # Each subclass will add to this DS_NAME_MAP = {} @@ -200,7 +197,6 @@ def create_datasets_list(): result = r.get(key) if result: - logger.debug("Redis cache hit") datasets = pickle.loads(result) if result is None: @@ -213,10 +209,6 @@ def create_datasets_list(): for dataset_type in type_dict: query = "SELECT Name FROM {}".format(type_dict[dataset_type]) for result in fetchall(query): - # The query at the beginning of this function isn't - # necessary here, but still would rather just reuse - # it logger.debug("type: {}\tname: - # {}".format(dataset_type, result.Name)) dataset = create_dataset(result.Name, dataset_type) datasets.append(dataset) @@ -259,9 +251,6 @@ class Markers: self.markers = markers def add_pvalues(self, p_values): - logger.debug("length of self.markers:", len(self.markers)) - logger.debug("length of p_values:", len(p_values)) - if isinstance(p_values, list): # THIS IS only needed for the case when we are limiting the number of p-values calculated # if len(self.markers) > len(p_values): @@ -664,8 +653,6 @@ class DataSet: """ % (query_args)) except TypeError: - logger.debug( - "Dataset {} is not yet available in GeneNetwork.".format(self.name)) pass def chunk_dataset(self, dataset, n): diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index 8f8e2b0a..d7e747aa 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -9,10 +9,6 @@ from utility.db_tools import escape from gn3.db_utils import database_connector -from utility.logger import getLogger -logger = getLogger(__name__) - - class MrnaAssayTissueData: def __init__(self, gene_symbols=None): diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py index 25b6cb8a..dd6fad04 100644 --- a/wqflask/base/webqtlCaseData.py +++ b/wqflask/base/webqtlCaseData.py @@ -21,9 +21,6 @@ # Created by GeneNetwork Core Team 2010/08/10 -from utility.logger import getLogger -logger = getLogger(__name__) - import utility.tools utility.tools.show_settings() diff --git a/wqflask/db/call.py b/wqflask/db/call.py index 1fe0772b..a6bbda54 100644 --- a/wqflask/db/call.py +++ b/wqflask/db/call.py @@ -31,9 +31,6 @@ GN_SERVER result when set (which should return a Tuple) res2 = func(result) else: res2 = result, - if LOG_SQL: - logger.debug("Replaced SQL call", query) - logger.debug(path, res2) return res2 else: return fetchone(query) @@ -75,5 +72,4 @@ def gn_server(path): res = urllib2.urlopen(GN_SERVER_URL + path) rest = res.read() res2 = json.loads(rest) - logger.debug(res2) return res2 diff --git a/wqflask/db/gn_server.py b/wqflask/db/gn_server.py index f9b01658..af6682ba 100644 --- a/wqflask/db/gn_server.py +++ b/wqflask/db/gn_server.py @@ -2,9 +2,6 @@ from db.call import gn_server -from utility.logger import getLogger -logger = getLogger(__name__) - def menu_main(): return gn_server("/int/menu/main.json") diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index 22d73ba3..0d9372ff 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -33,8 +33,6 @@ Redis = get_redis_conn() import urllib.parse from wqflask.database import database_connection -from utility.logger import getLogger -logger = getLogger(__name__) def parse_db_uri(): diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index d4256a46..df7156b4 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -33,8 +33,7 @@ from math import * import utility.corestats as corestats from base import webqtlConfig from utility.pillow_utils import draw_rotated_text -import utility.logger -logger = utility.logger.getLogger(__name__) + # ---- Define common colours ---- # BLUE = ImageColor.getrgb("blue") @@ -105,7 +104,6 @@ def find_outliers(vals): """ if vals: - #logger.debug("vals is:", pf(vals)) stats = corestats.Stats(vals) low_hinge = stats.percentile(25) up_hinge = stats.percentile(75) diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index e619b7b6..c7a1ea59 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -1,7 +1,3 @@ -import utility.logger -logger = utility.logger.getLogger(__name__) - - class genotype: """ Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 27dd0729..4229a91f 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -6,9 +6,6 @@ from utility import hmac from flask import g -import logging -logger = logging.getLogger(__name__) - def get_species_dataset_trait(self, start_vars): if "temp_trait" in list(start_vars.keys()): diff --git a/wqflask/utility/pillow_utils.py b/wqflask/utility/pillow_utils.py index 5713e155..e302df18 100644 --- a/wqflask/utility/pillow_utils.py +++ b/wqflask/utility/pillow_utils.py @@ -2,9 +2,6 @@ from PIL import Image, ImageColor, ImageDraw, ImageFont from utility.tools import TEMPDIR -import utility.logger -logger = utility.logger.getLogger(__name__) - BLACK = ImageColor.getrgb("black") WHITE = ImageColor.getrgb("white") diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index 641d973e..945efbbd 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -5,8 +5,6 @@ import datetime import redis # used for collections from utility.hmac import hmac_creation -from utility.logger import getLogger -logger = getLogger(__name__) def get_redis_conn(): diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index 5d239343..45d5739e 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -1,8 +1,5 @@ from gn3.db.species import get_all_species -import utility.logger -logger = utility.logger.getLogger(__name__) - def gen_dropdown_json(conn): """Generates and outputs (as json file) the data for the main dropdown menus on the home page @@ -31,7 +28,6 @@ def get_groups(species, conn): "InbredSet.FullName) ASC, IFNULL(InbredSet.Family, " "InbredSet.FullName) ASC, InbredSet.FullName ASC, " "InbredSet.MenuOrderId ASC").format(species_name) - # logger.debug(query) cursor.execute(query) results = cursor.fetchall() for result in results: diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index 3d33cc87..95cd2953 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -25,8 +25,6 @@ from utility.tools import flat_files from wqflask.database import database_connection -import utility.logger -logger = utility.logger.getLogger(__name__) version = "pre1" @@ -275,8 +273,6 @@ def get_dataset_info(dataset_name, group_name=None, file_format="json"): InbredSet.Name = "{0}" AND PublishXRef.Id = "{1}" """.format(group_name, dataset_name) - logger.debug("QUERY:", pheno_query) - pheno_results = g.db.execute(pheno_query) dataset = pheno_results.fetchone() diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 891da437..bb0973d5 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -21,9 +21,7 @@ from base.trait import retrieve_trait_info from base.trait import jsonable from base.data_set import create_dataset -from utility.logger import getLogger -logger = getLogger(__name__) Redis = get_redis_conn() @@ -48,8 +46,6 @@ def report_change(len_before, len_now): if new_length: flash("We've added {} to your collection.".format( numify(new_length, 'new trait', 'new traits'))) - else: - logger.debug("No new traits were added.") @app.route("/collections/store_trait_list", methods=('POST',)) diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index cafb9265..5df28c45 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -8,9 +8,6 @@ from utility import corr_result_helpers from scipy import stats import numpy as np -import utility.logger -logger = utility.logger.getLogger(__name__) - class CorrScatterPlot: """Page that displays a correlation scatterplot with a line fitted to it""" diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index bb928ec5..96a47eb8 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -24,8 +24,6 @@ from utility.tools import locate, GN2_BRANCH_URL from rpy2.robjects.packages import importr -import utility.logger -logger = utility.logger.getLogger(__name__) # Get pointers to some common R functions r_library = ro.r["library"] # Map the library function @@ -42,14 +40,9 @@ r_as_numeric = ro.r["as.numeric"] # Map the write.table function class CTL: def __init__(self): - logger.info("Initialization of CTL") - #log = r_file("/tmp/genenetwork_ctl.log", open = "wt") - # r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file - #r_sink(log, type = "message") # Load CTL - Should only be done once, since it is quite expensive r_library("ctl") r_options(stringsAsFactors=False) - logger.info("Initialization of CTL done, package loaded in R session") # Map the CTLscan function self.r_CTLscan = ro.r["CTLscan"] # Map the CTLsignificant function @@ -60,7 +53,6 @@ class CTL: self.r_plotCTLobject = ro.r["plot.CTLobject"] self.nodes_list = [] self.edges_list = [] - logger.info("Obtained pointers to CTL functions") self.gn2_url = GN2_BRANCH_URL @@ -85,21 +77,12 @@ class CTL: self.edges_list.append(edge_dict) def run_analysis(self, requestform): - logger.info("Starting CTL analysis on dataset") self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] self.trait_db_list = [x for x in self.trait_db_list if x] - - logger.debug("strategy:", requestform.get("strategy")) strategy = requestform.get("strategy") - - logger.debug("nperm:", requestform.get("nperm")) nperm = int(requestform.get("nperm")) - - logger.debug("parametric:", requestform.get("parametric")) parametric = bool(requestform.get("parametric")) - - logger.debug("significance:", requestform.get("significance")) significance = float(requestform.get("significance")) # Get the name of the .geno file belonging to the first phenotype @@ -109,7 +92,6 @@ class CTL: genofilelocation = locate(dataset.group.name + ".geno", "genotype") parser = genofile_parser.ConvertGenoFile(genofilelocation) parser.process_csv() - logger.debug("dataset group: ", dataset.group) # Create a genotype matrix individuals = parser.individuals markers = [] @@ -119,8 +101,6 @@ class CTL: markers.append(marker["genotypes"]) genotypes = list(itertools.chain(*markers)) - logger.debug(len(genotypes) / len(individuals), - "==", len(parser.markers)) rGeno = r_t(ro.r.matrix(r_unlist(genotypes), nrow=len(markernames), ncol=len( individuals), dimnames=r_list(markernames, individuals), byrow=True)) @@ -128,7 +108,6 @@ class CTL: # Create a phenotype matrix traits = [] for trait in self.trait_db_list: - logger.debug("retrieving data for", trait) if trait != "": ts = trait.split(':') gt = create_trait(name=ts[0], dataset_name=ts[1]) @@ -142,8 +121,6 @@ class CTL: rPheno = r_t(ro.r.matrix(r_as_numeric(r_unlist(traits)), nrow=len(self.trait_db_list), ncol=len( individuals), dimnames=r_list(self.trait_db_list, individuals), byrow=True)) - logger.debug(rPheno) - # Use a data frame to store the objects rPheno = r_data_frame(rPheno, check_names=False) rGeno = r_data_frame(rGeno, check_names=False) @@ -195,8 +172,6 @@ class CTL: # Create the interactive graph for cytoscape visualization (Nodes and Edges) if not isinstance(significant, ri.RNULLType): for x in range(len(significant[0])): - logger.debug(significant[0][x], significant[1] - [x], significant[2][x]) # Debug to console # Source tsS = significant[0][x].split(':') # Target @@ -231,7 +206,6 @@ class CTL: n = n + 1 def process_results(self, results): - logger.info("Processing CTL output") template_vars = {} template_vars["results"] = self.results template_vars["elements"] = self.elements diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index 938c453e..f52c30e4 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -6,9 +6,6 @@ import re from flask import Flask, g -from utility.logger import getLogger -logger = getLogger(__name__) - class InfoPage: def __init__(self, start_vars): diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index b0756361..97143486 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -13,10 +13,6 @@ import sys from db import webqtlDatabaseFunction from utility.tools import GN2_BASE_URL -import logging -from utility.logger import getLogger -logger = getLogger(__name__) - class DoSearch: """Parent class containing parameters/functions used for all searches""" @@ -41,7 +37,6 @@ class DoSearch: def execute(self, query): """Executes query and returns results""" query = self.normalize_spaces(query) - logger.sql(query) results = g.db.execute(query, no_parameters=True).fetchall() return results @@ -55,7 +50,6 @@ class DoSearch: def mescape(self, *items): """Multiple escape""" escaped = [escape(str(item)) for item in items] - logger.debug("escaped is:", escaped) return tuple(escaped) def normalize_spaces(self, stringy): @@ -69,8 +63,6 @@ class DoSearch: if 'key' in search_type and search_type['key'] != None: search_type_string += '_' + search_type['key'] - logger.debug("search_type_string is:", search_type_string) - if search_type_string in cls.search_types: return cls.search_types[search_type_string] else: @@ -177,8 +169,6 @@ class MrnaAssaySearch(DoSearch): def run_combined(self, from_clause='', where_clause=''): """Generates and runs a combined search of an mRNA expression dataset""" - - logger.debug("Running ProbeSetSearch") #query = self.base_query + from_clause + " WHERE " + where_clause from_clause = self.normalize_spaces(from_clause) @@ -197,11 +187,8 @@ class MrnaAssaySearch(DoSearch): def run(self): """Generates and runs a simple search of an mRNA expression dataset""" - - logger.debug("Running ProbeSetSearch") where_clause = self.get_where_clause() query = self.base_query + "WHERE " + where_clause + "ORDER BY ProbeSet.symbol ASC" - return self.execute(query) @@ -310,9 +297,6 @@ class PhenotypeSearch(DoSearch): def run_combined(self, from_clause, where_clause): """Generates and runs a combined search of an phenotype dataset""" - - logger.debug("Running PhenotypeSearch") - from_clause = self.normalize_spaces(from_clause) query = (self.base_query + @@ -370,7 +354,6 @@ class GenotypeSearch(DoSearch): where_clause.append('''%s REGEXP "%s"''' % ("%s.%s" % self.mescape(self.dataset.type, field), self.search_term)) - logger.debug("hello ;where_clause is:", pf(where_clause)) where_clause = "(%s) " % ' OR '.join(where_clause) return where_clause @@ -559,7 +542,6 @@ class LrsSearch(DoSearch): self.species_id) else: # Deal with >, <, >=, and <= - logger.debug("self.search_term is:", self.search_term) lrs_val = self.search_term[0] if self.search_type == "LOD": lrs_val = lrs_val * 4.61 @@ -794,7 +776,6 @@ class MeanSearch(MrnaAssaySearch): def run(self): self.where_clause = self.get_where_clause() - logger.debug("where_clause is:", pf(self.where_clause)) self.query = self.compile_final_query(where_clause=self.where_clause) @@ -824,9 +805,6 @@ class RangeSearch(MrnaAssaySearch): FROM ProbeSetData WHERE ProbeSetData.Id = ProbeSetXRef.dataId) > %s """ % (escape(self.search_term[0])) - - logger.debug("where_clause is:", pf(where_clause)) - return where_clause def run(self): @@ -932,11 +910,7 @@ class PvalueSearch(MrnaAssaySearch): self.search_operator, self.search_term[0]) - logger.debug("where_clause is:", pf(self.where_clause)) - self.query = self.compile_final_query(where_clause=self.where_clause) - - logger.sql(self.query) return self.execute(self.query) diff --git a/wqflask/wqflask/docs.py b/wqflask/wqflask/docs.py index 0a1a597d..9d58162e 100644 --- a/wqflask/wqflask/docs.py +++ b/wqflask/wqflask/docs.py @@ -2,9 +2,6 @@ import codecs from flask import g -from utility.logger import getLogger -logger = getLogger(__name__) - class Docs: diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 0c80e9a4..4b37c7f7 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -14,8 +14,6 @@ from gn3.computations.gemma import generate_hash_of_string from base.trait import create_trait, retrieve_trait_info -from utility.logger import getLogger -logger = getLogger(__name__) def export_traits(targs, export_type): if export_type == "collection": diff --git a/wqflask/wqflask/external_tools/send_to_bnw.py b/wqflask/wqflask/external_tools/send_to_bnw.py index c1b14ede..dfb59c63 100644 --- a/wqflask/wqflask/external_tools/send_to_bnw.py +++ b/wqflask/wqflask/external_tools/send_to_bnw.py @@ -21,9 +21,6 @@ from base.trait import GeneralTrait from utility import helper_functions, corr_result_helpers -import utility.logger -logger = utility.logger.getLogger(__name__) - class SendToBNW: def __init__(self, start_vars): diff --git a/wqflask/wqflask/external_tools/send_to_geneweaver.py b/wqflask/wqflask/external_tools/send_to_geneweaver.py index 9a4f7150..a8066b43 100644 --- a/wqflask/wqflask/external_tools/send_to_geneweaver.py +++ b/wqflask/wqflask/external_tools/send_to_geneweaver.py @@ -26,9 +26,6 @@ from base.trait import GeneralTrait, retrieve_trait_info from base.species import TheSpecies from utility import helper_functions, corr_result_helpers -import utility.logger -logger = utility.logger.getLogger(__name__) - class SendToGeneWeaver: def __init__(self, start_vars): diff --git a/wqflask/wqflask/external_tools/send_to_webgestalt.py b/wqflask/wqflask/external_tools/send_to_webgestalt.py index 6e74f4fe..4de684b0 100644 --- a/wqflask/wqflask/external_tools/send_to_webgestalt.py +++ b/wqflask/wqflask/external_tools/send_to_webgestalt.py @@ -26,9 +26,6 @@ from base.trait import GeneralTrait, retrieve_trait_info from base.species import TheSpecies from utility import helper_functions, corr_result_helpers -import utility.logger -logger = utility.logger.getLogger(__name__) - class SendToWebGestalt: def __init__(self, start_vars): diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 53a124d0..74964297 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -15,9 +15,6 @@ from utility.benchmark import Bench from utility.authentication_tools import check_resource_availability from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string -from utility.logger import getLogger -logger = getLogger(__name__) - class GSearch: @@ -65,7 +62,6 @@ class GSearch: LIMIT 6000 """ % (self.terms) with Bench("Running query"): - logger.sql(sql) re = g.db.execute(sql).fetchall() trait_list = [] @@ -210,7 +206,6 @@ class GSearch: ORDER BY Species.`Name`, InbredSet.`Name`, PublishXRef.`Id` LIMIT 6000 """.format(group_clause, search_term) - logger.sql(sql) re = g.db.execute(sql).fetchall() trait_list = [] with Bench("Creating trait objects"): diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 001bab3b..1c8a4ff6 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -8,12 +8,9 @@ from utility import helper_functions from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR from redis import Redis from flask import Flask, g -from utility.logger import getLogger Redis = Redis() -logger = getLogger(__name__) - class Heatmap: diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 6a62b45c..278d8a6b 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -47,12 +47,11 @@ from wqflask.interval_analyst import GeneUtil from base.webqtlConfig import GENERATED_IMAGE_DIR from utility.pillow_utils import draw_rotated_text, draw_open_polygon -import utility.logger + try: # Only import this for Python3 from functools import reduce except: pass -logger = utility.logger.getLogger(__name__) RED = ImageColor.getrgb("red") BLUE = ImageColor.getrgb("blue") @@ -247,8 +246,6 @@ class DisplayMappingResults: HELP_PAGE_REF = '/glossary.html' def __init__(self, start_vars): - logger.info("Running qtlreaper") - self.temp_uuid = start_vars['temp_uuid'] self.dataset = start_vars['dataset'] diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index 6b525733..646728ba 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -13,8 +13,6 @@ from utility.tools import TEMPDIR from utility.tools import WEBSERVER_MODE from gn3.computations.gemma import generate_hash_of_string -import utility.logger -logger = utility.logger.getLogger(__name__) GEMMAOPTS = "-debug" if WEBSERVER_MODE == 'PROD': diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index 2fa80841..75ee189e 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -5,9 +5,6 @@ from base.webqtlConfig import TMPDIR from utility import webqtlUtil from utility.tools import flat_files, PLINK_COMMAND -import utility.logger -logger = utility.logger.getLogger(__name__) - def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr( @@ -15,13 +12,11 @@ def run_plink(this_trait, dataset, species, vals, maf): gen_pheno_txt_file(dataset, vals) plink_command = f"{PLINK_COMMAND} --noweb --bfile {flat_files('mapping')}/{dataset.group.name} --no-pheno --no-fid --no-parents --no-sex --maf {maf} --out { TMPDIR}{plink_output_filename} --assoc " - logger.debug("plink_command:", plink_command) os.system(plink_command) count, p_values = parse_plink_output(plink_output_filename, species) - logger.debug("p_values:", p_values) dataset.group.markers.add_pvalues(p_values) return dataset.group.markers.markers diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index c4b495d7..4d5db2ee 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -10,9 +10,6 @@ from base.trait import GeneralTrait from base.data_set import create_dataset from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR -import utility.logger -logger = utility.logger.getLogger(__name__) - def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boot_check, num_bootstrap, do_control, control_marker, manhattan_plot, first_run=True, output_files=None): """Generates p-values for each marker using qtlreaper""" @@ -67,8 +64,6 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo opt_list), webqtlConfig.GENERATED_IMAGE_DIR, output_filename)) - - logger.debug("reaper_command:" + reaper_command) os.system(reaper_command) else: output_filename, permu_filename, bootstrap_filename = output_files diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 3bf06ea6..7d112c68 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -17,9 +17,6 @@ from base.webqtlConfig import TMPDIR from base.trait import create_trait from utility.tools import locate, GN3_LOCAL_URL -import utility.logger -logger = utility.logger.getLogger(__name__) - def run_rqtl(trait_name, vals, samples, dataset, pair_scan, mapping_scale, model, method, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, cofactors): """Run R/qtl by making a request to the GN3 endpoint and reading in the output file(s)""" diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 35678df9..7a424b17 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -41,9 +41,6 @@ from utility.tools import locate, locate_ignore_error, GEMMA_COMMAND, PLINK_COMM from utility.external import shell from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR -import utility.logger -logger = utility.logger.getLogger(__name__) - class RunMapping: @@ -273,7 +270,6 @@ class RunMapping: self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] - logger.info("Running qtlreaper") self.first_run = True self.output_files = None @@ -303,8 +299,6 @@ class RunMapping: results = plink_mapping.run_plink( self.this_trait, self.dataset, self.species, self.vals, self.maf) #results = self.run_plink() - else: - logger.debug("RUNNING NOTHING") self.no_results = False if len(results) == 0: diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index 7a808ac9..ddf48d90 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -21,9 +21,6 @@ import re from pprint import pformat as pf -from utility.logger import getLogger -logger = getLogger(__name__) - def parse(pstring): """ @@ -45,7 +42,6 @@ def parse(pstring): for item in pstring: splat = re.split(separators, item) - logger.debug("splat is:", splat) # splat is an array of 1 if no match, otherwise more than 1 if len(splat) > 1: @@ -73,7 +69,6 @@ def parse(pstring): search_term=[item]) items.append(term) - logger.debug("* items are:", pf(items) + "\n") return(items) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 7134cc24..bf7e3c93 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -25,8 +25,6 @@ from utility.authentication_tools import check_resource_availability from utility.tools import GN2_BASE_URL from utility.type_checking import is_str -from utility.logger import getLogger -logger = getLogger(__name__) class SearchResultPage: #maxReturn = 3000 @@ -56,7 +54,6 @@ class SearchResultPage: rx = re.compile( r'.*\W(href|http|sql|select|update)\W.*', re.IGNORECASE) if rx.match(search): - logger.debug("Regex failed search") self.search_term_exists = False return else: diff --git a/wqflask/wqflask/update_search_results.py b/wqflask/wqflask/update_search_results.py index 2e467dc8..6b1b4e97 100644 --- a/wqflask/wqflask/update_search_results.py +++ b/wqflask/wqflask/update_search_results.py @@ -7,9 +7,6 @@ from db import webqtlDatabaseFunction from utility.benchmark import Bench -from utility.logger import getLogger -logger = getLogger(__name__) - class GSearch: @@ -47,7 +44,6 @@ class GSearch: LIMIT 6000 """ % (self.terms) with Bench("Running query"): - logger.sql(sql) re = g.db.execute(sql).fetchall() self.trait_list = [] with Bench("Creating trait objects"): @@ -92,7 +88,6 @@ class GSearch: ORDER BY Species.`Name`, InbredSet.`Name`, PublishXRef.`Id` LIMIT 6000 """ % (self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms) - logger.sql(sql) re = g.db.execute(sql).fetchall() self.trait_list = [] with Bench("Creating trait objects"): diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index 45a12f77..ae61edb0 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -2,7 +2,6 @@ import os import hashlib import datetime import time -import logging import uuid import hmac import base64 @@ -21,9 +20,6 @@ from utility import hmac from utility.redis_tools import is_redis_available, get_redis_conn, get_user_id, get_user_by_unique_column, set_user_attribute, save_user, save_verification_code, check_verification_code, get_user_collections, save_collections Redis = get_redis_conn() -from utility.logger import getLogger -logger = getLogger(__name__) - from smtplib import SMTP from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_ALCHEMY, GN2_BRANCH_URL @@ -129,7 +125,6 @@ def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): server.login(SMTP_USERNAME, SMTP_PASSWORD) server.sendmail(fromaddr, toaddr, msg) server.quit() - logger.info("Successfully sent email to " + toaddr) def send_verification_email(user_details, template_name="email/user_verification.txt", key_prefix="verification_code", subject="GeneNetwork e-mail verification"): @@ -346,7 +341,6 @@ def get_github_user_details(access_token): @app.route("/n/logout") def logout(): - logger.debug("Logging out...") UserSession().delete_session() flash("You are now logged out. We hope you come back soon!") response = make_response(redirect(url_for('index_page'))) @@ -404,7 +398,6 @@ def forgot_password_submit(): email_address = params['email_address'] next_page = None if email_address != "": - logger.debug("Wants to send password E-mail to ", email_address) user_details = get_user_by_unique_column( "email_address", email_address) if user_details: @@ -425,8 +418,6 @@ def forgot_password_submit(): @app.route("/n/password_reset", methods=['GET']) def password_reset(): """Entry point after user clicks link in E-mail""" - logger.debug("in password_reset request.url is:", request.url) - verification_code = request.args.get('code') hmac = request.args.get('hm') @@ -446,8 +437,6 @@ def password_reset(): @app.route("/n/password_reset_step2", methods=('POST',)) def password_reset_step2(): """Handle confirmation E-mail for password reset""" - logger.debug("in password_reset request.url is:", request.url) - errors = [] user_email = request.form['user_encode'] user_id = get_user_id("email_address", user_email) @@ -515,7 +504,6 @@ def register(): params = params.to_dict(flat=True) if params: - logger.debug("Attempting to register the user...") errors = register_user(params) if len(errors) == 0: diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 00b268a7..1ed5b802 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -13,8 +13,6 @@ from utility import hmac from utility.redis_tools import get_redis_conn, get_user_id, get_user_by_unique_column, set_user_attribute, get_user_collections, save_collections Redis = get_redis_conn() -from utility.logger import getLogger -logger = getLogger(__name__) THREE_DAYS = 60 * 60 * 24 * 3 THIRTY_DAYS = 60 * 60 * 24 * 30 @@ -54,7 +52,6 @@ def create_signed_cookie(): the_uuid = str(uuid.uuid4()) signature = hmac.hmac_creation(the_uuid) uuid_signed = the_uuid + ":" + signature - logger.debug("uuid_signed:", uuid_signed) return the_uuid, uuid_signed -- cgit v1.2.3 From 9d0f492d8a999c1a3eb6525b0ae72bbbcaad4279 Mon Sep 17 00:00:00 2001 From: Munyoki Kilyungi Date: Tue, 30 Aug 2022 11:36:58 +0300 Subject: Remove "with Bench ..." calls "with Bench" instruments how long a function takes and generates time reports on as INFO logs. This should be done on a developer server. Should the log level be low enough, this bench marks will generate a lot of noise. Instrumentation should be done during development. * wqflask/base/data_set.py (create_datasets_list): Remove "with Bench...". * wqflask/db/call.py (fetchone): Ditto. (fetchall): Ditto. (gn_server): Ditto. * wqflask/wqflask/gsearch.py (GSearch.__init__): Ditto. * wqflask/wqflask/marker_regression/display_mapping_results.py (DisplayMappingResults.__init__): Ditto. * wqflask/wqflask/marker_regression/run_mapping.py (RunMapping.__init__): Ditto. * wqflask/wqflask/update_search_results.py (GSearch.__init__): Ditto. * wqflask/wqflask/views.py (search_page): Ditto. (heatmap_page): Ditto. (mapping_results_page): Ditto. --- wqflask/base/data_set.py | 20 +- wqflask/db/call.py | 38 ++-- wqflask/wqflask/gsearch.py | 240 ++++++++++----------- .../marker_regression/display_mapping_results.py | 6 +- wqflask/wqflask/marker_regression/run_mapping.py | 62 +++--- wqflask/wqflask/update_search_results.py | 34 ++- wqflask/wqflask/views.py | 68 +++--- 7 files changed, 216 insertions(+), 252 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8206b67c..33e6f353 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -32,7 +32,6 @@ from maintenance import get_group_samplelists from utility.tools import locate, locate_ignore_error, flat_files from utility import gen_geno_ob from utility import chunks -from utility.benchmark import Bench from utility import webqtlUtil from db import webqtlDatabaseFunction from base import species @@ -201,16 +200,15 @@ def create_datasets_list(): if result is None: datasets = list() - with Bench("Creating DataSets object"): - type_dict = {'Publish': 'PublishFreeze', - 'ProbeSet': 'ProbeSetFreeze', - 'Geno': 'GenoFreeze'} - - for dataset_type in type_dict: - query = "SELECT Name FROM {}".format(type_dict[dataset_type]) - for result in fetchall(query): - dataset = create_dataset(result.Name, dataset_type) - datasets.append(dataset) + type_dict = {'Publish': 'PublishFreeze', + 'ProbeSet': 'ProbeSetFreeze', + 'Geno': 'GenoFreeze'} + + for dataset_type in type_dict: + query = "SELECT Name FROM {}".format(type_dict[dataset_type]) + for result in fetchall(query): + dataset = create_dataset(result.Name, dataset_type) + datasets.append(dataset) if USE_REDIS: r.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL)) diff --git a/wqflask/db/call.py b/wqflask/db/call.py index a6bbda54..c27e0326 100644 --- a/wqflask/db/call.py +++ b/wqflask/db/call.py @@ -11,13 +11,10 @@ except: import urllib2 import json from utility.tools import USE_GN_SERVER, LOG_SQL, GN_SERVER_URL -from utility.benchmark import Bench from utility.logger import getLogger logger = getLogger(__name__) -# from inspect import stack - def fetch1(query, path=None, func=None): """Fetch one result as a Tuple using either a SQL query or the URI @@ -41,11 +38,10 @@ def fetchone(query): original fetchone, but with logging) """ - with Bench("SQL", LOG_SQL): - def helper(query): - res = g.db.execute(query) - return res.fetchone() - return logger.sql(query, helper) + def helper(query): + res = g.db.execute(query) + return res.fetchone() + return logger.sql(query, helper) def fetchall(query): @@ -53,23 +49,21 @@ def fetchall(query): original fetchall, but with logging) """ - with Bench("SQL", LOG_SQL): - def helper(query): - res = g.db.execute(query) - return res.fetchall() - return logger.sql(query, helper) + def helper(query): + res = g.db.execute(query) + return res.fetchall() + return logger.sql(query, helper) def gn_server(path): """Return JSON record by calling GN_SERVER """ - with Bench("GN_SERVER", LOG_SQL): - res = () - try: - res = urllib.request.urlopen(GN_SERVER_URL + path) - except: - res = urllib2.urlopen(GN_SERVER_URL + path) - rest = res.read() - res2 = json.loads(rest) - return res2 + res = () + try: + res = urllib.request.urlopen(GN_SERVER_URL + path) + except: + res = urllib2.urlopen(GN_SERVER_URL + path) + rest = res.read() + res2 = json.loads(rest) + return res2 diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 74964297..6d49bc79 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -11,7 +11,6 @@ from base import webqtlConfig from utility import hmac -from utility.benchmark import Bench from utility.authentication_tools import check_resource_availability from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string @@ -61,74 +60,72 @@ class GSearch: ORDER BY species_name, inbredset_name, tissue_name, probesetfreeze_name, probeset_name LIMIT 6000 """ % (self.terms) - with Bench("Running query"): - re = g.db.execute(sql).fetchall() + re = g.db.execute(sql).fetchall() trait_list = [] dataset_to_permissions = {} - with Bench("Creating trait objects"): - for i, line in enumerate(re): - this_trait = {} - this_trait['index'] = i + 1 - this_trait['name'] = line[5] - this_trait['dataset'] = line[3] - this_trait['dataset_fullname'] = line[4] - this_trait['hmac'] = hmac.data_hmac( - '{}:{}'.format(line[5], line[3])) - this_trait['species'] = line[0] - this_trait['group'] = line[1] - this_trait['tissue'] = line[2] - this_trait['symbol'] = "N/A" - if line[6]: - this_trait['symbol'] = line[6] - this_trait['description'] = "N/A" - if line[7]: - this_trait['description'] = line[7].decode( - 'utf-8', 'replace') - this_trait['location_repr'] = "N/A" - if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): - this_trait['location_repr'] = 'Chr%s: %.6f' % ( - line[8], float(line[9])) + for i, line in enumerate(re): + this_trait = {} + this_trait['index'] = i + 1 + this_trait['name'] = line[5] + this_trait['dataset'] = line[3] + this_trait['dataset_fullname'] = line[4] + this_trait['hmac'] = hmac.data_hmac( + '{}:{}'.format(line[5], line[3])) + this_trait['species'] = line[0] + this_trait['group'] = line[1] + this_trait['tissue'] = line[2] + this_trait['symbol'] = "N/A" + if line[6]: + this_trait['symbol'] = line[6] + this_trait['description'] = "N/A" + if line[7]: + this_trait['description'] = line[7].decode( + 'utf-8', 'replace') + this_trait['location_repr'] = "N/A" + if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): + this_trait['location_repr'] = 'Chr%s: %.6f' % ( + line[8], float(line[9])) - this_trait['LRS_score_repr'] = "N/A" - this_trait['additive'] = "N/A" - this_trait['mean'] = "N/A" + this_trait['LRS_score_repr'] = "N/A" + this_trait['additive'] = "N/A" + this_trait['mean'] = "N/A" - if line[11] != "" and line[11] != None: - this_trait['LRS_score_repr'] = f"{line[11]:.3f}" - if line[14] != "" and line[14] != None: - this_trait['additive'] = f"{line[14]:.3f}" - if line[10] != "" and line[10] != None: - this_trait['mean'] = f"{line[10]:.3f}" + if line[11] != "" and line[11] != None: + this_trait['LRS_score_repr'] = f"{line[11]:.3f}" + if line[14] != "" and line[14] != None: + this_trait['additive'] = f"{line[14]:.3f}" + if line[10] != "" and line[10] != None: + this_trait['mean'] = f"{line[10]:.3f}" - locus_chr = line[16] - locus_mb = line[17] + locus_chr = line[16] + locus_mb = line[17] - max_lrs_text = "N/A" - if locus_chr and locus_mb: - max_lrs_text = f"Chr{locus_chr}: {locus_mb}" - this_trait['max_lrs_text'] = max_lrs_text + max_lrs_text = "N/A" + if locus_chr and locus_mb: + max_lrs_text = f"Chr{locus_chr}: {locus_mb}" + this_trait['max_lrs_text'] = max_lrs_text - this_trait['additive'] = "N/A" - if line[14] != "" and line[14] != None: - this_trait['additive'] = '%.3f' % line[14] - this_trait['dataset_id'] = line[15] + this_trait['additive'] = "N/A" + if line[14] != "" and line[14] != None: + this_trait['additive'] = '%.3f' % line[14] + this_trait['dataset_id'] = line[15] - dataset_ob = SimpleNamespace( - id=this_trait["dataset_id"], type="ProbeSet", name=this_trait["dataset"], species=this_trait["species"]) - if dataset_ob.id not in dataset_to_permissions: - permissions = check_resource_availability(dataset_ob) - dataset_to_permissions[dataset_ob.id] = permissions - else: - pemissions = dataset_to_permissions[dataset_ob.id] - if type(permissions['data']) is list: - if "view" not in permissions['data']: - continue - else: - if permissions['data'] == 'no-access': - continue + dataset_ob = SimpleNamespace( + id=this_trait["dataset_id"], type="ProbeSet", name=this_trait["dataset"], species=this_trait["species"]) + if dataset_ob.id not in dataset_to_permissions: + permissions = check_resource_availability(dataset_ob) + dataset_to_permissions[dataset_ob.id] = permissions + else: + pemissions = dataset_to_permissions[dataset_ob.id] + if type(permissions['data']) is list: + if "view" not in permissions['data']: + continue + else: + if permissions['data'] == 'no-access': + continue - trait_list.append(this_trait) + trait_list.append(this_trait) self.trait_count = len(trait_list) self.trait_list = trait_list @@ -208,76 +205,75 @@ class GSearch: """.format(group_clause, search_term) re = g.db.execute(sql).fetchall() trait_list = [] - with Bench("Creating trait objects"): - for i, line in enumerate(re): - this_trait = {} - this_trait['index'] = i + 1 - this_trait['name'] = str(line[4]) - if len(str(line[12])) == 3: - this_trait['display_name'] = str( - line[12]) + "_" + this_trait['name'] - else: - this_trait['display_name'] = this_trait['name'] - this_trait['dataset'] = line[2] - this_trait['dataset_fullname'] = line[3] - this_trait['hmac'] = hmac.data_hmac( - '{}:{}'.format(line[4], line[2])) - this_trait['species'] = line[0] - this_trait['group'] = line[1] - if line[9] != None and line[6] != None: - this_trait['description'] = line[6].decode( - 'utf-8', 'replace') - elif line[5] != None: - this_trait['description'] = line[5].decode( - 'utf-8', 'replace') - else: - this_trait['description'] = "N/A" - this_trait['dataset_id'] = line[14] + for i, line in enumerate(re): + this_trait = {} + this_trait['index'] = i + 1 + this_trait['name'] = str(line[4]) + if len(str(line[12])) == 3: + this_trait['display_name'] = str( + line[12]) + "_" + this_trait['name'] + else: + this_trait['display_name'] = this_trait['name'] + this_trait['dataset'] = line[2] + this_trait['dataset_fullname'] = line[3] + this_trait['hmac'] = hmac.data_hmac( + '{}:{}'.format(line[4], line[2])) + this_trait['species'] = line[0] + this_trait['group'] = line[1] + if line[9] != None and line[6] != None: + this_trait['description'] = line[6].decode( + 'utf-8', 'replace') + elif line[5] != None: + this_trait['description'] = line[5].decode( + 'utf-8', 'replace') + else: + this_trait['description'] = "N/A" + this_trait['dataset_id'] = line[14] - this_trait['LRS_score_repr'] = "N/A" - this_trait['additive'] = "N/A" - this_trait['mean'] = "N/A" + this_trait['LRS_score_repr'] = "N/A" + this_trait['additive'] = "N/A" + this_trait['mean'] = "N/A" - if line[10] != "" and line[10] != None: - this_trait['LRS_score_repr'] = f"{line[10]:.3f}" - # Some Max LRS values in the DB are wrongly listed as 0.000, but shouldn't be displayed - if this_trait['LRS_score_repr'] == "0.000": - this_trait['LRS_score_repr'] = "N/A" - if line[11] != "" and line[11] != None: - this_trait['additive'] = f"{line[11]:.3f}" - if line[13] != "" and line[13] != None: - this_trait['mean'] = f"{line[13]:.3f}" + if line[10] != "" and line[10] != None: + this_trait['LRS_score_repr'] = f"{line[10]:.3f}" + # Some Max LRS values in the DB are wrongly listed as 0.000, but shouldn't be displayed + if this_trait['LRS_score_repr'] == "0.000": + this_trait['LRS_score_repr'] = "N/A" + if line[11] != "" and line[11] != None: + this_trait['additive'] = f"{line[11]:.3f}" + if line[13] != "" and line[13] != None: + this_trait['mean'] = f"{line[13]:.3f}" - locus_chr = line[15] - locus_mb = line[16] + locus_chr = line[15] + locus_mb = line[16] - max_lrs_text = "N/A" - if locus_chr and locus_mb: - max_lrs_text = f"Chr{locus_chr}: {locus_mb}" - this_trait['max_lrs_text'] = max_lrs_text + max_lrs_text = "N/A" + if locus_chr and locus_mb: + max_lrs_text = f"Chr{locus_chr}: {locus_mb}" + this_trait['max_lrs_text'] = max_lrs_text - this_trait['authors'] = line[7] - this_trait['year'] = line[8] - this_trait['pubmed_text'] = "N/A" - this_trait['pubmed_link'] = "N/A" - if this_trait['year'].isdigit(): - this_trait['pubmed_text'] = this_trait['year'] - if line[9] != "" and line[9] != None: - this_trait['pubmed_link'] = webqtlConfig.PUBMEDLINK_URL % line[8] - if line[12]: - this_trait['display_name'] = line[12] + \ - "_" + str(this_trait['name']) + this_trait['authors'] = line[7] + this_trait['year'] = line[8] + this_trait['pubmed_text'] = "N/A" + this_trait['pubmed_link'] = "N/A" + if this_trait['year'].isdigit(): + this_trait['pubmed_text'] = this_trait['year'] + if line[9] != "" and line[9] != None: + this_trait['pubmed_link'] = webqtlConfig.PUBMEDLINK_URL % line[8] + if line[12]: + this_trait['display_name'] = line[12] + \ + "_" + str(this_trait['name']) - dataset_ob = SimpleNamespace(id=this_trait["dataset_id"], type="Publish", species=this_trait["species"]) - permissions = check_resource_availability(dataset_ob, this_trait['name']) - if type(permissions['data']) is list: - if "view" not in permissions['data']: - continue - else: - if permissions['data'] == 'no-access': - continue + dataset_ob = SimpleNamespace(id=this_trait["dataset_id"], type="Publish", species=this_trait["species"]) + permissions = check_resource_availability(dataset_ob, this_trait['name']) + if type(permissions['data']) is list: + if "view" not in permissions['data']: + continue + else: + if permissions['data'] == 'no-access': + continue - trait_list.append(this_trait) + trait_list.append(this_trait) self.trait_count = len(trait_list) self.trait_list = trait_list diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 278d8a6b..ef0c357e 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -42,7 +42,6 @@ from base import webqtlConfig from base.GeneralObject import GeneralObject from utility import webqtlUtil from utility import Plot -from utility.benchmark import Bench from wqflask.interval_analyst import GeneUtil from base.webqtlConfig import GENERATED_IMAGE_DIR from utility.pillow_utils import draw_rotated_text, draw_open_polygon @@ -587,9 +586,8 @@ class DisplayMappingResults: ################################################################ showLocusForm = "" intCanvas = Image.new("RGBA", size=(self.graphWidth, self.graphHeight)) - with Bench("Drawing Plot"): - gifmap = self.plotIntMapping( - intCanvas, startMb=self.startMb, endMb=self.endMb, showLocusForm=showLocusForm) + gifmap = self.plotIntMapping( + intCanvas, startMb=self.startMb, endMb=self.endMb, showLocusForm=showLocusForm) self.gifmap = gifmap.__str__() diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 7a424b17..c57596e9 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -33,7 +33,6 @@ from utility import webqtlUtil from utility import helper_functions from utility import Plot, Bunch from utility import temp_data -from utility.benchmark import Bench from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping from wqflask.show_trait.SampleList import SampleList @@ -204,13 +203,12 @@ class RunMapping: self.first_run = False self.score_type = "-logP" self.manhattan_plot = True - with Bench("Running GEMMA"): - if self.use_loco == "True": - marker_obs, self.output_files = gemma_mapping.run_gemma( - self.this_trait, self.dataset, self.samples, self.vals, self.covariates, self.use_loco, self.maf, self.first_run, self.output_files) - else: - marker_obs, self.output_files = gemma_mapping.run_gemma( - self.this_trait, self.dataset, self.samples, self.vals, self.covariates, self.use_loco, self.maf, self.first_run, self.output_files) + if self.use_loco == "True": + marker_obs, self.output_files = gemma_mapping.run_gemma( + self.this_trait, self.dataset, self.samples, self.vals, self.covariates, self.use_loco, self.maf, self.first_run, self.output_files) + else: + marker_obs, self.output_files = gemma_mapping.run_gemma( + self.this_trait, self.dataset, self.samples, self.vals, self.covariates, self.use_loco, self.maf, self.first_run, self.output_files) results = marker_obs elif self.mapping_method == "rqtl_plink": results = self.run_rqtl_plink() @@ -372,33 +370,29 @@ class RunMapping: self.qtl_results.append(marker) total_markers = len(self.qtl_results) + export_mapping_results(self.dataset, self.this_trait, self.qtl_results, self.mapping_results_path, + self.mapping_method, self.mapping_scale, self.score_type, + self.transform, self.covariates, self.n_samples, self.vals_hash) + + if len(self.qtl_results) > 30000: + self.qtl_results = trim_markers_for_figure( + self.qtl_results) + self.results_for_browser = trim_markers_for_figure( + self.results_for_browser) + filtered_annotations = [] + for marker in self.results_for_browser: + for annot_marker in self.annotations_for_browser: + if annot_marker['rs'] == marker['rs']: + filtered_annotations.append(annot_marker) + break + self.annotations_for_browser = filtered_annotations + browser_files = write_input_for_browser( + self.dataset, self.results_for_browser, self.annotations_for_browser) + else: + browser_files = write_input_for_browser( + self.dataset, self.results_for_browser, self.annotations_for_browser) - with Bench("Exporting Results"): - export_mapping_results(self.dataset, self.this_trait, self.qtl_results, self.mapping_results_path, - self.mapping_method, self.mapping_scale, self.score_type, - self.transform, self.covariates, self.n_samples, self.vals_hash) - - with Bench("Trimming Markers for Figure"): - if len(self.qtl_results) > 30000: - self.qtl_results = trim_markers_for_figure( - self.qtl_results) - self.results_for_browser = trim_markers_for_figure( - self.results_for_browser) - filtered_annotations = [] - for marker in self.results_for_browser: - for annot_marker in self.annotations_for_browser: - if annot_marker['rs'] == marker['rs']: - filtered_annotations.append(annot_marker) - break - self.annotations_for_browser = filtered_annotations - browser_files = write_input_for_browser( - self.dataset, self.results_for_browser, self.annotations_for_browser) - else: - browser_files = write_input_for_browser( - self.dataset, self.results_for_browser, self.annotations_for_browser) - - with Bench("Trimming Markers for Table"): - self.trimmed_markers = trim_markers_for_table(results) + self.trimmed_markers = trim_markers_for_table(results) chr_lengths = get_chr_lengths( self.mapping_scale, self.mapping_method, self.dataset, self.qtl_results) diff --git a/wqflask/wqflask/update_search_results.py b/wqflask/wqflask/update_search_results.py index 6b1b4e97..f132e2c6 100644 --- a/wqflask/wqflask/update_search_results.py +++ b/wqflask/wqflask/update_search_results.py @@ -5,8 +5,6 @@ from base.data_set import create_dataset from base.trait import GeneralTrait from db import webqtlDatabaseFunction -from utility.benchmark import Bench - class GSearch: @@ -43,18 +41,15 @@ class GSearch: ORDER BY species_name, inbredset_name, tissue_name, probesetfreeze_name, probeset_name LIMIT 6000 """ % (self.terms) - with Bench("Running query"): - re = g.db.execute(sql).fetchall() + re = g.db.execute(sql).fetchall() self.trait_list = [] - with Bench("Creating trait objects"): - for line in re: - dataset = create_dataset( - line[3], "ProbeSet", get_samplelist=False) - trait_id = line[4] - # with Bench("Building trait object"): - this_trait = GeneralTrait( - dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) - self.trait_list.append(this_trait) + for line in re: + dataset = create_dataset( + line[3], "ProbeSet", get_samplelist=False) + trait_id = line[4] + this_trait = GeneralTrait( + dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) + self.trait_list.append(this_trait) elif self.type == "phenotype": sql = """ @@ -90,13 +85,12 @@ class GSearch: """ % (self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms, self.terms) re = g.db.execute(sql).fetchall() self.trait_list = [] - with Bench("Creating trait objects"): - for line in re: - dataset = create_dataset(line[2], "Publish") - trait_id = line[3] - this_trait = GeneralTrait( - dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) - self.trait_list.append(this_trait) + for line in re: + dataset = create_dataset(line[2], "Publish") + trait_id = line[3] + this_trait = GeneralTrait( + dataset=dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) + self.trait_list.append(this_trait) self.results = self.convert_to_json() diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index d80d557b..acd65587 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -95,7 +95,6 @@ from utility.redis_tools import get_redis_conn from base.webqtlConfig import GENERATED_IMAGE_DIR, DEFAULT_PRIVILEGES -from utility.benchmark import Bench from pprint import pformat as pf @@ -217,12 +216,11 @@ def twitter(filename): def search_page(): result = None if USE_REDIS: - with Bench("Trying Redis cache"): - key = "search_results:v1:" + \ - json.dumps(request.args, sort_keys=True) - result = Redis.get(key) - if result: - result = pickle.loads(result) + key = "search_results:v1:" + \ + json.dumps(request.args, sort_keys=True) + result = Redis.get(key) + if result: + result = pickle.loads(result) the_search = SearchResultPage(request.args) result = the_search.__dict__ valid_search = result['search_term_exists'] @@ -529,12 +527,10 @@ def heatmap_page(): version = "v5" key = "heatmap:{}:".format( version) + json.dumps(start_vars, sort_keys=True) - with Bench("Loading cache"): - result = Redis.get(key) + result = Redis.get(key) if result: - with Bench("Loading results"): - result = pickle.loads(result) + result = pickle.loads(result) else: template_vars = heatmap.Heatmap(request.form, temp_uuid) @@ -547,9 +543,7 @@ def heatmap_page(): pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) Redis.set(key, pickled_result) Redis.expire(key, 60 * 60) - - with Bench("Rendering template"): - rendered_template = render_template("heatmap.html", **result) + rendered_template = render_template("heatmap.html", **result) else: rendered_template = render_template( @@ -762,40 +756,36 @@ def mapping_results_page(): version = "v3" key = "mapping_results:{}:".format( version) + json.dumps(start_vars, sort_keys=True) - with Bench("Loading cache"): - result = None # Just for testing + result = None # Just for testing if result: - with Bench("Loading results"): - result = pickle.loads(result) + result = pickle.loads(result) else: - with Bench("Total time in RunMapping"): - try: - template_vars = run_mapping.RunMapping(start_vars, temp_uuid) - if template_vars.no_results: - rendered_template = render_template("mapping_error.html") - return rendered_template - except: + try: + template_vars = run_mapping.RunMapping(start_vars, temp_uuid) + if template_vars.no_results: rendered_template = render_template("mapping_error.html") return rendered_template + except: + rendered_template = render_template("mapping_error.html") + return rendered_template - if not template_vars.pair_scan: - template_vars.js_data = json.dumps(template_vars.js_data, - default=json_default_handler, - indent=" ") + if not template_vars.pair_scan: + template_vars.js_data = json.dumps(template_vars.js_data, + default=json_default_handler, + indent=" ") - result = template_vars.__dict__ + result = template_vars.__dict__ - if result['pair_scan']: - with Bench("Rendering template"): - rendered_template = render_template( - "pair_scan_results.html", **result) - else: - gn1_template_vars = display_mapping_results.DisplayMappingResults( - result).__dict__ + if result['pair_scan']: + rendered_template = render_template( + "pair_scan_results.html", **result) + else: + gn1_template_vars = display_mapping_results.DisplayMappingResults( + result).__dict__ - rendered_template = render_template( - "mapping_results.html", **gn1_template_vars) + rendered_template = render_template( + "mapping_results.html", **gn1_template_vars) return rendered_template -- cgit v1.2.3 From d9a6e9433e4fd5cc20ed45a53c34cab4357eac8e Mon Sep 17 00:00:00 2001 From: Munyoki Kilyungi Date: Tue, 30 Aug 2022 21:39:38 +0300 Subject: Replace fetchall, fetchone, fetch1 with database_connection * wqflask/base/data_set.py: Replace "db.call" import with "database_connection". (create_datasets_list): Use "database_connection" to fetch data. (DatasetGroup.__init__): Ditto. (DataSet.retrieve_other_names): Ditto. (PhenotypeDataSet.setup): Remove query escaping in string and format the string. (GenotypeDataSet.setup): Ditto. (MrnaAssayDataSet.setup): Ditto. * wqflask/db/webqtlDatabaseFunction.py: Remove db.call import. (retrieve_species): Use database_connection() to fetch data. (retrieve_species_id): Ditto. --- wqflask/base/data_set.py | 115 ++++++++++++++--------------------- wqflask/db/webqtlDatabaseFunction.py | 16 +++-- 2 files changed, 55 insertions(+), 76 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 33e6f353..57bd6be3 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -21,7 +21,6 @@ from dataclasses import dataclass from dataclasses import field from dataclasses import InitVar from typing import Optional, Dict, List -from db.call import fetchall, fetchone, fetch1 from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL from db.gn_server import menu_main from pprint import pformat as pf @@ -40,6 +39,7 @@ from flask import Flask, g from base.webqtlConfig import TMPDIR from urllib.parse import urlparse from utility.tools import SQL_URI +from wqflask.database import database_connection import os import math import string @@ -205,11 +205,14 @@ def create_datasets_list(): 'Geno': 'GenoFreeze'} for dataset_type in type_dict: - query = "SELECT Name FROM {}".format(type_dict[dataset_type]) - for result in fetchall(query): - dataset = create_dataset(result.Name, dataset_type) - datasets.append(dataset) - + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute("SELECT Name FROM %s", + (type_dict[dataset_type],)) + results = cursor.fetchall(query) + if results: + for result in results: + datasets.append( + create_dataset(result.Name, dataset_type)) if USE_REDIS: r.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL)) r.expire(key, 60 * 60) @@ -319,12 +322,16 @@ class DatasetGroup: def __init__(self, dataset, name=None): """This sets self.group and self.group_id""" - if name == None: - self.name, self.id, self.genetic_type, self.code = fetchone( - dataset.query_for_group) - else: - self.name, self.id, self.genetic_type, self.code = fetchone( - "SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode FROM InbredSet where Name='%s'" % name) + query = """SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, + InbredSet.InbredSetCode FROM InbredSet where Name=%s + """ + if not name: + query, name = dataset.query_for_group, self.name + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute(query, (name,)) + results = cursor.fetchone() + if results: + self.name, self.id, self.genetic_type, self.code = results if self.name == 'BXD300': self.name = "BXD" @@ -620,36 +627,25 @@ class DataSet: """ try: + query = "" + _vars = None if self.type == "ProbeSet": - query_args = tuple(escape(x) for x in ( - self.name, - self.name, - self.name)) - - self.id, self.name, self.fullname, self.shortname, self.data_scale, self.tissue = fetch1(""" - SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, ProbeSetFreeze.DataScale, Tissue.Name - FROM ProbeSetFreeze, ProbeFreeze, Tissue - WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id - AND ProbeFreeze.TissueId = Tissue.Id - AND (ProbeSetFreeze.Name = '%s' OR ProbeSetFreeze.FullName = '%s' OR ProbeSetFreeze.ShortName = '%s') - """ % (query_args), "/dataset/" + self.name + ".json", - lambda r: (r["id"], r["name"], r["full_name"], - r["short_name"], r["data_scale"], r["tissue"]) - ) + query = """ +SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, ProbeSetFreeze.DataScale, Tissue.Name +FROM ProbeSetFreeze, ProbeFreeze, Tissue +WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id +AND ProbeFreeze.TissueId = Tissue.Id +AND (ProbeSetFreeze.Name = %s OR ProbeSetFreeze.FullName = %s OR ProbeSetFreeze.ShortName = %s""" + _vars = (self.id, self.name, self.fullname, self.shortname, self.data_scale, self.tissue,) else: - query_args = tuple(escape(x) for x in ( - (self.type + "Freeze"), - self.name, - self.name, - self.name)) - + query = """ +SELECT Id, Name, FullName, ShortName FROM %s +WHERE (Name = %s OR FullName = '%s' OR ShortName = %s)""" self.tissue = "N/A" - self.id, self.name, self.fullname, self.shortname = fetchone(""" - SELECT Id, Name, FullName, ShortName - FROM %s - WHERE (Name = '%s' OR FullName = '%s' OR ShortName = '%s') - """ % (query_args)) - + _vars = (self.id, self.name, self.fullname, self.shortname,) + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute(query, (query_args,)) + _vars = cursor.fetchone() except TypeError: pass @@ -848,16 +844,8 @@ class PhenotypeDataSet(DataSet): 'Additive Effect'] self.type = 'Publish' - - self.query_for_group = ''' - SELECT - InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode - FROM - InbredSet, PublishFreeze - WHERE - PublishFreeze.InbredSetId = InbredSet.Id AND - PublishFreeze.Name = "%s" - ''' % escape(self.name) + self.query_for_group = """ +SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode FROM InbredSet, PublishFreeze WHERE PublishFreeze.InbredSetId = InbredSet.Id AND PublishFreeze.Name = %s""" def check_confidentiality(self): # (Urgently?) Need to write this @@ -964,16 +952,10 @@ class GenotypeDataSet(DataSet): # Todo: Obsolete or rename this field self.type = 'Geno' - - self.query_for_group = ''' - SELECT - InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode - FROM - InbredSet, GenoFreeze - WHERE - GenoFreeze.InbredSetId = InbredSet.Id AND - GenoFreeze.Name = "%s" - ''' % escape(self.name) + self.query_for_group = """ +SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode +FROM InbredSet, GenoFreeze WHERE GenoFreeze.InbredSetId = InbredSet.Id AND +GenoFreeze.Name = %s""" def check_confidentiality(self): return geno_mrna_confidentiality(self) @@ -1072,17 +1054,10 @@ class MrnaAssayDataSet(DataSet): # Todo: Obsolete or rename this field self.type = 'ProbeSet' - - self.query_for_group = ''' - SELECT - InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode - FROM - InbredSet, ProbeSetFreeze, ProbeFreeze - WHERE - ProbeFreeze.InbredSetId = InbredSet.Id AND - ProbeFreeze.Id = ProbeSetFreeze.ProbeFreezeId AND - ProbeSetFreeze.Name = "%s" - ''' % escape(self.name) + self.query_for_group = """ +SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType, InbredSet.InbredSetCode +FROM InbredSet, ProbeSetFreeze, ProbeFreeze WHERE ProbeFreeze.InbredSetId = InbredSet.Id AND +ProbeFreeze.Id = ProbeSetFreeze.ProbeFreezeId AND ProbeSetFreeze.Name = %s""" def check_confidentiality(self): return geno_mrna_confidentiality(self) diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index 9ec650a4..122c546f 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -20,19 +20,23 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from db.call import fetch1 +from wqflask.database import database_connection def retrieve_species(group): """Get the species of a group (e.g. returns string "mouse" on "BXD" """ - result = fetch1("select Species.Name from Species, InbredSet where InbredSet.Name = '%s' and InbredSet.SpeciesId = Species.Id" % ( - group), "/cross/" + group + ".json", lambda r: (r["species"],))[0] + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute( + "SELECT Species.Name FROM Species, InbredSet WHERE InbredSet.Name = %s AND InbredSet.SpeciesId = Species.Id", + (group,)) + return cursor.fetchone()[0] return result def retrieve_species_id(group): - result = fetch1("select SpeciesId from InbredSet where Name = '%s'" % ( - group), "/cross/" + group + ".json", lambda r: (r["species_id"],))[0] - return result + with database_connection() as conn, conn.cursor() as cursor: + cursor.execute("SELECT SpeciesId FROM InbredSet WHERE Name = %s", + (group,)) + return cursor.fetchone()[0] -- cgit v1.2.3 From 327506abbc9b4a368f5785b8f90086cc5e038eb8 Mon Sep 17 00:00:00 2001 From: Munyoki Kilyungi Date: Tue, 30 Aug 2022 21:49:28 +0300 Subject: Delete unused modules * wqflask/base/data_set.py: Delete "menu_main" import. * wqflask/db/call.py: Delete it. * wqflask/db/gn_server.py: Ditto. * wqflask/wqflask/submit_bnw.py: Ditto. --- wqflask/base/data_set.py | 1 - wqflask/db/call.py | 69 ------------------------------------------- wqflask/db/gn_server.py | 7 ----- wqflask/wqflask/submit_bnw.py | 10 ------- 4 files changed, 87 deletions(-) delete mode 100644 wqflask/db/call.py delete mode 100644 wqflask/db/gn_server.py delete mode 100644 wqflask/wqflask/submit_bnw.py (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 57bd6be3..433365fc 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -22,7 +22,6 @@ from dataclasses import field from dataclasses import InitVar from typing import Optional, Dict, List from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL -from db.gn_server import menu_main from pprint import pformat as pf from utility.db_tools import escape from utility.db_tools import mescape diff --git a/wqflask/db/call.py b/wqflask/db/call.py deleted file mode 100644 index c27e0326..00000000 --- a/wqflask/db/call.py +++ /dev/null @@ -1,69 +0,0 @@ -# Module for calling the backend - -from flask import g - -import string -try: # Python2 support - import urllib.request - import urllib.error - import urllib.parse -except: - import urllib2 -import json -from utility.tools import USE_GN_SERVER, LOG_SQL, GN_SERVER_URL - -from utility.logger import getLogger -logger = getLogger(__name__) - - -def fetch1(query, path=None, func=None): - """Fetch one result as a Tuple using either a SQL query or the URI -path to GN_SERVER (when USE_GN_SERVER is True). Apply func to -GN_SERVER result when set (which should return a Tuple) - - """ - if USE_GN_SERVER and path: - result = gn_server(path) - if func != None: - res2 = func(result) - else: - res2 = result, - return res2 - else: - return fetchone(query) - - -def fetchone(query): - """Return tuple containing one row by calling SQL directly (the -original fetchone, but with logging) - - """ - def helper(query): - res = g.db.execute(query) - return res.fetchone() - return logger.sql(query, helper) - - -def fetchall(query): - """Return row iterator by calling SQL directly (the -original fetchall, but with logging) - - """ - def helper(query): - res = g.db.execute(query) - return res.fetchall() - return logger.sql(query, helper) - - -def gn_server(path): - """Return JSON record by calling GN_SERVER - - """ - res = () - try: - res = urllib.request.urlopen(GN_SERVER_URL + path) - except: - res = urllib2.urlopen(GN_SERVER_URL + path) - rest = res.read() - res2 = json.loads(rest) - return res2 diff --git a/wqflask/db/gn_server.py b/wqflask/db/gn_server.py deleted file mode 100644 index af6682ba..00000000 --- a/wqflask/db/gn_server.py +++ /dev/null @@ -1,7 +0,0 @@ -# Backend query functions (logic) - -from db.call import gn_server - - -def menu_main(): - return gn_server("/int/menu/main.json") diff --git a/wqflask/wqflask/submit_bnw.py b/wqflask/wqflask/submit_bnw.py deleted file mode 100644 index b21a88cc..00000000 --- a/wqflask/wqflask/submit_bnw.py +++ /dev/null @@ -1,10 +0,0 @@ -from base.trait import GeneralTrait -from base import data_set -from utility import helper_functions - -import utility.logger -logger = utility.logger.getLogger(__name__) - - -def get_bnw_input(start_vars): - logger.debug("BNW VARS:", start_vars) -- cgit v1.2.3 From 2cd740180850c959ae54efb72b14255c358e181c Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Tue, 6 Sep 2022 11:55:07 +0300 Subject: Use dataset name if dataset group name is not set. --- wqflask/base/data_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 433365fc..da3c0628 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -325,7 +325,7 @@ class DatasetGroup: InbredSet.InbredSetCode FROM InbredSet where Name=%s """ if not name: - query, name = dataset.query_for_group, self.name + query, name = dataset.query_for_group, dataset.name with database_connection() as conn, conn.cursor() as cursor: cursor.execute(query, (name,)) results = cursor.fetchone() -- cgit v1.2.3 From a5b999a249cf5434e4a2a7fe21cf9b26357bc69f Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Tue, 6 Sep 2022 11:56:02 +0300 Subject: Set up the self.* variables from query results Set up the appropriate self.* variables from the results of running the appropriate query. --- wqflask/base/data_set.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'wqflask/base') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index da3c0628..f66bf186 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -628,23 +628,30 @@ class DataSet: try: query = "" _vars = None + query_args = (self.name, self.name, self.name) if self.type == "ProbeSet": - query = """ -SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, ProbeSetFreeze.DataScale, Tissue.Name -FROM ProbeSetFreeze, ProbeFreeze, Tissue -WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id -AND ProbeFreeze.TissueId = Tissue.Id -AND (ProbeSetFreeze.Name = %s OR ProbeSetFreeze.FullName = %s OR ProbeSetFreeze.ShortName = %s""" - _vars = (self.id, self.name, self.fullname, self.shortname, self.data_scale, self.tissue,) + query = ( + "SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, " + "ProbeSetFreeze.FullName, ProbeSetFreeze.ShortName, " + "ProbeSetFreeze.DataScale, Tissue.Name " + "FROM ProbeSetFreeze, ProbeFreeze, Tissue " + "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " + "AND ProbeFreeze.TissueId = Tissue.Id " + "AND (ProbeSetFreeze.Name = %s OR ProbeSetFreeze.FullName = %s OR ProbeSetFreeze.ShortName = %s)") else: - query = """ -SELECT Id, Name, FullName, ShortName FROM %s -WHERE (Name = %s OR FullName = '%s' OR ShortName = %s)""" + query = ( + "SELECT Id, Name, FullName, ShortName " + f"FROM {self.type}Freeze " + "WHERE (Name = %s OR FullName = %s OR ShortName = %s)") self.tissue = "N/A" - _vars = (self.id, self.name, self.fullname, self.shortname,) with database_connection() as conn, conn.cursor() as cursor: - cursor.execute(query, (query_args,)) + cursor.execute(query, query_args) _vars = cursor.fetchone() + if self.type == "ProbeSet": + (self.id, self.name, self.fullname, self.shortname, + self.data_scale, self.tissue) = _vars + else: + self.id, self.name, self.fullname, self.shortname = _vars except TypeError: pass -- cgit v1.2.3