diff options
| -rw-r--r-- | wqflask/wqflask/correlation/correlation_gn3_api.py | 6 | ||||
| -rw-r--r-- | wqflask/wqflask/correlation/rust_correlation.py | 18 | ||||
| -rw-r--r-- | wqflask/wqflask/ctl/gn3_ctl_analysis.py | 9 | ||||
| -rw-r--r-- | wqflask/wqflask/do_search.py | 1 | ||||
| -rw-r--r-- | wqflask/wqflask/gsearch.py | 10 | ||||
| -rw-r--r-- | wqflask/wqflask/heatmap/heatmap.py | 16 | ||||
| -rw-r--r-- | wqflask/wqflask/interval_analyst/GeneUtil.py | 5 | ||||
| -rw-r--r-- | wqflask/wqflask/marker_regression/plink_mapping.py | 1 | ||||
| -rw-r--r-- | wqflask/wqflask/marker_regression/qtlreaper_mapping.py | 26 | ||||
| -rw-r--r-- | wqflask/wqflask/network_graph/network_graph.py | 5 | ||||
| -rw-r--r-- | wqflask/wqflask/top_level_routes.py | 5 | ||||
| -rw-r--r-- | wqflask/wqflask/views.py | 42 | ||||
| -rw-r--r-- | wqflask/wqflask/wgcna/gn3_wgcna.py | 5 |
13 files changed, 77 insertions, 72 deletions
diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 64a17548..2553249f 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -3,7 +3,9 @@ import json import time from functools import wraps -from utility.tools import SQL_URI +from flask import current_app as app + +from utility.configuration import get_setting from wqflask.correlation import correlation_functions from base import data_set @@ -146,7 +148,7 @@ def lit_for_trait_list(corr_results, this_dataset, this_trait): geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict.items() if trait_lists.get(trait_name)} - with database_connection(SQL_URI) as conn: + with database_connection(get_setting(app, "SQL_URI")) as conn: correlation_results = compute_all_lit_correlation( conn=conn, trait_lists=list(geneid_dict.items()), species=species, gene_id=this_trait_geneid) diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py index 41dd77a1..cd3af26f 100644 --- a/wqflask/wqflask/correlation/rust_correlation.py +++ b/wqflask/wqflask/correlation/rust_correlation.py @@ -2,7 +2,9 @@ import json from functools import reduce -from utility.tools import SQL_URI +from flask import current_app as app + +from utility.configuration import get_setting from utility.db_tools import mescape from utility.db_tools import create_in_clause from wqflask.correlation.correlation_functions\ @@ -30,7 +32,7 @@ def query_probes_metadata(dataset, trait_list): if not bool(trait_list) or dataset.type != "ProbeSet": return [] - with database_connection(SQL_URI) as conn: + with database_connection(get_setting("SQL_URI")) as conn: with conn.cursor() as cursor: query = """ @@ -106,7 +108,7 @@ def chunk_dataset(dataset, steps, name): ProbeSetXRef.ProbeSetId = ProbeSet.Id """.format(name) - with database_connection(SQL_URI) as conn: + with database_connection(get_setting("SQL_URI")) as conn: with conn.cursor() as curr: curr.execute(query) traits_name_dict = dict(curr.fetchall()) @@ -130,7 +132,7 @@ def compute_top_n_sample(start_vars, dataset, trait_list): sample_data=json.loads(samples_vals), dataset_samples=dataset.group.all_samples_ordered()) - with database_connection(SQL_URI) as conn: + with database_connection(get_setting("SQL_URI")) as conn: with conn.cursor() as curr: curr.execute( """ @@ -148,7 +150,7 @@ def compute_top_n_sample(start_vars, dataset, trait_list): if len(trait_list) == 0: return {} - with database_connection(SQL_URI) as conn: + with database_connection(get_setting("SQL_URI")) as conn: with conn.cursor() as curr: # fetching strain data in bulk query = ( @@ -184,7 +186,7 @@ def compute_top_n_lit(corr_results, target_dataset, this_trait) -> dict: geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict.items() if corr_results.get(trait_name)} - with database_connection(SQL_URI) as conn: + with database_connection(get_setting("SQL_URI")) as conn: return reduce( lambda acc, corr: {**acc, **corr}, compute_all_lit_correlation( @@ -258,7 +260,7 @@ def __compute_sample_corr__( return {} if target_dataset.type == "ProbeSet" and start_vars.get("use_cache") == "true": - with database_connection(SQL_URI) as conn: + with database_connection(get_setting("SQL_URI")) as conn: file_path = fetch_text_file(target_dataset.name, conn) if file_path: (sample_vals, target_data) = read_text_file( @@ -339,7 +341,7 @@ def __compute_lit_corr__( (this_trait_geneid, geneid_dict, species) = do_lit_correlation( this_trait, target_dataset) - with database_connection(SQL_URI) as conn: + with database_connection(get_setting("SQL_URI")) as conn: return reduce( lambda acc, lit: {**acc, **lit}, compute_all_lit_correlation( diff --git a/wqflask/wqflask/ctl/gn3_ctl_analysis.py b/wqflask/wqflask/ctl/gn3_ctl_analysis.py index 8f790597..ca363f49 100644 --- a/wqflask/wqflask/ctl/gn3_ctl_analysis.py +++ b/wqflask/wqflask/ctl/gn3_ctl_analysis.py @@ -1,9 +1,10 @@ import requests import itertools +from flask import current_app as app + from utility import genofile_parser -from utility.tools import GN3_LOCAL_URL -from utility.tools import locate +from utility.configuration import locate, get_setting from base.trait import create_trait from base.trait import retrieve_sample_data @@ -33,7 +34,7 @@ def parse_geno_data(dataset_group_name) -> dict: @returns : dict with keys genotypes,markernames & individuals """ - genofile_location = locate(dataset_group_name + ".geno", "genotype") + genofile_location = locate(app, dataset_group_name + ".geno", "genotype") parser = genofile_parser.ConvertGenoFile(genofile_location) parser.process_csv() markers = [] @@ -100,7 +101,7 @@ def parse_form_data(form_data: dict): def run_ctl(requestform): """function to make an api call to gn3 and run ctl""" - ctl_api = f"{GN3_LOCAL_URL}/api/ctl/run_ctl" + ctl_api = f"{get_setting(app, 'GN3_LOCAL_URL')}/api/ctl/run_ctl" form_data = parse_form_data(requestform.to_dict()) trait_db_list = form_data["trait_db_list"] diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index fbeb7a49..b5ddb1dc 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -11,7 +11,6 @@ from pprint import pformat as pf import sys from db import webqtlDatabaseFunction -from utility.tools import GN2_BASE_URL class DoSearch: diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 72c55153..e0e79900 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -3,10 +3,11 @@ from urllib.parse import urlencode, urljoin from pymonad.maybe import Just, Maybe from pymonad.tools import curry import requests +from flask import current_app as app from gn3.monads import MonadicDict from utility.hmac import hmac_creation -from utility.tools import GN3_LOCAL_URL +from utility.configuration import get_setting from base import webqtlConfig # KLUDGE: Due to the lack of pagination, we hard-limit the maximum @@ -29,9 +30,10 @@ class GSearch: convert_lod = lambda x: x / 4.61 self.trait_list = [] for i, trait in enumerate(requests.get( - urljoin(GN3_LOCAL_URL, "/api/search?" + urlencode({"query": self.terms, - "type": self.type, - "per_page": MAX_SEARCH_RESULTS}))).json()): + urljoin(get_setting(app, "GN3_LOCAL_URL"), + "/api/search?" + urlencode({"query": self.terms, + "type": self.type, + "per_page": MAX_SEARCH_RESULTS}))).json()): trait = MonadicDict(trait) trait["index"] = Just(i) trait["location_repr"] = (Maybe.apply(chr_mb) diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 8ef85d3c..727fdaf8 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -1,13 +1,15 @@ -import string import os +import string import random + +from redis import Redis +from flask import g, Flask, current_app as app + from base import species from base import webqtlConfig from utility import helper_functions -from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR -from redis import Redis -from flask import Flask, g +from utility.configuration import flat_files, get_setting from wqflask.database import database_connection @@ -119,9 +121,9 @@ class Heatmap: ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) - reaper_command = REAPER_COMMAND + ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt -n 1000 -o {4}{5}.txt'.format(flat_files('genotype'), + reaper_command = get_setting(app, "REAPER_COMMAND") + ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt -n 1000 -o {4}{5}.txt'.format(flat_files(app, 'genotype'), genofile_name, - TEMPDIR, + get_setting(app, "TEMPDIR"), trait_filename, webqtlConfig.GENERATED_IMAGE_DIR, output_filename) @@ -145,7 +147,7 @@ class Heatmap: def gen_pheno_txt_file(samples, vals, filename): """Generates phenotype file for GEMMA""" - with open("{0}/gn2/{1}.txt".format(TEMPDIR, filename), "w") as outfile: + with open("{0}/gn2/{1}.txt".format(get_setting(app, "TEMPDIR"), filename), "w") as outfile: outfile.write("Trait\t") filtered_sample_list = [] diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index 5e3e9c9d..c194ad24 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -1,12 +1,13 @@ import string +from flask import current_app as app from wqflask.database import database_connection -from utility.tools import flat_files +from utility.configuration import flat_files def load_homology(chr_name, start_mb, end_mb, source_file): homology_list = [] - with open(flat_files("homology/") + source_file) as h_file: + with open(flat_files(app, "homology/") + source_file) as h_file: current_chr = 0 for line in h_file: line_items = line.split() diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index b7a296b7..a86e2919 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -5,7 +5,6 @@ from flask import current_app as app from utility import webqtlUtil from utility.configuration import flat_files, get_setting -from utility.tools import PLINK_COMMAND def run_plink(this_trait, dataset, species, vals, maf): diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index 4d5db2ee..1cf32f2c 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -5,10 +5,12 @@ import random import json import re +from flask import current_app as app + from base import webqtlConfig from base.trait import GeneralTrait from base.data_set import create_dataset -from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR +from utility.configuration import flat_files def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boot_check, num_bootstrap, do_control, control_marker, manhattan_plot, first_run=True, output_files=None): @@ -54,16 +56,16 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo if manhattan_plot != True: opt_list.append("--interval 1") - reaper_command = (REAPER_COMMAND + - ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt {4} -o {5}{6}.txt'.format(flat_files('genotype'), - - genofile_name, - TEMPDIR, - trait_filename, - " ".join( - opt_list), - webqtlConfig.GENERATED_IMAGE_DIR, - output_filename)) + reaper_command = ( + get_setting(app, REAPER_COMMAND) + + ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt {4} -o {5}{6}.txt'.format( + flat_files(app, 'genotype'), + genofile_name, + get_setting(app, 'TEMPDIR'), + trait_filename, + " ".join(opt_list), + webqtlConfig.GENERATED_IMAGE_DIR, + output_filename)) os.system(reaper_command) else: output_filename, permu_filename, bootstrap_filename = output_files @@ -84,7 +86,7 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo def gen_pheno_txt_file(samples, vals, trait_filename): """Generates phenotype file for GEMMA""" - with open(f"{TEMPDIR}/gn2/{trait_filename}.txt", "w") as outfile: + with open(f"{get_setting(app, 'TEMPDIR')}/gn2/{trait_filename}.txt", "w") as outfile: outfile.write("Trait\t") filtered_sample_list = [] diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index 9b70f03d..7b09d516 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -20,12 +20,13 @@ import scipy import simplejson as json +from flask import current_app as app from base.trait import create_trait from base import data_set from utility import helper_functions from utility import corr_result_helpers -from utility.tools import GN2_BRANCH_URL +from utility.configuration import get_setting class NetworkGraph: @@ -173,7 +174,7 @@ class NetworkGraph: self.nodes_list.append(node_dict) self.elements = json.dumps(self.nodes_list + self.edges_list) - self.gn2_url = GN2_BRANCH_URL + self.gn2_url = get_setting(app, "GN2_BRANCH_URL") groups = [] for sample in self.all_sample_list: diff --git a/wqflask/wqflask/top_level_routes.py b/wqflask/wqflask/top_level_routes.py index a1f26725..b733d955 100644 --- a/wqflask/wqflask/top_level_routes.py +++ b/wqflask/wqflask/top_level_routes.py @@ -2,7 +2,7 @@ from flask import Blueprint from .api import api_bp -# from .views import main_views +from .views import main_views from base.trait import trait_bp from .user_login import ulogin_bp from .user_session import usession_bp @@ -16,8 +16,7 @@ toplevel = Blueprint("toplevel", __name__) toplevel.register_blueprint(trait_bp) toplevel.register_blueprint(pcorrs_bp) toplevel.register_blueprint(ulogin_bp) -# toplevel.register_blueprint(main_views) +toplevel.register_blueprint(main_views) toplevel.register_blueprint(usession_bp) toplevel.register_blueprint(collections_bp) - toplevel.register_blueprint(api_bp, url_prefix="/api") diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index bb0e408f..ebd273af 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -80,14 +80,7 @@ from wqflask.oauth2.client import no_token_get from wqflask.oauth2.request_utils import process_error from utility import temp_data -from utility.tools import TEMPDIR -from utility.tools import USE_REDIS -from utility.tools import REDIS_URL -from utility.tools import GN_SERVER_URL -from utility.tools import GN3_LOCAL_URL -from utility.tools import GN_VERSION -from utility.tools import JS_TWITTER_POST_FETCHER_PATH -from utility.tools import JS_GUIX_PATH +from utility.configuration import get_setting from utility.helper_functions import get_species_groups from utility.redis_tools import get_redis_conn @@ -114,8 +107,8 @@ def no_access_page(): def index_page(): anon_id = session_info()["anon_id"] def __render__(colls): - return render_template("index_page.html", version=GN_VERSION, - gn_server_url=GN_SERVER_URL, + return render_template("index_page.html", version=get_setting(app, "GN_VERSION"), + gn_server_url=get_setting(app, "GN_SERVER_URL"), anon_collections=( colls if user_logged_in() else []), anon_id=anon_id) @@ -139,7 +132,7 @@ def tmp_page(img_path): @main_views.route("/js/<path:filename>") def js(filename): - js_path = JS_GUIX_PATH + js_path = get_setting(app, "JS_GUIX_PATH") name = filename if 'js_alt/' in filename: js_path = js_path.replace('genenetwork2/javascript', 'javascript') @@ -149,7 +142,7 @@ def js(filename): @main_views.route("/css/<path:filename>") def css(filename): - js_path = JS_GUIX_PATH + js_path = get_setting(app, "JS_GUIX_PATH") name = filename if 'js_alt/' in filename: js_path = js_path.replace('genenetwork2/javascript', 'javascript') @@ -159,13 +152,14 @@ def css(filename): @main_views.route("/twitter/<path:filename>") def twitter(filename): - return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename) + return send_from_directory(get_setting(app, "JS_TWITTER_POST_FETCHER_PATH"), + filename) @main_views.route("/search", methods=('GET',)) def search_page(): result = None - if USE_REDIS: + if get_setting(app, "USE_REDIS"): key = "search_results:v1:" + \ json.dumps(request.args, sort_keys=True) result = Redis.get(key) @@ -173,7 +167,7 @@ def search_page(): result = pickle.loads(result) result = SearchResultPage(request.args).__dict__ valid_search = result['search_term_exists'] - if USE_REDIS and valid_search: + if get_setting(app, "USE_REDIS") and valid_search: # Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) Redis.expire(key, 60 * 60) @@ -313,8 +307,8 @@ def submit_trait_form(): return render_template( "submit_trait.html", species_and_groups=species_and_groups, - gn_server_url=GN_SERVER_URL, - version=GN_VERSION) + gn_server_url=get_setting(app, "GN_SERVER_URL"), + version=get_setting(app, "GN_VERSION")) @main_views.route("/create_temp_trait", methods=('POST',)) @@ -866,7 +860,7 @@ def __handle_correlation_error__(exc): @main_views.route("/corr_compute", methods=('POST', 'GET')) def corr_compute_page(): - with Redis.from_url(REDIS_URL, decode_responses=True) as rconn: + with Redis.from_url(get_setting(app, "REDIS_URL"), decode_responses=True) as rconn: if request.method == "POST": request_received = datetime.datetime.utcnow() filename=hmac.hmac_creation(f"request_form_{request_received.isoformat()}") @@ -881,7 +875,7 @@ def corr_compute_page(): "request_received_time": request_received.isoformat(), "status": "queued" }) - jobs.run(job_id, REDIS_URL) + jobs.run(job_id, get_setting(app, "REDIS_URL")) return redirect(url_for("corr_compute_page", job_id=str(job_id))) @@ -992,7 +986,7 @@ def browser_inputs(): filename = request.args['filename'] - with open("{}/gn2/".format(TEMPDIR) + filename + ".json", "r") as the_file: + with open("{}/gn2/".format(get_setting(app, "TEMPDIR")) + filename + ".json", "r") as the_file: file_contents = json.load(the_file) return flask.jsonify(file_contents) @@ -1048,7 +1042,7 @@ def display_generif_page(symbol): """Fetch GeneRIF metadata from GN3 and display it""" entries = requests.get( urljoin( - GN3_LOCAL_URL, + get_setting(app, "GN3_LOCAL_URL"), f"/api/metadata/genewiki/{symbol}" ) ).json() @@ -1063,7 +1057,7 @@ def display_generif_page(symbol): def get_dataset(name): metadata = requests.get( urljoin( - GN3_LOCAL_URL, + get_setting(app, "GN3_LOCAL_URL"), f"/api/metadata/dataset/{name}") ).json() float_p = "" @@ -1086,7 +1080,7 @@ def get_dataset(name): def get_publication(name): metadata = requests.get( urljoin( - GN3_LOCAL_URL, + get_setting(app, "GN3_LOCAL_URL"), f"/api/metadata/publication/{name}") ).json() return render_template( @@ -1099,7 +1093,7 @@ def get_publication(name): def get_phenotype(name): metadata = requests.get( urljoin( - GN3_LOCAL_URL, + get_setting(app, "GN3_LOCAL_URL"), f"/api/metadata/phenotype/{name}") ).json() return render_template( diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index ab7fe46c..ec35ccc9 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -4,9 +4,10 @@ and process data to be rendered by datatables import requests from types import SimpleNamespace +from flask import current_app as app from utility.helper_functions import get_trait_db_obs -from utility.tools import GN3_LOCAL_URL +from utility.configuration import get_setting def fetch_trait_data(requestform): @@ -74,7 +75,7 @@ def process_image(response): def run_wgcna(form_data): """function to run wgcna""" - wgcna_api = f"{GN3_LOCAL_URL}/api/wgcna/run_wgcna" + wgcna_api = f"{get_setting(app, 'GN3_LOCAL_URL')}/api/wgcna/run_wgcna" trait_dataset = fetch_trait_data(form_data) form_data["minModuleSize"] = int(form_data["MinModuleSize"]) |