configs: Refactor to fetch configs consistently.

13 files changed, 77 insertions, 72 deletions

diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index 64a17548..2553249f 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -3,7 +3,9 @@ import json
 import time
 from functools import wraps
 
-from utility.tools import SQL_URI
+from flask import current_app as app
+
+from utility.configuration import get_setting
 
 from wqflask.correlation import correlation_functions
 from base import data_set
@@ -146,7 +148,7 @@ def lit_for_trait_list(corr_results, this_dataset, this_trait):
     geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict.items() if
                    trait_lists.get(trait_name)}
 
-    with database_connection(SQL_URI) as conn:
+    with database_connection(get_setting(app, "SQL_URI")) as conn:
         correlation_results = compute_all_lit_correlation(
             conn=conn, trait_lists=list(geneid_dict.items()),
             species=species, gene_id=this_trait_geneid)
diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py
index 41dd77a1..cd3af26f 100644
--- a/wqflask/wqflask/correlation/rust_correlation.py
+++ b/wqflask/wqflask/correlation/rust_correlation.py
@@ -2,7 +2,9 @@
 import json
 from functools import reduce
 
-from utility.tools import SQL_URI
+from flask import current_app as app
+
+from utility.configuration import get_setting
 from utility.db_tools import mescape
 from utility.db_tools import create_in_clause
 from wqflask.correlation.correlation_functions\
@@ -30,7 +32,7 @@ def query_probes_metadata(dataset, trait_list):
     if not bool(trait_list) or dataset.type != "ProbeSet":
         return []
 
-    with database_connection(SQL_URI) as conn:
+    with database_connection(get_setting("SQL_URI")) as conn:
         with conn.cursor() as cursor:
 
             query = """
@@ -106,7 +108,7 @@ def chunk_dataset(dataset, steps, name):
                   ProbeSetXRef.ProbeSetId = ProbeSet.Id
     """.format(name)
 
-    with database_connection(SQL_URI) as conn:
+    with database_connection(get_setting("SQL_URI")) as conn:
         with conn.cursor() as curr:
             curr.execute(query)
             traits_name_dict = dict(curr.fetchall())
@@ -130,7 +132,7 @@ def compute_top_n_sample(start_vars, dataset, trait_list):
             sample_data=json.loads(samples_vals),
             dataset_samples=dataset.group.all_samples_ordered())
 
-        with database_connection(SQL_URI) as conn:
+        with database_connection(get_setting("SQL_URI")) as conn:
             with conn.cursor() as curr:
                 curr.execute(
                     """
@@ -148,7 +150,7 @@ def compute_top_n_sample(start_vars, dataset, trait_list):
     if len(trait_list) == 0:
         return {}
 
-    with database_connection(SQL_URI) as conn:
+    with database_connection(get_setting("SQL_URI")) as conn:
         with conn.cursor() as curr:
             # fetching strain data in bulk
             query = (
@@ -184,7 +186,7 @@ def compute_top_n_lit(corr_results, target_dataset, this_trait) -> dict:
     geneid_dict = {trait_name: geneid for (trait_name, geneid)
                    in geneid_dict.items() if
                    corr_results.get(trait_name)}
-    with database_connection(SQL_URI) as conn:
+    with database_connection(get_setting("SQL_URI")) as conn:
         return reduce(
             lambda acc, corr: {**acc, **corr},
             compute_all_lit_correlation(
@@ -258,7 +260,7 @@ def __compute_sample_corr__(
         return {}
 
     if target_dataset.type == "ProbeSet" and start_vars.get("use_cache") == "true":
-        with database_connection(SQL_URI) as conn:
+        with database_connection(get_setting("SQL_URI")) as conn:
             file_path = fetch_text_file(target_dataset.name, conn)
             if file_path:
                 (sample_vals, target_data) = read_text_file(
@@ -339,7 +341,7 @@ def __compute_lit_corr__(
     (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
         this_trait, target_dataset)
 
-    with database_connection(SQL_URI) as conn:
+    with database_connection(get_setting("SQL_URI")) as conn:
         return reduce(
             lambda acc, lit: {**acc, **lit},
             compute_all_lit_correlation(
diff --git a/wqflask/wqflask/ctl/gn3_ctl_analysis.py b/wqflask/wqflask/ctl/gn3_ctl_analysis.py
index 8f790597..ca363f49 100644
--- a/wqflask/wqflask/ctl/gn3_ctl_analysis.py
+++ b/wqflask/wqflask/ctl/gn3_ctl_analysis.py
@@ -1,9 +1,10 @@
 import requests
 import itertools
 
+from flask import current_app as app
+
 from utility import genofile_parser
-from utility.tools import GN3_LOCAL_URL
-from utility.tools import locate
+from utility.configuration import locate, get_setting
 
 from base.trait import create_trait
 from base.trait import retrieve_sample_data
@@ -33,7 +34,7 @@ def parse_geno_data(dataset_group_name) -> dict:
 
     @returns : dict with keys genotypes,markernames & individuals
     """
-    genofile_location = locate(dataset_group_name + ".geno", "genotype")
+    genofile_location = locate(app, dataset_group_name + ".geno", "genotype")
     parser = genofile_parser.ConvertGenoFile(genofile_location)
     parser.process_csv()
     markers = []
@@ -100,7 +101,7 @@ def parse_form_data(form_data: dict):
 def run_ctl(requestform):
     """function to make an api call
     to gn3 and run ctl"""
-    ctl_api = f"{GN3_LOCAL_URL}/api/ctl/run_ctl"
+    ctl_api = f"{get_setting(app, 'GN3_LOCAL_URL')}/api/ctl/run_ctl"
 
     form_data = parse_form_data(requestform.to_dict())
     trait_db_list = form_data["trait_db_list"]
diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py
index fbeb7a49..b5ddb1dc 100644
--- a/wqflask/wqflask/do_search.py
+++ b/wqflask/wqflask/do_search.py
@@ -11,7 +11,6 @@ from pprint import pformat as pf
 import sys
 
 from db import webqtlDatabaseFunction
-from utility.tools import GN2_BASE_URL
 
 
 class DoSearch:
diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py
index 72c55153..e0e79900 100644
--- a/wqflask/wqflask/gsearch.py
+++ b/wqflask/wqflask/gsearch.py
@@ -3,10 +3,11 @@ from urllib.parse import urlencode, urljoin
 from pymonad.maybe import Just, Maybe
 from pymonad.tools import curry
 import requests
+from flask import current_app as app
 
 from gn3.monads import MonadicDict
 from utility.hmac import hmac_creation
-from utility.tools import GN3_LOCAL_URL
+from utility.configuration import get_setting
 from base import webqtlConfig
 
 # KLUDGE: Due to the lack of pagination, we hard-limit the maximum
@@ -29,9 +30,10 @@ class GSearch:
         convert_lod = lambda x: x / 4.61
         self.trait_list = []
         for i, trait in enumerate(requests.get(
-                urljoin(GN3_LOCAL_URL, "/api/search?" + urlencode({"query": self.terms,
-                                                                   "type": self.type,
-                                                                   "per_page": MAX_SEARCH_RESULTS}))).json()):
+                urljoin(get_setting(app, "GN3_LOCAL_URL"),
+                        "/api/search?" + urlencode({"query": self.terms,
+                                                    "type": self.type,
+                                                    "per_page": MAX_SEARCH_RESULTS}))).json()):
             trait = MonadicDict(trait)
             trait["index"] = Just(i)
             trait["location_repr"] = (Maybe.apply(chr_mb)
diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py
index 8ef85d3c..727fdaf8 100644
--- a/wqflask/wqflask/heatmap/heatmap.py
+++ b/wqflask/wqflask/heatmap/heatmap.py
@@ -1,13 +1,15 @@
-import string
 import os
+import string
 import random
+
+from redis import Redis
+from flask import g, Flask, current_app as app
+
 from base import species
 from base import webqtlConfig
 from utility import helper_functions
 
-from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR
-from redis import Redis
-from flask import Flask, g
+from utility.configuration import flat_files, get_setting
 
 from wqflask.database import database_connection
 
@@ -119,9 +121,9 @@ class Heatmap:
                 ''.join(random.choice(string.ascii_uppercase + string.digits)
                         for _ in range(6))
 
-            reaper_command = REAPER_COMMAND + ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt -n 1000 -o {4}{5}.txt'.format(flat_files('genotype'),
+            reaper_command = get_setting(app, "REAPER_COMMAND") + ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt -n 1000 -o {4}{5}.txt'.format(flat_files(app, 'genotype'),
                                                                                                                            genofile_name,
-                                                                                                                           TEMPDIR,
+                                                                                                                           get_setting(app, "TEMPDIR"),
                                                                                                                            trait_filename,
                                                                                                                            webqtlConfig.GENERATED_IMAGE_DIR,
                                                                                                                            output_filename)
@@ -145,7 +147,7 @@ class Heatmap:
 def gen_pheno_txt_file(samples, vals, filename):
     """Generates phenotype file for GEMMA"""
 
-    with open("{0}/gn2/{1}.txt".format(TEMPDIR, filename), "w") as outfile:
+    with open("{0}/gn2/{1}.txt".format(get_setting(app, "TEMPDIR"), filename), "w") as outfile:
         outfile.write("Trait\t")
 
         filtered_sample_list = []
diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py
index 5e3e9c9d..c194ad24 100644
--- a/wqflask/wqflask/interval_analyst/GeneUtil.py
+++ b/wqflask/wqflask/interval_analyst/GeneUtil.py
@@ -1,12 +1,13 @@
 import string
+from flask import current_app as app
 
 from wqflask.database import database_connection
 
-from utility.tools import flat_files
+from utility.configuration import flat_files
 
 def load_homology(chr_name, start_mb, end_mb, source_file):
     homology_list = []
-    with open(flat_files("homology/") + source_file) as h_file:
+    with open(flat_files(app, "homology/") + source_file) as h_file:
         current_chr = 0
         for line in h_file:
             line_items = line.split()
diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py
index b7a296b7..a86e2919 100644
--- a/wqflask/wqflask/marker_regression/plink_mapping.py
+++ b/wqflask/wqflask/marker_regression/plink_mapping.py
@@ -5,7 +5,6 @@ from flask import current_app as app
 
 from utility import webqtlUtil
 from utility.configuration import flat_files, get_setting
-from utility.tools import PLINK_COMMAND
 
 
 def run_plink(this_trait, dataset, species, vals, maf):
diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
index 4d5db2ee..1cf32f2c 100644
--- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
+++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
@@ -5,10 +5,12 @@ import random
 import json
 import re
 
+from flask import current_app as app
+
 from base import webqtlConfig
 from base.trait import GeneralTrait
 from base.data_set import create_dataset
-from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR
+from utility.configuration import flat_files
 
 
 def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boot_check, num_bootstrap, do_control, control_marker, manhattan_plot, first_run=True, output_files=None):
@@ -54,16 +56,16 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo
         if manhattan_plot != True:
             opt_list.append("--interval 1")
 
-        reaper_command = (REAPER_COMMAND +
-                          ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt {4} -o {5}{6}.txt'.format(flat_files('genotype'),
-
-                                                                                                   genofile_name,
-                                                                                                   TEMPDIR,
-                                                                                                   trait_filename,
-                                                                                                   " ".join(
-                              opt_list),
-                              webqtlConfig.GENERATED_IMAGE_DIR,
-                              output_filename))
+        reaper_command = (
+            get_setting(app, REAPER_COMMAND) +
+            ' --geno {0}/{1}.geno --traits {2}/gn2/{3}.txt {4} -o {5}{6}.txt'.format(
+                flat_files(app, 'genotype'),
+                genofile_name,
+                get_setting(app, 'TEMPDIR'),
+                trait_filename,
+                " ".join(opt_list),
+                webqtlConfig.GENERATED_IMAGE_DIR,
+                output_filename))
         os.system(reaper_command)
     else:
         output_filename, permu_filename, bootstrap_filename = output_files
@@ -84,7 +86,7 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo
 def gen_pheno_txt_file(samples, vals, trait_filename):
     """Generates phenotype file for GEMMA"""
 
-    with open(f"{TEMPDIR}/gn2/{trait_filename}.txt", "w") as outfile:
+    with open(f"{get_setting(app, 'TEMPDIR')}/gn2/{trait_filename}.txt", "w") as outfile:
         outfile.write("Trait\t")
 
         filtered_sample_list = []
diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py
index 9b70f03d..7b09d516 100644
--- a/wqflask/wqflask/network_graph/network_graph.py
+++ b/wqflask/wqflask/network_graph/network_graph.py
@@ -20,12 +20,13 @@
 
 import scipy
 import simplejson as json
+from flask import current_app as app
 
 from base.trait import create_trait
 from base import data_set
 from utility import helper_functions
 from utility import corr_result_helpers
-from utility.tools import GN2_BRANCH_URL
+from utility.configuration import get_setting
 
 
 class NetworkGraph:
@@ -173,7 +174,7 @@ class NetworkGraph:
             self.nodes_list.append(node_dict)
 
         self.elements = json.dumps(self.nodes_list + self.edges_list)
-        self.gn2_url = GN2_BRANCH_URL
+        self.gn2_url = get_setting(app, "GN2_BRANCH_URL")
 
         groups = []
         for sample in self.all_sample_list:
diff --git a/wqflask/wqflask/top_level_routes.py b/wqflask/wqflask/top_level_routes.py
index a1f26725..b733d955 100644
--- a/wqflask/wqflask/top_level_routes.py
+++ b/wqflask/wqflask/top_level_routes.py
@@ -2,7 +2,7 @@
 from flask import Blueprint
 
 from .api import api_bp
-# from .views import main_views
+from .views import main_views
 from base.trait import trait_bp
 from .user_login import ulogin_bp
 from .user_session import usession_bp
@@ -16,8 +16,7 @@ toplevel = Blueprint("toplevel", __name__)
 toplevel.register_blueprint(trait_bp)
 toplevel.register_blueprint(pcorrs_bp)
 toplevel.register_blueprint(ulogin_bp)
-# toplevel.register_blueprint(main_views)
+toplevel.register_blueprint(main_views)
 toplevel.register_blueprint(usession_bp)
 toplevel.register_blueprint(collections_bp)
-
 toplevel.register_blueprint(api_bp, url_prefix="/api")
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index bb0e408f..ebd273af 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -80,14 +80,7 @@ from wqflask.oauth2.client import no_token_get
 from wqflask.oauth2.request_utils import process_error
 
 from utility import temp_data
-from utility.tools import TEMPDIR
-from utility.tools import USE_REDIS
-from utility.tools import REDIS_URL
-from utility.tools import GN_SERVER_URL
-from utility.tools import GN3_LOCAL_URL
-from utility.tools import GN_VERSION
-from utility.tools import JS_TWITTER_POST_FETCHER_PATH
-from utility.tools import JS_GUIX_PATH
+from utility.configuration import get_setting
 from utility.helper_functions import get_species_groups
 from utility.redis_tools import get_redis_conn
 
@@ -114,8 +107,8 @@ def no_access_page():
 def index_page():
     anon_id = session_info()["anon_id"]
     def __render__(colls):
-        return render_template("index_page.html", version=GN_VERSION,
-                               gn_server_url=GN_SERVER_URL,
+        return render_template("index_page.html", version=get_setting(app, "GN_VERSION"),
+                               gn_server_url=get_setting(app, "GN_SERVER_URL"),
                                anon_collections=(
                                    colls if user_logged_in() else []),
                                anon_id=anon_id)
@@ -139,7 +132,7 @@ def tmp_page(img_path):
 
 @main_views.route("/js/<path:filename>")
 def js(filename):
-    js_path = JS_GUIX_PATH
+    js_path = get_setting(app, "JS_GUIX_PATH")
     name = filename
     if 'js_alt/' in filename:
         js_path = js_path.replace('genenetwork2/javascript', 'javascript')
@@ -149,7 +142,7 @@ def js(filename):
 
 @main_views.route("/css/<path:filename>")
 def css(filename):
-    js_path = JS_GUIX_PATH
+    js_path = get_setting(app, "JS_GUIX_PATH")
     name = filename
     if 'js_alt/' in filename:
         js_path = js_path.replace('genenetwork2/javascript', 'javascript')
@@ -159,13 +152,14 @@ def css(filename):
 
 @main_views.route("/twitter/<path:filename>")
 def twitter(filename):
-    return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename)
+    return send_from_directory(get_setting(app, "JS_TWITTER_POST_FETCHER_PATH"),
+                               filename)
 
 
 @main_views.route("/search", methods=('GET',))
 def search_page():
     result = None
-    if USE_REDIS:
+    if get_setting(app, "USE_REDIS"):
         key = "search_results:v1:" + \
             json.dumps(request.args, sort_keys=True)
         result = Redis.get(key)
@@ -173,7 +167,7 @@ def search_page():
             result = pickle.loads(result)
     result = SearchResultPage(request.args).__dict__
     valid_search = result['search_term_exists']
-    if USE_REDIS and valid_search:
+    if get_setting(app, "USE_REDIS") and valid_search:
         # Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL))
         Redis.expire(key, 60 * 60)
 
@@ -313,8 +307,8 @@ def submit_trait_form():
     return render_template(
         "submit_trait.html",
         species_and_groups=species_and_groups,
-        gn_server_url=GN_SERVER_URL,
-        version=GN_VERSION)
+        gn_server_url=get_setting(app, "GN_SERVER_URL"),
+        version=get_setting(app, "GN_VERSION"))
 
 
 @main_views.route("/create_temp_trait", methods=('POST',))
@@ -866,7 +860,7 @@ def __handle_correlation_error__(exc):
 
 @main_views.route("/corr_compute", methods=('POST', 'GET'))
 def corr_compute_page():
-    with Redis.from_url(REDIS_URL, decode_responses=True) as rconn:
+    with Redis.from_url(get_setting(app, "REDIS_URL"), decode_responses=True) as rconn:
         if request.method == "POST":
             request_received = datetime.datetime.utcnow()
             filename=hmac.hmac_creation(f"request_form_{request_received.isoformat()}")
@@ -881,7 +875,7 @@ def corr_compute_page():
                         "request_received_time": request_received.isoformat(),
                         "status": "queued"
                     })
-                jobs.run(job_id, REDIS_URL)
+                jobs.run(job_id, get_setting(app, "REDIS_URL"))
 
             return redirect(url_for("corr_compute_page", job_id=str(job_id)))
 
@@ -992,7 +986,7 @@ def browser_inputs():
 
     filename = request.args['filename']
 
-    with open("{}/gn2/".format(TEMPDIR) + filename + ".json", "r") as the_file:
+    with open("{}/gn2/".format(get_setting(app, "TEMPDIR")) + filename + ".json", "r") as the_file:
         file_contents = json.load(the_file)
 
     return flask.jsonify(file_contents)
@@ -1048,7 +1042,7 @@ def display_generif_page(symbol):
     """Fetch GeneRIF metadata from GN3 and display it"""
     entries = requests.get(
         urljoin(
-            GN3_LOCAL_URL,
+            get_setting(app, "GN3_LOCAL_URL"),
             f"/api/metadata/genewiki/{symbol}"
         )
     ).json()
@@ -1063,7 +1057,7 @@ def display_generif_page(symbol):
 def get_dataset(name):
     metadata = requests.get(
         urljoin(
-            GN3_LOCAL_URL,
+            get_setting(app, "GN3_LOCAL_URL"),
             f"/api/metadata/dataset/{name}")
     ).json()
     float_p = ""
@@ -1086,7 +1080,7 @@ def get_dataset(name):
 def get_publication(name):
     metadata = requests.get(
         urljoin(
-            GN3_LOCAL_URL,
+            get_setting(app, "GN3_LOCAL_URL"),
             f"/api/metadata/publication/{name}")
     ).json()
     return render_template(
@@ -1099,7 +1093,7 @@ def get_publication(name):
 def get_phenotype(name):
     metadata = requests.get(
         urljoin(
-            GN3_LOCAL_URL,
+            get_setting(app, "GN3_LOCAL_URL"),
             f"/api/metadata/phenotype/{name}")
     ).json()
     return render_template(
diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py
index ab7fe46c..ec35ccc9 100644
--- a/wqflask/wqflask/wgcna/gn3_wgcna.py
+++ b/wqflask/wqflask/wgcna/gn3_wgcna.py
@@ -4,9 +4,10 @@ and process data to be rendered by datatables
 
 import requests
 from types import SimpleNamespace
+from flask import current_app as app
 
 from utility.helper_functions import get_trait_db_obs
-from utility.tools import GN3_LOCAL_URL
+from utility.configuration import get_setting
 
 
 def fetch_trait_data(requestform):
@@ -74,7 +75,7 @@ def process_image(response):
 def run_wgcna(form_data):
     """function to run wgcna"""
 
-    wgcna_api = f"{GN3_LOCAL_URL}/api/wgcna/run_wgcna"
+    wgcna_api = f"{get_setting(app, 'GN3_LOCAL_URL')}/api/wgcna/run_wgcna"
 
     trait_dataset = fetch_trait_data(form_data)
     form_data["minModuleSize"] = int(form_data["MinModuleSize"])