Namespace all modules under gn2.

We move all modules under a gn2 directory. This is important for
"correct" packaging and deployment as a Guix service.

1 files changed, 1328 insertions, 0 deletions

diff --git a/gn2/wqflask/views.py b/gn2/wqflask/views.py
new file mode 100644
index 00000000..4f636ebc
--- /dev/null
+++ b/gn2/wqflask/views.py
@@ -0,0 +1,1328 @@
+"""Main routing table for GN2"""
+import array
+import base64
+import csv
+import datetime
+import flask
+import hashlib
+import io  # Todo: Use cStringIO?
+
+import json
+import numpy as np
+import os
+import pickle as pickle
+import random
+import requests
+import sys
+import traceback
+import uuid
+import xlsxwriter
+
+from functools import reduce
+
+from zipfile import ZipFile
+from zipfile import ZIP_DEFLATED
+
+from uuid import UUID
+
+from urllib.parse import urljoin
+
+from gn2.wqflask import app
+
+from gn3.computations.gemma import generate_hash_of_string
+from flask import current_app
+from flask import g
+from flask import Response
+from flask import request
+from flask import make_response
+from flask import render_template
+from flask import send_from_directory
+from flask import redirect
+from flask import send_file
+from flask import url_for
+from flask import flash
+from flask import session
+
+# Some of these (like collect) might contain endpoints, so they're still used.
+# Blueprints should probably be used instead.
+from gn2.wqflask import collect
+from gn2.wqflask import search_results
+from gn2.wqflask import server_side
+from gn2.base.data_set import create_dataset  # Used by YAML in marker_regression
+from gn2.wqflask.show_trait import show_trait
+from gn2.wqflask.show_trait import export_trait_data
+from gn2.wqflask.show_trait.show_trait import get_diff_of_vals
+from gn2.wqflask.heatmap import heatmap
+from gn2.wqflask.external_tools import send_to_bnw
+from gn2.wqflask.external_tools import send_to_webgestalt
+from gn2.wqflask.external_tools import send_to_geneweaver
+from gn2.wqflask.comparison_bar_chart import comparison_bar_chart
+from gn2.wqflask.marker_regression import run_mapping
+from gn2.wqflask.marker_regression.exceptions import NoMappingResultsError
+from gn2.wqflask.marker_regression import display_mapping_results
+from gn2.wqflask.network_graph import network_graph
+from gn2.wqflask.correlation.show_corr_results import set_template_vars
+from gn2.wqflask.correlation.correlation_gn3_api import compute_correlation
+from gn2.wqflask.correlation.rust_correlation import compute_correlation_rust
+from gn2.wqflask.correlation_matrix import show_corr_matrix
+from gn2.wqflask.correlation import corr_scatter_plot
+from gn2.wqflask.correlation.exceptions import WrongCorrelationType
+from gn2.wqflask.ctl.gn3_ctl_analysis import run_ctl
+
+from gn2.wqflask.wgcna.gn3_wgcna import run_wgcna
+from gn2.wqflask.snp_browser import snp_browser
+from gn2.wqflask.search_results import SearchResultPage
+from gn2.wqflask.export_traits import export_traits
+from gn2.wqflask.gsearch import GSearch
+from gn2.wqflask.update_search_results import GSearch as UpdateGSearch
+from gn2.wqflask.docs import Docs, update_text
+from gn2.wqflask.decorators import edit_access_required
+from gn2.wqflask.db_info import InfoPage
+
+from gn2.wqflask.oauth2 import client
+from gn2.wqflask.oauth2.client import no_token_get
+from gn2.wqflask.oauth2.request_utils import with_flash_error, with_flash_success
+
+from gn2.utility import temp_data
+from gn2.utility.tools import get_setting
+from gn2.utility.tools import TEMPDIR
+from gn2.utility.tools import USE_REDIS
+from gn2.utility.tools import REDIS_URL
+from gn2.utility.tools import GN_SERVER_URL
+from gn2.utility.tools import GN3_LOCAL_URL
+from gn2.utility.tools import GN_VERSION
+from gn2.utility.tools import JS_TWITTER_POST_FETCHER_PATH
+from gn2.utility.tools import JS_GUIX_PATH
+from gn2.utility.helper_functions import get_species_groups
+from gn2.utility.redis_tools import get_redis_conn
+
+import gn2.utility.hmac as hmac
+
+from gn2.base.webqtlConfig import TMPDIR
+from gn2.base.webqtlConfig import GENERATED_IMAGE_DIR
+
+from gn2.wqflask.database import database_connection
+
+import gn2.jobs.jobs as jobs
+
+from gn2.wqflask.oauth2.session import session_info
+from gn2.wqflask.oauth2.checks import user_logged_in
+
+from gn2.wqflask import requests as monad_requests
+
+Redis = get_redis_conn()
+
+
+@app.errorhandler(Exception)
+def handle_generic_exceptions(e):
+    import werkzeug
+    err_msg = str(e)
+    now = datetime.datetime.utcnow()
+    time_str = now.strftime('%l:%M%p UTC %b %d, %Y')
+    # get the stack trace and send it to the logger
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    formatted_lines = (f"{request.url} ({time_str}) \n"
+                       f"{traceback.format_exc()}")
+    _message_templates = {
+        werkzeug.exceptions.NotFound: ("404: Not Found: "
+                                       f"{time_str}: {request.url}"),
+        werkzeug.exceptions.BadRequest: ("400: Bad Request: "
+                                         f"{time_str}: {request.url}"),
+        werkzeug.exceptions.RequestTimeout: ("408: Request Timeout: "
+                                             f"{time_str}: {request.url}")}
+    # Default to the lengthy stack trace!
+    app.logger.error(_message_templates.get(exc_type,
+                                            formatted_lines))
+    # Handle random animations
+    # Use a cookie to have one animation on refresh
+    animation = request.cookies.get(err_msg[:32])
+    if not animation:
+        animation = random.choice([fn for fn in os.listdir(
+            "./wqflask/static/gif/error") if fn.endswith(".gif")])
+
+    resp = make_response(render_template("error.html", message=err_msg,
+                                         stack={formatted_lines},
+                                         error_image=animation,
+                                         version=GN_VERSION))
+    resp.set_cookie(err_msg[:32], animation)
+    return resp
+
+
+@app.route("/authentication_needed")
+def no_access_page():
+    return render_template("new_security/not_authenticated.html")
+
+
+@app.route("/")
+def index_page():
+    anon_id = session_info()["anon_id"]
+    def __render__(colls):
+        return render_template("index_page.html", version=GN_VERSION,
+                               gn_server_url=GN_SERVER_URL,
+                               anon_collections=(
+                                   colls if user_logged_in() else []),
+                               anon_id=anon_id)
+
+    return no_token_get(
+        f"auth/user/collections/{anon_id}/list").either(
+            lambda err: __render__([]),
+            __render__)
+
+
+@app.route("/tmp/<img_path>")
+def tmp_page(img_path):
+    initial_start_vars = request.form
+    imgfile = open(GENERATED_IMAGE_DIR + img_path, 'rb')
+    imgdata = imgfile.read()
+    imgB64 = base64.b64encode(imgdata)
+    bytesarray = array.array('B', imgB64)
+    return render_template("show_image.html",
+                           img_base64=bytesarray)
+
+
+@app.route("/js/<path:filename>")
+def js(filename):
+    js_path = JS_GUIX_PATH
+    name = filename
+    if 'js_alt/' in filename:
+        js_path = js_path.replace('genenetwork2/javascript', 'javascript')
+        name = name.replace('js_alt/', '')
+    return send_from_directory(js_path, name)
+
+
+@app.route("/css/<path:filename>")
+def css(filename):
+    js_path = JS_GUIX_PATH
+    name = filename
+    if 'js_alt/' in filename:
+        js_path = js_path.replace('genenetwork2/javascript', 'javascript')
+        name = name.replace('js_alt/', '')
+    return send_from_directory(js_path, name)
+
+
+@app.route("/twitter/<path:filename>")
+def twitter(filename):
+    return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename)
+
+
+@app.route("/search", methods=('GET',))
+def search_page():
+    result = None
+    if USE_REDIS:
+        key = "search_results:v1:" + \
+            json.dumps(request.args, sort_keys=True)
+        result = Redis.get(key)
+        if result:
+            result = pickle.loads(result)
+    result = SearchResultPage(request.args).__dict__
+    valid_search = result['search_term_exists']
+    if USE_REDIS and valid_search:
+        # Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL))
+        Redis.expire(key, 60 * 60)
+
+    if valid_search:
+        return render_template("search_result_page.html", **result)
+    else:
+        return render_template("search_error.html")
+
+
+@app.route("/search_table", methods=('GET',))
+def search_page_table():
+    the_search = search_results.SearchResultPage(request.args)
+    current_page = server_side.ServerSideTable(
+        len(the_search.trait_list),
+        the_search.trait_list,
+        the_search.header_data_names,
+        request.args,
+    ).get_page()
+
+    return flask.jsonify(current_page)
+
+
+@app.route("/gsearch", methods=('GET',))
+def gsearchact():
+    result = GSearch(request.args).__dict__
+    type = request.args['type']
+    if type == "gene":
+        return render_template("gsearch_gene.html", **result)
+    elif type == "phenotype":
+        return render_template("gsearch_pheno.html", **result)
+
+
+@app.route("/gsearch_table", methods=('GET',))
+def gsearchtable():
+    gsearch_table_data = GSearch(request.args)
+    current_page = server_side.ServerSideTable(
+        gsearch_table_data.trait_count,
+        gsearch_table_data.trait_list,
+        gsearch_table_data.header_data_names,
+        request.args,
+    ).get_page()
+
+    return flask.jsonify(current_page)
+
+
+
+@app.route("/gnqna",methods =["POST","GET"])
+def gnqna():
+    if request.method == "POST":
+        try:
+            def __error__(resp):
+                return resp.json()
+            def __success__(resp):
+
+                return render_template("gnqa_answer.html",**resp.json())
+            return monad_requests.post(
+                urljoin(GN_SERVER_URL,
+                        "llm/gnqna"),
+                json=dict(request.form),
+                ).then(
+                    lambda resp: resp
+                ).either(
+                    __error__, __success__)
+        except Exception as error:
+            return flask.jsonify({"error":str(error)})
+    return render_template("gnqa.html")
+
+
+@app.route("/gsearch_updating", methods=('POST',))
+def gsearch_updating():
+    result = UpdateGSearch(request.args).__dict__
+    return result['results']
+
+
+@app.route("/docedit")
+def docedit():
+    try:
+        if g.user_session.record['user_email_address'] == "zachary.a.sloan@gmail.com" or g.user_session.record['user_email_address'] == "labwilliams@gmail.com":
+            doc = Docs(request.args['entry'], request.args)
+            return render_template("docedit.html", **doc.__dict__)
+        else:
+            return "You shouldn't be here!"
+    except:
+        return "You shouldn't be here!"
+
+
+@app.route('/generated/<filename>')
+def generated_file(filename):
+    return send_from_directory(GENERATED_IMAGE_DIR, filename)
+
+
+@app.route("/help")
+def help():
+    doc = Docs("help", request.args)
+    return render_template("docs.html", **doc.__dict__)
+
+
+@app.route("/wgcna_setup", methods=('POST',))
+def wcgna_setup():
+    # We are going to get additional user input for the analysis
+    # Display them using the template
+    return render_template("wgcna_setup.html", **request.form)
+
+
+@app.route("/wgcna_results", methods=('POST',))
+def wcgna_results():
+    """call the gn3 api to get wgcna response data"""
+    results = run_wgcna(dict(request.form))
+    return render_template("gn3_wgcna_results.html", **results)
+
+
+@app.route("/ctl_setup", methods=('POST',))
+def ctl_setup():
+    # We are going to get additional user input for the analysis
+    # Display them using the template
+    return render_template("ctl_setup.html", **request.form)
+
+
+@app.route("/ctl_results", methods=["POST"])
+def ctl_results():
+    ctl_results = run_ctl(request.form)
+    return render_template("gn3_ctl_results.html", **ctl_results)
+
+
+@app.route("/ctl_network_files/<file_name>/<file_type>")
+def fetch_network_files(file_name, file_type):
+    file_path = f"{file_name}.{file_type}"
+
+    file_path = os.path.join("/tmp/", file_path)
+
+    return send_file(file_path)
+
+
+@app.route("/intro")
+def intro():
+    doc = Docs("intro", request.args)
+    return render_template("docs.html", **doc.__dict__)
+
+
+@app.route("/tutorials")
+def tutorials():
+    return render_template("tutorials.html")
+
+
+@app.route("/credits")
+def credits():
+    return render_template("credits.html")
+
+
+@app.route("/update_text", methods=('POST',))
+def update_page():
+    update_text(request.form)
+    doc = Docs(request.form['entry_type'], request.form)
+    return render_template("docs.html", **doc.__dict__)
+
+
+@app.route("/submit_trait")
+def submit_trait_form():
+    species_and_groups = get_species_groups()
+    return render_template(
+        "submit_trait.html",
+        species_and_groups=species_and_groups,
+        gn_server_url=GN_SERVER_URL,
+        version=GN_VERSION)
+
+
+@app.route("/create_temp_trait", methods=('POST',))
+def create_temp_trait():
+    doc = Docs("links")
+    return render_template("links.html", **doc.__dict__)
+
+
+@app.route('/export_trait_excel', methods=('POST',))
+def export_trait_excel():
+    """Excel file consisting of the sample data from the trait data and analysis page"""
+    trait_name, sample_data = export_trait_data.export_sample_table(
+        request.form)
+    app.logger.info(request.url)
+    buff = io.BytesIO()
+    workbook = xlsxwriter.Workbook(buff, {'in_memory': True})
+    worksheet = workbook.add_worksheet()
+    for i, row in enumerate(sample_data):
+        for j, column in enumerate(row):
+            worksheet.write(i, j, row[j])
+    workbook.close()
+    excel_data = buff.getvalue()
+    buff.close()
+
+    return Response(excel_data,
+                    mimetype='application/vnd.ms-excel',
+                    headers={"Content-Disposition": "attachment;filename=" + trait_name + ".xlsx"})
+
+
+@app.route('/export_trait_csv', methods=('POST',))
+def export_trait_csv():
+    """CSV file consisting of the sample data from the trait data and analysis page"""
+    trait_name, sample_data = export_trait_data.export_sample_table(
+        request.form)
+
+    buff = io.StringIO()
+    writer = csv.writer(buff)
+    for row in sample_data:
+        writer.writerow(row)
+    csv_data = buff.getvalue()
+    buff.close()
+
+    return Response(csv_data,
+                    mimetype='text/csv',
+                    headers={"Content-Disposition": "attachment;filename=" + trait_name + ".csv"})
+
+
+@app.route('/export_traits_csv', methods=('POST',))
+def export_traits_csv():
+    """CSV file consisting of the traits from the search result page"""
+    file_list = export_traits(request.form, "metadata")
+
+    if len(file_list) > 1:
+        now = datetime.datetime.now()
+        time_str = now.strftime('%H:%M_%d%B%Y')
+        filename = "export_{}".format(time_str)
+        memory_file = io.BytesIO()
+        with ZipFile(memory_file, mode='w', compression=ZIP_DEFLATED) as zf:
+            for the_file in file_list:
+                zf.writestr(the_file[0], the_file[1])
+
+        memory_file.seek(0)
+
+        return send_file(memory_file, attachment_filename=filename + ".zip", as_attachment=True)
+    else:
+        return Response(file_list[0][1],
+                        mimetype='text/csv',
+                        headers={"Content-Disposition": "attachment;filename=" + file_list[0][0]})
+
+
+@app.route('/export_collection', methods=('POST',))
+def export_collection_csv():
+    """CSV file consisting of trait list so collections can be exported/shared"""
+    out_file = export_traits(request.form, "collection")
+    return Response(out_file[1],
+                    mimetype='text/csv',
+                    headers={"Content-Disposition": "attachment;filename=" + out_file[0] + ".csv"})
+
+
+@app.route('/export_perm_data', methods=('POST',))
+def export_perm_data():
+    """CSV file consisting of the permutation data for the mapping results"""
+    perm_info = json.loads(request.form['perm_info'])
+
+    now = datetime.datetime.now()
+    time_str = now.strftime('%H:%M_%d%B%Y')
+
+    file_name = "Permutation_" + \
+        perm_info['num_perm'] + "_" + perm_info['trait_name'] + "_" + time_str
+
+    the_rows = [
+        ["#Permutation Test"],
+        ["#File_name: " + file_name],
+        ["#Metadata: From GeneNetwork.org"],
+        ["#Trait_ID: " + perm_info['trait_name']],
+        ["#Trait_description: " + perm_info['trait_description']],
+        ["#N_permutations: " + str(perm_info['num_perm'])],
+        ["#Cofactors: " + perm_info['cofactors']],
+        ["#N_cases: " + str(perm_info['n_samples'])],
+        ["#N_genotypes: " + str(perm_info['n_genotypes'])],
+        ["#Genotype_file: " + perm_info['genofile']],
+        ["#Units_linkage: " + perm_info['units_linkage']],
+        ["#Permutation_stratified_by: "
+            + ", ".join([str(cofactor) for cofactor in perm_info['strat_cofactors']])],
+        ["#RESULTS_1: Suggestive LRS(p=0.63) = "
+         + str(np.percentile(np.array(perm_info['perm_data']), 67))],
+        ["#RESULTS_2: Significant LRS(p=0.05) = " + str(
+            np.percentile(np.array(perm_info['perm_data']), 95))],
+        ["#RESULTS_3: Highly Significant LRS(p=0.01) = " + str(
+            np.percentile(np.array(perm_info['perm_data']), 99))],
+        ["#Comment: Results sorted from low to high peak linkage"]
+    ]
+
+    buff = io.StringIO()
+    writer = csv.writer(buff)
+    writer.writerows(the_rows)
+    for item in perm_info['perm_data']:
+        writer.writerow([item])
+    csv_data = buff.getvalue()
+    buff.close()
+
+    return Response(csv_data,
+                    mimetype='text/csv',
+                    headers={"Content-Disposition": "attachment;filename=" + file_name + ".csv"})
+
+
+@app.route("/show_temp_trait", methods=('POST',))
+def show_temp_trait_page():
+    with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor:
+        user_id = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8")
+                   or g.user_session.record.get("user_id") or "")
+        template_vars = show_trait.ShowTrait(cursor,
+                                             user_id=user_id,
+                                             kw=request.form)
+        template_vars.js_data = json.dumps(template_vars.js_data,
+                                           default=json_default_handler,
+                                           indent="   ")
+        return redirect(url_for("show_trait_page", dataset=template_vars.dataset.name, trait_id=template_vars.trait_id))
+
+
+@app.route("/show_trait")
+def show_trait_page():
+    def __show_trait__(privileges_data):
+        assert len(privileges_data) == 1
+        privileges_data = privileges_data[0]
+        trait_privileges = tuple(item for item in privileges_data["privileges"])
+        with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor:
+
+            user_id = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8")
+                       or g.user_session.record.get("user_id") or "")
+            template_vars = show_trait.ShowTrait(cursor,
+                                                 user_id=user_id,
+                                                 kw=request.args)
+            template_vars.js_data = json.dumps(template_vars.js_data,
+                                               default=json_default_handler,
+                                               indent="   ")
+            return render_template(
+                "show_trait.html",
+                **{
+                    **template_vars.__dict__,
+                    "user": privileges_data["user"],
+                    "trait_privileges": trait_privileges,
+                    "resource_id": privileges_data["resource_id"]
+                })
+    dataset = request.args["dataset"]
+    trait_id = request.args["trait_id"]
+
+    return client.post(
+        "auth/data/authorisation",
+        json={
+            "traits": [f"{dataset}::{trait_id}"]
+        }).either(with_flash_error(render_template("show_trait_error.html")),
+                  __show_trait__)
+
+
+@app.route("/heatmap", methods=('POST',))
+def heatmap_page():
+    start_vars = request.form
+    temp_uuid = uuid.uuid4()
+
+    traits = [trait.strip() for trait in start_vars['trait_list'].split(',')]
+    with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor:
+        if traits[0] != "":
+            version = "v5"
+            key = "heatmap:{}:".format(
+                version) + json.dumps(start_vars, sort_keys=True)
+            result = Redis.get(key)
+
+            if result:
+                result = pickle.loads(result)
+
+            else:
+                template_vars = heatmap.Heatmap(cursor, request.form, temp_uuid)
+                template_vars.js_data = json.dumps(template_vars.js_data,
+                                                   default=json_default_handler,
+                                                   indent="   ")
+
+                result = template_vars.__dict__
+
+                pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL)
+                Redis.set(key, pickled_result)
+                Redis.expire(key, 60 * 60)
+            rendered_template = render_template("heatmap.html", **result)
+
+        else:
+            rendered_template = render_template(
+                "empty_collection.html", **{'tool': 'Heatmap'})
+
+    return rendered_template
+
+
+@app.route("/bnw_page", methods=('POST',))
+def bnw_page():
+    start_vars = request.form
+
+    traits = [trait.strip() for trait in start_vars['trait_list'].split(',')]
+    if traits[0] != "":
+        template_vars = send_to_bnw.SendToBNW(request.form)
+
+        result = template_vars.__dict__
+        rendered_template = render_template("bnw_page.html", **result)
+    else:
+        rendered_template = render_template(
+            "empty_collection.html", **{'tool': 'BNW'})
+
+    return rendered_template
+
+
+@app.route("/webgestalt_page", methods=('POST',))
+def webgestalt_page():
+    start_vars = request.form
+
+    traits = [trait.strip() for trait in start_vars['trait_list'].split(',')]
+    if traits[0] != "":
+        template_vars = send_to_webgestalt.SendToWebGestalt(request.form)
+
+        result = template_vars.__dict__
+        rendered_template = render_template("webgestalt_page.html", **result)
+    else:
+        rendered_template = render_template(
+            "empty_collection.html", **{'tool': 'WebGestalt'})
+
+    return rendered_template
+
+
+@app.route("/geneweaver_page", methods=('POST',))
+def geneweaver_page():
+    start_vars = request.form
+
+    traits = [trait.strip() for trait in start_vars['trait_list'].split(',')]
+    if traits[0] != "":
+        template_vars = send_to_geneweaver.SendToGeneWeaver(request.form)
+
+        result = template_vars.__dict__
+        rendered_template = render_template("geneweaver_page.html", **result)
+    else:
+        rendered_template = render_template(
+            "empty_collection.html", **{'tool': 'GeneWeaver'})
+
+    return rendered_template
+
+
+@app.route("/comparison_bar_chart", methods=('POST',))
+def comp_bar_chart_page():
+    start_vars = request.form
+
+    traits = [trait.strip() for trait in start_vars['trait_list'].split(',')]
+    if traits[0] != "":
+        template_vars = comparison_bar_chart.ComparisonBarChart(request.form)
+        template_vars.js_data = json.dumps(template_vars.js_data,
+                                           default=json_default_handler,
+                                           indent="   ")
+
+        result = template_vars.__dict__
+        rendered_template = render_template(
+            "comparison_bar_chart.html", **result)
+    else:
+        rendered_template = render_template(
+            "empty_collection.html", **{'tool': 'Comparison Bar Chart'})
+
+    return rendered_template
+
+
+@app.route("/mapping_results_container")
+def mapping_results_container_page():
+    return render_template("mapping_results_container.html")
+
+
+@app.route("/loading", methods=('POST',))
+def loading_page():
+    initial_start_vars = request.form
+    start_vars_container = {}
+    n_samples = 0  # ZS: So it can be displayed on loading page
+    if 'wanted_inputs' in initial_start_vars:
+        wanted = initial_start_vars['wanted_inputs'].split(",")
+        start_vars = {}
+        for key, value in list(initial_start_vars.items()):
+            if key in wanted:
+                start_vars[key] = value
+
+        sample_vals_dict = json.loads(start_vars['sample_vals'])
+        if 'n_samples' in start_vars:
+            n_samples = int(start_vars['n_samples'])
+        else:
+            if 'group' in start_vars:
+                dataset = create_dataset(
+                    start_vars['dataset'], group_name=start_vars['group'])
+            else:
+                dataset = create_dataset(start_vars['dataset'])
+            start_vars['trait_name'] = start_vars['trait_id']
+            if dataset.type == "Publish":
+                start_vars['trait_name'] = f"{dataset.group.code}_{start_vars['trait_name']}"
+            samples = dataset.group.samplelist
+            if 'genofile' in start_vars:
+                if start_vars['genofile'] != "":
+                    genofile_string = start_vars['genofile']
+                    dataset.group.genofile = genofile_string.split(":")[0]
+                    genofile_samples = run_mapping.get_genofile_samplelist(
+                        dataset)
+                    if len(genofile_samples) > 1:
+                        samples = genofile_samples
+
+            for sample in samples:
+                if sample in sample_vals_dict:
+                    if sample_vals_dict[sample] != "x":
+                        n_samples += 1
+
+        start_vars['n_samples'] = n_samples
+        start_vars['vals_hash'] = generate_hash_of_string(
+            str(sample_vals_dict))
+        if start_vars['dataset'] != "Temp":  # Currently can't get diff for temp traits
+            start_vars['vals_diff'] = get_diff_of_vals(sample_vals_dict, str(
+                start_vars['trait_id'] + ":" + str(start_vars['dataset'])))
+
+        start_vars['wanted_inputs'] = initial_start_vars['wanted_inputs']
+
+        start_vars_container['start_vars'] = start_vars
+    else:
+        start_vars_container['start_vars'] = initial_start_vars
+
+    rendered_template = render_template("loading.html", **start_vars_container)
+
+    return rendered_template
+
+
+@app.route("/run_mapping", methods=('POST',))
+@app.route("/run_mapping/<path:hash_of_inputs>")
+def mapping_results_page(hash_of_inputs=None):
+    if hash_of_inputs:
+        initial_start_vars = json.loads(Redis.get(hash_of_inputs))
+        initial_start_vars['hash_of_inputs'] = hash_of_inputs
+    else:
+        initial_start_vars = request.form
+
+        # Get hash of inputs (as JSON) for sharing results
+        inputs_json = json.dumps(initial_start_vars, sort_keys=True)
+        dhash = hashlib.md5()
+        dhash.update(inputs_json.encode())
+        hash_of_inputs = dhash.hexdigest()
+
+        # Just store for one hour on initial load; will be stored for longer if user clicks Share
+        Redis.set(hash_of_inputs, inputs_json, ex=60*60*24*30)
+
+    temp_uuid = initial_start_vars['temp_uuid']
+    wanted = (
+        'trait_id',
+        'dataset',
+        'group',
+        'species',
+        'samples',
+        'vals',
+        'sample_vals',
+        'vals_hash',
+        'first_run',
+        'output_files',
+        'geno_db_exists',
+        'method',
+        'mapping_results_path',
+        'trimmed_markers',
+        'selected_chr',
+        'chromosomes',
+        'mapping_scale',
+        'plotScale',
+        'score_type',
+        'suggestive',
+        'significant',
+        'num_perm',
+        'permCheck',
+        'perm_strata',
+        'categorical_vars',
+        'perm_output',
+        'num_bootstrap',
+        'bootCheck',
+        'bootstrap_results',
+        'LRSCheck',
+        'covariates',
+        'maf',
+        'use_loco',
+        'manhattan_plot',
+        'color_scheme',
+        'manhattan_single_color',
+        'control_marker',
+        'do_control',
+        'genofile',
+        'genofile_string',
+        'pair_scan',
+        'startMb',
+        'endMb',
+        'graphWidth',
+        'lrsMax',
+        'additiveCheck',
+        'showSNP',
+        'showHomology',
+        'showGenes',
+        'viewLegend',
+        'haplotypeAnalystCheck',
+        'mapmethod_rqtl',
+        'mapmodel_rqtl',
+        'temp_trait',
+        'n_samples',
+        'transform',
+        'hash_of_inputs',
+        'dataid'
+    )
+    start_vars = {}
+    for key, value in list(initial_start_vars.items()):
+        if key in wanted:
+            start_vars[key] = value
+
+    start_vars['hash_of_inputs'] = hash_of_inputs
+
+    # Store trait sample data in Redis, so additive effect scatterplots can include edited values
+    dhash = hashlib.md5()
+    dhash.update(start_vars['sample_vals'].encode())
+    samples_hash = dhash.hexdigest()
+    Redis.set(samples_hash, start_vars['sample_vals'], ex=7*24*60*60)
+    start_vars['dataid'] = samples_hash
+
+    version = "v3"
+    key = "mapping_results:{}:".format(
+        version) + json.dumps(start_vars, sort_keys=True)
+    result = None  # Just for testing
+
+    if result:
+        result = pickle.loads(result)
+    else:
+        template_vars = run_mapping.RunMapping(start_vars, temp_uuid)
+        if template_vars.no_results:
+            raise NoMappingResultsError(
+                start_vars["trait_id"], start_vars["dataset"], start_vars["method"])
+
+        if not template_vars.pair_scan:
+            template_vars.js_data = json.dumps(template_vars.js_data,
+                                               default=json_default_handler,
+                                               indent="   ")
+
+        result = template_vars.__dict__
+
+        if result['pair_scan']:
+            rendered_template = render_template(
+                "pair_scan_results.html", **result)
+        else:
+            gn1_template_vars = display_mapping_results.DisplayMappingResults(
+                result).__dict__
+
+            rendered_template = render_template(
+                "mapping_results.html", **gn1_template_vars)
+
+    return rendered_template
+
+@app.route("/cache_mapping_inputs", methods=('POST',))
+def cache_mapping_inputs():
+    cache_id = request.form.get("inputs_hash")
+    inputs_json = Redis.get(cache_id)
+    Redis.set(cache_id, inputs_json)
+
+    return "Success"
+
+@app.route("/export_mapping_results", methods=('POST',))
+def export_mapping_results():
+    file_path = request.form.get("results_path")
+    results_csv = open(file_path, "r").read()
+    response = Response(results_csv,
+                        mimetype='text/csv',
+                        headers={"Content-Disposition": "attachment;filename=" + os.path.basename(file_path)})
+
+    return response
+
+
+@app.route("/export_corr_matrix", methods=('POST',))
+def export_corr_matrix():
+    file_path = request.form.get("export_filepath")
+    file_name = request.form.get("export_filename")
+    results_csv = open(file_path, "r").read()
+    response = Response(results_csv,
+                        mimetype='text/csv',
+                        headers={"Content-Disposition": "attachment;filename=" + file_name + ".csv"})
+
+    return response
+
+
+@app.route("/export", methods=('POST',))
+def export():
+    svg_xml = request.form.get("data", "Invalid data")
+    filename = request.form.get("filename", "manhattan_plot_snp")
+    response = Response(svg_xml, mimetype="image/svg+xml")
+    response.headers["Content-Disposition"] = "attachment; filename=%s" % filename
+    return response
+
+
+@app.route("/export_pdf", methods=('POST',))
+def export_pdf():
+    import cairosvg
+    svg_xml = request.form.get("data", "Invalid data")
+    filename = request.form.get("filename", "interval_map_pdf")
+    pdf_file = cairosvg.svg2pdf(bytestring=svg_xml)
+    response = Response(pdf_file, mimetype="application/pdf")
+    response.headers["Content-Disposition"] = "attachment; filename=%s" % filename
+    return response
+
+
+@app.route("/network_graph", methods=('POST',))
+def network_graph_page():
+    start_vars = request.form
+    traits = [trait.strip() for trait in start_vars['trait_list'].split(',')]
+    if traits[0] != "":
+        template_vars = network_graph.NetworkGraph(start_vars)
+        template_vars.js_data = json.dumps(template_vars.js_data,
+                                           default=json_default_handler,
+                                           indent="   ")
+
+        return render_template("network_graph.html", **template_vars.__dict__)
+    else:
+        return render_template("empty_collection.html", **{'tool': 'Network Graph'})
+
+def __handle_correlation_error__(exc):
+    return render_template(
+        "correlation_error_page.html",
+        error = {
+            "error-type": {
+                "WrongCorrelationType": "Wrong Correlation Type"
+            }[type(exc).__name__],
+            "error-message": exc.args[0]
+        })
+
+@app.route("/corr_compute", methods=('POST', 'GET'))
+def corr_compute_page():
+    with Redis.from_url(REDIS_URL, decode_responses=True) as rconn:
+        if request.method == "POST":
+            request_received = datetime.datetime.utcnow()
+            filename=hmac.hmac_creation(f"request_form_{request_received.isoformat()}")
+            filepath = f"{TMPDIR}{filename}"
+            with open(filepath, "wb") as pfile:
+                pickle.dump(request.form, pfile, protocol=pickle.HIGHEST_PROTOCOL)
+                job_id = jobs.queue(
+                    rconn, {
+                        "command": [
+                            sys.executable, "-m", "scripts.corr_compute", filepath,
+                            g.user_session.user_id],
+                        "request_received_time": request_received.isoformat(),
+                        "status": "queued"
+                    })
+                jobs.run(job_id, REDIS_URL)
+
+            return redirect(url_for("corr_compute_page", job_id=str(job_id)))
+
+        job = jobs.job(
+            rconn, UUID(request.args.get("job_id"))).maybe(
+                {}, lambda the_job: the_job)
+
+        if jobs.completed_successfully(job):
+            output = json.loads(job.get("stdout", "{}"))
+            return render_template("correlation_page.html", **output)
+
+        if jobs.completed_erroneously(job):
+            try:
+                error_output = {
+                    "error-type": "ComputeError",
+                    "error-message": "There was an error computing the correlations",
+                    **json.loads(job.get("stdout") or "{}"),
+                    "stderr-output": job.get("stderr", "").split("\n")
+                }
+                return render_template(
+                    "correlation_error_page.html", error=error_output)
+            except json.decoder.JSONDecodeError as jde:
+                raise Exception(f"STDOUT: {job.get('stdout')}") from jde
+
+        return render_template("loading_corrs.html")
+
+
+@app.route("/corr_matrix", methods=('POST',))
+def corr_matrix_page():
+    start_vars = request.form
+    traits = [trait.strip() for trait in start_vars['trait_list'].split(',')]
+    if len(traits) > 1:
+        template_vars = show_corr_matrix.CorrelationMatrix(start_vars)
+        template_vars.js_data = json.dumps(template_vars.js_data,
+                                           default=json_default_handler,
+                                           indent="   ")
+
+        return render_template("correlation_matrix.html", **template_vars.__dict__)
+    else:
+        return render_template("empty_collection.html", **{'tool': 'Correlation Matrix'})
+
+
+@app.route("/corr_scatter_plot")
+def corr_scatter_plot_page():
+    template_vars = corr_scatter_plot.CorrScatterPlot(request.args)
+    template_vars.js_data = json.dumps(template_vars.js_data,
+                                       default=json_default_handler,
+                                       indent="   ")
+    return render_template("corr_scatterplot.html", **template_vars.__dict__)
+
+
+@app.route("/snp_browser", methods=('GET',))
+def snp_browser_page():
+    with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor:
+        template_vars = snp_browser.SnpBrowser(cursor, request.args)
+        return render_template("snp_browser.html", **template_vars.__dict__)
+
+
+@app.route("/db_info", methods=('GET',))
+def db_info_page():
+    template_vars = InfoPage(request.args)
+
+    return render_template("info_page.html", **template_vars.__dict__)
+
+
+@app.route("/snp_browser_table", methods=('GET',))
+def snp_browser_table():
+    with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor:
+        snp_table_data = snp_browser.SnpBrowser(cursor, request.args)
+        current_page = server_side.ServerSideTable(
+            snp_table_data.rows_count,
+            snp_table_data.table_rows,
+            snp_table_data.header_data_names,
+            request.args,
+        ).get_page()
+
+        return flask.jsonify(current_page)
+
+
+@app.route("/tutorial/WebQTLTour", methods=('GET',))
+def tutorial_page():
+    # ZS: Currently just links to GN1
+    return redirect("http://gn1.genenetwork.org/tutorial/WebQTLTour/")
+
+
+@app.route("/tutorial/security", methods=('GET',))
+def security_tutorial_page():
+    # ZS: Currently just links to GN1
+    return render_template("admin/security_help.html")
+
+
+@app.route("/submit_bnw", methods=('POST',))
+def submit_bnw():
+    return render_template("empty_collection.html", **{'tool': 'Correlation Matrix'})
+
+# Take this out or secure it before putting into production
+
+
+@app.route("/get_temp_data")
+def get_temp_data():
+    temp_uuid = request.args['key']
+    return flask.jsonify(temp_data.TempData(temp_uuid).get_all())
+
+
+@app.route("/browser_input", methods=('GET',))
+def browser_inputs():
+    """  Returns JSON from tmp directory for the purescript genome browser"""
+
+    filename = request.args['filename']
+
+    with open("{}/gn2/".format(TEMPDIR) + filename + ".json", "r") as the_file:
+        file_contents = json.load(the_file)
+
+    return flask.jsonify(file_contents)
+
+
+def json_default_handler(obj):
+    """Based on http://stackoverflow.com/a/2680060/1175849"""
+    # Handle datestamps
+    if hasattr(obj, 'isoformat'):
+        return obj.isoformat()
+    # Handle integer keys for dictionaries
+    elif isinstance(obj, int) or isinstance(obj, uuid.UUID):
+        return str(obj)
+    # Handle custom objects
+    if hasattr(obj, '__dict__'):
+        return obj.__dict__
+    else:
+        raise TypeError('Object of type %s with value of %s is not JSON serializable' % (
+            type(obj), repr(obj)))
+
+
+@app.route("/admin/data-sample/diffs/")
+@edit_access_required
+def display_diffs_admin():
+    TMPDIR = current_app.config.get("TMPDIR")
+    DIFF_DIR = f"{TMPDIR}/sample-data/diffs"
+    files = []
+    if os.path.exists(DIFF_DIR):
+        files = os.listdir(DIFF_DIR)
+        files = filter(lambda x: not(x.endswith((".approved", ".rejected"))),
+                       files)
+    return render_template("display_files_admin.html",
+                           files=files)
+
+
+@app.route("/user/data-sample/diffs/")
+def display_diffs_users():
+    TMPDIR = current_app.config.get("TMPDIR")
+    DIFF_DIR = f"{TMPDIR}/sample-data/diffs"
+    files = []
+    author = g.user_session.record.get(b'user_name').decode("utf-8")
+    if os.path.exists(DIFF_DIR):
+        files = os.listdir(DIFF_DIR)
+        files = filter(lambda x: not(x.endswith((".approved", ".rejected")))
+                       and author in x,
+                       files)
+    return render_template("display_files_user.html",
+                           files=files)
+
+
+@app.route("/genewiki/<symbol>")
+def display_generif_page(symbol):
+    """Fetch GeneRIF metadata from GN3 and display it"""
+    entries = requests.get(
+        urljoin(
+            GN3_LOCAL_URL,
+            f"/api/metadata/genewiki/{symbol}"
+        )
+    ).json()
+    return render_template(
+        "generif.html",
+        symbol=symbol,
+        entries=entries
+    )
+
+
+@app.route("/datasets/<name>", methods=('GET',))
+def get_dataset(name):
+    metadata = requests.get(
+        urljoin(
+            GN3_LOCAL_URL,
+            f"/api/metadata/datasets/{name}")
+    ).json()
+    return render_template(
+        "dataset.html",
+        name=name,
+        dataset=metadata
+    )
+
+@app.route("/datasets/search", methods=('POST',))
+def search_for_dataset():
+    search = request.form.get('search')
+    return render_template(
+        "metadata/dataset_search_results.html",
+        results=search
+    )
+
+@app.route("/publications/<name>", methods=('GET',))
+def get_publication(name):
+    metadata = requests.get(
+        urljoin(
+            GN3_LOCAL_URL,
+            f"/api/metadata/publications/{name}")
+    ).json()
+    return render_template(
+        "publication.html",
+        metadata=metadata,
+    )
+
+
+@app.route("/phenotypes/<name>", methods=('GET',))
+@app.route("/phenotypes/<group>/<name>", methods=('GET',))
+def get_phenotype(name, group=None):
+    if group:
+        name = f"{group}_{name}"
+    metadata = requests.get(
+        urljoin(
+            GN3_LOCAL_URL,
+            f"/api/metadata/phenotypes/{name}")
+    ).json()
+    return render_template(
+        "phenotype.html",
+        metadata=metadata,
+    )
+
+
+@app.route("/genotypes/<name>", methods=('GET',))
+def get_genotype(name):
+    metadata = requests.get(
+        urljoin(
+            GN3_LOCAL_URL,
+            f"/api/metadata/genotypes/{name}")
+    ).json()
+    return render_template(
+        "genotype.html",
+        metadata=metadata,
+    )
+
+@app.route("/case-attribute/<int:inbredset_id>/edit", methods=["GET", "POST"])
+def edit_case_attributes(inbredset_id: int) -> Response:
+    """
+    Edit the case-attributes for InbredSet group identified by `inbredset_id`.
+    """
+    if request.method == "POST":
+        form = request.form
+        def __process_data__(acc, item):
+            _new, strain, calabel = tuple(val.strip() for val in item[0].split(":"))
+            old_row = acc.get(strain, {})
+            return {
+                **acc,
+                strain: {
+                    **old_row, "case-attributes": {
+                        **old_row.get("case-attributes", {}),
+                        calabel: item[1]
+                    }
+                }
+            }
+
+        edit_case_attributes_page = redirect(url_for(
+            "edit_case_attributes", inbredset_id=inbredset_id))
+        token = session_info()["user"]["token"].either(
+            lambda err: err, lambda tok: tok["access_token"])
+        def flash_success(resp):
+            def __succ__(remote_resp):
+                flash(f"Success: {remote_resp.json()['message']}", "alert-success")
+                return resp
+            return __succ__
+        return monad_requests.post(
+            urljoin(
+                current_app.config["GN_SERVER_URL"],
+                f"/api/case-attribute/{inbredset_id}/edit"),
+            json={
+                "edit-data": reduce(__process_data__, form.items(), {})
+            },
+            headers={
+                "Authorization": f"Bearer {token}"}).either(
+                with_flash_error(edit_case_attributes_page),
+                flash_success(edit_case_attributes_page))
+
+    def __fetch_strains__(inbredset_group):
+        return monad_requests.get(urljoin(
+            current_app.config["GN_SERVER_URL"],
+            f"/api/case-attribute/{inbredset_id}/strains")).then(
+                lambda resp: {**inbredset_group, "strains": resp.json()})
+
+    def __fetch_names__(strains):
+        return monad_requests.get(urljoin(
+            current_app.config["GN_SERVER_URL"],
+            f"/api/case-attribute/{inbredset_id}/names")).then(
+                lambda resp: {**strains, "case_attribute_names": resp.json()})
+
+    def __fetch_values__(canames):
+        return monad_requests.get(urljoin(
+            current_app.config["GN_SERVER_URL"],
+            f"/api/case-attribute/{inbredset_id}/values")).then(
+                lambda resp: {**canames, "case_attribute_values": {
+                    value["StrainName"]: value for value in resp.json()}})
+
+    return monad_requests.get(urljoin(
+        current_app.config["GN_SERVER_URL"],
+        f"/api/case-attribute/{inbredset_id}")).then(
+            lambda resp: {"inbredset_group": resp.json()}).then(
+                __fetch_strains__).then(__fetch_names__).then(
+                    __fetch_values__).either(
+                        lambda err: err, ## TODO: Handle error better
+                        lambda values: render_template(
+                            "edit_case_attributes.html", inbredset_id=inbredset_id, **values))
+
+@app.route("/case-attribute/<int:inbredset_id>/list-diffs", methods=["GET"])
+def list_case_attribute_diffs(inbredset_id: int) -> Response:
+    """List any diffs awaiting review."""
+    return monad_requests.get(urljoin(
+        current_app.config["GN_SERVER_URL"],
+        f"/api/case-attribute/{inbredset_id}/diff/list")).then(
+            lambda resp: resp.json()).either(
+                lambda err: render_template(
+                    "list_case_attribute_diffs_error.html",
+                    inbredset_id=inbredset_id,
+                    error=err),
+                lambda diffs: render_template(
+                "list_case_attribute_diffs.html",
+                    inbredset_id=inbredset_id,
+                    diffs=diffs))
+
+@app.route("/case-attribute/<int:inbredset_id>/diff/<int:diff_id>/view", methods=["GET"])
+def view_diff(inbredset_id:int, diff_id: int) -> Response:
+    """View the pending diff."""
+    token = session_info()["user"]["token"].either(
+        lambda err: err, lambda tok: tok["access_token"])
+    return monad_requests.get(
+        urljoin(current_app.config["GN_SERVER_URL"],
+                f"/api/case-attribute/{inbredset_id}/diff/{diff_id}/view"),
+        headers={"Authorization": f"Bearer {token}"}).then(
+            lambda resp: resp.json()).either(
+                lambda err: render_template(
+                    "view_case_attribute_diff_error.html", error=err.json()),
+                lambda diff: render_template(
+                    "view_case_attribute_diff.html", diff=diff))
+
+@app.route("/case-attribute/diff/approve-reject", methods=["POST"])
+def approve_reject_diff() -> Response:
+    """Approve/Reject the diff."""
+    try:
+        form = request.form
+        action = form["action"]
+        assert action in ("approve", "reject")
+        diff_data = json.loads(form["diff_data"])
+        diff_data = {
+            **diff_data,
+            "created": datetime.datetime.fromisoformat(diff_data["created"])}
+        inbredset_id = diff_data["inbredset_id"]
+        filename = (
+            f"{inbredset_id}:::{diff_data['user_id']}:::"
+            f"{diff_data['created'].isoformat()}.json")
+
+        list_diffs_page = url_for("list_case_attribute_diffs",
+                                  inbredset_id=inbredset_id)
+        token = session_info()["user"]["token"].either(
+            lambda err: err, lambda tok: tok["access_token"])
+        def __error__(resp):
+            error = resp.json()
+            flash((f"{resp.status_code} {error['error']}: "
+                   f"{error['error_description']}"),
+                  "alert-danger")
+            return redirect(list_diffs_page)
+        def __success__(results):
+            flash(results["message"], "alert-success")
+            return redirect(list_diffs_page)
+        return monad_requests.post(
+            urljoin(current_app.config["GN_SERVER_URL"],
+                    f"/api/case-attribute/{action}/{filename}"),
+            headers={"Authorization": f"Bearer {token}"}).then(
+                lambda resp: resp.json()
+            ).either(
+                __error__, __success__)
+    except AssertionError as _ae:
+        flash("Invalid action! Expected either 'approve' or 'reject'.",
+              "alert-danger")
+        return redirect(url_for("view_diff",
+                                inbredset_id=inbredset_id,
+                                diff_id=form["diff_id"]))