diff options
| -rw-r--r-- | wqflask/scripts/corr_compute.py | 78 | ||||
| -rw-r--r-- | wqflask/wqflask/templates/loading_corrs.html | 28 | ||||
| -rw-r--r-- | wqflask/wqflask/views.py | 59 |
3 files changed, 148 insertions, 17 deletions
diff --git a/wqflask/scripts/corr_compute.py b/wqflask/scripts/corr_compute.py new file mode 100644 index 00000000..c98a66a4 --- /dev/null +++ b/wqflask/scripts/corr_compute.py @@ -0,0 +1,78 @@ +"""Compute the correlations.""" + +import sys +import json +import pickle +import pathlib +import datetime + +from flask import g + +from wqflask import app +from wqflask.user_session import UserSession +from wqflask.correlation.show_corr_results import set_template_vars +from wqflask.correlation.correlation_gn3_api import compute_correlation + +class UserSessionSimulator(): + + def __init__(self, user_id): + self._user_id = user_id + + @property + def user_id(self): + return self._user_id + +error_types = { + "WrongCorrelationType": "Wrong Correlation Type", + "CalledProcessError": "Called Process Error" +} + +def e_time(): + return datetime.datetime.utcnow().isoformat() + +def compute(form): + import subprocess + from gn3.settings import CORRELATION_COMMAND + try: + correlation_results = compute_correlation(form, compute_all=True) + except Exception as exc: + return { + "error-type": error_types[type(exc).__name__], + "error-message": exc.args[0] + } + + return set_template_vars(form, correlation_results) + +if __name__ == "__main__": + ARGS_COUNT = 3 + if len(sys.argv) < ARGS_COUNT: + print(f"{e_time()}: You need to pass the file with the picked form", + file=sys.stderr) + sys.exit(1) + + if len(sys.argv) > ARGS_COUNT: + print(f"{e_time()}: Unknown arguments {sys.argv[ARGS_COUNT:]}", + file=sys.stderr) + sys.exit(1) + + filepath = pathlib.Path(sys.argv[1]) + if not filepath.exists(): + print(f"File not found '{filepath}'", file=sys.stderr) + sys.exit(2) + + with open(filepath, "rb") as pfile: + form = pickle.Unpickler(pfile).load() + + with app.app_context(): + g.user_session = UserSessionSimulator(sys.argv[2]) + results = compute(form) + + print(json.dumps(results), file=sys.stdout) + + if "error-type" in results: + print( + f"{results['error-type']}: {results['error-message']}", + file=sys.stderr) + sys.exit(3) + + sys.exit(0) diff --git a/wqflask/wqflask/templates/loading_corrs.html b/wqflask/wqflask/templates/loading_corrs.html new file mode 100644 index 00000000..8abd5464 --- /dev/null +++ b/wqflask/wqflask/templates/loading_corrs.html @@ -0,0 +1,28 @@ +<!DOCTYPE html> +<html> + + <head> + <title>Loading Correlation Results</title> + + <meta charset="utf-8" /> + <meta http-equiv="refresh" content="5"> + + <link rel="stylesheet" type="text/css" + href="{{url_for('css', filename='bootstrap/css/bootstrap.css')}}" /> + <link rel="stylesheet" type="text/css" + href="/static/new/css/bootstrap-custom.css" /> + </head> + + <body> + <div style="margin: 0; position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%);"> + <h1> Correlation Computation in progress ...</h1> + <div style="text-align: center;"> + <img align="center" src="/static/gif/waitAnima2.gif"> + </div> + </div> + + <script src="{{ url_for('js', filename='jquery/jquery.min.js') }}" type="text/javascript"></script> + <script src="{{ url_for('js', filename='bootstrap/js/bootstrap.min.js') }}" type="text/javascript"></script> + </body> + +</html> diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 0d8e5e64..211a8f13 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -19,6 +19,8 @@ import xlsxwriter from zipfile import ZipFile from zipfile import ZIP_DEFLATED +from uuid import UUID + from wqflask import app from gn3.computations.gemma import generate_hash_of_string @@ -31,6 +33,7 @@ from flask import render_template from flask import send_from_directory from flask import redirect from flask import send_file +from flask import url_for # Some of these (like collect) might contain endpoints, so they're still used. # Blueprints should probably be used instead. @@ -71,6 +74,7 @@ from wqflask.db_info import InfoPage from utility import temp_data from utility.tools import TEMPDIR from utility.tools import USE_REDIS +from utility.tools import REDIS_URL from utility.tools import GN_SERVER_URL from utility.tools import GN_VERSION from utility.tools import JS_TWITTER_POST_FETCHER_PATH @@ -78,11 +82,16 @@ from utility.tools import JS_GUIX_PATH from utility.helper_functions import get_species_groups from utility.redis_tools import get_redis_conn +import utility.hmac as hmac + +from base.webqtlConfig import TMPDIR from base.webqtlConfig import GENERATED_IMAGE_DIR from wqflask.database import database_connection +import jobs.jobs as jobs + Redis = get_redis_conn() @@ -822,24 +831,40 @@ def __handle_correlation_error__(exc): "error-message": exc.args[0] }) -@app.route("/corr_compute", methods=('POST',)) +@app.route("/corr_compute", methods=('POST', 'GET')) def corr_compute_page(): - import subprocess - from gn3.settings import CORRELATION_COMMAND - try: - correlation_results = compute_correlation( - request.form, compute_all=True) - except WrongCorrelationType as exc: - return __handle_correlation_error__(exc) - except subprocess.CalledProcessError as cpe: - actual_command = ( - os.readlink(CORRELATION_COMMAND) - if os.path.islink(CORRELATION_COMMAND) - else CORRELATION_COMMAND) - raise Exception(actual_command, cpe.output, cpe.stdout, cpe.stderr) from cpe - - correlation_results = set_template_vars(request.form, correlation_results) - return render_template("correlation_page.html", **correlation_results) + with Redis.from_url(REDIS_URL, decode_responses=True) as rconn: + if request.method == "POST": + request_received = datetime.datetime.utcnow() + filename=hmac.hmac_creation(f"request_form_{request_received.isoformat()}") + filepath = f"{TMPDIR}{filename}" + with open(filepath, "wb") as pfile: + pickle.dump(request.form, pfile, protocol=pickle.HIGHEST_PROTOCOL) + job_id = jobs.queue( + rconn, { + "command": [ + sys.executable, "-m", "scripts.corr_compute", filepath, + g.user_session.user_id], + "request_received_time": request_received.isoformat(), + "status": "queued" + }) + jobs.run(job_id, REDIS_URL) + + return redirect(url_for("corr_compute_page", job_id=str(job_id))) + + job = jobs.job( + rconn, UUID(request.args.get("job_id"))).maybe( + {}, lambda the_job: the_job) + + if jobs.completed_successfully(job): + output = json.loads(job.get("stdout", "{}")) + return render_template("correlation_page.html", **output) + + if jobs.completed_erroneously(job): + output = json.loads(job.get("stdout", "{}")) + return render_template("correlation_error_page.html", error=output) + + return render_template("loading_corrs.html") @app.route("/corr_matrix", methods=('POST',)) |