Refactor: run correlation computation externally

* wqflask/scripts/corr_compute.py: move correlation computation to external script. * wqflask/wqflask/templates/loading_corrs.html: Provide UI to display while computation is still not complete. * wqflask/wqflask/views.py: Dispatch correlation computation to external script and display appropriate UI for each valid state of the computation.
author: Frederick Muriuki Muriithi 2022-10-28 11:51:50 +0300
committer: Frederick Muriuki Muriithi 2022-10-28 15:58:01 +0300
commit: 0bb49f9873e958cd480f7c9fc20f3a8db6f62003 (patch)
tree: 1e13625ddb9b581ad16db98ab75619ef1fdacdce
parent: 2357452ccdcf475bcad3a5da7de43f4d481e296b (diff)
download: genenetwork2-0bb49f9873e958cd480f7c9fc20f3a8db6f62003.tar.gz
3 files changed, 148 insertions, 17 deletions
diff --git a/wqflask/scripts/corr_compute.py b/wqflask/scripts/corr_compute.py
new file mode 100644
index 00000000..c98a66a4
--- /dev/null
+++ b/wqflask/scripts/corr_compute.py
@@ -0,0 +1,78 @@
+"""Compute the correlations."""
+
+import sys
+import json
+import pickle
+import pathlib
+import datetime
+
+from flask import g
+
+from wqflask import app
+from wqflask.user_session import UserSession
+from wqflask.correlation.show_corr_results import set_template_vars
+from wqflask.correlation.correlation_gn3_api import compute_correlation
+
+class UserSessionSimulator():
+
+    def __init__(self, user_id):
+        self._user_id = user_id
+
+    @property
+    def user_id(self):
+        return self._user_id
+
+error_types = {
+    "WrongCorrelationType": "Wrong Correlation Type",
+    "CalledProcessError": "Called Process Error"
+}
+
+def e_time():
+    return datetime.datetime.utcnow().isoformat()
+
+def compute(form):
+    import subprocess
+    from gn3.settings import CORRELATION_COMMAND
+    try:
+        correlation_results = compute_correlation(form, compute_all=True)
+    except Exception as exc:
+        return {
+            "error-type": error_types[type(exc).__name__],
+            "error-message": exc.args[0]
+        }
+
+    return set_template_vars(form, correlation_results)
+
+if __name__ == "__main__":
+    ARGS_COUNT = 3
+    if len(sys.argv) < ARGS_COUNT:
+        print(f"{e_time()}: You need to pass the file with the picked form",
+              file=sys.stderr)
+        sys.exit(1)
+
+    if len(sys.argv) > ARGS_COUNT:
+        print(f"{e_time()}: Unknown arguments {sys.argv[ARGS_COUNT:]}",
+              file=sys.stderr)
+        sys.exit(1)
+
+    filepath = pathlib.Path(sys.argv[1])
+    if not filepath.exists():
+        print(f"File not found '{filepath}'", file=sys.stderr)
+        sys.exit(2)
+
+    with open(filepath, "rb") as pfile:
+        form = pickle.Unpickler(pfile).load()
+
+    with app.app_context():
+        g.user_session = UserSessionSimulator(sys.argv[2])
+        results = compute(form)
+
+    print(json.dumps(results), file=sys.stdout)
+
+    if "error-type" in results:
+        print(
+            f"{results['error-type']}: {results['error-message']}",
+            file=sys.stderr)
+        sys.exit(3)
+
+    sys.exit(0)
diff --git a/wqflask/wqflask/templates/loading_corrs.html b/wqflask/wqflask/templates/loading_corrs.html
new file mode 100644
index 00000000..8abd5464
--- /dev/null
+++ b/wqflask/wqflask/templates/loading_corrs.html
@@ -0,0 +1,28 @@
+<!DOCTYPE html>
+<html>
+
+  <head>
+    <title>Loading Correlation Results</title>
+
+    <meta charset="utf-8" />
+    <meta http-equiv="refresh" content="5">
+
+    <link rel="stylesheet" type="text/css"
+	  href="{{url_for('css', filename='bootstrap/css/bootstrap.css')}}" />
+    <link rel="stylesheet" type="text/css"
+	  href="/static/new/css/bootstrap-custom.css" />
+  </head>
+
+  <body>
+    <div style="margin: 0; position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%);">
+      <h1>&nbsp;Correlation Computation in progress ...</h1>
+      <div style="text-align: center;">
+	<img align="center" src="/static/gif/waitAnima2.gif">
+      </div>
+    </div>
+
+    <script src="{{ url_for('js', filename='jquery/jquery.min.js') }}" type="text/javascript"></script>
+    <script src="{{ url_for('js', filename='bootstrap/js/bootstrap.min.js') }}" type="text/javascript"></script>
+  </body>
+
+</html>
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 0d8e5e64..211a8f13 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -19,6 +19,8 @@ import xlsxwriter
 from zipfile import ZipFile
 from zipfile import ZIP_DEFLATED
 
+from uuid import UUID
+
 from wqflask import app
 
 from gn3.computations.gemma import generate_hash_of_string
@@ -31,6 +33,7 @@ from flask import render_template
 from flask import send_from_directory
 from flask import redirect
 from flask import send_file
+from flask import url_for
 
 # Some of these (like collect) might contain endpoints, so they're still used.
 # Blueprints should probably be used instead.
@@ -71,6 +74,7 @@ from wqflask.db_info import InfoPage
 from utility import temp_data
 from utility.tools import TEMPDIR
 from utility.tools import USE_REDIS
+from utility.tools import REDIS_URL
 from utility.tools import GN_SERVER_URL
 from utility.tools import GN_VERSION
 from utility.tools import JS_TWITTER_POST_FETCHER_PATH
@@ -78,11 +82,16 @@ from utility.tools import JS_GUIX_PATH
 from utility.helper_functions import get_species_groups
 from utility.redis_tools import get_redis_conn
 
+import utility.hmac as hmac
+
 
+from base.webqtlConfig import TMPDIR
 from base.webqtlConfig import GENERATED_IMAGE_DIR
 
 from wqflask.database import database_connection
 
+import jobs.jobs as jobs
+
 
 Redis = get_redis_conn()
 
@@ -822,24 +831,40 @@ def __handle_correlation_error__(exc):
             "error-message": exc.args[0]
         })
 
-@app.route("/corr_compute", methods=('POST',))
+@app.route("/corr_compute", methods=('POST', 'GET'))
 def corr_compute_page():
-    import subprocess
-    from gn3.settings import CORRELATION_COMMAND
-    try:
-        correlation_results = compute_correlation(
-            request.form, compute_all=True)
-    except WrongCorrelationType as exc:
-        return __handle_correlation_error__(exc)
-    except subprocess.CalledProcessError as cpe:
-        actual_command = (
-            os.readlink(CORRELATION_COMMAND)
-            if os.path.islink(CORRELATION_COMMAND)
-            else CORRELATION_COMMAND)
-        raise Exception(actual_command, cpe.output, cpe.stdout, cpe.stderr) from cpe
-
-    correlation_results = set_template_vars(request.form, correlation_results)
-    return render_template("correlation_page.html", **correlation_results)
+    with Redis.from_url(REDIS_URL, decode_responses=True) as rconn:
+        if request.method == "POST":
+            request_received = datetime.datetime.utcnow()
+            filename=hmac.hmac_creation(f"request_form_{request_received.isoformat()}")
+            filepath = f"{TMPDIR}{filename}"
+            with open(filepath, "wb") as pfile:
+                pickle.dump(request.form, pfile, protocol=pickle.HIGHEST_PROTOCOL)
+                job_id = jobs.queue(
+                    rconn, {
+                        "command": [
+                            sys.executable, "-m", "scripts.corr_compute", filepath,
+                            g.user_session.user_id],
+                        "request_received_time": request_received.isoformat(),
+                        "status": "queued"
+                    })
+                jobs.run(job_id, REDIS_URL)
+
+            return redirect(url_for("corr_compute_page", job_id=str(job_id)))
+
+        job = jobs.job(
+            rconn, UUID(request.args.get("job_id"))).maybe(
+                {}, lambda the_job: the_job)
+
+        if jobs.completed_successfully(job):
+            output = json.loads(job.get("stdout", "{}"))
+            return render_template("correlation_page.html", **output)
+
+        if jobs.completed_erroneously(job):
+            output = json.loads(job.get("stdout", "{}"))
+            return render_template("correlation_error_page.html", error=output)
+
+        return render_template("loading_corrs.html")
 
 
 @app.route("/corr_matrix", methods=('POST',))
author	Frederick Muriuki Muriithi	2022-10-28 11:51:50 +0300
committer	Frederick Muriuki Muriithi	2022-10-28 15:58:01 +0300
commit	0bb49f9873e958cd480f7c9fc20f3a8db6f62003 (patch)
tree	1e13625ddb9b581ad16db98ab75619ef1fdacdce
parent	2357452ccdcf475bcad3a5da7de43f4d481e296b (diff)
download	genenetwork2-0bb49f9873e958cd480f7c9fc20f3a8db6f62003.tar.gz