Timestamp	Editor	Field	Diff
{{ timestamp_ }}	{{ author }}	{{ i.diff.field }}	{{ i.diff.diff }}

Timestamp	Editor	Field	Diff
{{ timestamp_ }}	{{ author }}	{{ i.diff.field }}	{{ i.diff.diff }}

Timestamp	Editor	Field	Diff
{{ timestamp_ }}	{{ author }}	{{ i.diff.field }}	{{ i.diff.diff }}

") +def display_file(name): + with open(os.path.join("/tmp/sample-data/diffs", name), 'r') as myfile: + content = myfile.read() + return Response(content, mimetype='text/json') -- cgit v1.2.3 From 9b68883041b581fbd01e5568d36d82581dec7772 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 26 Jul 2021 14:20:44 +0300 Subject: templates: edit_phenotype: Add enctype to form cause of file uploads --- wqflask/wqflask/templates/edit_phenotype.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 7d4c65f8..5edddada 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -53,7 +53,7 @@ Submit Trait | Reset {% endif %} -

Trait Information:

Pubmed ID: -- cgit v1.2.3 From 6eab9f07e5eaf75eaa4f406b1ed3a3cfec9575f8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 26 Jul 2021 14:22:32 +0300 Subject: templates: edit_phenotype: Add file upload and "approval" buttons --- wqflask/wqflask/templates/edit_phenotype.html | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 5edddada..691ad27b 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -206,11 +206,31 @@ Submit Trait | Reset

+ +

+ Sequence: +

+ + +

+ +

+ + Data Approval + +

+ +

+ + Download csv of sample data for editing +

- + -- cgit v1.2.3 From 4482a1c5dc61dc4813911f1439721d0407286e44 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Jul 2021 22:43:16 +0300 Subject: edit_phenotype: Remove duplicate html --- wqflask/wqflask/templates/edit_phenotype.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 691ad27b..9530fad0 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -208,11 +208,6 @@ Submit Trait | Reset

- Sequence: -

- - -

-- cgit v1.2.3 From 097f4168901741d7913e79da09c1f3907c3d1312 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Jul 2021 22:47:29 +0300 Subject: templates: edit_phenotype: add some form of padding upload button --- wqflask/wqflask/templates/edit_phenotype.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 9530fad0..7961c188 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -208,7 +208,7 @@ Submit Trait | Reset

- +

-- cgit v1.2.3 From 0793a41cc8534b5084bbf35b437d956681a8c19d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Jul 2021 23:08:50 +0300 Subject: templates: edit_phenotype: Use proper page header for this page --- wqflask/wqflask/templates/edit_phenotype.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 7961c188..0e7c4f63 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -2,8 +2,9 @@ {% block title %}Trait Submission{% endblock %} {% block content %} -Edit Trait for Published Database -Submit Trait | Reset +

Edit Trait for Published Database

{% if diff %} -- cgit v1.2.3 From 399d388b6704d39d778617cedebec31341428b75 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Jul 2021 23:09:34 +0300 Subject: templates: edit_phenotype: Properly centre form controls --- wqflask/wqflask/templates/edit_phenotype.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 0e7c4f63..9810b7ab 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -223,7 +223,7 @@ Download csv of sample data for editing

-- cgit v1.2.3 From f5677ef162fec18603fce6d903fcdfd31b3541cc Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Jul 2021 23:10:07 +0300 Subject: templates: edit_phenotype: Add some margin to links for csv download --- wqflask/wqflask/templates/edit_phenotype.html | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 9810b7ab..83daba58 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -212,15 +212,12 @@

- +

+ Data Approval -

- -

- - Download csv of sample data for editing + + Sample Data(CSV Download)

-- cgit v1.2.3 From cf278e6c119871d08f68c13e07b190b8c14e0414 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Jul 2021 23:12:14 +0300 Subject: wqflask: views: Replace `update_raw` with `update_sample_data` --- wqflask/wqflask/views.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 12b47ce9..4746fcc2 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -33,7 +33,6 @@ from gn3.db import fetchall from gn3.db import fetchone from gn3.db import insert from gn3.db import update -from gn3.db import update_raw from gn3.db.metadata_audit import MetadataAudit from gn3.db.phenotypes import Phenotype from gn3.db.phenotypes import Probeset @@ -41,6 +40,7 @@ from gn3.db.phenotypes import Publication from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import probeset_mapping from gn3.db.traits import get_trait_csv_sample_data +from gn3.db.traits import update_sample_data from flask import current_app @@ -1398,10 +1398,11 @@ def approve_data(name): host=current_app.config.get("DB_HOST")) with open(os.path.join("/tmp/sample-data/diffs", name), 'r') as myfile: sample_data = json.load(myfile) + PUBLISH_ID = sample_data.get("publishdata_id") modifications = [d for d in sample_data.get("Modifications")] for modification in modifications: - if modifications.get("current"): - (strain_id, publish_id, + if modification.get("Current"): + (strain_id, strain_name, value, se, count) = modification.get("Current").split(",") row_counts = update_sample_data( @@ -1419,11 +1420,10 @@ def approve_data(name): dataset_id=sample_data.get("publishdata_id"), editor=sample_data.get("author"), json_data=json.dumps(sample_data))) - - # Once data is approved, rename it! - os.rename(os.path.join("/tmp/sample-data/diffs", name), - os.path.join("/tmp/sample-data/diffs", - f"{name}.approved")) + # Once data is approved, rename it! + os.rename(os.path.join("/tmp/sample-data/diffs", name), + os.path.join("/tmp/sample-data/diffs", + f"{name}.approved")) return redirect("/data/approve/") -- cgit v1.2.3 From 52d53eb90aa0a4fbaab713704f4894e652cb91f3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Jul 2021 23:12:51 +0300 Subject: wqflask: views: Filter out approved/ rejected files during display * wqflask/wqflask/views.py (approve_data): Only show files that have not been approved/ rejected. --- wqflask/wqflask/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 4746fcc2..9a961f26 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1383,8 +1383,9 @@ def display_diffs_admin(): DIFF_DIR = "/tmp/sample-data/diffs" files = [] if os.path.exists(DIFF_DIR): + files = os.listdir(DIFF_DIR) files = filter(lambda x: not(x.endswith((".approved", ".rejected"))), - os.listdir(DIFF_DIR)) + files) return render_template("display_files.html", files=files) -- cgit v1.2.3 From 3804d64d6eed6c9b5fbb95758a84014ff026705d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Jul 2021 23:14:44 +0300 Subject: wqflask: views: Get the publishdata_id from the file --- wqflask/wqflask/views.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 9a961f26..0f1df123 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -572,11 +572,19 @@ def update_phenotype(): f"{trait_name}.{phenotype_id}." f"{current_time}.csv") file_.save(new_file_name) + publishdata_id = "" + lines = [] + with open(new_file_name, "r") as f: + lines = f.read() + first_line = lines.split('\n', 1)[0] + publishdata_id = first_line.split("Id:")[-1].strip() + with open(new_file_name, "w") as f: + f.write(lines.split("\n\n")[-1]) csv_ = get_trait_csv_sample_data(conn=conn, trait_name=str(trait_name), phenotype_id=str(phenotype_id)) with open(uploaded_file_name, "w") as f_: - f_.write(csv_) + f_.write(csv_.split("\n\n")[-1]) r = run_cmd(cmd=("/home/bonface/opt/genenetwork3/bin/csvdiff " f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) @@ -584,9 +592,9 @@ def update_phenotype(): f"{trait_name}." f"{phenotype_id}.{current_time}.json") with open(diff_output, "w") as f: - print(r.get("output")) dict_ = json.loads(r.get("output")) dict_.update({"author": author.decode('utf-8')}) + dict_.update({"publishdata_id": publishdata_id}) dict_.update({"timestamp": datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S")}) f.write(json.dumps(dict_)) -- cgit v1.2.3 From 305a2fc37bc233c9842bb12346c7e476c1d81be7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 30 Jul 2021 16:38:43 +0300 Subject: wqflask: views: Flash messages during edits --- wqflask/wqflask/templates/edit_phenotype.html | 9 +++++++++ wqflask/wqflask/views.py | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 83daba58..0d7071f7 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -2,6 +2,15 @@ {% block title %}Trait Submission{% endblock %} {% block content %} +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +

+{% endfor %} +{% endif %} +{% endwith %}

Edit Trait for Published Database

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 0f1df123..18d3e99f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -546,7 +546,7 @@ def update_phenotype(): data_ = request.form.to_dict() author = g.user_session.record.get(b'user_name') if 'file' not in request.files: - flash("No sample data has been uploaded") + flash("No sample-data has been uploaded", "warning") else: file_ = request.files['file'] trait_name = str(data_.get('dataset-name')) @@ -598,6 +598,7 @@ def update_phenotype(): dict_.update({"timestamp": datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S")}) f.write(json.dumps(dict_)) + flash("Sample-data has been successfully uploaded", "success") # Run updates: phenotype_ = { "pre_pub_description": data_.get("pre-pub-desc"), @@ -649,6 +650,7 @@ def update_phenotype(): data=MetadataAudit(dataset_id=data_.get("dataset-name"), editor=author.decode("utf-8"), json_data=json.dumps(diff_data))) + flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") return redirect(f"/trait/{data_.get('dataset-name')}" f"/edit/phenotype-id/{data_.get('phenotype-id')}") -- cgit v1.2.3 From d9e6c4d67aa5cb18cf199f5e27fbc1ac73906c89 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 17:49:44 +0300 Subject: templates: edit_phenotype: Remove link to the approve page --- wqflask/wqflask/templates/edit_phenotype.html | 3 --- 1 file changed, 3 deletions(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 0d7071f7..a7d6f1b0 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -222,9 +222,6 @@

- - Data Approval - Sample Data(CSV Download) -- cgit v1.2.3 From b5fc6068a1dbe493a3d7b240df236ada36984570 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 17:51:35 +0300 Subject: templates: edit_phenotype: Put csv "oownload" button above "Browse" --- wqflask/wqflask/templates/edit_phenotype.html | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index a7d6f1b0..7a841793 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -216,16 +216,15 @@

Sample Data(CSV Download)

+ +

-- cgit v1.2.3 From 299847f03a84c5a74c51d8cb6bea6763431773c2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 21:08:31 +0300 Subject: templates: display_files: Add a a reject button --- wqflask/wqflask/templates/display_files.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html index e7210c1b..30974300 100644 --- a/wqflask/wqflask/templates/display_files.html +++ b/wqflask/wqflask/templates/display_files.html @@ -7,7 +7,8 @@ Show files for approval

{{ file }}
{{ file }}
+

-- cgit v1.2.3 From f175c8806b4fbf372cbe65790ce7816b98e32695 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 21:24:10 +0300 Subject: wqflask: views: Add author name to generated diff's filename --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 18d3e99f..ed5398a2 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -589,7 +589,7 @@ def update_phenotype(): f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) diff_output = ("/tmp/sample-data/diffs/" - f"{trait_name}." + f"{trait_name}.{author.decode('utf-8')}." f"{phenotype_id}.{current_time}.json") with open(diff_output, "w") as f: dict_ = json.loads(r.get("output")) -- cgit v1.2.3 From 79d6dc5ac4610ea50e61bca80598b87d42911490 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 21:24:39 +0300 Subject: wqflask: views: Add dataset_id to diff json that'll be stored --- wqflask/wqflask/views.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index ed5398a2..11e1bfaa 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -593,10 +593,13 @@ def update_phenotype(): f"{phenotype_id}.{current_time}.json") with open(diff_output, "w") as f: dict_ = json.loads(r.get("output")) - dict_.update({"author": author.decode('utf-8')}) - dict_.update({"publishdata_id": publishdata_id}) - dict_.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) + dict_.update({ + "author": author.decode('utf-8'), + "publishdata_id": publishdata_id, + "dataset_id": data_.get("dataset-name"), + "timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S") + }) f.write(json.dumps(dict_)) flash("Sample-data has been successfully uploaded", "success") # Run updates: -- cgit v1.2.3 From 2c0b0d7132ff67709de13d986a92293b68e275e2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 21:25:30 +0300 Subject: wqflask: views: use dataset_id when inserting into metadata_audit --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 11e1bfaa..760d5947 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1431,7 +1431,7 @@ def approve_data(name): insert(conn, table="metadata_audit", data=MetadataAudit( - dataset_id=sample_data.get("publishdata_id"), + dataset_id=name.split(".")[0], # use the dataset name editor=sample_data.get("author"), json_data=json.dumps(sample_data))) # Once data is approved, rename it! -- cgit v1.2.3 From e200245b2a1666b7a16f4484891c7c2b143fefe9 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 21:44:30 +0300 Subject: wqflask: views: Rename endpoint to "/admin/data-sample/diffs/" --- wqflask/wqflask/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 760d5947..1a8536ac 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1391,7 +1391,7 @@ def get_sample_data_as_csv(trait_name: int, phenotype_id: int): ) -@app.route("/data/approve/") +@app.route("/admin/data-sample/diffs/") def display_diffs_admin(): DIFF_DIR = "/tmp/sample-data/diffs" files = [] @@ -1438,7 +1438,7 @@ def approve_data(name): os.rename(os.path.join("/tmp/sample-data/diffs", name), os.path.join("/tmp/sample-data/diffs", f"{name}.approved")) - return redirect("/data/approve/") + return redirect("/admin/data-sample/diffs/") @app.route("/display-file/") -- cgit v1.2.3 From 4594743baf6e6fd4b50f68665d009084f605bc78 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 21:45:17 +0300 Subject: wqflask: views: After approval, rename the file at the right point --- wqflask/wqflask/views.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 1a8536ac..c7582782 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1434,10 +1434,11 @@ def approve_data(name): dataset_id=name.split(".")[0], # use the dataset name editor=sample_data.get("author"), json_data=json.dumps(sample_data))) - # Once data is approved, rename it! - os.rename(os.path.join("/tmp/sample-data/diffs", name), - os.path.join("/tmp/sample-data/diffs", - f"{name}.approved")) + if modifications: + # Once data is approved, rename it! + os.rename(os.path.join("/tmp/sample-data/diffs", name), + os.path.join("/tmp/sample-data/diffs", + f"{name}.approved")) return redirect("/admin/data-sample/diffs/") -- cgit v1.2.3 From 366c9c433a0b18db25f5cb28d7c7dc1ed17b3c67 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 21:46:07 +0300 Subject: Display a flash message after a file(and data) approval squash! Display a flash message after a file(and data) approval --- wqflask/wqflask/templates/display_files.html | 9 +++++++++ wqflask/wqflask/views.py | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html index 30974300..f8bc188f 100644 --- a/wqflask/wqflask/templates/display_files.html +++ b/wqflask/wqflask/templates/display_files.html @@ -2,6 +2,15 @@ {% block title %}Trait Submission{% endblock %} {% block content %} +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +

+{% endfor %} +{% endif %} +{% endwith %} Show files for approval

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index c7582782..11c8b347 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1414,12 +1414,13 @@ def approve_data(name): sample_data = json.load(myfile) PUBLISH_ID = sample_data.get("publishdata_id") modifications = [d for d in sample_data.get("Modifications")] + row_counts = len(modifications) for modification in modifications: if modification.get("Current"): (strain_id, strain_name, value, se, count) = modification.get("Current").split(",") - row_counts = update_sample_data( + update_sample_data( conn=conn, strain_name=strain_name, strain_id=int(strain_id), @@ -1439,6 +1440,9 @@ def approve_data(name): os.rename(os.path.join("/tmp/sample-data/diffs", name), os.path.join("/tmp/sample-data/diffs", f"{name}.approved")) + flash((f"Just updated data from: {name}; {row_counts} " + "row(s) modified!"), + "success") return redirect("/admin/data-sample/diffs/") -- cgit v1.2.3 From 3241306917277278a218270a6691d103df6fee4e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 21:57:41 +0300 Subject: Reject diff data on click --- wqflask/wqflask/templates/display_files.html | 3 ++- wqflask/wqflask/views.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html index f8bc188f..4b4babc4 100644 --- a/wqflask/wqflask/templates/display_files.html +++ b/wqflask/wqflask/templates/display_files.html @@ -17,7 +17,8 @@ Show files for approval

{{ file }}
-

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 11c8b347..69b432f5 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1446,6 +1446,15 @@ def approve_data(name): return redirect("/admin/data-sample/diffs/") +@app.route("/data-samples/reject/") +def reject_data(name): + os.rename(os.path.join("/tmp/sample-data/diffs", name), + os.path.join("/tmp/sample-data/diffs", + f"{name}.rejected")) + flash(f"{name} has been rejected!", "success") + return redirect("/admin/data-sample/diffs/") + + @app.route("/display-file/") def display_file(name): with open(os.path.join("/tmp/sample-data/diffs", name), 'r') as myfile: -- cgit v1.2.3 From fbe52a11f4471d928e398f09edc2e243bbbab58a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 22:07:06 +0300 Subject: Rename display_files.html -> display_files_admin.html --- wqflask/wqflask/templates/display_files.html | 32 ---------------------- wqflask/wqflask/templates/display_files_admin.html | 32 ++++++++++++++++++++++ wqflask/wqflask/views.py | 3 +- 3 files changed, 34 insertions(+), 33 deletions(-) delete mode 100644 wqflask/wqflask/templates/display_files.html create mode 100644 wqflask/wqflask/templates/display_files_admin.html diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html deleted file mode 100644 index 4b4babc4..00000000 --- a/wqflask/wqflask/templates/display_files.html +++ /dev/null @@ -1,32 +0,0 @@ -{% extends "base.html" %} -{% block title %}Trait Submission{% endblock %} -{% block content %} - -{% with messages = get_flashed_messages(with_categories=true) %} -{% if messages %} -{% for category, message in messages %} -

-{% endfor %} -{% endif %} -{% endwith %} -Show files for approval - -

{{ file }}
- -

-{%endblock%} - -{% block js %} - -{% endblock %} diff --git a/wqflask/wqflask/templates/display_files_admin.html b/wqflask/wqflask/templates/display_files_admin.html new file mode 100644 index 00000000..4b4babc4 --- /dev/null +++ b/wqflask/wqflask/templates/display_files_admin.html @@ -0,0 +1,32 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} + +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +

+{% endfor %} +{% endif %} +{% endwith %} +Show files for approval + +

{{ file }}
+ +

+{%endblock%} + +{% block js %} + +{% endblock %} diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 69b432f5..56f9a5eb 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1399,7 +1399,8 @@ def display_diffs_admin(): files = os.listdir(DIFF_DIR) files = filter(lambda x: not(x.endswith((".approved", ".rejected"))), files) - return render_template("display_files.html", + return render_template("display_files_admin.html", + files=files) files=files) -- cgit v1.2.3 From 72bdbc512ee250ffaa1fa0c1ba91b2a065a485d0 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 22:08:25 +0300 Subject: wqflask: views: Only show user's diff --- wqflask/wqflask/templates/display_files_user.html | 31 +++++++++++++++++++++++ wqflask/wqflask/views.py | 14 ++++++++++ 2 files changed, 45 insertions(+) create mode 100644 wqflask/wqflask/templates/display_files_user.html diff --git a/wqflask/wqflask/templates/display_files_user.html b/wqflask/wqflask/templates/display_files_user.html new file mode 100644 index 00000000..b6bab709 --- /dev/null +++ b/wqflask/wqflask/templates/display_files_user.html @@ -0,0 +1,31 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} + +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +

+{% endfor %} +{% endif %} +{% endwith %} +Show files for approval + +

{{ file }}
+

+{%endblock%} + +{% block js %} + +{% endblock %} diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 56f9a5eb..78da024f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1392,6 +1392,7 @@ def get_sample_data_as_csv(trait_name: int, phenotype_id: int): @app.route("/admin/data-sample/diffs/") +@admin_login_required def display_diffs_admin(): DIFF_DIR = "/tmp/sample-data/diffs" files = [] @@ -1401,6 +1402,19 @@ def display_diffs_admin(): files) return render_template("display_files_admin.html", files=files) + + +@app.route("/user/data-sample/diffs/") +def display_diffs_users(): + DIFF_DIR = "/tmp/sample-data/diffs" + files = [] + author = g.user_session.record.get(b'user_name').decode("utf-8") + if os.path.exists(DIFF_DIR): + files = os.listdir(DIFF_DIR) + files = filter(lambda x: not(x.endswith((".approved", ".rejected"))) \ + and author in x, + files) + return render_template("display_files_user.html", files=files) -- cgit v1.2.3 From cbe5d6edacafe7f1afba49e20c644a34fa9aaf7d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 22:19:57 +0300 Subject: Replace TMPDIR hard-coded val with one set from flask --- wqflask/wqflask/views.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 78da024f..2f704ac9 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -544,6 +544,7 @@ def update_phenotype(): passwd=current_app.config.get("DB_PASS"), host=current_app.config.get("DB_HOST")) data_ = request.form.to_dict() + TMPDIR = current_app.config.get("TMPDIR") author = g.user_session.record.get(b'user_name') if 'file' not in request.files: flash("No sample-data has been uploaded", "warning") @@ -551,7 +552,7 @@ def update_phenotype(): file_ = request.files['file'] trait_name = str(data_.get('dataset-name')) phenotype_id = str(data_.get('phenotype-id', 35)) - SAMPLE_DATADIR = "/tmp/sample-data/" + SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") if not os.path.exists(SAMPLE_DATADIR): os.makedirs(SAMPLE_DATADIR) if not os.path.exists(os.path.join(SAMPLE_DATADIR, @@ -563,14 +564,17 @@ def update_phenotype(): os.makedirs(os.path.join(SAMPLE_DATADIR, "updated")) current_time = str(datetime.datetime.now().isoformat()) - new_file_name = ("/tmp/sample-data/updated/" - f"{author.decode('utf-8')}." - f"{trait_name}.{phenotype_id}." - f"{current_time}.csv") - uploaded_file_name = ("/tmp/sample-data/updated/" - f"updated.{author.decode('utf-8')}." - f"{trait_name}.{phenotype_id}." - f"{current_time}.csv") + new_file_name = (os.path.join(TMPDIR, + "sample-data/updated/", + (f"{author.decode('utf-8')}." + f"{trait_name}.{phenotype_id}." + f"{current_time}.csv"))) + uploaded_file_name = (os.path.join( + TMPDIR, + "sample-data/updated/", + (f"updated.{author.decode('utf-8')}." + f"{trait_name}.{phenotype_id}." + f"{current_time}.csv"))) file_.save(new_file_name) publishdata_id = "" lines = [] @@ -1425,7 +1429,9 @@ def approve_data(name): user=current_app.config.get("DB_USER"), passwd=current_app.config.get("DB_PASS"), host=current_app.config.get("DB_HOST")) - with open(os.path.join("/tmp/sample-data/diffs", name), 'r') as myfile: + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + name), 'r') as myfile: sample_data = json.load(myfile) PUBLISH_ID = sample_data.get("publishdata_id") modifications = [d for d in sample_data.get("Modifications")] @@ -1452,8 +1458,8 @@ def approve_data(name): json_data=json.dumps(sample_data))) if modifications: # Once data is approved, rename it! - os.rename(os.path.join("/tmp/sample-data/diffs", name), - os.path.join("/tmp/sample-data/diffs", + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), + os.path.join(f"{TMPDIR}/sample-data/diffs", f"{name}.approved")) flash((f"Just updated data from: {name}; {row_counts} " "row(s) modified!"), @@ -1463,8 +1469,9 @@ def approve_data(name): @app.route("/data-samples/reject/") def reject_data(name): - os.rename(os.path.join("/tmp/sample-data/diffs", name), - os.path.join("/tmp/sample-data/diffs", + TMPDIR = current_app.config.get("TMPDIR") + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), + os.path.join(f"{TMPDIR}/sample-data/diffs", f"{name}.rejected")) flash(f"{name} has been rejected!", "success") return redirect("/admin/data-sample/diffs/") @@ -1472,6 +1479,8 @@ def reject_data(name): @app.route("/display-file/") def display_file(name): - with open(os.path.join("/tmp/sample-data/diffs", name), 'r') as myfile: + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + name), 'r') as myfile: content = myfile.read() return Response(content, mimetype='text/json') -- cgit v1.2.3 From 759b04823818cd09f5e8e78299f270b3f7b73042 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 2 Aug 2021 22:28:00 +0300 Subject: Remove hard coded PATH for csvdiff binary --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 2f704ac9..a40e0d44 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -589,7 +589,7 @@ def update_phenotype(): phenotype_id=str(phenotype_id)) with open(uploaded_file_name, "w") as f_: f_.write(csv_.split("\n\n")[-1]) - r = run_cmd(cmd=("/home/bonface/opt/genenetwork3/bin/csvdiff " + r = run_cmd(cmd=("csvdiff " f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) diff_output = ("/tmp/sample-data/diffs/" -- cgit v1.2.3 From a24ea03594d6879adf92cd4db2b52108ddb352bd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 4 Aug 2021 14:51:20 +0300 Subject: wqflask: views: Replace hard-coded "/tmp" dir --- wqflask/wqflask/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index a40e0d44..fe81bb6c 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -592,7 +592,7 @@ def update_phenotype(): r = run_cmd(cmd=("csvdiff " f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) - diff_output = ("/tmp/sample-data/diffs/" + diff_output = (f"{TMPDIR}/sample-data/diffs/" f"{trait_name}.{author.decode('utf-8')}." f"{phenotype_id}.{current_time}.json") with open(diff_output, "w") as f: @@ -1398,7 +1398,8 @@ def get_sample_data_as_csv(trait_name: int, phenotype_id: int): @app.route("/admin/data-sample/diffs/") @admin_login_required def display_diffs_admin(): - DIFF_DIR = "/tmp/sample-data/diffs" + TMPDIR = current_app.config.get("TMPDIR") + DIFF_DIR = f"{TMPDIR}/sample-data/diffs" files = [] if os.path.exists(DIFF_DIR): files = os.listdir(DIFF_DIR) @@ -1410,7 +1411,8 @@ def display_diffs_admin(): @app.route("/user/data-sample/diffs/") def display_diffs_users(): - DIFF_DIR = "/tmp/sample-data/diffs" + TMPDIR = current_app.config.get("TMPDIR") + DIFF_DIR = f"{TMPDIR}/sample-data/diffs" files = [] author = g.user_session.record.get(b'user_name').decode("utf-8") if os.path.exists(DIFF_DIR): -- cgit v1.2.3 From 457296f004e576a632d2e77c656d41d9ceed0c47 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 4 Aug 2021 16:28:34 +0300 Subject: wqflask: views: Remove unnecessary casting for "count" Sometimes, the "count" value can be an "x", as opposed to an int. With the cast, an un-handled error could be thrown. --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index fe81bb6c..76fe029f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1450,7 +1450,7 @@ def approve_data(name): publish_data_id=int(PUBLISH_ID), value=value, error=se, - count=int(count) + count=count ) insert(conn, table="metadata_audit", -- cgit v1.2.3 From b8c88668dbf1bc96c2faf14c483e3af35babc3e5 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 5 Aug 2021 20:51:46 +0000 Subject: Generate phenotype and covariate filenames for GEMMA from a hash of values and dataset name - previously the covariate filename was static and the phenotype filename was just random --- wqflask/wqflask/marker_regression/gemma_mapping.py | 35 ++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index f88c5ac8..efd8bba8 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -11,6 +11,7 @@ from utility.tools import flat_files from utility.tools import GEMMA_WRAPPER_COMMAND from utility.tools import TEMPDIR from utility.tools import WEBSERVER_MODE +from gn3.computations.gemma import generate_hash_of_string import utility.logger logger = utility.logger.getLogger(__name__) @@ -34,10 +35,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, genofile_name = this_dataset.group.name if first_run: - trait_filename = (f"{str(this_dataset.group.name)}_" - f"{str(this_trait.name)}_" - f"{generate_random_n_string(6)}") - gen_pheno_txt_file(this_dataset, genofile_name, vals, trait_filename) + pheno_filename = gen_pheno_txt_file(this_dataset, genofile_name, vals) if not os.path.isfile(f"{webqtlConfig.GENERATED_IMAGE_DIR}" f"{genofile_name}_output.assoc.txt"): @@ -56,13 +54,13 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, chr_list_string = ",".join(this_chromosomes_name) if covariates != "": - gen_covariates_file(this_dataset, covariates, samples) + covar_filename = gen_covariates_file(this_dataset, covariates, samples) if use_loco == "True": generate_k_command = (f"{GEMMA_WRAPPER_COMMAND} --json --loco " f"{chr_list_string} -- {GEMMAOPTS} " f"-g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt -a " + f"{TEMPDIR}/gn2/{pheno_filename}.txt -a " f"{flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt -gk > " f"{TEMPDIR}/gn2/{k_output_filename}.json") @@ -73,10 +71,10 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"-- {GEMMAOPTS} " f"-g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt " - f"-p {TEMPDIR}/gn2/{trait_filename}.txt ") + f"-p {TEMPDIR}/gn2/{pheno_filename}.txt ") if covariates != "": gemma_command += (f"-c {flat_files('mapping')}/" - f"{this_dataset.group.name}_covariates.txt " + f"{covar_filename}.txt " f"-a {flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt " f"-lmm 9 -maf {maf} > {TEMPDIR}/gn2/" @@ -92,7 +90,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"{GEMMAOPTS} " f" -g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt -a " + f"{TEMPDIR}/gn2/{pheno_filename}.txt -a " f"{flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt -gk > " f"{TEMPDIR}/gn2/{k_output_filename}.json") @@ -106,12 +104,11 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"{genofile_name}_snps.txt " f"-lmm 9 -g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt ") + f"{TEMPDIR}/gn2/{pheno_filename}.txt ") if covariates != "": gemma_command += (f" -c {flat_files('mapping')}/" - f"{this_dataset.group.name}" - f"_covariates.txt > " + f"{covar_filename}.txt > " f"{TEMPDIR}/gn2/{gwa_output_filename}.json") else: gemma_command += f" > {TEMPDIR}/gn2/{gwa_output_filename}.json" @@ -129,16 +126,20 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, return marker_obs, gwa_output_filename -def gen_pheno_txt_file(this_dataset, genofile_name, vals, trait_filename): +def gen_pheno_txt_file(this_dataset, genofile_name, vals): """Generates phenotype file for GEMMA""" - with open(f"{TEMPDIR}/gn2/{trait_filename}.txt", "w") as outfile: + filename = "PHENO_" + generate_hash_of_string(this_dataset.name + str(vals)) + + with open(f"{TEMPDIR}/gn2/{filename}.txt", "w") as outfile: for value in vals: if value == "x": outfile.write("NA\n") else: outfile.write(value + "\n") + return filename + def gen_covariates_file(this_dataset, covariates, samples): covariate_list = covariates.split(",") @@ -168,14 +169,18 @@ def gen_covariates_file(this_dataset, covariates, samples): this_covariate_data.append("-9") covariate_data_object.append(this_covariate_data) + filename = "COVAR_" + generate_hash_of_string(this_dataset.name + str(covariate_data_object)) + with open((f"{flat_files('mapping')}/" - f"{this_dataset.group.name}_covariates.txt"), + f"{filename}.txt"), "w") as outfile: for i in range(len(covariate_data_object[0])): for this_covariate in covariate_data_object: outfile.write(str(this_covariate[i]) + "\t") outfile.write("\n") + return filename + def parse_loco_output(this_dataset, gwa_output_filename, loco="True"): -- cgit v1.2.3 From d29dfb72ceb005ab045203a36b1fc1552544b1e2 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 6 Aug 2021 12:56:07 +0200 Subject: gunicorn production settings: increase process refresh rate to clean up RAM. Also use the jitter switch. --- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5f4e0f9a..2b94b2a2 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -209,7 +209,7 @@ if [ "$1" = '-gunicorn-prod' ] ; then echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi PID=$TMPDIR/gunicorn.$USER.pid - cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 1000 --timeout 1200 wsgi" + cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 100 --max-requests-jitter 30 --timeout 1200 wsgi" echo RUNNING gunicorn $cmd gunicorn $cmd exit $? -- cgit v1.2.3 From db0bdf52156b4f75402a7a92584d162aa5d9b151 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 9 Aug 2021 18:34:31 +0000 Subject: Removed some paths that have been throwing errors in the logs constantly --- wqflask/wqflask/static/new/css/bootstrap-custom.css | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wqflask/wqflask/static/new/css/bootstrap-custom.css b/wqflask/wqflask/static/new/css/bootstrap-custom.css index 7c8549e1..a0d3ff6a 100644 --- a/wqflask/wqflask/static/new/css/bootstrap-custom.css +++ b/wqflask/wqflask/static/new/css/bootstrap-custom.css @@ -327,7 +327,7 @@ th { font-family: 'Glyphicons Halflings'; src: url('../fonts/glyphicons-halflings-regular.eot'); - src: url('../fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('../fonts/glyphicons-halflings-regular.woff') format('woff'), url('../fonts/glyphicons-halflings-regular.ttf') format('truetype'), url('../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg'); + src: url('../fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg'); } .glyphicon { @@ -7554,5 +7554,3 @@ button.close { display: none !important; } } - -/*# sourceMappingURL=bootstrap.css.map */ \ No newline at end of file -- cgit v1.2.3 From 162851e8f92ed6d01c25d4f6034bcac4986e8094 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 10 Aug 2021 19:01:47 +0000 Subject: Replaced forward slashes with underscores in the hashed filenames used by GEMMA, since the forward slashes make the paths not work properly --- wqflask/wqflask/marker_regression/gemma_mapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index efd8bba8..623ab87f 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -129,7 +129,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, def gen_pheno_txt_file(this_dataset, genofile_name, vals): """Generates phenotype file for GEMMA""" - filename = "PHENO_" + generate_hash_of_string(this_dataset.name + str(vals)) + filename = "PHENO_" + generate_hash_of_string(this_dataset.name + str(vals)).replace("/", "_") with open(f"{TEMPDIR}/gn2/{filename}.txt", "w") as outfile: for value in vals: @@ -169,7 +169,7 @@ def gen_covariates_file(this_dataset, covariates, samples): this_covariate_data.append("-9") covariate_data_object.append(this_covariate_data) - filename = "COVAR_" + generate_hash_of_string(this_dataset.name + str(covariate_data_object)) + filename = "COVAR_" + generate_hash_of_string(this_dataset.name + str(covariate_data_object)).replace("/", "_") with open((f"{flat_files('mapping')}/" f"{filename}.txt"), -- cgit v1.2.3 From 10ef0e532324e838c7be5a0f7f451dcaec9c2ce8 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 10 Aug 2021 20:19:27 +0000 Subject: Fixed issue that caused correlations to not work with temp traits after switching to using the GN3 code --- wqflask/wqflask/correlation/correlation_gn3_api.py | 7 +++++-- wqflask/wqflask/correlation/show_corr_results.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index c96da8ee..f7c06a46 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -18,7 +18,10 @@ from gn3.db_utils import database_connector def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" - this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) + if start_vars['dataset'] == "Temp": + this_dataset = data_set.create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + else: + this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( dataset_name=start_vars['corr_dataset']) this_trait = create_trait(dataset=this_dataset, @@ -306,4 +309,4 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): "trait_symbol_dict": trait_symbol_dict, "symbol_tissue_vals_dict": corr_result_tissue_vals_dict } - return (primary_tissue_data, target_tissue_data) \ No newline at end of file + return (primary_tissue_data, target_tissue_data) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index e2daa991..d73965da 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -30,7 +30,10 @@ def set_template_vars(start_vars, correlation_data): corr_type = start_vars['corr_type'] corr_method = start_vars['corr_sample_method'] - this_dataset_ob = create_dataset(dataset_name=start_vars['dataset']) + if start_vars['dataset'] == "Temp": + this_dataset_ob = create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + else: + this_dataset_ob = create_dataset(dataset_name=start_vars['dataset']) this_trait = create_trait(dataset=this_dataset_ob, name=start_vars['trait_id']) -- cgit v1.2.3 From 2dce5e20b5dcfa032a32a486bb4e4a4390c5f2b7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 10 Aug 2021 20:44:13 +0000 Subject: Fixed the way n_samples is calculated for the loading page and mapping figure; previously it would includes parents/f1s --- wqflask/wqflask/views.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 76fe029f..44560427 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1033,7 +1033,7 @@ def loading_page(): start_vars['dataset'], group_name=start_vars['group']) else: dataset = create_dataset(start_vars['dataset']) - samples = start_vars['primary_samples'].split(",") + samples = dataset.group.samplelist if 'genofile' in start_vars: if start_vars['genofile'] != "": genofile_string = start_vars['genofile'] @@ -1178,9 +1178,8 @@ def mapping_results_page(): gn1_template_vars = display_mapping_results.DisplayMappingResults( result).__dict__ - with Bench("Rendering template"): - rendered_template = render_template( - "mapping_results.html", **gn1_template_vars) + rendered_template = render_template( + "mapping_results.html", **gn1_template_vars) return rendered_template -- cgit v1.2.3 From 9a1e7270bc9c2130a4cb9405f2de24053dea6582 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 11 Aug 2021 20:26:45 +0000 Subject: Sort by attribute ID instead of name in initialize_show_trait_tables.js when defining attribute columns --- wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 6ca92fb6..49311f87 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -93,7 +93,7 @@ build_columns = function() { ); } - attr_keys = Object.keys(js_data.attributes).sort((a, b) => (js_data.attributes[a].name.toLowerCase() > js_data.attributes[b].name.toLowerCase()) ? 1 : -1) + attr_keys = Object.keys(js_data.attributes).sort((a, b) => (js_data.attributes[a].id > js_data.attributes[b].id) ? 1 : -1) for (i = 0; i < attr_keys.length; i++){ column_list.push( { @@ -101,7 +101,7 @@ build_columns = function() { 'type': "natural", 'data': null, 'render': function(data, type, row, meta) { - attr_name = Object.keys(data.extra_attributes).sort()[meta.col - data.first_attr_col] + attr_name = Object.keys(data.extra_attributes).sort((a, b) => (parseInt(a) > parseInt(b)) ? 1 : -1)[meta.col - data.first_attr_col] if (attr_name != null && attr_name != undefined){ if (Array.isArray(data.extra_attributes[attr_name])){ -- cgit v1.2.3 From 67bab7f9cd12ceb7ea0b0ac4b0b98c24935f3ab5 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 11 Aug 2021 20:28:35 +0000 Subject: Fixed issue that caused case attribute columns to become unaligned if some strains didn't have values for all case attributes + changed ordering to use attribute ID instead of name --- wqflask/wqflask/show_trait/SampleList.py | 44 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 92cea550..e4bbe5b9 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -32,7 +32,7 @@ class SampleList: for counter, sample_name in enumerate(sample_names, 1): sample_name = sample_name.replace("_2nd_", "") - # ZS: self.this_trait will be a list if it is a Temp trait + # self.this_trait will be a list if it is a Temp trait if isinstance(self.this_trait, list): sample = webqtlCaseData.webqtlCaseData(name=sample_name) if counter <= len(self.this_trait): @@ -47,7 +47,7 @@ class SampleList: name=sample_name, value=float(self.this_trait[counter - 1])) else: - # ZS - If there's no value for the sample/strain, + # If there's no value for the sample/strain, # create the sample object (so samples with no value # are still displayed in the table) try: @@ -63,29 +63,29 @@ class SampleList: sample.this_id = str(counter) - # ZS: For extra attribute columns; currently only used by + # For extra attribute columns; currently only used by # several datasets if self.sample_attribute_values: sample.extra_attributes = self.sample_attribute_values.get( sample_name, {}) - # ZS: Add a url so RRID case attributes can be displayed as links - if 'rrid' in sample.extra_attributes: + # Add a url so RRID case attributes can be displayed as links + if '36' in sample.extra_attributes: if self.dataset.group.species == "mouse": - if len(sample.extra_attributes['rrid'].split(":")) > 1: - the_rrid = sample.extra_attributes['rrid'].split(":")[ + if len(sample.extra_attributes['36'].split(":")) > 1: + the_rrid = sample.extra_attributes['36'].split(":")[ 1] - sample.extra_attributes['rrid'] = [ - sample.extra_attributes['rrid']] - sample.extra_attributes['rrid'].append( + sample.extra_attributes['36'] = [ + sample.extra_attributes['36']] + sample.extra_attributes['36'].append( webqtlConfig.RRID_MOUSE_URL % the_rrid) elif self.dataset.group.species == "rat": - if len(str(sample.extra_attributes['rrid'])): - the_rrid = sample.extra_attributes['rrid'].split("_")[ + if len(str(sample.extra_attributes['36'])): + the_rrid = sample.extra_attributes['36'].split("_")[ 1] - sample.extra_attributes['rrid'] = [ - sample.extra_attributes['rrid']] - sample.extra_attributes['rrid'].append( + sample.extra_attributes['36'] = [ + sample.extra_attributes['36']] + sample.extra_attributes['36'].append( webqtlConfig.RRID_RAT_URL % the_rrid) self.sample_list.append(sample) @@ -128,12 +128,13 @@ class SampleList: FROM CaseAttribute, CaseAttributeXRefNew WHERE CaseAttributeXRefNew.CaseAttributeId = CaseAttribute.Id AND CaseAttributeXRefNew.InbredSetId = %s - ORDER BY lower(CaseAttribute.Name)''', (str(self.dataset.group.id),)) + ORDER BY CaseAttribute.Id''', (str(self.dataset.group.id),)) self.attributes = {} for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name)): key, name = attr self.attributes[key] = Bunch() + self.attributes[key].id = key self.attributes[key].name = name self.attributes[key].distinct_values = [ item.Value for item in values] @@ -168,10 +169,13 @@ class SampleList: for sample_name, items in itertools.groupby(results.fetchall(), lambda row: row.SampleName): attribute_values = {} + # Make a list of attr IDs without values (that have values for other samples) + valueless_attr_ids = [self.attributes[key].id for key in self.attributes.keys()] for item in items: + valueless_attr_ids.remove(item.Id) attribute_value = item.Value - # ZS: If it's an int, turn it into one for sorting + # If it's an int, turn it into one for sorting # (for example, 101 would be lower than 80 if # they're strings instead of ints) try: @@ -179,8 +183,10 @@ class SampleList: except ValueError: pass - attribute_values[self.attributes[item.Id].name.lower( - )] = attribute_value + attribute_values[str(item.Id)] = attribute_value + for attr_id in valueless_attr_ids: + attribute_values[str(attr_id)] = "" + self.sample_attribute_values[sample_name] = attribute_values def get_first_attr_col(self): -- cgit v1.2.3 From eecaad2dac6efa885540287a7ef5a273c6d18be2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 21:55:26 +0000 Subject: Get case attribute descriptions from DB and display them as mouseover titles in the sample table headers for the trait page --- wqflask/wqflask/show_trait/SampleList.py | 7 ++++--- .../wqflask/static/new/javascript/initialize_show_trait_tables.js | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index e4bbe5b9..ae30aa59 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -124,18 +124,19 @@ class SampleList: # Get attribute names and distinct values for each attribute results = g.db.execute(''' - SELECT DISTINCT CaseAttribute.Id, CaseAttribute.Name, CaseAttributeXRefNew.Value + SELECT DISTINCT CaseAttribute.Id, CaseAttribute.Name, CaseAttribute.Description, CaseAttributeXRefNew.Value FROM CaseAttribute, CaseAttributeXRefNew WHERE CaseAttributeXRefNew.CaseAttributeId = CaseAttribute.Id AND CaseAttributeXRefNew.InbredSetId = %s ORDER BY CaseAttribute.Id''', (str(self.dataset.group.id),)) self.attributes = {} - for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name)): - key, name = attr + for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name, row.Description)): + key, name, description = attr self.attributes[key] = Bunch() self.attributes[key].id = key self.attributes[key].name = name + self.attributes[key].description = description self.attributes[key].distinct_values = [ item.Value for item in values] self.attributes[key].distinct_values = natural_sort( diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 49311f87..4de1b0ac 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -97,7 +97,7 @@ build_columns = function() { for (i = 0; i < attr_keys.length; i++){ column_list.push( { - 'title': "

" + js_data.attributes[attr_keys[i]].name + "

", + 'title': "

" + js_data.attributes[attr_keys[i]].name + "

", 'type': "natural", 'data': null, 'render': function(data, type, row, meta) { -- cgit v1.2.3 From f4fbb6d53419a19c6ee67977d18605cdcbb09c0e Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 22:35:53 +0000 Subject: add function for reading in JSON file that lists sample lists unique to each study within a group (in this case only BXD Longevity for now) --- wqflask/base/data_set.py | 9 +++++++++ wqflask/wqflask/show_trait/show_trait.py | 1 + 2 files changed, 10 insertions(+) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 4cb82665..b8f2f9fb 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -398,6 +398,15 @@ class DatasetGroup: if maternal and paternal: self.parlist = [maternal, paternal] + def get_study_samplelists(self): + study_sample_file = "%s/study_sample_lists/%s.json" % (webqtlConfig.GENODIR, self.name) + try: + f = open(study_sample_file) + except: + return None + study_samples = json.load(f) + return study_samples + def get_genofiles(self): jsonfile = "%s/%s.json" % (webqtlConfig.GENODIR, self.name) try: diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index c07430dd..d3356bc3 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -192,6 +192,7 @@ class ShowTrait: [self.dataset.species.chromosomes.chromosomes[this_chr].name, i]) self.genofiles = self.dataset.group.get_genofiles() + self.study_samplelists = self.dataset.group.get_study_samplelists() # ZS: No need to grab scales from .geno file unless it's using # a mapping method that reads .geno files -- cgit v1.2.3 From 308860f05b87e5fc3899230b7312be9543e6c299 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 23:16:02 +0000 Subject: Add option to filter samples by study to template --- .../templates/show_trait_transform_and_filter.html | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/wqflask/wqflask/templates/show_trait_transform_and_filter.html b/wqflask/wqflask/templates/show_trait_transform_and_filter.html index 20f78b48..56c3c30e 100644 --- a/wqflask/wqflask/templates/show_trait_transform_and_filter.html +++ b/wqflask/wqflask/templates/show_trait_transform_and_filter.html @@ -45,6 +45,25 @@

{% endif %} + {% if study_samplelists|length > 0 %} +

+ Filter samples by study: + + + +

+ {% endif %}

Filter samples by {% if (numerical_var_list|length == 0) and (not js_data.se_exists) %}value{% endif %} {% if (numerical_var_list|length > 0) or js_data.se_exists %} -- cgit v1.2.3 From 98683bd5cc809aa03e0bd58a67733498b4f56a9d Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 23:17:04 +0000 Subject: Fix the way the study_sample_lists path is set and checked --- wqflask/base/data_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index b8f2f9fb..1042e1bd 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -399,7 +399,7 @@ class DatasetGroup: self.parlist = [maternal, paternal] def get_study_samplelists(self): - study_sample_file = "%s/study_sample_lists/%s.json" % (webqtlConfig.GENODIR, self.name) + study_sample_file = locate_ignore_error(self.name + ".json", 'study_sample_lists') try: f = open(study_sample_file) except: -- cgit v1.2.3 From 810b2ace0a9cb2511cf0ef6f0c01f70a0ce11915 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 23:19:38 +0000 Subject: Return empty list instead of None in get_study_samplelists --- wqflask/base/data_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 1042e1bd..0ea61faa 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -403,7 +403,7 @@ class DatasetGroup: try: f = open(study_sample_file) except: - return None + return [] study_samples = json.load(f) return study_samples -- cgit v1.2.3 From 11d5e9ffbf48c6a2fa7a59f10a9d7fe44cee0c23 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 23:20:28 +0000 Subject: Get list of study titles to use for the dropdown menu and store the full list of study samplelists as a hddn input (don't like this, but no easy alternative right now) --- wqflask/wqflask/show_trait/show_trait.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index d3356bc3..bc97d417 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -192,7 +192,8 @@ class ShowTrait: [self.dataset.species.chromosomes.chromosomes[this_chr].name, i]) self.genofiles = self.dataset.group.get_genofiles() - self.study_samplelists = self.dataset.group.get_study_samplelists() + study_samplelist_json = self.dataset.group.get_study_samplelists() + self.study_samplelists = [study["title"] for study in study_samplelist_json] # ZS: No need to grab scales from .geno file unless it's using # a mapping method that reads .geno files @@ -281,6 +282,7 @@ class ShowTrait: hddn['selected_chr'] = -1 hddn['mapping_display_all'] = True hddn['suggestive'] = 0 + hddn['study_samplelists'] = json.dumps(study_samplelist_json) hddn['num_perm'] = 0 hddn['categorical_vars'] = "" if categorical_var_list: -- cgit v1.2.3 From 88c9f4f9bd2adca23052fc3f5c5b965accf6722d Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 23:20:52 +0000 Subject: Add JS for filtering sample table rows by study samplelist --- wqflask/wqflask/static/new/javascript/show_trait.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 77ef1720..3957175b 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -713,10 +713,24 @@ block_by_index = function() { for (_k = 0, _len1 = index_list.length; _k < _len1; _k++) { index = index_list[_k]; val_nodes[index - 1].childNodes[0].value = "x"; - } }; +filter_by_study = function() { + let this_study = $('#filter_study').val(); + let block_group = $('#filter_study_group').val(); + let study_sample_data = JSON.parse($('input[name=study_samplelists]').val()) + let filter_samples = study_sample_data[parseInt(this_study)]['samples'] + let sample_nodes = table_api.column(2).nodes().to$(); + let val_nodes = table_api.column(3).nodes().to$(); + for (i = 0; i < sample_nodes.length; i++) { + this_sample = sample_nodes[i].childNodes[0].innerText; + if (!filter_samples.includes(this_sample)){ + val_nodes[i].childNodes[0].value = "x"; + } + } +} + filter_by_value = function() { let filter_logic = $('#filter_logic').val(); let filter_column = $('#filter_column').val(); @@ -1690,6 +1704,11 @@ $('#block_by_index').click(function(){ edit_data_change(); }); +$('#filter_by_study').click(function(){ + filter_by_study(); + edit_data_change(); +}) + $('#filter_by_value').click(function(){ filter_by_value(); edit_data_change(); -- cgit v1.2.3 From 9ae4c8d0370137c3a698d57566305d815929d4ce Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 23:28:08 +0000 Subject: Fixed issue where 1 was added to the loop.index instead of subtracted + only show the sample group selection if there are more than one sample groups --- wqflask/wqflask/templates/show_trait_transform_and_filter.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_transform_and_filter.html b/wqflask/wqflask/templates/show_trait_transform_and_filter.html index 56c3c30e..064fd3e5 100644 --- a/wqflask/wqflask/templates/show_trait_transform_and_filter.html +++ b/wqflask/wqflask/templates/show_trait_transform_and_filter.html @@ -50,9 +50,10 @@ Filter samples by study: + {% if sample_groups|length != 1 %} + {% endif %}

{% endif %} -- cgit v1.2.3 From 6acafff1b22089916b8323afb872e2ebed37943a Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 12 Aug 2021 23:28:49 +0000 Subject: Account for user selecting one of the two sample groups if more than one exist --- wqflask/wqflask/static/new/javascript/show_trait.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 3957175b..845ed907 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -718,9 +718,19 @@ block_by_index = function() { filter_by_study = function() { let this_study = $('#filter_study').val(); - let block_group = $('#filter_study_group').val(); + let study_sample_data = JSON.parse($('input[name=study_samplelists]').val()) let filter_samples = study_sample_data[parseInt(this_study)]['samples'] + + if ($('#filter_study_group').length){ + let block_group = $('#filter_study_group').val(); + if (block_group === "other") { + table_api = $('#samples_other').DataTable(); + } else { + table_api = $('#samples_primary').DataTable(); + } + } + let sample_nodes = table_api.column(2).nodes().to$(); let val_nodes = table_api.column(3).nodes().to$(); for (i = 0; i < sample_nodes.length; i++) { -- cgit v1.2.3 From e2ee1c489f0eb59c3efbf81d42f43a1f74cd1adb Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 6 Aug 2021 15:31:30 +0300 Subject: wqflask: show_trait:: Remove debug comment --- wqflask/wqflask/show_trait/show_trait.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index c07430dd..b640ec27 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -520,6 +520,9 @@ class ShowTrait: sample_group_type='primary', header="%s Only" % (self.dataset.group.name)) self.sample_groups = (primary_samples,) + print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") + print(self.sample_groups) + print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") self.primary_sample_names = primary_sample_names self.dataset.group.allsamples = all_samples_ordered -- cgit v1.2.3 From 579ae94b08f0b1cef00350b54dadc12869ad70de Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 6 Aug 2021 16:15:17 +0300 Subject: base: data_set: Remove unnecessary comments and logging statements --- wqflask/base/data_set.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 4cb82665..edc22540 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -277,7 +277,6 @@ class Markers: filtered_markers = [] for marker in self.markers: if marker['name'] in p_values: - # logger.debug("marker {} IS in p_values".format(i)) marker['p_value'] = p_values[marker['name']] if math.isnan(marker['p_value']) or (marker['p_value'] <= 0): marker['lod_score'] = 0 @@ -298,7 +297,6 @@ class HumanMarkers(Markers): self.markers = [] for line in marker_data_fh: splat = line.strip().split() - # logger.debug("splat:", splat) if len(specified_markers) > 0: if splat[1] in specified_markers: marker = {} @@ -737,7 +735,6 @@ class DataSet: and Strain.SpeciesId=Species.Id and Species.name = '{}' """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) - logger.sql(query) results = dict(g.db.execute(query).fetchall()) sample_ids = [results[item] for item in self.samplelist] @@ -908,7 +905,6 @@ class PhenotypeDataSet(DataSet): Geno.Name = '%s' and Geno.SpeciesId = Species.Id """ % (species, this_trait.locus) - logger.sql(query) result = g.db.execute(query).fetchone() if result: @@ -938,7 +934,6 @@ class PhenotypeDataSet(DataSet): Order BY Strain.Name """ - logger.sql(query) results = g.db.execute(query, (trait, self.id)).fetchall() return results @@ -1005,7 +1000,6 @@ class GenotypeDataSet(DataSet): Order BY Strain.Name """ - logger.sql(query) results = g.db.execute(query, (webqtlDatabaseFunction.retrieve_species_id(self.group.name), trait, self.name)).fetchall() @@ -1126,8 +1120,6 @@ class MrnaAssayDataSet(DataSet): ProbeSet.Name = '%s' """ % (escape(str(this_trait.dataset.id)), escape(this_trait.name))) - - logger.sql(query) result = g.db.execute(query).fetchone() mean = result[0] if result else 0 @@ -1147,7 +1139,6 @@ class MrnaAssayDataSet(DataSet): Geno.Name = '{}' and Geno.SpeciesId = Species.Id """.format(species, this_trait.locus) - logger.sql(query) result = g.db.execute(query).fetchone() if result: @@ -1179,7 +1170,6 @@ class MrnaAssayDataSet(DataSet): Order BY Strain.Name """ % (escape(trait), escape(self.name)) - logger.sql(query) results = g.db.execute(query).fetchall() return results @@ -1190,7 +1180,6 @@ class MrnaAssayDataSet(DataSet): where ProbeSetXRef.ProbeSetFreezeId = %s and ProbeSetXRef.ProbeSetId=ProbeSet.Id; """ % (column_name, escape(str(self.id))) - logger.sql(query) results = g.db.execute(query).fetchall() return dict(results) @@ -1224,7 +1213,6 @@ def geno_mrna_confidentiality(ob): query = '''SELECT Id, Name, FullName, confidentiality, AuthorisedUsers FROM %s WHERE Name = "%s"''' % (dataset_table, ob.name) - logger.sql(query) result = g.db.execute(query) (dataset_id, -- cgit v1.2.3 From 7f2a1f9d1b91b9d0d24da56afcfe26197a434af3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 16 Aug 2021 11:13:40 +0300 Subject: Add "Description" column to CaseAttribute See: eecaad2d --- scripts/add_missing_columns.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/add_missing_columns.sh b/scripts/add_missing_columns.sh index 70d5fdeb..611e2dd6 100644 --- a/scripts/add_missing_columns.sh +++ b/scripts/add_missing_columns.sh @@ -13,6 +13,9 @@ ALTER TABLE PublishXRef ADD mean double AFTER DataId; + ALTER TABLE CaseAttribute + ADD Description varchar(255) AFTER Name; + -- This takes some time ALTER TABLE ProbeSet ADD UniProtID varchar(20) AFTER ProteinName; -- cgit v1.2.3 From 2071107566d89a12fe27fdcc94dd16b7b8680f83 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 16 Aug 2021 11:38:05 +0300 Subject: wqflask: views: Remove commented out code blocks --- wqflask/wqflask/views.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 44560427..08e895cb 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -365,20 +365,6 @@ def wcgna_setup(): return render_template("wgcna_setup.html", **request.form) -# @app.route("/wgcna_results", methods=('POST',)) -# def wcgna_results(): -# logger.info("In wgcna, request.form is:", request.form) -# logger.info(request.url) -# # Start R, load the package and pointers and create the analysis -# wgcna = wgcna_analysis.WGCNA() -# # Start the analysis, a wgcnaA object should be a separate long running thread -# wgcnaA = wgcna.run_analysis(request.form) -# # After the analysis is finished store the result -# result = wgcna.process_results(wgcnaA) -# # Display them using the template -# return render_template("wgcna_results.html", **result) - - @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): # We are going to get additional user input for the analysis @@ -388,20 +374,6 @@ def ctl_setup(): return render_template("ctl_setup.html", **request.form) -# @app.route("/ctl_results", methods=('POST',)) -# def ctl_results(): -# logger.info("In ctl, request.form is:", request.form) -# logger.info(request.url) -# # Start R, load the package and pointers and create the analysis -# ctl = ctl_analysis.CTL() -# # Start the analysis, a ctlA object should be a separate long running thread -# ctlA = ctl.run_analysis(request.form) -# # After the analysis is finished store the result -# result = ctl.process_results(ctlA) -# # Display them using the template -# return render_template("ctl_results.html", **result) - - @app.route("/intro") def intro(): doc = Docs("intro", request.args) -- cgit v1.2.3 From 923d849d640e3048422e3cdfa6ce24716cfa7904 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 16 Aug 2021 18:54:16 +0300 Subject: wqflask: views: Use "inbredset_id" in the "Edit Trait" URL --- wqflask/wqflask/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 08e895cb..3f0ab93f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -408,9 +408,9 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/trait//edit/phenotype-id/") +@app.route("/trait//edit/inbredset-id/") @admin_login_required -def edit_phenotype(name, phenotype_id): +def edit_phenotype(name, inbredset_id): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), passwd=current_app.config.get("DB_PASS"), @@ -419,7 +419,7 @@ def edit_phenotype(name, phenotype_id): conn=conn, table="PublishXRef", where=PublishXRef(id_=name, - phenotype_id=phenotype_id)) + inbred_set_id=inbredset_id)) phenotype_ = fetchone( conn=conn, table="Phenotype", -- cgit v1.2.3 From 82a7ccf6d2e5e18dcbd83f8070fd88ddc34ad419 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 16 Aug 2021 19:00:34 +0300 Subject: Update URL for editing published trait --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index bb30c54c..53e16aa0 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -236,7 +236,7 @@ {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} {% if this_trait.dataset.type == 'Publish' %} - + {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} -- cgit v1.2.3 From 83af88a4bbf9d8a67f4d11c3546ba64f09efb6d2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 17 Aug 2021 09:15:04 +0300 Subject: wqflask: wsgi: Remove it * wqflask/wsgi.py: Delete file. Not used anywhere. --- wqflask/wsgi.py | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 wqflask/wsgi.py diff --git a/wqflask/wsgi.py b/wqflask/wsgi.py deleted file mode 100644 index 755da333..00000000 --- a/wqflask/wsgi.py +++ /dev/null @@ -1,4 +0,0 @@ -from run_gunicorn import app as application # expect application as a name - -if __name__ == "__main__": - application.run() -- cgit v1.2.3 From b0610024e7cb5dec73742fd461cd662881869214 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 17 Aug 2021 09:15:47 +0300 Subject: wqflask: views: Re-enable admin access in edit probeset --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3f0ab93f..7095c392 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -466,7 +466,7 @@ def edit_phenotype(name, inbredset_id): @app.route("/trait/edit/probeset-name/") -# @admin_login_required +@admin_login_required def edit_probeset(dataset_name): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), -- cgit v1.2.3 From d89a13ef9f3f4da47432eb3555df3ada7bbfbb8b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 17 Aug 2021 12:41:24 +0300 Subject: wqflask: runserver: Add DebugToolbar when running in DEBUG mode --- wqflask/runserver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index df957bd9..8198b921 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -23,6 +23,9 @@ app_config() werkzeug_logger = logging.getLogger('werkzeug') if WEBSERVER_MODE == 'DEBUG': + from flask_debugtoolbar import DebugToolbarExtension + app.debug = True + toolbar = DebugToolbarExtension(app) app.run(host='0.0.0.0', port=SERVER_PORT, debug=True, -- cgit v1.2.3 From b70bf1250bc93a47c3e3e32d4d2f4a116121d11f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 17 Aug 2021 12:43:50 +0300 Subject: wqflask:.DS_Store: Delete it --- wqflask/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 wqflask/.DS_Store diff --git a/wqflask/.DS_Store b/wqflask/.DS_Store deleted file mode 100644 index d992942f..00000000 Binary files a/wqflask/.DS_Store and /dev/null differ -- cgit v1.2.3 From 94f278b68817fb22d363fdfd1b416f9c674c3226 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 18 Aug 2021 09:42:48 +0300 Subject: Revert "wqflask: wsgi: Remove it" This reverts commit 83af88a4bbf9d8a67f4d11c3546ba64f09efb6d2. --- wqflask/wsgi.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 wqflask/wsgi.py diff --git a/wqflask/wsgi.py b/wqflask/wsgi.py new file mode 100644 index 00000000..755da333 --- /dev/null +++ b/wqflask/wsgi.py @@ -0,0 +1,4 @@ +from run_gunicorn import app as application # expect application as a name + +if __name__ == "__main__": + application.run() -- cgit v1.2.3 From f5f1b3e64db18dbdde39860f1f86a4bf408e3dd6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 16 Aug 2021 19:50:24 +0000 Subject: Replace textarea with multiple select element for displaying mapping covariates --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 3dd44c85..3f0557aa 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -77,7 +77,9 @@

- + {% endif %} @@ -320,7 +322,9 @@ - + {% endif %} -- cgit v1.2.3 From eb7863b540bda56f3e0a769440ebc16c675d4adb Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 16 Aug 2021 19:51:07 +0000 Subject: Update get_covariates_from_collection.js to properly interact with the select element instead of the previous textarea --- .../javascript/get_covariates_from_collection.js | 55 +++++++++++++++------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js index 3e414034..62590aaf 100644 --- a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js +++ b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js @@ -65,10 +65,8 @@ if ( ! $.fn.DataTable.isDataTable( '#collection_table' ) ) { collection_click = function() { var this_collection_url; - console.log("Clicking on:", $(this)); this_collection_url = $(this).find('.collection_name').prop("href"); this_collection_url += "&json"; - console.log("this_collection_url", this_collection_url); collection_list = $("#collections_holder").html(); return $.ajax({ dataType: "json", @@ -79,32 +77,57 @@ collection_click = function() { submit_click = function() { var covariates_string = ""; - var covariates_display_string = ""; + var covariates_as_set = new Set(); + $(".selected-covariates:first option").each(function() { + if ($(this).val() != ""){ + covariates_as_set.add($(this).val() + "," + $(this).text()); + } + }); $('#collections_holder').find('input[type=checkbox]:checked').each(function() { var this_dataset, this_trait; this_trait = $(this).parents('tr').find('.trait').text(); this_trait_display = $(this).parents('tr').find('.trait').data("display_name"); this_description = $(this).parents('tr').find('.description').text(); - console.log("this_trait is:", this_trait_display); this_dataset = $(this).parents('tr').find('.dataset').data("dataset"); - console.log("this_dataset is:", this_dataset); - covariates_string += this_trait + ":" + this_dataset + "," - //this_covariate_display_string = this_trait + ": " + this_description this_covariate_display_string = this_trait_display if (this_covariate_display_string.length > 50) { this_covariate_display_string = this_covariate_display_string.substring(0, 45) + "..." } - covariates_display_string += this_covariate_display_string + "\n" + covariates_as_set.add(this_trait + ":" + this_dataset + "," + this_covariate_display_string) + }); + + covariates_as_list = Array.from(covariates_as_set) + + // Removed the starting "No covariates selected" option before adding options for each covariate + if (covariates_as_list.length > 0){ + $(".selected-covariates option[value='']").each(function() { + $(this).remove(); + }); + } + + $(".selected-covariates option").each(function() { + $(this).remove(); }); - // Trim the last newline from display_string - covariates_display_string = covariates_display_string.replace(/\n$/, "") - // Trim the last comma - covariates_string = covariates_string.substring(0, covariates_string.length - 1) - //covariates_display_string = covariates_display_string.substring(0, covariates_display_string.length - 2) + covariate_list_for_form = [] + $.each(covariates_as_list, function (index, value) { + option_value = value.split(",")[0] + option_text = value.split(",")[1] + $(".selected-covariates").append($(" -- cgit v1.2.3 From e98ef60db6d0986360e6c8aa920241bbb268af8e Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 17 Aug 2021 03:07:18 +0000 Subject: Add some CSS changing the way the colorbox pop-up for adding covariates looks on the trait page --- wqflask/wqflask/static/new/css/show_trait.css | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/wqflask/wqflask/static/new/css/show_trait.css b/wqflask/wqflask/static/new/css/show_trait.css index 782dabc2..b0514e01 100644 --- a/wqflask/wqflask/static/new/css/show_trait.css +++ b/wqflask/wqflask/static/new/css/show_trait.css @@ -260,3 +260,33 @@ input.trait-value-input { div.inline-div { display: inline; } + +/* div.colorbox_border { + border: 1px solid grey; +} */ +div#cboxContent { + /* box-shadow: + 0 2.8px 2.2px rgba(0, 0, 0, 0.034), + 0 6.7px 5.3px rgba(0, 0, 0, 0.048), + 0 12.5px 10px rgba(0, 0, 0, 0.06), + 0 22.3px 17.9px rgba(0, 0, 0, 0.072), + 0 41.8px 33.4px rgba(0, 0, 0, 0.086), + 0 100px 80px rgba(0, 0, 0, 0.12) */ + + padding: 10px 10px 5px 10px; + + -moz-box-shadow: 3px 3px 5px #535353; + -webkit-box-shadow: 3px 3px 5px #535353; + box-shadow: 3px 3px 5px #535353; + + -moz-border-radius: 6px 6px 6px 6px; + -webkit-border-radius: 6px; + border-radius: 6px 6px 6px 6px; + + /* border: 2px solid grey; */ +} + +#cboxClose { + margin-right: 5px; + margin-bottom: 2px; +} -- cgit v1.2.3 From 5c879337593366005d07f390b8e0da901ec23881 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 17 Aug 2021 18:34:50 +0000 Subject: Changed Remove All button to Clear and changed Select and Clear button colors --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index cc472914..e63d0195 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -74,9 +74,9 @@ No collections available. Please add traits to a collection to use them as covariates. {% else %}

- + - +

- + {% endif %} -- cgit v1.2.3 From fc15b67ca15a8f08833a8b1e479c152bc4a2b723 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 17 Aug 2021 20:35:50 +0000 Subject: Added option to make collection default in the colorbox pop-up --- wqflask/wqflask/templates/collections/add.html | 29 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/templates/collections/add.html b/wqflask/wqflask/templates/collections/add.html index cb4172ac..8640fdb8 100644 --- a/wqflask/wqflask/templates/collections/add.html +++ b/wqflask/wqflask/templates/collections/add.html @@ -5,7 +5,7 @@ or add to an existing collection.

- + {% if traits is defined %} {% else %} @@ -14,10 +14,8 @@ {% if collections|length > 0 %}

1. Add to an existing collection -

- - {% for col in collections %} {% if loop.index == 1 %} @@ -26,8 +24,9 @@ {% endif %} {% endfor %} -
- + +

+

{% endif %} @@ -35,7 +34,6 @@

{% if collections|length > 0 %}2. {% else %}{% endif %}Create a new collection

- @@ -54,6 +52,21 @@ parent.jQuery.colorbox.close(); }); + make_default = function() { + alert("The current collection is now your default collection.") + let uc_id = $('[name=existing_collection] option:selected').val().split(":")[0] + $.cookie('default_collection', uc_id, { + expires: 365, + path: '/' + }); + + let default_collection_id = $.cookie('default_collection'); + }; + + $("#make_default").on("click", function(){ + make_default(); + }); + apply_default = function() { let default_collection_id = $.cookie('default_collection'); if (default_collection_id) { -- cgit v1.2.3 From 1393d777bd0fee4a25ba4b1f6af6ccb9ae24c0f8 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 17 Aug 2021 20:49:16 +0000 Subject: Added hash of sample values and table of changed values to the loading page --- wqflask/wqflask/templates/loading.html | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 6d6136ac..b02605ee 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -25,6 +25,8 @@
transformation = {{ start_vars.transform }} {% endif %} +
+ hash of sample values = {{ start_vars.vals_hash }}

Mapping Metadata
@@ -68,6 +70,29 @@

+ {% if start_vars.vals_diff|length != 0 and start_vars.transform == "" %} +

+ +
+ + {% endif %}

Sample	New Value	Old Value
{{ sample }}	{{ start_vars.vals_diff[sample].new_val }}	{{ start_vars.vals_diff[sample].old_val }}

@@ -79,4 +104,15 @@ $("#loading_form").attr("action", "{{ start_vars.form_url }}"); setTimeout(function(){ $("#loading_form").submit()}, 350); + +$('#show_full_diff').click(function() { + if ($('#diff_table_container').is(':visible')){ + $('#diff_table_container').hide(); + } else { + $('#diff_table_container').show(); + } +}) + + + -- cgit v1.2.3 From 26958694322855f4ee91beb3b01e73afe75066cf Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 17 Aug 2021 20:50:31 +0000 Subject: Added function for getting the diff of sample values before and after user changes to show_trait.py --- wqflask/wqflask/show_trait/show_trait.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 4120c801..96f6901a 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -1,3 +1,5 @@ +from typing import Dict + import string import datetime import uuid @@ -805,3 +807,41 @@ def get_scales_from_genofile(file_location): return [["physic", "Mb"], ["morgan", "cM"]] else: return [["physic", "Mb"]] + + + +def get_diff_of_vals(new_vals: Dict, trait_id: str) -> Dict: + """ Get the diff between current sample values and the values in the DB + + Given a dict of the changed values and the trait/dataset ID, return a Dict + with keys corresponding to each sample with a changed value and a value + that is a dict with keys for the old_value and new_value + + """ + + trait_name = trait_id.split(":")[0] + dataset_name = trait_id.split(":")[1] + trait_ob = create_trait(name=trait_name, dataset_name=dataset_name) + + old_vals = {sample : trait_ob.data[sample].value for sample in trait_ob.data} + + shared_samples = set.union(set(new_vals.keys()), set(old_vals.keys())) + + diff_dict = {} + for sample in shared_samples: + try: + new_val = float(new_vals[sample]) + except: + new_val = "x" + try: + old_val = float(old_vals[sample]) + except: + old_val = "x" + + if new_val != old_val: + diff_dict[sample] = { + "new_val": new_val, + "old_val": old_val + } + + return diff_dict -- cgit v1.2.3 From 71e6c7483c8675e96376295927133c370478bbba Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 17 Aug 2021 20:51:26 +0000 Subject: Removed some unnecessary empty lines from loading.html JS --- wqflask/wqflask/templates/loading.html | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index b02605ee..1edde31e 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -101,7 +101,6 @@ -- cgit v1.2.3 From ab3b838b99b58693f3f187093e78f492e7cf392b Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 18 Aug 2021 00:13:55 +0000 Subject: Exclude attributes with only one distinct value as well --- wqflask/wqflask/templates/show_trait_transform_and_filter.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_transform_and_filter.html b/wqflask/wqflask/templates/show_trait_transform_and_filter.html index 064fd3e5..8d21b0b0 100644 --- a/wqflask/wqflask/templates/show_trait_transform_and_filter.html +++ b/wqflask/wqflask/templates/show_trait_transform_and_filter.html @@ -25,7 +25,7 @@ Block samples by group: {% endif %} -- cgit v1.2.3 From 041354b4c278c4784119da60ace2c3d4ad981354 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 19 Aug 2021 18:28:31 +0000 Subject: Fixed issue in JS that caused filter_by_value to not work for case attributes --- wqflask/wqflask/static/new/javascript/show_trait.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index be9e2089..f050d4ae 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -818,7 +818,7 @@ filter_by_value = function() { var this_col_value = filter_val_nodes[i].childNodes[0].value; } else { if (filter_val_nodes[i].childNodes[0] !== undefined){ - var this_col_value = filter_val_nodes[i].childNodes[0].data; + var this_col_value = filter_val_nodes[i].innerText; } else { continue } -- cgit v1.2.3 From 0e0f25a71d2fab7395af007643b908af51bd7ca2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 19 Aug 2021 18:29:45 +0000 Subject: Fixed R/qtl 'select covariates' buttons to be the same as the changed ones for GEMMA --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index ddcc3edf..4c0efecb 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -320,9 +320,9 @@ No collections available. Please add traits to a collection to use them as covariates. {% else %}

- + - +

+ {% if categorical_vars is defined %} + + {% endif %} -- cgit v1.2.3 From 0bbe9072566397942a6db034cb11203bcb43b78f Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 23 Aug 2021 19:03:10 +0000 Subject: Fix the way categorical_var_list is set to account for case attribute Ids being keys now instead of names --- wqflask/wqflask/show_trait/show_trait.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 96f6901a..36e69fa3 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -701,7 +701,7 @@ def get_categorical_variables(this_trait, sample_list) -> list: if len(sample_list.attributes) > 0: for attribute in sample_list.attributes: if len(sample_list.attributes[attribute].distinct_values) < 10: - categorical_var_list.append(sample_list.attributes[attribute].name) + categorical_var_list.append(str(sample_list.attributes[attribute].id)) return categorical_var_list -- cgit v1.2.3 From 654b35c2050e89f9ba533a3713d40465cf6fcd47 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 23 Aug 2021 19:05:10 +0000 Subject: Fix get_perm_strata to usethe Ids for case attributes instead of names + allow perm_strata to be passed to the template --- wqflask/wqflask/marker_regression/run_mapping.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index f601201b..ebad7d36 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -220,7 +220,7 @@ class RunMapping: elif self.mapping_method == "rqtl_plink": results = self.run_rqtl_plink() elif self.mapping_method == "rqtl_geno": - perm_strata = [] + self.perm_strata = [] if "perm_strata" in start_vars and "categorical_vars" in start_vars: self.categorical_vars = start_vars["categorical_vars"].split( ",") @@ -229,7 +229,7 @@ class RunMapping: sample_names=self.samples, this_trait=self.this_trait) - perm_strata = get_perm_strata( + self.perm_strata = get_perm_strata( self.this_trait, primary_samples, self.categorical_vars, self.samples) self.score_type = "LOD" self.control_marker = start_vars['control_marker'] @@ -243,10 +243,10 @@ class RunMapping: # self.pair_scan = True if self.permCheck and self.num_perm > 0: self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl( - self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) + self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, self.num_perm, self.perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) else: results = rqtl_mapping.run_rqtl(self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, - self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) + self.num_perm, self.perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) elif self.mapping_method == "reaper": if "startMb" in start_vars: # ZS: Check if first time page loaded, so it can default to ON if "additiveCheck" in start_vars: @@ -765,9 +765,9 @@ def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): if sample in list(sample_list.sample_attribute_values.keys()): combined_string = "" for var in categorical_vars: - if var.lower() in sample_list.sample_attribute_values[sample]: + if var in sample_list.sample_attribute_values[sample]: combined_string += str( - sample_list.sample_attribute_values[sample][var.lower()]) + sample_list.sample_attribute_values[sample][var]) else: combined_string += "NA" else: -- cgit v1.2.3 From fff90d1a1b194fe5e65e99506bc4fe6fffc9a255 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 23 Aug 2021 19:06:01 +0000 Subject: Account for situations where the minimum permutation value is also above webqtlConfig.MAXLRS; previously threw an error --- wqflask/utility/Plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 9b2c6735..d4256a46 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -139,7 +139,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab max_D = max(data) min_D = min(data) # add by NL 06-20-2011: fix the error: when max_D is infinite, log function in detScale will go wrong - if max_D == float('inf') or max_D > webqtlConfig.MAXLRS: + if (max_D == float('inf') or max_D > webqtlConfig.MAXLRS) and min_D < webqtlConfig.MAXLRS: max_D = webqtlConfig.MAXLRS # maximum LRS value xLow, xTop, stepX = detScale(min_D, max_D) @@ -156,7 +156,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab j += step for i, item in enumerate(data): - if item == float('inf') or item > webqtlConfig.MAXLRS: + if (item == float('inf') or item > webqtlConfig.MAXLRS) and min_D < webqtlConfig.MAXLRS: item = webqtlConfig.MAXLRS # maximum LRS value j = int((item - xLow) / step) Count[j] += 1 -- cgit v1.2.3 From 7546da17cc5b7a2f97b490e1593b41609b281a43 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 23 Aug 2021 19:29:38 +0000 Subject: Fixed issue with the way categorical_vars and perm_strata were being passed to the template --- wqflask/wqflask/templates/mapping_results.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index 7baac000..81eb1ba1 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -45,8 +45,10 @@ {% if categorical_vars is defined %} - - + + {% endif %} + {% if perm_strata is defined %} + {% endif %} -- cgit v1.2.3 From 629aab0ef51c2c6f7d4f479e51a7168d9aa14d41 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 23 Aug 2021 19:31:18 +0000 Subject: Removed unused variable from list of 'wanted' variables for mapping --- wqflask/wqflask/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 957c34be..11a9380c 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1067,7 +1067,6 @@ def mapping_results_page(): 'num_perm', 'permCheck', 'perm_strata', - 'strat_var', 'categorical_vars', 'perm_output', 'num_bootstrap', -- cgit v1.2.3 From fe345c9f2e99be748511f6889420d6560a553a37 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 24 Aug 2021 09:55:05 +0200 Subject: README: install with guix profile --- doc/README.org | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/doc/README.org b/doc/README.org index 43c92e3c..fb1781aa 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,7 +2,8 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#install][Install]] + - [[#installing-guix-packages][Installing Guix packages]] + - [[#creating-a-gnu-guix-profile][Creating a GNU Guix profile]] - [[#running-gn2][Running GN2]] - [[#run-gn-proxy][Run gn-proxy]] - [[#run-redis][Run Redis]] @@ -37,27 +38,79 @@ tree. Current supported versions can be found as the SHA values of For a full view of runtime dependencies as defined by GNU Guix, see an example of the [[#gn2-dependency-graph][GN2 Dependency Graph]]. -* Install +* Installing Guix packages Make sure to install GNU Guix using the binary download instructions on the main website. Follow the instructions on [[GUIX-Reproducible-from-source.org]] to download pre-built binaries. Note -the download amounts to several GBs of data. +the download amounts to several GBs of data. Debian-derived distros +may support + +: apt-get install guix + +* Creating a GNU Guix profile + +We run a GNU Guix channel with packages at [[https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics][guix-bioinformatics]]. The +README has instructions for hosting a channel, but typically we use +the GUIX_PACKAGE_PATH instead. First upgrade to a recent guix with + +: mkdir ~/opt +: guix pull -p ~/opt/guix-pull + +It should upgrade (ignore the locales warnings). You can optionally +specify the specific git checkout of guix with + +: guix pull -p ~/opt/guix-pull --commit=f04883d + +which is useful when you ned to roll back to an earlier version +(sometimes our channel goes out of sync). Next, we install +GeneNetwork2 with + +: source ~/opt/guix-pull/etc/profile +: git clone https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics.git ~/guix-bioinformatics +: cd ~/guix-bioinformatics +: git pull +: env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics guix package -i genenetwork2 -p ~/opt/genenetwork2 + +you probably also need guix-past (the upstream channel for older packages): + +: git clone https://gitlab.inria.fr/guix-hpc/guix-past.git ~/guix-past +: cd ~/guix-past +: git pull +: env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics:$HOME/guix-past/modules ~/opt/guix-pull/bin/guix package -i genenetwork2 -p ~/opt/genenetwork2 + +ignore the warnings. Guix should install the software without trying +to build everything. If you system insists on building all packages, +try the `--dry-run` switch and fix the [[https://guix.gnu.org/manual/en/html_node/Substitute-Server-Authorization.html][substitutes]]. You may add the +`--substitute-urls="http://guix.genenetwork.org https://ci.guix.gnu.org https://mirror.hydra.gnu.org"` switch. + +The guix.genenetwork.org has most of our packages pre-built(!). To use +it on your own machine the public key is + +#+begin_src scheme +(public-key + (ecc + (curve Ed25519) + (q #E50F005E6DA2F85749B9AA62C8E86BB551CE2B541DC578C4DBE613B39EC9E750#))) +#+end_src + +Once we have a GNU Guix profile, a running database (see below) and the file storage, +we should be ready to fire up GeneNetwork: * Running GN2 -Default settings for GN2 are listed in a file called -[[../etc/default_settings.py][default_settings.py]]. You can copy this file and pass it as a new -parameter to the genenetwork2 command, e.g. +Check out the source with git: -: genenetwork2 mysettings.py +: git clone git@github.com:genenetwork/genenetwork2.git +: cd genenetwork2 -or you can set environment variables to override individual parameters, e.g. +Run GN2 with above Guix profile -: env SERVER_PORT=5004 SQL_URI=mysql://user:pwd@dbhostname/db_webqtl genenetwork2 +: export GN2_PROFILE=$HOME/opt/genenetwork2 +: env TMPDIR=$HOME/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/data/genenetwork/genotype_files SQL_URI=mysql://webqtlout:webqtlout@localhost/db_webqtl ./bin/genenetwork2 -gunicorn-dev the debug and logging switches can be particularly useful when -developing GN2. +developing GN2. Location and files are the current ones for Penguin2. * Run gn-proxy -- cgit v1.2.3 From ba1dd2c354863536f2c12a25b5e59aed56a683b6 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 24 Aug 2021 10:22:43 +0200 Subject: README: minor edits --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bfb16ddb..003cfd7b 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ [![DOI](https://zenodo.org/badge/5591/genenetwork/genenetwork2.svg)](https://zenodo.org/badge/latestdoi/5591/genenetwork/genenetwork2) [![JOSS](http://joss.theoj.org/papers/10.21105/joss.00025/status.svg)](http://joss.theoj.org/papers/10.21105/joss.00025) [![Actions Status](https://github.com/genenetwork/genenetwork2/workflows/tests/badge.svg)](https://github.com/genenetwork/genenetwork2/actions) + # GeneNetwork This repository contains the current source code for GeneNetwork (GN) (https://www.genenetwork.org/ (version 2). GN2 is a Web -2.0-style framework that includes data and computational tools for online genetics and genomic analysis of -many different populations and many types of molecular, cellular, and physiological data. +2.0-style framework that includes data and computational tools for online genetics and genomic analysis of +many different populations and many types of molecular, cellular, and physiological data. The system is used by scientists and clinians in the field of precision health care and systems genetics. GN and its predecessors have been in operation since Jan 1994, making it one of the longest-lived web services in biomedical research (https://en.wikipedia.org/wiki/GeneNetwork, and see a partial list of publications using GN and its predecessor, WebQTL (https://genenetwork.org/references/). @@ -26,7 +27,7 @@ interface genenetwork2 ``` -(default is http://localhost:5003/). A quick example is +A quick example is ```sh env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 GENENETWORK_FILES=~/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev -- cgit v1.2.3 From 8a8dcba497e8f4210329f42cd4b4e766d95a36fc Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 24 Aug 2021 10:54:01 +0200 Subject: README: on installing GN2 --- doc/README.org | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/doc/README.org b/doc/README.org index fb1781aa..1236016e 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,6 +2,7 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] + - [[#check-list][Check list]] - [[#installing-guix-packages][Installing Guix packages]] - [[#creating-a-gnu-guix-profile][Creating a GNU Guix profile]] - [[#running-gn2][Running GN2]] @@ -38,6 +39,17 @@ tree. Current supported versions can be found as the SHA values of For a full view of runtime dependencies as defined by GNU Guix, see an example of the [[#gn2-dependency-graph][GN2 Dependency Graph]]. +* Check list + +To run GeneNetwork the following services need to function: + +1. [ ] GNU Guix with a guix profile for genenetwork2 +1. [ ] A path to the (static) genotype files +1. [ ] Gn-proxy for authentication +1. [ ] The genenetwork3 service +1. [ ] Redis +1. [ ] Mariadb + * Installing Guix packages Make sure to install GNU Guix using the binary download instructions @@ -107,15 +119,26 @@ Check out the source with git: Run GN2 with above Guix profile : export GN2_PROFILE=$HOME/opt/genenetwork2 -: env TMPDIR=$HOME/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/data/genenetwork/genotype_files SQL_URI=mysql://webqtlout:webqtlout@localhost/db_webqtl ./bin/genenetwork2 -gunicorn-dev +: env TMPDIR=$HOME/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5012 GENENETWORK_FILES=/export/data/genenetwork/genotype_files SQL_URI=mysql://webqtlout:webqtlout@localhost/db_webqtl ./bin/genenetwork2 etc/default_settings.py -gunicorn-dev the debug and logging switches can be particularly useful when developing GN2. Location and files are the current ones for Penguin2. +It may be useful to tunnel the web server to your local browser with +an ssh tunnel: + +If you want to test a service running on the server on a certain +port (say 8202) use + + ssh -L 8202:127.0.0.1:8202 -f -N myname@penguin2.genenetwork.org + +And browse on your local machine to http://localhost:8202/ + * Run gn-proxy GeneNetwork requires a separate gn-proxy server which handles -authorisation and access control. For instructions see the [[https://github.com/genenetwork/gn-proxy][README]]. +authorisation and access control. For instructions see the +[[https://github.com/genenetwork/gn-proxy][README]]. Note it may already be running on our servers! * Run Redis -- cgit v1.2.3 From 3fd1d766fcf38f5a147c01b1c397469139c9080d Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 25 Aug 2021 17:39:02 +0000 Subject: Fix issue that could cause mapping_scale to not be set correctly in show_trait.py --- wqflask/wqflask/show_trait/show_trait.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 36e69fa3..80f5d117 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -300,7 +300,7 @@ class ShowTrait: hddn['compare_traits'] = [] hddn['export_data'] = "" hddn['export_format'] = "excel" - if len(self.scales_in_geno) < 2: + if len(self.scales_in_geno) < 2 and bool(self.scales_in_geno): hddn['mapping_scale'] = self.scales_in_geno[list( self.scales_in_geno.keys())[0]][0][0] -- cgit v1.2.3 From ae634a36891ddba9748e0f75ff1cf51e95e4a8a7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 25 Aug 2021 17:39:16 +0000 Subject: Fix issue that could cause an error when doing cM mapping --- wqflask/wqflask/marker_regression/display_mapping_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index f941267e..3986c441 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2113,7 +2113,7 @@ class DisplayMappingResults: thisChr.append( [_locus.name, _locus.cM - Locus0CM]) else: - for j in (0, nLoci / 4, nLoci / 2, nLoci * 3 / 4, -1): + for j in (0, round(nLoci / 4), round(nLoci / 2), round(nLoci * 3 / 4), -1): while _chr[j].name == ' - ': j += 1 if _chr[j].cM != preLpos: -- cgit v1.2.3 From 3327c9dc95d2c4d295dc8dbf4839c01294b53ada Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 25 Aug 2021 17:46:02 +0000 Subject: Add map scale option that was missing for groups with only a single genotype file --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 4c0efecb..5365140d 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -229,6 +229,17 @@ + {% else %} +

+ Map Scale +

+ +

{% endif %}

Permutations -- cgit v1.2.3 From 582893c0cc87f87e5dfc7fe0209b2e4a3f85d4a4 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 25 Aug 2021 18:02:31 +0000 Subject: Fixed issue in get_diff_of_vals that caused the diff to be calculated wrong (due to one set of values being rounded to 3 digits and the other not --- wqflask/wqflask/show_trait/show_trait.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 80f5d117..3f93bae0 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -830,11 +830,11 @@ def get_diff_of_vals(new_vals: Dict, trait_id: str) -> Dict: diff_dict = {} for sample in shared_samples: try: - new_val = float(new_vals[sample]) + new_val = round(float(new_vals[sample]), 3) except: new_val = "x" try: - old_val = float(old_vals[sample]) + old_val = round(float(old_vals[sample]), 3) except: old_val = "x" -- cgit v1.2.3 From cc7c392198f73ac8e8419be1367ce566d85a873c Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 25 Aug 2021 19:05:07 +0000 Subject: Fix issue where correlation results weren't included parents/f1s --- wqflask/wqflask/correlation/correlation_gn3_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index a78cb0b9..d0d4bcba 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -148,7 +148,7 @@ def lit_for_trait_list(corr_results, this_dataset, this_trait): def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset): sample_data = process_samples( - start_vars, this_dataset.group.samplelist) + start_vars, this_dataset.group.all_samples_ordered()) if target_dataset.type == "ProbeSet": target_dataset.get_probeset_data(list(sample_data.keys())) -- cgit v1.2.3 From 5ebe36d5ff685e6b663b14c130606aa60b0123c2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 3 Sep 2021 14:58:05 +0000 Subject: Fix issue where values written to phenotype file for R/qtl sometimes had trailing decimal values by grounding to 3 places past the decimal --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 09afb8d1..cd578870 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -89,7 +89,7 @@ def write_phenotype_file(trait_name: str, for i, sample in enumerate(samples): this_row = [sample] if vals[i] != "x": - this_row.append(vals[i]) + this_row.append(str(round(float(vals[i]), 3))) else: this_row.append("NA") for cofactor in cofactor_data: @@ -126,7 +126,7 @@ def cofactors_to_dict(cofactors: str, dataset_ob, samples) -> Dict: sample_data = trait_ob.data for index, sample in enumerate(samples): if sample in sample_data: - sample_value = sample_data[sample].value + sample_value = str(round(float(sample_data[sample].value), 3)) cofactor_dict[cofactor_name].append(sample_value) else: cofactor_dict[cofactor_name].append("NA") -- cgit v1.2.3 From 8af212bc7eac5b39e8e6838df446a4003633a55e Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 3 Sep 2021 14:59:14 +0000 Subject: Fix issue that caused javascript to not work on the R/qtl mapping result page when permutations weren't used (because it wrongly expected the permutation histogram to always exist) --- wqflask/wqflask/templates/mapping_results.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index 81eb1ba1..d446745d 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -529,7 +529,7 @@ }); {% endif %} - {% if mapping_method != "gemma" and mapping_method != "plink" %} + {% if mapping_method != "gemma" and mapping_method != "plink" and nperm > 0 and permChecked == "ON" %} $('#download_perm').click(function(){ perm_info_dict = { perm_data: js_data.perm_results, -- cgit v1.2.3 From f4297bc9e4c49f58e0759d02d3e7aea392ba5615 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 3 Sep 2021 15:01:34 +0000 Subject: Allow categorical_var_list to be passed as a template variable --- wqflask/wqflask/show_trait/show_trait.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 3f93bae0..c4d1ae1c 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -178,11 +178,11 @@ class ShowTrait: self.sample_group_types['samples_primary'] = self.dataset.group.name sample_lists = [group.sample_list for group in self.sample_groups] - categorical_var_list = [] + self.categorical_var_list = [] self.numerical_var_list = [] if not self.temp_trait: # ZS: Only using first samplelist, since I think mapping only uses those samples - categorical_var_list = get_categorical_variables( + self.categorical_var_list = get_categorical_variables( self.this_trait, self.sample_groups[0]) self.numerical_var_list = get_numerical_variables( self.this_trait, self.sample_groups[0]) @@ -287,8 +287,8 @@ class ShowTrait: hddn['study_samplelists'] = json.dumps(study_samplelist_json) hddn['num_perm'] = 0 hddn['categorical_vars'] = "" - if categorical_var_list: - hddn['categorical_vars'] = ",".join(categorical_var_list) + if self.categorical_var_list: + hddn['categorical_vars'] = ",".join(self.categorical_var_list) hddn['manhattan_plot'] = "" hddn['control_marker'] = "" if not self.temp_trait: @@ -323,7 +323,7 @@ class ShowTrait: has_num_cases=self.has_num_cases, attributes=self.sample_groups[0].attributes, categorical_attr_exists=self.categorical_attr_exists, - categorical_vars=",".join(categorical_var_list), + categorical_vars=",".join(self.categorical_var_list), num_values=self.num_values, qnorm_values=self.qnorm_vals, zscore_values=self.z_scores, -- cgit v1.2.3 From f81a5629ab6ea7e39893af00e59c3ac6d79d7892 Mon Sep 17 00:00:00 2001 From: zsloan Date: Sun, 5 Sep 2021 17:03:51 +0000 Subject: Fixed issue that caused sample data to not be fetched correctly; there's something wrong with the 'get_probeset_data' function (not sure why this function exists) --- wqflask/wqflask/correlation/correlation_gn3_api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index d0d4bcba..a18bceaf 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -150,10 +150,7 @@ def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset): sample_data = process_samples( start_vars, this_dataset.group.all_samples_ordered()) - if target_dataset.type == "ProbeSet": - target_dataset.get_probeset_data(list(sample_data.keys())) - else: - target_dataset.get_trait_data(list(sample_data.keys())) + target_dataset.get_trait_data(list(sample_data.keys())) this_trait = retrieve_sample_data(this_trait, this_dataset) this_trait_data = { "trait_sample_data": sample_data, -- cgit v1.2.3 From 25a5fe8027a00a64513855630a4365480cf567d7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 7 Sep 2021 18:21:34 +0000 Subject: Add timer to loading page to track how long the process has been running --- wqflask/wqflask/templates/loading.html | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 1edde31e..ccf810b0 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -12,6 +12,8 @@ {% if start_vars.tool_used == "Mapping" %}

Computing the Maps

+ Time Elapsed: +
Trait Metadata
species = {{ start_vars.species[0] | upper }}{{ start_vars.species[1:] }} @@ -101,9 +103,6 @@ -- cgit v1.2.3 From d243e3a69b26d60709fe10ab0b70a0e1d53ba50d Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 7 Sep 2021 18:22:44 +0000 Subject: Add trait hash and datetime to mapping figure --- .../marker_regression/display_mapping_results.py | 68 ++++++++++++++-------- wqflask/wqflask/marker_regression/run_mapping.py | 1 + .../new/javascript/show_trait_mapping_tools.js | 10 ++-- wqflask/wqflask/templates/mapping_results.html | 1 + wqflask/wqflask/views.py | 1 + 5 files changed, 51 insertions(+), 30 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 3986c441..5f5fe6a3 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -24,6 +24,7 @@ # # Last updated by Zach 12/14/2010 +import datetime import string from math import * from PIL import Image @@ -271,6 +272,7 @@ class DisplayMappingResults: # Needing for form submission when doing single chr # mapping or remapping after changing options self.sample_vals = start_vars['sample_vals'] + self.vals_hash= start_vars['vals_hash'] self.sample_vals_dict = json.loads(self.sample_vals) self.transform = start_vars['transform'] @@ -651,7 +653,7 @@ class DisplayMappingResults: btminfo.append( 'Mapping using genotype data as a trait will result in infinity LRS at one locus. In order to display the result properly, all LRSs higher than 100 are capped at 100.') - def plotIntMapping(self, canvas, offset=(80, 120, 90, 100), zoom=1, startMb=None, endMb=None, showLocusForm=""): + def plotIntMapping(self, canvas, offset=(80, 120, 110, 100), zoom=1, startMb=None, endMb=None, showLocusForm=""): im_drawer = ImageDraw.Draw(canvas) # calculating margins xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset @@ -661,7 +663,7 @@ class DisplayMappingResults: if self.legendChecked: yTopOffset += 10 if self.covariates != "" and self.controlLocus and self.doControl != "false": - yTopOffset += 20 + yTopOffset += 25 if len(self.transform) > 0: yTopOffset += 5 else: @@ -1195,43 +1197,47 @@ class DisplayMappingResults: dataset_label = "%s - %s" % (self.dataset.group.name, self.dataset.fullname) - string1 = 'Dataset: %s' % (dataset_label) + + current_datetime = datetime.datetime.now() + string1 = 'UTC Timestamp: %s' % (current_datetime.strftime("%b %d %Y %H:%M:%S")) + string2 = 'Dataset: %s' % (dataset_label) + string3 = 'Trait Hash: %s' % (self.vals_hash) if self.genofile_string == "": - string2 = 'Genotype File: %s.geno' % self.dataset.group.name + string4 = 'Genotype File: %s.geno' % self.dataset.group.name else: - string2 = 'Genotype File: %s' % self.genofile_string + string4 = 'Genotype File: %s' % self.genofile_string - string4 = '' + string6 = '' if self.mapping_method == "gemma" or self.mapping_method == "gemma_bimbam": if self.use_loco == "True": - string3 = 'Using GEMMA mapping method with LOCO and ' + string5 = 'Using GEMMA mapping method with LOCO and ' else: - string3 = 'Using GEMMA mapping method with ' + string5 = 'Using GEMMA mapping method with ' if self.covariates != "": - string3 += 'the cofactors below:' + string5 += 'the cofactors below:' cofactor_names = ", ".join( [covar.split(":")[0] for covar in self.covariates.split(",")]) - string4 = cofactor_names + string6 = cofactor_names else: - string3 += 'no cofactors' + string5 += 'no cofactors' elif self.mapping_method == "rqtl_plink" or self.mapping_method == "rqtl_geno": - string3 = 'Using R/qtl mapping method with ' + string5 = 'Using R/qtl mapping method with ' if self.covariates != "": - string3 += 'the cofactors below:' + string5 += 'the cofactors below:' cofactor_names = ", ".join( [covar.split(":")[0] for covar in self.covariates.split(",")]) - string4 = cofactor_names + string6 = cofactor_names elif self.controlLocus and self.doControl != "false": - string3 += '%s as control' % self.controlLocus + string5 += '%s as control' % self.controlLocus else: - string3 += 'no cofactors' + string5 += 'no cofactors' else: - string3 = 'Using Haldane mapping function with ' + string5 = 'Using Haldane mapping function with ' if self.controlLocus and self.doControl != "false": - string3 += '%s as control' % self.controlLocus + string5 += '%s as control' % self.controlLocus else: - string3 += 'no control for other QTLs' + string5 += 'no control for other QTLs' y_constant = 10 if self.this_trait.name: @@ -1260,7 +1266,9 @@ class DisplayMappingResults: d = 4 + max( im_drawer.textsize(identification, font=labelFont)[0], im_drawer.textsize(string1, font=labelFont)[0], - im_drawer.textsize(string2, font=labelFont)[0]) + im_drawer.textsize(string2, font=labelFont)[0], + im_drawer.textsize(string3, font=labelFont)[0], + im_drawer.textsize(string4, font=labelFont)[0]) im_drawer.text( text=identification, xy=(xLeftOffset, y_constant * fontZoom), font=labelFont, @@ -1269,7 +1277,9 @@ class DisplayMappingResults: else: d = 4 + max( im_drawer.textsize(string1, font=labelFont)[0], - im_drawer.textsize(string2, font=labelFont)[0]) + im_drawer.textsize(string2, font=labelFont)[0], + im_drawer.textsize(string3, font=labelFont)[0], + im_drawer.textsize(string4, font=labelFont)[0]) if len(self.transform) > 0: transform_text = "Transform - " @@ -1296,14 +1306,22 @@ class DisplayMappingResults: text=string2, xy=(xLeftOffset, y_constant * fontZoom), font=labelFont, fill=labelColor) y_constant += 15 - if string3 != '': + im_drawer.text( + text=string3, xy=(xLeftOffset, y_constant * fontZoom), + font=labelFont, fill=labelColor) + y_constant += 15 + im_drawer.text( + text=string4, xy=(xLeftOffset, y_constant * fontZoom), + font=labelFont, fill=labelColor) + y_constant += 15 + if string4 != '': im_drawer.text( - text=string3, xy=(xLeftOffset, y_constant * fontZoom), + text=string5, xy=(xLeftOffset, y_constant * fontZoom), font=labelFont, fill=labelColor) y_constant += 15 - if string4 != '': + if string5 != '': im_drawer.text( - text=string4, xy=(xLeftOffset, y_constant * fontZoom), + text=string6, xy=(xLeftOffset, y_constant * fontZoom), font=labelFont, fill=labelColor) def drawGeneBand(self, canvas, gifmap, plotXScale, offset=(40, 120, 80, 10), zoom=1, startMb=None, endMb=None): diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index ebad7d36..2f90b475 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -75,6 +75,7 @@ class RunMapping: self.vals = [] self.samples = [] self.sample_vals = start_vars['sample_vals'] + self.vals_hash = start_vars['vals_hash'] sample_val_dict = json.loads(self.sample_vals) samples = sample_val_dict.keys() if (len(genofile_samplelist) != 0): diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index 09e9d024..b75d658e 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -141,11 +141,11 @@ $('input[name=display_all]').change((function(_this) { })(this)); //ZS: This is a list of inputs to be passed to the loading page, since not all inputs on the trait page are relevant to mapping -var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'sample_vals', - 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', - 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'do_control', 'genofile', - 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', - 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] +var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', + 'sample_vals', 'vals_hash', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', + 'num_bootstrap', 'bootCheck', 'bootstrap_results', 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', + 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', + 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] $(".rqtl-geno-tab, #rqtl_geno_compute").on("click", (function(_this) { return function() { diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index d446745d..162ae810 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -34,6 +34,7 @@ + diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 11a9380c..707b18e1 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1051,6 +1051,7 @@ def mapping_results_page(): 'samples', 'vals', 'sample_vals', + 'vals_hash', 'first_run', 'output_files', 'geno_db_exists', -- cgit v1.2.3 From 1f2e32a91727abab77ecdf501fcc5040b17dfece Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 9 Sep 2021 17:07:06 +0000 Subject: Replaced trait name with trait display name in display_mapping_results so the group codes will be includes in phenotype IDs --- wqflask/wqflask/marker_regression/display_mapping_results.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 5f5fe6a3..e9ba7dff 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -1249,18 +1249,18 @@ class DisplayMappingResults: if self.this_trait.symbol: identification += "Trait: %s - %s" % ( - self.this_trait.name, self.this_trait.symbol) + self.this_trait.display_name, self.this_trait.symbol) elif self.dataset.type == "Publish": if self.this_trait.post_publication_abbreviation: identification += "Trait: %s - %s" % ( - self.this_trait.name, self.this_trait.post_publication_abbreviation) + self.this_trait.display_name, self.this_trait.post_publication_abbreviation) elif self.this_trait.pre_publication_abbreviation: identification += "Trait: %s - %s" % ( - self.this_trait.name, self.this_trait.pre_publication_abbreviation) + self.this_trait.display_name, self.this_trait.pre_publication_abbreviation) else: - identification += "Trait: %s" % (self.this_trait.name) + identification += "Trait: %s" % (self.this_trait.display_name) else: - identification += "Trait: %s" % (self.this_trait.name) + identification += "Trait: %s" % (self.this_trait.display_name) identification += " with %s samples" % (self.n_samples) d = 4 + max( -- cgit v1.2.3 From 5020158b3ab7cf14a5809af65d2c616d32714a22 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 9 Sep 2021 17:07:47 +0000 Subject: Change trait name to display name in the metadata at the top of the page (so group codes will be includes in phenotype IDs --- wqflask/wqflask/templates/mapping_results.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index 162ae810..7f865056 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -68,7 +68,7 @@

Map Viewer: Whole Genome

Population: {{ dataset.group.species|capitalize }} {{ dataset.group.name }}
Database: {{ dataset.fullname }}
- {% if dataset.type == "ProbeSet" %}Trait ID:{% else %}Record ID:{% endif %} {{ this_trait.name }}
+ {% if dataset.type == "ProbeSet" %}Trait ID:{% else %}Record ID:{% endif %} {{ this_trait.display_name }}
{% if dataset.type == "ProbeSet" %} Gene Symbol: {{ this_trait.symbol }}
Location: Chr {{ this_trait.chr }} @ {{ this_trait.mb }} Mb
-- cgit v1.2.3 From f51569c80a4e00ad7dcd99ed9bf3ed6d9aaf4051 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 10 Sep 2021 18:54:44 +0000 Subject: Removed encoding, since it's apparently not needed since the Python 3 switchover (and was causing there to be no matches between user IDs and groups) --- wqflask/utility/redis_tools.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index ff125bd2..99295c67 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -135,10 +135,8 @@ def get_user_groups(user_id): for key in groups_list: try: group_ob = json.loads(groups_list[key]) - group_admins = set([this_admin.encode( - 'utf-8') if this_admin else None for this_admin in group_ob['admins']]) - group_members = set([this_member.encode( - 'utf-8') if this_member else None for this_member in group_ob['members']]) + group_admins = set([this_admin if this_admin else None for this_admin in group_ob['admins']]) + group_members = set([this_member if this_member else None for this_member in group_ob['members']]) if user_id in group_admins: admin_group_ids.append(group_ob['id']) elif user_id in group_members: -- cgit v1.2.3 From d76e1ec30195074cf8dfd0f8396285d31faa8984 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 10 Sep 2021 19:02:10 +0000 Subject: Fix issue with the way the template was checking if genofile_string was set; it was receiving an empty string, so the previous logic made it think there was a genofile string when there wasn't one --- wqflask/wqflask/templates/mapping_results.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index 7f865056..7a222d0c 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -73,7 +73,7 @@ Gene Symbol: {{ this_trait.symbol }}
Location: Chr {{ this_trait.chr }} @ {{ this_trait.mb }} Mb
{% endif %} - {% if genofile_string is defined %} + {% if genofile_string != "" %} Genotypes: {{ genofile_string.split(":")[1] }} {% endif %}
-- cgit v1.2.3 From f414cd4109229546bfb8c56fea95b6729121f0b3 Mon Sep 17 00:00:00 2001 From: jgart Date: Thu, 9 Sep 2021 22:28:02 -0500 Subject: Add Jupyter Notebook Launcher to Tools dropdown menu --- wqflask/wqflask/templates/base.html | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 049ebe6d..14e6bc88 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -87,6 +87,7 @@

Systems Genetics PheWAS

Genome Browser

BXD Power Calculator

Jupyter Notebook Launcher

Interplanetary File System

-- cgit v1.2.3 From 38a8544553506651a263b0134ab5372c6a62c629 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 13 Sep 2021 18:58:10 +0000 Subject: Temporarily point the View in GN1 button to gn1-lily.genenetwork.org since some features aren't working on gn1.genenetwork.org yet --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 53e16aa0..36d3c15a 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -233,7 +233,7 @@ {% endif %} {% endif %} - + {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} {% if this_trait.dataset.type == 'Publish' %} -- cgit v1.2.3 From 6a94f7a3a4893f087af53c2d39685d9dcb4b733d Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 13 Sep 2021 19:41:19 +0000 Subject: Change Genotype File text to only show the meaningful genotype deescription --- wqflask/wqflask/marker_regression/display_mapping_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index e9ba7dff..a9ebeb64 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -1206,7 +1206,7 @@ class DisplayMappingResults: if self.genofile_string == "": string4 = 'Genotype File: %s.geno' % self.dataset.group.name else: - string4 = 'Genotype File: %s' % self.genofile_string + string4 = 'Genotype File: %s' % self.genofile_string.split(":")[1] string6 = '' if self.mapping_method == "gemma" or self.mapping_method == "gemma_bimbam": -- cgit v1.2.3 From 75132a5382d32d9332f743aca0e406ae6137d7e5 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 13 Sep 2021 19:49:05 +0000 Subject: Include some extra metadata in the mapping page details (in addition to the figure itself) --- wqflask/wqflask/marker_regression/display_mapping_results.py | 4 ++-- wqflask/wqflask/templates/mapping_results.html | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index a9ebeb64..77d6e2db 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -1198,8 +1198,8 @@ class DisplayMappingResults: self.dataset.fullname) - current_datetime = datetime.datetime.now() - string1 = 'UTC Timestamp: %s' % (current_datetime.strftime("%b %d %Y %H:%M:%S")) + self.current_datetime = datetime.datetime.now().strftime("%b %d %Y %H:%M:%S") + string1 = 'UTC Timestamp: %s' % (self.current_datetime) string2 = 'Dataset: %s' % (dataset_label) string3 = 'Trait Hash: %s' % (self.vals_hash) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index 7a222d0c..f2d11e89 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -69,14 +69,15 @@ Population: {{ dataset.group.species|capitalize }} {{ dataset.group.name }}
Database: {{ dataset.fullname }}
{% if dataset.type == "ProbeSet" %}Trait ID:{% else %}Record ID:{% endif %} {{ this_trait.display_name }}
+ Trait Hash: {{ vals_hash }}
{% if dataset.type == "ProbeSet" %} Gene Symbol: {{ this_trait.symbol }}
Location: Chr {{ this_trait.chr }} @ {{ this_trait.mb }} Mb
{% endif %} {% if genofile_string != "" %} - Genotypes: {{ genofile_string.split(":")[1] }} + Genotypes: {{ genofile_string.split(":")[1] }}
{% endif %} -
+ Current Date/Time: {{ current_datetime }}

Download Full Results

-- cgit v1.2.3 From 2ef7cad06cc1dcde61c91d2eafef002d25cb194e Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 13 Sep 2021 19:58:43 +0000 Subject: Add trait name and trait hash to exported mapping results --- wqflask/wqflask/marker_regression/run_mapping.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 2f90b475..b101b948 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -422,8 +422,9 @@ class RunMapping: total_markers = len(self.qtl_results) with Bench("Exporting Results"): - export_mapping_results(self.dataset, self.this_trait, self.qtl_results, self.mapping_results_path, - self.mapping_scale, self.score_type, self.transform, self.covariates, self.n_samples) + export_mapping_results(self.dataset, self.this_trait, self.qtl_results, + self.mapping_results_path, self.mapping_scale, self.score_type, + self.transform, self.covariates, self.n_samples, self.vals_hash) with Bench("Trimming Markers for Figure"): if len(self.qtl_results) > 30000: @@ -541,13 +542,15 @@ class RunMapping: return trimmed_genotype_data -def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, score_type, transform, covariates, n_samples): +def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, score_type, transform, covariates, n_samples, vals_hash): with open(results_path, "w+") as output_file: output_file.write( "Time/Date: " + datetime.datetime.now().strftime("%x / %X") + "\n") output_file.write( "Population: " + dataset.group.species.title() + " " + dataset.group.name + "\n") output_file.write("Data Set: " + dataset.fullname + "\n") + output_file.write("Trait: " + trait.display_name + "\n") + output_file.write("Trait Hash: " + vals_hash + "\n") output_file.write("N Samples: " + str(n_samples) + "\n") if len(transform) > 0: transform_text = "Transform - " -- cgit v1.2.3 From 91d04e5b573107aead1da8482dd9184d9869b82a Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 13 Sep 2021 20:03:45 +0000 Subject: Change mapping export filename to use the trait hash --- wqflask/wqflask/marker_regression/run_mapping.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index b101b948..361c413b 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -104,9 +104,7 @@ class RunMapping: if "results_path" in start_vars: self.mapping_results_path = start_vars['results_path'] else: - mapping_results_filename = self.dataset.group.name + "_" + \ - ''.join(random.choice(string.ascii_uppercase + string.digits) - for _ in range(6)) + mapping_results_filename = "_".join([self.dataset.group.name, self.vals_hash]) self.mapping_results_path = "{}{}.csv".format( webqtlConfig.GENERATED_IMAGE_DIR, mapping_results_filename) -- cgit v1.2.3 From 24e0de14969edb220a03e22d1c9b7e6ed33f6b82 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 13 Sep 2021 20:04:30 +0000 Subject: Change mapping export filename to use the path passed from run_mapping, instead of the generic mapping_results.csv --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 707b18e1..54314a20 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1170,7 +1170,7 @@ def export_mapping_results(): results_csv = open(file_path, "r").read() response = Response(results_csv, mimetype='text/csv', - headers={"Content-Disposition": "attachment;filename=mapping_results.csv"}) + headers={"Content-Disposition": "attachment;filename=" + os.path.basename(file_path)}) return response -- cgit v1.2.3 From c0fbbb42a79baa823b517ee8f537cec1edc057c8 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 14 Sep 2021 18:09:36 +0000 Subject: Replace / with _ in the file hashes in rqtl_mapping.py, since they get translated to directories --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index cd578870..a148b49c 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -61,6 +61,7 @@ def get_hash_of_textio(the_file: TextIO) -> str: the_file.seek(0) hash_of_file = hashlib.md5(the_file.read().encode()).hexdigest() + hash_of_file = hash_of_file.replace("/", "_") # Replace / with _ to prevent issue with filenames being translated to directories return hash_of_file -- cgit v1.2.3 From 8d54d1871e1319f7db46132d3eed61a817d5dc4c Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 14 Sep 2021 18:14:33 +0000 Subject: Replace / with _ in the mapping results filename, due to / causing issues with being translated to directories --- wqflask/wqflask/marker_regression/run_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 361c413b..1df53fef 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -104,7 +104,7 @@ class RunMapping: if "results_path" in start_vars: self.mapping_results_path = start_vars['results_path'] else: - mapping_results_filename = "_".join([self.dataset.group.name, self.vals_hash]) + mapping_results_filename = "_".join([self.dataset.group.name, self.vals_hash]).replace("/", "_") self.mapping_results_path = "{}{}.csv".format( webqtlConfig.GENERATED_IMAGE_DIR, mapping_results_filename) -- cgit v1.2.3 From b6c0885007d516b257d1027b97df66c11b8672dd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 16 Sep 2021 13:26:53 +0300 Subject: wqflask: views: Redirect to the correct URL after phenotype update --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 54314a20..85aa6b17 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -633,7 +633,7 @@ def update_phenotype(): json_data=json.dumps(diff_data))) flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") return redirect(f"/trait/{data_.get('dataset-name')}" - f"/edit/phenotype-id/{data_.get('phenotype-id')}") + f"/edit/inbredset-id/{data_.get('inbred-set-id')}") @app.route("/probeset/update", methods=["POST"]) -- cgit v1.2.3 From 719b256996facb1f4bee2a0288bce1f18168f99b Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 16 Sep 2021 17:59:49 +0000 Subject: Added back the option to edit privileges on trait pages, since the metadata edit button had replaced it --- wqflask/wqflask/templates/show_trait_details.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 36d3c15a..6a541c8c 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -242,6 +242,9 @@ {% if this_trait.dataset.type == 'ProbeSet' %} {% endif %} + {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + + {% endif %} {% endif %} -- cgit v1.2.3 From a9509a01191883e61cd5013453d18e89db022df2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 20 Sep 2021 17:34:26 +0000 Subject: Removed the reaper_version selection option, since original qtlreaper is no longer supported --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 9 --------- 1 file changed, 9 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 5365140d..c6b6c0e1 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -95,15 +95,6 @@ {% elif mapping_method == "QTLReaper" %}

- Version^? -

- -

Chromosome

-- cgit v1.2.3 From 2ad5b87f1c9bc5ce5bd3f54f7d83f18123ce3c3e Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 20 Sep 2021 17:37:18 +0000 Subject: Returned the GN1 url to gn1.genenetwork.org since it's apparently working well enough again now that some issues have been fixed --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 6a541c8c..2a21dd24 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -233,7 +233,7 @@ {% endif %} {% endif %} - + {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} {% if this_trait.dataset.type == 'Publish' %} -- cgit v1.2.3 From 92df90d68c3d3067f6856c6894be92c7c5dbcaa6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 17:41:53 +0000 Subject: Remove references to reaper_version and code for running old qtlreaper --- .../marker_regression/display_mapping_results.py | 3 +- .../wqflask/marker_regression/qtlreaper_mapping.py | 95 ---------------------- wqflask/wqflask/marker_regression/run_mapping.py | 59 +++++--------- .../new/javascript/show_trait_mapping_tools.js | 2 +- wqflask/wqflask/views.py | 1 - 5 files changed, 24 insertions(+), 136 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 77d6e2db..6254b9b9 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -357,8 +357,7 @@ class DisplayMappingResults: if 'use_loco' in list(start_vars.keys()) and self.mapping_method == "gemma": self.use_loco = start_vars['use_loco'] - if 'reaper_version' in list(start_vars.keys()) and self.mapping_method == "reaper": - self.reaper_version = start_vars['reaper_version'] + if self.mapping_method == "reaper": if 'output_files' in start_vars: self.output_files = ",".join( [(the_file if the_file is not None else "") for the_file in start_vars['output_files']]) diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index 4d6715ba..801674e1 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -178,101 +178,6 @@ def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename): return marker_obs, permu_vals, bootstrap_vals -def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot): - genotype = dataset.group.read_genotype_file(use_reaper=True) - - if manhattan_plot != True: - genotype = genotype.addinterval() - - trimmed_samples = [] - trimmed_values = [] - for i in range(0, len(samples_before)): - try: - trimmed_values.append(float(trait_vals[i])) - trimmed_samples.append(str(samples_before[i])) - except: - pass - - perm_output = [] - bootstrap_results = [] - - if num_perm < 100: - suggestive = 0 - significant = 0 - else: - perm_output = genotype.permutation( - strains=trimmed_samples, trait=trimmed_values, nperm=num_perm) - suggestive = perm_output[int(num_perm * 0.37 - 1)] - significant = perm_output[int(num_perm * 0.95 - 1)] - # highly_significant = perm_output[int(num_perm*0.99-1)] #ZS: Currently not used, but leaving it here just in case - - json_data['suggestive'] = suggestive - json_data['significant'] = significant - - if control_marker != "" and do_control == "true": - reaper_results = genotype.regression(strains=trimmed_samples, - trait=trimmed_values, - control=str(control_marker)) - if bootCheck: - control_geno = [] - control_geno2 = [] - _FIND = 0 - for _chr in genotype: - for _locus in _chr: - if _locus.name == control_marker: - control_geno2 = _locus.genotype - _FIND = 1 - break - if _FIND: - break - if control_geno2: - _prgy = list(genotype.prgy) - for _strain in trimmed_samples: - _idx = _prgy.index(_strain) - control_geno.append(control_geno2[_idx]) - - bootstrap_results = genotype.bootstrap(strains=trimmed_samples, - trait=trimmed_values, - control=control_geno, - nboot=num_bootstrap) - else: - reaper_results = genotype.regression(strains=trimmed_samples, - trait=trimmed_values) - - if bootCheck: - bootstrap_results = genotype.bootstrap(strains=trimmed_samples, - trait=trimmed_values, - nboot=num_bootstrap) - - json_data['chr'] = [] - json_data['pos'] = [] - json_data['lod.hk'] = [] - json_data['markernames'] = [] - # if self.additive: - # self.json_data['additive'] = [] - - # Need to convert the QTL objects that qtl reaper returns into a json serializable dictionary - qtl_results = [] - for qtl in reaper_results: - reaper_locus = qtl.locus - # ZS: Convert chr to int - converted_chr = reaper_locus.chr - if reaper_locus.chr != "X" and reaper_locus.chr != "X/Y": - converted_chr = int(reaper_locus.chr) - json_data['chr'].append(converted_chr) - json_data['pos'].append(reaper_locus.Mb) - json_data['lod.hk'].append(qtl.lrs) - json_data['markernames'].append(reaper_locus.name) - # if self.additive: - # self.json_data['additive'].append(qtl.additive) - locus = {"name": reaper_locus.name, "chr": reaper_locus.chr, - "cM": reaper_locus.cM, "Mb": reaper_locus.Mb} - qtl = {"lrs_value": qtl.lrs, "chr": converted_chr, "Mb": reaper_locus.Mb, - "cM": reaper_locus.cM, "name": reaper_locus.name, "additive": qtl.additive, "dominance": qtl.dominance} - qtl_results.append(qtl) - return qtl_results, json_data, perm_output, suggestive, significant, bootstrap_results - - def natural_sort(marker_list): """ Function to naturally sort numbers + strings, adopted from user Mark Byers here: https://stackoverflow.com/questions/4836710/does-python-have-a-built-in-function-for-string-natural-sort diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 1df53fef..290c4a14 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -271,47 +271,32 @@ class RunMapping: self.bootCheck = False self.num_bootstrap = 0 - self.reaper_version = start_vars['reaper_version'] - self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] logger.info("Running qtlreaper") - if self.reaper_version == "new": - self.first_run = True - self.output_files = None - # ZS: check if first run so existing result files can be used if it isn't (for example zooming on a chromosome, etc) - if 'first_run' in start_vars: - self.first_run = False - if 'output_files' in start_vars: - self.output_files = start_vars['output_files'].split( - ",") - - results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results, self.output_files = qtlreaper_mapping.run_reaper(self.this_trait, - self.dataset, - self.samples, - self.vals, - self.json_data, - self.num_perm, - self.bootCheck, - self.num_bootstrap, - self.do_control, - self.control_marker, - self.manhattan_plot, - self.first_run, - self.output_files) - else: - results, self.json_data, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = qtlreaper_mapping.run_original_reaper(self.this_trait, - self.dataset, - self.samples, - self.vals, - self.json_data, - self.num_perm, - self.bootCheck, - self.num_bootstrap, - self.do_control, - self.control_marker, - self.manhattan_plot) + self.first_run = True + self.output_files = None + # ZS: check if first run so existing result files can be used if it isn't (for example zooming on a chromosome, etc) + if 'first_run' in start_vars: + self.first_run = False + if 'output_files' in start_vars: + self.output_files = start_vars['output_files'].split( + ",") + + results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results, self.output_files = qtlreaper_mapping.run_reaper(self.this_trait, + self.dataset, + self.samples, + self.vals, + self.json_data, + self.num_perm, + self.bootCheck, + self.num_bootstrap, + self.do_control, + self.control_marker, + self.manhattan_plot, + self.first_run, + self.output_files) elif self.mapping_method == "plink": self.score_type = "-logP" self.manhattan_plot = True diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index b75d658e..e42fe8c4 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -145,7 +145,7 @@ var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form 'sample_vals', 'vals_hash', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', - 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] + 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'primary_samples'] $(".rqtl-geno-tab, #rqtl_geno_compute").on("click", (function(_this) { return function() { diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 85aa6b17..980b9362 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1097,7 +1097,6 @@ def mapping_results_page(): 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', - 'reaper_version', 'n_samples', 'transform' ) -- cgit v1.2.3 From 7cb23fe9c03e4f8051340e2a8f32645e88acce3a Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 17:52:37 +0000 Subject: Added type='button' to the Delete Collection button within collections/view, since without it it attempts to submit the form encapsulating the button (export in this case) --- wqflask/wqflask/templates/collections/view.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 9ec98ab1..a3090bcf 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -49,7 +49,7 @@ - +

-- cgit v1.2.3 From 9bd85361d58eb67ef9e59eff6031ed9fd00a3411 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 18:30:13 +0000 Subject: Only show most search page options if results exist; otherwise provide a link back to the home page --- wqflask/wqflask/templates/search_result_page.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 7ec335d5..c499aa8f 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -53,6 +53,7 @@ A total of {{ results|count }} records were found.

+ {% if results|count > 0 %} {% if go_term is not none %}

The associated genes include:

{% for word in search_terms %}{{ word.search_term[0] }}{% endfor %}

{% endif %} @@ -133,8 +134,11 @@

{% endif %} + {% else %} +
+ + {% endif %}

@@ -171,6 +175,7 @@ return params; }; + {% if results|count > 0 %} //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters trait_table = $('#trait_table').DataTable( { 'drawCallback': function( settings ) { @@ -412,6 +417,7 @@ var table = $('#trait_table').DataTable(); table.colReorder.reset() }); + {% endif %} submit_special = function(url) { $("#trait_submission_form").attr("action", url); -- cgit v1.2.3 From cbb9905aa4406551973bfe47dcdb00f6b8ce9628 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 18:39:38 +0000 Subject: Fixed issue where the control marker parameter was being passed incorrectly to GN3 (should be just 'control' but was previously passed as 'control_marker') --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index a148b49c..1dca1b1b 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -39,7 +39,7 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, n } if do_control == "true" and control_marker: - post_data["control_marker"] = control_marker + post_data["control"] = control_marker if not manhattan_plot: post_data["interval"] = True -- cgit v1.2.3 From cb9422ec5fc985530fcb4fb44d2729e460424d60 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 18:44:01 +0000 Subject: Removed the Control for (marker) option from R/qtl for now since Danny mentioned it isn't working correctly --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index c6b6c0e1..3af94ed6 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -253,21 +253,6 @@

{% endif %} -

- Control for -

- - - - Yes - - - - No - -

Model

-- cgit v1.2.3 From 3494a9af7f82691f419ba1e3bd616be5ce68a00a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Sep 2021 13:07:11 +0300 Subject: Remove lengthy stack trace for 404, 400, 408 status codes The stack trace causes the log file to grow into unmanageable sizes over time. --- wqflask/wqflask/views.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 980b9362..1139ad43 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -160,18 +160,26 @@ def shutdown_session(exception=None): @app.errorhandler(Exception) -def handle_bad_request(e): +def handle_generic_exceptions(e): + import werkzeug err_msg = str(e) - logger.error(err_msg) - logger.error(request.url) - # get the stack trace and send it to the logger - exc_type, exc_value, exc_traceback = sys.exc_info() - logger.error(traceback.format_exc()) now = datetime.datetime.utcnow() time_str = now.strftime('%l:%M%p UTC %b %d, %Y') - formatted_lines = [request.url - + " (" + time_str + ")"] + traceback.format_exc().splitlines() - + # get the stack trace and send it to the logger + exc_type, exc_value, exc_traceback = sys.exc_info() + formatted_lines = {f"{request.url} ({time_str}) " + f" {traceback.format_exc().splitlines()}"} + + _message_templates = { + werkzeug.exceptions.NotFound: ("404: Not Found: " + f"{time_str}: {request.url}"), + werkzeug.exceptions.BadRequest: ("400: Bad Request: " + f"{time_str}: {request.url}"), + werkzeug.exceptions.RequestTimeout: ("408: Request Timeout: " + f"{time_str}: {request.url}")} + # Default to the lengthy stack trace! + logger.error(_message_templates.get(exc_type, + formatted_lines)) # Handle random animations # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) -- cgit v1.2.3 From 8135bd22898130ae6f58aa74cb80c83ab334d0d4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Sep 2021 13:08:49 +0300 Subject: Remove variable-- named after a python keyword-- called "list" --- wqflask/wqflask/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 1139ad43..007b0f9e 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -184,9 +184,8 @@ def handle_generic_exceptions(e): # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) if not animation: - list = [fn for fn in os.listdir( - "./wqflask/static/gif/error") if fn.endswith(".gif")] - animation = random.choice(list) + animation = random.choice([fn for fn in os.listdir( + "./wqflask/static/gif/error") if fn.endswith(".gif")]) resp = make_response(render_template("error.html", message=err_msg, stack=formatted_lines, error_image=animation, version=GN_VERSION)) -- cgit v1.2.3 From 16947a15bd955be7da45bb5f078870649b1b63e5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Sep 2021 13:09:24 +0300 Subject: Apply pep-8 formatting --- wqflask/wqflask/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 007b0f9e..b297da08 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -188,7 +188,9 @@ def handle_generic_exceptions(e): "./wqflask/static/gif/error") if fn.endswith(".gif")]) resp = make_response(render_template("error.html", message=err_msg, - stack=formatted_lines, error_image=animation, version=GN_VERSION)) + stack=formatted_lines, + error_image=animation, + version=GN_VERSION)) # logger.error("Set cookie %s with %s" % (err_msg, animation)) resp.set_cookie(err_msg[:32], animation) -- cgit v1.2.3 From 266d4c4a425ca0a215c8d789e2978d213d5ff37d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 21 Sep 2021 21:25:13 +0300 Subject: Rename "admin_login_required" to "edit_access_required" --- wqflask/wqflask/decorators.py | 2 +- wqflask/wqflask/views.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index f0978fd3..f6e3eb8a 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -3,7 +3,7 @@ from flask import g from functools import wraps -def admin_login_required(f): +def edit_access_required(f): """Use this for endpoints where admins are required""" @wraps(f) def wrap(*args, **kwargs): diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b297da08..5067ca0e 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -85,7 +85,7 @@ from wqflask.export_traits import export_search_results_csv from wqflask.gsearch import GSearch from wqflask.update_search_results import GSearch as UpdateGSearch from wqflask.docs import Docs, update_text -from wqflask.decorators import admin_login_required +from wqflask.decorators import edit_access_required from wqflask.db_info import InfoPage from utility import temp_data @@ -420,7 +420,7 @@ def submit_trait_form(): @app.route("/trait//edit/inbredset-id/") -@admin_login_required +@edit_access_required def edit_phenotype(name, inbredset_id): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -477,7 +477,7 @@ def edit_phenotype(name, inbredset_id): @app.route("/trait/edit/probeset-name/") -@admin_login_required +@edit_access_required def edit_probeset(dataset_name): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -520,7 +520,7 @@ def edit_probeset(dataset_name): @app.route("/trait/update", methods=["POST"]) -@admin_login_required +@edit_access_required def update_phenotype(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -646,7 +646,7 @@ def update_phenotype(): @app.route("/probeset/update", methods=["POST"]) -@admin_login_required +@edit_access_required def update_probeset(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -1381,7 +1381,7 @@ def get_sample_data_as_csv(trait_name: int, phenotype_id: int): @app.route("/admin/data-sample/diffs/") -@admin_login_required +@edit_access_required def display_diffs_admin(): TMPDIR = current_app.config.get("TMPDIR") DIFF_DIR = f"{TMPDIR}/sample-data/diffs" -- cgit v1.2.3 From 7f317126d7d422b073cb4e4a8698757fe1e763f3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 21 Sep 2021 21:36:32 +0300 Subject: Replace hard-coded e-mails with gn-proxy queries * wqflask/wqflask/decorators.py (edit_access_required.wrap): Query the proxy to see the access rights of a given user. --- wqflask/wqflask/decorators.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index f6e3eb8a..54aa6795 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,14 +1,36 @@ """This module contains gn2 decorators""" from flask import g +from typing import Dict from functools import wraps +from utility.hmac import hmac_creation + +import json +import requests def edit_access_required(f): """Use this for endpoints where admins are required""" @wraps(f) def wrap(*args, **kwargs): - if g.user_session.record.get(b"user_email_address") not in [ - b"labwilliams@gmail.com"]: + resource_id: str = "" + if kwargs.get("inbredset_id"): # data type: dataset-publish + resource_id = hmac_creation("dataset-publish:" + f"{kwargs.get('inbredset_id')}:" + f"{kwargs.get('name')}") + if kwargs.get("dataset_name"): # data type: dataset-probe + resource_id = hmac_creation("dataset-probeset:" + f"{kwargs.get('dataset_name')}") + response: Dict = {} + try: + _user_id = g.user_session.record.get(b"user_id", + "").decode("utf-8") + response = json.loads( + requests.get("http://localhost:8080/" + "available?resource=" + f"{resource_id}&user={_user_id}").content) + except: + response = {} + if "edit" not in response.get("data", []): return "You need to be admin", 401 return f(*args, **kwargs) return wrap -- cgit v1.2.3 From ac14326be6695f185f843d29bf3ff016f5eb3016 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Sep 2021 14:30:09 +0300 Subject: new_security: login_user.html: Delete commented out block --- .../wqflask/templates/new_security/login_user.html | 26 ---------------------- 1 file changed, 26 deletions(-) diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 095036f0..88eab6bc 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -114,31 +114,5 @@ label.error,div.error{ {% endblock %} {% block js %} - - - {% include "new_security/_scripts.html" %} {% endblock %} -- cgit v1.2.3 From 84a0cce8a341b8b45b3b0037379818c32d5614b2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Sep 2021 14:32:03 +0300 Subject: Remove "_scripts.html" and all it's references --- wqflask/wqflask/templates/new_security/_scripts.html | 1 - wqflask/wqflask/templates/new_security/forgot_password.html | 1 - wqflask/wqflask/templates/new_security/forgot_password_step2.html | 1 - wqflask/wqflask/templates/new_security/password_reset.html | 1 - wqflask/wqflask/templates/new_security/register_user.html | 1 - wqflask/wqflask/templates/new_security/registered.html | 1 - wqflask/wqflask/templates/new_security/thank_you.html | 1 - wqflask/wqflask/templates/new_security/verification_still_needed.html | 1 - 8 files changed, 8 deletions(-) delete mode 100644 wqflask/wqflask/templates/new_security/_scripts.html diff --git a/wqflask/wqflask/templates/new_security/_scripts.html b/wqflask/wqflask/templates/new_security/_scripts.html deleted file mode 100644 index 5fefe305..00000000 --- a/wqflask/wqflask/templates/new_security/_scripts.html +++ /dev/null @@ -1 +0,0 @@ - diff --git a/wqflask/wqflask/templates/new_security/forgot_password.html b/wqflask/wqflask/templates/new_security/forgot_password.html index e5c42a45..60a221da 100644 --- a/wqflask/wqflask/templates/new_security/forgot_password.html +++ b/wqflask/wqflask/templates/new_security/forgot_password.html @@ -48,6 +48,5 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/forgot_password_step2.html b/wqflask/wqflask/templates/new_security/forgot_password_step2.html index b4bf41c7..1835fd4c 100644 --- a/wqflask/wqflask/templates/new_security/forgot_password_step2.html +++ b/wqflask/wqflask/templates/new_security/forgot_password_step2.html @@ -20,7 +20,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/password_reset.html b/wqflask/wqflask/templates/new_security/password_reset.html index 684c12b1..e21f075c 100644 --- a/wqflask/wqflask/templates/new_security/password_reset.html +++ b/wqflask/wqflask/templates/new_security/password_reset.html @@ -73,7 +73,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/register_user.html b/wqflask/wqflask/templates/new_security/register_user.html index 3ae4488b..c2895517 100644 --- a/wqflask/wqflask/templates/new_security/register_user.html +++ b/wqflask/wqflask/templates/new_security/register_user.html @@ -100,7 +100,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/registered.html b/wqflask/wqflask/templates/new_security/registered.html index f2f58ec1..29889a97 100644 --- a/wqflask/wqflask/templates/new_security/registered.html +++ b/wqflask/wqflask/templates/new_security/registered.html @@ -19,7 +19,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/thank_you.html b/wqflask/wqflask/templates/new_security/thank_you.html index 0ff7ee8d..d4f5e574 100644 --- a/wqflask/wqflask/templates/new_security/thank_you.html +++ b/wqflask/wqflask/templates/new_security/thank_you.html @@ -18,7 +18,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/verification_still_needed.html b/wqflask/wqflask/templates/new_security/verification_still_needed.html index dc0f9e68..1f91fd8d 100644 --- a/wqflask/wqflask/templates/new_security/verification_still_needed.html +++ b/wqflask/wqflask/templates/new_security/verification_still_needed.html @@ -21,7 +21,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} -- cgit v1.2.3 From 6e0ea75ca427721aed0a5f394b501b2cde9bf769 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 23 Sep 2021 13:17:55 +0300 Subject: Add script for creating/ updating groups during authorisation --- scripts/authentication/group.py | 130 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 scripts/authentication/group.py diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py new file mode 100644 index 00000000..b89bc3ec --- /dev/null +++ b/scripts/authentication/group.py @@ -0,0 +1,130 @@ +"""A script for adding users to a specific group. + +Example: + +Assuming there are no groups and 'test@bonfacemunyoki.com' does not +exist in Redis: + +.. code-block:: bash + python group.py -g "editors" -m "test@bonfacemunyoki.com" + +results in:: + + Successfully created the group: 'editors' + Data: '{"admins": [], "members": []}' + +If 'me@bonfacemunyoki.com' exists in 'users' in Redis and we run: + +.. code-block:: bash + python group.py -g "editors" -m "me@bonfacemunyoki.com" + +now results in:: + + No new group was created. + Updated Data: {'admins': [], 'members': ['me@bonfacemunyoki.com']} + +""" + +import argparse +import redis +import json + +from typing import Dict, List, Optional, Set +from glom import glom # type: ignore + + +def create_group_data(users: Dict, target_group: str, + members: Optional[str] = None, + admins: Optional[str] = None) -> Dict: + """Return a dictionary that contains the following keys: "key", + "field", and "value" that can be used in a redis hash as follows: + HSET key field value + + Parameters: + + - `users`: a list of users for example: + + {'8ad942fe-490d-453e-bd37-56f252e41603': + '{"email_address": "me@test.com", + "full_name": "John Doe", + "organization": "Genenetwork", + "password": {"algorithm": "pbkdf2", + "hashfunc": "sha256", + "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", + "iterations": 100000, "keylength": 32, + "created_timestamp": "2021-09-22T11:32:44.971912", + "password": "edcdaa60e84526c6"}, + "user_id": "8ad942fe", "confirmed": 1, + "registration_info": { + "timestamp": "2021-09-22T11:32:45.028833", + "ip_address": "127.0.0.1", + "user_agent": "Mozilla/5.0"}}'} + + - `target_group`: the group name that will be stored inside the + "groups" hash in Redis. + + - `members`: a comma-separated list of values that contain members + of the `target_group` e.g. "me@test1.com, me@test2.com, + me@test3.com" + + - `admins`: a comma-separated list of values that contain + administrators of the `target_group` e.g. "me@test1.com, + me@test2.com, me@test3.com" + + """ + _members = "".join(members.split()).split(",") if members else [] + _admins: List = "".join(admins.split()).split(",") if admins else [] + + user_emails: Set = {glom(json.loads(user_details), "email_address") + for _, user_details in users.items()} + + return {"key": "groups", + "field": target_group, + "value": json.dumps({ + "admins": [admin for admin in _admins + if admin in user_emails], + "members": [member for member in _members + if member in user_emails] + })} + + +if __name__ == "__main__": + # Initialising the parser CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("-g", "--group-name", + help="This is the name of the GROUP mask") + parser.add_argument("-m", "--members", + help="Members of the GROUP mask") + parser.add_argument("-a", "--admins", + help="Admins of the GROUP mask") + args = parser.parse_args() + + if not args.group_name: + exit("\nExiting. Please specify a group name to use!\n") + + members = args.members if args.members else None + admins = args.admins if args.admins else None + REDIS_CONN = redis.Redis() + USERS = {key.decode(): val.decode() + for key, val in REDIS_CONN.hgetall("users").items()} + + if not any([members, admins]): + exit("\nExiting. Please provide a value for " + "MEMBERS(-m) or ADMINS(-a)!\n") + + data = create_group_data( + users=USERS, + target_group=args.group_name, + members=members, + admins=admins) + created_p = REDIS_CONN.hset(data.get("key", ""), + data.get("field", ""), + data.get("value", "")) + + groups = json.loads(REDIS_CONN.hget("groups", + args.group_name)) # type: ignore + if created_p: + exit(f"\nSuccessfully created the group: '{args.group_name}'\n" + f"Data: {groups}\n") + exit("\nNo new group was created.\n" + f"Updated Data: {groups}\n") -- cgit v1.2.3 From c585945c5516092b362efecc16325ad9ecc54291 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 23 Sep 2021 15:39:32 +0300 Subject: Add script that adds "editors" group to all resources in Redis --- scripts/authentication/resource.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 scripts/authentication/resource.py diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py new file mode 100644 index 00000000..75ef9e93 --- /dev/null +++ b/scripts/authentication/resource.py @@ -0,0 +1,25 @@ +"""A script that adds the group: 'editors' to every +resource. 'editors' should have the right to edit both metadata and +data. + +To use this script, simply run: + +.. code-block:: python + python resource.py + +""" +import json +import redis + + +if __name__ == "__main__": + REDIS_CONN = redis.Redis() + resources = REDIS_CONN.hgetall("resources_clone") + for resource_id, resource in resources.items(): + deserialized_resource = json.loads(resource) + deserialized_resource["group_masks"] = { + "editors": {"metadata": "edit", + "data": "edit"}} + REDIS_CONN.hset("resources_clone", + resource_id, + json.dumps(deserialized_resource)) -- cgit v1.2.3 From edb6fe1b9dd98d84deb0925a2d83726e739d8677 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Sep 2021 16:28:12 +0300 Subject: wqflask: resource_manager: Remove logger --- wqflask/wqflask/resource_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index b28c1b04..c54dd0b3 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -8,8 +8,6 @@ from wqflask import app from utility.authentication_tools import check_owner_or_admin from utility.redis_tools import get_resource_info, get_group_info, get_groups_like_unique_column, get_user_id, get_user_by_unique_column, get_users_like_unique_column, add_access_mask, add_resource, change_resource_owner -from utility.logger import getLogger -logger = getLogger(__name__) @app.route("/resources/manage", methods=('GET', 'POST')) -- cgit v1.2.3 From df487791c91a5aa1a9a3b4e1a6c9ce17a58eafe6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 12:21:16 +0300 Subject: Modify resource editing script to enable data backups & restoration --- scripts/authentication/resource.py | 101 ++++++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 13 deletions(-) diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 75ef9e93..8fcf09d7 100644 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -1,25 +1,100 @@ -"""A script that adds the group: 'editors' to every -resource. 'editors' should have the right to edit both metadata and -data. +"""A script that: -To use this script, simply run: +- Optionally restores data from a json file. + +- By default, without any args provided, adds the group: 'editors' to +every resource. 'editors' should have the right to edit both metadata +and data. + +- Optionally creates a back-up every time you edit a resource. + + +To restore a back-up: + +.. code-block:: python + python resource.py --restore + +To add editors to every resource without creating a back-up: .. code-block:: python python resource.py +To add editors to every resource while creating a back-up before any +destructive edits: + +.. code-block:: python + python resource.py --enable-backup + """ +import argparse import json import redis +import os + +from datetime import datetime + + +def recover_hash(name: str, file_path: str, set_function) -> bool: + """Recover back-ups using the `set_function` + + Parameters: + + - `name`: Redis hash where `file_path` will be restored + + - `file_path`: File path where redis hash is sourced from + + - `set_function`: Function used to do the Redis backup for + example: HSET + + """ + try: + with open(file_path, "r") as f: + resources = json.load(f) + for resource_id, resource in resources.items(): + set_function(name=name, + key=resource_id, + value=resource) + return True + except Exception as e: + print(e) + return False if __name__ == "__main__": - REDIS_CONN = redis.Redis() - resources = REDIS_CONN.hgetall("resources_clone") - for resource_id, resource in resources.items(): - deserialized_resource = json.loads(resource) - deserialized_resource["group_masks"] = { - "editors": {"metadata": "edit", - "data": "edit"}} - REDIS_CONN.hset("resources_clone", + # Initialising the parser CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("--restore", + help="Restore from a given backup") + parser.add_argument("--enable-backup", action="store_true", + help="Create a back up before edits") + args = parser.parse_args() + + if args.restore: + if recover_hash(name="resources", + file_path=args.back_up, + set_function=redis.Redis(decode_responses=True).hset): + exit(f"\n Done restoring {args.back_up}!\n") + else: + exit(f"\n There was an error restoring {args.back_up}!\n") + + REDIS_CONN = redis.Redis(decode_responses=True) + RESOURCES = REDIS_CONN.hgetall("resources") + BACKUP_DIR = os.path.join(os.getenv("HOME"), "redis") + if args.enable_backup: + FILENAME = ("resources-" + f"{datetime.now().strftime('%Y-%m-%d-%I:%M:%S-%p')}" + ".json") + if not os.path.exists(BACKUP_DIR): + os.mkdir(BACKUP_DIR) + with open(os.path.join(BACKUP_DIR, FILENAME), "w") as f: + json.dump(RESOURCES, f, indent=4) + print(f"\nDone backing upto {FILENAME}") + + for resource_id, resource in RESOURCES.items(): + _resource = json.loads(resource) # str -> dict conversion + _resource["group_masks"] = {"editors": {"metadata": "edit", + "data": "edit"}} + REDIS_CONN.hset("resources", resource_id, - json.dumps(deserialized_resource)) + json.dumps(_resource)) + exit("Done updating `resources`\n") -- cgit v1.2.3 From c5215d1ed224480a274476933beded9d2ba7f7dc Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:40:06 +0300 Subject: Decode redis response by default --- scripts/authentication/group.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index b89bc3ec..265e8664 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -104,9 +104,8 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None - REDIS_CONN = redis.Redis() - USERS = {key.decode(): val.decode() - for key, val in REDIS_CONN.hgetall("users").items()} + REDIS_CONN = redis.Redis(decode_responses=True) + USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): exit("\nExiting. Please provide a value for " -- cgit v1.2.3 From 8fae92c83d49042da68638319385df02061df44b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:45:59 +0300 Subject: scripts: group.py: Remove "glom" dependency --- scripts/authentication/group.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 265e8664..02f782b3 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -30,7 +30,6 @@ import redis import json from typing import Dict, List, Optional, Set -from glom import glom # type: ignore def create_group_data(users: Dict, target_group: str, @@ -74,10 +73,12 @@ def create_group_data(users: Dict, target_group: str, """ _members = "".join(members.split()).split(",") if members else [] _admins: List = "".join(admins.split()).split(",") if admins else [] - - user_emails: Set = {glom(json.loads(user_details), "email_address") - for _, user_details in users.items()} - + user_emails: Set = set() + for _, user_details in users.items(): + _details = json.loads(user_details) + if _details.get("email_address"): + user_emails.add(_details.get("email_address")) + print(user_emails) return {"key": "groups", "field": target_group, "value": json.dumps({ -- cgit v1.2.3 From 7c1dd1211f96ca1021debc27a80d1700e70b9c6b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:49:12 +0300 Subject: scripts: group.py: Modify exit message when displaying updated data --- scripts/authentication/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 02f782b3..cc0b037e 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -125,6 +125,6 @@ if __name__ == "__main__": args.group_name)) # type: ignore if created_p: exit(f"\nSuccessfully created the group: '{args.group_name}'\n" - f"Data: {groups}\n") + f"`HGETALL groups {args.group_name}`: {groups}\n") exit("\nNo new group was created.\n" - f"Updated Data: {groups}\n") + f"`HGETALL groups {args.group_name}`: {groups}\n") -- cgit v1.2.3 From 609865fc42b7436d8c34cdcefd159c3352c5d91c Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 20:58:14 +0000 Subject: Add group link for user member groups --- wqflask/wqflask/templates/admin/group_manager.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html index c0b99e75..692a7abc 100644 --- a/wqflask/wqflask/templates/admin/group_manager.html +++ b/wqflask/wqflask/templates/admin/group_manager.html @@ -81,7 +81,7 @@ {{ loop.index }} - {{ group.name }} + {{ group.name }} {{ group.admins|length + group.members|length }} {{ group.created_timestamp }} {{ group.changed_timestamp }} -- cgit v1.2.3 From 5b116de4aaf796be138ee0ad06c2242b3f3c33c7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 21:02:37 +0000 Subject: Changed get_user_groups to pull both the ID and details in the for loop from group_list by using group_list.items() --- wqflask/utility/redis_tools.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index 99295c67..de9dde46 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -127,20 +127,20 @@ def check_verification_code(code): def get_user_groups(user_id): - # ZS: Get the groups where a user is an admin or a member and + # Get the groups where a user is an admin or a member and # return lists corresponding to those two sets of groups - admin_group_ids = [] # ZS: Group IDs where user is an admin - user_group_ids = [] # ZS: Group IDs where user is a regular user + admin_group_ids = [] # Group IDs where user is an admin + user_group_ids = [] # Group IDs where user is a regular user groups_list = Redis.hgetall("groups") - for key in groups_list: + for group_id, group_details in groups_list.items(): try: - group_ob = json.loads(groups_list[key]) - group_admins = set([this_admin if this_admin else None for this_admin in group_ob['admins']]) - group_members = set([this_member if this_member else None for this_member in group_ob['members']]) + _details = json.loads(group_details) + group_admins = set([this_admin if this_admin else None for this_admin in _details['admins']]) + group_members = set([this_member if this_member else None for this_member in _details['members']]) if user_id in group_admins: - admin_group_ids.append(group_ob['id']) + admin_group_ids.append(group_id) elif user_id in group_members: - user_group_ids.append(group_ob['id']) + user_group_ids.append(group_id) else: continue except: -- cgit v1.2.3 From 35a970adba5ee1d60769a81b446122a60eac9494 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 21:03:52 +0000 Subject: Changed the group.py script to replace user e-mails with IDs and to include id, name, changed_timestamp, and created_timestamp in group details --- scripts/authentication/group.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index cc0b037e..a5f75aad 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -26,11 +26,13 @@ now results in:: """ import argparse +import datetime import redis import json from typing import Dict, List, Optional, Set +REDIS_CONN = redis.Redis(decode_responses=True) def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, @@ -71,21 +73,26 @@ def create_group_data(users: Dict, target_group: str, me@test2.com, me@test3.com" """ - _members = "".join(members.split()).split(",") if members else [] + + _members: List = "".join(members.split()).split(",") if members else [] _admins: List = "".join(admins.split()).split(",") if admins else [] - user_emails: Set = set() - for _, user_details in users.items(): + + user_ids: Dict = dict() + for user_id, user_details in users.items(): _details = json.loads(user_details) if _details.get("email_address"): - user_emails.add(_details.get("email_address")) - print(user_emails) + user_ids[_details.get("email_address")] = user_id + print(user_ids) return {"key": "groups", "field": target_group, "value": json.dumps({ - "admins": [admin for admin in _admins - if admin in user_emails], - "members": [member for member in _members - if member in user_emails] + "id": target_group, + "name": target_group, + "admins": [user_ids[admin] for admin in _admins + if admin in user_ids], + "members": [user_ids[member] for member in _members + if member in user_ids], + "changed_timestamp": datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') })} @@ -105,7 +112,6 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None - REDIS_CONN = redis.Redis(decode_responses=True) USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): @@ -117,6 +123,12 @@ if __name__ == "__main__": target_group=args.group_name, members=members, admins=admins) + + if not REDIS_CONN.hget("groups", data.get("field", "")): + updated_data = json.loads(data["value"]) + updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + data["value"] = json.dumps(updated_data) + created_p = REDIS_CONN.hset(data.get("key", ""), data.get("field", ""), data.get("value", "")) -- cgit v1.2.3 From 4c6a7e46dd7afe311c0bed38c4a69ddadf3cb416 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 21:09:15 +0000 Subject: Moved REDIS_CONN back into if __name__ == '__main__' since it doesn't need to be globally accessed anymore (I think I intiially moved it because I was calling it in create_group_data, but that ended up being unnecessary --- scripts/authentication/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index a5f75aad..76c7fb4f 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -32,8 +32,6 @@ import json from typing import Dict, List, Optional, Set -REDIS_CONN = redis.Redis(decode_responses=True) - def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, admins: Optional[str] = None) -> Dict: @@ -112,6 +110,8 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None + + REDIS_CONN = redis.Redis(decode_responses=True) USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): -- cgit v1.2.3

Update History

Trait Information:

Update History

Trait Information:

Update History

Probeset Information:

Trait Information:

Edit Trait for Published Database

Edit Trait for Published Database

Computing the Maps

Map Viewer: Whole Genome