From ca23d4ed6943d25c14ffac767b64fd60bded515e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 17:10:33 +0300 Subject: Move endpoints for viewing metadata to own module --- wqflask/wqflask/__init__.py | 3 + wqflask/wqflask/metadata_edits.py | 147 ++++++++++++++++++++++++++++++++++++++ wqflask/wqflask/views.py | 115 +---------------------------- 3 files changed, 153 insertions(+), 112 deletions(-) create mode 100644 wqflask/wqflask/metadata_edits.py (limited to 'wqflask') diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 5b2d05d1..a5097287 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -11,6 +11,8 @@ from utility import formatting from wqflask.resource_manager import resource_management +from wqflask.metadata_edits import metadata_edit + from wqflask.api.markdown import glossary_blueprint from wqflask.api.markdown import references_blueprint from wqflask.api.markdown import links_blueprint @@ -60,6 +62,7 @@ app.register_blueprint(news_blueprint, url_prefix="/news") app.register_blueprint(resource_management, url_prefix="/resource-management") +app.register_blueprint(metadata_edit, url_prefix="/datasets/") @app.before_request def before_request(): diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py new file mode 100644 index 00000000..94e2710b --- /dev/null +++ b/wqflask/wqflask/metadata_edits.py @@ -0,0 +1,147 @@ +import MySQLdb +import os +import json +import difflib + + +from collections import namedtuple +from flask import Blueprint, current_app, render_template, request +from itertools import groupby + +from wqflask.decorators import edit_access_required + +from gn3.db import diff_from_dict +from gn3.db import fetchall +from gn3.db import fetchone +from gn3.db import insert +from gn3.db import update +from gn3.db.metadata_audit import MetadataAudit +from gn3.db.phenotypes import Phenotype +from gn3.db.phenotypes import Probeset +from gn3.db.phenotypes import Publication +from gn3.db.phenotypes import PublishXRef +from gn3.db.phenotypes import probeset_mapping + + +metadata_edit = Blueprint('metadata_edit', __name__) + + +def edit_phenotype(conn, name, dataset_id): + publish_xref = fetchone( + conn=conn, + table="PublishXRef", + where=PublishXRef(id_=name, + inbred_set_id=dataset_id)) + phenotype_ = fetchone( + conn=conn, + table="Phenotype", + where=Phenotype(id_=publish_xref.phenotype_id)) + publication_ = fetchone( + conn=conn, + table="Publication", + where=Publication(id_=publish_xref.publication_id)) + json_data = fetchall( + conn, + "metadata_audit", + where=MetadataAudit(dataset_id=publish_xref.id_)) + Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) + Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) + diff_data = [] + for data in json_data: + json_ = json.loads(data.json_data) + timestamp = json_.get("timestamp") + author = json_.get("author") + for key, value in json_.items(): + if isinstance(value, dict): + for field, data_ in value.items(): + diff_data.append( + Diff(author=author, + diff=Edit(field, + data_.get("old"), + data_.get("new"), + "\n".join(difflib.ndiff( + [data_.get("old")], + [data_.get("new")]))), + timestamp=timestamp)) + diff_data_ = None + if len(diff_data) > 0: + diff_data_ = groupby(diff_data, lambda x: x.timestamp) + return { + "diff": diff_data_, + "publish_xref": publish_xref, + "phenotype": phenotype_, + "publication": publication_, + } + + +def edit_probeset(conn, name): + probeset_ = fetchone(conn=conn, + table="ProbeSet", + columns=list(probeset_mapping.values()), + where=Probeset(name=name)) + json_data = fetchall( + conn, + "metadata_audit", + where=MetadataAudit(dataset_id=probeset_.id_)) + Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) + Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) + diff_data = [] + for data in json_data: + json_ = json.loads(data.json_data) + timestamp = json_.get("timestamp") + author = json_.get("author") + for key, value in json_.items(): + if isinstance(value, dict): + for field, data_ in value.items(): + diff_data.append( + Diff(author=author, + diff=Edit(field, + data_.get("old"), + data_.get("new"), + "\n".join(difflib.ndiff( + [data_.get("old")], + [data_.get("new")]))), + timestamp=timestamp)) + diff_data_ = None + if len(diff_data) > 0: + diff_data_ = groupby(diff_data, lambda x: x.timestamp) + return { + "diff": diff_data_, + "probeset": probeset_, + } + + +@metadata_edit.route("//traits//edit") +@edit_access_required +def display_phenotype_metadata(dataset_id: str, name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + _d = edit_phenotype(conn=conn, name=name, dataset_id=dataset_id) + return render_template( + "edit_phenotype.html", + diff=_d.get("diff"), + publish_xref=_d.get("publish_xref"), + phenotype=_d.get("phenotype"), + publication=_d.get("publication"), + resource_id=request.args.get("resource-id"), + version=os.environ.get("GN_VERSION"), + ) + + +@metadata_edit.route("/traits//edit") +@edit_access_required +def display_probeset_metadata(name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + _d = edit_probeset(conn=conn, name=name) + return render_template( + "edit_probeset.html", + diff=_d.get("diff"), + probeset=_d.get("probeset"), + resource_id=request.args.get("resource-id"), + version=os.environ.get("GN_VERSION"), + ) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b0da1f21..463b7c3a 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -4,7 +4,6 @@ import MySQLdb import array import base64 import csv -import difflib import datetime import flask import io # Todo: Use cStringIO? @@ -20,8 +19,6 @@ import traceback import uuid import xlsxwriter -from itertools import groupby -from collections import namedtuple from zipfile import ZipFile from zipfile import ZIP_DEFLATED @@ -30,19 +27,12 @@ from wqflask import app from gn3.commands import run_cmd from gn3.computations.gemma import generate_hash_of_string from gn3.db import diff_from_dict -from gn3.db import fetchall -from gn3.db import fetchone from gn3.db import insert from gn3.db import update from gn3.db.metadata_audit import MetadataAudit from gn3.db.phenotypes import Phenotype from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication -from gn3.db.phenotypes import PublishXRef -from gn3.db.phenotypes import probeset_mapping -# from gn3.db.traits import get_trait_csv_sample_data -# from gn3.db.traits import update_sample_data - from flask import current_app from flask import g @@ -426,106 +416,6 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/trait//edit/inbredset-id/") -@edit_access_required -def edit_phenotype(name, inbredset_id): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - publish_xref = fetchone( - conn=conn, - table="PublishXRef", - where=PublishXRef(id_=name, - inbred_set_id=inbredset_id)) - phenotype_ = fetchone( - conn=conn, - table="Phenotype", - where=Phenotype(id_=publish_xref.phenotype_id)) - publication_ = fetchone( - conn=conn, - table="Publication", - where=Publication(id_=publish_xref.publication_id)) - json_data = fetchall( - conn, - "metadata_audit", - where=MetadataAudit(dataset_id=publish_xref.id_)) - - Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) - Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) - diff_data = [] - for data in json_data: - json_ = json.loads(data.json_data) - timestamp = json_.get("timestamp") - author = json_.get("author") - for key, value in json_.items(): - if isinstance(value, dict): - for field, data_ in value.items(): - diff_data.append( - Diff(author=author, - diff=Edit(field, - data_.get("old"), - data_.get("new"), - "\n".join(difflib.ndiff( - [data_.get("old")], - [data_.get("new")]))), - timestamp=timestamp)) - diff_data_ = None - if len(diff_data) > 0: - diff_data_ = groupby(diff_data, lambda x: x.timestamp) - return render_template( - "edit_phenotype.html", - diff=diff_data_, - publish_xref=publish_xref, - phenotype=phenotype_, - publication=publication_, - version=GN_VERSION, - ) - - -@app.route("/trait/edit/probeset-name/") -@edit_access_required -def edit_probeset(dataset_name): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - probeset_ = fetchone(conn=conn, - table="ProbeSet", - columns=list(probeset_mapping.values()), - where=Probeset(name=dataset_name)) - json_data = fetchall( - conn, - "metadata_audit", - where=MetadataAudit(dataset_id=probeset_.id_)) - Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) - Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) - diff_data = [] - for data in json_data: - json_ = json.loads(data.json_data) - timestamp = json_.get("timestamp") - author = json_.get("author") - for key, value in json_.items(): - if isinstance(value, dict): - for field, data_ in value.items(): - diff_data.append( - Diff(author=author, - diff=Edit(field, - data_.get("old"), - data_.get("new"), - "\n".join(difflib.ndiff( - [data_.get("old")], - [data_.get("new")]))), - timestamp=timestamp)) - diff_data_ = None - if len(diff_data) > 0: - diff_data_ = groupby(diff_data, lambda x: x.timestamp) - return render_template( - "edit_probeset.html", - diff=diff_data_, - probeset=probeset_) - - @app.route("/trait/update", methods=["POST"]) @edit_access_required def update_phenotype(): @@ -653,7 +543,6 @@ def update_phenotype(): @app.route("/probeset/update", methods=["POST"]) -@edit_access_required def update_probeset(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -691,7 +580,9 @@ def update_probeset(): where=Probeset(id_=data_.get("id"))) diff_data = {} - author = g.user_session.record.get(b'user_name') + author = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) if updated_probeset: diff_data.update({"Probeset": diff_from_dict(old={ k: data_.get(f"old_{k}") for k, v in probeset_.items() -- cgit v1.2.3 From 433f62500408d84a49153628384f4b4c3e9a7b2e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 17:10:50 +0300 Subject: Get "resource-id" from query parameters instead of computing it --- wqflask/wqflask/decorators.py | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 843539ee..a4ff7ce3 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,9 +1,7 @@ """This module contains gn2 decorators""" -import hashlib -import hmac import redis -from flask import current_app, g +from flask import current_app, g, request from typing import Dict from urllib.parse import urljoin from functools import wraps @@ -14,18 +12,12 @@ import json import requests -def create_hmac(data: str, secret: str) -> str: - return hmac.new(bytearray(secret, "latin-1"), - bytearray(data, "utf-8"), - hashlib.sha1).hexdigest()[:20] - - def login_required(f): """Use this for endpoints where login is required""" @wraps(f) def wrap(*args, **kwargs): user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or + b"").decode("utf-8") or g.user_session.record.get("user_id", "")) redis_conn = redis.from_url(current_app.config["REDIS_URL"], decode_responses=True) @@ -40,28 +32,21 @@ def edit_access_required(f): @wraps(f) def wrap(*args, **kwargs): resource_id: str = "" - if kwargs.get("inbredset_id"): # data type: dataset-publish - resource_id = create_hmac( - data=("dataset-publish:" - f"{kwargs.get('inbredset_id')}:" - f"{kwargs.get('name')}"), - secret=current_app.config.get("SECRET_HMAC_CODE")) - if kwargs.get("dataset_name"): # data type: dataset-probe - resource_id = create_hmac( - data=("dataset-probeset:" - f"{kwargs.get('dataset_name')}"), - secret=current_app.config.get("SECRET_HMAC_CODE")) - if kwargs.get("resource_id"): # The resource_id is already provided + if request.args.get("resource-id"): + resource_id = request.args.get("resource-id") + elif kwargs.get("resource_id"): resource_id = kwargs.get("resource_id") response: Dict = {} try: - _user_id = g.user_session.record.get(b"user_id", - "").decode("utf-8") + _user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) response = json.loads( requests.get(urljoin( current_app.config.get("GN2_PROXY"), ("available?resource=" f"{resource_id}&user={_user_id}"))).content) + except: response = {} if max([DataRole(role) for role in response.get( @@ -78,8 +63,9 @@ def edit_admins_access_required(f): resource_id: str = kwargs.get("resource_id", "") response: Dict = {} try: - _user_id = g.user_session.record.get(b"user_id", - "").decode("utf-8") + _user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) response = json.loads( requests.get(urljoin( current_app.config.get("GN2_PROXY"), @@ -92,4 +78,3 @@ def edit_admins_access_required(f): return "You need to have edit-admins access", 401 return f(*args, **kwargs) return wrap - -- cgit v1.2.3 From 50553aa93b6fb0df15f30235c70cff7f08d93ef4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 17:21:21 +0300 Subject: Remove debug statement --- wqflask/wqflask/show_trait/show_trait.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index c4d1ae1c..d6355921 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -525,10 +525,6 @@ class ShowTrait: sample_group_type='primary', header="%s Only" % (self.dataset.group.name)) self.sample_groups = (primary_samples,) - print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") - print(self.sample_groups) - print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") - self.primary_sample_names = primary_sample_names self.dataset.group.allsamples = all_samples_ordered -- cgit v1.2.3 From a2a8f5da64a52d52354eded0b1b3adf149a234b3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 20:51:19 +0300 Subject: Use new "get_user_access_roles" when showing a trait * wqflask/wqflask/show_trait/show_trait.py (ShowTrait): Replace `check_owner_or_admin` with `get_user_access_roles` * wqflask/wqflask/views.py (show_temp_trait): Pass `user_id` as extra arg to ShowTrait. (show_trait_page): Ditto. --- wqflask/wqflask/show_trait/show_trait.py | 22 +++++++++++----------- wqflask/wqflask/views.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 13 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index d6355921..777efd02 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -20,9 +20,13 @@ from base import data_set from utility import helper_functions from utility.authentication_tools import check_owner_or_admin from utility.tools import locate_ignore_error +from utility.tools import GN_PROXY_URL from utility.redis_tools import get_redis_conn, get_resource_id from utility.logger import getLogger +from wqflask.access_roles import AdminRole +from wqflask.access_roles import DataRole +from wqflask.resource_manager import get_user_access_roles Redis = get_redis_conn() ONE_YEAR = 60 * 60 * 24 * 365 @@ -38,14 +42,11 @@ logger = getLogger(__name__) class ShowTrait: - def __init__(self, kw): + def __init__(self, user_id, kw): if 'trait_id' in kw and kw['dataset'] != "Temp": self.temp_trait = False self.trait_id = kw['trait_id'] helper_functions.get_species_dataset_trait(self, kw) - self.resource_id = get_resource_id(self.dataset, self.trait_id) - self.admin_status = check_owner_or_admin( - resource_id=self.resource_id) elif 'group' in kw: self.temp_trait = True self.trait_id = "Temp_" + kw['species'] + "_" + kw['group'] + \ @@ -62,9 +63,6 @@ class ShowTrait: self.this_trait = create_trait(dataset=self.dataset, name=self.trait_id, cellid=None) - - self.admin_status = check_owner_or_admin( - dataset=self.dataset, trait_id=self.trait_id) else: self.temp_trait = True self.trait_id = kw['trait_id'] @@ -75,11 +73,13 @@ class ShowTrait: self.this_trait = create_trait(dataset=self.dataset, name=self.trait_id, cellid=None) - self.trait_vals = Redis.get(self.trait_id).split() - self.admin_status = check_owner_or_admin( - dataset=self.dataset, trait_id=self.trait_id) - + self.resource_id = get_resource_id(self.dataset, + self.trait_id) + self.admin_status = get_user_access_roles( + user_id=user_id, + resource_id=(self.resource_id or ""), + gn_proxy_url=GN_PROXY_URL) # ZS: Get verify/rna-seq link URLs try: blatsequence = self.this_trait.blatseq diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 463b7c3a..6b00bf34 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -735,7 +735,11 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): logger.info(request.url) - template_vars = show_trait.ShowTrait(request.form) + user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + template_vars = show_trait.ShowTrait(user_id=user_id, + kw=request.form) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") @@ -745,7 +749,11 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): logger.info(request.url) - template_vars = show_trait.ShowTrait(request.args) + user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + template_vars = show_trait.ShowTrait(user_id=user_id, + kw=request.args) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") -- cgit v1.2.3 From 2b82f284efe60767c0e3f30e095094cee9c10a81 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 20:56:11 +0300 Subject: Delete noisy logging --- wqflask/wqflask/show_trait/show_trait.py | 4 ---- wqflask/wqflask/views.py | 2 -- 2 files changed, 6 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 777efd02..fa1206c9 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -22,7 +22,6 @@ from utility.authentication_tools import check_owner_or_admin from utility.tools import locate_ignore_error from utility.tools import GN_PROXY_URL from utility.redis_tools import get_redis_conn, get_resource_id -from utility.logger import getLogger from wqflask.access_roles import AdminRole from wqflask.access_roles import DataRole @@ -31,8 +30,6 @@ from wqflask.resource_manager import get_user_access_roles Redis = get_redis_conn() ONE_YEAR = 60 * 60 * 24 * 365 -logger = getLogger(__name__) - ############################################### # # Todo: Put in security to ensure that user has permission to access @@ -610,7 +607,6 @@ def get_nearest_marker(this_trait, this_db): GenoFreeze.Id = GenoXRef.GenoFreezeId AND GenoFreeze.Name = '{}' ORDER BY ABS( Geno.Mb - {}) LIMIT 1""".format(this_chr, this_db.group.name + "Geno", this_mb) - logger.sql(query) result = g.db.execute(query).fetchall() if result == []: diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 6b00bf34..08c88a25 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -734,7 +734,6 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): - logger.info(request.url) user_id = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or g.user_session.record.get("user_id", "")) @@ -748,7 +747,6 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): - logger.info(request.url) user_id = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or g.user_session.record.get("user_id", "")) -- cgit v1.2.3 From 46a9f5e522b96346034044b946ff85ac71197699 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 20:57:01 +0300 Subject: Make DataRole and AdminRole available to all jinja templates --- wqflask/wqflask/__init__.py | 11 +++++++++++ wqflask/wqflask/resource_manager.py | 3 +-- 2 files changed, 12 insertions(+), 2 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index a5097287..169192c7 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -9,6 +9,7 @@ from typing import Tuple from urllib.parse import urlparse from utility import formatting +from wqflask.access_roles import DataRole, AdminRole from wqflask.resource_manager import resource_management from wqflask.metadata_edits import metadata_edit @@ -70,6 +71,16 @@ def before_request(): g.request_time = lambda: "%.5fs" % (time.time() - g.request_start_time) +@app.context_processor +def include_admin_role_class(): + return {'AdminRole': AdminRole} + + +@app.context_processor +def include_data_role_class(): + return {'DataRole': DataRole} + + from wqflask.api import router from wqflask import group_manager from wqflask import resource_manager diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 3371e59d..e338a22d 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -147,8 +147,7 @@ def view_resource(resource_id: str): access_role=get_user_access_roles( resource_id=resource_id, user_id=user_id, - gn_proxy_url=current_app.config.get("GN2_PROXY")), - DataRole=DataRole, AdminRole=AdminRole) + gn_proxy_url=current_app.config.get("GN2_PROXY"))) @resource_management.route("/resources//make-public", -- cgit v1.2.3 From 3f0f6dfd00fe2cf302a7dac2c7507d82f02e8a35 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 20:58:31 +0300 Subject: show_trait_details.html: Update html links for editing metadata --- wqflask/wqflask/templates/show_trait_details.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 2a21dd24..f824d6e2 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -236,11 +236,12 @@ {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} {% if this_trait.dataset.type == 'Publish' %} - + {{ this_trait }} + {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} - + {% endif %} {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} -- cgit v1.2.3 From 5e3f63c45baba9ba8967144dc3bc4fbe0f798b07 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 21:01:05 +0300 Subject: show_trait_details.html: Use new link for managing privileges --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index f824d6e2..183c4c61 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -244,7 +244,7 @@ {% endif %} {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} - + {% endif %} {% endif %} -- cgit v1.2.3 From 8a3faf5a975895a05d0bf4c9c3a77079b1b6bf5a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 21:01:05 +0300 Subject: show_trait_details.html: Use new acccess roles in template --- wqflask/wqflask/templates/show_trait_details.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 183c4c61..b0639879 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -234,7 +234,7 @@ {% endif %} {% endif %} - {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + {% if admin_status.get('metadata', Datarole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} {{ this_trait }} @@ -243,7 +243,7 @@ {% if this_trait.dataset.type == 'ProbeSet' %} {% endif %} - {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% endif %} {% endif %} -- cgit v1.2.3 From 0e0da16c9bd5b0ea6366201cb34de6547dbe4080 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 22:37:04 +0300 Subject: metadata_edits: Add "POST //traits//update" --- wqflask/wqflask/metadata_edits.py | 136 +++++++++++++++++++++++++++++++++++++- wqflask/wqflask/views.py | 126 ----------------------------------- 2 files changed, 135 insertions(+), 127 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 94e2710b..bab1fa71 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -1,11 +1,13 @@ import MySQLdb import os import json +import datetime import difflib from collections import namedtuple -from flask import Blueprint, current_app, render_template, request +from flask import (Blueprint, current_app, redirect, + flash, g, render_template, request) from itertools import groupby from wqflask.decorators import edit_access_required @@ -21,6 +23,9 @@ from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import probeset_mapping +from gn3.commands import run_cmd +from gn3.db.traits import get_trait_csv_sample_data +from gn3.db.traits import update_sample_data metadata_edit = Blueprint('metadata_edit', __name__) @@ -125,6 +130,7 @@ def display_phenotype_metadata(dataset_id: str, name: str): publish_xref=_d.get("publish_xref"), phenotype=_d.get("phenotype"), publication=_d.get("publication"), + dataset_id=dataset_id, resource_id=request.args.get("resource-id"), version=os.environ.get("GN_VERSION"), ) @@ -145,3 +151,131 @@ def display_probeset_metadata(name: str): resource_id=request.args.get("resource-id"), version=os.environ.get("GN_VERSION"), ) + + +@metadata_edit.route("//traits//update", methods=("POST",)) +@edit_access_required +def update_phenotype(dataset_id: str, name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + data_ = request.form.to_dict() + TMPDIR = current_app.config.get("TMPDIR") + author = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + phenotype_id = str(data_.get('phenotype-id')) + if 'file' not in request.files: + flash("No sample-data has been uploaded", "warning") + else: + file_ = request.files['file'] + SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") + if not os.path.exists(SAMPLE_DATADIR): + os.makedirs(SAMPLE_DATADIR) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "diffs")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "diffs")) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "updated")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "updated")) + current_time = str(datetime.datetime.now().isoformat()) + new_file_name = (os.path.join(TMPDIR, + "sample-data/updated/", + (f"{author}." + f"{name}.{phenotype_id}." + f"{current_time}.csv"))) + uploaded_file_name = (os.path.join( + TMPDIR, + "sample-data/updated/", + (f"updated.{author}." + f"{request.args.get('resource-id')}." + f"{current_time}.csv"))) + file_.save(new_file_name) + publishdata_id = "" + lines = [] + with open(new_file_name, "r") as f: + lines = f.read() + first_line = lines.split('\n', 1)[0] + publishdata_id = first_line.split("Id:")[-1].strip() + with open(new_file_name, "w") as f: + f.write(lines.split("\n\n")[-1]) + csv_ = get_trait_csv_sample_data(conn=conn, + trait_name=str(name), + phenotype_id=str(phenotype_id)) + with open(uploaded_file_name, "w") as f_: + f_.write(csv_.split("\n\n")[-1]) + r = run_cmd(cmd=("csvdiff " + f"'{uploaded_file_name}' '{new_file_name}' " + "--format json")) + diff_output = (f"{TMPDIR}/sample-data/diffs/" + f"{author}.{request.args.get('resource-id')}." + f"{current_time}.json") + with open(diff_output, "w") as f: + dict_ = json.loads(r.get("output")) + dict_.update({ + "author": author, + "publishdata_id": publishdata_id, + "dataset_id": data_.get("dataset-name"), + "timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S") + }) + f.write(json.dumps(dict_)) + flash("Sample-data has been successfully uploaded", "success") + # Run updates: + phenotype_ = { + "pre_pub_description": data_.get("pre-pub-desc"), + "post_pub_description": data_.get("post-pub-desc"), + "original_description": data_.get("orig-desc"), + "units": data_.get("units"), + "pre_pub_abbreviation": data_.get("pre-pub-abbrev"), + "post_pub_abbreviation": data_.get("post-pub-abbrev"), + "lab_code": data_.get("labcode"), + "submitter": data_.get("submitter"), + "owner": data_.get("owner"), + "authorized_users": data_.get("authorized-users"), + } + updated_phenotypes = update( + conn, "Phenotype", + data=Phenotype(**phenotype_), + where=Phenotype(id_=data_.get("phenotype-id"))) + diff_data = {} + if updated_phenotypes: + diff_data.update({"Phenotype": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in phenotype_.items() + if v is not None}, new=phenotype_)}) + publication_ = { + "abstract": data_.get("abstract"), + "authors": data_.get("authors"), + "title": data_.get("title"), + "journal": data_.get("journal"), + "volume": data_.get("volume"), + "pages": data_.get("pages"), + "month": data_.get("month"), + "year": data_.get("year") + } + updated_publications = update( + conn, "Publication", + data=Publication(**publication_), + where=Publication(id_=data_.get("pubmed-id", + data_.get("old_id_")))) + if updated_publications: + diff_data.update({"Publication": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in publication_.items() + if v is not None}, new=publication_)}) + if diff_data: + diff_data.update({"dataset_id": name}) + diff_data.update({"resource_id": request.args.get('resource-id')}) + diff_data.update({"author": author}) + diff_data.update({"timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S")}) + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=name, + editor=author, + json_data=json.dumps(diff_data))) + flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") + return redirect(f"/datasets/{dataset_id}/traits/{name}/edit" + f"?resource-id={request.args.get('resource-id')}") diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 08c88a25..963e48b7 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -416,132 +416,6 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/trait/update", methods=["POST"]) -@edit_access_required -def update_phenotype(): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - data_ = request.form.to_dict() - TMPDIR = current_app.config.get("TMPDIR") - author = g.user_session.record.get(b'user_name') - if 'file' not in request.files: - flash("No sample-data has been uploaded", "warning") - else: - file_ = request.files['file'] - trait_name = str(data_.get('dataset-name')) - phenotype_id = str(data_.get('phenotype-id', 35)) - SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") - if not os.path.exists(SAMPLE_DATADIR): - os.makedirs(SAMPLE_DATADIR) - if not os.path.exists(os.path.join(SAMPLE_DATADIR, - "diffs")): - os.makedirs(os.path.join(SAMPLE_DATADIR, - "diffs")) - if not os.path.exists(os.path.join(SAMPLE_DATADIR, - "updated")): - os.makedirs(os.path.join(SAMPLE_DATADIR, - "updated")) - current_time = str(datetime.datetime.now().isoformat()) - new_file_name = (os.path.join(TMPDIR, - "sample-data/updated/", - (f"{author.decode('utf-8')}." - f"{trait_name}.{phenotype_id}." - f"{current_time}.csv"))) - uploaded_file_name = (os.path.join( - TMPDIR, - "sample-data/updated/", - (f"updated.{author.decode('utf-8')}." - f"{trait_name}.{phenotype_id}." - f"{current_time}.csv"))) - file_.save(new_file_name) - publishdata_id = "" - lines = [] - with open(new_file_name, "r") as f: - lines = f.read() - first_line = lines.split('\n', 1)[0] - publishdata_id = first_line.split("Id:")[-1].strip() - with open(new_file_name, "w") as f: - f.write(lines.split("\n\n")[-1]) - csv_ = get_trait_csv_sample_data(conn=conn, - trait_name=str(trait_name), - phenotype_id=str(phenotype_id)) - with open(uploaded_file_name, "w") as f_: - f_.write(csv_.split("\n\n")[-1]) - r = run_cmd(cmd=("csvdiff " - f"'{uploaded_file_name}' '{new_file_name}' " - "--format json")) - diff_output = (f"{TMPDIR}/sample-data/diffs/" - f"{trait_name}.{author.decode('utf-8')}." - f"{phenotype_id}.{current_time}.json") - with open(diff_output, "w") as f: - dict_ = json.loads(r.get("output")) - dict_.update({ - "author": author.decode('utf-8'), - "publishdata_id": publishdata_id, - "dataset_id": data_.get("dataset-name"), - "timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S") - }) - f.write(json.dumps(dict_)) - flash("Sample-data has been successfully uploaded", "success") - # Run updates: - phenotype_ = { - "pre_pub_description": data_.get("pre-pub-desc"), - "post_pub_description": data_.get("post-pub-desc"), - "original_description": data_.get("orig-desc"), - "units": data_.get("units"), - "pre_pub_abbreviation": data_.get("pre-pub-abbrev"), - "post_pub_abbreviation": data_.get("post-pub-abbrev"), - "lab_code": data_.get("labcode"), - "submitter": data_.get("submitter"), - "owner": data_.get("owner"), - "authorized_users": data_.get("authorized-users"), - } - updated_phenotypes = update( - conn, "Phenotype", - data=Phenotype(**phenotype_), - where=Phenotype(id_=data_.get("phenotype-id"))) - diff_data = {} - if updated_phenotypes: - diff_data.update({"Phenotype": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in phenotype_.items() - if v is not None}, new=phenotype_)}) - publication_ = { - "abstract": data_.get("abstract"), - "authors": data_.get("authors"), - "title": data_.get("title"), - "journal": data_.get("journal"), - "volume": data_.get("volume"), - "pages": data_.get("pages"), - "month": data_.get("month"), - "year": data_.get("year") - } - updated_publications = update( - conn, "Publication", - data=Publication(**publication_), - where=Publication(id_=data_.get("pubmed-id", - data_.get("old_id_")))) - if updated_publications: - diff_data.update({"Publication": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in publication_.items() - if v is not None}, new=publication_)}) - if diff_data: - diff_data.update({"dataset_id": data_.get("dataset-name")}) - diff_data.update({"author": author.decode('utf-8')}) - diff_data.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) - insert(conn, - table="metadata_audit", - data=MetadataAudit(dataset_id=data_.get("dataset-name"), - editor=author.decode("utf-8"), - json_data=json.dumps(diff_data))) - flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") - return redirect(f"/trait/{data_.get('dataset-name')}" - f"/edit/inbredset-id/{data_.get('inbred-set-id')}") - - @app.route("/probeset/update", methods=["POST"]) def update_probeset(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), -- cgit v1.2.3 From e640c85bdb5174d86f2ba5adf112bbf9259fdc94 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 22:38:05 +0300 Subject: edit_phenotype.html: Update form element for submitting POST data --- wqflask/wqflask/templates/edit_phenotype.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 7a841793..5166a903 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -62,8 +62,7 @@ {% endif %} - -
+

Trait Information:

@@ -226,7 +225,6 @@
- -- cgit v1.2.3 From 0618b9f78d6ca2517f6265a9cfe7db64b6ae12ef Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:20:20 +0300 Subject: Default to an empty string if a session_id is empty "user_id" is jumbled up; it's either bytes or plain strings. This correctly parses it. --- wqflask/wqflask/decorators.py | 23 +++++++++++------------ wqflask/wqflask/metadata_edits.py | 5 ++--- 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index a4ff7ce3..0c3c2a89 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -16,9 +16,9 @@ def login_required(f): """Use this for endpoints where login is required""" @wraps(f) def wrap(*args, **kwargs): - user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") redis_conn = redis.from_url(current_app.config["REDIS_URL"], decode_responses=True) if not redis_conn.hget("users", user_id): @@ -38,15 +38,14 @@ def edit_access_required(f): resource_id = kwargs.get("resource_id") response: Dict = {} try: - _user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") response = json.loads( requests.get(urljoin( current_app.config.get("GN2_PROXY"), ("available?resource=" - f"{resource_id}&user={_user_id}"))).content) - + f"{resource_id}&user={user_id}"))).content) except: response = {} if max([DataRole(role) for role in response.get( @@ -63,14 +62,14 @@ def edit_admins_access_required(f): resource_id: str = kwargs.get("resource_id", "") response: Dict = {} try: - _user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") response = json.loads( requests.get(urljoin( current_app.config.get("GN2_PROXY"), ("available?resource=" - f"{resource_id}&user={_user_id}"))).content) + f"{resource_id}&user={user_id}"))).content) except: response = {} if max([AdminRole(role) for role in response.get( diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index bab1fa71..7e237976 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -162,9 +162,8 @@ def update_phenotype(dataset_id: str, name: str): host=current_app.config.get("DB_HOST")) data_ = request.form.to_dict() TMPDIR = current_app.config.get("TMPDIR") - author = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") phenotype_id = str(data_.get('phenotype-id')) if 'file' not in request.files: flash("No sample-data has been uploaded", "warning") -- cgit v1.2.3 From fb7456a0e850290961c0f8be59c09fdd425f15f1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:22:53 +0300 Subject: Rename metadata edit endpoints --- wqflask/wqflask/metadata_edits.py | 8 ++++---- wqflask/wqflask/templates/edit_phenotype.html | 2 +- wqflask/wqflask/templates/edit_probeset.html | 2 +- wqflask/wqflask/templates/show_trait_details.html | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 7e237976..08ff4704 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -116,7 +116,7 @@ def edit_probeset(conn, name): } -@metadata_edit.route("//traits//edit") +@metadata_edit.route("//traits/") @edit_access_required def display_phenotype_metadata(dataset_id: str, name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), @@ -136,7 +136,7 @@ def display_phenotype_metadata(dataset_id: str, name: str): ) -@metadata_edit.route("/traits//edit") +@metadata_edit.route("/traits/") @edit_access_required def display_probeset_metadata(name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), @@ -153,7 +153,7 @@ def display_probeset_metadata(name: str): ) -@metadata_edit.route("//traits//update", methods=("POST",)) +@metadata_edit.route("//traits/", methods=("POST",)) @edit_access_required def update_phenotype(dataset_id: str, name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), @@ -276,5 +276,5 @@ def update_phenotype(dataset_id: str, name: str): editor=author, json_data=json.dumps(diff_data))) flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") - return redirect(f"/datasets/{dataset_id}/traits/{name}/edit" + return redirect(f"/datasets/{dataset_id}/traits/{name}" f"?resource-id={request.args.get('resource-id')}") diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 5166a903..6b08163e 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -62,7 +62,7 @@
{% endif %} - +

Trait Information:

diff --git a/wqflask/wqflask/templates/edit_probeset.html b/wqflask/wqflask/templates/edit_probeset.html index 85d49561..cd7b1379 100644 --- a/wqflask/wqflask/templates/edit_probeset.html +++ b/wqflask/wqflask/templates/edit_probeset.html @@ -53,7 +53,7 @@ Submit Trait | Reset {% endif %} - +Probeset Information:
diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index b0639879..7a09f52e 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -237,11 +237,11 @@ {% if admin_status.get('metadata', Datarole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} {{ this_trait }} - + {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} - + {% endif %} {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} -- cgit v1.2.3 From d15d381fd9018e9883d17046496250321327763b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:23:15 +0300 Subject: Fix typo --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 7a09f52e..b4b1860d 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -234,7 +234,7 @@ {% endif %} {% endif %} - {% if admin_status.get('metadata', Datarole.VIEW) > DataRole.VIEW %} + {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} {{ this_trait }} -- cgit v1.2.3 From c125dfbbc34a78dda7b5b9a62763aacbf52f98ff Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:23:24 +0300 Subject: Use correct predicate to see if a resource belongs to anyone --- wqflask/wqflask/templates/admin/manage_resource.html | 2 +- wqflask/wqflask/views.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 613aa70e..9ca75906 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -8,7 +8,7 @@ {% set METADATA_ACCESS = access_role.get('metadata') %} {% set ADMIN_STATUS = access_role.get('admin') %}

Resource Manager

- {% if resource_info.get('owner_id') != 'none'%} + {% if resource_info.get('owner_id') %} {% set user_details = resource_info.get('owner_details') %}

Current Owner: {{ user_details.get('full_name') }} diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 963e48b7..665aabb0 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -608,9 +608,8 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): - user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") template_vars = show_trait.ShowTrait(user_id=user_id, kw=request.form) template_vars.js_data = json.dumps(template_vars.js_data, @@ -621,9 +620,8 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): - user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") template_vars = show_trait.ShowTrait(user_id=user_id, kw=request.args) template_vars.js_data = json.dumps(template_vars.js_data, -- cgit v1.2.3 From c8938c9b3d9318f257083857aa9a0f79e3fbe220 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:25:26 +0300 Subject: manage_resource.html: Add missing opening tag --- wqflask/wqflask/templates/admin/manage_resource.html | 1 + 1 file changed, 1 insertion(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 9ca75906..5f6140f0 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -3,6 +3,7 @@ {% block content %}
+
{{ flash_me() }} {% set DATA_ACCESS = access_role.get('data') %} {% set METADATA_ACCESS = access_role.get('metadata') %} -- cgit v1.2.3 From 106a6a4932151347318fd22ccfe87c2fe2ac70d6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:26:04 +0300 Subject: manage_resource.html: Set ADMIN_STATUS variable --- wqflask/wqflask/templates/admin/manage_resource.html | 1 + 1 file changed, 1 insertion(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 5f6140f0..46753f01 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -27,6 +27,7 @@ {% endif %} {% endif %}
+ {% set ADMIN_STATUS = access_role.get('admin') %}
-- cgit v1.2.3 From ce864679a5724ce55caccdf9c520a81da52d8f75 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:26:24 +0300 Subject: show_trait_details.html: Delete debug variable --- wqflask/wqflask/templates/show_trait_details.html | 1 - 1 file changed, 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index b4b1860d..fa892853 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -236,7 +236,6 @@ {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} - {{ this_trait }} {% endif %} -- cgit v1.2.3 From d49536e8ec945d852a713d36e1b6db51d951c295 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:26:56 +0300 Subject: manage_resource.html: Apply indentation --- .../wqflask/templates/admin/manage_resource.html | 76 +++++++++++----------- 1 file changed, 38 insertions(+), 38 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 46753f01..64d4b6eb 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -5,29 +5,29 @@
{{ flash_me() }} - {% set DATA_ACCESS = access_role.get('data') %} - {% set METADATA_ACCESS = access_role.get('metadata') %} - {% set ADMIN_STATUS = access_role.get('admin') %} -

Resource Manager

- {% if resource_info.get('owner_id') %} - {% set user_details = resource_info.get('owner_details') %} -

- Current Owner: {{ user_details.get('full_name') }} -

- {% if user_details.get('organization') %} -

- Organization: {{ user_details.get('organization')}} -

- {% endif %} - {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} - - Change Owner - - {% endif %} - {% endif %} -
+ {% set DATA_ACCESS = access_role.get('data') %} + {% set METADATA_ACCESS = access_role.get('metadata') %} {% set ADMIN_STATUS = access_role.get('admin') %} + {% set ADMIN_STATUS = access_role.get('admin') %} +

Resource Manager

+ {% if resource_info.get('owner_id') %} + {% set user_details = resource_info.get('owner_details') %} +

+ Current Owner: {{ user_details.get('full_name') }} +

+ {% if user_details.get('organization') %} +

+ Organization: {{ user_details.get('organization')}} +

+ {% endif %} + {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} + + Change Owner + + {% endif %} + {% endif %} +
@@ -53,7 +53,7 @@ +

@@ -100,25 +100,25 @@
{% endif %} - + - + -{% endblock %} -{% block js %} + {% endblock %} + {% block js %} -{% endblock %} + {% endblock %} -- cgit v1.2.3 From 7547801f579ded3b8f94d98bc954a530d5275682 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:27:28 +0300 Subject: decorators.py: Auto-pep8 file --- wqflask/wqflask/decorators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 0c3c2a89..1ef8c188 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -28,7 +28,8 @@ def login_required(f): def edit_access_required(f): - """Use this for endpoints where people with admin or edit privileges are required""" + """Use this for endpoints where people with admin or edit privileges +are required""" @wraps(f) def wrap(*args, **kwargs): resource_id: str = "" -- cgit v1.2.3 From 4bc8c4d2b5fcfc164026e9a924b3b784af8957ea Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:28:12 +0300 Subject: Move endpoint for updating probeset to metadata_edits.py as a bp --- wqflask/wqflask/metadata_edits.py | 59 +++++++++++++++++++++++++++++++++++++++ wqflask/wqflask/views.py | 58 -------------------------------------- 2 files changed, 59 insertions(+), 58 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 08ff4704..ef9dc127 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -278,3 +278,62 @@ def update_phenotype(dataset_id: str, name: str): flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") return redirect(f"/datasets/{dataset_id}/traits/{name}" f"?resource-id={request.args.get('resource-id')}") + + +@metadata_edit.route("/traits/", methods=["POST"]) +@edit_access_required +def update_probeset(name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + data_ = request.form.to_dict() + probeset_ = { + "id_": data_.get("id"), + "symbol": data_.get("symbol"), + "description": data_.get("description"), + "probe_target_description": data_.get("probe_target_description"), + "chr_": data_.get("chr"), + "mb": data_.get("mb"), + "alias": data_.get("alias"), + "geneid": data_.get("geneid"), + "homologeneid": data_.get("homologeneid"), + "unigeneid": data_.get("unigeneid"), + "omim": data_.get("OMIM"), + "refseq_transcriptid": data_.get("refseq_transcriptid"), + "blatseq": data_.get("blatseq"), + "targetseq": data_.get("targetseq"), + "strand_probe": data_.get("Strand_Probe"), + "probe_set_target_region": data_.get("probe_set_target_region"), + "probe_set_specificity": data_.get("probe_set_specificity"), + "probe_set_blat_score": data_.get("probe_set_blat_score"), + "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), + "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), + "probe_set_strand": data_.get("probe_set_strand"), + "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), + "flag": data_.get("flag") + } + diff_data = {} + author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + if (updated_probeset := update( + conn, "ProbeSet", + data=Probeset(**probeset_), + where=Probeset(id_=data_.get("id")))): + diff_data.update({"Probeset": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in probeset_.items() + if v is not None}, new=probeset_)}) + if diff_data: + diff_data.update({"probeset_name": data_.get("probeset_name")}) + diff_data.update({"author": author}) + diff_data.update({"resource_id": request.args.get('resource-id')}) + diff_data.update({"timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S")}) + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=data_.get("id"), + editor=author.decode("utf-8"), + json_data=json.dumps(diff_data))) + return redirect(f"/datasets/traits/{name}" + f"?resource-id={request.args.get('resource-id')}") + diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 665aabb0..220d9b87 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -416,64 +416,6 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/probeset/update", methods=["POST"]) -def update_probeset(): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - data_ = request.form.to_dict() - probeset_ = { - "id_": data_.get("id"), - "symbol": data_.get("symbol"), - "description": data_.get("description"), - "probe_target_description": data_.get("probe_target_description"), - "chr_": data_.get("chr"), - "mb": data_.get("mb"), - "alias": data_.get("alias"), - "geneid": data_.get("geneid"), - "homologeneid": data_.get("homologeneid"), - "unigeneid": data_.get("unigeneid"), - "omim": data_.get("OMIM"), - "refseq_transcriptid": data_.get("refseq_transcriptid"), - "blatseq": data_.get("blatseq"), - "targetseq": data_.get("targetseq"), - "strand_probe": data_.get("Strand_Probe"), - "probe_set_target_region": data_.get("probe_set_target_region"), - "probe_set_specificity": data_.get("probe_set_specificity"), - "probe_set_blat_score": data_.get("probe_set_blat_score"), - "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), - "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), - "probe_set_strand": data_.get("probe_set_strand"), - "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), - "flag": data_.get("flag") - } - updated_probeset = update( - conn, "ProbeSet", - data=Probeset(**probeset_), - where=Probeset(id_=data_.get("id"))) - - diff_data = {} - author = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) - if updated_probeset: - diff_data.update({"Probeset": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in probeset_.items() - if v is not None}, new=probeset_)}) - if diff_data: - diff_data.update({"probeset_name": data_.get("probeset_name")}) - diff_data.update({"author": author.decode('utf-8')}) - diff_data.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) - insert(conn, - table="metadata_audit", - data=MetadataAudit(dataset_id=data_.get("id"), - editor=author.decode("utf-8"), - json_data=json.dumps(diff_data))) - return redirect(f"/trait/edit/probeset-name/{data_.get('probeset_name')}") - - @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): logger.info(request.url) -- cgit v1.2.3 From 77865c2ffbc921abb2fe04a64d03e51d47b2ecce Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:34:11 +0300 Subject: show_trait_details.html: Fix broken "edit-phenotype" link --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index fa892853..3e59a3ee 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -240,7 +240,7 @@ {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} - + {% endif %} {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} -- cgit v1.2.3 From f7e18594b5a60ea966a38fc3424c32d104ce7a77 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:04:02 +0300 Subject: edit_probeset.html: Apply indentation --- wqflask/wqflask/templates/edit_probeset.html | 70 ++++++++++++++-------------- 1 file changed, 35 insertions(+), 35 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/edit_probeset.html b/wqflask/wqflask/templates/edit_probeset.html index cd7b1379..ab91b701 100644 --- a/wqflask/wqflask/templates/edit_probeset.html +++ b/wqflask/wqflask/templates/edit_probeset.html @@ -9,52 +9,52 @@ Submit Trait | Reset
- -

Update History

- - - - - - - - - - {% set ns = namespace(display_cell=True) %} + +

Update History

+ +
TimestampEditorFieldDiff
+ + + + + + + + {% set ns = namespace(display_cell=True) %} - {% for timestamp, group in diff %} - {% set ns.display_cell = True %} - {% for i in group %} - - {% if ns.display_cell and i.timestamp == timestamp %} + {% for timestamp, group in diff %} + {% set ns.display_cell = True %} + {% for i in group %} + + {% if ns.display_cell and i.timestamp == timestamp %} - {% set author = i.author %} - {% set timestamp_ = i.timestamp %} + {% set author = i.author %} + {% set timestamp_ = i.timestamp %} - {% else %} + {% else %} - {% set author = "" %} - {% set timestamp_ = "" %} + {% set author = "" %} + {% set timestamp_ = "" %} - {% endif %} - - - - - {% set ns.display_cell = False %} - - {% endfor %} - {% endfor %} - -
TimestampEditorFieldDiff
{{ timestamp_ }}{{ author }}{{ i.diff.field }}
{{ i.diff.diff }}
+ {% endif %} + {{ timestamp_ }} + {{ author }} + {{ i.diff.field }} + 
{{ i.diff.diff }}
+ {% set ns.display_cell = False %} + + {% endfor %} + {% endfor %} + +
{% endif %} -
Probeset Information: + +

Probeset Information:

-- cgit v1.2.3 From b0946c079d0d4b953285c3f5781482a3d7a9f87d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:04:42 +0300 Subject: Fix html --- wqflask/wqflask/templates/edit_phenotype.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 6b08163e..c3cde391 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -62,7 +62,7 @@
{% endif %} - +

Trait Information:

-- cgit v1.2.3 From 178aae274982b8d7bc7720310e53c60dc35b9701 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:04:57 +0300 Subject: Remove decoding when parsing author variable --- wqflask/wqflask/metadata_edits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index ef9dc127..b03c999e 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -332,7 +332,7 @@ def update_probeset(name: str): insert(conn, table="metadata_audit", data=MetadataAudit(dataset_id=data_.get("id"), - editor=author.decode("utf-8"), + editor=author, json_data=json.dumps(diff_data))) return redirect(f"/datasets/traits/{name}" f"?resource-id={request.args.get('resource-id')}") -- cgit v1.2.3 From bf71e24e7e471c99b650c5250d0ee9e692e4a6fd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:05:18 +0300 Subject: Pass "name" to "edit_probeset.html" template --- wqflask/wqflask/metadata_edits.py | 1 + 1 file changed, 1 insertion(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index b03c999e..690f04c2 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -148,6 +148,7 @@ def display_probeset_metadata(name: str): "edit_probeset.html", diff=_d.get("diff"), probeset=_d.get("probeset"), + name=name, resource_id=request.args.get("resource-id"), version=os.environ.get("GN_VERSION"), ) -- cgit v1.2.3 From ea1de9bffd36dbd7417ff24b9a0caab1e3449081 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:05:47 +0300 Subject: Prefer tuples over lists --- wqflask/wqflask/metadata_edits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 690f04c2..d232b32b 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -281,7 +281,7 @@ def update_phenotype(dataset_id: str, name: str): f"?resource-id={request.args.get('resource-id')}") -@metadata_edit.route("/traits/", methods=["POST"]) +@metadata_edit.route("/traits/", methods=("POST",)) @edit_access_required def update_probeset(name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), -- cgit v1.2.3 From 230184bf1484ed672bf66c29110bcb47e556f72f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 10:57:17 +0300 Subject: Remove the use of '@deprecated' This causes noisy logging. --- wqflask/utility/authentication_tools.py | 2 -- wqflask/utility/hmac.py | 3 --- wqflask/utility/redis_tools.py | 2 -- 3 files changed, 7 deletions(-) (limited to 'wqflask') diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index c4801c8c..afea69e1 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -1,7 +1,6 @@ import json import requests -from deprecated import deprecated from flask import g from base import webqtlConfig @@ -127,7 +126,6 @@ def check_owner(dataset=None, trait_id=None, resource_id=None): return False -@deprecated def check_owner_or_admin(dataset=None, trait_id=None, resource_id=None): if not resource_id: if dataset.type == "Temp": diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py index d6e515ed..29891677 100644 --- a/wqflask/utility/hmac.py +++ b/wqflask/utility/hmac.py @@ -1,14 +1,11 @@ import hmac import hashlib -from deprecated import deprecated from flask import url_for from wqflask import app -@deprecated("This function leads to circular imports. " - "If possible use wqflask.decorators.create_hmac instead.") def hmac_creation(stringy): """Helper function to create the actual hmac""" diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index c2a3b057..a6c5875f 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -4,7 +4,6 @@ import datetime import redis # used for collections -from deprecated import deprecated from utility.hmac import hmac_creation from utility.logger import getLogger logger = getLogger(__name__) @@ -252,7 +251,6 @@ def get_resource_id(dataset, trait_id=None): return resource_id -@deprecated def get_resource_info(resource_id): resource_info = Redis.hget("resources", resource_id) if resource_info: -- cgit v1.2.3 From 03caa57ad209f3bdd135be9d6516b94261c9b8de Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 11:05:19 +0300 Subject: Remove all elasticsearch references in gn2 --- bin/genenetwork2 | 7 - doc/elasticsearch.org | 247 ------------------------------ test/requests/parametrized_test.py | 32 ---- test/requests/test-website.py | 1 - test/requests/test_forgot_password.py | 29 +--- test/requests/test_registration.py | 36 ++--- wqflask/maintenance/quantile_normalize.py | 18 --- wqflask/utility/elasticsearch_tools.py | 121 --------------- wqflask/utility/tools.py | 5 +- wqflask/wqflask/user_session.py | 1 - 10 files changed, 23 insertions(+), 474 deletions(-) delete mode 100644 doc/elasticsearch.org delete mode 100644 test/requests/parametrized_test.py delete mode 100644 wqflask/utility/elasticsearch_tools.py (limited to 'wqflask') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 2b94b2a2..5f714d2e 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -101,13 +101,6 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead -# if [ -z $ELASTICSEARCH_PROFILE ]; then -# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; -# else -# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python3.8/site-packages" -# fi - if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) diff --git a/doc/elasticsearch.org b/doc/elasticsearch.org deleted file mode 100644 index 864a8363..00000000 --- a/doc/elasticsearch.org +++ /dev/null @@ -1,247 +0,0 @@ -* Elasticsearch - -** Introduction - -GeneNetwork uses elasticsearch (ES) for all things considered -'state'. One example is user collections, another is user management. - -** Example - -To get the right environment, first you can get a python REPL with something like - -: env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ../etc/default_settings.py -cli python - -(make sure to use the correct GN2_PROFILE!) - -Next try - -#+BEGIN_SRC python - -from elasticsearch import Elasticsearch, TransportError - -es = Elasticsearch([{ "host": 'localhost', "port": '9200' }]) - -# Dump all data - -es.search("*") - -# To fetch an E-mail record from the users index - -record = es.search( - index = 'users', doc_type = 'local', body = { - "query": { "match": { "email_address": "myname@email.com" } } - }) - -# It is also possible to do wild card matching - -q = { "query": { "wildcard" : { "full_name" : "pjot*" } }} -es.search(index = 'users', doc_type = 'local', body = q) - -# To get elements from that record: - -record['hits']['hits'][0][u'_source']['full_name'] -u'Pjotr' - -record['hits']['hits'][0][u'_source']['email_address'] -u"myname@email.com" - -#+END_SRC - -** Health - -ES provides support for checking its health: - -: curl -XGET http://localhost:9200/_cluster/health?pretty=true - -#+BEGIN_SRC json - - - { - "cluster_name" : "asgard", - "status" : "yellow", - "timed_out" : false, - "number_of_nodes" : 1, - "number_of_data_nodes" : 1, - "active_primary_shards" : 5, - "active_shards" : 5, - "relocating_shards" : 0, - "initializing_shards" : 0, - "unassigned_shards" : 5 - } - -#+END_SRC - -Yellow means just one instance is running (no worries). - -To get full cluster info - -: curl -XGET "localhost:9200/_cluster/stats?human&pretty" - -#+BEGIN_SRC json -{ - "_nodes" : { - "total" : 1, - "successful" : 1, - "failed" : 0 - }, - "cluster_name" : "elasticsearch", - "timestamp" : 1529050366452, - "status" : "yellow", - "indices" : { - "count" : 3, - "shards" : { - "total" : 15, - "primaries" : 15, - "replication" : 0.0, - "index" : { - "shards" : { - "min" : 5, - "max" : 5, - "avg" : 5.0 - }, - "primaries" : { - "min" : 5, - "max" : 5, - "avg" : 5.0 - }, - "replication" : { - "min" : 0.0, - "max" : 0.0, - "avg" : 0.0 - } - } - }, - "docs" : { - "count" : 14579, - "deleted" : 0 - }, - "store" : { - "size" : "44.7mb", - "size_in_bytes" : 46892794 - }, - "fielddata" : { - "memory_size" : "0b", - "memory_size_in_bytes" : 0, - "evictions" : 0 - }, - "query_cache" : { - "memory_size" : "0b", - "memory_size_in_bytes" : 0, - "total_count" : 0, - "hit_count" : 0, - "miss_count" : 0, - "cache_size" : 0, - "cache_count" : 0, - "evictions" : 0 - }, - "completion" : { - "size" : "0b", - "size_in_bytes" : 0 - }, - "segments" : { - "count" : 24, - "memory" : "157.3kb", - "memory_in_bytes" : 161112, - "terms_memory" : "122.6kb", - "terms_memory_in_bytes" : 125569, - "stored_fields_memory" : "15.3kb", - "stored_fields_memory_in_bytes" : 15728, - "term_vectors_memory" : "0b", - "term_vectors_memory_in_bytes" : 0, - "norms_memory" : "10.8kb", - "norms_memory_in_bytes" : 11136, - "points_memory" : "111b", - "points_memory_in_bytes" : 111, - "doc_values_memory" : "8.3kb", - "doc_values_memory_in_bytes" : 8568, - "index_writer_memory" : "0b", - "index_writer_memory_in_bytes" : 0, - "version_map_memory" : "0b", - "version_map_memory_in_bytes" : 0, - "fixed_bit_set" : "0b", - "fixed_bit_set_memory_in_bytes" : 0, - "max_unsafe_auto_id_timestamp" : -1, - "file_sizes" : { } - } - }, - "nodes" : { - "count" : { - "total" : 1, - "data" : 1, - "coordinating_only" : 0, - "master" : 1, - "ingest" : 1 - }, - "versions" : [ - "6.2.1" - ], - "os" : { - "available_processors" : 16, - "allocated_processors" : 16, - "names" : [ - { - "name" : "Linux", - "count" : 1 - } - ], - "mem" : { - "total" : "125.9gb", - "total_in_bytes" : 135189286912, - "free" : "48.3gb", - "free_in_bytes" : 51922628608, - "used" : "77.5gb", - "used_in_bytes" : 83266658304, - "free_percent" : 38, - "used_percent" : 62 - } - }, - "process" : { - "cpu" : { - "percent" : 0 - }, - "open_file_descriptors" : { - "min" : 415, - "max" : 415, - "avg" : 415 - } - }, - "jvm" : { - "max_uptime" : "1.9d", - "max_uptime_in_millis" : 165800616, - "versions" : [ - { - "version" : "9.0.4", - "vm_name" : "OpenJDK 64-Bit Server VM", - "vm_version" : "9.0.4+11", - "vm_vendor" : "Oracle Corporation", - "count" : 1 - } - ], - "mem" : { - "heap_used" : "1.1gb", - "heap_used_in_bytes" : 1214872032, - "heap_max" : "23.8gb", - "heap_max_in_bytes" : 25656426496 - }, - "threads" : 110 - }, - "fs" : { - "total" : "786.4gb", - "total_in_bytes" : 844400918528, - "free" : "246.5gb", - "free_in_bytes" : 264688160768, - "available" : "206.5gb", - "available_in_bytes" : 221771468800 - }, - "plugins" : [ ], - "network_types" : { - "transport_types" : { - "netty4" : 1 - }, - "http_types" : { - "netty4" : 1 - } - } - } -} -#+BEGIN_SRC json diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py deleted file mode 100644 index 50003850..00000000 --- a/test/requests/parametrized_test.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging -import unittest -from wqflask import app -from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column -from elasticsearch import Elasticsearch, TransportError - -class ParametrizedTest(unittest.TestCase): - - def __init__(self, methodName='runTest', gn2_url="http://localhost:5003", es_url="localhost:9200"): - super(ParametrizedTest, self).__init__(methodName=methodName) - self.gn2_url = gn2_url - self.es_url = es_url - - def setUp(self): - self.es = get_elasticsearch_connection() - self.es_cleanup = [] - - es_logger = logging.getLogger("elasticsearch") - es_logger.setLevel(app.config.get("LOG_LEVEL")) - es_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) - es_trace_logger = logging.getLogger("elasticsearch.trace") - es_trace_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) - - def tearDown(self): - from time import sleep - self.es.delete_by_query( - index="users" - , doc_type="local" - , body={"query":{"match":{"email_address":"test@user.com"}}}) - sleep(1) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 8bfb47c2..d619a7d5 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -43,7 +43,6 @@ def dummy(args_obj, parser): def integration_tests(args_obj, parser): gn2_url = args_obj.host - es_url = app.config.get("ELASTICSEARCH_HOST")+":"+str(app.config.get("ELASTICSEARCH_PORT")) run_integration_tests(gn2_url, es_url) def initTest(klass, gn2_url, es_url): diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py index 346524bc..65b061f8 100644 --- a/test/requests/test_forgot_password.py +++ b/test/requests/test_forgot_password.py @@ -1,25 +1,22 @@ import requests -from utility.elasticsearch_tools import get_user_by_unique_column from parameterized import parameterized from parametrized_test import ParametrizedTest passwork_reset_link = '' forgot_password_page = None -class TestForgotPassword(ParametrizedTest): +class TestForgotPassword(ParametrizedTest): def setUp(self): super(TestForgotPassword, self).setUp() self.forgot_password_url = self.gn2_url+"/n/forgot_password_submit" + def send_email(to_addr, msg, fromaddr="no-reply@genenetwork.org"): print("CALLING: send_email_mock()") email_data = { - "to_addr": to_addr - , "msg": msg - , "fromaddr": from_addr} + "to_addr": to_addr, "msg": msg, "fromaddr": from_addr} data = { - "es_connection": self.es, "email_address": "test@user.com", "full_name": "Test User", "organization": "Test Organisation", @@ -27,24 +24,12 @@ class TestForgotPassword(ParametrizedTest): "password_confirm": "test_password" } - def testWithoutEmail(self): data = {"email_address": ""} - error_notification = '
You MUST provide an email
' + error_notification = ('
' + 'You MUST provide an email
') result = requests.post(self.forgot_password_url, data=data) self.assertEqual(result.url, self.gn2_url+"/n/forgot_password") self.assertTrue( - result.content.find(error_notification) >= 0 - , "Error message should be displayed but was not") - - def testWithNonExistingEmail(self): - # Monkey patching doesn't work, so simply test that getting by email - # returns the correct data - user = get_user_by_unique_column(self.es, "email_address", "non-existent@domain.com") - self.assertTrue(user is None, "Should not find non-existent user") - - def testWithExistingEmail(self): - # Monkey patching doesn't work, so simply test that getting by email - # returns the correct data - user = get_user_by_unique_column(self.es, "email_address", "test@user.com") - self.assertTrue(user is not None, "Should find user") + result.content.find(error_notification) >= 0, + "Error message should be displayed but was not") diff --git a/test/requests/test_registration.py b/test/requests/test_registration.py index 0047e8a6..5d08bf58 100644 --- a/test/requests/test_registration.py +++ b/test/requests/test_registration.py @@ -1,31 +1,25 @@ import sys import requests -from parametrized_test import ParametrizedTest class TestRegistration(ParametrizedTest): - def tearDown(self): - for item in self.es_cleanup: - self.es.delete(index="users", doc_type="local", id=item["_id"]) def testRegistrationPage(self): - if self.es.ping(): - data = { - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - requests.post(self.gn2_url+"/n/register", data) - response = self.es.search( - index="users" - , doc_type="local" - , body={ - "query": {"match": {"email_address": "test@user.com"}}}) - self.assertEqual(len(response["hits"]["hits"]), 1) - else: - self.skipTest("The elasticsearch server is down") + data = { + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + requests.post(self.gn2_url+"/n/register", data) + response = self.es.search( + index="users" + , doc_type="local" + , body={ + "query": {"match": {"email_address": "test@user.com"}}}) + self.assertEqual(len(response["hits"]["hits"]), 1) + def main(gn2, es): import unittest diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 0cc963e5..32780ca6 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -5,14 +5,10 @@ import urllib.parse import numpy as np import pandas as pd -from elasticsearch import Elasticsearch, TransportError -from elasticsearch.helpers import bulk from flask import Flask, g, request from wqflask import app -from utility.elasticsearch_tools import get_elasticsearch_connection -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI def parse_db_uri(): @@ -106,20 +102,6 @@ if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() - # es = Elasticsearch([{ - # "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT - # }], timeout=60) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - - es = get_elasticsearch_connection(for_user=False) - - #input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" - #input_df = create_dataframe(input_filename) - #output_df = quantileNormalize(input_df) - - #output_df.to_csv('quant_norm.csv', sep='\t') - - #out_filename = sys.argv[1][:-4] + '_quantnorm.txt' - success, _ = bulk(es, set_data(sys.argv[1])) response = es.search( diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py deleted file mode 100644 index eae3ba03..00000000 --- a/wqflask/utility/elasticsearch_tools.py +++ /dev/null @@ -1,121 +0,0 @@ -# Elasticsearch support -# -# Some helpful commands to view the database: -# -# You can test the server being up with -# -# curl -H 'Content-Type: application/json' http://localhost:9200 -# -# List all indices -# -# curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v' -# -# To see the users index 'table' -# -# curl http://localhost:9200/users -# -# To list all user ids -# -# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' -# { -# "query" : { -# "match_all" : {} -# }, -# "stored_fields": [] -# }' -# -# To view a record -# -# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' -# { -# "query" : { -# "match" : { "email_address": "pjotr2017@thebird.nl"} -# } -# }' -# -# -# To delete the users index and data (dangerous!) -# -# curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users' - - -from elasticsearch import Elasticsearch, TransportError -import logging - -from utility.logger import getLogger -logger = getLogger(__name__) - -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT - - -def test_elasticsearch_connection(): - es = Elasticsearch(['http://' + ELASTICSEARCH_HOST + \ - ":" + str(ELASTICSEARCH_PORT) + '/'], verify_certs=True) - if not es.ping(): - logger.warning("Elasticsearch is DOWN") - - -def get_elasticsearch_connection(for_user=True): - """Return a connection to ES. Returns None on failure""" - logger.info("get_elasticsearch_connection") - es = None - try: - assert(ELASTICSEARCH_HOST) - assert(ELASTICSEARCH_PORT) - logger.info("ES HOST", ELASTICSEARCH_HOST) - - es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT - }], timeout=30, retry_on_timeout=True) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - - if for_user: - setup_users_index(es) - - es_logger = logging.getLogger("elasticsearch") - es_logger.setLevel(logging.INFO) - es_logger.addHandler(logging.NullHandler()) - except Exception as e: - logger.error("Failed to get elasticsearch connection", e) - es = None - - return es - - -def setup_users_index(es_connection): - if es_connection: - index_settings = { - "properties": { - "email_address": { - "type": "keyword"}}} - - es_connection.indices.create(index='users', ignore=400) - es_connection.indices.put_mapping( - body=index_settings, index="users", doc_type="local") - - -def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): - return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) - - -def save_user(es, user, user_id): - es_save_data(es, "users", "local", user, user_id) - - -def get_item_by_unique_column(es, column_name, column_value, index, doc_type): - item_details = None - try: - response = es.search( - index=index, doc_type=doc_type, body={ - "query": {"match": {column_name: column_value}} - }) - if len(response["hits"]["hits"]) > 0: - item_details = response["hits"]["hits"][0]["_source"] - except TransportError as te: - pass - return item_details - - -def es_save_data(es, index, doc_type, data_item, data_id,): - from time import sleep - es.create(index, doc_type, body=data_item, id=data_id) - sleep(1) # Delay 1 second to allow indexing diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 0efe8ca9..f28961ec 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -287,6 +287,7 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') +GITHUB_AUTH_URL = "" if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET: GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \ GITHUB_CLIENT_ID + "&client_secret=" + GITHUB_CLIENT_SECRET @@ -301,10 +302,6 @@ if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: "&redirect_uri=" + GN2_BRANCH_URL + "n/login/orcid_oauth2" ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') -# import utility.elasticsearch_tools as es -# es.test_elasticsearch_connection() SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 67e2e158..d3c4a62f 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -10,7 +10,6 @@ from flask import (Flask, g, render_template, url_for, request, make_response, from wqflask import app from utility import hmac -#from utility.elasticsearch_tools import get_elasticsearch_connection from utility.redis_tools import get_redis_conn, get_user_id, get_user_by_unique_column, set_user_attribute, get_user_collections, save_collections Redis = get_redis_conn() -- cgit v1.2.3