From a302a2b0ac0e7c0f26a0d063c3f2b057f61d47f1 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 5 Jun 2020 16:52:56 -0500 Subject: Commiting other current group/resource management code, plus the new files --- wqflask/base/trait.py | 2 + wqflask/maintenance/set_resource_defaults.py | 155 +++++++++++++++++++++ wqflask/utility/authentication_tools.py | 46 ++++++ wqflask/utility/redis_tools.py | 37 +++-- wqflask/wqflask/group_manager.py | 77 ++++++++++ wqflask/wqflask/resource_manager.py | 72 ++++++++++ .../wqflask/static/new/javascript/group_manager.js | 38 +++++ wqflask/wqflask/templates/admin/create_group.html | 89 ++++++++++++ wqflask/wqflask/templates/admin/group_manager.html | 68 ++++----- .../wqflask/templates/admin/manage_resource.html | 92 ++++++++++++ .../wqflask/templates/admin/search_for_groups.html | 64 +++++++++ .../templates/admin/select_group_to_add.html | 54 +++++++ .../templates/new_security/not_authenticated.html | 11 ++ wqflask/wqflask/templates/show_trait_details.html | 5 + wqflask/wqflask/views.py | 3 +- 15 files changed, 764 insertions(+), 49 deletions(-) create mode 100644 wqflask/maintenance/set_resource_defaults.py create mode 100644 wqflask/utility/authentication_tools.py create mode 100644 wqflask/wqflask/group_manager.py create mode 100644 wqflask/wqflask/resource_manager.py create mode 100644 wqflask/wqflask/static/new/javascript/group_manager.js create mode 100644 wqflask/wqflask/templates/admin/create_group.html create mode 100644 wqflask/wqflask/templates/admin/manage_resource.html create mode 100644 wqflask/wqflask/templates/admin/search_for_groups.html create mode 100644 wqflask/wqflask/templates/admin/select_group_to_add.html create mode 100644 wqflask/wqflask/templates/new_security/not_authenticated.html diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 405c4ebf..2a945588 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -391,6 +391,8 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if response.strip() == "no-access": trait.view = False return trait + else: + trait_info = json.loads(response) except: resource_info = get_resource_info(resource_id) default_permissions = resource_info['default_mask']['data'] diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py new file mode 100644 index 00000000..ba102d9c --- /dev/null +++ b/wqflask/maintenance/set_resource_defaults.py @@ -0,0 +1,155 @@ +""" + +Script that sets default resource access masks for use with the DB proxy + +Defaults will be: +Owner - omni_gn +Mask - Public/non-confidential: { data: "view", + metadata: "view", + admin: "not-admin" } + Private/confidentia: { data: "no-access", + metadata: "no-access", + admin: "not-admin" } + +To run: +./bin/genenetwork2 ~/my_settings.py -c ./wqflask/maintenance/gen_select_dataset.py + +""" + +from __future__ import print_function, division + +import sys +import json + +# NEW: Note we prepend the current path - otherwise a guix instance of GN2 may be used instead +sys.path.insert(0,'./') + +# NEW: import app to avoid a circular dependency on utility.tools +from wqflask import app + +from utility.tools import SQL_URI +from utility.redis_tools import get_redis_conn, get_user_id, add_resource, get_resources +Redis = get_redis_conn() + +import MySQLdb + +import urlparse + +from utility.logger import getLogger +logger = getLogger(__name__) + +def parse_db_uri(): + """Converts a database URI to the db name, host name, user name, and password""" + + parsed_uri = urlparse.urlparse(SQL_URI) + + db_conn_info = dict( + db = parsed_uri.path[1:], + host = parsed_uri.hostname, + user = parsed_uri.username, + passwd = parsed_uri.password) + + print(db_conn_info) + return db_conn_info + +def insert_probeset_resources(default_owner_id): + current_resources = Redis.hgetall("resources") + Cursor.execute(""" SELECT + ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.confidentiality, ProbeSetFreeze.public + FROM + ProbeSetFreeze""") + + resource_results = Cursor.fetchall() + for i, resource in enumerate(resource_results): + if i % 20 == 0: + print(i) + resource_ob = {} + resource_ob['name'] = resource[1] + resource_ob['owner_id'] = default_owner_id + resource_ob['data'] = { "dataset" : str(resource[0])} + resource_ob['type'] = "dataset-probeset" + if resource[2] < 1 and resource[3] > 0: + resource_ob['default_mask'] = { "data": ["no-access", "view"] } + else: + resource_ob['default_mask'] = { "data": ["no-access"] } + resource_ob['group_masks'] = {} + + add_resource(resource_ob) + +def insert_publish_resources(default_owner_id): + current_resources = Redis.hgetall("resources") + Cursor.execute(""" SELECT + PublishXRef.Id, PublishFreeze.Id, InbredSet.InbredSetCode + FROM + PublishXRef, PublishFreeze, InbredSet, Publication + WHERE + PublishFreeze.InbredSetId = PublishXRef.InbredSetId AND + InbredSet.Id = PublishXRef.InbredSetId AND + Publication.Id = PublishXRef.PublicationId""") + + resource_results = Cursor.fetchall() + for resource in resource_results: + if resource[2]: + resource_ob = {} + if resource[2]: + resource_ob['name'] = resource[2] + "_" + str(resource[0]) + else: + resource_ob['name'] = str(resource[0]) + resource_ob['owner_id'] = default_owner_id + resource_ob['data'] = { "dataset" : str(resource[1]) , + "trait" : str(resource[0])} + resource_ob['type'] = "dataset-publish" + resource_ob['default_mask'] = { "data": "view" } + + resource_ob['group_masks'] = {} + + add_resource(resource_ob) + else: + continue + +def insert_geno_resources(default_owner_id): + current_resources = Redis.hgetall("resources") + Cursor.execute(""" SELECT + GenoFreeze.Id, GenoFreeze.ShortName, GenoFreeze.confidentiality + FROM + GenoFreeze""") + + resource_results = Cursor.fetchall() + for i, resource in enumerate(resource_results): + if i % 20 == 0: + print(i) + resource_ob = {} + resource_ob['name'] = resource[1] + resource_ob['owner_id'] = default_owner_id + resource_ob['data'] = { "dataset" : str(resource[0]) } + resource_ob['type'] = "dataset-geno" + if resource[2] < 1: + resource_ob['default_mask'] = { "data": "view" } + else: + resource_ob['default_mask'] = { "data": "no-access" } + resource_ob['group_masks'] = {} + + add_resource(resource_ob) + +def insert_resources(default_owner_id): + current_resources = get_resources() + print("START") + insert_publish_resources(default_owner_id) + print("AFTER PUBLISH") + insert_geno_resources(default_owner_id) + print("AFTER GENO") + insert_probeset_resources(default_owner_id) + print("AFTER PROBESET") + +def main(): + """Generates and outputs (as json file) the data for the main dropdown menus on the home page""" + + Redis.delete("resources") + + owner_id = get_user_id("email_address", "zachary.a.sloan@gmail.com") + insert_resources(owner_id) + +if __name__ == '__main__': + Conn = MySQLdb.Connect(**parse_db_uri()) + Cursor = Conn.cursor() + main() \ No newline at end of file diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py new file mode 100644 index 00000000..537881a5 --- /dev/null +++ b/wqflask/utility/authentication_tools.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import, print_function, division + +import json +import requests + +from base import data_set + +from utility import hmac +from utility.redis_tools import get_redis_conn, get_resource_info, get_resource_id + +from flask import Flask, g, redirect, url_for + +import logging +logger = logging.getLogger(__name__ ) + +def check_resource_availability(dataset, trait_id=None): + resource_id = get_resource_id(dataset, trait_id) + + if resource_id: + the_url = "http://localhost:8080/available?resource={}&user={}".format(resource_id, g.user_session.user_id) + try: + response = json.loads(requests.get(the_url).content)['data'] + except: + resource_info = get_resource_info(resource_id) + response = resource_info['default_mask']['data'] + + if 'view' in response: + return True + else: + return redirect(url_for("no_access_page")) + + return True + +def check_owner(dataset=None, trait_id=None, resource_id=None): + if resource_id: + resource_info = get_resource_info(resource_id) + if g.user_session.user_id == resource_info['owner_id']: + return resource_id + else: + resource_id = get_resource_id(dataset, trait_id) + if resource_id: + resource_info = get_resource_info(resource_id) + if g.user_session.user_id == resource_info['owner_id']: + return resource_id + + return False \ No newline at end of file diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index 0ad96879..bc30a0af 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -95,14 +95,17 @@ def get_user_groups(user_id): user_group_ids = [] #ZS: Group IDs where user is a regular user groups_list = Redis.hgetall("groups") for key in groups_list: - group_ob = json.loads(groups_list[key]) - group_admins = set(group_ob['admins']) - group_members = set(group_ob['members']) - if user_id in group_admins: - admin_group_ids.append(group_ob['id']) - elif user_id in group_members: - user_group_ids.append(group_ob['id']) - else: + try: + group_ob = json.loads(groups_list[key]) + group_admins = set(group_ob['admins']) + group_members = set(group_ob['members']) + if user_id in group_admins: + admin_group_ids.append(group_ob['id']) + elif user_id in group_members: + user_group_ids.append(group_ob['id']) + else: + continue + except: continue admin_groups = [] @@ -122,6 +125,24 @@ def get_group_info(group_id): return group_info +def get_group_by_unique_column(column_name, column_value): + """ Get group by column; not sure if there's a faster way to do this """ + + matched_groups = [] + + all_group_list = Redis.hgetall("groups") + for key in all_group_list: + group_info = json.loads(all_group_list[key]) + if column_name == "admins" or column_name == "members": #ZS: Since these fields are lists, search in the list + if column_value in group_info[column_name]: + matched_groups.append(group_info) + else: + if group_info[column_name] == column_value: + matched_groups.append(group_info) + + return matched_groups + + def create_group(admin_user_ids, member_user_ids = [], group_name = "Default Group Name"): group_id = str(uuid.uuid4()) new_group = { diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py new file mode 100644 index 00000000..f41ae56d --- /dev/null +++ b/wqflask/wqflask/group_manager.py @@ -0,0 +1,77 @@ + +from __future__ import print_function, division, absolute_import + +from flask import (Flask, g, render_template, url_for, request, make_response, + redirect, flash) + +from wqflask import app +from wqflask.user_login import send_verification_email + +from utility.redis_tools import get_user_groups, get_group_info, create_group, delete_group, add_users_to_group, remove_users_from_group, \ + change_group_name, save_verification_code, check_verification_code, get_user_by_unique_column + +from utility.logger import getLogger +logger = getLogger(__name__) + +@app.route("/groups/manage", methods=('GET', 'POST')) +def manage_groups(): + params = request.form if request.form else request.args + if "add_new_group" in params: + return redirect(url_for('add_group')) + else: + admin_groups, user_groups = get_user_groups(g.user_session.user_id) + return render_template("admin/group_manager.html", admin_groups=admin_groups, user_groups=user_groups) + +@app.route("/groups/remove", methods=('POST',)) +def remove_groups(): + group_ids_to_remove = request.form['selected_group_ids'] + for group_id in group_ids_to_remove.split(":"): + delete_group(g.user_session.user_id, group_id) + + return redirect(url_for('manage_groups')) + +@app.route("/groups/create", methods=('GET', 'POST')) +def add_group(): + params = request.form if request.form else request.args + if "group_name" in params: + member_user_ids = set() + admin_user_ids = set() + admin_user_ids.add(g.user_session.user_id) #ZS: Always add the user creating the group as an admin + if "admin_emails" in params: + admin_emails = params['admin_emails_to_add'].split(",") + for email in admin_emails: + user_details = get_user_by_unique_column("email_address", email) + if user_details: + admin_user_ids.add(user_details['user_id']) + #send_group_invites(params['group_id'], user_email_list = admin_emails, user_type="admins") + if "user_emails" in params: + member_emails = params['member_emails_to_add'].split(",") + for email in member_emails: + user_details = get_user_by_unique_column("email_address", email) + if user_details: + member_user_ids.add(user_details['user_id']) + #send_group_invites(params['group_id'], user_email_list = user_emails, user_type="members") + + create_group(list(admin_user_ids), list(member_user_ids), params['group_name']) + return redirect(url_for('manage_groups')) + else: + return render_template("admin/create_group.html") + +#ZS: Will integrate this later, for now just letting users be added directly +def send_group_invites(group_id, user_email_list = [], user_type="members"): + for user_email in user_email_list: + user_details = get_user_by_unique_column("email_address", user_email) + if user_details: + group_info = get_group_info(group_id) + #ZS: Probably not necessary since the group should normally always exist if group_id is being passed here, + # but it's technically possible to hit it if Redis is cleared out before submitting the new users or something + if group_info: + #ZS: Don't add user if they're already an admin or if they're being added a regular user and are already a regular user, + # but do add them if they're a regular user and are added as an admin + if (user_details['user_id'] in group_info['admins']) or \ + ((user_type == "members") and (user_details['user_id'] in group_info['members'])): + continue + else: + send_verification_email(user_details, template_name = "email/group_verification.txt", key_prefix = "verification_code", subject = "You've been invited to join a GeneNetwork user group") + +#@app.route() \ No newline at end of file diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py new file mode 100644 index 00000000..7d88b8ed --- /dev/null +++ b/wqflask/wqflask/resource_manager.py @@ -0,0 +1,72 @@ +from __future__ import print_function, division, absolute_import + +from flask import (Flask, g, render_template, url_for, request, make_response, + redirect, flash) + +from wqflask import app + +from utility.authentication_tools import check_owner +from utility.redis_tools import get_resource_info, get_group_info, get_group_by_unique_column, get_user_id + +from utility.logger import getLogger +logger = getLogger(__name__) + +@app.route("/resources/manage", methods=('GET', 'POST')) +def view_resource(): + params = request.form if request.form else request.args + if 'resource_id' in request.args: + resource_id = request.args['resource_id'] + if check_owner(resource_id=resource_id): + resource_info = get_resource_info(resource_id) + group_masks = resource_info['group_masks'] + group_masks_with_names = get_group_names(group_masks) + default_mask = resource_info['default_mask']['data'] + return render_template("admin/manage_resource.html", resource_id = resource_id, resource_info=resource_info, default_mask=default_mask, group_masks=group_masks_with_names) + else: + return redirect(url_for("no_access_page")) + +@app.route("/resources/add_group", methods=('POST',)) +def add_group_to_resource(): + resource_id = request.form['resource_id'] + if check_owner(resource_id=resource_id): + if all(key in request.form for key in ('group_id', 'group_name', 'user_name', 'user_email')): + group_list = [] + if request.form['group_id'] != "": + the_group = get_group_info(request.form['group_id']) + if the_group: + group_list.append(the_group) + if request.form['group_name'] != "": + matched_groups = get_group_by_unique_column("name", request.form['group_name']) + for group in matched_groups: + group_list.append(group) + if request.form['user_name'] != "": + user_id = get_user_id("user_name", request.form['user_name']) + if user_id: + matched_groups = get_group_by_unique_column("admins", user_id) + matched_groups += get_group_by_unique_column("members", user_id) + for group in matched_groups: + group_list.append(group) + if request.form['user_email'] != "": + user_id = get_user_id("email_address", request.form['user_email']) + if user_id: + matched_groups = get_group_by_unique_column("admins", user_id) + matched_groups += get_group_by_unique_column("members", user_id) + for group in matched_groups: + group_list.append(group) + return render_template("admin/select_group_to_add.html", group_list=group_list, resource_id = resource_id) + elif 'selected_group' in request.form: + group_id = request.form['selected_group'] + return render_template("admin/set_group_privileges.html", resource_id = resource_id, group_id = group_id) + else: + return render_template("admin/search_for_groups.html", resource_id = resource_id) + else: + return redirect(url_for("no_access_page")) + +def get_group_names(group_masks): + group_masks_with_names = {} + for group_id, group_mask in group_masks.iteritems(): + this_mask = group_mask + group_name = get_group_info(group_id)['name'] + this_mask['name'] = group_name + + return group_masks_with_names \ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/group_manager.js b/wqflask/wqflask/static/new/javascript/group_manager.js new file mode 100644 index 00000000..5e82d104 --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/group_manager.js @@ -0,0 +1,38 @@ +$('#add_to_admins').click(function() { + add_emails('admin') +}) + +$('#add_to_members').click(function() { + add_emails('member') +}) + +$('#clear_admins').click(function(){ + clear_emails('admin') +}) + +$('#clear_members').click(function(){ + clear_emails('member') +}) + + +function add_emails(user_type){ + var email_address = $('input[name=user_email]').val(); + var email_list_string = $('input[name=' + user_type + '_emails_to_add]').val() + console.log(email_list_string) + if (email_list_string == ""){ + var email_set = new Set(); + } else { + var email_set = new Set(email_list_string.split(",")) + } + email_set.add(email_address) + + $('input[name=' + user_type + '_emails_to_add]').val(Array.from(email_set).join(',')) + + var emails_display_string = Array.from(email_set).join('\n') + $('.added_' + user_type + 's').val(emails_display_string) +} + +function clear_emails(user_type){ + $('input[name=' + user_type + '_emails_to_add]').val("") + $('.added_' + user_type + 's').val("") +} \ No newline at end of file diff --git a/wqflask/wqflask/templates/admin/create_group.html b/wqflask/wqflask/templates/admin/create_group.html new file mode 100644 index 00000000..55c3fa0b --- /dev/null +++ b/wqflask/wqflask/templates/admin/create_group.html @@ -0,0 +1,89 @@ +{% extends "base.html" %} +{% block title %}Group Manager{% endblock %} +{% block content %} + +
+
+

Create Group

+
+
+ + +
+
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+ + + + + +{% endblock %} + +{% block js %} + + + + + +{% endblock %} diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html index b7df1aad..23d8205a 100644 --- a/wqflask/wqflask/templates/admin/group_manager.html +++ b/wqflask/wqflask/templates/admin/group_manager.html @@ -1,15 +1,23 @@ {% extends "base.html" %} {% block title %}Group Manager{% endblock %} +{% block css %} + + + +{% endblock %} {% block content %}

Manage Groups

- +
+ + +
-
+
{% if admin_groups|length == 0 and user_groups|length == 0 %}

You currently aren't a member or admin of any groups.


@@ -20,7 +28,7 @@ {% if admin_groups|length == 0 %}

You currently aren't the administrator of any groups.

{% else %} - +
@@ -29,17 +37,19 @@ + {% for group in admin_groups %} - + - + + {% endfor %} @@ -47,13 +57,13 @@ {% endif %}
-
+

User Groups

{% if user_groups|length == 0 %}

You currently aren't a member of any groups.

{% else %} -
# Members Created Last ChangedGroup ID
{{ loop.index }}{{ loop.index }} {{ group.name }}{{ group.admins|length + group.users|length }}{{ group.admins|length + group.users|length }} {{ group.created_timestamp }} {{ group.changed_timestamp }}{{ group.id }}
+
@@ -88,48 +98,26 @@ {% endblock %} {% block js %} - - - + + + + +{% endblock %} diff --git a/wqflask/wqflask/templates/admin/search_for_groups.html b/wqflask/wqflask/templates/admin/search_for_groups.html new file mode 100644 index 00000000..89eb11dd --- /dev/null +++ b/wqflask/wqflask/templates/admin/search_for_groups.html @@ -0,0 +1,64 @@ +{% extends "base.html" %} +{% block title %}Resource Manager{% endblock %} +{% block content %} + +
+
+

Find Groups

+
+ + +
+
+
+
+

Search by:

+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+
+ +
+ + + +{% endblock %} + +{% block js %} + + + + + +{% endblock %} diff --git a/wqflask/wqflask/templates/admin/select_group_to_add.html b/wqflask/wqflask/templates/admin/select_group_to_add.html new file mode 100644 index 00000000..df70fb2f --- /dev/null +++ b/wqflask/wqflask/templates/admin/select_group_to_add.html @@ -0,0 +1,54 @@ +{% extends "base.html" %} +{% block title %}Matched Groups{% endblock %} +{% block css %} + + + +{% endblock %} +{% block content %} + +
+

The following groups were found:

+
+
+ +
+ {% if group_list|length > 0 %} + +
+ + + + + + + + + + {% for group in group_list %} + + + + + + + {% endfor %} + +
NameCreatedLast Changed
{% if 'name' in group %}{{ group.name }}{% else %}N/A{% endif %}{% if 'created_timestamp' in group %}{{ group.created_timestamp }}{% else %}N/A{% endif %}{% if 'changed_timestamp' in group %}{{ group.changed_timestamp }}{% else %}N/A{% endif %}
+ {% else %} +

No matching groups were found.

+ {% endif %} +
+ +
+ + + +{% endblock %} + +{% block js %} + + +{% endblock %} diff --git a/wqflask/wqflask/templates/new_security/not_authenticated.html b/wqflask/wqflask/templates/new_security/not_authenticated.html new file mode 100644 index 00000000..7d0d3060 --- /dev/null +++ b/wqflask/wqflask/templates/new_security/not_authenticated.html @@ -0,0 +1,11 @@ +{% extends "base.html" %} +{% block title %}Authentication Needed{% endblock %} +{% block content %} +
+
+

You lack the permissions to view this data.

+
+

Please contact the data's owner or GN administrators if you believe you should have access to this data.

+
+ +{% endblock %} \ No newline at end of file diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 878b6ced..5c315878 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -248,6 +248,11 @@ + {% if resource_id %} + + + + {% endif %}
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 24a4dcee..ee827ba3 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -30,6 +30,7 @@ import sqlalchemy from wqflask import app from flask import g, Response, request, make_response, render_template, send_from_directory, jsonify, redirect, url_for from wqflask import group_manager +from wqflask import resource_manager from wqflask import search_results from wqflask import export_traits from wqflask import gsearch @@ -89,13 +90,13 @@ def connect_db(): @app.before_request def check_access_permissions(): logger.debug("@app.before_request check_access_permissions") + available = True if "temp_trait" in request.args: if request.args['temp_trait'] == "True": pass else: if 'dataset' in request.args: dataset = create_dataset(request.args['dataset']) - logger.debug("USER:", Redis.hget("users")) if 'trait_id' in request.args: available = check_resource_availability(dataset, request.args['trait_id']) else: -- cgit v1.2.3