From 04b32f95279602da4cfb00b4f356a04eee48af51 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 9 Mar 2020 11:54:34 -0500 Subject: I think this should complete consolidating all the collections code. --- wqflask/base/trait.py | 8 +- wqflask/utility/hmac.py | 56 ++- wqflask/wqflask/collect.py | 247 ++------- wqflask/wqflask/correlation/corr_scatter_plot.py | 9 +- wqflask/wqflask/gsearch.py | 4 +- wqflask/wqflask/search_results.py | 2 +- wqflask/wqflask/templates/base.html | 4 - wqflask/wqflask/templates/collections/list.html | 24 - wqflask/wqflask/templates/collections/view.html | 9 +- .../wqflask/templates/new_security/login_user.html | 1 + .../templates/show_trait_mapping_tools.html | 11 - .../wqflask/templates/show_trait_statistics.html | 6 - wqflask/wqflask/user_login.py | 8 +- wqflask/wqflask/user_session.py | 551 +++++++++++---------- wqflask/wqflask/views.py | 2 +- 15 files changed, 375 insertions(+), 567 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index c9dfe780..8e779a11 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -303,7 +303,7 @@ def jsonable_table_row(trait, dataset_name, index): additive = "N/A" else: additive = "%.3f" % round(float(trait.additive), 2) - return ['', + return ['', index, ''+str(trait.name)+'', trait.symbol, @@ -319,7 +319,7 @@ def jsonable_table_row(trait, dataset_name, index): else: additive = "%.2f" % round(float(trait.additive), 2) if trait.pubmed_id: - return ['', + return ['', index, ''+str(trait.name)+'', trait.description_display, @@ -329,7 +329,7 @@ def jsonable_table_row(trait, dataset_name, index): trait.LRS_location_repr, additive] else: - return ['', + return ['', index, ''+str(trait.name)+'', trait.description_display, @@ -339,7 +339,7 @@ def jsonable_table_row(trait, dataset_name, index): trait.LRS_location_repr, additive] elif dataset.type == "Geno": - return ['', + return ['', index, ''+str(trait.name)+'', trait.location_repr] diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py index 47001e54..d8a0eace 100644 --- a/wqflask/utility/hmac.py +++ b/wqflask/utility/hmac.py @@ -1,18 +1,38 @@ -from __future__ import print_function, division, absolute_import - -import hmac - -from wqflask import app - -def hmac_creation(stringy): - """Helper function to create the actual hmac""" - - secret = app.config['SECRET_HMAC_CODE'] - - hmaced = hmac.new(secret, stringy, hashlib.sha1) - hm = hmaced.hexdigest() - # ZS: Leaving the below comment here to ask Pjotr about - # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." - # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html - hm = hm[:20] - return hm \ No newline at end of file +from __future__ import print_function, division, absolute_import + +import hmac +import hashlib + +from wqflask import app + +def hmac_creation(stringy): + """Helper function to create the actual hmac""" + + secret = app.config['SECRET_HMAC_CODE'] + + hmaced = hmac.new(secret, stringy, hashlib.sha1) + hm = hmaced.hexdigest() + # ZS: Leaving the below comment here to ask Pjotr about + # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." + # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html + hm = hm[:20] + return hm + +def data_hmac(stringy): + """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with""" + return stringy + ":" + hmac_creation(stringy) + +def url_for_hmac(endpoint, **values): + """Like url_for but adds an hmac at the end to insure the url hasn't been tampered with""" + + url = url_for(endpoint, **values) + + hm = hmac_creation(url) + if '?' in url: + combiner = "&" + else: + combiner = "?" + return url + combiner + "hm=" + hm + +app.jinja_env.globals.update(url_for_hmac=url_for_hmac, + data_hmac=data_hmac) \ No newline at end of file diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index c273af59..e5c24873 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -8,7 +8,6 @@ import time import uuid import hashlib -import hmac import base64 import urlparse @@ -28,9 +27,8 @@ from pprint import pformat as pf from wqflask.database import db_session from wqflask import model -from wqflask import user_manager -from utility import Bunch, Struct +from utility import Bunch, Struct, hmac from utility.formatting import numify from base import trait @@ -40,119 +38,15 @@ import logging from utility.logger import getLogger logger = getLogger(__name__) - -class AnonCollection(object): - """User is not logged in""" - def __init__(self, collection_name): - anon_user = user_manager.AnonUser() - self.key = anon_user.key - self.name = collection_name - self.id = None - self.created_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') - self.changed_timestamp = self.created_timestamp #ZS: will be updated when changes are made - - #ZS: Find id and set it if the collection doesn't already exist - if Redis.get(self.key) == "None" or Redis.get(self.key) == None: - Redis.set(self.key, "None") #ZS: For some reason I get the error "Operation against a key holding the wrong kind of value" if I don't do this - else: - collections_list = json.loads(Redis.get(self.key)) - collection_position = 0 #ZS: Position of collection in collection_list, if it exists - collection_exists = False - for i, collection in enumerate(collections_list): - if collection['name'] == self.name: - collection_position = i - collection_exists = True - self.id = collection['id'] - break - - if self.id == None: - self.id = str(uuid.uuid4()) - - def get_members(self): - traits = [] - collections_list = json.loads(Redis.get(self.key)) - for collection in collections_list: - if collection['id'] == self.id: - traits = collection['members'] - return traits - - @property - def num_members(self): - num_members = 0 - collections_list = json.loads(Redis.get(self.key)) - for collection in collections_list: - if collection['id'] == self.id: - num_members = collection['num_members'] - return num_members - - def add_traits(self, unprocessed_traits): - #assert collection_name == "Default", "Unexpected collection name for anonymous user" - self.traits = list(process_traits(unprocessed_traits)) - existing_collections = Redis.get(self.key) - logger.debug("existing_collections:", existing_collections) - if existing_collections != None and existing_collections != "None": - collections_list = json.loads(existing_collections) - collection_position = 0 #ZS: Position of collection in collection_list, if it exists - collection_exists = False - for i, collection in enumerate(collections_list): - if collection['id'] == self.id: - collection_position = i - collection_exists = True - break - if collection_exists: - collections_list[collection_position]['members'] += list(set(self.traits) - set(collections_list[collection_position]['members'])) - collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members']) - collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') - else: - collection_dict = {"id" : self.id, - "name" : self.name, - "created_timestamp" : self.created_timestamp, - "changed_timestamp" : self.changed_timestamp, - "num_members" : len(self.traits), - "members" : self.traits} - collections_list.append(collection_dict) - else: - collections_list = [] - collection_dict = {"id" : self.id, - "name" : self.name, - "created_timestamp" : self.created_timestamp, - "changed_timestamp" : self.changed_timestamp, - "num_members" : len(self.traits), - "members" : self.traits} - collections_list.append(collection_dict) - - Redis.set(self.key, json.dumps(collections_list)) - - def remove_traits(self, params): - traits_to_remove = [(":").join(trait.split(":")[:2]) for trait in params.getlist('traits[]')] - existing_collections = Redis.get(self.key) - collection_position = 0 - collections_list = json.loads(existing_collections) - for i, collection in enumerate(collections_list): - if collection['id'] == self.id: - collection_position = i - collection_exists = True - break - collections_list[collection_position]['members'] = [trait for trait in collections_list[collection_position]['members'] if trait not in traits_to_remove] - collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members']) - collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') - len_now = collections_list[collection_position]['num_members'] - Redis.set(self.key, json.dumps(collections_list)) - - # We need to return something so we'll return this...maybe in the future - # we can use it to check the results - return str(len_now) - def process_traits(unprocessed_traits): if isinstance(unprocessed_traits, basestring): unprocessed_traits = unprocessed_traits.split(",") traits = set() for trait in unprocessed_traits: - #print("trait is:", trait) data, _separator, hmac = trait.rpartition(':') data = data.strip() if g.user_session.logged_in: - assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?" + assert hmac == hmac.data_hmac(data), "Data tampering?" traits.add(str(data)) return traits @@ -160,8 +54,6 @@ def process_traits(unprocessed_traits): def report_change(len_before, len_now): new_length = len_now - len_before if new_length: - logger.debug("We've added {} to your collection.".format( - numify(new_length, 'new trait', 'new traits'))) flash("We've added {} to your collection.".format( numify(new_length, 'new trait', 'new traits'))) else: @@ -180,18 +72,14 @@ def store_traits_list(): @app.route("/collections/add") def collections_add(): - if g.user_session.logged_in: + + collections = g.user_session.user_collections + if len(collections) < 1: + collection_name = "Default Collection" + uc_id = g.user_session.add_collection(collection_name, set()) collections = g.user_session.user_collections - if len(collections) < 1: - collection_name = "Default Collection" - uc_id = g.user_session.add_collection(collection_name, set()) - collections = g.user_session.user_collections - else: - anon_collections = user_manager.AnonUser().get_collections() - collections = [] - for collection in anon_collections: - collections.append({'id':collection['id'], 'name':collection['name']}) + #ZS: One of these might be unnecessary if 'traits' in request.args: traits=request.args['traits'] return render_template("collections/add.html", @@ -212,19 +100,13 @@ def collections_new(): if "sign_in" in params: return redirect(url_for('login')) if "create_new" in params: - logger.debug("in create_new") collection_name = params['new_collection'] if collection_name.strip() == "": collection_name = datetime.datetime.utcnow().strftime('Collection_%b_%d_%H:%M') return create_new(collection_name) elif "add_to_existing" in params: - logger.debug("in add to existing") if 'existing_collection' not in params: - default_collection_exists = False - if g.user_session.logged_in: - collections = g.user_session.user_collections - else: - collections = user_manager.AnonUser().get_collections() + collections = g.user_session.user_collections for collection in collections: if collection["name"] == "Default Collection": collection_id = collection["id"] @@ -235,22 +117,14 @@ def collections_new(): else: collection_id = params['existing_collection'].split(":")[0] collection_name = params['existing_collection'].split(":")[1] - if g.user_session.logged_in: - if "hash" in params: - unprocessed_traits = Redis.get(params['hash']) - else: - unprocessed_traits = params['traits'] - traits = list(process_traits(unprocessed_traits)) - g.user_session.add_traits_to_collection(collection_id, traits) - return redirect(url_for('view_collection', uc_id=collection_id)) + + if "hash" in params: + unprocessed_traits = Redis.get(params['hash']) else: - ac = AnonCollection(collection_name) - if "hash" in params: - unprocessed_traits = Redis.get(params['hash']) - else: - unprocessed_traits = params['traits'] - ac.add_traits(unprocessed_traits) - return redirect(url_for('view_collection', collection_id=ac.id)) + unprocessed_traits = params['traits'] + traits = list(process_traits(unprocessed_traits)) + g.user_session.add_traits_to_collection(collection_id, traits) + return redirect(url_for('view_collection', uc_id=collection_id)) else: CauseAnError @@ -265,50 +139,30 @@ def create_new(collection_name): traits = process_traits(unprocessed_traits) - if g.user_session.logged_in: - uc_id = g.user_session.add_collection(collection_name, traits) + uc_id = g.user_session.add_collection(collection_name, traits) - return redirect(url_for('view_collection', uc_id=uc_id)) - else: - ac = AnonCollection(collection_name) - ac.changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') - ac.add_traits(unprocessed_traits) - return redirect(url_for('view_collection', collection_id=ac.id)) + return redirect(url_for('view_collection', uc_id=uc_id)) @app.route("/collections/list") def list_collections(): params = request.args - if g.user_session.logged_in: - user_collections = list(g.user_session.user_collections) - #logger.debug("user_collections are:", user_collections) - return render_template("collections/list.html", - params = params, - collections = user_collections, - ) - else: - anon_collections = user_manager.AnonUser().get_collections() - #logger.debug("anon_collections are:", anon_collections) - return render_template("collections/list.html", - params = params, - collections = anon_collections) - + user_collections = list(g.user_session.user_collections) + return render_template("collections/list.html", + params = params, + collections = user_collections, + ) @app.route("/collections/remove", methods=('POST',)) def remove_traits(): params = request.form - if "uc_id" in params: - uc_id = params['uc_id'] - traits_to_remove = params.getlist('traits[]') - traits_to_remove = process_traits(traits_to_remove) - logger.debug("\n\n after processing, traits_to_remove:", traits_to_remove) - - members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove) - else: - collection_name = params['collection_name'] - members_now = AnonCollection(collection_name).remove_traits(params) + uc_id = params['uc_id'] + traits_to_remove = params.getlist('traits[]') + traits_to_remove = process_traits(traits_to_remove) + logger.debug("\n\n after processing, traits_to_remove:", traits_to_remove) + members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove) # We need to return something so we'll return this...maybe in the future # we can use it to check the results @@ -319,21 +173,13 @@ def remove_traits(): def delete_collection(): params = request.form uc_id = "" - if g.user_session.logged_in: - uc_id = params['uc_id'] - if len(uc_id.split(":")) > 1: - for this_uc_id in uc_id.split(":"): - collection_name = g.user_session.delete_collection(this_uc_id) - else: - collection_name = g.user_session.delete_collection(uc_id) + + uc_id = params['uc_id'] + if len(uc_id.split(":")) > 1: + for this_uc_id in uc_id.split(":"): + collection_name = g.user_session.delete_collection(this_uc_id) else: - if "collection_name" in params: - collection_name = params['collection_name'] - user_manager.AnonUser().delete_collection(collection_name) - else: - uc_id = params['uc_id'] - for this_collection in uc_id.split(":"): - user_manager.AnonUser().delete_collection(this_collection) + collection_name = g.user_session.delete_collection(uc_id) if uc_id != "": if len(uc_id.split(":")) > 1: @@ -350,19 +196,9 @@ def delete_collection(): def view_collection(): params = request.args - if g.user_session.logged_in and "uc_id" in params: - uc_id = params['uc_id'] - uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next() - traits = uc["members"] - else: - user_collections = json.loads(Redis.get(user_manager.AnonUser().key)) - this_collection = {} - for collection in user_collections: - if collection['id'] == params['collection_id']: - this_collection = collection - break - - traits = this_collection['members'] + uc_id = params['uc_id'] + uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next() + traits = uc["members"] trait_obs = [] json_version = [] @@ -381,12 +217,9 @@ def view_collection(): json_version.append(trait.jsonable(trait_ob)) - if "uc_id" in params: - collection_info = dict(trait_obs=trait_obs, - uc = uc) - else: - collection_info = dict(trait_obs=trait_obs, - collection_name=this_collection['name']) + collection_info = dict(trait_obs=trait_obs, + uc = uc) + if "json" in params: return json.dumps(json_version) else: diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index f8eca386..dfb81c54 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -71,15 +71,8 @@ class CorrScatterPlot(object): sr_intercept_coords = get_intercept_coords(srslope, srintercept, sr_range, sr_range) - #vals_3 = [] - #for sample in self.trait_3.data: - # vals_3.append(self.trait_3.data[sample].value) - self.collections_exist = "False" - if g.user_session.logged_in: - if g.user_session.num_collections > 0: - self.collections_exist = "True" - elif g.cookie_session.display_num_collections() != "": + if g.user_session.num_collections > 0: self.collections_exist = "True" self.js_data = dict( diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index ed3b74b8..12813e9a 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -70,7 +70,7 @@ class GSearch(object): this_trait['name'] = line[5] this_trait['dataset'] = line[3] this_trait['dataset_fullname'] = line[4] - this_trait['hmac'] = hmac.hmac_creation('{}:{}'.format(line[5], line[3])) + this_trait['hmac'] = hmac.data_hmac('{}:{}'.format(line[5], line[3])) this_trait['species'] = line[0] this_trait['group'] = line[1] this_trait['tissue'] = line[2] @@ -174,7 +174,7 @@ class GSearch(object): this_trait['display_name'] = this_trait['name'] this_trait['dataset'] = line[2] this_trait['dataset_fullname'] = line[3] - this_trait['hmac'] = hmac.hmac_creation('{}:{}'.format(line[4], line[2])) + this_trait['hmac'] = hmac.data_hmac('{}:{}'.format(line[4], line[2])) this_trait['species'] = line[0] this_trait['group'] = line[1] if line[9] != None and line[6] != None: diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 902317a4..34c647f4 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -120,7 +120,7 @@ views.py). else: trait_dict['display_name'] = this_trait.name trait_dict['dataset'] = this_trait.dataset.name - trait_dict['hmac'] = hmac.hmac_creation('{}:{}'.format(this_trait.name, this_trait.dataset.name)) + trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) if this_trait.dataset.type == "ProbeSet": trait_dict['symbol'] = this_trait.symbol trait_dict['description'] = this_trait.description_display.decode('utf-8', 'replace') diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 04cf718a..2c95bf5e 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -73,11 +73,7 @@