From 57b00317168eb3a84c489c2613133170a191aef6 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 15 Oct 2013 02:50:25 -0500 Subject: * Redid mail sending service to set a ping showing it's alive * Redid server so it wouldn't start unless mail service recently pinged * Wrote a bunch of forgot password code * Refactored sending email about verification and forgotten passwords --- wqflask/secure_server.py | 40 ++++- wqflask/wqflask/send_mail.py | 39 +++-- .../wqflask/templates/email/forgot_password.txt | 5 + wqflask/wqflask/templates/email/verification.txt | 3 +- .../templates/new_security/forgot_password.html | 65 ++++++++ .../wqflask/templates/new_security/login_user.html | 55 ++++--- wqflask/wqflask/user_manager.py | 165 ++++++++++++--------- 7 files changed, 258 insertions(+), 114 deletions(-) create mode 100644 wqflask/wqflask/templates/email/forgot_password.txt create mode 100644 wqflask/wqflask/templates/new_security/forgot_password.html diff --git a/wqflask/secure_server.py b/wqflask/secure_server.py index a77abf7e..d5f1a291 100644 --- a/wqflask/secure_server.py +++ b/wqflask/secure_server.py @@ -1,12 +1,18 @@ from __future__ import absolute_import, division, print_function +import time +import sys + from wqflask import app from flask import Flask, render_template +import redis +Redis = redis.StrictRedis() + # Setup mail -from flask.ext.mail import Mail -mail = Mail(app) +#from flask.ext.mail import Mail +#mail = Mail(app) from wqflask.model import * @@ -33,8 +39,38 @@ app.wsgi_app = ProxyFix(app.wsgi_app) #print("app.config is:", app.config) + + +def check_send_mail_running(): + """Ensure send_mail.py is running before we start the site + + It would be really easy to accidentally run the site + without our mail program running + This will make sure our mail program is running...or at least recently run... + + """ + error_msg = "Make sure your are running send_mail.py" + send_mail_ping = Redis.get("send_mail:ping") + print("send_mail_ping is:", send_mail_ping) + if not send_mail_ping: + sys.exit(error_msg) + + last_ping = time.time() - float(send_mail_ping) + if not (0 < last_ping < 100): + sys.exit(error_msg) + + + print("send_mail.py seems to be running...") + + if __name__ == '__main__': #create_user() + + + + check_send_mail_running() + + app.run(host='0.0.0.0', port=app.config['SERVER_PORT'], use_debugger=False, diff --git a/wqflask/wqflask/send_mail.py b/wqflask/wqflask/send_mail.py index be51ad0d..bf5d0dd8 100644 --- a/wqflask/wqflask/send_mail.py +++ b/wqflask/wqflask/send_mail.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, division, print_function import datetime +import time import simplejson as json @@ -12,34 +13,40 @@ import mailer def timestamp(): ts = datetime.datetime.utcnow() return ts.isoformat() - + def main(): while True: - print("Waiting for message to show up in queue...") - msg = Redis.blpop("mail_queue") - - # Queue name is the first element, we want the second, which is the actual message - msg = msg[1] - - print("\nGot a msg in queue at {}: {}".format(timestamp(), msg)) - # Todo: Truncate mail_processed when it gets to long - Redis.rpush("mail_processed", msg) - process_message(msg) - + print("I'm alive!") + + # Set something so we know it's running (or at least been running recently) + Redis.setex("send_mail:ping", 300, time.time()) + + msg = Redis.blpop("mail_queue", 30) + + if msg: + # Queue name is the first element, we want the second, which is the actual message + msg = msg[1] + + print("\n\nGot a msg in queue at {}: {}".format(timestamp(), msg)) + # Todo: Truncate mail_processed when it gets to long + Redis.rpush("mail_processed", msg) + process_message(msg) + + def process_message(msg): msg = json.loads(msg) - + message = mailer.Message() message.From = msg['From'] message.To = msg['To'] message.Subject = msg['Subject'] message.Body = msg['Body'] - + sender = mailer.Mailer('localhost') sender.send(message) print("Sent message at {}: {}\n".format(timestamp(), msg)) - + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/wqflask/wqflask/templates/email/forgot_password.txt b/wqflask/wqflask/templates/email/forgot_password.txt new file mode 100644 index 00000000..e7d1389b --- /dev/null +++ b/wqflask/wqflask/templates/email/forgot_password.txt @@ -0,0 +1,5 @@ +Sorry to hear you lost your GeneNetwork password. + +To reset your password please click the following link, or cut and paste it into your browser window: + +{{ url_for_hmac("password_reset", code = verification_code, _external=True )}} diff --git a/wqflask/wqflask/templates/email/verification.txt b/wqflask/wqflask/templates/email/verification.txt index 29229c68..76149a3a 100644 --- a/wqflask/wqflask/templates/email/verification.txt +++ b/wqflask/wqflask/templates/email/verification.txt @@ -4,5 +4,4 @@ We need to verify your email address. To do that please click the following link, or cut and paste it into your browser window: -{{ url_for_hmac("verify", code = verification_code, _external=True )}} - +{{ url_for_hmac("verify_email", code = verification_code, _external=True )}} diff --git a/wqflask/wqflask/templates/new_security/forgot_password.html b/wqflask/wqflask/templates/new_security/forgot_password.html new file mode 100644 index 00000000..39e51f96 --- /dev/null +++ b/wqflask/wqflask/templates/new_security/forgot_password.html @@ -0,0 +1,65 @@ +{% extends "base.html" %} +{% block title %}Forgot Password{% endblock %} +{% block content %} + + {{ header("Forgot Password", "Easily reset your password.") }} + +
+ + + +
+ +

Enter your email address

+ +
And we'll send you a link to reset your password
+ + + +
+
+ + +
+ +
+ +
+
+ + +
+
+ + + +
+
+ +
+ +
+ +
+
Has your email address changed?
+ + If you no longer use the email address connected to your account, you can contact us for assistance. + +
+ +
+
+
+ + + {% endblock %} + +{% block js %} + + + {% include "new_security/_scripts.html" %} + +{% endblock %} diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index f232ccbc..4e308c75 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -2,82 +2,81 @@ {% block title %}Register{% endblock %} {% block content %} - {{ header("Login", "Gain access to GeneNetwork.") }} + {{ header("Sign in", "Gain access to GeneNetwork") }}
- +
- +

Don't have an account?

- - + + Create a new account - - + +
- +

Already have an account?

- +
Sign in here
- - - + + +
- - + +
- + - - + +
- - + +
- +
- - + +
- +
- + {% endblock %} -{% block js %} +{% block js %} - + {% include "new_security/_scripts.html" %} {% endblock %} - diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index a2dff7f2..1df2c153 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -19,8 +19,11 @@ import base64 import simplejson as json -from redis import StrictRedis -Redis = StrictRedis() +from sqlalchemy import orm + +#from redis import StrictRedis +import redis +Redis = redis.StrictRedis() from flask import (Flask, g, render_template, url_for, request, make_response, @@ -53,7 +56,7 @@ def timestamp(): class UserSession(object): cookie_name = 'session_id' - + def __init__(self): cookie = request.cookies.get(self.cookie_name) if not cookie: @@ -70,13 +73,13 @@ class UserSession(object): self.record = Redis.hgetall(self.redis_key) print("record is:", self.record) self.logged_in = True - - + + def delete_session(self): # And more importantly delete the redis record Redis.delete(self.cookie_name) print("At end of delete_session") - + @app.before_request def before_request(): g.user_session = UserSession() @@ -105,22 +108,22 @@ class UserManager(object): print(" ID:", dataset.id) print(" Confidential:", dataset.check_confidentiality()) #print(" ---> self.datasets:", self.datasets) - + class RegisterUser(object): def __init__(self, kw): self.thank_you_mode = False self.errors = [] self.user = Bunch() - + self.user.email_address = kw.get('email_address', '').strip() if not (5 <= len(self.user.email_address) <= 50): self.errors.append('Email Address needs to be between 5 and 50 characters.') - + self.user.full_name = kw.get('full_name', '').strip() if not (5 <= len(self.user.full_name) <= 50): self.errors.append('Full Name needs to be between 5 and 50 characters.') - + self.user.organization = kw.get('organization', '').strip() if self.user.organization and not (5 <= len(self.user.organization) <= 50): self.errors.append('Organization needs to be empty or between 5 and 50 characters.') @@ -128,48 +131,49 @@ class RegisterUser(object): password = str(kw.get('password', '')) if not (6 <= len(password)): self.errors.append('Password needs to be at least 6 characters.') - + if kw.get('password_confirm') != password: self.errors.append("Passwords don't match.") - + if self.errors: return - + print("No errors!") - + self.set_password(password) - + self.user.registration_info = json.dumps(basic_info(), sort_keys=True) - + self.new_user = model.User(**self.user.__dict__) db_session.add(self.new_user) db_session.commit() - + print("Adding verification email to queue") - self.send_email_verification() + #self.send_email_verification() + VerificationEmail(self.new_user) print("Added verification email to queue") - + self.thank_you_mode = True - - + + def set_password(self, password): pwfields = Bunch() - + pwfields.algorithm = "pbkdf2" pwfields.hashfunc = "sha256" #hashfunc = getattr(hashlib, pwfields.hashfunc) - - # Encoding it to base64 makes storing it in json much easier + + # Encoding it to base64 makes storing it in json much easier pwfields.salt = base64.b64encode(os.urandom(32)) # https://forums.lastpass.com/viewtopic.php?t=84104 - pwfields.iterations = 100000 + pwfields.iterations = 100000 pwfields.keylength = 32 - + pwfields.created_ts = timestamp() # One more check on password length assert len(password) >= 6, "Password shouldn't be so short here" - + print("pwfields:", vars(pwfields)) print("locals:", locals()) @@ -178,31 +182,42 @@ class RegisterUser(object): pwfields.iterations, pwfields.keylength, pwfields.hashfunc) - + pwfields.password = enc_password.password pwfields.encrypt_time = enc_password.encrypt_time - + self.user.password = json.dumps(pwfields.__dict__, - sort_keys=True, + sort_keys=True, ) - - def send_email_verification(self): + +class VerificationEmail(object): + template_name = "email/verification.txt" + key_preface = "verification_code" + subject = "GeneNetwork email verification" + + def __init__(self, user): verification_code = str(uuid.uuid4()) - key = "verification_code:" + verification_code - - data = json.dumps(dict(id=self.new_user.id, + key = self.key_preface + "verification_code:" + verification_code + + data = json.dumps(dict(id=user.id, timestamp=timestamp()) ) - + Redis.set(key, data) two_days = 60 * 60 * 24 * 2 - Redis.expire(key, two_days) - to = self.user.email_address - subject = "GeneNetwork email verification" - body = render_template("email/verification.txt", + Redis.expire(key, two_days) + to = user.email_address + subject = self.subject + body = render_template(self.template_name, verification_code = verification_code) send_email(to, subject, body) +class ForgotPasswordEmail(VerificationEmail): + template_name = "email/forgot_password.txt" + key_preface = "forgot_password_code" + subject = "GeneNetwork password reset" + + class Password(object): def __init__(self, unencrypted_password, salt, iterations, keylength, hashfunc): hashfunc = getattr(hashlib, hashfunc) @@ -215,13 +230,14 @@ class Password(object): salt, iterations, keylength, hashfunc) self.encrypt_time = round(time.time() - start_time, 3) print("Creating password took:", self.encrypt_time) - - + + def basic_info(): return dict(timestamp = timestamp(), ip_address = request.remote_addr, user_agent = request.headers.get('User-Agent')) +@app.route("/manage/verify_email") def verify_email(): print("in verify_email request.url is:", request.url) verify_url_hmac(request.url) @@ -232,7 +248,7 @@ def verify_email(): user = model.User.query.get(data['id']) user.confirmed = json.dumps(basic_info(), sort_keys=True) db_session.commit() - + def login(): params = request.form if request.form else request.args print("in login params are:", params) @@ -250,10 +266,10 @@ def login(): print("\n\nComparing:\n{}\n{}\n".format(encrypted.password, pwfields.password)) valid = pbkdf2.safe_str_cmp(encrypted.password, pwfields.password) print("valid is:", valid) - + login_rec = model.Login(user) - - + + if valid: login_rec.successful = True login_rec.session_id = str(uuid.uuid4()) @@ -261,17 +277,17 @@ def login(): session_id_signature = actual_hmac_creation(login_rec.session_id) session_id_signed = login_rec.session_id + ":" + session_id_signature print("session_id_signed:", session_id_signed) - + session = dict(login_time = time.time(), user_id = user.id, user_email_address = user.email_address) - + flash("Thank you for logging in.", "alert-success") - + key = "session_id:" + login_rec.session_id print("Key when signing:", key) Redis.hmset(key, session) - + response = make_response(redirect(url_for('index_page'))) response.set_cookie(UserSession.cookie_name, session_id_signed) else: @@ -281,8 +297,8 @@ def login(): db_session.add(login_rec) db_session.commit() return response - -@app.route("/n/logout") + +@app.route("/n/logout") def logout(): print("Logging out...") UserSession().delete_session() @@ -291,15 +307,36 @@ def logout(): # Delete the cookie response.set_cookie(UserSession.cookie_name, '', expires=0) return response - - + + +@app.route("/n/forgot_password") +def forgot_password(): + return render_template("new_security/forgot_password.html") + +@app.route("/n/forgot_password_submit", methods=('POST',)) +def forgot_password_submit(): + params = request.form + email_address = params['email_address'] + try: + user = model.User.query.filter_by(email_address=email_address).one() + except orm.exc.NoResultFound: + flash("Couldn't find a user associated with the email address {}. Sorry.".format( + email_address)) + return redirect(url_for("login")) + ForgotPasswordEmail(user) + +@app.route("/n/password_reset") +def password_reset(): + pass + + ################################# Sign and unsign ##################################### def url_for_hmac(endpoint, **values): """Like url_for but adds an hmac at the end to insure the url hasn't been tampered with""" url = url_for(endpoint, **values) - + hm = actual_hmac_creation(url) if '?' in url: combiner = "&" @@ -323,10 +360,10 @@ def verify_url_hmac(url): hm = actual_hmac_creation(url) assert hm == hmac, "Unexpected url (stage 3)" - + def actual_hmac_creation(stringy): """Helper function to create the actual hmac""" - + secret = app.config['SECRET_HMAC_CODE'] hmaced = hmac.new(secret, stringy, hashlib.sha1) @@ -339,14 +376,14 @@ def actual_hmac_creation(stringy): app.jinja_env.globals.update(url_for_hmac=url_for_hmac) ####################################################################################### - + def send_email(to, subject, body): msg = json.dumps(dict(From="no-reply@genenetwork.org", To=to, Subject=subject, Body=body)) Redis.rpush("mail_queue", msg) - + #def combined_salt(user_salt): # """Combine the master salt with the user salt...we use two seperate salts so that if the database is compromised, the @@ -358,7 +395,7 @@ def send_email(to, subject, body): # for x, y in user_salt, secret_salt: # combined = combined + x + y # return combined - + class GroupsManager(object): @@ -374,9 +411,5 @@ class RolesManager(object): #class Password(object): # """To generate a master password: dd if=/dev/urandom bs=32 count=1 > master_salt""" -# -# master_salt = - - - - +# +# master_salt = -- cgit v1.2.3 From 612bc953cc82bee292a6a230b6604cc3070ea6d8 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 15 Oct 2013 16:47:08 -0500 Subject: Before getting rid of query_string in DecodeUser --- .../new_security/forgot_password_step2.html | 33 +++++++++ .../templates/new_security/password_reset.html | 78 ++++++++++++++++++++++ .../templates/new_security/register_user.html | 56 +++++++--------- .../wqflask/templates/new_security/registered.html | 11 ++- 4 files changed, 141 insertions(+), 37 deletions(-) create mode 100644 wqflask/wqflask/templates/new_security/forgot_password_step2.html create mode 100644 wqflask/wqflask/templates/new_security/password_reset.html diff --git a/wqflask/wqflask/templates/new_security/forgot_password_step2.html b/wqflask/wqflask/templates/new_security/forgot_password_step2.html new file mode 100644 index 00000000..1295e589 --- /dev/null +++ b/wqflask/wqflask/templates/new_security/forgot_password_step2.html @@ -0,0 +1,33 @@ +{% extends "base.html" %} +{% block title %}Register{% endblock %} +{% block content %} +
+
+

Password Reset

+

+ Check your email. +

+
+
+ +
+ + +

You will receive an email with the subject "{{ subject }}".

+ +

You must click the link in the email to reset the password.

+ +

If you don't see the email, check your spam folder.

+
+ +{% endblock %} + +{% block js %} + + + {% include "new_security/_scripts.html" %} + + +{% endblock %} diff --git a/wqflask/wqflask/templates/new_security/password_reset.html b/wqflask/wqflask/templates/new_security/password_reset.html new file mode 100644 index 00000000..c66ddd07 --- /dev/null +++ b/wqflask/wqflask/templates/new_security/password_reset.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} +{% block title %}Register{% endblock %} +{% block content %} + + {{ header("Password Reset", "Create a new password.") }} + + +
+ + +
+ + +

Enter your new password

+ + + {% if errors %} +
+ Please note: +
    + {% for error in errors %} +
  • {{error}}
  • + {% endfor %} +
+
+ {% endif %} + +
+
+ + + +
+ +
+ +
+
+ + + +
+ +
+ +
+
+ +
+
+ +
+
+ +
+ +
+
+
+ +{% endblock %} + +{% block js %} + + + {% include "new_security/_scripts.html" %} + + +{% endblock %} diff --git a/wqflask/wqflask/templates/new_security/register_user.html b/wqflask/wqflask/templates/new_security/register_user.html index 2a02e7ca..998d2a7b 100644 --- a/wqflask/wqflask/templates/new_security/register_user.html +++ b/wqflask/wqflask/templates/new_security/register_user.html @@ -1,36 +1,31 @@ {% extends "base.html" %} {% block title %}Register{% endblock %} {% block content %} -
-
-

Register

-

- It's easy and fast to make an account. -

-
-
+ + {{ header("Register", "It's fast and easy to make an account.") }} +
- +

Already have an account?

- - + + Sign in using existing account - - + +
- +

Don't have an account?

- +
Register here
- + {% if errors %} -
+
Please note:
    {% for error in errors %} @@ -39,11 +34,11 @@
{% endif %} - +
- +
@@ -51,7 +46,7 @@ data-trigger="change" data-required="true" data-type="email" data-maxlength="50">
- +
@@ -59,28 +54,28 @@ data-trigger="change" data-required="true" data-minlength="5" data-maxlength="50">
- +
- +
-
- +
+ - +
@@ -88,26 +83,25 @@ data-trigger="change" data-required="true" data-equalto="#password">
- +
- + - +
{% endblock %} -{% block js %} +{% block js %} - + {% include "new_security/_scripts.html" %} {% endblock %} - diff --git a/wqflask/wqflask/templates/new_security/registered.html b/wqflask/wqflask/templates/new_security/registered.html index 391a6044..49dc961f 100644 --- a/wqflask/wqflask/templates/new_security/registered.html +++ b/wqflask/wqflask/templates/new_security/registered.html @@ -15,20 +15,19 @@

One last step

-

You will receive an email with the subject "GeneNetwork email verification".

- +

You will receive an email with the subject "{{ subject }}".

+

You must click the link in the email to complete registration.

- +

If you don't see the email, check your spam folder.

{% endblock %} -{% block js %} +{% block js %} - + {% include "new_security/_scripts.html" %} {% endblock %} - -- cgit v1.2.3 From 5b7d03c4af45a6e9b556368746d3ff04b6942a57 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 15 Oct 2013 18:17:02 -0500 Subject: Password reset now works...but needs some error checking still --- .../templates/new_security/password_reset.html | 2 + wqflask/wqflask/user_manager.py | 191 ++++++++++++++++----- wqflask/wqflask/views.py | 64 +------ 3 files changed, 154 insertions(+), 103 deletions(-) diff --git a/wqflask/wqflask/templates/new_security/password_reset.html b/wqflask/wqflask/templates/new_security/password_reset.html index c66ddd07..cda1e477 100644 --- a/wqflask/wqflask/templates/new_security/password_reset.html +++ b/wqflask/wqflask/templates/new_security/password_reset.html @@ -29,8 +29,10 @@
+
+
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 1df2c153..766f49df 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -17,6 +17,8 @@ import hashlib import hmac import base64 +import urlparse + import simplejson as json from sqlalchemy import orm @@ -140,7 +142,7 @@ class RegisterUser(object): print("No errors!") - self.set_password(password) + set_password(password, user) self.user.registration_info = json.dumps(basic_info(), sort_keys=True) @@ -156,48 +158,49 @@ class RegisterUser(object): self.thank_you_mode = True - def set_password(self, password): - pwfields = Bunch() +def set_password(password, user): + pwfields = Bunch() + + pwfields.algorithm = "pbkdf2" + pwfields.hashfunc = "sha256" + #hashfunc = getattr(hashlib, pwfields.hashfunc) - pwfields.algorithm = "pbkdf2" - pwfields.hashfunc = "sha256" - #hashfunc = getattr(hashlib, pwfields.hashfunc) + # Encoding it to base64 makes storing it in json much easier + pwfields.salt = base64.b64encode(os.urandom(32)) - # Encoding it to base64 makes storing it in json much easier - pwfields.salt = base64.b64encode(os.urandom(32)) + # https://forums.lastpass.com/viewtopic.php?t=84104 + pwfields.iterations = 100000 + pwfields.keylength = 32 - # https://forums.lastpass.com/viewtopic.php?t=84104 - pwfields.iterations = 100000 - pwfields.keylength = 32 + pwfields.created_ts = timestamp() + # One more check on password length + assert len(password) >= 6, "Password shouldn't be so short here" - pwfields.created_ts = timestamp() - # One more check on password length - assert len(password) >= 6, "Password shouldn't be so short here" + print("pwfields:", vars(pwfields)) + print("locals:", locals()) - print("pwfields:", vars(pwfields)) - print("locals:", locals()) + enc_password = Password(password, + pwfields.salt, + pwfields.iterations, + pwfields.keylength, + pwfields.hashfunc) - enc_password = Password(password, - pwfields.salt, - pwfields.iterations, - pwfields.keylength, - pwfields.hashfunc) + pwfields.password = enc_password.password + pwfields.encrypt_time = enc_password.encrypt_time - pwfields.password = enc_password.password - pwfields.encrypt_time = enc_password.encrypt_time + user.password = json.dumps(pwfields.__dict__, + sort_keys=True, + ) - self.user.password = json.dumps(pwfields.__dict__, - sort_keys=True, - ) class VerificationEmail(object): template_name = "email/verification.txt" - key_preface = "verification_code" + key_prefix = "verification_code" subject = "GeneNetwork email verification" def __init__(self, user): verification_code = str(uuid.uuid4()) - key = self.key_preface + "verification_code:" + verification_code + key = self.key_prefix + ":" + verification_code data = json.dumps(dict(id=user.id, timestamp=timestamp()) @@ -214,7 +217,7 @@ class VerificationEmail(object): class ForgotPasswordEmail(VerificationEmail): template_name = "email/forgot_password.txt" - key_preface = "forgot_password_code" + key_prefix = "forgot_password_code" subject = "GeneNetwork password reset" @@ -239,16 +242,71 @@ def basic_info(): @app.route("/manage/verify_email") def verify_email(): - print("in verify_email request.url is:", request.url) - verify_url_hmac(request.url) - verification_code = request.args['code'] - data = Redis.get("verification_code:" + verification_code) - data = json.loads(data) - print("data is:", data) - user = model.User.query.get(data['id']) + user = DecodeUser(VerificationEmail.key_prefix).user user.confirmed = json.dumps(basic_info(), sort_keys=True) db_session.commit() +@app.route("/n/password_reset") +def password_reset(): + print("in password_reset request.url is:", request.url) + + # We do this mainly just to assert that it's in proper form for displaying next page + # Really not necessary but doesn't hurt + user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() + + return render_template("new_security/password_reset.html", user_encode=user_encode) + +@app.route("/n/password_reset_step2", methods=('POST',)) +def password_reset_step2(): + print("in password_reset request.url is:", request.url) + + errors = [] + + user_encode = request.form['user_encode'] + verification_code, separator, hmac = user_encode.partition(':') + + hmac_verified = actual_hmac_creation(verification_code) + print("locals are:", locals()) + + + assert hmac == hmac_verified, "Someone has been naughty" + + user = DecodeUser.actual_get_user(ForgotPasswordEmail.key_prefix, verification_code) + print("user is:", user) + + password = request.form['password'] + + set_password(password, user) + db_session.commit() + + flash("Password changed successfully. You can now sign in.", "alert-info") + response = make_response(redirect(url_for('login'))) + + return response + +class DecodeUser(object): + + def __init__(self, code_prefix): + verify_url_hmac(request.url) + + #params = urlparse.parse_qs(url) + + self.verification_code = request.args['code'] + self.user = self.actual_get_user(code_prefix, self.verification_code) + + def reencode_standalone(self): + hmac = actual_hmac_creation(self.verification_code) + return self.verification_code + ":" + hmac + + @staticmethod + def actual_get_user(code_prefix, verification_code): + data = Redis.get(code_prefix + ":" + verification_code) + print("in get_coded_user, data is:", data) + data = json.loads(data) + print("data is:", data) + return model.User.query.get(data['id']) + +@app.route("/n/login", methods=('GET', 'POST')) def login(): params = request.form if request.form else request.args print("in login params are:", params) @@ -282,7 +340,7 @@ def login(): user_id = user.id, user_email_address = user.email_address) - flash("Thank you for logging in.", "alert-success") + flash("Thank you for logging in {}.".format(user.full_name), "alert-success") key = "session_id:" + login_rec.session_id print("Key when signing:", key) @@ -324,10 +382,63 @@ def forgot_password_submit(): email_address)) return redirect(url_for("login")) ForgotPasswordEmail(user) + return render_template("new_security/forgot_password_step2.html", + subject=ForgotPasswordEmail.subject) + + + +@app.route("/manage/users") +def manage_users(): + template_vars = UsersManager() + return render_template("admin/user_manager.html", **template_vars.__dict__) + +@app.route("/manage/user") +def manage_user(): + template_vars = UserManager(request.args) + return render_template("admin/ind_user_manager.html", **template_vars.__dict__) + +@app.route("/manage/groups") +def manage_groups(): + template_vars = GroupsManager(request.args) + return render_template("admin/group_manager.html", **template_vars.__dict__) + + +@app.route("/n/register", methods=('GET', 'POST')) +def register(): + params = None + errors = None + + #if request.form: + # params = request.form + #else: + # params = request.args + + params = request.form if request.form else request.args + + if params: + print("Attempting to register the user...") + result = RegisterUser(params) + errors = result.errors + + if result.thank_you_mode: + assert not errors, "Errors while in thank you mode? That seems wrong..." + return render_template("new_security/registered.html", + subject=VerificationEmail.subject) + + return render_template("new_security/register_user.html", values=params, errors=errors) + + + + +#@app.route("/n/login", methods=('GET', 'POST')) +#def login(): +# return user_manager.login() +# +#@app.route("/manage/verify") +#def verify(): +# user_manager.verify_email() +# return render_template("new_security/verified.html") -@app.route("/n/password_reset") -def password_reset(): - pass ################################# Sign and unsign ##################################### diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index deccf459..a060ba3c 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -51,7 +51,7 @@ from wqflask import user_manager @app.before_request def connect_db(): g.db = sqlalchemy.create_engine(app.config['DB_URI']) - + #@app.before_request #def trace_it(): # from wqflask import tracer @@ -277,68 +277,6 @@ def get_temp_data(): ################################################################################################### -@app.route("/manage/users") -def manage_users(): - template_vars = user_manager.UsersManager() - return render_template("admin/user_manager.html", **template_vars.__dict__) - -@app.route("/manage/user") -def manage_user(): - template_vars = user_manager.UserManager(request.args) - return render_template("admin/ind_user_manager.html", **template_vars.__dict__) - -@app.route("/manage/groups") -def manage_groups(): - template_vars = user_manager.GroupsManager(request.args) - return render_template("admin/group_manager.html", **template_vars.__dict__) - - -@app.route("/n/register", methods=('GET', 'POST')) -def register(): - params = None - errors = None - - #if request.form: - # params = request.form - #else: - # params = request.args - - params = request.form if request.form else request.args - - if params: - print("Attempting to register the user...") - result = user_manager.RegisterUser(params) - errors = result.errors - - if result.thank_you_mode: - assert not errors, "Errors while in thank you mode? That seems wrong..." - return render_template("new_security/registered.html") - - return render_template("new_security/register_user.html", values=params, errors=errors) - -#@app.route("/n/register_submit", methods=('POST',)) -#def register_submit(): -# print("request.args are: ", request.args) -# result = user_manager.RegisterUser(request.form) -# if result.errors: -# print("Redirecting") -# # 307 preserves the post on the redirect (maybe) -# errors = result.errors -# #errors = json.dumps(errors) -# print("request.args are: ", request.args) -# return render_template("new_security/register_user.html", errors=errors, values=request.form) -# #return redirect(url_for('new_register', errors=errors), code=307) - - -@app.route("/n/login", methods=('GET', 'POST')) -def login(): - return user_manager.login() - -@app.route("/manage/verify") -def verify(): - user_manager.verify_email() - return render_template("new_security/verified.html") - ########################################################################## -- cgit v1.2.3 From ad94850e0916346af8cdb72c77f4ef7889d6ee95 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 16 Oct 2013 17:49:38 -0500 Subject: security code cleanup --- .../new_security/forgot_password_step2.html | 11 +--- .../wqflask/templates/new_security/registered.html | 13 +--- .../wqflask/templates/new_security/thank_you.html | 18 ++---- .../wqflask/templates/new_security/verified.html | 32 --------- wqflask/wqflask/templates/security/_macros.html | 39 ----------- wqflask/wqflask/templates/security/_menu.html | 15 ----- wqflask/wqflask/templates/security/_messages.html | 9 --- wqflask/wqflask/templates/security/_scripts.html | 3 - .../templates/security/change_password.html | 11 ---- .../templates/security/email/change_notice.html | 4 -- .../templates/security/email/change_notice.txt | 5 -- .../security/email/confirmation_instructions.html | 5 -- .../security/email/confirmation_instructions.txt | 5 -- .../security/email/login_instructions.html | 5 -- .../security/email/login_instructions.txt | 5 -- .../security/email/reset_instructions.html | 1 - .../security/email/reset_instructions.txt | 3 - .../templates/security/email/reset_notice.html | 1 - .../templates/security/email/reset_notice.txt | 1 - .../wqflask/templates/security/email/welcome.html | 9 --- .../wqflask/templates/security/email/welcome.txt | 9 --- .../templates/security/forgot_password.html | 9 --- wqflask/wqflask/templates/security/login_user.html | 72 --------------------- .../wqflask/templates/security/register_user.html | 75 ---------------------- .../wqflask/templates/security/reset_password.html | 15 ----- .../templates/security/send_confirmation.html | 34 ---------- wqflask/wqflask/templates/security/send_login.html | 9 --- wqflask/wqflask/templates/security/thank_you.html | 8 --- wqflask/wqflask/user_manager.py | 37 ++++------- 29 files changed, 23 insertions(+), 440 deletions(-) delete mode 100644 wqflask/wqflask/templates/new_security/verified.html delete mode 100644 wqflask/wqflask/templates/security/_macros.html delete mode 100644 wqflask/wqflask/templates/security/_menu.html delete mode 100644 wqflask/wqflask/templates/security/_messages.html delete mode 100644 wqflask/wqflask/templates/security/_scripts.html delete mode 100644 wqflask/wqflask/templates/security/change_password.html delete mode 100644 wqflask/wqflask/templates/security/email/change_notice.html delete mode 100644 wqflask/wqflask/templates/security/email/change_notice.txt delete mode 100644 wqflask/wqflask/templates/security/email/confirmation_instructions.html delete mode 100644 wqflask/wqflask/templates/security/email/confirmation_instructions.txt delete mode 100644 wqflask/wqflask/templates/security/email/login_instructions.html delete mode 100644 wqflask/wqflask/templates/security/email/login_instructions.txt delete mode 100644 wqflask/wqflask/templates/security/email/reset_instructions.html delete mode 100644 wqflask/wqflask/templates/security/email/reset_instructions.txt delete mode 100644 wqflask/wqflask/templates/security/email/reset_notice.html delete mode 100644 wqflask/wqflask/templates/security/email/reset_notice.txt delete mode 100644 wqflask/wqflask/templates/security/email/welcome.html delete mode 100644 wqflask/wqflask/templates/security/email/welcome.txt delete mode 100644 wqflask/wqflask/templates/security/forgot_password.html delete mode 100644 wqflask/wqflask/templates/security/login_user.html delete mode 100644 wqflask/wqflask/templates/security/register_user.html delete mode 100644 wqflask/wqflask/templates/security/reset_password.html delete mode 100644 wqflask/wqflask/templates/security/send_confirmation.html delete mode 100644 wqflask/wqflask/templates/security/send_login.html delete mode 100644 wqflask/wqflask/templates/security/thank_you.html diff --git a/wqflask/wqflask/templates/new_security/forgot_password_step2.html b/wqflask/wqflask/templates/new_security/forgot_password_step2.html index 1295e589..888dcad4 100644 --- a/wqflask/wqflask/templates/new_security/forgot_password_step2.html +++ b/wqflask/wqflask/templates/new_security/forgot_password_step2.html @@ -1,14 +1,9 @@ {% extends "base.html" %} {% block title %}Register{% endblock %} {% block content %} -
-
-

Password Reset

-

- Check your email. -

-
-
+ + {{ header("Password Reset", "Check your email.") }} +
+ + diff --git a/wqflask/wqflask/templates/new_security/forgot_password_step2.html b/wqflask/wqflask/templates/new_security/forgot_password_step2.html index 888dcad4..e32ef753 100644 --- a/wqflask/wqflask/templates/new_security/forgot_password_step2.html +++ b/wqflask/wqflask/templates/new_security/forgot_password_step2.html @@ -10,7 +10,7 @@

One last step

-

You will receive an email with the subject "{{ subject }}".

+

You will receive an email with the subject {{ subject }}.

You must click the link in the email to reset the password.

diff --git a/wqflask/wqflask/templates/new_security/registered.html b/wqflask/wqflask/templates/new_security/registered.html index 5c2b2347..349cbdd4 100644 --- a/wqflask/wqflask/templates/new_security/registered.html +++ b/wqflask/wqflask/templates/new_security/registered.html @@ -1,7 +1,7 @@ {% extends "base.html" %} {% block title %}Register{% endblock %} {% block content %} - {{ header("Almost Done", "Thanks for registering")} + {{ header("Almost Done", "Thanks for registering") }}
{% endblock %} diff --git a/wqflask/wqflask/templates/new_security/verification_still_needed.html b/wqflask/wqflask/templates/new_security/verification_still_needed.html new file mode 100644 index 00000000..817fa963 --- /dev/null +++ b/wqflask/wqflask/templates/new_security/verification_still_needed.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} +{% block title %}Verification{% endblock %} +{% block content %} + {{ header("Verification", "You still need to verify") }} + +
+ +

You still need to verify your e-mail address before you can sign in.

+ +

We've resent the verificaiton email. + +

Please check for an email with the subject {{ subject }}.

+ +

You must click the link in the email to complete registration.

+ +

If you don't see the email, check your spam folder.

+
+ +{% endblock %} + +{% block js %} + + + {% include "new_security/_scripts.html" %} + + +{% endblock %} diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 1fe7cce9..b29e6482 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -27,7 +27,7 @@ Check records below and click Add button to add to selection.

- +
{% for header in header_fields %} @@ -38,10 +38,10 @@ {% for this_trait in trait_list %} - + - - - {% for trait in correlation_results %} - - - - - - - - - - - - - - - - - {% endfor %} - -
- + Select - - - + + + + +
+ {% endblock %} + +{% block js %} + +{% endblock %} diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 70aa111e..f519aed9 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -246,6 +246,14 @@ def verify_email(): user.confirmed = json.dumps(basic_info(), sort_keys=True) db_session.commit() + # As long as they have access to the email account + # We might as well log them in + + session_id_signed = successful_login(user) + response = make_response(render_template("new_security/thank_you.html")) + response.set_cookie(UserSession.cookie_name, session_id_signed) + return response + @app.route("/n/password_reset") def password_reset(): print("in password_reset request.url is:", request.url) @@ -325,45 +333,51 @@ def login(): valid = pbkdf2.safe_str_cmp(encrypted.password, pwfields.password) print("valid is:", valid) - login_rec = model.Login(user) - - if valid and not user.confirmed: - # User needs to confirm before we log them in... - flash("You still need to verify your email address." - "We've resent the verification email. " - "Please check your email and follow the instructions.", "alert-error") - VerificationEmail(user) - return redirect((url_for('login'))) - elif valid: - login_rec.successful = True - login_rec.session_id = str(uuid.uuid4()) - #session_id = "session_id:{}".format(login_rec.session_id) - session_id_signature = actual_hmac_creation(login_rec.session_id) - session_id_signed = login_rec.session_id + ":" + session_id_signature - print("session_id_signed:", session_id_signed) - - session = dict(login_time = time.time(), - user_id = user.id, - user_email_address = user.email_address) - - flash("Thank you for logging in {}.".format(user.full_name), "alert-success") + return render_template("new_security/verification_still_needed.html", + subject=VerificationEmail.subject) - key = "session_id:" + login_rec.session_id - print("Key when signing:", key) - Redis.hmset(key, session) + if valid: + session_id_signed = successful_login(user) + flash("Thank you for logging in {}.".format(user.full_name), "alert-success") response = make_response(redirect(url_for('index_page'))) response.set_cookie(UserSession.cookie_name, session_id_signed) else: - login_rec.successful = False + unsuccessful_login(user) flash("Invalid email-address or password. Please try again.", "alert-error") response = make_response(redirect(url_for('login'))) - db_session.add(login_rec) - db_session.commit() + return response + +def successful_login(user): + login_rec = model.Login(user) + login_rec.successful = True + login_rec.session_id = str(uuid.uuid4()) + #session_id = "session_id:{}".format(login_rec.session_id) + session_id_signature = actual_hmac_creation(login_rec.session_id) + session_id_signed = login_rec.session_id + ":" + session_id_signature + print("session_id_signed:", session_id_signed) + + session = dict(login_time = time.time(), + user_id = user.id, + user_email_address = user.email_address) + + key = "session_id:" + login_rec.session_id + print("Key when signing:", key) + Redis.hmset(key, session) + db_session.add(login_rec) + db_session.commit() + return session_id_signed + +def unsuccessful_login(user): + login_rec = model.Login(user) + login_rec.successful = False + db_session.add(login_rec) + db_session.commit() + @app.route("/n/logout") def logout(): print("Logging out...") @@ -459,6 +473,11 @@ def url_for_hmac(endpoint, **values): combiner = "?" return url + combiner + "hm=" + hm +def data_hmac(stringy): + """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with""" + return stringy + ":" + actual_hmac_creation(stringy) + + def verify_url_hmac(url): """Pass in a url that was created with url_hmac and this assures it hasn't been tampered with""" print("url passed in to verify is:", url) @@ -488,7 +507,8 @@ def actual_hmac_creation(stringy): hm = hm[:20] return hm -app.jinja_env.globals.update(url_for_hmac=url_for_hmac) +app.jinja_env.globals.update(url_for_hmac=url_for_hmac, + data_hmac=data_hmac) ####################################################################################### diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index a060ba3c..6c9addbc 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -43,6 +43,7 @@ from utility.benchmark import Bench from pprint import pformat as pf from wqflask import user_manager +from wqflask import collect #import logging #logging.basicConfig(filename="/tmp/gn_log", level=logging.INFO) -- cgit v1.2.3 From 5c5671f3c3353f6adc1ce5d9cab8f9f4de5a4e05 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 23 Oct 2013 16:00:40 -0500 Subject: Adding new collections works --- wqflask/wqflask/collect.py | 14 ++++++++-- wqflask/wqflask/model.py | 65 +++++++++------------------------------------- 2 files changed, 24 insertions(+), 55 deletions(-) diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 4ec37014..f61ea3dc 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -51,7 +51,10 @@ def collections_add(): @app.route("/collections/new") def collections_new(): - new_collection = request.args['new_collection'] + uc = model.UserCollection() + uc.name = request.args['new_collection'] + print("user_session:", g.user_session.__dict__) + uc.user = g.user_session.record['user_id'] unprocessed_traits = request.args['traits'] print("unprocessed_traits are:", unprocessed_traits) unprocessed_traits = unprocessed_traits.split(",") @@ -64,5 +67,12 @@ def collections_new(): assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?" traits.add(str(data)) + uc.members = json.dumps(list(traits)) print("traits are:", traits) - return "Created: " + new_collection + + db_session.add(uc) + db_session.commit() + + + + return "Created: " + uc.name diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py index ca1364d7..c89dc80a 100644 --- a/wqflask/wqflask/model.py +++ b/wqflask/wqflask/model.py @@ -7,10 +7,6 @@ import simplejson as json from flask import request from flask.ext.sqlalchemy import SQLAlchemy -#from flask.ext.security import Security, SQLAlchemyUserDatastore, UserMixin, RoleMixin - -#from flask_security.forms import TextField -#from flask_security.forms import RegisterForm from wqflask import app @@ -19,33 +15,7 @@ from sqlalchemy.orm import relationship, backref from wqflask.database import Base, init_db -# Create database connection object -#db = SQLAlchemy(app) - - -# Is this right? -Sam -#from sqlalchemy.ext.declarative import declarative_base -#Base = declarative_base() - -#@classmethod -#def get(cls, key): -# """Convenience get method using the primary key -# -# If record doesn't exist, returns None -# -# Allows the following: User.get('121') -# -# """ -# print("in get cls is:", cls) -# print(" key is {} : {}".format(type(key), key)) -# query = Model.query(cls) -# print("query is: ", query) -# record = query.get(key) -# return record -# -# -#print("Model is:", vars(Model)) -#Model.get = get + # Define models #roles_users = Table('roles_users', @@ -83,10 +53,7 @@ class User(Base): @property def login_count(self): return self.logins.filter_by(successful=True).count() - #return self.query.filter - #return len(self.logins) - #return 8 - #return len(self.logins.query.filter(User.logins.has(successful=True))) + @property def confirmed_at(self): @@ -104,16 +71,10 @@ class User(Base): return None - - #last_login_at = Column(DateTime()) - #current_login_at = Column(DateTime()) - #last_login_ip = Column(Unicode(39)) - #current_login_ip = Column(Unicode(39)) - #login_count = Column(Integer()) - #roles = relationship('Role', secondary=roles_users, # backref=backref('users', lazy='dynamic')) + class Login(Base): __tablename__ = "login" id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) @@ -127,15 +88,13 @@ class Login(Base): self.user = user.id self.ip_address = request.remote_addr -# Setup Flask-Security -#user_datastore = SQLAlchemyUserDatastore(db, User, Role) - -#class ExtendedRegisterForm(RegisterForm): -# name = TextField('name') -# #print("name is:", name['_name'], vars(name)) -# organization = TextField('organization') -# -#security = Security(app, user_datastore, register_form=ExtendedRegisterForm) - +################################################################################################## -#user_datastore.create_role(name="Genentech", description="Genentech Beta Project(testing)") +class UserCollection(Base): + __tablename__ = "user_collection" + id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) + user = Column(Unicode(36), ForeignKey('user.id')) + name = Column(Text) + created_timestamp = Column(DateTime(), default=lambda: datetime.datetime.utcnow()) + changed_timestamp = Column(DateTime(), default=lambda: datetime.datetime.utcnow()) + members = Column(Text) # We're going to store them as a json list -- cgit v1.2.3 From 4eb2c2244b083d6522db02dff5ddb036af496392 Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Thu, 24 Oct 2013 17:59:11 -0500 Subject: Added a dynamically resortable bar chart to the trait page Improved the look/feel of the correlation page somewhat --- wqflask/other_config/nginx_conf/penguin.conf | 2 +- wqflask/wqflask/correlation/show_corr_results.py | 15 ++ wqflask/wqflask/static/new/css/bar_chart.css | 14 ++ .../static/new/javascript/show_trait.coffee | 184 ++++++++++++++++++++- .../wqflask/static/new/javascript/show_trait.js | 183 +++++++++++++++++++- wqflask/wqflask/templates/correlation_page.html | 119 +++++++------ wqflask/wqflask/templates/quick_search.html | 7 +- wqflask/wqflask/templates/search_result_page.html | 3 +- wqflask/wqflask/templates/show_trait.html | 3 +- .../templates/show_trait_statistics_new.html | 16 ++ wqflask/wqflask/views.py | 2 +- 11 files changed, 481 insertions(+), 67 deletions(-) create mode 100644 wqflask/wqflask/static/new/css/bar_chart.css create mode 100644 wqflask/wqflask/templates/show_trait_statistics_new.html diff --git a/wqflask/other_config/nginx_conf/penguin.conf b/wqflask/other_config/nginx_conf/penguin.conf index 822556d3..5c380da8 100644 --- a/wqflask/other_config/nginx_conf/penguin.conf +++ b/wqflask/other_config/nginx_conf/penguin.conf @@ -2,7 +2,7 @@ server { # Modeled after http://flask.pocoo.org/docs/deploying/wsgi-standalone/ listen 80; - server_name penguin.uthsc.edu; + server_name gn2python.genenetwork.org; access_log /var/log/nginx/access.log; error_log /var/log/nginx/error.log; diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 8f23165c..0b66bc61 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -102,6 +102,7 @@ class CorrelationResults(object): self.sample_data = {} self.corr_type = start_vars['corr_type'] self.corr_method = start_vars['corr_sample_method'] + self.get_formatted_corr_type() self.return_number = 50 #The two if statements below append samples to the sample list based upon whether the user @@ -239,6 +240,20 @@ class CorrelationResults(object): ############################################################################################################################################ + def get_formatted_corr_type(self): + self.formatted_corr_type = "" + if self.corr_type == "lit": + self.formatted_corr_type += "Literature Correlation " + elif self.corr_type == "tissue": + self.formatted_corr_type += "Tissue Correlation " + elif self.corr_type == "sample": + self.formatted_corr_type += "Genetic Correlation " + + if self.corr_method == "pearson": + self.formatted_corr_type += "(Pearson's r)" + elif self.corr_method == "spearman": + self.formatted_corr_type += "(Spearman's rho)" + def do_tissue_correlation_for_trait_list(self, tissue_dataset_id=1): """Given a list of correlation results (self.correlation_results), gets the tissue correlation value for each""" diff --git a/wqflask/wqflask/static/new/css/bar_chart.css b/wqflask/wqflask/static/new/css/bar_chart.css new file mode 100644 index 00000000..ba14fe4e --- /dev/null +++ b/wqflask/wqflask/static/new/css/bar_chart.css @@ -0,0 +1,14 @@ +.axis path, +.axis line { + fill: none; + stroke: #000; + shape-rendering: crispEdges; +} + +.bar { + fill: steelblue; +} + +.x.axis path { + display: none; +} \ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/show_trait.coffee b/wqflask/wqflask/static/new/javascript/show_trait.coffee index 0f16ac68..2e049e6a 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.coffee +++ b/wqflask/wqflask/static/new/javascript/show_trait.coffee @@ -56,16 +56,191 @@ Stat_Table_Rows = [ url: "/glossary.html#Interquartile" digits: 2 } - ] $ -> + class Histogram + constructor: (@sample_list, @sample_group) -> + @get_samples() + console.log("sample names:", @sample_names) + + #Used to calculate the bottom margin so sample names aren't cut off + longest_sample_name = d3.max(sample.length for sample in @sample_names) + + @margin = {top: 20, right: 20, bottom: longest_sample_name * 7, left: 40} + @plot_width = @sample_vals.length * 15 - @margin.left - @margin.right + @plot_height = 500 - @margin.top - @margin.bottom + + @x_buffer = @plot_width/20 + @y_buffer = @plot_height/20 + + @y_min = d3.min(@sample_vals) + @y_max = d3.max(@sample_vals) * 1.1 + + @svg = @create_svg() + + @plot_height -= @y_buffer + @create_scales() + @create_graph() + + d3.select("#update_bar_chart").on("click", => + if $("#update_bar_chart").html() == 'Sort By Value' + $("#update_bar_chart").html('Sort By Name') + sortItems = (a, b) -> + return a[1] - b[1] + + @svg.selectAll(".bar") + .data(@sorted_samples()) + .transition() + .duration(1000) + .attr("y", (d) => + return @y_scale(d[1]) + ) + .attr("height", (d) => + return @plot_height - @y_scale(d[1]) + ) + sorted_sample_names = (sample[0] for sample in @sorted_samples()) + x_scale = d3.scale.ordinal() + .domain(sorted_sample_names) + .rangeRoundBands([0, @plot_width], .1) + $('.x.axis').remove() + @add_x_axis(x_scale) + else + $("#update_bar_chart").html('Sort By Value') + @svg.selectAll(".bar") + .data(@sample_vals) + .transition() + .duration(1000) + .attr("y", (d) => + return @y_scale(d) + ) + .attr("height", (d) => + return @plot_height - @y_scale(d) + ) + x_scale = d3.scale.ordinal() + .domain(@sample_names) + .rangeRoundBands([0, @plot_width], .1) + $('.x.axis').remove() + @add_x_axis(x_scale) + + ) + + get_samples: () -> + @sample_names = (sample.name for sample in @sample_list when sample.value != null) + @sample_vals = (sample.value for sample in @sample_list when sample.value != null) + + create_svg: () -> + svg = d3.select("#bar_chart") + .append("svg") + .attr("class", "bar_chart") + .attr("width", @plot_width + @margin.left + @margin.right) + .attr("height", @plot_height + @margin.top + @margin.bottom) + .append("g") + .attr("transform", "translate(" + @margin.left + "," + @margin.top + ")") + + return svg + + create_scales: () -> + @x_scale = d3.scale.ordinal() + .domain(@sample_names) + .rangeRoundBands([0, @plot_width], .1) + + @y_scale = d3.scale.linear() + .domain([@y_min * 0.75, @y_max]) + .range([@plot_height, @y_buffer]) + + create_graph: () -> + + #@add_border() + @add_x_axis(@x_scale) + @add_y_axis() + + @add_bars() + + add_x_axis: (scale) -> + xAxis = d3.svg.axis() + .scale(scale) + .orient("bottom"); + + @svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + @plot_height + ")") + .call(xAxis) + .selectAll("text") + .style("text-anchor", "end") + .style("font-size", "12px") + .attr("dx", "-.8em") + .attr("dy", "-.3em") + .attr("transform", (d) => + return "rotate(-90)" + ) + + add_y_axis: () -> + yAxis = d3.svg.axis() + .scale(@y_scale) + .orient("left") + .ticks(5) + + @svg.append("g") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + + add_bars: () -> + @svg.selectAll(".bar") + .data(_.zip(@sample_names, @sample_vals)) + .enter().append("rect") + .attr("class", "bar") + .attr("x", (d) => + return @x_scale(d[0]) + ) + .attr("width", @x_scale.rangeBand()) + .attr("y", (d) => + return @y_scale(d[1]) + ) + .attr("height", (d) => + return @plot_height - @y_scale(d[1]) + ) + .append("svg:title") + .text((d) => + return d[1] + ) + + sorted_samples: () -> + sample_list = _.zip(@sample_names, @sample_vals) + sorted = _.sortBy(sample_list, (sample) => + return sample[1] + ) + console.log("sorted:", sorted) + return sorted + + sample_lists = js_data.sample_lists + sample_group_types = js_data.sample_group_types + + new Histogram(sample_lists[0]) + + $('.stats_samples_group').change -> + $('#bar_chart').remove() + $('#bar_chart_container').append('
') + group = $(this).val() + console.log("group:", group) + if group == "samples_primary" + new Histogram(sample_lists[0]) + else if group == "samples_other" + new Histogram(sample_lists[1]) + else if group == "samples_all" + all_samples = sample_lists[0].concat sample_lists[1] + new Histogram(all_samples) + + hide_tabs = (start) -> for x in [start..10] $("#stats_tabs" + x).hide() - #hide_tabs(1) - # Changes stats table between all, bxd only and non-bxd, etc. stats_mdp_change = -> selected = $(this).val() @@ -81,7 +256,6 @@ $ -> current_value = parseFloat($(in_box)).toFixed(decimal_places) - console.log("urgh:", category, value_type) the_value = sample_sets[category][value_type]() console.log("After running sample_sets, the_value is:", the_value) if decimal_places > 0 @@ -121,7 +295,6 @@ $ -> tables = ['samples_primary', 'samples_other'] for table in tables rows = $("#" + table).find('tr') - console.log("[fuji3] rows:", rows) for row in rows name = $(row).find('.edit_sample_sample_name').html() name = $.trim(name) @@ -180,7 +353,6 @@ $ -> $("#stats_table").append(table) - process_id = (values...) -> ### Make an id or a class valid javascript by, for example, eliminating spaces ### processed = "" diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index f554267f..e6fcbd7b 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -56,7 +56,186 @@ ]; $(function() { - var block_by_attribute_value, block_by_index, block_outliers, change_stats_value, create_value_dropdown, edit_data_change, export_sample_table_data, get_sample_table_data, hide_no_value, hide_tabs, make_table, on_corr_method_change, populate_sample_attributes_values_dropdown, process_id, reset_samples_table, show_hide_outliers, stats_mdp_change, update_stat_values; + var Histogram, block_by_attribute_value, block_by_index, block_outliers, change_stats_value, create_value_dropdown, edit_data_change, export_sample_table_data, get_sample_table_data, hide_no_value, hide_tabs, make_table, on_corr_method_change, populate_sample_attributes_values_dropdown, process_id, reset_samples_table, sample_group_types, sample_lists, show_hide_outliers, stats_mdp_change, update_stat_values; + Histogram = (function() { + + function Histogram(sample_list, sample_group) { + var longest_sample_name, sample, + _this = this; + this.sample_list = sample_list; + this.sample_group = sample_group; + this.get_samples(); + console.log("sample names:", this.sample_names); + longest_sample_name = d3.max((function() { + var _i, _len, _ref, _results; + _ref = this.sample_names; + _results = []; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + sample = _ref[_i]; + _results.push(sample.length); + } + return _results; + }).call(this)); + this.margin = { + top: 20, + right: 20, + bottom: longest_sample_name * 7, + left: 40 + }; + this.plot_width = this.sample_vals.length * 15 - this.margin.left - this.margin.right; + this.plot_height = 500 - this.margin.top - this.margin.bottom; + this.x_buffer = this.plot_width / 20; + this.y_buffer = this.plot_height / 20; + this.y_min = d3.min(this.sample_vals); + this.y_max = d3.max(this.sample_vals) * 1.1; + this.svg = this.create_svg(); + this.plot_height -= this.y_buffer; + this.create_scales(); + this.create_graph(); + d3.select("#update_bar_chart").on("click", function() { + var sortItems, sorted_sample_names, x_scale; + if ($("#update_bar_chart").html() === 'Sort By Value') { + $("#update_bar_chart").html('Sort By Name'); + sortItems = function(a, b) { + return a[1] - b[1]; + }; + _this.svg.selectAll(".bar").data(_this.sorted_samples()).transition().duration(1000).attr("y", function(d) { + return _this.y_scale(d[1]); + }).attr("height", function(d) { + return _this.plot_height - _this.y_scale(d[1]); + }); + sorted_sample_names = (function() { + var _i, _len, _ref, _results; + _ref = this.sorted_samples(); + _results = []; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + sample = _ref[_i]; + _results.push(sample[0]); + } + return _results; + }).call(_this); + x_scale = d3.scale.ordinal().domain(sorted_sample_names).rangeRoundBands([0, _this.plot_width], .1); + $('.x.axis').remove(); + return _this.add_x_axis(x_scale); + } else { + $("#update_bar_chart").html('Sort By Value'); + _this.svg.selectAll(".bar").data(_this.sample_vals).transition().duration(1000).attr("y", function(d) { + return _this.y_scale(d); + }).attr("height", function(d) { + return _this.plot_height - _this.y_scale(d); + }); + x_scale = d3.scale.ordinal().domain(_this.sample_names).rangeRoundBands([0, _this.plot_width], .1); + $('.x.axis').remove(); + return _this.add_x_axis(x_scale); + } + }); + } + + Histogram.prototype.get_samples = function() { + var sample; + this.sample_names = (function() { + var _i, _len, _ref, _results; + _ref = this.sample_list; + _results = []; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + sample = _ref[_i]; + if (sample.value !== null) { + _results.push(sample.name); + } + } + return _results; + }).call(this); + return this.sample_vals = (function() { + var _i, _len, _ref, _results; + _ref = this.sample_list; + _results = []; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + sample = _ref[_i]; + if (sample.value !== null) { + _results.push(sample.value); + } + } + return _results; + }).call(this); + }; + + Histogram.prototype.create_svg = function() { + var svg; + svg = d3.select("#bar_chart").append("svg").attr("class", "bar_chart").attr("width", this.plot_width + this.margin.left + this.margin.right).attr("height", this.plot_height + this.margin.top + this.margin.bottom).append("g").attr("transform", "translate(" + this.margin.left + "," + this.margin.top + ")"); + return svg; + }; + + Histogram.prototype.create_scales = function() { + this.x_scale = d3.scale.ordinal().domain(this.sample_names).rangeRoundBands([0, this.plot_width], .1); + return this.y_scale = d3.scale.linear().domain([this.y_min * 0.75, this.y_max]).range([this.plot_height, this.y_buffer]); + }; + + Histogram.prototype.create_graph = function() { + this.add_x_axis(this.x_scale); + this.add_y_axis(); + return this.add_bars(); + }; + + Histogram.prototype.add_x_axis = function(scale) { + var xAxis, + _this = this; + xAxis = d3.svg.axis().scale(scale).orient("bottom"); + return this.svg.append("g").attr("class", "x axis").attr("transform", "translate(0," + this.plot_height + ")").call(xAxis).selectAll("text").style("text-anchor", "end").style("font-size", "12px").attr("dx", "-.8em").attr("dy", "-.3em").attr("transform", function(d) { + return "rotate(-90)"; + }); + }; + + Histogram.prototype.add_y_axis = function() { + var yAxis; + yAxis = d3.svg.axis().scale(this.y_scale).orient("left").ticks(5); + return this.svg.append("g").attr("class", "y axis").call(yAxis).append("text").attr("transform", "rotate(-90)").attr("y", 6).attr("dy", ".71em").style("text-anchor", "end"); + }; + + Histogram.prototype.add_bars = function() { + var _this = this; + return this.svg.selectAll(".bar").data(_.zip(this.sample_names, this.sample_vals)).enter().append("rect").attr("class", "bar").attr("x", function(d) { + return _this.x_scale(d[0]); + }).attr("width", this.x_scale.rangeBand()).attr("y", function(d) { + return _this.y_scale(d[1]); + }).attr("height", function(d) { + return _this.plot_height - _this.y_scale(d[1]); + }).append("svg:title").text(function(d) { + return d[1]; + }); + }; + + Histogram.prototype.sorted_samples = function() { + var sample_list, sorted, + _this = this; + sample_list = _.zip(this.sample_names, this.sample_vals); + sorted = _.sortBy(sample_list, function(sample) { + return sample[1]; + }); + console.log("sorted:", sorted); + return sorted; + }; + + return Histogram; + + })(); + sample_lists = js_data.sample_lists; + sample_group_types = js_data.sample_group_types; + new Histogram(sample_lists[0]); + $('.stats_samples_group').change(function() { + var all_samples, group; + $('#bar_chart').remove(); + $('#bar_chart_container').append('
'); + group = $(this).val(); + console.log("group:", group); + if (group === "samples_primary") { + return new Histogram(sample_lists[0]); + } else if (group === "samples_other") { + return new Histogram(sample_lists[1]); + } else if (group === "samples_all") { + all_samples = sample_lists[0].concat(sample_lists[1]); + return new Histogram(all_samples); + } + }); hide_tabs = function(start) { var x, _i, _results; _results = []; @@ -77,7 +256,6 @@ console.log("the_id:", id); in_box = $(id).html; current_value = parseFloat($(in_box)).toFixed(decimal_places); - console.log("urgh:", category, value_type); the_value = sample_sets[category][value_type](); console.log("After running sample_sets, the_value is:", the_value); if (decimal_places > 0) { @@ -127,7 +305,6 @@ for (_i = 0, _len = tables.length; _i < _len; _i++) { table = tables[_i]; rows = $("#" + table).find('tr'); - console.log("[fuji3] rows:", rows); for (_j = 0, _len1 = rows.length; _j < _len1; _j++) { row = rows[_j]; name = $(row).find('.edit_sample_sample_name').html(); diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 7e149506..d675b801 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -9,55 +9,76 @@ {{ header("Correlation", 'Trait: {} Dataset: {}'.format(this_trait.name, dataset.name)) }} - - - - - - - - - - - - {% if corr_method == 'pearson' %} - - - - - - - {% else %} - - - - - - {% endif %} +
+ + +

Values of record {{ this_trait.name }} in the {{ dataset.fullname }} + dataset were compared to all records in the {{ target_dataset.fullname }} + dataset. The top {{ return_number }} correlations ranked by the {{ formatted_corr_type }} are displayed. + You can resort this list by clicking the headers. Select the Record ID to open the trait data + and analysis page. +

+ +
+
TraitSymbolAliasDescriptionLocationMean ExprMax LRSMax LRS LocationSample rN CasesSample p(r)Lit CorrTissue rTissue p(r)Sample rhoSample p(rho)Lit CorrTissue rhoTissue p(rho)
+ + + + + + + + + + {% if corr_method == 'pearson' %} + + + + + + + {% else %} + + + + + + + {% endif %} + + + + {% for trait in correlation_results %} + + + + + + + + + + + + + + + + {% endfor %} + +
TraitSymbolDescriptionLocationMean ExprMax LRSMax LRS LocationSample rN CasesSample p(r)Lit CorrTissue rTissue p(r)Sample rhoN CasesSample p(rho)Lit CorrTissue rhoTissue p(rho)
{{ trait.name }}{{ trait.symbol }}{{ trait.description }}

Aliases: {{ trait.alias }}
Chr{{ trait.chr }}: {{'%0.3f'|format(trait.mb) }}{{'%0.3f'|format(trait.mean)}}{{'%0.3f'|format(trait.lrs)}}Chr{{ trait.locus_chr }}: {{'%0.3f'|format(trait.locus_mb) }}{{'%0.3f'|format(trait.sample_r)}}{{ trait.num_overlap }}{{'%0.3e'|format(trait.sample_p)}}{{'%0.3f'|format(trait.lit_corr)}}{{'%0.3f'|format(trait.tissue_corr)}}{{'%0.3e'|format(trait.tissue_pvalue)}}
-
{{ trait.name }}{{ trait.symbol }}{{ trait.alias }}{{ trait.description }}Chr{{ trait.chr }}:{{trait.mb}}{{'%0.3f'|format(trait.mean)}}{{'%0.3f'|format(trait.lrs)}}Chr{{ trait.locus_chr }}:{{'%0.6f'|format(trait.locus_mb)}}{{'%0.3f'|format(trait.sample_r)}}{{ trait.num_overlap }}{{'%0.3e'|format(trait.sample_p)}}{{'%0.3f'|format(trait.lit_corr)}}{{'%0.3f'|format(trait.tissue_corr)}}{{'%0.3e'|format(trait.tissue_pvalue)}}
+
+ + +
+
{% endblock %} {% block js %} @@ -92,4 +113,4 @@ console.timeEnd("Creating table"); }); -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/wqflask/wqflask/templates/quick_search.html b/wqflask/wqflask/templates/quick_search.html index 2f268c5a..fe6f3f65 100644 --- a/wqflask/wqflask/templates/quick_search.html +++ b/wqflask/wqflask/templates/quick_search.html @@ -3,15 +3,14 @@ {% block content %} - {{ header("QuickSearch Results", - 'GeneNetwork found {}.'.format(numify(results|count, "record", "records"))) }} + {{ header("QuickSearch Results") }}
-

We across all data sets to find all records that match:

+

We searched across all data sets to find all records that match:

-

We searched {{ dataset.fullname }} +

We searched {{ dataset.fullname }}

To find all records that match: