path: root/server.py

#!/bin/env  python3
#
# This is the main flask server

from __future__ import print_function

import hashlib
import json
import os
import random
import re
import shutil
import string
import tempfile
from os import listdir

import bcrypt
import nltk
import numpy as np
import pandas as pd
import pytz
from flask import (Flask, Response, flash, jsonify, redirect, render_template,
                   request, session, url_for)
from flask_sqlalchemy import SQLAlchemy
from numpy import array

# nltk.download('punkt') -- let's not just download data

import pickle
from collections import Counter
from datetime import datetime

import tensorflow
import tensorflow.keras
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from tensorflow.keras import backend as K
from tensorflow.keras import metrics, optimizers
from tensorflow.keras.layers import *
from tensorflow.keras.layers import Dense, Embedding, Flatten
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.preprocessing.text import Tokenizer

from more_functions import *

app=Flask(__name__)
#datadir="/export/ratspub/"
#datadir = "."
datadir="./"

app.config['SECRET_KEY'] = '#DtfrL98G5t1dC*4'
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+datadir+'userspub.sqlite'
db = SQLAlchemy(app)
nltk.data.path.append("./nlp/")

VERSION=None

def version():
    global VERSION
    if VERSION is None:
        with open("VERSION", 'r') as file:
            VERSION = file.read()
    return VERSION

# Sqlite database
class users(db.Model):
    __tablename__='user'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(80), nullable=False)
    email = db.Column(db.String(80), unique=True, nullable=False)
    password = db.Column(db.String(128), nullable=False)
    date_created = db.Column(db.DateTime, default=datetime.utcnow)

# Preprocessing of words for CNN
def clean_doc(doc, vocab):
    doc = doc.lower()
    tokens = doc.split()
    re_punc = re.compile('[%s]' % re.escape(string.punctuation))
    tokens = [re_punc.sub('' , w) for w in tokens]
    tokens = [word for word in tokens if len(word) > 1]
    stop_words = set(stopwords.words('english'))
    tokens = [w for w in tokens if not w in stop_words]
    porter = PorterStemmer()
    stemmed = [porter.stem(word) for word in tokens]
    return tokens

# Load tokenizer
with open('./nlp/tokenizer.pickle', 'rb') as handle:
    tokenizer = pickle.load(handle)

# Load vocabulary
with open('./nlp/vocabulary.txt', 'r') as vocab:
    vocab = vocab.read()

def tf_auc_score(y_true, y_pred):
    return tensorflow.metrics.auc(y_true, y_pred)[1]

K.clear_session()

# Create the CNN model
def create_model(vocab_size, max_length):
    model = Sequential()
    model.add(Embedding(vocab_size, 32, input_length=max_length))
    model.add(Conv1D(filters=16, kernel_size=4, activation='relu'))
    model.add(MaxPooling1D(pool_size=2))
    model.add(Flatten())
    model.add(Dense(10, activation='relu'))
    model.add(Dense(1, activation='sigmoid'))
    opt = tensorflow.keras.optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999)
    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[tf_auc_score])
    return model

# Use addiction ontology by default
onto_cont=open("addiction.onto","r").read()
dictionary=ast.literal_eval(onto_cont)


@app.route("/")
def root():
    if 'email' in session:
        ontoarchive()
        onto_len_dir = session['onto_len_dir']
        onto_list = session['onto_list']
    else:
        onto_len_dir = 0
        onto_list = ''

    onto_cont=open("addiction.onto","r").read()
    dict_onto=ast.literal_eval(onto_cont)
    return render_template('index.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())


@app.route("/login", methods=["POST", "GET"])
def login():
    onto_len_dir = 0
    onto_list = ''
    onto_cont=open("addiction.onto","r").read()
    dict_onto=ast.literal_eval(onto_cont)
    email = None

    if request.method == "POST":
        email = request.form['email']
        password = request.form['password']
        found_user = users.query.filter_by(email=email).first()
        if (found_user and (bcrypt.checkpw(password.encode('utf8'), found_user.password))):
            session['email'] = found_user.email
            #print(bcrypt.hashpw(session['email'].encode('utf8'), bcrypt.gensalt()))
            session['hashed_email'] = hashlib.md5(session['email'] .encode('utf-8')).hexdigest()
            session['name'] = found_user.name
            session['id'] = found_user.id
            flash("Login Succesful!")
            ontoarchive()
            onto_len_dir = session['onto_len_dir']
            onto_list = session['onto_list']
        else:
            flash("Invalid username or password!", "inval")
            return render_template('signup.html',version=version())
    return render_template('index.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())


@app.route("/signup", methods=["POST", "GET"])
def signup():
    onto_len_dir = 0
    onto_list = ''
    onto_cont=open("addiction.onto","r").read()
    dict_onto=ast.literal_eval(onto_cont)

    if request.method == "POST":
        name = request.form['name']
        email = request.form['email']
        password = request.form['password']
        found_user = users.query.filter_by(email=email).first()

        if (found_user and (bcrypt.checkpw(password.encode('utf8'), found_user.password)==False)):
            flash("Already registered, but wrong password!", "inval")
            return render_template('signup.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())

        session['email'] = email
        session['hashed_email'] = hashlib.md5(session['email'] .encode('utf-8')).hexdigest()
        session['name'] = name
        password = bcrypt.hashpw(password.encode('utf8'), bcrypt.gensalt())
        user = users(name=name, email=email, password = password)
        if found_user:
            session['email'] = found_user.email
            session['hashed_email'] = hashlib.md5(session['email'] .encode('utf-8')).hexdigest()
            session['id'] = found_user.id
            found_user.name = name
            db.session.commit()
            ontoarchive()
            onto_len_dir = session['onto_len_dir']
            onto_list = session['onto_list']
        else:
            db.session.add(user)
            db.session.commit()
            newuser = users.query.filter_by(email=session['email']).first()
            session['id'] = newuser.id
            os.makedirs(datadir+"/user/"+str(session['hashed_email']))
            session['user_folder'] = datadir+"/user/"+str(session['hashed_email'])
            os.makedirs(session['user_folder']+"/ontology/")

        flash("Login Succesful!")
        return render_template('index.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())
    else:
        if 'email' in session:
            flash("Already Logged In!")
            return render_template('index.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())
        return render_template('signup.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())


@app.route("/signin", methods=["POST", "GET"])
def signin():
    email = None
    if request.method == "POST":
        email = request.form['email']
        password = request.form['password']
        found_user = users.query.filter_by(email=email).first()

        if (found_user and (bcrypt.checkpw(password.encode('utf8'), found_user.password))):
            session['email'] = found_user.email
            session['hashed_email'] = hashlib.md5(session['email'].encode('utf-8')).hexdigest()
            session['name'] = found_user.name
            session['id'] = found_user.id
            flash("Login Succesful!")
            #onto_len_dir = 0
            #onto_list = ''
            onto_cont=open("addiction.onto","r").read()
            ontoarchive()
            onto_len_dir = session['onto_len_dir']
            onto_list = session['onto_list']
            dict_onto=ast.literal_eval(onto_cont)
            return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())
        else:
            flash("Invalid username or password!", "inval")
            return render_template('signup.html',version=version())
    return render_template('signin.html',version=version())

# change password
@app.route("/<nm_passwd>", methods=["POST", "GET"])
def profile(nm_passwd):
    try:
        if "_" in str(nm_passwd):
            user_name = str(nm_passwd).split("_")[0]
            user_passwd = str(nm_passwd).split("_")[1]
            user_passwd = "b\'"+user_passwd+"\'"
            found_user = users.query.filter_by(name=user_name).first()

            if request.method == "POST":
                password = request.form['password']
                session['email'] = found_user.email
                session['hashed_email'] = hashlib.md5(session['email'] .encode('utf-8')).hexdigest()
                session['name'] = found_user.name
                session['id'] = found_user.id
                password = bcrypt.hashpw(password.encode('utf8'), bcrypt.gensalt())
                found_user.password = password
                db.session.commit()
                flash("Your password is changed!", "inval")
                onto_len_dir = 0
                onto_list = ''
                onto_cont=open("addiction.onto","r").read()
                dict_onto=ast.literal_eval(onto_cont)
                return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())
            # remove reserved characters from the hashed passwords
            reserved = (";", "/", "?", ":", "@", "=", "&", ".")
            def replace_reserved(fullstring):
                for replace_str in reserved:
                    fullstring = fullstring.replace(replace_str,"")
                return fullstring
            replaced_passwd = replace_reserved(str(found_user.password))

            if replaced_passwd == user_passwd:
                return render_template("/passwd_change.html", name=user_name,version=version())
            else:
                return "This url does not exist"
        else:
            return "This url does not exist"
    except (AttributeError):
        return "This url does not exist"


@app.route("/logout")
def logout():
    onto_len_dir = 0
    onto_list = ''
    onto_cont=open("addiction.onto","r").read()
    dict_onto=ast.literal_eval(onto_cont)

    if 'email' in session:
        global user1
        if session['name'] != '':
            user1 = session['name']
        else:
            user1 = session['email']
    flash("You have been logged out, {user1}", "inval")
    session.pop('email', None)
    session.clear()
    return render_template('index.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())


@app.route("/about")
def about():
    return render_template('about.html',version=version)

# Ontology selection
@app.route("/index_ontology", methods=["POST", "GET"])
def index_ontology():
    namecat2 = request.args.get('onto')
    session['namecat']=namecat2

    if (namecat2 == 'addiction' or namecat2 == 'Select your ontology' ):
        session['namecat']='addiction'
        onto_cont=open("addiction.onto","r").read()
    else:
        dirlist = os.listdir(session['user_folder']+"/ontology/")
        for filename in dirlist:
            onto_name = filename.split('_0_')[1]
            if namecat2 == onto_name:
                onto_cont = open(session['user_folder']+"/ontology/"+filename+"/"+namecat2+".onto", "r").read()
                break
    dict_onto=ast.literal_eval(onto_cont)
    onto_len_dir = session['onto_len_dir']
    onto_list = session['onto_list']
    return render_template('index.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = session['namecat'], dict_onto=dict_onto ,version=version())


@app.route("/ontology", methods=["POST", "GET"])
def ontology():
    namecat2 = request.args.get('onto')
    select_date=request.args.get('selected_date')
    namecat_exist=0

    if select_date != None:
        time_extension = str(select_date)
        time_extension = time_extension.split('_0_')[0]
        namecat = str(select_date).split('_0_')[1]
        time_extension = time_extension.replace(':', '_')
        time_extension = time_extension.replace('-', '_')

        if ('email' in session):
            session['namecat'] = session['user_folder']+"/ontology/"+str(time_extension)+"_0_"+namecat+"/"+namecat
        else:
            session['namecat']=tempfile.gettempdir()+'/'+namecat
        onto_cont = open(session['namecat']+".onto","r").read()

        if onto_cont=='':
            dict_onto={}
        else:
            dict_onto=ast.literal_eval(onto_cont)
    elif (('email' in session) and (namecat2 == 'addiction')):
        namecat='addiction'
        session['namecat']=namecat
        onto_cont = open("addiction.onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)
    else:
        if (('email' in session) and ((namecat2 != None) and (namecat2 != 'choose your ontology'))):
            namecat=namecat2
            dirlist = os.listdir(session['user_folder']+"/ontology/")

            for filename in dirlist:
                onto_name = filename.split('_0_')[1]
                if onto_name==namecat:
                    namecat_exist=1
                    namecat_filename=filename
                    break
            if namecat_exist==1:
                session['namecat'] = session['user_folder']+"/ontology/"+namecat_filename+'/'+namecat
            else:
                onto_cont = open("addiction.onto","r").read()
                dict_onto=ast.literal_eval(onto_cont)
        else:
            namecat='addiction'
            session['namecat']=namecat
            onto_cont = open("addiction.onto","r").read()
            dict_onto=ast.literal_eval(onto_cont)

    if request.method == "POST":
        maincat = request.form['maincat']
        subcat = request.form['subcat']
        keycat = request.form['keycat']
        namecat = request.form['namecat']
        namecat_exist=0
        maincat=re.sub('[^,a-zA-Z0-9 \n]', '', maincat)
        subcat=re.sub('[^,a-zA-Z0-9 \n]', '', subcat)
        keycat=re.sub('[^,a-zA-Z0-9 \n]', '', keycat)
        keycat = keycat.replace(',', '|')
        keycat = re.sub("\s+", ' ', keycat)
        keycat = keycat.replace(' |', '|')
        keycat = keycat.replace('| ', '|')
        namecat=re.sub('[^,a-zA-Z0-9 \n]', '', namecat)

        # Generate a unique session ID depending on timestamp to track the results
        timestamp = datetime.utcnow().replace(microsecond=0)
        timestamp = timestamp.replace(tzinfo=pytz.utc)
        timestamp = timestamp.astimezone(pytz.timezone("America/Chicago"))
        timeextension = str(timestamp)
        timeextension = timeextension.replace(':', '_')
        timeextension = timeextension.replace('-', '_')
        timeextension = timeextension.replace(' ', '_')
        timeextension = timeextension.replace('_06_00', '')
        session['timeextension'] = timeextension

        if request.form['submit'] == 'add':  # Add new keywords or create a new ontology
            if ('email' in session):
                session['namecat']=namecat
                if (namecat=='addiction'):
                    flash("You cannot change addiction keywords but a new ontology will be saved as 'addictionnew', instead","inval")
                    namecat='addictionnew'
                    session['namecat']=namecat
                session['user_folder'] = datadir+"/user/"+str(session['hashed_email'])
                dirlist = os.listdir(session['user_folder']+"/ontology/")
                for filename in dirlist:
                    onto_name = filename.split('_0_')[1]
                    if onto_name==namecat:
                        namecat_exist=1  # Add new keywords
                        namecat_filename=filename
                        break
                if namecat_exist==0:  # Create a new ontology folder
                    os.makedirs(session['user_folder']+"/ontology/"+str(timeextension)+"_0_"+namecat,exist_ok=True)
                    session['namecat'] = session['user_folder']+"/ontology/"+str(timeextension)+"_0_"+namecat+"/"+namecat
                    if namecat=='addictionnew':
                        with open("addiction.onto","r") as f1:
                            with open(session['namecat']+".onto", "w") as f2:
                                for line in f1:
                                    f2.write(line)
                    else:
                        f= open(session['namecat']+".onto","w")
                        dict_onto={}
                else:
                    session['namecat'] = session['user_folder']+"/ontology/"+namecat_filename+'/'+namecat

                onto_cont=open(session['namecat']+".onto",'r').read()
                if onto_cont=='':
                    dict_onto={}
                else:
                    dict_onto=ast.literal_eval(onto_cont)

                flag_kw=0
                if (',' in maincat) or (',' in subcat):
                    flash("Only one word can be added to the category and subcategory at a time.","inval")
                elif maincat in dict_onto.keys():  # Layer 2, main category
                    if subcat in dict_onto[maincat].keys():  # Layer 3, keywords shown in results
                        keycat_ls = keycat.split('|')
                        for kw in str.split(next(iter(dict_onto[maincat][subcat])), '|'):  # Layer 4, synonyms
                            for keycat_word in keycat_ls:
                                if kw==keycat_word:
                                    flash("\""+kw+"\" is already in keywords under the subcategory \""+ subcat \
                                        + "\" that is under the category \""+ maincat+"\"","inval")
                                    flag_kw=1
                        if flag_kw==0:
                            dict_onto[maincat][subcat]= '{'+next(iter(dict_onto[maincat][subcat]))+'|'+keycat+'}'
                            dict_onto=str(dict_onto).replace('\'{','{\'')
                            dict_onto=str(dict_onto).replace('}\'','\'}')
                            dict_onto=str(dict_onto).replace('}},','}},\n')
                            with open(session['namecat']+'.onto', 'w') as file3:
                                file3.write(str(dict_onto))
                    else:
                        dict_onto[maincat][subcat]='{'+subcat+'|'+keycat+'}'
                        dict_onto=str(dict_onto).replace('\'{','{\'')
                        dict_onto=str(dict_onto).replace('}\'','\'}')
                        dict_onto=str(dict_onto).replace('}},','}},\n')
                        with open(session['namecat']+'.onto', 'w') as file3:
                            file3.write(str(dict_onto))
                else:
                    dict_onto[maincat]= '{'+subcat+'\': {\''+keycat+'}'+'}'
                    dict_onto=str(dict_onto).replace('\"{','{\'')
                    dict_onto=str(dict_onto).replace('}\"','\'}')
                    dict_onto=str(dict_onto).replace('\'{','{\'')
                    dict_onto=str(dict_onto).replace('}\'','\'}')
                    dict_onto=str(dict_onto).replace('}},','}},\n')
                    with open(session['namecat']+'.onto', 'w') as file3:
                        file3.write(str(dict_onto))
            else:
                if namecat=='addiction':
                    flash("You must login to change the addiction ontology.")
                else:
                    flash("You must login to create a new ontology.")

        if request.form['submit'] == 'remove':
            if ('email' in session):
                session['namecat']=namecat
                if (namecat=='addiction'):
                    flash("You cannot change addiction keywords but a new ontology will be saved as 'addictionnew', instead","inval")
                    namecat='addictionnew'
                    session['namecat']=namecat
                session['user_folder'] = datadir+"/user/"+str(session['hashed_email'])
                dirlist = os.listdir(session['user_folder']+"/ontology/")
                for filename in dirlist:
                    onto_name = filename.split('_0_')[1]
                    if onto_name==namecat:
                        namecat_exist=1
                        namecat_filename=filename
                        break
                if namecat_exist==0:
                    os.makedirs(session['user_folder']+"/ontology/"+str(timeextension)+"_0_"+namecat,exist_ok=True)
                    session['namecat'] = session['user_folder']+"/ontology/"+str(timeextension)+"_0_"+namecat+"/"+namecat
                    if namecat=='addictionnew':
                        with open("addiction.onto","r") as f1:
                            with open(session['namecat']+".onto", "w") as f2:
                                for line in f1:
                                    f2.write(line)
                    else:
                        f= open(session['namecat']+".onto","w")
                        dict_onto={}

                else:
                    session['namecat'] = session['user_folder']+"/ontology/"+namecat_filename+'/'+namecat

                onto_cont=open(session['namecat']+".onto",'r').read()
                if onto_cont=='':
                    dict_onto={}
                else:
                    dict_onto=ast.literal_eval(onto_cont)

                flag_kw=0
                if maincat in dict_onto.keys():  # Layer 2, main category
                    if subcat in dict_onto[maincat].keys():  # Layer 3, keywords shown in results
                        for kw in str.split(next(iter(dict_onto[maincat][subcat])), '|'):
                            keycat_ls = keycat.split('|')
                            for keycat_word in keycat_ls:  # Layer 4, synonyms
                                if kw==keycat_word:
                                    dict_onto[maincat][subcat]=re.sub(r'\|'+keycat_word+'\'', '\'', str(dict_onto[maincat][subcat]))
                                    dict_onto[maincat][subcat]=re.sub(r'\''+keycat_word+'\|', '\'', str(dict_onto[maincat][subcat]))
                                    dict_onto[maincat][subcat]=re.sub(r'\|'+keycat_word+'\|', '|', str(dict_onto[maincat][subcat]))
                                    dict_onto[maincat][subcat]=re.sub(r'\''+keycat_word+'\'', '', str(dict_onto[maincat][subcat]))
                                    flag_kw=1
                        if '{}' in dict_onto[maincat][subcat]:
                            dict_onto[maincat]=re.sub(r', \''+subcat+'\': \'{}\' ', '', str(dict_onto[maincat]))
                            dict_onto[maincat]=re.sub(r'\''+subcat+'\': \'{}\', ', '', str(dict_onto[maincat]))
                            dict_onto[maincat]=re.sub(r'\''+subcat+'\': \'{}\'', '', str(dict_onto[maincat]))
                        if '{}' in dict_onto[maincat]:
                            dict_onto=re.sub(r', \''+maincat+'\': \'{}\'', '', str(dict_onto))
                        dict_onto=str(dict_onto).replace('\"{','{')
                        dict_onto=str(dict_onto).replace('}\"','}')
                        dict_onto=str(dict_onto).replace('\'{','{')
                        dict_onto=str(dict_onto).replace('}\'','}')
                        with open(session['namecat']+'.onto', 'w') as file3:
                            file3.write(str(dict_onto))
                        if flag_kw==0:
                            flash("\""+keycat+"\" is not a keyword.","inval")
                    else:
                        flash("\""+subcat+"\" is not a subcategory.","inval")
                else:
                    flash("\""+subcat+"\" is not a category.","inval")
            else:
                if namecat=='addiction':
                    flash("You must login to change the addiction ontology.")
                else:
                    flash("You must login to create a new ontology.")

    if 'namecat' in session:
        file2 = open(session['namecat']+".onto","r")
        onto_cont=file2.read()
        if onto_cont=='':
            dict_onto={}
        else:
            dict_onto=ast.literal_eval(onto_cont)
    else:
        session['namecat']='addiction'
        file2 = open(session['namecat']+".onto","r")
        onto_cont=file2.read()
        dict_onto=ast.literal_eval(onto_cont)
    name_to_html = str(session['namecat']).split('/')[-1]

    if ('email' in session):
        ontoarchive()
        onto_len_dir = session['onto_len_dir']
        onto_list = session['onto_list']
    else:
        onto_len_dir=0
        onto_list=''
    return render_template('ontology.html',no_footer=True, dict_onto=dict_onto, namecat=name_to_html, onto_len_dir=onto_len_dir, onto_list=onto_list,version=version())


@app.route("/ontoarchive")
def ontoarchive():
    session['onto_len_dir'] = 0
    session['onto_list'] = ''
    if ('email' in session):
        if os.path.exists(datadir+"/user/"+str(session['hashed_email'])+"/ontology") == False:
            flash("Ontology history doesn't exist!")
            onto_len_dir = 0
            onto_list = ''
            onto_cont=open("addiction.onto","r").read()
            dict_onto=ast.literal_eval(onto_cont)
            return render_template('index.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())
        else:
            session['user_folder'] = datadir+"/user/"+str(session['hashed_email'])
    else:
        flash("You logged out!")
        onto_len_dir = 0
        onto_list = ''
        onto_cont=open("addiction.onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)
        return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())

    session_id=session['id']
    def sorted_alphanumeric(data):
        convert = lambda text: int(text) if text.isdigit() else text.lower()
        alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
        return sorted(data, key=alphanum_key)

    dirlist = sorted_alphanumeric(os.listdir(session['user_folder']+"/ontology/"))
    onto_folder_list = []
    onto_directory_list = []
    onto_list=[]

    for filename in dirlist:
        onto_folder_list.append(filename)
        onto_name = filename.split('_0_')[1]
        onto_name = onto_name.replace('_', ', ')
        onto_list.append(onto_name)
        onto_name=""
        filename=filename[0:4]+"-"+filename[5:7]+"-"+filename[8:13]+":"+filename[14:16]+":"+filename[17:19]
        onto_directory_list.append(filename)

    onto_len_dir = len(onto_directory_list)
    session['onto_len_dir'] = onto_len_dir
    session['onto_list'] = onto_list
    message3="<ul><li> Click on the Date/Time to view archived results. <li>The Date/Time are based on US Central time zone.</ul> "
    return render_template('ontoarchive.html', onto_len_dir=onto_len_dir, onto_list = onto_list, onto_folder_list=onto_folder_list, onto_directory_list=onto_directory_list, session_id=session_id, message3=message3,version=version())


# Remove an ontology folder
@app.route('/removeonto', methods=['GET', 'POST'])
def removeonto():
    if('email' in session):
        remove_folder = request.args.get('remove_folder')
        shutil.rmtree(datadir+"/user/"+str(session['hashed_email']+"/ontology/"+remove_folder), ignore_errors=True)
        return redirect(url_for('ontoarchive'))
    else:
        flash("You logged out!")
        onto_len_dir = 0
        onto_list = ''
        onto_cont=open("addiction.onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)
        return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())


@app.route('/progress')
def progress():
    genes=request.args.get('query')
    genes=genes.replace(",", " ")
    genes=genes.replace(";", " ")
    genes=re.sub(r'\bLOC\d*?\b', "", genes, flags=re.I)
    genes=genes.replace(" \'", " \"")
    genes=genes.replace("\' ", "\" ")
    genes=genes.replace("\'", "-")
    genes1 = [f[1:-1] for f in re.findall('".+?"', genes)]
    genes2 = [p for p in re.findall(r'([^""]+)',genes) if p not in genes1]
    genes2_str = ''.join(genes2)
    genes2 = genes2_str.split()
    genes3 = genes1 + genes2
    genes = [re.sub("\s+", '-', s) for s in genes3]

    # Only 1-200 terms are allowed
    if len(genes)>=200:
        if ('email' in session):
            onto_len_dir = session['onto_len_dir']
            onto_list = session['onto_list']
        else:
            onto_len_dir = 0
            onto_list = ''
        onto_cont=open("addiction.onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)
        message="<span class='text-danger'>Up to 200 terms can be searched at a time</span>"
        return render_template('index.html' ,onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto, message=message,version=version())

    if len(genes)==0:
        if ('email' in session):
            onto_len_dir = session['onto_len_dir']
            onto_list = session['onto_list']
        else:
            onto_len_dir = 0
            onto_list = ''
        onto_cont=open("addiction.onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)
        message="<span class='text-danger'>Please enter a search term </span>"
        return render_template('index.html',onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto, message=message,version=version())

    tf_path=tempfile.gettempdir()
    genes_for_folder_name =""
    if len(genes) == 1:
        marker = ""
        genes_for_folder_name =str(genes[0])
    elif len(genes) == 2:
        marker = ""
        genes_for_folder_name =str(genes[0])+"_"+str(genes[1])
    elif len(genes) == 3:
        marker = ""
        genes_for_folder_name =str(genes[0])+"_"+str(genes[1])+"_"+str(genes[2])
    else:
        genes_for_folder_name =str(genes[0])+"_"+str(genes[1])+"_"+str(genes[2])
        marker="_m"

    # Generate a unique session ID depending on timestamp to track the results
    timestamp = datetime.utcnow().replace(microsecond=0)
    timestamp = timestamp.replace(tzinfo=pytz.utc)
    timestamp = timestamp.astimezone(pytz.timezone("America/Chicago"))
    timeextension = str(timestamp)
    timeextension = timeextension.replace(':', '_')
    timeextension = timeextension.replace('-', '_')
    timeextension = timeextension.replace(' ', '_')
    timeextension = timeextension.replace('_06_00', '')
    session['timeextension'] = timeextension
    namecat_exist=0

    # Create a folder for the search
    if ('email' in session):
        try:
            namecat=session['namecat']
        except:
            namecat = 'addiction'
            session['namecat'] = namecat
        if namecat=='choose your ontology' or namecat=='addiction' or namecat == 'addiction':
            session['namecat']='addiction'
            onto_cont=open("addiction.onto","r").read()
            dictionary=ast.literal_eval(onto_cont)
            search_type = request.args.getlist('type')
            if (search_type == []):
                search_type = ['GWAS', 'function', 'addiction', 'drug', 'brain', 'stress', 'psychiatric', 'cell']
            session['search_type'] = search_type
        else:
            dirlist = os.listdir(session['user_folder']+"/ontology/")
            for filename in dirlist:
                onto_name = filename.split('_0_')[1]
                if onto_name==namecat:
                    namecat_exist=1
                    namecat_filename=filename
                    break
            if (namecat_exist==1):
                session['namecat'] = session['user_folder']+"/ontology/"+namecat_filename+'/'+namecat
                onto_cont=open(session['namecat']+".onto","r").read()
                dict_onto=ast.literal_eval(onto_cont)
                search_type = request.args.getlist('type')
                if (search_type == []):
                    search_type = list(dict_onto.keys())
                session['search_type'] = search_type

        # Save the ontology name in the user search history table
        if session['namecat']=='addiction':
            onto_name_archive = session['namecat']
        elif ('/' in session['namecat']):
            onto_name_archive = session['namecat'].split('/')[-1]
        else:
            onto_name_archive = '-'

        os.makedirs(datadir + "/user/"+str(session['hashed_email'])+"/"+str(timeextension)+"_0_"+genes_for_folder_name+marker+"_0_"+onto_name_archive,exist_ok=True)
        session['path_user'] = datadir+"/user/"+str(session['hashed_email'])+"/"+str(timeextension)+"_0_"+genes_for_folder_name+marker+"_0_"+onto_name_archive+"/"
        session['rnd'] = timeextension+"_0_"+genes_for_folder_name+marker+"_0_"+onto_name_archive
        rnd = session['rnd']
    else:
        rnd = "tmp" + ''.join(random.choice(string.ascii_letters) for x in range(6))
        session['path']=tf_path+ "/" + rnd
        os.makedirs(session['path'])
        search_type = request.args.getlist('type')

        if (search_type == []):
            search_type = ['GWAS', 'function', 'addiction', 'drug', 'brain', 'stress', 'psychiatric', 'cell']
        session['search_type'] = search_type
    genes_session = ''

    for gen in genes:
        genes_session += str(gen) + "_"
    genes_session = genes_session[:-1]
    session['query']=genes
    return render_template('progress.html', url_in="search", url_out="cytoscape/?rnd="+rnd+"&genequery="+genes_session,version=version())


@app.route("/search")
def search():
    genes=session['query']
    percent_ratio=len(genes)+1

    if(len(genes)==1):
        percent_ratio=2
    timeextension=session['timeextension']
    percent=100/percent_ratio-0.00000001 # 7 categories + 1 at the beginning

    if ('email' in session):
        sessionpath = session['path_user'] + timeextension
        path_user=session['path_user']
    else:
        sessionpath = session['path']
        path_user=session['path']+"/"

    snt_file=sessionpath+"_snt"
    cysdata=open(sessionpath+"_cy","w+")
    sntdata=open(snt_file,"w+")
    zeroLinkNode=open(sessionpath+"_0link","w+")
    search_type = session['search_type']
    temp_nodes = ""
    json_nodes = "{\"data\":["

    n_num=0
    d={}
    nodecolor={}
    nodecolor['GWAS'] = "hsl(0, 0%, 70%)"
    nodes_list = []

    if 'namecat' in session:
        namecat_flag=1
        ses_namecat = session['namecat']
        onto_cont = open(session['namecat']+".onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)

        for ky in dict_onto.keys():
            nodecolor[ky] = "hsl("+str((n_num+1)*int(360/len(dict_onto.keys())))+", 70%, 80%)"
            d["nj{0}".format(n_num)]=generate_nodes_json(dict_onto[ky],str(ky),nodecolor[ky])
            n_num+=1

            if (ky in search_type):
                temp_nodes += generate_nodes(dict_onto[ky],str(ky),nodecolor[ky])

                for nd in dict_onto[ky]:
                    nodes_list.append(nd)
                json_nodes += generate_nodes_json(dict_onto[ky],str(ky),nodecolor[ky] )
        d["nj{0}".format(n_num)]=''
    else:
        namecat_flag=0
        for ky in dictionary.keys():
            nodecolor[ky] = "hsl("+str((n_num+1)*int(360/len(dictionary.keys())))+", 70%, 80%)"
            d["nj{0}".format(n_num)]=generate_nodes_json(dictionary[ky],str(ky),nodecolor[ky])
            n_num+=1

            if (ky in search_type):
                temp_nodes += generate_nodes(dictionary[ky],str(ky),nodecolor[ky])

                for nd in dictionary[ky]:
                    nodes_list.append(nd)
                json_nodes += generate_nodes_json(dictionary[ky],str(ky),nodecolor[ky])
        d["nj{0}".format(n_num)]=''

    json_nodes = json_nodes[:-2]
    json_nodes =json_nodes+"]}"
    def generate(genes, tf_name):
        with app.test_request_context():
            sentences=str()
            edges=str()
            nodes = temp_nodes
            progress=0
            searchCnt=0
            nodesToHide=str()
            json_edges = str()
            #genes_or = ' [tiab] or '.join(genes)
            all_d=''

            if namecat_flag==1:
                onto_cont = open(ses_namecat+".onto","r").read()
                dict_onto=ast.literal_eval(onto_cont)

                for ky in dict_onto.keys():
                    if (ky in search_type):
                        all_d_ls=undic(list(dict_onto[ky].values()))
                        all_d = all_d+'|'+all_d_ls
            else:
                for ky in dictionary.keys():
                    if (ky in search_type):
                        all_d_ls=undic(list(dictionary[ky].values()))
                        all_d = all_d+'|'+all_d_ls
            all_d=all_d[1:]
            if ("GWAS" in search_type):
                datf = pd.read_csv('./utility/gwas_used.csv',sep='\t')
            progress+=percent
            yield "data:"+str(progress)+"\n\n"
            for gene in genes:
                abstracts_raw = getabstracts(gene,all_d)
                #print(abstracts_raw)
                sentences_ls=[]

                for row in abstracts_raw.split("\n"):
                    tiab=row.split("\t")
                    pmid = tiab.pop(0)
                    tiab= " ".join(tiab)
                    sentences_tok = sent_tokenize(tiab)
                    for sent_tok in sentences_tok:
                        sent_tok = pmid + ' ' + sent_tok
                        sentences_ls.append(sent_tok)
                gene=gene.replace("-"," ")

                geneEdges = ""

                if namecat_flag==1:
                    onto_cont = open(ses_namecat+".onto","r").read()
                    dict_onto=ast.literal_eval(onto_cont)
                else:
                    dict_onto = dictionary

                for ky in dict_onto.keys():
                    if (ky in search_type):
                        if (ky=='addiction') and ('addiction' in dict_onto.keys())\
                            and ('drug' in dict_onto.keys()) and ('addiction' in dict_onto['addiction'].keys())\
                            and ('aversion' in dict_onto['addiction'].keys()) and ('intoxication' in dict_onto['addiction'].keys()):
                            #addiction terms must present with at least one drug
                            addiction_flag=1
                            #addiction=undic0(addiction_d) +") AND ("+undic0(drug_d)
                            sent=gene_category(gene, addiction_d, "addiction", sentences_ls,addiction_flag,dict_onto)
                            if ('addiction' in search_type):
                                geneEdges += generate_edges(sent, tf_name)
                                json_edges += generate_edges_json(sent, tf_name)
                        else:
                            addiction_flag=0
                            if namecat_flag==1:
                                onto_cont = open(ses_namecat+".onto","r").read()
                                dict_onto=ast.literal_eval(onto_cont)
                                #ky_d=undic(list(dict_onto[ky].values()))
                                sent=gene_category(gene,ky,str(ky), sentences_ls, addiction_flag,dict_onto)
                            else:
                                #ky_d=undic(list(dict_onto[ky].values()))
                                #print(sentences_ls)
                                sent=gene_category(gene,ky,str(ky), sentences_ls, addiction_flag,dict_onto)
                                #print(sent)
                            yield "data:"+str(progress)+"\n\n"

                            geneEdges += generate_edges(sent, tf_name)
                            json_edges += generate_edges_json(sent, tf_name)
                        sentences+=sent
                if ("GWAS" in search_type):
                    gwas_sent=[]
                    print (datf)
                    datf_sub1 = datf[datf["MAPPED_GENE"].str.contains('(?:\s|^)'+gene+'(?:\s|$)', flags=re.IGNORECASE)
                                    | (datf["REPORTED GENE(S)"].str.contains('(?:\s|^)'+gene+'(?:\s|$)', flags=re.IGNORECASE))]
                    print (datf_sub1)
                    for nd2 in dict_onto['GWAS'].keys():
                        for nd1 in dict_onto['GWAS'][nd2]:
                            for nd in nd1.split('|'):
                                gwas_text=''
                                datf_sub = datf_sub1[datf_sub1['DISEASE/TRAIT'].str.contains('(?:\s|^)'+nd+'(?:\s|$)', flags=re.IGNORECASE)]
                                    #& (datf['REPORTED GENE(S)'].str.contains('(?:\s|^)'+gene+'(?:\s|$)', flags=re.IGNORECASE)
                                    #| (datf['MAPPED_GENE'].str.contains('(?:\s|^)'+gene+'(?:\s|$)', flags=re.IGNORECASE)))]
                                if not datf_sub.empty:
                                    for index, row in datf_sub.iterrows():
                                        gwas_text = "SNP:<b>"+str(row['SNPS'])+"</b>, P value: <b>"+str(row['P-VALUE'])\
                                            +"</b>, Disease/trait:<b> "+str(row['DISEASE/TRAIT'])+"</b>, Mapped trait:<b> "\
                                            +str(row['MAPPED_TRAIT'])+"</b><br>"
                                        gwas_sent.append(gene+"\t"+"GWAS"+"\t"+nd+"_GWAS\t"+str(row['PUBMEDID'])+"\t"+gwas_text)
                    cys, gwas_json, sn_file = searchArchived('GWAS', gene , 'json',gwas_sent, path_user)
                    with open(path_user+"gwas_results.tab", "a") as gwas_edges:
                        gwas_edges.write(sn_file)
                    geneEdges += cys
                    json_edges += gwas_json
                # report progress immediately
                progress+=percent
                yield "data:"+str(progress)+"\n\n"

                if len(geneEdges) >0:
                    edges+=geneEdges
                    nodes+="{ data: { id: '" + gene +  "', nodecolor:'#E74C3C', fontweight:700, url:'/synonyms?node="+gene+"'} },\n"
                else:
                    nodesToHide+=gene +  " "

                searchCnt+=1
                if (searchCnt==len(genes)):
                    progress=100
                    sntdata.write(sentences)
                    sntdata.close()
                    cysdata.write(nodes+edges)
                    cysdata.close()
                    zeroLinkNode.write(nodesToHide)
                    zeroLinkNode.close()
                yield "data:"+str(progress)+"\n\n"

           # Edges in json format
            json_edges="{\"data\":["+json_edges
            json_edges = json_edges[:-2]
            json_edges =json_edges+"]}"

            # Write edges to txt file in json format also in user folder
            with open(path_user+"edges.json", "w") as temp_file_edges:
                temp_file_edges.write(json_edges)
    with open(path_user+"nodes.json", "w") as temp_file_nodes:
        temp_file_nodes.write(json_nodes)
    return Response(generate(genes, snt_file), mimetype='text/event-stream')


@app.route("/tableview/")
def tableview():
    genes_url=request.args.get('genequery')
    rnd_url=request.args.get('rnd')
    tf_path=tempfile.gettempdir()

    if ('email' in session):
        filename = rnd_url.split("_0_")[0]
        genes_session_tmp = datadir+"/user/"+str(session['hashed_email'])+"/"+rnd_url+"/"+filename
        gene_url_tmp = "/user/"+str(session['hashed_email'])+"/"+rnd_url

        try:
            with open(datadir+gene_url_tmp+"/nodes.json") as jsonfile:
                jnodes = json.load(jsonfile)
        except FileNotFoundError:
            flash("You logged out!")
            onto_len_dir = 0
            onto_list = ''
            onto_cont=open("addiction.onto","r").read()
            dict_onto=ast.literal_eval(onto_cont)
            return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())

        jedges =''
        file_edges = open(datadir+gene_url_tmp +'/edges.json', 'r')
        for line in file_edges.readlines():
            if ':' not in line:
                nodata_temp = 1
            else:
                nodata_temp = 0
                with open(datadir+gene_url_tmp +"/edges.json") as edgesjsonfile:
                    jedges = json.load(edgesjsonfile)
                break
    else:
        genes_session_tmp=tf_path+"/"+rnd_url
        gene_url_tmp = genes_session_tmp
        try:
            with open(gene_url_tmp+"/nodes.json") as jsonfile:
                jnodes = json.load(jsonfile)
        except FileNotFoundError:
            flash("You logged out!")
            onto_len_dir = 0
            onto_list = ''
            onto_cont=open("addiction.onto","r").read()
            dict_onto=ast.literal_eval(onto_cont)
            return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())
        jedges =''
        file_edges = open(gene_url_tmp +'/edges.json', 'r')
        for line in file_edges.readlines():
            if ':' not in line:
                nodata_temp = 1
            else:
                nodata_temp = 0
                with open(gene_url_tmp +"/edges.json") as edgesjsonfile:
                    jedges = json.load(edgesjsonfile)
                break
    genename=genes_url.split("_")
    if len(genename)>3:
        genename = genename[0:3]
        added = ",..."
    else:
        added = ""
    gene_name = str(genename)[1:]
    gene_name=gene_name[:-1]
    gene_name=gene_name.replace("'","")
    gene_name = gene_name+added
    num_gene = gene_name.count(',')+1

    message3="<ul><li> <font color=\"#E74C3C\">Click on the abstract count to read sentences linking the keyword and the gene</font>  <li> Click on a keyword to see the terms included in the search. <li>View the results in <a href='\\cytoscape/?rnd={}&genequery={}'\ ><b> a graph.</b></a> </ul> Links will be preserved when the table is copy-n-pasted into a spreadsheet.".format(rnd_url,genes_url)
    return render_template('tableview.html', no_footer=True, genes_session_tmp = genes_session_tmp, nodata_temp=nodata_temp, num_gene=num_gene, jedges=jedges, jnodes=jnodes,gene_name=gene_name, message3=message3, rnd_url=rnd_url, genes_url=genes_url,version=version())


# Table for the zero abstract counts
@app.route("/tableview0/")
def tableview0():
    genes_url=request.args.get('genequery')
    rnd_url=request.args.get('rnd')
    tf_path=tempfile.gettempdir()

    if ('email' in session):
        filename = rnd_url.split("_0_")[0]
        genes_session_tmp = datadir+"/user/"+str(session['hashed_email'])+"/"+rnd_url+"/"+filename
        gene_url_tmp = "/user/"+str(session['hashed_email'])+"/"+rnd_url
        try:
            with open(datadir+gene_url_tmp+"/nodes.json") as jsonfile:
                jnodes = json.load(jsonfile)
        except FileNotFoundError:
            flash("You logged out!")
            onto_len_dir = 0
            onto_list = ''
            onto_cont=open("addiction.onto","r").read()
            dict_onto=ast.literal_eval(onto_cont)
            return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())

        jedges =''
        file_edges = open(datadir+gene_url_tmp+'/edges.json', 'r')
        for line in file_edges.readlines():
            if ':' not in line:
                nodata_temp = 1
            else:
                nodata_temp = 0
                with open(datadir+gene_url_tmp+"/edges.json") as edgesjsonfile:
                    jedges = json.load(edgesjsonfile)
                break
    else:
        genes_session_tmp=tf_path+"/"+rnd_url
        gene_url_tmp = genes_session_tmp
        try:
            with open(gene_url_tmp+"/nodes.json") as jsonfile:
                jnodes = json.load(jsonfile)
        except FileNotFoundError:
            flash("You logged out!")
            onto_len_dir = 0
            onto_list = ''
            onto_cont=open("addiction.onto","r").read()
            dict_onto=ast.literal_eval(onto_cont)
            return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())

        jedges =''
        file_edges = open(gene_url_tmp+'/edges.json', 'r')
        for line in file_edges.readlines():
            if ':' not in line:
                nodata_temp = 1
            else:
                nodata_temp = 0
                with open(gene_url_tmp+"/edges.json") as edgesjsonfile:
                    jedges = json.load(edgesjsonfile)
                break
    genes_url=request.args.get('genequery')
    genename=genes_url.split("_")
    if len(genename)>3:
        genename = genename[0:3]
        added = ",..."
    else:
        added = ""

    gene_name = str(genename)[1:]
    gene_name=gene_name[:-1]
    gene_name=gene_name.replace("'","")
    gene_name = gene_name+added
    num_gene = gene_name.count(',')+1
    message4="<b> Notes: </b><li> These are the keywords that have <b>zero</b> abstract counts. <li>View all the results in <a href='\\cytoscape/?rnd={}&genequery={}'><b> a graph.</b></a> ".format(rnd_url,genes_url)
    return render_template('tableview0.html',nodata_temp=nodata_temp, num_gene=num_gene, jedges=jedges, jnodes=jnodes,gene_name=gene_name, message4=message4,version=version())


@app.route("/userarchive")
def userarchive():
    onto_len_dir = 0
    onto_list = ''
    onto_cont=open("addiction.onto","r").read()
    dict_onto=ast.literal_eval(onto_cont)

    if ('email' in session):
        if os.path.exists(datadir+"/user/"+str(session['hashed_email'])) == False:
            flash("Search history doesn't exist!")
            return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())
        else:
            session['user_folder'] = datadir+"/user/"+str(session['hashed_email'])
    else:
        onto_name_archive=''
        flash("You logged out!")
        onto_len_dir = 0
        onto_list = ''
        onto_cont=open("addiction.onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)
        return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())

    session_id=session['id']
    def sorted_alphanumeric(data):
        convert = lambda text: int(text) if text.isdigit() else text.lower()
        alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
        return sorted(data, key=alphanum_key)
    dirlist = sorted_alphanumeric(os.listdir(session['user_folder']))
    folder_list = []
    directory_list = []
    gene_list=[]
    onto_list=[]

    for filename in dirlist:
        if ('_0_'  in filename):
            folder_list.append(filename)
            gene_name = filename.split('_0_')[1]
            onto_name = filename.split('_0_')[2]
            if gene_name[-2:] == '_m':
                gene_name = gene_name[:-2]
                gene_name = gene_name + ", ..."
            gene_name = gene_name.replace('_', ', ')
            gene_list.append(gene_name)
            onto_list.append(onto_name)
            onto_name=""
            gene_name=""
            filename=filename[0:4]+"-"+filename[5:7]+"-"+filename[8:13]+":"+filename[14:16]+":"+filename[17:19]
            directory_list.append(filename)
    len_dir = len(directory_list)
    message3="<ul><li> Click on the Date/Time to view archived results. <li>The Date/Time are based on US Central time zone.</ul> "
    return render_template('userarchive.html', len_dir=len_dir, gene_list = gene_list, onto_list = onto_list, folder_list=folder_list, directory_list=directory_list, session_id=session_id, message3=message3,version=version())


# Remove the search directory
@app.route('/remove', methods=['GET', 'POST'])
def remove():
    if('email' in session):
        remove_folder = request.args.get('remove_folder')
        shutil.rmtree(datadir+"/user/"+str(session['hashed_email']+"/"+remove_folder), ignore_errors=True)
        return redirect(url_for('userarchive'))
    else:
        flash("You logged out!")
        onto_len_dir = 0
        onto_list = ''
        onto_cont=open("addiction.onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)
        return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())


@app.route('/date', methods=['GET', 'POST'])
def date():
    select_date = request.args.get('selected_date')
    # Open the cache folder for the user
    tf_path=datadir+"/user"
    if ('email' in session):
        time_extension = str(select_date)
        time_extension = time_extension.split('_0_')[0]
        gene_name1 = str(select_date).split('_0_')[1]
        time_extension = time_extension.replace(':', '_')
        time_extension = time_extension.replace('-', '_')
        session['user_folder'] = tf_path+"/"+str(session['hashed_email'])
        genes_session_tmp = tf_path+"/"+str(session['hashed_email'])+"/"+select_date+"/"+time_extension
        with open(tf_path+"/"+str(session['hashed_email'])+"/"+select_date+"/nodes.json", "r") as jsonfile:
            jnodes = json.load(jsonfile)
        jedges =''
        file_edges = open(tf_path+"/"+str(session['hashed_email'])+"/"+select_date+"/edges.json", "r")
        for line in file_edges.readlines():
            if ':' not in line:
                nodata_temp = 1
            else:
                nodata_temp = 0
                with open(tf_path+"/"+str(session['hashed_email'])+"/"+select_date+"/edges.json", "r") as edgesjsonfile:
                    jedges = json.load(edgesjsonfile)
                break
        gene_list_all=[]
        gene_list=[]
        if nodata_temp == 0:
            for p in jedges['data']:
                if p['source'] not in gene_list:
                    gene_list_all.append(p['source'])
                    gene_list.append(p['source'])
            if len(gene_list)>3:
                gene_list = gene_list[0:3]
                added = ",..."
            else:
                added = ""
            gene_name = str(gene_list)[1:]
            gene_name=gene_name[:-1]
            gene_name=gene_name.replace("'","")
            gene_name = gene_name+added
            num_gene = gene_name.count(',')+1
        else:
            gene_name1 = gene_name1.replace("_", ", ")
            gene_name = gene_name1
            num_gene = gene_name1.count(',')+1
            for i in range(0,num_gene):
                gene_list.append(gene_name1.split(',')[i])
        genes_session = ''
        for gen in gene_list_all:
            genes_session += str(gen) + "_"
        genes_session = genes_session[:-1]
    else:
        flash("You logged out!")
        onto_len_dir = 0
        onto_list = ''
        onto_cont=open("addiction.onto","r").read()
        dict_onto=ast.literal_eval(onto_cont)
        return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())
    message3="<ul><li> <font color=\"#E74C3C\">Click on the abstract count to read sentences linking the keyword and the gene</font> <li> Click on a keyword to see the terms included in the search. <li>View the results in <a href='\\cytoscape/?rnd={}&genequery={}'\ ><b> a graph.</b></a> </ul> Links will be preserved when the table is copy-n-pasted into a spreadsheet.".format(select_date,genes_session)
    return render_template('tableview.html',nodata_temp=nodata_temp, num_gene=num_gene,genes_session_tmp = genes_session_tmp, rnd_url=select_date ,jedges=jedges, jnodes=jnodes,gene_name=gene_name, genes_url=genes_session, message3=message3,version=version())

@app.route('/cytoscape/')
def cytoscape():
    genes_url=request.args.get('genequery')
    rnd_url=request.args.get('rnd')
    tf_path=tempfile.gettempdir()
    genes_session_tmp=tf_path + "/" + genes_url
    rnd_url_tmp=tf_path +"/" + rnd_url
    message2="<ul><li><font color=\"#E74C3C\">Click on a line to read the sentences </font> <li>Click on a keyword to see the terms included in the search<li>Hover a pointer over a node to hide other links <li>Move the nodes around to adjust visibility <li> Reload the page to restore the default layout<li>View the results in <a href='\\tableview/?rnd={}&genequery={}'\ ><b>a table. </b></a></ul>".format(rnd_url,genes_url)

    if ('email' in session):
        filename = rnd_url.split("_0_")[0]
        rnd_url_tmp = datadir+"/user/"+str(session['hashed_email'])+"/"+rnd_url+"/"+filename
        try:
            with open(rnd_url_tmp+"_cy","r") as f:
                elements=f.read()
        except FileNotFoundError:
            flash("You logged out!")
            onto_len_dir = 0
            onto_list = ''
            onto_cont=open("addiction.onto","r").read()
            dict_onto=ast.literal_eval(onto_cont)
            return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())

        with open(rnd_url_tmp+"_0link","r") as z:
            zeroLink=z.read()
            if (len(zeroLink)>0):
                message2+="<span style=\"color:darkred;\">No result was found for these genes: " + zeroLink + "</span>"
    else:
        rnd_url_tmp=tf_path +"/" + rnd_url
        try:
            rnd_url_tmp.replace("\"", "")
            with open(rnd_url_tmp+"_cy","r") as f:
                elements=f.read()
        except FileNotFoundError:
            flash("You logged out!")
            onto_len_dir = 0
            onto_list = ''
            onto_cont=open("addiction.onto","r").read()
            dict_onto=ast.literal_eval(onto_cont)
            return render_template('index.html', onto_len_dir=onto_len_dir, onto_list=onto_list, ontol = 'addiction', dict_onto = dict_onto,version=version())

        with open(rnd_url_tmp+"_0link","r") as z:
            zeroLink=z.read()
            if (len(zeroLink)>0):
                message2+="<span style=\"color:darkred;\">No result was found for these genes: " + zeroLink + "</span>"
    return render_template('cytoscape.html', elements=elements, message2=message2,version=version())


@app.route("/sentences")
def sentences():
    def predict_sent(sent_for_pred):
        max_length = 64
        tokens = clean_doc(sent_for_pred, vocab)
        tokens = [w for w in tokens if w in vocab]
        # convert to line
        line = ' '.join(tokens)
        line = [line]
        tokenized_sent = tokenizer.texts_to_sequences(line)
        tokenized_sent = pad_sequences(tokenized_sent, maxlen=max_length, padding='post')
        predict_sent = model.predict(tokenized_sent, verbose=0)
        percent_sent = predict_sent[0,0]
        if round(percent_sent) == 0:
            return 'neg'
        else:
            return 'pos'
    pmid_list=[]
    pmid_string=''
    edge=request.args.get('edgeID')
    (tf_name, gene0, cat0)=edge.split("|")

    if(cat0=='stress'):
        model = create_model(23154, 64)
        model.load_weights("./nlp/weights.ckpt")
    out3=""
    out_pos = ""
    out_neg = ""
    num_abstract = 0
    stress_cellular = "<br><br><br>"+"</ol><b>Sentence(s) describing celluar stress (classified using a deep learning model):</b><hr><ol>"
    stress_systemic = "<b></ol>Sentence(s) describing systemic stress (classified using a deep learning model):</b><hr><ol>"
    with open(tf_name, "r") as df:
        all_sents=df.read()

    for sent in all_sents.split("\n"):
        if len(sent.strip())!=0:
            (gene,nouse,cat, pmid, text)=sent.split("\t")
            if (gene.upper() == gene0.upper() and cat.upper() == cat0.upper()) :
                out3+= "<li> "+ text + " <a href=\"https://www.ncbi.nlm.nih.gov/pubmed/?term=" + pmid +"\" target=_new>PMID:"+pmid+"<br></a>"
                num_abstract += 1
                if(pmid+cat0 not in pmid_list):
                    pmid_string = pmid_string + ' ' + pmid
                    pmid_list.append(pmid+cat0)
                if(cat0=='stress'):
                    out4 = predict_sent(text)
                    if(out4 == 'pos'):
                        out_pred_pos = "<li> "+ text + " <a href=\"https://www.ncbi.nlm.nih.gov/pubmed/?term=" + pmid +"\" target=_new>PMID:"+pmid+"<br></a>"
                        out_pos += out_pred_pos
                    else:
                        out_pred_neg = "<li>"+ text + " <a href=\"https://www.ncbi.nlm.nih.gov/pubmed/?term=" + pmid +"\" target=_new>PMID:"+pmid+"<br></a>"
                        out_neg += out_pred_neg
    out1="<h3>"+gene0 + " and " + cat0  + "</h3>\n"
    if len(pmid_list)>1:
        out2 = str(num_abstract) + ' sentences in ' + " <a href=\"https://www.ncbi.nlm.nih.gov/pubmed/?term=" + pmid_string +"\" target=_new>"+ str(len(pmid_list)) + ' studies' +"<br></a>" + "<br><br>"
    else:
        out2 = str(num_abstract) + ' sentence(s) in '+ " <a href=\"https://www.ncbi.nlm.nih.gov/pubmed/?term=" + pmid_string +"\" target=_new>"+ str(len(pmid_list)) + ' study' +"<br></a>" "<br><br>"
    if(out_neg == "" and out_pos == ""):
        out= out1+ out2 +out3
    elif(out_pos != "" and out_neg!=""):
        out = out1 + out2 + stress_systemic+out_pos + stress_cellular + out_neg
    elif(out_pos != "" and out_neg ==""):
        out= out1+ out2 + stress_systemic + out_pos
    elif(out_neg != "" and out_pos == ""):
        out = out1 +out2+stress_cellular+out_neg
    K.clear_session()
    return render_template('sentences.html', no_footer=True, sentences="<ol>"+out+"</ol><p>",version=version())


# Show the cytoscape graph for one gene from the top gene list
@app.route("/showTopGene")
def showTopGene():
    query=request.args.get('topGene')
    nodesEdges=searchArchived('topGene',query, 'cys','','')[0]
    message2="<li><strong>"+query + "</strong> is one of the top addiction genes. <li> An archived search is shown. Click on the blue circle to update the results and include keywords for brain region and gene function. <strong> The update may take a long time to finish.</strong> "
    return render_template("cytoscape.html", elements=nodesEdges, message="Top addiction genes", message2=message2,version=version())


@app.route("/shownode")
def shownode():
    node=request.args.get('node')
    if 'namecat' in session:
        file2 = open(session['namecat']+".onto","r")
        onto_cont=file2.read()
        dict_onto=ast.literal_eval(onto_cont)
        for ky in dict_onto.keys():
            if node in dict_onto[ky].keys():
                out="<p>"+node.upper()+"<hr><li>"+ next(iter(dict_onto[ky][node])).replace("|", "<li>")
    else:
        for ky in dictionary.keys():
            if node in dictionary[ky].keys():
                out="<p>"+node.upper()+"<hr><li>"+ next(iter(dictionary[ky][node])).replace("|", "<li>")
    return render_template('sentences.html', sentences=out+"<p>",version=version())


@app.route("/synonyms")
def synonyms():
    node=request.args.get('node')
    node=node.upper()
    allnodes={**genes}
    try:
        synonym_list = list(allnodes[node].split("|"))
        session['synonym_list'] = synonym_list
        session['main_gene'] = node.upper()
        out="<hr><li>"+ allnodes[node].replace("|", "<li>")
        synonym_list_str = ';'.join([str(syn) for syn in synonym_list])
        synonym_list_str +=';' + node
        case = 1
        return render_template('genenames.html', case = case, gene = node.upper(), version=version(), synonym_list = synonym_list, synonym_list_str=synonym_list_str)
    except:
        try:
            synonym_list = session['synonym_list']
            synonym_list_str = ';'.join([str(syn) for syn in synonym_list])
            synonym_list_str +=';' + node
            case = 1
            return render_template('genenames.html', case=case, gene = session['main_gene'] , synonym_list = synonym_list, synonym_list_str=synonym_list_str,version=version())
        except:
            case = 2
            return render_template('genenames.html', gene = node, case = case,version=version())


@app.route("/startGeneGene")
def startGeneGene():
    session['forTopGene']=request.args.get('forTopGene')
    return render_template('progress.html', url_in="searchGeneGene", url_out="showGeneTopGene",version=version())


@app.route("/searchGeneGene")
def gene_gene():
    tmp_ggPMID=session['path']+"_ggPMID"
    gg_file=session['path']+"_ggSent" # Gene_gene
    result_file=session['path']+"_ggResult"

    def generate(query):
        progress=1
        yield "data:"+str(progress)+"\n\n"
        os.system("esearch -db pubmed -query \"" +  query + "\" | efetch -format uid |sort >" + tmp_ggPMID)
        abstracts=os.popen("comm -1 -2 topGene_uniq.pmid " + tmp_ggPMID + " |fetch-pubmed -path "+pubmed_path+ " | xtract -pattern PubmedArticle -element MedlineCitation/PMID,ArticleTitle,AbstractText|sed \"s/-/ /g\"").read()
        os.system("rm "+tmp_ggPMID)
        progress=10
        yield "data:"+str(progress)+"\n\n"
        topGenes=dict()
        out=str()
        hitGenes=dict()
        with open("topGene_symb_alias.txt", "r") as top_f:
            for line in top_f:
                (symb, alias)=line.strip().split("\t")
                topGenes[symb]=alias.replace("; ","|")
        allAbstracts= abstracts.split("\n")
        abstractCnt=len(allAbstracts)
        rowCnt=0

        for row in allAbstracts:
            rowCnt+=1
            if rowCnt/10==int(rowCnt/10):
                progress=10+round(rowCnt/abstractCnt,2)*80
                yield "data:"+str(progress)+"\n\n"
            tiab=row.split("\t")
            pmid = tiab.pop(0)
            tiab= " ".join(tiab)
            sentences = sent_tokenize(tiab)
            ## keep the sentence only if it contains the gene
            for sent in sentences:
                if findWholeWord(query)(sent):
                    sent=re.sub(r'\b(%s)\b' % query, r'<strong>\1</strong>', sent, flags=re.I)
                    for symb in topGenes:
                        allNames=symb+"|"+topGenes[symb]
                        if findWholeWord(allNames)(sent) :
                            sent=sent.replace("<b>","").replace("</b>","")
                            sent=re.sub(r'\b(%s)\b' % allNames, r'<b>\1</b>', sent, flags=re.I)
                            out+=query+"\t"+"gene\t" + symb+"\t"+pmid+"\t"+sent+"\n"
                            if symb in hitGenes.keys():
                                hitGenes[symb]+=1
                            else:
                                hitGenes[symb]=1
        progress=95
        yield "data:"+str(progress)+"\n\n"
        with open(gg_file, "w+") as gg:
            gg.write(out)
            gg.close()
        results="<h4>"+query+" vs top addiction genes</h4> Click on the number of sentences will show those sentences. Click on the <span style=\"background-color:#FcF3cf\">top addiction genes</span> will show an archived search for that gene.<hr>"
        topGeneHits={}
        for key in hitGenes.keys():
            url=gg_file+"|"+query+"|"+key
            if hitGenes[key]==1:
                sentword="sentence"
            else:
                sentword="sentences"
            topGeneHits[ "<li> <a href=/sentences?edgeID=" + url+ " target=_new>" + "Show " + str(hitGenes[key]) + " " + sentword +" </a> about "+query+" and <a href=/showTopGene?topGene="+key+" target=_gene><span style=\"background-color:#FcF3cf\">"+key+"</span></a>" ]=hitGenes[key]
        topSorted = [(k, topGeneHits[k]) for k in sorted(topGeneHits, key=topGeneHits.get, reverse=True)]

        for k,v in topSorted:
            results+=k
        saveResult=open(result_file, "w+")
        saveResult.write(results)
        saveResult.close()
        progress=100
        yield "data:"+str(progress)+"\n\n"

    # Start the run
    query=session['forTopGene']
    return Response(generate(query), mimetype='text/event-stream')


@app.route('/showGeneTopGene')
def showGeneTopGene ():
    with open(session['path']+"_ggResult", "r") as result_f:
        results=result_f.read()
    return render_template('sentences.html', sentences=results+"<p><br>",version=version())


# Generate a page that lists all the top 150 addiction genes with links to cytoscape graph.
@app.route("/allTopGenes")
def top150genes():
    return render_template("topAddictionGene.html",no_footer=True,version=version())


if __name__ == '__main__':
    db.create_all()
    app.run(debug=True, port=4200)