diff options
143 files changed, 2987 insertions, 2823 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fd9a886..f27feb5f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ on: jobs: unittest: runs-on: ubuntu-latest - container: bonfacekilz/python2-genenetwork2:latest + container: bonfacekilz/python3-genenetwork2:latest steps: # First start with mariadb set then checkout. The checkout gives diff --git a/bin/genenetwork2 b/bin/genenetwork2 index dd6db7d6..5f4e0f9a 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -60,7 +60,7 @@ GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) echo GN2_BASE_DIR=$GN2_BASE_DIR -GUIX_SITE=$GN2_BASE_DIR/lib/python2.7/site-packages +GUIX_SITE=$GN2_BASE_DIR/lib/python3.8/site-packages if [ -d $GUIX_SITE ]; then echo INFO: GN2 is running from GNU Guix GN2_BASE_DIR=$GUIX_SITE @@ -105,7 +105,7 @@ echo GN2_SETTINGS=$settings # if [ -z $ELASTICSEARCH_PROFILE ]; then # echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; # else -# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python3.8/site-packages" # fi if [ -z $GN2_PROFILE ] ; then @@ -121,7 +121,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$PYTHON_GN_PATH:$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!! + export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! export R_LIBS_SITE=$GN2_PROFILE/site-library export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index 19e4d14f..fffa9571 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -167,7 +167,7 @@ the Guix suggested environment vars. Check the output of #+begin_src bash guix package --search-paths -export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages" +export PYTHONPATH="/root/.guix-profile/lib/python3.8/site-packages" export R_LIBS_SITE="/root/.guix-profile/site-library/" #+end_src @@ -265,7 +265,7 @@ software. If something is not working, take a hint from the settings file that comes in the Guix installation. It sits in something like -: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py +: cat ~/.guix-profile/lib/python3.8/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py ** Set up nginx port forwarding @@ -380,7 +380,7 @@ After setting the paths for the server #+begin_src bash export PATH=~/.guix-profile/bin:$PATH -export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +export PYTHONPATH="$HOME/.guix-profile/lib/python3.8/site-packages" export R_LIBS_SITE="$HOME/.guix-profile/site-library/" export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0" export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0" diff --git a/doc/README.org b/doc/README.org index 46df03c7..43c92e3c 100644 --- a/doc/README.org +++ b/doc/README.org @@ -218,7 +218,7 @@ information given by guix: On one system: -: export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +: export PYTHONPATH="$HOME/.guix-profile/lib/python3.8/site-packages" : export R_LIBS_SITE="$HOME/.guix-profile/site-library/" : export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0" diff --git a/doc/development.org b/doc/development.org index e65ccd58..cd3beea3 100644 --- a/doc/development.org +++ b/doc/development.org @@ -67,11 +67,11 @@ You can install a Python package locally with pip, e.g. pip install hjson #+END_SRC -This installed in ~$HOME/.local/lib/python2.7/site-packages~. To add +This installed in ~$HOME/.local/lib/python3.8/site-packages~. To add the search path for GeneNetwork use the environment variable #+BEGIN_SRC sh -export PYTHON_GN_PATH=$HOME/.local/lib/python2.7/site-packages +export PYTHON_GN_PATH=$HOME/.local/lib/python3.8/site-packages #+END_SRC Now you should be able to do diff --git a/etc/default_settings.py b/etc/default_settings.py index f368237b..27522187 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -19,12 +19,12 @@ # # For GNU Guix deployment also check the paths in # -# ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py +# ~/.guix-profile/lib/python3.8/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py import os import sys -GN_VERSION = open("../etc/VERSION","r").read() +GN_VERSION = open("../etc/VERSION", "r").read() GN_SERVER_URL = "http://localhost:8880/" # REST API server # ---- MySQL diff --git a/scripts/maintenance/QTL_Reaper_v6.py b/scripts/maintenance/QTL_Reaper_v6.py index e50dbd40..35f2d1a1 100755 --- a/scripts/maintenance/QTL_Reaper_v6.py +++ b/scripts/maintenance/QTL_Reaper_v6.py @@ -7,7 +7,7 @@ import reaper import MySQLdb import time -con = MySQLdb.Connect(db='db_webqtl',user='username',passwd='', host="localhost") +con = MySQLdb.Connect(db='db_webqtl', user='username', passwd='', host="localhost") cursor = con.cursor() genotypeDir = '/gnshare/gn/web/genotypes/' @@ -23,7 +23,7 @@ for item in results: ProbeSetFreezeIds=sys.argv[1:] if ProbeSetFreezeIds: #####convert the Ids to integer - ProbeSetFreezeIds=map(int, ProbeSetFreezeIds) + ProbeSetFreezeIds=list(map(int, ProbeSetFreezeIds)) else: #####get all of the dataset that need be updated @@ -53,7 +53,7 @@ for ProbeSetFreezeId in ProbeSetFreezeIds: #if InbredSetId==12: # InbredSetId=2 - print ProbeSetFreezeId, InbredSets[InbredSetId] + print((ProbeSetFreezeId, InbredSets[InbredSetId])) genotype_1.read(InbredSets[InbredSetId]) locuses = [] @@ -102,7 +102,7 @@ for ProbeSetFreezeId in ProbeSetFreezeIds: kj += 1 if kj%1000==0: - print ProbeSetFreezeId, InbredSets[InbredSetId],kj + print((ProbeSetFreezeId, InbredSets[InbredSetId], kj)) - print ProbeSetFreezeIds + print(ProbeSetFreezeIds) diff --git a/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py b/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py index 0f8602c9..bf796df4 100644 --- a/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py +++ b/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py @@ -24,4 +24,4 @@ for row in csv_data: #close the connection to the database. mydb.commit() cursor.close() -print "Done"
\ No newline at end of file +print("Done")
\ No newline at end of file diff --git a/scripts/maintenance/delete_genotypes.py b/scripts/maintenance/delete_genotypes.py index fa693f0f..b7f83758 100755 --- a/scripts/maintenance/delete_genotypes.py +++ b/scripts/maintenance/delete_genotypes.py @@ -8,26 +8,26 @@ import genotypes def main(argv): # config config = utilities.get_config(argv[1]) - print "config:" + print("config:") for item in config.items('config'): - print "\t%s" % (str(item)) + print(("\t%s" % (str(item)))) # var - print "variable:" + print("variable:") inbredsetid = config.get('config', 'inbredsetid') - print "\tinbredsetid: %s" % inbredsetid + print(("\tinbredsetid: %s" % inbredsetid)) # datafile datafile = open(config.get('config', 'datafile'), 'r') datafile = csv.reader(datafile, delimiter='\t', quotechar='"') - datafile.next() + next(datafile) delrowcount = 0 for row in datafile: if len(row) == 0: continue genoname = row[0] delrowcount += genotypes.delete(genoname, inbredsetid) - print "deleted %d genotypes" % (delrowcount) + print(("deleted %d genotypes" % (delrowcount))) if __name__ == "__main__": - print "command line arguments:\n\t%s" % sys.argv + print(("command line arguments:\n\t%s" % sys.argv)) main(sys.argv) - print "exit successfully" + print("exit successfully") diff --git a/scripts/maintenance/delete_phenotypes.py b/scripts/maintenance/delete_phenotypes.py index 326c466e..60dbec61 100755 --- a/scripts/maintenance/delete_phenotypes.py +++ b/scripts/maintenance/delete_phenotypes.py @@ -8,13 +8,13 @@ import phenotypes def main(argv): # config config = utilities.get_config(argv[1]) - print "config:" + print("config:") for item in config.items('config'): - print "\t%s" % (str(item)) + print(("\t%s" % (str(item)))) # var - print "variable:" + print("variable:") inbredsetid = config.get('config', 'inbredsetid') - print "\tinbredsetid: %s" % inbredsetid + print(("\tinbredsetid: %s" % inbredsetid)) # datafile datafile = open(config.get('config', 'datafile'), 'r') datafile = csv.reader(datafile, delimiter='\t', quotechar='"') @@ -27,9 +27,9 @@ def main(argv): except: continue delrowcount += phenotypes.delete(publishxrefid=publishxrefid, inbredsetid=inbredsetid) - print "deleted %d phenotypes" % (delrowcount) + print(("deleted %d phenotypes" % (delrowcount))) if __name__ == "__main__": - print "command line arguments:\n\t%s" % sys.argv + print(("command line arguments:\n\t%s" % sys.argv)) main(sys.argv) - print "exit successfully" + print("exit successfully") diff --git a/scripts/maintenance/load_genotypes.py b/scripts/maintenance/load_genotypes.py index 338483f4..51278d48 100755 --- a/scripts/maintenance/load_genotypes.py +++ b/scripts/maintenance/load_genotypes.py @@ -8,7 +8,7 @@ def main(argv): config = utilities.get_config(argv[1]) print("config file:") for item in config.items('config'): - print("\t%s" % str(item)) + print(("\t%s" % str(item))) parse_genofile(config, fetch_parameters(config)) def fetch_parameters(config): @@ -19,8 +19,8 @@ def fetch_parameters(config): config_dic['dataid'] = datastructure.get_nextdataid_genotype() config_dic['genofile'] = config.get('config', 'genofile') print("config dictionary:") - for k, v in config_dic.items(): - print("\t%s: %s" % (k, v)) + for k, v in list(config_dic.items()): + print(("\t%s: %s" % (k, v))) return config_dic def parse_genofile(config, config_dic): @@ -42,10 +42,10 @@ def parse_genofile(config, config_dic): if line.lower().startswith("chr"): # print("geno file meta dictionary:") - for k, v in meta_dic.items(): - print("\t%s: %s" % (k, v)) + for k, v in list(meta_dic.items()): + print(("\t%s: %s" % (k, v))) # - print("geno file head:\n\t%s" % line) + print(("geno file head:\n\t%s" % line)) strainnames = line.split()[4:] config_dic['strains'] = datastructure.get_strains_bynames(inbredsetid=config_dic['inbredsetid'], strainnames=strainnames, updatestrainxref="yes") continue @@ -81,7 +81,7 @@ def check_or_insert_geno(config_dic, marker_dic): result = cursor.fetchone() if result: genoid = result[0] - print("get geno record: %d" % genoid) + print(("get geno record: %d" % genoid)) else: sql = """ INSERT INTO Geno @@ -95,7 +95,7 @@ def check_or_insert_geno(config_dic, marker_dic): cursor.execute(sql, (config_dic['speciesid'], marker_dic['locus'], marker_dic['locus'], marker_dic['chromosome'], marker_dic['mb'])) rowcount = cursor.rowcount genoid = con.insert_id() - print("INSERT INTO Geno: %d record: %d" % (rowcount, genoid)) + print(("INSERT INTO Geno: %d record: %d" % (rowcount, genoid))) return genoid def check_genoxref(config_dic, marker_dic): @@ -146,9 +146,9 @@ def insert_genoxref(config_dic, marker_dic): """ cursor.execute(sql, (config_dic['genofreezeid'], marker_dic['genoid'], config_dic['dataid'], marker_dic['cm'], 'N')) rowcount = cursor.rowcount - print("INSERT INTO GenoXRef: %d record" % (rowcount)) + print(("INSERT INTO GenoXRef: %d record" % (rowcount))) if __name__ == "__main__": - print("command line arguments:\n\t%s" % sys.argv) + print(("command line arguments:\n\t%s" % sys.argv)) main(sys.argv) print("exit successfully") diff --git a/scripts/maintenance/load_phenotypes.py b/scripts/maintenance/load_phenotypes.py index c3c6570b..759d2eec 100755 --- a/scripts/maintenance/load_phenotypes.py +++ b/scripts/maintenance/load_phenotypes.py @@ -7,37 +7,37 @@ import datastructure def main(argv): # config config = utilities.get_config(argv[1]) - print "config:" + print("config:") for item in config.items('config'): - print "\t%s" % (str(item)) + print(("\t%s" % (str(item)))) # var inbredsetid = config.get('config', 'inbredsetid') - print "inbredsetid: %s" % inbredsetid + print(("inbredsetid: %s" % inbredsetid)) species = datastructure.get_species(inbredsetid) speciesid = species[0] - print "speciesid: %s" % speciesid + print(("speciesid: %s" % speciesid)) dataid = datastructure.get_nextdataid_phenotype() - print "next data id: %s" % dataid + print(("next data id: %s" % dataid)) cursor, con = utilities.get_cursor() # datafile datafile = open(config.get('config', 'datafile'), 'r') phenotypedata = csv.reader(datafile, delimiter='\t', quotechar='"') - phenotypedata_head = phenotypedata.next() - print "phenotypedata head:\n\t%s" % phenotypedata_head + phenotypedata_head = next(phenotypedata) + print(("phenotypedata head:\n\t%s" % phenotypedata_head)) strainnames = phenotypedata_head[1:] strains = datastructure.get_strains_bynames(inbredsetid=inbredsetid, strainnames=strainnames, updatestrainxref="yes") # metafile metafile = open(config.get('config', 'metafile'), 'r') phenotypemeta = csv.reader(metafile, delimiter='\t', quotechar='"') - phenotypemeta_head = phenotypemeta.next() - print "phenotypemeta head:\n\t%s" % phenotypemeta_head - print + phenotypemeta_head = next(phenotypemeta) + print(("phenotypemeta head:\n\t%s" % phenotypemeta_head)) + print() # load for metarow in phenotypemeta: # - datarow_value = phenotypedata.next() - datarow_se = phenotypedata.next() - datarow_n = phenotypedata.next() + datarow_value = next(phenotypedata) + datarow_se = next(phenotypedata) + datarow_n = next(phenotypedata) # Phenotype sql = """ INSERT INTO Phenotype @@ -67,7 +67,7 @@ def main(argv): )) rowcount = cursor.rowcount phenotypeid = con.insert_id() - print "INSERT INTO Phenotype: %d record: %d" % (rowcount, phenotypeid) + print(("INSERT INTO Phenotype: %d record: %d" % (rowcount, phenotypeid))) # Publication publicationid = None # reset pubmed_id = utilities.to_db_string(metarow[0], None) @@ -81,7 +81,7 @@ def main(argv): re = cursor.fetchone() if re: publicationid = re[0] - print "get Publication record: %d" % publicationid + print(("get Publication record: %d" % publicationid)) if not publicationid: sql = """ INSERT INTO Publication @@ -109,7 +109,7 @@ def main(argv): )) rowcount = cursor.rowcount publicationid = con.insert_id() - print "INSERT INTO Publication: %d record: %d" % (rowcount, publicationid) + print(("INSERT INTO Publication: %d record: %d" % (rowcount, publicationid))) # data for index, strain in enumerate(strains): # @@ -158,14 +158,14 @@ def main(argv): cursor.execute(sql, (inbredsetid, phenotypeid, publicationid, dataid, "")) rowcount = cursor.rowcount publishxrefid = con.insert_id() - print "INSERT INTO PublishXRef: %d record: %d" % (rowcount, publishxrefid) + print(("INSERT INTO PublishXRef: %d record: %d" % (rowcount, publishxrefid))) # for loop next dataid += 1 - print + print() # release con.close() if __name__ == "__main__": - print "command line arguments:\n\t%s" % sys.argv + print(("command line arguments:\n\t%s" % sys.argv)) main(sys.argv) - print "exit successfully" + print("exit successfully") diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index e9c8f25c..43f084f4 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -9,19 +9,17 @@ import sys import MySQLdb import getpass import time -#import pdb -#pdb.set_trace() ######################################################################## def translateAlias(str): - if str == "B6": - return "C57BL/6J" - elif str == "D2": - return "DBA/2J" - else: - return str + if str == "B6": + return "C57BL/6J" + elif str == "D2": + return "DBA/2J" + else: + return str ######################################################################## # @@ -29,23 +27,25 @@ def translateAlias(str): # ######################################################################## + dataStart = 1 -GeneChipId = int( raw_input("Enter GeneChipId:") ) -ProbeSetFreezeId = int( raw_input("Enter ProbeSetFreezeId:") ) -input_file_name = raw_input("Enter file name with suffix:") +GeneChipId = int(input("Enter GeneChipId:")) +ProbeSetFreezeId = int(input("Enter ProbeSetFreezeId:")) +input_file_name = input("Enter file name with suffix:") fp = open("%s" % input_file_name, 'rb') try: - passwd = getpass.getpass('Please enter mysql password here : ') - con = MySQLdb.Connect(db='db_webqtl',host='localhost', user='username',passwd=passwd) + passwd = getpass.getpass('Please enter mysql password here : ') + con = MySQLdb.Connect(db='db_webqtl', host='localhost', + user='username', passwd=passwd) - db = con.cursor() - print "You have successfully connected to mysql.\n" + db = con.cursor() + print("You have successfully connected to mysql.\n") except: - print "You entered incorrect password.\n" - sys.exit(0) + print("You entered incorrect password.\n") + sys.exit(0) time0 = time.time() @@ -55,163 +55,163 @@ time0 = time.time() # generate the gene list of expression data here # ######################################################################### -print 'Checking if each line have same number of members' +print('Checking if each line have same number of members') GeneList = [] isCont = 1 header = fp.readline() -header = string.split(string.strip(header),'\t') -header = map(string.strip, header) +header = header.strip().split('\t') +header = [x.strip() for x in header] nfield = len(header) line = fp.readline() -kj=0 +kj = 0 while line: - line2 = string.split(string.strip(line),'\t') - line2 = map(string.strip, line2) - if len(line2) != nfield: - print "Error : " + line - isCont = 0 + line2 = line.strip().split('\t') + line2 = [x.strip() for x in line2] + if len(line2) != nfield: + print(("Error : " + line)) + isCont = 0 - GeneList.append(line2[0]) - line = fp.readline() + GeneList.append(line2[0]) + line = fp.readline() - kj+=1 - if kj%100000 == 0: - print 'checked ',kj,' lines' + kj += 1 + if kj % 100000 == 0: + print(('checked ', kj, ' lines')) -GeneList = map(string.lower, GeneList) -GeneList.sort() - -if isCont==0: - sys.exit(0) +GeneList = sorted(map(string.lower, GeneList)) +if isCont == 0: + sys.exit(0) -print 'used ',time.time()-time0,' seconds' + +print(('used ', time.time()-time0, ' seconds')) ######################################################################### # # Check if each strain exist in database # generate the string id list of expression data here # ######################################################################### -print 'Checking if each strain exist in database' +print('Checking if each strain exist in database') isCont = 1 fp.seek(0) header = fp.readline() -header = string.split(string.strip(header),'\t') -header = map(string.strip, header) -header = map(translateAlias, header) +header = header.strip().split('\t') +header = [x.strip() for x in header] +header = list(map(translateAlias, header)) header = header[dataStart:] Ids = [] for item in header: - try: - db.execute('select Id from Strain where Name = "%s"' % item) - Ids.append(db.fetchall()[0][0]) - except: - print item,'does not exist, check the if the strain name is correct' - isCont=0 + try: + db.execute('select Id from Strain where Name = "%s"' % item) + Ids.append(db.fetchall()[0][0]) + except: + print((item, 'does not exist, check the if the strain name is correct')) + isCont = 0 -if isCont==0: - sys.exit(0) +if isCont == 0: + sys.exit(0) -print 'used ',time.time()-time0,' seconds' +print(('used ', time.time()-time0, ' seconds')) ######################################################################## # # Check if each ProbeSet exist in database # ######################################################################## -print 'Check if each ProbeSet exist in database' +print('Check if each ProbeSet exist in database') ##---- find PID is name or target ----## line = fp.readline() line = fp.readline() -line2 = string.split(string.strip(line),'\t') -line2 = map(string.strip, line2) +line2 = line.strip().split('\t') +line2 = [x.strip() for x in line2] PId = line2[0] -db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % (PId, GeneChipId) ) +db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % + (PId, GeneChipId)) results = db.fetchall() IdStr = 'TargetId' -if len(results)>0: - IdStr = 'Name' +if len(results) > 0: + IdStr = 'Name' ##---- get Name/TargetId list from database ----## -db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % (IdStr, GeneChipId, IdStr)) +db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % ( + IdStr, GeneChipId, IdStr)) results = db.fetchall() - + Names = [] for item in results: - Names.append(item[0]) - -print Names + Names.append(item[0]) -Names = map(string.lower, Names) +print(Names) -Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# +Names = sorted(map(string.lower, Names)) ##---- compare genelist with names ----## -x=y=0 -x1=-1 -GeneList2=[] -while x<len(GeneList) and y<len(Names): - if GeneList[x]==Names[y]: - x += 1 - y += 1 - elif GeneList[x]<Names[y]: - if x!=x1: - GeneList2.append(GeneList[x]) - x1 = x - x += 1 - elif GeneList[x]>Names[y]: - y += 1 - - if x%100000==0: - print 'check Name, checked %d lines'%x - -while x<len(GeneList): - GeneList2.append(GeneList[x]) - x += 1 - -isCont=1 +x = y = 0 +x1 = -1 +GeneList2 = [] +while x < len(GeneList) and y < len(Names): + if GeneList[x] == Names[y]: + x += 1 + y += 1 + elif GeneList[x] < Names[y]: + if x != x1: + GeneList2.append(GeneList[x]) + x1 = x + x += 1 + elif GeneList[x] > Names[y]: + y += 1 + + if x % 100000 == 0: + print(('check Name, checked %d lines' % x)) + +while x < len(GeneList): + GeneList2.append(GeneList[x]) + x += 1 + +isCont = 1 ferror = open("ProbeSetError.txt", "wb") for item in GeneList2: - ferror.write(item + " doesn't exist \n") - print item, " doesn't exist, check if the ProbeSet name is correct" - isCont = 0 - -if isCont==0: - sys.exit(0) + ferror.write(item + " doesn't exist \n") + print((item, " doesn't exist, check if the ProbeSet name is correct")) + isCont = 0 + +if isCont == 0: + sys.exit(0) -print 'used ',time.time()-time0,' seconds' +print(('used ', time.time()-time0, ' seconds')) ######################################################################### # # Insert data into database # ######################################################################### -print 'getting ProbeSet/Id' +print('getting ProbeSet/Id') #---- get Name/Id map ----# -db.execute('select %s, Id from ProbeSet where ChipId=%d order by %s' % (IdStr, GeneChipId, IdStr)) +db.execute('select %s, Id from ProbeSet where ChipId=%d order by %s' % + (IdStr, GeneChipId, IdStr)) results = db.fetchall() NameIds = {} for item in results: - NameIds[item[0]] = item[1] -print 'used ',time.time()-time0,' seconds' + NameIds[item[0]] = item[1] +print(('used ', time.time()-time0, ' seconds')) -print 'inserting data' +print('inserting data') ##---- get old max dataId ----## db.execute('select max(Id) from ProbeSetData') maxDataId = int(db.fetchall()[0][0]) bmax = maxDataId -print "old_max = %d\n" % bmax +print(("old_max = %d\n" % bmax)) ##---- insert data ----## fp.seek(0) @@ -222,53 +222,51 @@ kj = 0 values1 = [] values2 = [] while line: - line2 = string.split(string.strip(line),'\t') - line2 = map(string.strip, line2) - PId = line2[0] - recordId = NameIds[PId] - - maxDataId += 1 - datasorig = line2[dataStart:] - - ###### Data Table items ###### - i=0 - for item in datasorig: - try: - values1.append('(%d,%d,%s)' % (maxDataId, Ids[i], float(item))) - except: - pass - i += 1 - - values2.append("(%d,%d,%d)" % (ProbeSetFreezeId, recordId, maxDataId)) - - - ##---- insert into table ----## - kj += 1 - if kj % 100 == 0: - cmd = ','.join(values1) - cmd = 'insert into ProbeSetData values %s' % cmd - db.execute(cmd) - - cmd = ','.join(values2) - cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd - db.execute(cmd) - - values1=[] - values2=[] - print 'Inserted ', kj,' lines' - print 'used ',time.time()-time0,' seconds' - - line = fp.readline() - - - -if len(values1)>0: - cmd = ','.join(values1) - cmd = 'insert into ProbeSetData values %s' % cmd - db.execute(cmd) - - cmd = ','.join(values2) - cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd - db.execute(cmd) + line2 = line.strip().split('\t') + line2 = [x.strip() for x in line2] + PId = line2[0] + recordId = NameIds[PId] + + maxDataId += 1 + datasorig = line2[dataStart:] + + ###### Data Table items ###### + i = 0 + for item in datasorig: + try: + values1.append('(%d,%d,%s)' % (maxDataId, Ids[i], float(item))) + except: + pass + i += 1 + + values2.append("(%d,%d,%d)" % (ProbeSetFreezeId, recordId, maxDataId)) + + ##---- insert into table ----## + kj += 1 + if kj % 100 == 0: + cmd = ','.join(values1) + cmd = 'insert into ProbeSetData values %s' % cmd + db.execute(cmd) + + cmd = ','.join(values2) + cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd + db.execute(cmd) + + values1 = [] + values2 = [] + print(('Inserted ', kj, ' lines')) + print(('used ', time.time()-time0, ' seconds')) + + line = fp.readline() + + +if len(values1) > 0: + cmd = ','.join(values1) + cmd = 'insert into ProbeSetData values %s' % cmd + db.execute(cmd) + + cmd = ','.join(values2) + cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd + db.execute(cmd) con.close() diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index fd6f0bb8..2cfe2e07 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -1,254 +1,254 @@ -#!/usr/bin/python2
-"""This script use the nearest marker to the transcript as control, increasing permutation rounds according to the p-value"""
-########################################################################
-# Last Updated Sep 27, 2011 by Xiaodong
-# This version fix the bug that incorrectly exclude the first 2 probesetIDs
-########################################################################
-
-import string
-import sys
-import MySQLdb
-import getpass
-import time
-
-
-def translateAlias(str):
- if str == "B6":
- return "C57BL/6J"
- elif str == "D2":
- return "DBA/2J"
- else:
- return str
-
-########################################################################
-#
-# Indicate Data Start Position, ProbeFreezeId, GeneChipId, DataFile
-#
-########################################################################
-
-dataStart = 1
-
-GeneChipId = int( raw_input("Enter GeneChipId:") )
-ProbeSetFreezeId = int( raw_input("Enter ProbeSetFreezeId:") )
-input_file_name = raw_input("Enter file name with suffix:")
-
-fp = open("%s" % input_file_name, 'rb')
-
-
-try:
- passwd = getpass.getpass('Please enter mysql password here : ')
- con = MySQLdb.Connect(db='db_webqtl',host='localhost', user='username',passwd=passwd)
-
- db = con.cursor()
- print "You have successfully connected to mysql.\n"
-except:
- print "You entered incorrect password.\n"
- sys.exit(0)
-
-time0 = time.time()
-########################################################################
-#
-# Indicate Data Start Position, ProbeFreezeId, GeneChipId, DataFile
-#
-########################################################################
-
-#GeneChipId = 4
-#dataStart = 1
-#ProbeSetFreezeId = 359 #JAX Liver 6C Affy M430 2.0 (Jul11) MDP
-#fp = open("GSE10493_AllSamples_6C_Z_AvgSE.txt", 'rb')
-
-
-#########################################################################
-#
-# Check if each line have same number of members
-# generate the gene list of expression data here
-#
-#########################################################################
-print 'Checking if each line have same number of members'
-
-GeneList = []
-isCont = 1
-header = fp.readline()
-header = string.split(string.strip(header),'\t')
-header = map(string.strip, header)
-nfield = len(header)
-line = fp.readline()
-
-kj=0
-while line:
- line2 = string.split(string.strip(line),'\t')
- line2 = map(string.strip, line2)
- if len(line2) != nfield:
- print "Error : " + line
- isCont = 0
-
- GeneList.append(line2[0])
- line = fp.readline()
-
- kj+=1
- if kj%100000 == 0:
- print 'checked ',kj,' lines'
-
-GeneList = map(string.lower, GeneList)
-GeneList.sort()
-
-if isCont==0:
- sys.exit(0)
-
-
-print 'used ',time.time()-time0,' seconds'
-#########################################################################
-#
-# Check if each strain exist in database
-# generate the string id list of expression data here
-#
-#########################################################################
-print 'Checking if each strain exist in database'
-
-isCont = 1
-fp.seek(0)
-header = fp.readline()
-header = string.split(string.strip(header),'\t')
-header = map(string.strip, header)
-header = map(translateAlias, header)
-header = header[dataStart:]
-Ids = []
-for item in header:
- try:
- db.execute('select Id from Strain where Name = "%s"' % item)
- Ids.append(db.fetchall()[0][0])
- except:
- print item,'does not exist, check the if the strain name is correct'
- isCont=0
-
-if isCont==0:
- sys.exit(0)
-
-
-print 'used ',time.time()-time0,' seconds'
-########################################################################
-#
-# Check if each ProbeSet exist in database
-#
-########################################################################
-print 'Check if each ProbeSet exist in database'
-
-##---- find PID is name or target ----##
-line = fp.readline()
-line = fp.readline()
-line2 = string.split(string.strip(line),'\t')
-line2 = map(string.strip, line2)
-PId = line2[0]
-
-db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % (PId, GeneChipId))
-results = db.fetchall()
-IdStr = 'TargetId'
-if len(results)>0:
- IdStr = 'Name'
-
-
-##---- get Name/TargetId list from database ----##
-db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % (IdStr, GeneChipId, IdStr))
-results = db.fetchall()
-
-Names = []
-for item in results:
- Names.append(item[0])
-Names = map(string.lower, Names)
-Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --#
-
-##---- compare genelist with names ----##
-x=y=0
-x1=-1
-GeneList2=[]
-while x<len(GeneList) and y<len(Names):
- if GeneList[x]==Names[y]:
- x += 1
- y += 1
- elif GeneList[x]<Names[y]:
- if x!=x1:
- GeneList2.append(GeneList[x])
- x1 = x
- x += 1
- elif GeneList[x]>Names[y]:
- y += 1
-
- if x%100000==0:
- print 'check Name, checked %d lines'%x
-
-while x<len(GeneList):
- GeneList2.append(GeneList[x])
- x += 1
-
-isCont=1
-ferror = open("ProbeSetError.txt", "wb")
-for item in GeneList2:
- ferror.write(item + " doesn't exist \n")
- print item, " doesn't exist"
- isCont = 0
-
-if isCont==0:
- sys.exit(0)
-
-
-print 'used ',time.time()-time0,' seconds'
-#############################
-#Insert new Data into SE
-############################
-db.execute("""
- select ProbeSet.%s, ProbeSetXRef.DataId from ProbeSet, ProbeSetXRef
- where ProbeSet.Id=ProbeSetXRef.ProbeSetId and ProbeSetXRef.ProbeSetFreezeId=%d"""
- % (IdStr, ProbeSetFreezeId))
-results = db.fetchall()
-
-ProbeNameId = {}
-for Name, Id in results:
- ProbeNameId[Name] = Id
-
-ferror = open("ProbeError.txt", "wb")
-
-DataValues = []
-
-fp.seek(0) #XZ add this line
-line = fp.readline() #XZ add this line
-line = fp.readline()
-
-kj = 0
-while line:
- line2 = string.split(string.strip(line),'\t')
- line2 = map(string.strip, line2)
-
- CellId = line2[0]
- if not ProbeNameId.has_key(CellId):
- ferror.write(CellId + " doesn't exist\n")
- print CellId, " doesn't exist"
- else:
- DataId = ProbeNameId[CellId]
- datasorig = line2[dataStart:]
-
- i = 0
- for item in datasorig:
- if item != '':
- value = '('+str(DataId)+','+str(Ids[i])+','+str(item)+')'
- DataValues.append(value)
- i += 1
-
- kj += 1
- if kj % 100 == 0:
- Dataitems = ','.join(DataValues)
- cmd = 'insert ProbeSetSE values %s' % Dataitems
- db.execute(cmd)
-
- DataValues = []
- print 'inserted ',kj,' lines'
- print 'used ',time.time()-time0,' seconds'
- line = fp.readline()
-
-if len(DataValues)>0:
- DataValues = ','.join(DataValues)
- cmd = 'insert ProbeSetSE values %s' % DataValues
- db.execute(cmd)
-
-con.close()
-
-
+#!/usr/bin/python2 +"""This script use the nearest marker to the transcript as control, increasing permutation rounds according to the p-value""" +######################################################################## +# Last Updated Sep 27, 2011 by Xiaodong +# This version fix the bug that incorrectly exclude the first 2 probesetIDs +######################################################################## + +import string +import sys +import MySQLdb +import getpass +import time + + +def translateAlias(str): + if str == "B6": + return "C57BL/6J" + elif str == "D2": + return "DBA/2J" + else: + return str + +######################################################################## +# +# Indicate Data Start Position, ProbeFreezeId, GeneChipId, DataFile +# +######################################################################## + + +dataStart = 1 + +GeneChipId = int(input("Enter GeneChipId:")) +ProbeSetFreezeId = int(input("Enter ProbeSetFreezeId:")) +input_file_name = input("Enter file name with suffix:") + +fp = open("%s" % input_file_name, 'rb') + + +try: + passwd = getpass.getpass('Please enter mysql password here : ') + con = MySQLdb.Connect(db='db_webqtl', host='localhost', + user='username', passwd=passwd) + + db = con.cursor() + print("You have successfully connected to mysql.\n") +except: + print("You entered incorrect password.\n") + sys.exit(0) + +time0 = time.time() +######################################################################## +# +# Indicate Data Start Position, ProbeFreezeId, GeneChipId, DataFile +# +######################################################################## + +#GeneChipId = 4 +#dataStart = 1 +# ProbeSetFreezeId = 359 #JAX Liver 6C Affy M430 2.0 (Jul11) MDP +#fp = open("GSE10493_AllSamples_6C_Z_AvgSE.txt", 'rb') + + +######################################################################### +# +# Check if each line have same number of members +# generate the gene list of expression data here +# +######################################################################### +print('Checking if each line have same number of members') + +GeneList = [] +isCont = 1 +header = fp.readline() +header = header.strip().split('\t') +header = [item.strip() for item in header] +nfield = len(header) +line = fp.readline() + +kj = 0 +while line: + line2 = line.strip().split('\t') + line2 = [item.strip() for item in line2] + if len(line2) != nfield: + isCont = 0 + print(("Error : " + line)) + + GeneList.append(line2[0]) + line = fp.readline() + + kj += 1 + if kj % 100000 == 0: + print(('checked ', kj, ' lines')) + +GeneList = sorted(map(string.lower, GeneList)) + +if isCont == 0: + sys.exit(0) + + +print(('used ', time.time()-time0, ' seconds')) +######################################################################### +# +# Check if each strain exist in database +# generate the string id list of expression data here +# +######################################################################### +print('Checking if each strain exist in database') + +isCont = 1 +fp.seek(0) +header = fp.readline() +header = header.strip().split('\t') +header = [item.strip() for item in header] +header = list(map(translateAlias, header)) +header = header[dataStart:] +Ids = [] +for item in header: + try: + db.execute('select Id from Strain where Name = "%s"' % item) + Ids.append(db.fetchall()[0][0]) + except: + isCont = 0 + print((item, 'does not exist, check the if the strain name is correct')) + +if isCont == 0: + sys.exit(0) + + +print(('used ', time.time()-time0, ' seconds')) +######################################################################## +# +# Check if each ProbeSet exist in database +# +######################################################################## +print('Check if each ProbeSet exist in database') + +##---- find PID is name or target ----## +line = fp.readline() +line = fp.readline() +line2 = line.strip().split('\t') +line2 = [x.strip() for x in line2] +PId = line2[0] + +db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % + (PId, GeneChipId)) +results = db.fetchall() +IdStr = 'TargetId' +if len(results) > 0: + IdStr = 'Name' + + +##---- get Name/TargetId list from database ----## +db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % ( + IdStr, GeneChipId, IdStr)) +results = db.fetchall() + +Names = [] +for item in results: + Names.append(item[0]) + Names = sorted(map(string.lower, Names)) + +##---- compare genelist with names ----## +x = y = 0 +x1 = -1 +GeneList2 = [] +while x < len(GeneList) and y < len(Names): + if GeneList[x] == Names[y]: + x += 1 + y += 1 + elif GeneList[x] < Names[y]: + if x != x1: + GeneList2.append(GeneList[x]) + x1 = x + x += 1 + elif GeneList[x] > Names[y]: + y += 1 + + if x % 100000 == 0: + print(('check Name, checked %d lines' % x)) + +while x < len(GeneList): + GeneList2.append(GeneList[x]) + x += 1 + +isCont = 1 +ferror = open("ProbeSetError.txt", "wb") +for item in GeneList2: + ferror.write(item + " doesn't exist \n") + isCont = 0 + + print((item, " doesn't exist")) +if isCont == 0: + sys.exit(0) + + +print(('used ', time.time()-time0, ' seconds')) +############################# +# Insert new Data into SE +############################ +db.execute(""" + select ProbeSet.%s, ProbeSetXRef.DataId from ProbeSet, ProbeSetXRef + where ProbeSet.Id=ProbeSetXRef.ProbeSetId and ProbeSetXRef.ProbeSetFreezeId=%d""" + % (IdStr, ProbeSetFreezeId)) +results = db.fetchall() + +ProbeNameId = {} +for Name, Id in results: + ProbeNameId[Name] = Id + +ferror = open("ProbeError.txt", "wb") + +DataValues = [] + +fp.seek(0) # XZ add this line +line = fp.readline() # XZ add this line +line = fp.readline() + +kj = 0 +while line: + line2 = line.strip().split('\t') + line2 = [x.strip() for x in line2] + + CellId = line2[0] + if CellId not in ProbeNameId: + ferror.write(CellId + " doesn't exist\n") + else: + DataId = ProbeNameId[CellId] + datasorig = line2[dataStart:] + + i = 0 + for item in datasorig: + if item != '': + value = '('+str(DataId)+','+str(Ids[i])+','+str(item)+')' + DataValues.append(value) + i += 1 + + kj += 1 + if kj % 100 == 0: + Dataitems = ','.join(DataValues) + cmd = 'insert ProbeSetSE values %s' % Dataitems + db.execute(cmd) + + DataValues = [] + line = fp.readline() + print((CellId, " doesn't exist")) + print(('inserted ', kj, ' lines')) + print(('used ', time.time()-time0, ' seconds')) + +if len(DataValues) > 0: + DataValues = ','.join(DataValues) + cmd = 'insert ProbeSetSE values %s' % DataValues + db.execute(cmd) + +con.close() @@ -9,7 +9,7 @@ setup(name='genenetwork2', url = "https://github.com/genenetwork/genenetwork2/blob/master/README.md", description = 'Website and tools for genetics.', include_package_data=True, - packages=['wqflask','etc'], + packages=['wqflask', 'etc'], scripts=['bin/genenetwork2'], # package_data = { # 'etc': ['*.py'] diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index d040ba54..6ac26ba7 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import requests from lxml.html import parse @@ -23,12 +22,9 @@ def is_in_page_link(link): return pattern.match(link) def get_links(doc): - return filter( - lambda x: not ( + return [x for x in [y.get("href") for y in doc.cssselect("a")] if not ( is_root_link(x) - or is_mailto_link(x)) - , map(lambda y: y.get("href") - , doc.cssselect("a"))) + or is_mailto_link(x))] def verify_link(link): if link[0] == "#": @@ -58,7 +54,7 @@ def verify_static_file(link): try: result = requests.get(link, timeout=20, verify=False) if (result.status_code == 200 and - result.content.find("Error: 404 Not Found") <= 0): + result.content.find(bytes("Error: 404 Not Found", "utf-8")) <= 0): print(link+" ==> OK") else: print("ERROR: link {}".format(link)) @@ -72,9 +68,9 @@ def check_page(host, start_url): print("Checking links host "+host+" in page `"+start_url+"`") doc = parse(start_url).getroot() links = get_links(doc) - in_page_links = filter(is_in_page_link, links) - internal_links = filter(is_internal_link, links) - external_links = filter(lambda x: not (is_internal_link(x) or is_in_page_link(x)), links) + in_page_links = list(filter(is_in_page_link, links)) + internal_links = list(filter(is_internal_link, links)) + external_links = [x for x in links if not (is_internal_link(x) or is_in_page_link(x))] for link in internal_links: verify_link(host+link) diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py index d4c3b1ad..28033ad5 100644 --- a/test/requests/main_web_functionality.py +++ b/test/requests/main_web_functionality.py @@ -1,9 +1,7 @@ -from __future__ import print_function -import re import requests from lxml.html import parse from link_checker import check_page -from requests.exceptions import ConnectionError + def check_home(url): doc = parse(url).getroot() @@ -13,18 +11,20 @@ def check_home(url): def check_search_page(host): data = dict( - species="mouse" - , group="BXD" - , type="Hippocampus mRNA" - , dataset="HC_M2_0606_P" - , search_terms_or="" - , search_terms_and="MEAN=(15 16) LRS=(23 46)") + species="mouse", + group="BXD", + type="Hippocampus mRNA", + dataset="HC_M2_0606_P", + search_terms_or="", + search_terms_and="MEAN=(15 16) LRS=(23 46)") result = requests.get(host+"/search", params=data) found = result.text.find("records were found") assert(found >= 0) assert(result.status_code == 200) print("OK") - check_traits_page(host, "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") + check_traits_page(host, ("/show_trait?trait_id=1435395_" + "s_at&dataset=HC_M2_0606_P")) + def check_traits_page(host, traits_url): doc = parse(host+traits_url).getroot() @@ -33,6 +33,7 @@ def check_traits_page(host, traits_url): print("OK") check_page(host, host+traits_url) + def check_main_web_functionality(args_obj, parser): print("") print("Checking main web functionality...") diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py index 5748a2a3..19b22c21 100644 --- a/test/requests/mapping_tests.py +++ b/test/requests/mapping_tests.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import copy import json diff --git a/test/requests/navigation_tests.py b/test/requests/navigation_tests.py index eda27324..6b91c1fd 100644 --- a/test/requests/navigation_tests.py +++ b/test/requests/navigation_tests.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import requests from lxml.html import parse diff --git a/test/requests/test-website.py b/test/requests/test-website.py index f90d1843..8bfb47c2 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -3,7 +3,7 @@ # env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 # # Mostly to pick up the Guix GN2_PROFILE and python modules -from __future__ import print_function + import argparse from link_checker import check_links from link_checker import check_packaged_js_files diff --git a/webtests/browser_run.py b/webtests/browser_run.py index 2ec299c5..7ee540b7 100644 --- a/webtests/browser_run.py +++ b/webtests/browser_run.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - __all__ = ('sleep', 'testmod', 'test') from doctest import testmod @@ -71,4 +69,4 @@ class Test(object): -test = Test()
\ No newline at end of file +test = Test() diff --git a/webtests/correlation_matrix_test.py b/webtests/correlation_matrix_test.py index 8529c265..97114890 100644 --- a/webtests/correlation_matrix_test.py +++ b/webtests/correlation_matrix_test.py @@ -65,8 +65,6 @@ text: 0.608\n71 """ -from __future__ import absolute_import, division, print_function - from browser_run import * testmod() diff --git a/webtests/correlation_test.py b/webtests/correlation_test.py index aad3a69f..311bb847 100644 --- a/webtests/correlation_test.py +++ b/webtests/correlation_test.py @@ -44,8 +44,6 @@ text: 1.000 """ -from __future__ import absolute_import, division, print_function - from browser_run import * testmod() diff --git a/webtests/marker_regression_test.py b/webtests/marker_regression_test.py index c4f76f53..9b4a4acb 100644 --- a/webtests/marker_regression_test.py +++ b/webtests/marker_regression_test.py @@ -48,8 +48,6 @@ text: 11.511 """ -from __future__ import absolute_import, division, print_function - from browser_run import * testmod() diff --git a/webtests/show_trait_js_test.py b/webtests/show_trait_js_test.py index 0fd2c16c..34ffd3b7 100644 --- a/webtests/show_trait_js_test.py +++ b/webtests/show_trait_js_test.py @@ -35,8 +35,6 @@ style: display: none; """ -from __future__ import absolute_import, division, print_function - from browser_run import * testmod() diff --git a/webtests/test_runner.py b/webtests/test_runner.py index ef6d0d69..b5b590a6 100644 --- a/webtests/test_runner.py +++ b/webtests/test_runner.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import unittest import doctest import glob @@ -25,4 +23,4 @@ def main(): runner.run(suite) if __name__ == '__main__': - main()
\ No newline at end of file + main() diff --git a/wqflask/base/GeneralObject.py b/wqflask/base/GeneralObject.py index 0fccaab3..0122ee32 100644 --- a/wqflask/base/GeneralObject.py +++ b/wqflask/base/GeneralObject.py @@ -33,7 +33,7 @@ class GeneralObject: def __init__(self, *args, **kw): self.contents = list(args) - for name, value in kw.items(): + for name, value in list(kw.items()): setattr(self, name, value) def __setitem__(self, key, value): @@ -50,16 +50,17 @@ class GeneralObject: def __str__(self): s = '' - for key in self.__dict__.keys(): + for key in list(self.__dict__.keys()): if key != 'contents': s += '%s = %s\n' % (key, self.__dict__[key]) return s def __repr__(self): s = '' - for key in self.__dict__.keys(): + for key in list(self.__dict__.keys()): s += '%s = %s\n' % (key, self.__dict__[key]) return s - def __cmp__(self, other): - return len(self.__dict__.keys()).__cmp__(len(other.__dict__.keys())) + def __eq__(self, other): + return (len(list(self.__dict__.keys())) == + len(list(other.__dict__.keys()))) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 2f1549ae..0d4ac24b 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -18,13 +18,14 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division from db.call import fetchall, fetchone, fetch1 from utility.logger import getLogger from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL from db.gn_server import menu_main from pprint import pformat as pf -from MySQLdb import escape_string as escape +from utility.db_tools import escape +from utility.db_tools import mescape +from utility.db_tools import create_in_clause from maintenance import get_group_samplelists from utility.tools import locate, locate_ignore_error, flat_files from utility import gen_geno_ob @@ -34,7 +35,6 @@ from utility import webqtlUtil from db import webqtlDatabaseFunction from base import species from base import webqtlConfig -import reaper from flask import Flask, g import os import math @@ -45,7 +45,7 @@ import codecs import json import requests import gzip -import cPickle as pickle +import pickle as pickle import itertools from redis import Redis @@ -209,20 +209,6 @@ def create_datasets_list(): return datasets -def create_in_clause(items): - """Create an in clause for mysql""" - in_clause = ', '.join("'{}'".format(x) for x in mescape(*items)) - in_clause = '( {} )'.format(in_clause) - return in_clause - - -def mescape(*items): - """Multiple escape""" - escaped = [escape(str(item)) for item in items] - #logger.debug("escaped is:", escaped) - return escaped - - class Markers(object): """Todo: Build in cacheing so it saves us reading the same file more than once""" @@ -257,12 +243,12 @@ class Markers(object): logger.debug("length of self.markers:", len(self.markers)) logger.debug("length of p_values:", len(p_values)) - if type(p_values) is list: + if isinstance(p_values, list): # THIS IS only needed for the case when we are limiting the number of p-values calculated # if len(self.markers) > len(p_values): # self.markers = self.markers[:len(p_values)] - for marker, p_value in itertools.izip(self.markers, p_values): + for marker, p_value in zip(self.markers, p_values): if not p_value: continue marker['p_value'] = float(p_value) @@ -273,7 +259,7 @@ class Markers(object): marker['lod_score'] = -math.log10(marker['p_value']) # Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 - elif type(p_values) is dict: + elif isinstance(p_values, dict): filtered_markers = [] for marker in self.markers: #logger.debug("marker[name]", marker['name']) @@ -459,12 +445,7 @@ class DatasetGroup(object): full_filename = str(locate(self.genofile, 'genotype')) else: full_filename = str(locate(self.name + '.geno', 'genotype')) - - if use_reaper: - genotype_1 = reaper.Dataset() - genotype_1.read(full_filename) - else: - genotype_1 = gen_geno_ob.genotype(full_filename) + genotype_1 = gen_geno_ob.genotype(full_filename) if genotype_1.type == "group" and self.parlist: genotype_2 = genotype_1.add( @@ -707,7 +688,7 @@ class DataSet(object): else: query = "SELECT {}.Name,".format(escape(dataset_type)) data_start_pos = 1 - query += string.join(temp, ', ') + query += ', '.join(temp) query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(dataset_type, self.type, self.type)) @@ -1053,9 +1034,9 @@ class MrnaAssayDataSet(DataSet): # XZ, 12/08/2008: description # XZ, 06/05/2009: Rob asked to add probe target description - description_string = unicode( + description_string = str( str(this_trait.description).strip(codecs.BOM_UTF8), 'utf-8') - target_string = unicode( + target_string = str( str(this_trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') if len(description_string) > 1 and description_string != 'None': diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index 6fec5dcd..f1929518 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import collections from flask import g @@ -7,9 +5,8 @@ from flask import g from utility import db_tools from utility import Bunch -from MySQLdb import escape_string as escape +from utility.db_tools import escape -from pprint import pformat as pf from utility.logger import getLogger logger = getLogger(__name__ ) @@ -92,4 +89,4 @@ class MrnaAssayTissueData(object): else: symbol_values_dict[result.Symbol.lower()].append(result.value) - return symbol_values_dict
\ No newline at end of file + return symbol_values_dict diff --git a/wqflask/base/species.py b/wqflask/base/species.py index 6d99af65..2771d116 100644 --- a/wqflask/base/species.py +++ b/wqflask/base/species.py @@ -1,14 +1,7 @@ -from __future__ import absolute_import, print_function, division - import collections from flask import Flask, g -#from MySQLdb import escape_string as escape - -from utility import Bunch - -from pprint import pformat as pf from utility.logger import getLogger logger = getLogger(__name__ ) @@ -59,4 +52,4 @@ class Chromosomes(object): results = g.db.execute(query).fetchall() for item in results: - self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length)
\ No newline at end of file + self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 7ebbc4bb..cfc02f8b 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -1,42 +1,30 @@ -from __future__ import absolute_import, division, print_function -from utility.logger import getLogger -from flask import Flask, g, request, url_for, redirect, make_response, render_template -from pprint import pformat as pf -from MySQLdb import escape_string as escape +import requests import simplejson as json from wqflask import app -import os -import string -import resource -import codecs -import requests -import random -import urllib - from base import webqtlConfig from base.webqtlCaseData import webqtlCaseData from base.data_set import create_dataset -from db import webqtlDatabaseFunction -from utility import webqtlUtil from utility import hmac from utility.authentication_tools import check_resource_availability -from utility.tools import GN2_BASE_URL, GN_VERSION -from utility.redis_tools import get_redis_conn -from utility.redis_tools import get_resource_id -from utility.redis_tools import get_resource_info +from utility.tools import GN2_BASE_URL +from utility.redis_tools import get_redis_conn, get_resource_id -Redis = get_redis_conn() +from utility.db_tools import escape + +from flask import g, request, url_for +from utility.logger import getLogger logger = getLogger(__name__) +Redis = get_redis_conn() + def create_trait(**kw): assert bool(kw.get('dataset')) != bool( kw.get('dataset_name')), "Needs dataset ob. or name" - permitted = True if kw.get('name'): if kw.get('dataset_name'): if kw.get('dataset_name') != "Temp": @@ -55,7 +43,9 @@ def create_trait(**kw): the_trait = GeneralTrait(**kw) if the_trait.dataset.type != "Temp": the_trait = retrieve_trait_info( - the_trait, the_trait.dataset, get_qtl_info=kw.get('get_qtl_info')) + the_trait, + the_trait.dataset, + get_qtl_info=kw.get('get_qtl_info')) return the_trait else: return None @@ -78,7 +68,9 @@ class GeneralTrait(object): if kw.get('dataset_name') == "Temp": temp_group = self.name.split("_")[2] self.dataset = create_dataset( - dataset_name="Temp", dataset_type="Temp", group_name=temp_group) + dataset_name="Temp", + dataset_type="Temp", + group_name=temp_group) else: self.dataset = create_dataset(kw.get('dataset_name')) else: @@ -113,9 +105,10 @@ class GeneralTrait(object): elif len(name2) == 3: self.dataset, self.name, self.cellid = name2 - # Todo: These two lines are necessary most of the time, but perhaps not all of the time - # So we could add a simple if statement to short-circuit this if necessary - if get_sample_info != False: + # Todo: These two lines are necessary most of the time, but + # perhaps not all of the time So we could add a simple if + # statement to short-circuit this if necessary + if get_sample_info is not False: self = retrieve_sample_data(self, self.dataset) def export_informative(self, include_variance=0): @@ -128,9 +121,9 @@ class GeneralTrait(object): vals = [] the_vars = [] sample_aliases = [] - for sample_name, sample_data in self.data.items(): - if sample_data.value != None: - if not include_variance or sample_data.variance != None: + for sample_name, sample_data in list(self.data.items()): + if sample_data.value is not None: + if not include_variance or sample_data.variance is not None: samples.append(sample_name) vals.append(sample_data.value) the_vars.append(sample_data.variance) @@ -154,7 +147,8 @@ class GeneralTrait(object): formatted = self.post_publication_description else: formatted = "Not available" - + if isinstance(formatted, bytes): + formatted = formatted.decode("utf-8") return formatted @property @@ -163,8 +157,8 @@ class GeneralTrait(object): alias = 'Not available' if getattr(self, "alias", None): - alias = string.replace(self.alias, ";", " ") - alias = string.join(string.split(alias), ", ") + alias = self.alias.replace(";", " ") + alias = ", ".join(alias.split()) return alias @@ -183,7 +177,8 @@ class GeneralTrait(object): if human_response and mouse_response and other_response: alias_list = json.loads(human_response.content) + json.loads( - mouse_response.content) + json.loads(other_response.content) + mouse_response.content) + \ + json.loads(other_response.content) filtered_aliases = [] seen = set() @@ -201,7 +196,8 @@ class GeneralTrait(object): def location_fmt(self): """Return a text formatted location - While we're at it we set self.location in case we need it later (do we?) + While we're at it we set self.location in case we need it + later (do we?) """ @@ -223,7 +219,7 @@ class GeneralTrait(object): def retrieve_sample_data(trait, dataset, samplelist=None): - if samplelist == None: + if samplelist is None: samplelist = [] if dataset.type == "Temp": @@ -278,7 +274,9 @@ def get_sample_data(): trait_dict['pubmed_link'] = trait_ob.pubmed_link trait_dict['pubmed_text'] = trait_ob.pubmed_text - return json.dumps([trait_dict, {key: value.value for key, value in trait_ob.data.iteritems()}]) + return json.dumps([trait_dict, {key: value.value for + key, value in list( + trait_ob.data.items())}]) else: return None @@ -289,7 +287,8 @@ def jsonable(trait): Actual turning into json doesn't happen here though""" dataset = create_dataset(dataset_name=trait.dataset.name, - dataset_type=trait.dataset.type, group_name=trait.dataset.group.name) + dataset_type=trait.dataset.type, + group_name=trait.dataset.group.name) if dataset.type == "ProbeSet": return dict(name=trait.name, @@ -471,8 +470,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): # XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name # to avoid the problem of same marker name from different species. elif dataset.type == 'Geno': - display_fields_string = string.join( - dataset.display_fields, ',Geno.') + display_fields_string = ',Geno.'.join(dataset.display_fields) display_fields_string = 'Geno.' + display_fields_string query = """ SELECT %s @@ -491,13 +489,15 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): query = """SELECT %s FROM %s WHERE Name = %s""" logger.sql(query) trait_info = g.db.execute(query, - (string.join(dataset.display_fields, ','), - dataset.type, trait.name)).fetchone() + ','.join(dataset.display_fields), + dataset.type, trait.name).fetchone() if trait_info: trait.haveinfo = True for i, field in enumerate(dataset.display_fields): holder = trait_info[i] + if isinstance(holder, bytes): + holder = holder.decode("utf-8", errors="ignore") setattr(trait, field, holder) if dataset.type == 'Publish': @@ -523,13 +523,6 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): else: trait.description_display = "" - trait.abbreviation = unicode(str(trait.abbreviation).strip( - codecs.BOM_UTF8), 'utf-8', errors="replace") - trait.description_display = unicode(str(trait.description_display).strip( - codecs.BOM_UTF8), 'utf-8', errors="replace") - trait.authors = unicode(str(trait.authors).strip( - codecs.BOM_UTF8), 'utf-8', errors="replace") - if not trait.year.isdigit(): trait.pubmed_text = "N/A" else: @@ -539,10 +532,8 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id if dataset.type == 'ProbeSet' and dataset.group: - description_string = unicode( - str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') - target_string = unicode( - str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') + description_string = trait.description + target_string = trait.probe_target_description if str(description_string or "") != "" and description_string != 'None': description_display = description_string @@ -645,6 +636,6 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if str(trait.lrs or "") != "": trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs else: - raise KeyError, `trait.name`+ ' information is not found in the database.' - + raise KeyError(repr(trait.name) + + ' information is not found in the database.') return trait diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py index 3cf2d80d..aa55470f 100644 --- a/wqflask/base/webqtlCaseData.py +++ b/wqflask/base/webqtlCaseData.py @@ -41,8 +41,6 @@ class webqtlCaseData: self.this_id = None # Set a sane default (can't be just "id" cause that's a reserved word) self.outlier = None # Not set to True/False until later - self.first_attr_col = self.get_first_attr_col() - def __repr__(self): case_data_string = "<webqtlCaseData> " if self.value is not None: @@ -80,13 +78,4 @@ class webqtlCaseData: def display_num_cases(self): if self.num_cases is not None: return "%s" % self.num_cases - return "x" - - def get_first_attr_col(self): - col_num = 4 - if self.variance is not None: - col_num += 2 - if self.num_cases is not None: - col_num += 1 - - return col_num
\ No newline at end of file + return "x"
\ No newline at end of file diff --git a/wqflask/db/call.py b/wqflask/db/call.py index 1a1b3adc..0971d2a2 100644 --- a/wqflask/db/call.py +++ b/wqflask/db/call.py @@ -3,7 +3,10 @@ from flask import g import string -import urllib2 +try: # Python2 support + import urllib.request, urllib.error, urllib.parse +except: + import urllib2 import json from utility.tools import USE_GN_SERVER, LOG_SQL, GN_SERVER_URL from utility.benchmark import Bench @@ -26,8 +29,8 @@ GN_SERVER result when set (which should return a Tuple) else: res2 = result, if LOG_SQL: - logger.debug("Replaced SQL call",query) - logger.debug(path,res2) + logger.debug("Replaced SQL call", query) + logger.debug(path, res2) return res2 else: return fetchone(query) @@ -37,7 +40,7 @@ def fetchone(query): original fetchone, but with logging) """ - with Bench("SQL",LOG_SQL): + with Bench("SQL", LOG_SQL): def helper(query): res = g.db.execute(query) return res.fetchone() @@ -48,7 +51,7 @@ def fetchall(query): original fetchall, but with logging) """ - with Bench("SQL",LOG_SQL): + with Bench("SQL", LOG_SQL): def helper(query): res = g.db.execute(query) return res.fetchall() @@ -58,8 +61,12 @@ def gn_server(path): """Return JSON record by calling GN_SERVER """ - with Bench("GN_SERVER",LOG_SQL): - res = urllib2.urlopen(GN_SERVER_URL+path) + with Bench("GN_SERVER", LOG_SQL): + res = () + try: + res = urllib.request.urlopen(GN_SERVER_URL+path) + except: + res = urllib2.urlopen(GN_SERVER_URL+path) rest = res.read() res2 = json.loads(rest) logger.debug(res2) diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index 8a9dc79d..2805febd 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -35,13 +35,13 @@ def retrieve_species(group): """Get the species of a group (e.g. returns string "mouse" on "BXD" """ - result = fetch1("select Species.Name from Species, InbredSet where InbredSet.Name = '%s' and InbredSet.SpeciesId = Species.Id" % (group),"/cross/"+group+".json",lambda r: (r["species"],))[0] - logger.debug("retrieve_species result:",result) + result = fetch1("select Species.Name from Species, InbredSet where InbredSet.Name = '%s' and InbredSet.SpeciesId = Species.Id" % (group), "/cross/"+group+".json", lambda r: (r["species"],))[0] + logger.debug("retrieve_species result:", result) return result def retrieve_species_id(group): - result = fetch1("select SpeciesId from InbredSet where Name = '%s'" % (group),"/cross/"+group+".json",lambda r: (r["species_id"],))[0] - logger.debug("retrieve_species_id result:",result) + result = fetch1("select SpeciesId from InbredSet where Name = '%s'" % (group), "/cross/"+group+".json", lambda r: (r["species_id"],))[0] + logger.debug("retrieve_species_id result:", result) return result diff --git a/wqflask/maintenance/convert_dryad_to_bimbam.py b/wqflask/maintenance/convert_dryad_to_bimbam.py index e833b395..12ce35e9 100644 --- a/wqflask/maintenance/convert_dryad_to_bimbam.py +++ b/wqflask/maintenance/convert_dryad_to_bimbam.py @@ -6,7 +6,6 @@ Convert data dryad files to a BIMBAM _geno and _snps file """ -from __future__ import print_function, division, absolute_import import sys sys.path.append("..") @@ -67,4 +66,4 @@ def convert_dryad_to_bimbam(filename): if __name__=="__main__": input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" - convert_dryad_to_bimbam(input_filename)
\ No newline at end of file + convert_dryad_to_bimbam(input_filename) diff --git a/wqflask/maintenance/convert_geno_to_bimbam.py b/wqflask/maintenance/convert_geno_to_bimbam.py index 528b98cf..d49742f2 100644 --- a/wqflask/maintenance/convert_geno_to_bimbam.py +++ b/wqflask/maintenance/convert_geno_to_bimbam.py @@ -9,7 +9,6 @@ code """ -from __future__ import print_function, division, absolute_import import sys sys.path.append("..") import os @@ -187,4 +186,4 @@ if __name__=="__main__": #convertob = ConvertGenoFile("/home/zas1024/gene/genotype_files/genotypes/SRxSHRSPF2.geno", "/home/zas1024/gene/genotype_files/new_genotypes/SRxSHRSPF2.json") #convertob.convert() ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory) - #ConvertGenoFiles(Geno_Directory)
\ No newline at end of file + #ConvertGenoFiles(Geno_Directory) diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index 647e58a2..544e2fd1 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -30,18 +30,10 @@ It needs to be run manually when database has been changed. Run it as # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import print_function, division - -#from flask import config -# -#cdict = {} -#config = config.Config(cdict).from_envvar('WQFLASK_SETTINGS') -#print("cdict is:", cdict) - import sys # NEW: Note we prepend the current path - otherwise a guix instance of GN2 may be used instead -sys.path.insert(0,'./') +sys.path.insert(0, './') # NEW: import app to avoid a circular dependency on utility.tools from wqflask import app @@ -50,7 +42,7 @@ from utility.tools import locate, locate_ignore_error, TEMPDIR, SQL_URI import MySQLdb import simplejson as json -import urlparse +import urllib.parse #import sqlalchemy as sa @@ -66,7 +58,7 @@ from pprint import pformat as pf def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" - parsed_uri = urlparse.urlparse(SQL_URI) + parsed_uri = urllib.parse.urlparse(SQL_URI) db_conn_info = dict( db = parsed_uri.path[1:], @@ -108,7 +100,7 @@ def get_types(groups): """Build types list""" types = {} #print("Groups: ", pf(groups)) - for species, group_dict in groups.iteritems(): + for species, group_dict in list(groups.items()): types[species] = {} for group_name, _group_full_name in group_dict: # make group an alias to shorten the code @@ -195,9 +187,9 @@ def build_types(species, group): def get_datasets(types): """Build datasets list""" datasets = {} - for species, group_dict in types.iteritems(): + for species, group_dict in list(types.items()): datasets[species] = {} - for group, type_list in group_dict.iteritems(): + for group, type_list in list(group_dict.items()): datasets[species][group] = {} for type_name in type_list: these_datasets = build_datasets(species, group, type_name[0]) @@ -319,4 +311,4 @@ def _test_it(): if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() - main()
\ No newline at end of file + main() diff --git a/wqflask/maintenance/generate_kinship_from_bimbam.py b/wqflask/maintenance/generate_kinship_from_bimbam.py index b53f5dda..60257b28 100644 --- a/wqflask/maintenance/generate_kinship_from_bimbam.py +++ b/wqflask/maintenance/generate_kinship_from_bimbam.py @@ -8,7 +8,6 @@ and uses GEMMA to generate their corresponding kinship/relatedness matrix file """ -from __future__ import print_function, division, absolute_import import sys sys.path.append("..") import os @@ -58,4 +57,4 @@ if __name__=="__main__": Bimbam_Directory = """/export/local/home/zas1024/genotype_files/genotype/bimbam/""" GenerateKinshipMatrices.process_all(Geno_Directory, Bimbam_Directory) - #./gemma -g /home/zas1024/genotype_files/genotype/bimbam/BXD_geno.txt -p /home/zas1024/genotype_files/genotype/bimbam/BXD_pheno.txt -gk 1 -o BXD
\ No newline at end of file + #./gemma -g /home/zas1024/genotype_files/genotype/bimbam/BXD_geno.txt -p /home/zas1024/genotype_files/genotype/bimbam/BXD_pheno.txt -gk 1 -o BXD diff --git a/wqflask/maintenance/generate_probesetfreeze_file.py b/wqflask/maintenance/generate_probesetfreeze_file.py index b7b2dc8e..b1e41e9a 100644 --- a/wqflask/maintenance/generate_probesetfreeze_file.py +++ b/wqflask/maintenance/generate_probesetfreeze_file.py @@ -1,7 +1,5 @@ #!/usr/bin/python -from __future__ import absolute_import, print_function, division - import sys # sys.path.insert(0, "..") - why? @@ -82,7 +80,7 @@ def get_probeset_vals(cursor, dataset_name): def trim_strains(strains, probeset_vals): trimmed_strains = [] #print("probeset_vals is:", pf(probeset_vals)) - first_probeset = list(probeset_vals.itervalues())[0] + first_probeset = list(probeset_vals.values())[0] print("\n**** first_probeset is:", pf(first_probeset)) for strain in strains: print("\n**** strain is:", pf(strain)) diff --git a/wqflask/maintenance/geno_to_json.py b/wqflask/maintenance/geno_to_json.py index 9579812a..7e7fd241 100644 --- a/wqflask/maintenance/geno_to_json.py +++ b/wqflask/maintenance/geno_to_json.py @@ -9,7 +9,6 @@ code """ -from __future__ import print_function, division, absolute_import import sys sys.path.append("..") import os @@ -194,4 +193,4 @@ if __name__=="__main__": ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory) #ConvertGenoFiles(Geno_Directory) - #process_csv(Input_File, Output_File)
\ No newline at end of file + #process_csv(Input_File, Output_File) diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py index fb22898a..3f9d0278 100644 --- a/wqflask/maintenance/get_group_samplelists.py +++ b/wqflask/maintenance/get_group_samplelists.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import os import glob import gzip diff --git a/wqflask/maintenance/print_benchmark.py b/wqflask/maintenance/print_benchmark.py index ae327cf3..b24ce4f1 100644 --- a/wqflask/maintenance/print_benchmark.py +++ b/wqflask/maintenance/print_benchmark.py @@ -1,7 +1,5 @@ #!/usr/bin/python -from __future__ import absolute_import, print_function, division - import time from pprint import pformat as pf @@ -40,4 +38,4 @@ def new_main(): print(pf(TheCounter.Counters)) if __name__ == '__main__': - new_main()
\ No newline at end of file + new_main() diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 41a3aad8..701b2b50 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -1,12 +1,7 @@ -from __future__ import absolute_import, print_function, division - import sys -sys.path.insert(0,'./') - -from itertools import izip - +sys.path.insert(0, './') import MySQLdb -import urlparse +import urllib.parse import numpy as np import pandas as pd @@ -22,7 +17,7 @@ from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" - parsed_uri = urlparse.urlparse(SQL_URI) + parsed_uri = urllib.parse.urlparse(SQL_URI) db_conn_info = dict( db = parsed_uri.path[1:], @@ -37,7 +32,7 @@ def create_dataframe(input_file): with open(input_file) as f: ncols = len(f.readline().split("\t")) - input_array = np.loadtxt(open(input_file, "rb"), delimiter="\t", skiprows=1, usecols=range(1, ncols)) + input_array = np.loadtxt(open(input_file, "rb"), delimiter="\t", skiprows=1, usecols=list(range(1, ncols))) return pd.DataFrame(input_array) #This function taken from https://github.com/ShawnLYU/Quantile_Normalize @@ -60,7 +55,7 @@ def set_data(dataset_name): sample_list = [] with open(orig_file, 'r') as orig_fh, open('/home/zas1024/cfw_data/quant_norm.csv', 'r') as quant_fh: - for i, (line1, line2) in enumerate(izip(orig_fh, quant_fh)): + for i, (line1, line2) in enumerate(zip(orig_fh, quant_fh)): trait_dict = {} sample_list = [] if i == 0: @@ -126,4 +121,4 @@ if __name__ == '__main__': } ) - print(response)
\ No newline at end of file + print(response) diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index 54fd8e7e..4177c124 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -16,13 +16,11 @@ To run: """ -from __future__ import print_function, division - import sys import json # NEW: Note we prepend the current path - otherwise a guix instance of GN2 may be used instead -sys.path.insert(0,'./') +sys.path.insert(0, './') # NEW: import app to avoid a circular dependency on utility.tools from wqflask import app @@ -34,7 +32,7 @@ Redis = get_redis_conn() import MySQLdb -import urlparse +import urllib.parse from utility.logger import getLogger logger = getLogger(__name__) @@ -42,7 +40,7 @@ logger = getLogger(__name__) def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" - parsed_uri = urlparse.urlparse(SQL_URI) + parsed_uri = urllib.parse.urlparse(SQL_URI) db_conn_info = dict( db = parsed_uri.path[1:], @@ -163,4 +161,4 @@ def main(): if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() - main()
\ No newline at end of file + main() diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index adffdca3..58108e03 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -7,7 +7,7 @@ # from flask import Flask # application = Flask(__name__) -print "===> Starting up Gunicorn process" +print("===> Starting up Gunicorn process") from wqflask import app from utility.startup_config import app_config diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 15572d97..df957bd9 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -8,28 +8,20 @@ # /sbin/iptables -A INPUT -p tcp -i eth0 -s ! 71.236.239.43 --dport 5003 -j DROP from wqflask import app - +from utility.startup_config import app_config +from utility.tools import WEBSERVER_MODE, SERVER_PORT import logging -import utility.logger -logger = utility.logger.getLogger(__name__ ) - -import signal -signal.signal(signal.SIGPIPE, signal.SIG_DFL) -BLUE = '\033[94m' +BLUE = '\033[94m' GREEN = '\033[92m' -BOLD = '\033[1m' -ENDC = '\033[0m' - -from utility.startup_config import app_config +BOLD = '\033[1m' +ENDC = '\033[0m' app_config() werkzeug_logger = logging.getLogger('werkzeug') -from utility.tools import WEBSERVER_MODE, SERVER_PORT - if WEBSERVER_MODE == 'DEBUG': app.run(host='0.0.0.0', port=SERVER_PORT, @@ -47,7 +39,7 @@ elif WEBSERVER_MODE == 'DEV': threaded=False, processes=0, use_reloader=True) -else: # staging/production modes +else: # staging/production modes app.run(host='0.0.0.0', port=SERVER_PORT, debug=False, diff --git a/wqflask/tests/base/test_data_set.py b/wqflask/tests/base/test_data_set.py index dd7f5051..96563a16 100644 --- a/wqflask/tests/base/test_data_set.py +++ b/wqflask/tests/base/test_data_set.py @@ -1,10 +1,10 @@ """Tests for wqflask/base/data_set.py""" import unittest -import mock +from unittest import mock from wqflask import app -from data import gen_menu_json +from .data import gen_menu_json from base.data_set import DatasetType @@ -59,9 +59,14 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("BXDGeno"), "Geno") self.assertEqual(data_set("BXDPublish"), "Publish") self.assertEqual(data_set("HLC_0311"), "ProbeSet") + redis_mock.set.assert_called_once_with( "dataset_structure", - '{"BXDGeno": "Geno", "BXDPublish": "Publish", "HLCPublish": "Publish", "HLC_0311": "ProbeSet", "HC_M2_0606_P": "ProbeSet"}') + ('{"HLC_0311": "ProbeSet", ' + '"HLCPublish": "Publish", ' + '"BXDGeno": "Geno", ' + '"HC_M2_0606_P": "ProbeSet", ' + '"BXDPublish": "Publish"}')) @mock.patch('base.data_set.g') def test_set_dataset_key_mrna(self, db_mock): @@ -74,8 +79,17 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("Test"), "ProbeSet") redis_mock.set.assert_called_once_with( "dataset_structure", - '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "ProbeSet", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') - expected_db_call = """""" + ('{"AD-cases-controls-MyersGeno": "Geno", ' + '"AD-cases-controls-MyersPublish": "Publish", ' + '"AKXDGeno": "Geno", ' + '"AXBXAGeno": "Geno", ' + '"AXBXAPublish": "Publish", ' + '"Aging-Brain-UCIPublish": "Publish", ' + '"All Phenotypes": "Publish", ' + '"B139_K_1206_M": "ProbeSet", ' + '"B139_K_1206_R": "ProbeSet", ' + '"Test": "ProbeSet"}')) + db_mock.db.execute.assert_called_with( ("SELECT ProbeSetFreeze.Id FROM ProbeSetFreeze " + "WHERE ProbeSetFreeze.Name = \"Test\" ") @@ -92,12 +106,21 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("Test"), "Publish") redis_mock.set.assert_called_once_with( "dataset_structure", - '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Publish", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') + ('{"AD-cases-controls-MyersGeno": "Geno", ' + '"AD-cases-controls-MyersPublish": "Publish", ' + '"AKXDGeno": "Geno", ' + '"AXBXAGeno": "Geno", ' + '"AXBXAPublish": "Publish", ' + '"Aging-Brain-UCIPublish": "Publish", ' + '"All Phenotypes": "Publish", ' + '"B139_K_1206_M": "ProbeSet", ' + '"B139_K_1206_R": "ProbeSet", ' + '"Test": "Publish"}')) db_mock.db.execute.assert_called_with( - ("SELECT InfoFiles.GN_AccesionId " + - "FROM InfoFiles, PublishFreeze, InbredSet " + + ("SELECT InfoFiles.GN_AccesionId " + "FROM InfoFiles, PublishFreeze, InbredSet " "WHERE InbredSet.Name = 'Test' AND " - "PublishFreeze.InbredSetId = InbredSet.Id AND " + + "PublishFreeze.InbredSetId = InbredSet.Id AND " "InfoFiles.InfoPageName = PublishFreeze.Name") ) @@ -110,9 +133,20 @@ class TestDataSetTypes(unittest.TestCase): data_set = DatasetType(redis_mock) data_set.set_dataset_key("other_pheno", "Test") self.assertEqual(data_set("Test"), "Publish") + redis_mock.set.assert_called_once_with( "dataset_structure", - '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Publish", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') + ('{"AD-cases-controls-MyersGeno": "Geno", ' + '"AD-cases-controls-MyersPublish": "Publish", ' + '"AKXDGeno": "Geno", ' + '"AXBXAGeno": "Geno", ' + '"AXBXAPublish": "Publish", ' + '"Aging-Brain-UCIPublish": "Publish", ' + '"All Phenotypes": "Publish", ' + '"B139_K_1206_M": "ProbeSet", ' + '"B139_K_1206_R": "ProbeSet", ' + '"Test": "Publish"}')) + db_mock.db.execute.assert_called_with( ("SELECT PublishFreeze.Name " + "FROM PublishFreeze, InbredSet " + @@ -131,8 +165,17 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("Test"), "Geno") redis_mock.set.assert_called_once_with( "dataset_structure", - '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Geno", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') - expected_db_call = """""" + ('{"AD-cases-controls-MyersGeno": "Geno", ' + '"AD-cases-controls-MyersPublish": "Publish", ' + '"AKXDGeno": "Geno", ' + '"AXBXAGeno": "Geno", ' + '"AXBXAPublish": "Publish", ' + '"Aging-Brain-UCIPublish": "Publish", ' + '"All Phenotypes": "Publish", ' + '"B139_K_1206_M": "ProbeSet", ' + '"B139_K_1206_R": "ProbeSet", ' + '"Test": "Geno"}')) + db_mock.db.execute.assert_called_with( - ("SELECT GenoFreeze.Id FROM GenoFreeze WHERE GenoFreeze.Name = \"Test\" ") - ) + ("SELECT GenoFreeze.Id FROM " + "GenoFreeze WHERE GenoFreeze.Name = \"Test\" ")) diff --git a/wqflask/tests/base/test_general_object.py b/wqflask/tests/base/test_general_object.py index c7701021..00fd3c72 100644 --- a/wqflask/tests/base/test_general_object.py +++ b/wqflask/tests/base/test_general_object.py @@ -17,9 +17,9 @@ class TestGeneralObjectTests(unittest.TestCase): def test_object_dict(self): """Test whether the base class is printed properly""" test_obj = GeneralObject("a", name="test", value=1) - self.assertEqual(str(test_obj), "value = 1\nname = test\n") + self.assertEqual(str(test_obj), "name = test\nvalue = 1\n") self.assertEqual( - repr(test_obj), "value = 1\nname = test\ncontents = ['a']\n") + repr(test_obj), "contents = ['a']\nname = test\nvalue = 1\n") self.assertEqual(len(test_obj), 2) self.assertEqual(test_obj["value"], 1) test_obj["test"] = 1 @@ -36,6 +36,5 @@ class TestGeneralObjectTests(unittest.TestCase): test_obj1 = GeneralObject("a", name="test", value=1) test_obj2 = GeneralObject("b", name="test2", value=2) test_obj3 = GeneralObject("a", name="test", x=1, y=2) - self.assertTrue(test_obj1 == test_obj2 ) - self.assertFalse(test_obj1 == test_obj3 ) - + self.assertTrue(test_obj1 == test_obj2) + self.assertFalse(test_obj1 == test_obj3) diff --git a/wqflask/tests/base/test_trait.py b/wqflask/tests/base/test_trait.py index 1a3820f2..826ccefd 100644 --- a/wqflask/tests/base/test_trait.py +++ b/wqflask/tests/base/test_trait.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- """Tests wqflask/base/trait.py""" import unittest -import mock +from unittest import mock +from wqflask import app from base.trait import GeneralTrait from base.trait import retrieve_trait_info @@ -31,6 +32,14 @@ class MockTrait(GeneralTrait): class TestRetrieveTraitInfo(unittest.TestCase): """Tests for 'retrieve_trait_info'""" + + def setUp(self): + self.app_context = app.app_context() + self.app_context.push() + + def tearDown(self): + self.app_context.pop() + def test_retrieve_trait_info_with_empty_dataset(self): """Test that an exception is raised when dataset is empty""" with self.assertRaises(AssertionError): @@ -38,9 +47,8 @@ class TestRetrieveTraitInfo(unittest.TestCase): dataset={}) @mock.patch('base.trait.requests.get') - @mock.patch('base.trait.g') + @mock.patch('base.trait.g', mock.Mock()) def test_retrieve_trait_info_with_empty_trait_info(self, - g_mock, requests_mock): """Empty trait info""" requests_mock.return_value = TestNilResponse() @@ -49,9 +57,8 @@ class TestRetrieveTraitInfo(unittest.TestCase): dataset=mock.MagicMock()) @mock.patch('base.trait.requests.get') - @mock.patch('base.trait.g') + @mock.patch('base.trait.g', mock.Mock()) def test_retrieve_trait_info_with_non_empty_trait_info(self, - g_mock, requests_mock): """Test that attributes are set""" mock_dataset = mock.MagicMock() @@ -66,9 +73,8 @@ class TestRetrieveTraitInfo(unittest.TestCase): self.assertEqual(test_trait.d, 4) @mock.patch('base.trait.requests.get') - @mock.patch('base.trait.g') + @mock.patch('base.trait.g', mock.Mock()) def test_retrieve_trait_info_utf8_parsing(self, - g_mock, requests_mock): """Test that utf-8 strings are parsed correctly""" utf_8_string = "test_string" @@ -96,9 +102,9 @@ class TestRetrieveTraitInfo(unittest.TestCase): test_trait = retrieve_trait_info(trait=mock_trait, dataset=mock_dataset) self.assertEqual(test_trait.abbreviation, - "ファイルを画面毎に見て行くには、次のコマンドを使います。".decode('utf-8')) + "ファイルを画面毎に見て行くには、次のコマンドを使います。") self.assertEqual(test_trait.authors, - "Jane Doe かいと".decode('utf-8')) + "Jane Doe かいと") @mock.patch('base.trait.requests.get') @mock.patch('base.trait.g') @@ -107,7 +113,7 @@ class TestRetrieveTraitInfo(unittest.TestCase): resource_id_mock, g_mock, requests_mock): - """Test """ + """Test retrieve trait info when lrs has a value""" resource_id_mock.return_value = 1 g_mock.db.execute.return_value.fetchone = mock.Mock() g_mock.db.execute.return_value.fetchone.side_effect = [ diff --git a/wqflask/tests/utility/test_authentication_tools.py b/wqflask/tests/utility/test_authentication_tools.py index 99c74245..5c391be5 100644 --- a/wqflask/tests/utility/test_authentication_tools.py +++ b/wqflask/tests/utility/test_authentication_tools.py @@ -1,6 +1,6 @@ """Tests for authentication tools""" import unittest -import mock +from unittest import mock from utility.authentication_tools import check_resource_availability from utility.authentication_tools import add_new_resource @@ -38,17 +38,15 @@ class TestCheckResourceAvailability(unittest.TestCase): """Test methods related to checking the resource availability""" @mock.patch('utility.authentication_tools.add_new_resource') @mock.patch('utility.authentication_tools.Redis') - @mock.patch('utility.authentication_tools.g') + @mock.patch('utility.authentication_tools.g', mock.Mock()) @mock.patch('utility.authentication_tools.get_resource_id') def test_check_resource_availability_default_mask( self, resource_id_mock, - g_mock, redis_mock, add_new_resource_mock): """Test the resource availability with default mask""" resource_id_mock.return_value = 1 - g_mock.return_value = mock.Mock() redis_mock.smembers.return_value = [] test_dataset = mock.MagicMock() type(test_dataset).type = mock.PropertyMock(return_value="Test") @@ -58,18 +56,16 @@ class TestCheckResourceAvailability(unittest.TestCase): @mock.patch('utility.authentication_tools.requests.get') @mock.patch('utility.authentication_tools.add_new_resource') @mock.patch('utility.authentication_tools.Redis') - @mock.patch('utility.authentication_tools.g') + @mock.patch('utility.authentication_tools.g', TestUserSession()) @mock.patch('utility.authentication_tools.get_resource_id') def test_check_resource_availability_non_default_mask( self, resource_id_mock, - g_mock, redis_mock, add_new_resource_mock, requests_mock): """Test the resource availability with default mask""" resource_id_mock.return_value = 1 - g_mock.return_value = mock.Mock() redis_mock.smembers.return_value = [] add_new_resource_mock.return_value = {"default_mask": 2} requests_mock.return_value = TestResponse() diff --git a/wqflask/tests/utility/test_hmac.py b/wqflask/tests/utility/test_hmac.py index 16b50771..4e3652f8 100644 --- a/wqflask/tests/utility/test_hmac.py +++ b/wqflask/tests/utility/test_hmac.py @@ -2,7 +2,7 @@ """Test hmac utility functions""" import unittest -import mock +from unittest import mock from utility.hmac import data_hmac from utility.hmac import url_for_hmac @@ -17,6 +17,19 @@ class TestHmacUtil(unittest.TestCase): """Test hmac creation with a utf-8 string""" self.assertEqual(hmac_creation("ファイ"), "7410466338cfe109e946") + @mock.patch("utility.hmac.app.config", + {'SECRET_HMAC_CODE': ('\x08\xdf\xfa\x93N\x80' + '\xd9\\H@\\\x9f`\x98d^' + '\xb4a;\xc6OM\x946a\xbc' + '\xfc\x80:*\xebc')}) + def test_hmac_creation_with_cookie(self): + """Test hmac creation with a cookie""" + cookie = "3f4c1dbf-5b56-4260-87d6-f35445bda37e:af4fcf5eace9e7c864ce" + uuid_, _, signature = cookie.partition(":") + self.assertEqual( + hmac_creation(uuid_), + "af4fcf5eace9e7c864ce") + @mock.patch("utility.hmac.app.config", {'SECRET_HMAC_CODE': "secret"}) def test_data_hmac(self): """Test data_hmac fn with a utf-8 string""" diff --git a/wqflask/tests/wqflask/api/test_gen_menu.py b/wqflask/tests/wqflask/api/test_gen_menu.py index 239484aa..84898bd1 100644 --- a/wqflask/tests/wqflask/api/test_gen_menu.py +++ b/wqflask/tests/wqflask/api/test_gen_menu.py @@ -1,7 +1,8 @@ """Test cases for wqflask.api.gen_menu""" import unittest -import mock +from unittest import mock +from wqflask import app from wqflask.api.gen_menu import gen_dropdown_json from wqflask.api.gen_menu import get_species from wqflask.api.gen_menu import get_groups @@ -17,6 +18,8 @@ class TestGenMenu(unittest.TestCase): """Tests for the gen_menu module""" def setUp(self): + self.app_context = app.app_context() + self.app_context.push() self.test_group = { 'mouse': [ ['H_T1', @@ -67,6 +70,9 @@ class TestGenMenu(unittest.TestCase): } } + def tearDown(self): + self.app_context.pop() + @mock.patch('wqflask.api.gen_menu.g') def test_get_species(self, db_mock): """Test that assertion is raised when dataset and dataset_name diff --git a/wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py b/wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py index 6f791df1..8ae0f09f 100644 --- a/wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py +++ b/wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py @@ -1,6 +1,6 @@ import unittest -from htmlgen import HTMLgen2 as HT +import htmlgen as HT from wqflask.marker_regression.display_mapping_results import ( DisplayMappingResults, HtmlGenWrapper @@ -26,9 +26,9 @@ class TestHtmlGenWrapper(unittest.TestCase): width="10", height="13", usemap="#webqtlmap")), - ("""<IMG src="test.png" height="13" width="10" """ - """alt="random" border="0" """ - """usemap="#webqtlmap">""") + ("""<img alt="random" border="0" height="13" """ + """src="test.png" usemap="#webqtlmap" """ + """width="10"/>""") ) def test_create_form(self): @@ -37,7 +37,7 @@ class TestHtmlGenWrapper(unittest.TestCase): cgi="/testing/", enctype='multipart/form-data', name="formName", - submit=HT.Input(type='hidden') + submit=HtmlGenWrapper.create_input_tag(type_='hidden', name='Default_Name') ) test_image = HtmlGenWrapper.create_image_tag( src="test.png", @@ -49,10 +49,10 @@ class TestHtmlGenWrapper(unittest.TestCase): ) self.assertEqual( str(test_form).replace("\n", ""), - ("""<FORM METHOD="POST" ACTION="/testing/" """ - """ENCTYPE="multipart/form-data" """ - """NAME="formName"><INPUT TYPE="hidden" """ - """NAME="Default_Name"></FORM>""")) + ("""<form action="/testing/" enctype="multipart/form-data" """ + """method="POST" """ + """name="formName"><input name="Default_Name" """ + """type="hidden"/></form>""")) hddn = { 'FormID': 'showDatabase', 'ProbeSetID': '_', @@ -62,21 +62,26 @@ class TestHtmlGenWrapper(unittest.TestCase): 'incparentsf1': 'ON' } for key in hddn.keys(): - test_form.append(HT.Input(name=key, value=hddn[key], - type='hidden')) + test_form.append( + HtmlGenWrapper.create_input_tag( + name=key, + value=hddn[key], + type_='hidden')) test_form.append(test_image) + self.assertEqual(str(test_form).replace("\n", ""), ( - """<FORM METHOD="POST" ACTION="/testing/" """ - """ENCTYPE="multipart/form-data" NAME="formName">""" - """<INPUT TYPE="hidden" NAME="database" VALUE="TestGeno">""" - """<INPUT TYPE="hidden" NAME="incparentsf1" VALUE="ON">""" - """<INPUT TYPE="hidden" NAME="FormID" VALUE="showDatabase">""" - """<INPUT TYPE="hidden" NAME="ProbeSetID" VALUE="_">""" - """<INPUT TYPE="hidden" NAME="RISet" VALUE="Test">""" - """<INPUT TYPE="hidden" NAME="CellID" VALUE="_">""" - """<IMG src="test.png" height="13" width="10" alt="random" """ - """border="0" usemap="#webqtlmap">""" - """<INPUT TYPE="hidden" NAME="Default_Name"></FORM>""")) + """<form action="/testing/" enctype="multipart/form-data" """ + """method="POST" name="formName">""" + """<input name="Default_Name" type="hidden"/>""" + """<input name="FormID" type="hidden" value="showDatabase"/>""" + """<input name="ProbeSetID" type="hidden" value="_"/>""" + """<input name="database" type="hidden" value="TestGeno"/>""" + """<input name="CellID" type="hidden" value="_"/>""" + """<input name="RISet" type="hidden" value="Test"/>""" + """<input name="incparentsf1" type="hidden" value="ON"/>""" + """<img alt="random" border="0" height="13" src="test.png" """ + """usemap="#webqtlmap" width="10"/>""" + """</form>""")) def test_create_paragraph(self): """Test HT.Paragraph method""" @@ -89,48 +94,48 @@ class TestHtmlGenWrapper(unittest.TestCase): ) self.assertEqual( str(test_p_element), - """<P id="smallSize"></P>""" + """<p id="smallSize"></p>""" ) - test_p_element.append(HT.BR()) + test_p_element.append(HtmlGenWrapper.create_br_tag()) test_p_element.append(par_text) self.assertEqual( str(test_p_element), - """<P id="smallSize"><BR>{}</P>""".format(par_text) + """<p id="smallSize"><br/>{}</p>""".format(par_text) ) def test_create_br_tag(self): """Test HT.BR() method""" self.assertEqual(str(HtmlGenWrapper.create_br_tag()), - "<BR>") + "<br/>") def test_create_input_tag(self): """Test HT.Input method""" self.assertEqual( str(HtmlGenWrapper.create_input_tag( - type="hidden", + type_="hidden", name="name", value="key", Class="trait trait_")).replace("\n", ""), - ("""<INPUT TYPE="hidden" NAME="name" """ - """class="trait trait_" VALUE="key">""")) + ("""<input class="trait trait_" name="name" """ + """type="hidden" value="key"/>""")) def test_create_map_tag(self): """Test HT.Map method""" self.assertEqual(str(HtmlGenWrapper.create_map_tag( name="WebqTLImageMap")).replace("\n", ""), - """<MAP NAME="WebqTLImageMap"></MAP>""") - gifmap = HtmlGenWrapper.create_map_tag(areas=[]) - gifmap.areas.append(HT.Area(shape="rect", - coords='1 2 3', href='#area1')) - gifmap.areas.append(HT.Area(shape="rect", - coords='1 2 3', href='#area2')) + """<map name="WebqTLImageMap"></map>""") + gifmap = HtmlGenWrapper.create_map_tag(name="test") + gifmap.append(HtmlGenWrapper.create_area_tag(shape="rect", + coords='1 2 3', href='#area1')) + gifmap.append(HtmlGenWrapper.create_area_tag(shape="rect", + coords='1 2 3', href='#area2')) self.assertEqual( str(gifmap).replace("\n", ""), - ("""<MAP NAME="">""" - """<AREA coords="1 2 3" """ - """href="#area1" shape="rect">""" - """<AREA coords="1 2 3" href="#area2" shape="rect">""" - """</MAP>""")) + ("""<map name="test">""" + """<area coords="1 2 3" """ + """href="#area1" shape="rect"/>""" + """<area coords="1 2 3" href="#area2" shape="rect"/>""" + """</map>""")) def test_create_area_tag(self): """Test HT.Area method""" @@ -140,12 +145,12 @@ class TestHtmlGenWrapper(unittest.TestCase): coords="1 2", href="http://test.com", title="Some Title")).replace("\n", ""), - ("""<AREA coords="1 2" href="http://test.com" """ - """shape="rect" title="Some Title">""")) + ("""<area coords="1 2" href="http://test.com" """ + """shape="rect" title="Some Title"/>""")) def test_create_link_tag(self): """Test HT.HREF method""" self.assertEqual( str(HtmlGenWrapper.create_link_tag( "www.test.com", "test", target="_blank")).replace("\n", ""), - """<A HREF="www.test.com" TARGET="_blank">test</A>""") + """<a href="www.test.com" target="_blank">test</a>""") diff --git a/wqflask/tests/wqflask/show_trait/__init__.py b/wqflask/tests/wqflask/show_trait/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wqflask/tests/wqflask/show_trait/__init__.py diff --git a/wqflask/tests/wqflask/show_trait/test_export_trait_data.py b/wqflask/tests/wqflask/show_trait/test_export_trait_data.py new file mode 100644 index 00000000..98d599b1 --- /dev/null +++ b/wqflask/tests/wqflask/show_trait/test_export_trait_data.py @@ -0,0 +1,94 @@ +import unittest +from wqflask.show_trait.export_trait_data import dict_to_sorted_list +from wqflask.show_trait.export_trait_data import cmp_samples + + +class TestExportTraits(unittest.TestCase): + """Test methods related to converting dict to sortedlist""" + + def test_dict_to_sortedlist(self): + """test for conversion of dict to sorted list""" + sample1 = { + "other": "exp1", + "name": "exp2" + } + sample2 = { + "se": 1, + "num_cases": 4, + "value": 6, + "name": 3 + + } + rever = { + "name": 3, + "value": 6, + "num_cases": 4, + "se": 1 + } + oneItem = { + "item1": "one" + } + + self.assertEqual(["exp2", "exp1"], dict_to_sorted_list(sample1)) + self.assertEqual([3, 6, 1, 4], dict_to_sorted_list(sample2)) + self.assertEqual([3, 6, 1, 4], dict_to_sorted_list(rever)) + self.assertEqual(["one"], dict_to_sorted_list(oneItem)) + """test that the func returns the values not the keys""" + self.assertFalse(["other", "name"] == dict_to_sorted_list(sample1)) + + def test_cmp_samples(self): + """test for comparing samples function""" + sampleA = [ + [ + ("value", "other"), + ("name", "test_name") + ] + ] + sampleB = [ + [ + ("value", "other"), + ("unknown", "test_name") + ] + ] + sampleC = [ + [("other", "value"), + ("name", "value") + ], + [ + ("name", "value"), + ("value", "name") + ], + [ + ("other", "value"), + ("name", "value" + )], + [ + ("name", "name1"), + ("se", "valuex") + ], + [( + "value", "name1"), + ("se", "valuex") + ], + [( + "other", "name1"), + ("se", "valuex" + ) + ], + [( + "name", "name_val"), + ("num_cases", "num_val") + ], + [( + "other_a", "val_a"), + ("other_b", "val" + ) + ] + ] + results = [cmp_samples(val[0], val[1]) for val in sampleA] + resultB = [cmp_samples(val[0], val[1]) for val in sampleB] + resultC = [cmp_samples(val[0], val[1]) for val in sampleC] + + self.assertEqual(1, *results) + self.assertEqual(-1, *resultB) + self.assertEqual([1, -1, 1, -1, -1, 1, -1, -1], resultC) diff --git a/wqflask/tests/wqflask/test_collect.py b/wqflask/tests/wqflask/test_collect.py index 6b8d7931..9a36132d 100644 --- a/wqflask/tests/wqflask/test_collect.py +++ b/wqflask/tests/wqflask/test_collect.py @@ -1,7 +1,7 @@ """Test cases for some methods in collect.py""" import unittest -import mock +from unittest import mock from flask import Flask from wqflask.collect import process_traits @@ -41,10 +41,10 @@ class TestCollect(unittest.TestCase): self.app_context.pop() @mock.patch("wqflask.collect.g", MockFlaskG()) - def test_process_traits_when_user_is_logged_out(self): + def test_process_traits_with_bytestring(self): """ Test that the correct traits are returned when the user is logged - out + out and bytes are used. """ self.assertEqual(process_traits( b'1452452_at:HC_M2_0606_P:163d04f7db7c9e110de6,' @@ -55,3 +55,19 @@ class TestCollect(unittest.TestCase): '1451401_a_at:HC_M2_0606_P', '1452447_at:HC_M2_0606_P', '1452452_at:HC_M2_0606_P'])) + + @mock.patch("wqflask.collect.g", MockFlaskG()) + def test_process_traits_with_normal_string(self): + """ + Test that the correct traits are returned when the user is logged + out and a normal string is used. + """ + self.assertEqual(process_traits( + '1452452_at:HC_M2_0606_P:163d04f7db7c9e110de6,' + '1452447_at:HC_M2_0606_P:eeece8fceb67072debea,' + '1451401_a_at:HC_M2_0606_P:a043d23b3b3906d8318e,' + '1429252_at:HC_M2_0606_P:6fa378b349bc9180e8f5'), + set(['1429252_at:HC_M2_0606_P', + '1451401_a_at:HC_M2_0606_P', + '1452447_at:HC_M2_0606_P', + '1452452_at:HC_M2_0606_P'])) diff --git a/wqflask/tests/wqflask/test_pbkdf2.py b/wqflask/tests/wqflask/test_pbkdf2.py new file mode 100644 index 00000000..a33fbd4f --- /dev/null +++ b/wqflask/tests/wqflask/test_pbkdf2.py @@ -0,0 +1,61 @@ +"""Test cases pbkdf2""" + +import unittest +from wqflask.pbkdf2 import pbkdf2_hex + + +class TestPbkdf2(unittest.TestCase): + def test_pbkdf2_hex(self): + """ + Test pbkdf2_hex function + """ + + for password, salt, iterations, keylen, expected_value in [ + ('password', 'salt', 1, 20, + '0c60c80f961f0e71f3a9b524af6012062fe037a6'), + ('password', 'salt', 2, 20, + 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'), + ('password', 'salt', 4096, 20, + '4b007901b765489abead49d926f721d065a429c1'), + ('passwordPASSWORDpassword', + 'saltSALTsaltSALTsaltSALTsaltSALTsalt', + 4096, 25, + '3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038'), + ('pass\x00word', 'sa\x00lt', 4096, 16, + '56fa6aa75548099dcc37d7f03425e0c3'), + ('password', 'ATHENA.MIT.EDUraeburn', 1, 16, + 'cdedb5281bb2f801565a1122b2563515'), + ('password', 'ATHENA.MIT.EDUraeburn', 1, 32, + ('cdedb5281bb2f80' + '1565a1122b256351' + '50ad1f7a04bb9f3a33' + '3ecc0e2e1f70837')), + ('password', 'ATHENA.MIT.EDUraeburn', 2, 16, + '01dbee7f4a9e243e988b62c73cda935d'), + ('password', 'ATHENA.MIT.EDUraeburn', 2, 32, + ('01dbee7f4a9e243e9' + '88b62c73cda935da05' + '378b93244ec8f48a99' + 'e61ad799d86')), + ('password', 'ATHENA.MIT.EDUraeburn', 1200, 32, + ('5c08eb61fdf71e' + '4e4ec3cf6ba1f55' + '12ba7e52ddbc5e51' + '42f708a31e2e62b1e13')), + ('X' * 64, 'pass phrase equals block size', 1200, 32, + ('139c30c0966bc32ba' + '55fdbf212530ac9c5' + 'ec59f1a452f5cc9ad' + '940fea0598ed1')), + ('X' * 65, 'pass phrase exceeds block size', 1200, 32, + ('9ccad6d468770cd' + '51b10e6a68721be6' + '11a8b4d282601db3' + 'b36be9246915ec82a')) + ]: + self.assertEqual( + pbkdf2_hex(data=password, + salt=salt, + iterations=iterations, + keylen=keylen), + expected_value) diff --git a/wqflask/tests/wqflask/test_user_login.py b/wqflask/tests/wqflask/test_user_login.py new file mode 100644 index 00000000..61cd9ab9 --- /dev/null +++ b/wqflask/tests/wqflask/test_user_login.py @@ -0,0 +1,21 @@ +"""Test cases for some methods in login.py""" + +import unittest +from wqflask.user_login import encode_password + + +class TestUserLogin(unittest.TestCase): + def test_encode_password(self): + """ + Test encode password + """ + pass_gen_fields = { + "salt": "salt", + "hashfunc": "sha1", + "iterations": 4096, + "keylength": 20, + } + self.assertEqual( + encode_password(pass_gen_fields, + "password").get("password"), + '4b007901b765489abead49d926f721d065a429c1') diff --git a/wqflask/tests/wqflask/test_user_session.py b/wqflask/tests/wqflask/test_user_session.py new file mode 100644 index 00000000..ebb0334a --- /dev/null +++ b/wqflask/tests/wqflask/test_user_session.py @@ -0,0 +1,15 @@ +"""Test cases for some methods in user_session.py""" + +import unittest +from wqflask.user_session import verify_cookie + + +class TestUserSession(unittest.TestCase): + def test_verify_cookie(self): + """ + Test cookie verification + """ + self.assertEqual( + "3f4c1dbf-5b56-4260-87d6-f35445bda37e", + verify_cookie(("3f4c1dbf-5b56-4260-87d6-" + "f35445bda37e:af4fcf5eace9e7c864ce"))) diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index b9b71129..61f408d2 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -24,15 +24,13 @@ # # Last updated by GeneNetwork Core Team 2010/10/20 -from __future__ import print_function - from PIL import ImageColor from PIL import ImageDraw from PIL import ImageFont from math import * -import corestats +import utility.corestats as corestats from base import webqtlConfig from utility.pillow_utils import draw_rotated_text import utility.logger @@ -54,7 +52,7 @@ def cformat(d, rank=0): strD = "%2.6f" % d if rank == 0: - while strD[-1] in ('0','.'): + while strD[-1] in ('0', '.'): if strD[-1] == '0' and strD[-2] == '.' and len(strD) <= 4: break elif strD[-1] == '.': @@ -82,7 +80,7 @@ def frange(start, end=None, inc=1.0): # Need to adjust the count. AFAICT, it always comes up one short. count += 1 L = [start] * count - for i in xrange(1, count): + for i in range(1, count): L[i] = start + i * inc return L @@ -93,7 +91,7 @@ def find_outliers(vals): >>> find_outliers([3.504, 5.234, 6.123, 7.234, 3.542, 5.341, 7.852, 4.555, 12.537]) (11.252500000000001, 0.5364999999999993) - >>> >>> find_outliers([9,12,15,17,31,50,7,5,6,8]) + >>> find_outliers([9,12,15,17,31,50,7,5,6,8]) (32.0, -8.0) If there are no vals, returns None for the upper and lower bounds, @@ -158,7 +156,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab j = int((item-xLow)/step) Count[j] += 1 - yLow, yTop, stepY=detScale(0,max(Count)) + yLow, yTop, stepY=detScale(0, max(Count)) #draw data xScale = plotWidth/(xTop-xLow) @@ -170,7 +168,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab xc = (dataXY[i]-xLow)*xScale+xLeftOffset yc =-(count-yLow)*yScale+yTopOffset+plotHeight im_drawer.rectangle( - xy=((xc+2,yc),(xc+barWidth-2,yTopOffset+plotHeight)), + xy=((xc+2, yc), (xc+barWidth-2, yTopOffset+plotHeight)), outline=barColor, fill=barColor) #draw drawing region @@ -179,81 +177,81 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab ) #draw scale - scaleFont=ImageFont.truetype(font=COUR_FILE,size=11) + scaleFont=ImageFont.truetype(font=COUR_FILE, size=11) x=xLow for i in range(int(stepX)+1): xc=xLeftOffset+(x-xLow)*xScale im_drawer.line( - xy=((xc,yTopOffset+plotHeight),(xc,yTopOffset+plotHeight+5)), + xy=((xc, yTopOffset+plotHeight), (xc, yTopOffset+plotHeight+5)), fill=axesColor) strX = cformat(d=x, rank=0) im_drawer.text( text=strX, - xy=(xc-im_drawer.textsize(strX,font=scaleFont)[0]/2, - yTopOffset+plotHeight+14),font=scaleFont) + xy=(xc-im_drawer.textsize(strX, font=scaleFont)[0]/2, + yTopOffset+plotHeight+14), font=scaleFont) x+= (xTop - xLow)/stepX y=yLow for i in range(int(stepY)+1): yc=yTopOffset+plotHeight-(y-yLow)*yScale - im_drawer.line(xy=((xLeftOffset,yc),(xLeftOffset-5,yc)), fill=axesColor) + im_drawer.line(xy=((xLeftOffset, yc), (xLeftOffset-5, yc)), fill=axesColor) strY = "%d" %y im_drawer.text( text=strY, - xy=(xLeftOffset-im_drawer.textsize(strY,font=scaleFont)[0]-6,yc+5), + xy=(xLeftOffset-im_drawer.textsize(strY, font=scaleFont)[0]-6, yc+5), font=scaleFont) y+= (yTop - yLow)/stepY #draw label - labelFont=ImageFont.truetype(font=TAHOMA_FILE,size=17) + labelFont=ImageFont.truetype(font=TAHOMA_FILE, size=17) if XLabel: im_drawer.text( text=XLabel, xy=(xLeftOffset+( - plotWidth-im_drawer.textsize(XLabel,font=labelFont)[0])/2.0, + plotWidth-im_drawer.textsize(XLabel, font=labelFont)[0])/2.0, yTopOffset+plotHeight+yBottomOffset-10), - font=labelFont,fill=labelColor) + font=labelFont, fill=labelColor) if YLabel: draw_rotated_text(canvas, text=YLabel, xy=(19, yTopOffset+plotHeight-( plotHeight-im_drawer.textsize( - YLabel,font=labelFont)[0])/2.0), + YLabel, font=labelFont)[0])/2.0), font=labelFont, fill=labelColor, angle=90) - labelFont=ImageFont.truetype(font=VERDANA_FILE,size=16) + labelFont=ImageFont.truetype(font=VERDANA_FILE, size=16) if title: im_drawer.text( text=title, xy=(xLeftOffset+(plotWidth-im_drawer.textsize( - title,font=labelFont)[0])/2.0, + title, font=labelFont)[0])/2.0, 20), - font=labelFont,fill=labelColor) + font=labelFont, fill=labelColor) # This function determines the scale of the plot -def detScaleOld(min,max): +def detScaleOld(min, max): if min>=max: return None elif min == -1.0 and max == 1.0: - return [-1.2,1.2,12] + return [-1.2, 1.2, 12] else: a=max-min b=floor(log10(a)) - c=pow(10.0,b) + c=pow(10.0, b) if a < c*5.0: c/=2.0 #print a,b,c low=c*floor(min/c) high=c*ceil(max/c) - return [low,high,round((high-low)/c)] + return [low, high, round((high-low)/c)] def detScale(min=0,max=0): if min>=max: return None elif min == -1.0 and max == 1.0: - return [-1.2,1.2,12] + return [-1.2, 1.2, 12] else: a=max-min if max != 0: @@ -265,7 +263,7 @@ def detScale(min=0,max=0): min -= 0.1*a a=max-min b=floor(log10(a)) - c=pow(10.0,b) + c=pow(10.0, b) low=c*floor(min/c) high=c*ceil(max/c) n = round((high-low)/c) @@ -283,7 +281,7 @@ def detScale(min=0,max=0): high=c*ceil(max/c) n = round((high-low)/c) - return [low,high,n] + return [low, high, n] def bluefunc(x): return 1.0 / (1.0 + exp(-10*(x-0.6))) @@ -292,7 +290,7 @@ def redfunc(x): return 1.0 / (1.0 + exp(10*(x-0.5))) def greenfunc(x): - return 1 - pow(redfunc(x+0.2),2) - bluefunc(x-0.3) + return 1 - pow(redfunc(x+0.2), 2) - bluefunc(x-0.3) def colorSpectrum(n=100): multiple = 10 diff --git a/wqflask/utility/__init__.py b/wqflask/utility/__init__.py index d9856eed..204ff59a 100644 --- a/wqflask/utility/__init__.py +++ b/wqflask/utility/__init__.py @@ -19,7 +19,7 @@ class Struct(object): ''' def __init__(self, obj): - for k, v in obj.iteritems(): + for k, v in list(obj.items()): if isinstance(v, dict): setattr(self, k, Struct(v)) else: @@ -30,6 +30,6 @@ class Struct(object): def __repr__(self): return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for - (k, v) in self.__dict__.iteritems())) + (k, v) in list(self.__dict__.items()))) diff --git a/wqflask/utility/after.py b/wqflask/utility/after.py index b628a0a4..06091ecb 100644 --- a/wqflask/utility/after.py +++ b/wqflask/utility/after.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - """ See: http://flask.pocoo.org/docs/patterns/deferredcallbacks/#deferred-callbacks @@ -13,4 +11,4 @@ def after_this_request(f): if not hasattr(g, 'after_request_callbacks'): g.after_request_callbacks = [] g.after_request_callbacks.append(f) - return f
\ No newline at end of file + return f diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 239b08e3..ce0c0749 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -1,34 +1,37 @@ -from __future__ import absolute_import, print_function, division -import logging -from flask import Flask, g, redirect, url_for - import json import requests -from base import data_set, webqtlConfig - -from utility import hmac -from utility.redis_tools import get_redis_conn, get_resource_info, get_resource_id, add_resource -Redis = get_redis_conn() +from flask import g +from base import webqtlConfig -logger = logging.getLogger(__name__) +from utility.redis_tools import (get_redis_conn, + get_resource_info, + get_resource_id, + add_resource) +Redis = get_redis_conn() def check_resource_availability(dataset, trait_id=None): - # At least for now assume temporary entered traits are accessible if type(dataset) == str or dataset.type == "Temp": return webqtlConfig.DEFAULT_PRIVILEGES resource_id = get_resource_id(dataset, trait_id) - if resource_id: # ZS: This should never be false, but it's technically possible if a non-Temp dataset somehow had a type other than Publish/ProbeSet/Geno + # ZS: This should never be false, but it's technically possible if + # a non-Temp dataset somehow had a type other than + # Publish/ProbeSet/Geno + if resource_id: resource_info = get_resource_info(resource_id) - if not resource_info: # ZS: If resource isn't already in redis, add it with default privileges + + # ZS: If resource isn't already in redis, add it with default + # privileges + if not resource_info: resource_info = add_new_resource(dataset, trait_id) - # ZS: Check if super-user - we should probably come up with some way to integrate this into the proxy + # ZS: Check if super-user - we should probably come up with some + # way to integrate this into the proxy if g.user_session.user_id in Redis.smembers("super_users"): return webqtlConfig.SUPER_PRIVILEGES @@ -52,7 +55,10 @@ def add_new_resource(dataset, trait_id=None): } if dataset.type == "Publish": - resource_ob['name'] = get_group_code(dataset) + "_" + str(trait_id) + group_code = get_group_code(dataset) + if group_code is None: + group_code = "" + resource_ob['name'] = group_code + "_" + str(trait_id) resource_ob['data'] = { 'dataset': dataset.id, 'trait': trait_id @@ -77,8 +83,9 @@ def add_new_resource(dataset, trait_id=None): def get_group_code(dataset): - results = g.db.execute("SELECT InbredSetCode from InbredSet where Name='{}'".format( - dataset.group.name)).fetchone() + results = g.db.execute( + "SELECT InbredSetCode from InbredSet where Name='{}'".format( + dataset.group.name)).fetchone() if results[0]: return results[0] else: diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py index 8f1c916b..ea5a0ab6 100644 --- a/wqflask/utility/benchmark.py +++ b/wqflask/utility/benchmark.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import collections import inspect import time @@ -38,9 +36,9 @@ class Bench(object): @classmethod def report(cls): - total_time = sum((time_taken for time_taken in cls.entries.itervalues())) + total_time = sum((time_taken for time_taken in list(cls.entries.values()))) print("\nTiming report\n") - for name, time_taken in cls.entries.iteritems(): + for name, time_taken in list(cls.entries.items()): percent = int(round((time_taken/total_time) * 100)) print("[{}%] {}: {}".format(percent, name, time_taken)) print() diff --git a/wqflask/utility/chunks.py b/wqflask/utility/chunks.py index d91b9bf4..9a7db102 100644 --- a/wqflask/utility/chunks.py +++ b/wqflask/utility/chunks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import math import time diff --git a/wqflask/utility/db_tools.py b/wqflask/utility/db_tools.py index 4034f39c..6e19778f 100644 --- a/wqflask/utility/db_tools.py +++ b/wqflask/utility/db_tools.py @@ -1,6 +1,5 @@ -from __future__ import absolute_import, print_function, division +from MySQLdb import escape_string as escape_ -from MySQLdb import escape_string as escape def create_in_clause(items): """Create an in clause for mysql""" @@ -8,8 +7,11 @@ def create_in_clause(items): in_clause = '( {} )'.format(in_clause) return in_clause + def mescape(*items): """Multiple escape""" - escaped = [escape(str(item)) for item in items] - #print("escaped is:", escaped) - return escaped + return [escape_(str(item)).decode('utf8') for item in items] + + +def escape(string_): + return escape_(string_).decode('utf8') diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 15cdd0bc..a5580811 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -59,7 +59,7 @@ def get_elasticsearch_connection(for_user=True): try: assert(ELASTICSEARCH_HOST) assert(ELASTICSEARCH_PORT) - logger.info("ES HOST",ELASTICSEARCH_HOST) + logger.info("ES HOST", ELASTICSEARCH_HOST) es = Elasticsearch([{ "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index 23b0b650..81085ffe 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import utility.logger logger = utility.logger.getLogger(__name__ ) @@ -175,7 +173,7 @@ class Locus(object): start_pos = 3 for allele in marker_row[start_pos:]: - if allele in geno_table.keys(): + if allele in list(geno_table.keys()): self.genotype.append(geno_table[allele]) else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown - self.genotype.append("U")
\ No newline at end of file + self.genotype.append("U") diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py index af306731..0b736176 100644 --- a/wqflask/utility/genofile_parser.py +++ b/wqflask/utility/genofile_parser.py @@ -1,7 +1,6 @@ # CTL analysis for GN2 # Author / Maintainer: Danny Arends <Danny.Arends@gmail.com> -from __future__ import print_function, division, absolute_import import sys import os import glob diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 9ce809b6..7eb7f013 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from base import data_set from base.trait import create_trait from base.species import TheSpecies @@ -13,7 +11,7 @@ logger = logging.getLogger(__name__ ) def get_species_dataset_trait(self, start_vars): #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype" - if "temp_trait" in start_vars.keys(): + if "temp_trait" in list(start_vars.keys()): if start_vars['temp_trait'] == "True": self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) else: @@ -34,7 +32,7 @@ def get_species_dataset_trait(self, start_vars): #self.genotype = self.dataset.group.genotype def get_trait_db_obs(self, trait_db_list): - if isinstance(trait_db_list, basestring): + if isinstance(trait_db_list, str): trait_db_list = trait_db_list.split(",") self.trait_list = [] diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py index fd75803e..29891677 100644 --- a/wqflask/utility/hmac.py +++ b/wqflask/utility/hmac.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import hmac import hashlib @@ -12,7 +10,9 @@ def hmac_creation(stringy): """Helper function to create the actual hmac""" secret = app.config['SECRET_HMAC_CODE'] - hmaced = hmac.new(secret, stringy, hashlib.sha1) + hmaced = hmac.new(bytearray(secret, "latin-1"), + bytearray(stringy, "utf-8"), + hashlib.sha1) hm = hmaced.hexdigest() # ZS: Leaving the below comment here to ask Pjotr about # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py index 510b1041..e904eb94 100644 --- a/wqflask/utility/logger.py +++ b/wqflask/utility/logger.py @@ -42,10 +42,10 @@ class GNLogger: """ - def __init__(self,name): + def __init__(self, name): self.logger = logging.getLogger(name) - def setLevel(self,value): + def setLevel(self, value): """Set the undelying log level""" self.logger.setLevel(value) @@ -54,7 +54,7 @@ class GNLogger: level=num to filter on LOG_LEVEL_DEBUG. """ - self.collect(self.logger.debug,*args) + self.collect(self.logger.debug, *args) def debug20(self,*args): """Call logging.debug for multiple args. Use level=num to filter on @@ -63,15 +63,15 @@ LOG_LEVEL_DEBUG (NYI). """ if level <= LOG_LEVEL_DEBUG: if self.logger.getEffectiveLevel() < 20: - self.collect(self.logger.debug,*args) + self.collect(self.logger.debug, *args) def info(self,*args): """Call logging.info for multiple args""" - self.collect(self.logger.info,*args) + self.collect(self.logger.info, *args) def warning(self,*args): """Call logging.warning for multiple args""" - self.collect(self.logger.warning,*args) + self.collect(self.logger.warning, *args) # self.logger.warning(self.collect(*args)) def error(self,*args): @@ -79,13 +79,13 @@ LOG_LEVEL_DEBUG (NYI). now = datetime.datetime.utcnow() time_str = now.strftime('%H:%M:%S UTC %Y%m%d') l = [time_str]+list(args) - self.collect(self.logger.error,*l) + self.collect(self.logger.error, *l) def infof(self,*args): """Call logging.info for multiple args lazily""" # only evaluate function when logging if self.logger.getEffectiveLevel() < 30: - self.collectf(self.logger.debug,*args) + self.collectf(self.logger.debug, *args) def debugf(self,level=0,*args): """Call logging.debug for multiple args lazily and handle @@ -95,15 +95,15 @@ LOG_LEVEL_DEBUG (NYI). # only evaluate function when logging if level <= LOG_LEVEL_DEBUG: if self.logger.getEffectiveLevel() < 20: - self.collectf(self.logger.debug,*args) + self.collectf(self.logger.debug, *args) def sql(self, sqlcommand, fun = None): """Log SQL command, optionally invoking a timed fun""" if LOG_SQL: caller = stack()[1][3] - if caller in ['fetchone','fetch1','fetchall']: + if caller in ['fetchone', 'fetch1', 'fetchall']: caller = stack()[2][3] - self.info(caller,sqlcommand) + self.info(caller, sqlcommand) if fun: result = fun(sqlcommand) if LOG_SQL: @@ -119,7 +119,7 @@ LOG_LEVEL_DEBUG (NYI). if isinstance(a, str): out = out + a else: - out = out + pf(a,width=160) + out = out + pf(a, width=160) fun(out) def collectf(self,fun,*args): @@ -134,7 +134,7 @@ LOG_LEVEL_DEBUG (NYI). if isinstance(a, str): out = out + a else: - out = out + pf(a,width=160) + out = out + pf(a, width=160) fun(out) # Get the module logger. You can override log levels at the diff --git a/wqflask/utility/pillow_utils.py b/wqflask/utility/pillow_utils.py index 0c2ce7af..c486abba 100644 --- a/wqflask/utility/pillow_utils.py +++ b/wqflask/utility/pillow_utils.py @@ -12,9 +12,9 @@ WHITE = ImageColor.getrgb("white") def draw_rotated_text(canvas, text, font, xy, fill=BLACK, angle=-90): # type: (Image, str, ImageFont, tuple, ImageColor, int) """Utility function draw rotated text""" - tmp_img = Image.new("RGBA", font.getsize(text), color=(0,0,0,0)) + tmp_img = Image.new("RGBA", font.getsize(text), color=(0, 0, 0, 0)) draw_text = ImageDraw.Draw(tmp_img) - draw_text.text(text=text, xy=(0,0), font=font, fill=fill) + draw_text.text(text=text, xy=(0, 0), font=font, fill=fill) tmp_img2 = tmp_img.rotate(angle, expand=1) tmp_img2.save("/{0}/{1}.png".format(TEMPDIR, text), format="png") canvas.paste(im=tmp_img2, box=tuple([int(i) for i in xy])) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index ef02268e..d855a7fa 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import uuid import simplejson as json import datetime diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 817284dd..f1aaebb6 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -27,7 +27,7 @@ def app_config(): port = get_setting_int("SERVER_PORT") if get_setting_bool("USE_GN_SERVER"): - print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]") + print(("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]")) import requests page = requests.get(get_setting("GN_SERVER_URL")) if page.status_code != 200: @@ -36,4 +36,4 @@ def app_config(): # import utility.elasticsearch_tools as es # es.test_elasticsearch_connection() - print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) + print(("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE, str(port), ENDC, get_setting("WEBSERVER_URL")))) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index db13b9d1..b92cc2d1 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -25,54 +25,56 @@ # Last updated by GeneNetwork Core Team 2010/10/20 #!/usr/bin/env python -##Copyright (c) 2002, Fedor Baart & Hans de Wit (Stichting Farmaceutische Kengetallen) -##All rights reserved. +# Copyright (c) 2002, Fedor Baart & Hans de Wit (Stichting Farmaceutische Kengetallen) +# All rights reserved. ## -##Redistribution and use in source and binary forms, with or without modification, -##are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: ## -##Redistributions of source code must retain the above copyright notice, this -##list of conditions and the following disclaimer. +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. ## -##Redistributions in binary form must reproduce the above copyright notice, -##this list of conditions and the following disclaimer in the documentation and/or -##other materials provided with the distribution. +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. ## -##Neither the name of the Stichting Farmaceutische Kengetallen nor the names of -##its contributors may be used to endorse or promote products derived from this -##software without specific prior written permission. +# Neither the name of the Stichting Farmaceutische Kengetallen nor the names of +# its contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. ## -##THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -##AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -##IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -##DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -##FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -##DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -##SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -##CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -##OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -##OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Thanks to Gerald Rosennfellner for his help and useful comments. -##Thanks to Gerald Rosennfellner for his help and useful comments. - -__doc__="""Use SVGdraw to generate your SVGdrawings. +import sys +import exceptions +__doc__ = """Use SVGdraw to generate your SVGdrawings. SVGdraw uses an object model drawing and a method toXML to create SVG graphics by using easy to use classes and methods usualy you start by creating a drawing eg d=drawing() - #then you create a SVG root element + # then you create a SVG root element s=svg() - #then you add some elements eg a circle and add it to the svg root element + # then you add some elements eg a circle and add it to the svg root element c=circle() - #you can supply attributes by using named arguments. + # you can supply attributes by using named arguments. c=circle(fill='red',stroke='blue') - #or by updating the attributes attribute: + # or by updating the attributes attribute: c.attributes['stroke-width']=1 s.addElement(c) - #then you add the svg root element to the drawing + # then you add the svg root element to the drawing d.setSVG(s) - #and finaly you xmlify the drawing + # and finaly you xmlify the drawing d.toXml() @@ -82,7 +84,7 @@ This module was created using the SVG specification of www.w3c.org and the O'Reilly (www.oreilly.com) python books as information sources. A svg viewer is available from www.adobe.com""" -__version__="1.0" +__version__ = "1.0" # there are two possibilities to generate svg: # via a dom implementation and directly using <element>text</element> strings @@ -93,33 +95,34 @@ __version__="1.0" # Note that PyXML is required for using the dom implementation. # It is also possible to use the standard minidom. But I didn't try that one. # Anyway the text based approach is about 60 times faster than using the full dom implementation. -use_dom_implementation=0 +use_dom_implementation = 0 -import exceptions -if use_dom_implementation<>0: +if use_dom_implementation != 0: try: from xml.dom import implementation from xml.dom.ext import PrettyPrint except: - raise exceptions.ImportError, "PyXML is required for using the dom implementation" -#The implementation is used for the creating the XML document. -#The prettyprint module is used for converting the xml document object to a xml file + raise exceptions.ImportError( + "PyXML is required for using the dom implementation") +# The implementation is used for the creating the XML document. +# The prettyprint module is used for converting the xml document object to a xml file + +assert sys.version_info[0] >= 2 +if sys.version_info[1] < 2: + True = 1 + False = 0 + file = open + +sys.setrecursionlimit = 50 +# The recursion limit is set conservative so mistakes like s=svg() s.addElement(s) +# won't eat up too much processor time. + +# the following code is pasted form xml.sax.saxutils +# it makes it possible to run the code without the xml sax package installed +# To make it possible to have <rubbish> in your text elements, it is necessary to escape the texts + -import sys -assert sys.version_info[0]>=2 -if sys.version_info[1]<2: - True=1 - False=0 - file=open - -sys.setrecursionlimit=50 -#The recursion limit is set conservative so mistakes like s=svg() s.addElement(s) -#won't eat up too much processor time. - -#the following code is pasted form xml.sax.saxutils -#it makes it possible to run the code without the xml sax package installed -#To make it possible to have <rubbish> in your text elements, it is necessary to escape the texts def _escape(data, entities={}): """Escape &, <, and > in a string of data. @@ -127,13 +130,14 @@ def _escape(data, entities={}): the optional entities parameter. The keys and values must all be strings; each key will be replaced with its corresponding value. """ - #data = data.replace("&", "&") + # data = data.replace("&", "&") data = data.replace("<", "<") data = data.replace(">", ">") - for chars, entity in entities.items(): + for chars, entity in list(entities.items()): data = data.replace(chars, entity) return data + def _quoteattr(data, entities={}): """Escape and quote an attribute value. @@ -156,96 +160,121 @@ def _quoteattr(data, entities={}): return data - def _xypointlist(a): """formats a list of xy pairs""" - s='' - for e in a: #this could be done more elegant - s+=str(e)[1:-1] +' ' + s = '' + for e in a: # this could be done more elegant + s += str(e)[1:-1] + ' ' return s + def _viewboxlist(a): """formats a tuple""" - s='' + s = '' for e in a: - s+=str(e)+' ' + s += str(e)+' ' return s + def _pointlist(a): """formats a list of numbers""" return str(a)[1:-1] + class pathdata: """class used to create a pathdata object which can be used for a path. although most methods are pretty straightforward it might be useful to look at the SVG specification.""" - #I didn't test the methods below. - def __init__(self,x=None,y=None): - self.path=[] + # I didn't test the methods below. + + def __init__(self, x=None, y=None): + self.path = [] if x is not None and y is not None: self.path.append('M '+str(x)+' '+str(y)) + def closepath(self): """ends the path""" self.path.append('z') - def move(self,x,y): + + def move(self, x, y): """move to absolute""" self.path.append('M '+str(x)+' '+str(y)) - def relmove(self,x,y): + + def relmove(self, x, y): """move to relative""" self.path.append('m '+str(x)+' '+str(y)) - def line(self,x,y): + + def line(self, x, y): """line to absolute""" self.path.append('L '+str(x)+' '+str(y)) - def relline(self,x,y): + + def relline(self, x, y): """line to relative""" self.path.append('l '+str(x)+' '+str(y)) - def hline(self,x): + + def hline(self, x): """horizontal line to absolute""" self.path.append('H'+str(x)) - def relhline(self,x): + + def relhline(self, x): """horizontal line to relative""" self.path.append('h'+str(x)) - def vline(self,y): + + def vline(self, y): """verical line to absolute""" self.path.append('V'+str(y)) - def relvline(self,y): + + def relvline(self, y): """vertical line to relative""" self.path.append('v'+str(y)) - def bezier(self,x1,y1,x2,y2,x,y): + + def bezier(self, x1, y1, x2, y2, x, y): """bezier with xy1 and xy2 to xy absolut""" - self.path.append('C'+str(x1)+','+str(y1)+' '+str(x2)+','+str(y2)+' '+str(x)+','+str(y)) - def relbezier(self,x1,y1,x2,y2,x,y): + self.path.append('C'+str(x1)+','+str(y1)+' '+str(x2) + + ','+str(y2)+' '+str(x)+','+str(y)) + + def relbezier(self, x1, y1, x2, y2, x, y): """bezier with xy1 and xy2 to xy relative""" - self.path.append('c'+str(x1)+','+str(y1)+' '+str(x2)+','+str(y2)+' '+str(x)+','+str(y)) - def smbezier(self,x2,y2,x,y): + self.path.append('c'+str(x1)+','+str(y1)+' '+str(x2) + + ','+str(y2)+' '+str(x)+','+str(y)) + + def smbezier(self, x2, y2, x, y): """smooth bezier with xy2 to xy absolut""" self.path.append('S'+str(x2)+','+str(y2)+' '+str(x)+','+str(y)) - def relsmbezier(self,x2,y2,x,y): + + def relsmbezier(self, x2, y2, x, y): """smooth bezier with xy2 to xy relative""" self.path.append('s'+str(x2)+','+str(y2)+' '+str(x)+','+str(y)) - def qbezier(self,x1,y1,x,y): + + def qbezier(self, x1, y1, x, y): """quadratic bezier with xy1 to xy absolut""" self.path.append('Q'+str(x1)+','+str(y1)+' '+str(x)+','+str(y)) - def relqbezier(self,x1,y1,x,y): + + def relqbezier(self, x1, y1, x, y): """quadratic bezier with xy1 to xy relative""" self.path.append('q'+str(x1)+','+str(y1)+' '+str(x)+','+str(y)) - def smqbezier(self,x,y): + + def smqbezier(self, x, y): """smooth quadratic bezier to xy absolut""" self.path.append('T'+str(x)+','+str(y)) - def relsmqbezier(self,x,y): + + def relsmqbezier(self, x, y): """smooth quadratic bezier to xy relative""" self.path.append('t'+str(x)+','+str(y)) - def ellarc(self,rx,ry,xrot,laf,sf,x,y): + + def ellarc(self, rx, ry, xrot, laf, sf, x, y): """elliptival arc with rx and ry rotating with xrot using large-arc-flag and sweep-flag to xy absolut""" - self.path.append('A'+str(rx)+','+str(ry)+' '+str(xrot)+' '+str(laf)+' '+str(sf)+' '+str(x)+' '+str(y)) - def relellarc(self,rx,ry,xrot,laf,sf,x,y): + self.path.append('A'+str(rx)+','+str(ry)+' '+str(xrot) + + ' '+str(laf)+' '+str(sf)+' '+str(x)+' '+str(y)) + + def relellarc(self, rx, ry, xrot, laf, sf, x, y): """elliptival arc with rx and ry rotating with xrot using large-arc-flag and sweep-flag to xy relative""" - self.path.append('a'+str(rx)+','+str(ry)+' '+str(xrot)+' '+str(laf)+' '+str(sf)+' '+str(x)+' '+str(y)) + self.path.append('a'+str(rx)+','+str(ry)+' '+str(xrot) + + ' '+str(laf)+' '+str(sf)+' '+str(x)+' '+str(y)) + def __repr__(self): return ' '.join(self.path) - - class SVGelement: """SVGelement(type,attributes,elements,text,namespace,**args) Creates a arbitrary svg element and is intended to be subclassed not used on its own. @@ -256,52 +285,56 @@ class SVGelement: namespace. Note the elements==None, if elements = None:self.elements=[] construction. This is done because if you default to elements=[] every object has a reference to the same empty list.""" - def __init__(self,type='',attributes=None,elements=None,text='',namespace='',cdata=None, **args): - self.type=type - if attributes==None: - self.attributes={} + + def __init__(self, type='', attributes=None, elements=None, text='', namespace='', cdata=None, **args): + self.type = type + if attributes == None: + self.attributes = {} else: - self.attributes=attributes - if elements==None: - self.elements=[] + self.attributes = attributes + if elements == None: + self.elements = [] else: - self.elements=elements - self.text=text - self.namespace=namespace - self.cdata=cdata - for arg in args.keys(): + self.elements = elements + self.text = text + self.namespace = namespace + self.cdata = cdata + for arg in list(args.keys()): arg2 = arg.replace("__", ":") arg2 = arg2.replace("_", "-") - self.attributes[arg2]=args[arg] - def addElement(self,SVGelement): + self.attributes[arg2] = args[arg] + + def addElement(self, SVGelement): """adds an element to a SVGelement SVGelement.addElement(SVGelement) """ self.elements.append(SVGelement) - def toXml(self,level,f): + def toXml(self, level, f): f.write('\t'*level) f.write('<'+self.type) - for attkey in self.attributes.keys(): - f.write(' '+_escape(str(attkey))+'='+_quoteattr(str(self.attributes[attkey]))) + for attkey in list(self.attributes.keys()): + f.write(' '+_escape(str(attkey))+'=' + + _quoteattr(str(self.attributes[attkey]))) if self.namespace: - f.write(' xmlns="'+ _escape(str(self.namespace))+'" xmlns:xlink="http://www.w3.org/1999/xlink"') + f.write(' xmlns="' + _escape(str(self.namespace)) + + '" xmlns:xlink="http://www.w3.org/1999/xlink"') if self.elements or self.text or self.cdata: f.write('>') if self.elements: f.write('\n') for element in self.elements: - element.toXml(level+1,f) + element.toXml(level+1, f) if self.cdata: f.write('\n'+'\t'*(level+1)+'<![CDATA[') for line in self.cdata.splitlines(): f.write('\n'+'\t'*(level+2)+line) f.write('\n'+'\t'*(level+1)+']]>\n') if self.text: - if type(self.text)==type(''): #If the text is only text + if isinstance(self.text, type('')): # If the text is only text f.write(_escape(str(self.text))) - else: #If the text is a spannedtext class + else: # If the text is a spannedtext class f.write(str(self.text)) if self.elements: f.write('\t'*level+'</'+self.type+'>\n') @@ -312,6 +345,7 @@ class SVGelement: else: f.write('/>\n') + class tspan(SVGelement): """ts=tspan(text='',**args) @@ -323,19 +357,22 @@ class tspan(SVGelement): st.addtspan(ts) t=text(3,5,st) """ - def __init__(self,text=None,**args): - SVGelement.__init__(self,'tspan',**args) - if self.text<>None: - self.text=text + + def __init__(self, text=None, **args): + SVGelement.__init__(self, 'tspan', **args) + if self.text != None: + self.text = text + def __repr__(self): - s="<tspan" - for key,value in self.attributes.items(): - s+= ' %s="%s"' % (key,value) - s+='>' - s+=self.text - s+='</tspan>' + s = "<tspan" + for key, value in list(self.attributes.items()): + s += ' %s="%s"' % (key, value) + s += '>' + s += self.text + s += '</tspan>' return s + class tref(SVGelement): """tr=tref(link='',**args) @@ -346,16 +383,19 @@ class tref(SVGelement): st.addtref(tr) t=text(3,5,st) """ - def __init__(self,link,**args): - SVGelement.__init__(self,'tref',{'xlink:href':link},**args) + + def __init__(self, link, **args): + SVGelement.__init__(self, 'tref', {'xlink:href': link}, **args) + def __repr__(self): - s="<tref" + s = "<tref" - for key,value in self.attributes.items(): - s+= ' %s="%s"' % (key,value) - s+='/>' + for key, value in list(self.attributes.items()): + s += ' %s="%s"' % (key, value) + s += '/>' return s + class spannedtext: """st=spannedtext(textlist=[]) @@ -374,46 +414,49 @@ class spannedtext: st.addtext('This text is not bold') t=text(3,5,st) """ - def __init__(self,textlist=None): - if textlist==None: - self.textlist=[] + + def __init__(self, textlist=None): + if textlist == None: + self.textlist = [] else: - self.textlist=textlist - def addtext(self,text=''): + self.textlist = textlist + + def addtext(self, text=''): self.textlist.append(text) - def addtspan(self,tspan): + + def addtspan(self, tspan): self.textlist.append(tspan) - def addtref(self,tref): + + def addtref(self, tref): self.textlist.append(tref) + def __repr__(self): - s="" + s = "" for element in self.textlist: - s+=str(element) + s += str(element) return s + class rect(SVGelement): """r=rect(width,height,x,y,fill,stroke,stroke_width,**args) a rectangle is defined by a width and height and a xy pair """ - def __init__(self,x=None,y=None,width=None,height=None,fill=None,stroke=None,stroke_width=None,**args): - if width==None or height==None: - if width<>None: - raise ValueError, 'height is required' - if height<>None: - raise ValueError, 'width is required' - else: - raise ValueError, 'both height and width are required' - SVGelement.__init__(self,'rect',{'width':width,'height':height},**args) - if x<>None: + + def __init__(self, x=None, y=None, width=None, height=None, fill=None, stroke=None, stroke_width=None, **args): + if width == None or height == None: + raise ValueError('both height and width are required') + + SVGelement.__init__(self, 'rect', {'width':width,'height':height}, **args) + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width class ellipse(SVGelement): @@ -423,22 +466,18 @@ class ellipse(SVGelement): """ def __init__(self,cx=None,cy=None,rx=None,ry=None,fill=None,stroke=None,stroke_width=None,**args): if rx==None or ry== None: - if rx<>None: - raise ValueError, 'rx is required' - if ry<>None: - raise ValueError, 'ry is required' - else: - raise ValueError, 'both rx and ry are required' - SVGelement.__init__(self,'ellipse',{'rx':rx,'ry':ry},**args) - if cx<>None: + raise ValueError('both rx and ry are required') + + SVGelement.__init__(self, 'ellipse', {'rx':rx,'ry':ry}, **args) + if cx!=None: self.attributes['cx']=cx - if cy<>None: + if cy!=None: self.attributes['cy']=cy - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width @@ -449,17 +488,17 @@ class circle(SVGelement): """ def __init__(self,cx=None,cy=None,r=None,fill=None,stroke=None,stroke_width=None,**args): if r==None: - raise ValueError, 'r is required' - SVGelement.__init__(self,'circle',{'r':r},**args) - if cx<>None: + raise ValueError('r is required') + SVGelement.__init__(self, 'circle', {'r':r}, **args) + if cx!=None: self.attributes['cx']=cx - if cy<>None: + if cy!=None: self.attributes['cy']=cy - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width class point(circle): @@ -469,7 +508,7 @@ class point(circle): very small rectangle if you use many points because a circle is difficult to render. """ def __init__(self,x,y,fill='black',**args): - circle.__init__(self,x,y,1,fill,**args) + circle.__init__(self, x, y, 1, fill, **args) class line(SVGelement): """l=line(x1,y1,x2,y2,stroke,stroke_width,**args) @@ -477,18 +516,18 @@ class line(SVGelement): A line is defined by a begin x,y pair and an end x,y pair """ def __init__(self,x1=None,y1=None,x2=None,y2=None,stroke=None,stroke_width=None,**args): - SVGelement.__init__(self,'line',**args) - if x1<>None: + SVGelement.__init__(self, 'line', **args) + if x1!=None: self.attributes['x1']=x1 - if y1<>None: + if y1!=None: self.attributes['y1']=y1 - if x2<>None: + if x2!=None: self.attributes['x2']=x2 - if y2<>None: + if y2!=None: self.attributes['y2']=y2 - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke class polyline(SVGelement): @@ -497,12 +536,12 @@ class polyline(SVGelement): a polyline is defined by a list of xy pairs """ def __init__(self,points,fill=None,stroke=None,stroke_width=None,**args): - SVGelement.__init__(self,'polyline',{'points':_xypointlist(points)},**args) - if fill<>None: + SVGelement.__init__(self, 'polyline', {'points':_xypointlist(points)}, **args) + if fill!=None: self.attributes['fill']=fill - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke class polygon(SVGelement): @@ -511,12 +550,12 @@ class polygon(SVGelement): a polygon is defined by a list of xy pairs """ def __init__(self,points,fill=None,stroke=None,stroke_width=None,**args): - SVGelement.__init__(self,'polygon',{'points':_xypointlist(points)},**args) - if fill<>None: + SVGelement.__init__(self, 'polygon', {'points':_xypointlist(points)}, **args) + if fill!=None: self.attributes['fill']=fill - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke class path(SVGelement): @@ -525,14 +564,14 @@ class path(SVGelement): a path is defined by a path object and optional width, stroke and fillcolor """ def __init__(self,pathdata,fill=None,stroke=None,stroke_width=None,id=None,**args): - SVGelement.__init__(self,'path',{'d':str(pathdata)},**args) - if stroke<>None: + SVGelement.__init__(self, 'path', {'d':str(pathdata)}, **args) + if stroke!=None: self.attributes['stroke']=stroke - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width - if id<>None: + if id!=None: self.attributes['id']=id @@ -542,18 +581,18 @@ class text(SVGelement): a text element can bge used for displaying text on the screen """ def __init__(self,x=None,y=None,text=None,font_size=None,font_family=None,text_anchor=None,**args): - SVGelement.__init__(self,'text',**args) - if x<>None: + SVGelement.__init__(self, 'text', **args) + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y - if font_size<>None: + if font_size!=None: self.attributes['font-size']=font_size - if font_family<>None: + if font_family!=None: self.attributes['font-family']=font_family - if text<>None: + if text!=None: self.text=text - if text_anchor<>None: + if text_anchor!=None: self.attributes['text-anchor']=text_anchor @@ -563,8 +602,8 @@ class textpath(SVGelement): a textpath places a text on a path which is referenced by a link. """ def __init__(self,link,text=None,**args): - SVGelement.__init__(self,'textPath',{'xlink:href':link},**args) - if text<>None: + SVGelement.__init__(self, 'textPath', {'xlink:href':link}, **args) + if text!=None: self.text=text class pattern(SVGelement): @@ -575,16 +614,16 @@ class pattern(SVGelement): in x and y to cover the areas to be painted. """ def __init__(self,x=None,y=None,width=None,height=None,patternUnits=None,**args): - SVGelement.__init__(self,'pattern',**args) - if x<>None: + SVGelement.__init__(self, 'pattern', **args) + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y - if width<>None: + if width!=None: self.attributes['width']=width - if height<>None: + if height!=None: self.attributes['height']=height - if patternUnits<>None: + if patternUnits!=None: self.attributes['patternUnits']=patternUnits class title(SVGelement): @@ -594,8 +633,8 @@ class title(SVGelement): add at least one to the root svg element """ def __init__(self,text=None,**args): - SVGelement.__init__(self,'title',**args) - if text<>None: + SVGelement.__init__(self, 'title', **args) + if text!=None: self.text=text class description(SVGelement): @@ -605,8 +644,8 @@ class description(SVGelement): Add this element before adding other elements. """ def __init__(self,text=None,**args): - SVGelement.__init__(self,'desc',**args) - if text<>None: + SVGelement.__init__(self, 'desc', **args) + if text!=None: self.text=text class lineargradient(SVGelement): @@ -616,16 +655,16 @@ class lineargradient(SVGelement): stop elements van be added to define the gradient colors. """ def __init__(self,x1=None,y1=None,x2=None,y2=None,id=None,**args): - SVGelement.__init__(self,'linearGradient',**args) - if x1<>None: + SVGelement.__init__(self, 'linearGradient', **args) + if x1!=None: self.attributes['x1']=x1 - if y1<>None: + if y1!=None: self.attributes['y1']=y1 - if x2<>None: + if x2!=None: self.attributes['x2']=x2 - if y2<>None: + if y2!=None: self.attributes['y2']=y2 - if id<>None: + if id!=None: self.attributes['id']=id class radialgradient(SVGelement): @@ -635,18 +674,18 @@ class radialgradient(SVGelement): stop elements van be added to define the gradient colors. """ def __init__(self,cx=None,cy=None,r=None,fx=None,fy=None,id=None,**args): - SVGelement.__init__(self,'radialGradient',**args) - if cx<>None: + SVGelement.__init__(self, 'radialGradient', **args) + if cx!=None: self.attributes['cx']=cx - if cy<>None: + if cy!=None: self.attributes['cy']=cy - if r<>None: + if r!=None: self.attributes['r']=r - if fx<>None: + if fx!=None: self.attributes['fx']=fx - if fy<>None: + if fy!=None: self.attributes['fy']=fy - if id<>None: + if id!=None: self.attributes['id']=id class stop(SVGelement): @@ -655,8 +694,8 @@ class stop(SVGelement): Puts a stop color at the specified radius """ def __init__(self,offset,stop_color=None,**args): - SVGelement.__init__(self,'stop',{'offset':offset},**args) - if stop_color<>None: + SVGelement.__init__(self, 'stop', {'offset':offset}, **args) + if stop_color!=None: self.attributes['stop-color']=stop_color class style(SVGelement): @@ -665,7 +704,7 @@ class style(SVGelement): Add a CDATA element to this element for defing in line stylesheets etc.. """ def __init__(self,type,cdata=None,**args): - SVGelement.__init__(self,'style',{'type':type},cdata=cdata, **args) + SVGelement.__init__(self, 'style', {'type':type}, cdata=cdata, **args) class image(SVGelement): @@ -675,16 +714,11 @@ class image(SVGelement): """ def __init__(self,url,x=None,y=None,width=None,height=None,**args): if width==None or height==None: - if width<>None: - raise ValueError, 'height is required' - if height<>None: - raise ValueError, 'width is required' - else: - raise ValueError, 'both height and width are required' - SVGelement.__init__(self,'image',{'xlink:href':url,'width':width,'height':height},**args) - if x<>None: + raise ValueError('both height and width are required') + SVGelement.__init__(self, 'image', {'xlink:href':url,'width':width,'height':height}, **args) + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y class cursor(SVGelement): @@ -693,7 +727,7 @@ class cursor(SVGelement): defines a custom cursor for a element or a drawing """ def __init__(self,url,**args): - SVGelement.__init__(self,'cursor',{'xlink:href':url},**args) + SVGelement.__init__(self, 'cursor', {'xlink:href':url}, **args) class marker(SVGelement): @@ -703,18 +737,18 @@ class marker(SVGelement): add an element to it which should be used as a marker. """ def __init__(self,id=None,viewBox=None,refx=None,refy=None,markerWidth=None,markerHeight=None,**args): - SVGelement.__init__(self,'marker',**args) - if id<>None: + SVGelement.__init__(self, 'marker', **args) + if id!=None: self.attributes['id']=id - if viewBox<>None: + if viewBox!=None: self.attributes['viewBox']=_viewboxlist(viewBox) - if refx<>None: + if refx!=None: self.attributes['refX']=refx - if refy<>None: + if refy!=None: self.attributes['refY']=refy - if markerWidth<>None: + if markerWidth!=None: self.attributes['markerWidth']=markerWidth - if markerHeight<>None: + if markerHeight!=None: self.attributes['markerHeight']=markerHeight class group(SVGelement): @@ -724,8 +758,8 @@ class group(SVGelement): g.addElement(SVGelement) """ def __init__(self,id=None,**args): - SVGelement.__init__(self,'g',**args) - if id<>None: + SVGelement.__init__(self, 'g', **args) + if id!=None: self.attributes['id']=id class symbol(SVGelement): @@ -738,10 +772,10 @@ class symbol(SVGelement): """ def __init__(self,id=None,viewBox=None,**args): - SVGelement.__init__(self,'symbol',**args) - if id<>None: + SVGelement.__init__(self, 'symbol', **args) + if id!=None: self.attributes['id']=id - if viewBox<>None: + if viewBox!=None: self.attributes['viewBox']=_viewboxlist(viewBox) class defs(SVGelement): @@ -750,7 +784,7 @@ class defs(SVGelement): container for defining elements """ def __init__(self,**args): - SVGelement.__init__(self,'defs',**args) + SVGelement.__init__(self, 'defs', **args) class switch(SVGelement): """sw=switch(**args) @@ -760,7 +794,7 @@ class switch(SVGelement): Refer to the SVG specification for details. """ def __init__(self,**args): - SVGelement.__init__(self,'switch',**args) + SVGelement.__init__(self, 'switch', **args) class use(SVGelement): @@ -769,15 +803,15 @@ class use(SVGelement): references a symbol by linking to its id and its position, height and width """ def __init__(self,link,x=None,y=None,width=None,height=None,**args): - SVGelement.__init__(self,'use',{'xlink:href':link},**args) - if x<>None: + SVGelement.__init__(self, 'use', {'xlink:href':link}, **args) + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y - if width<>None: + if width!=None: self.attributes['width']=width - if height<>None: + if height!=None: self.attributes['height']=height @@ -788,15 +822,15 @@ class link(SVGelement): a.addElement(SVGelement) """ def __init__(self,link='',**args): - SVGelement.__init__(self,'a',{'xlink:href':link},**args) + SVGelement.__init__(self, 'a', {'xlink:href':link}, **args) class view(SVGelement): """v=view(id,**args) a view can be used to create a view with different attributes""" def __init__(self,id=None,**args): - SVGelement.__init__(self,'view',**args) - if id<>None: + SVGelement.__init__(self, 'view', **args) + if id!=None: self.attributes['id']=id class script(SVGelement): @@ -806,7 +840,7 @@ class script(SVGelement): """ def __init__(self,type,cdata=None,**args): - SVGelement.__init__(self,'script',{'type':type},cdata=cdata,**args) + SVGelement.__init__(self, 'script', {'type':type}, cdata=cdata, **args) class animate(SVGelement): """an=animate(attribute,from,to,during,**args) @@ -814,12 +848,12 @@ class animate(SVGelement): animates an attribute. """ def __init__(self,attribute,fr=None,to=None,dur=None,**args): - SVGelement.__init__(self,'animate',{'attributeName':attribute},**args) - if fr<>None: + SVGelement.__init__(self, 'animate', {'attributeName':attribute}, **args) + if fr!=None: self.attributes['from']=fr - if to<>None: + if to!=None: self.attributes['to']=to - if dur<>None: + if dur!=None: self.attributes['dur']=dur class animateMotion(SVGelement): @@ -828,10 +862,10 @@ class animateMotion(SVGelement): animates a SVGelement over the given path in dur seconds """ def __init__(self,pathdata,dur,**args): - SVGelement.__init__(self,'animateMotion',**args) - if pathdata<>None: + SVGelement.__init__(self, 'animateMotion', **args) + if pathdata!=None: self.attributes['path']=str(pathdata) - if dur<>None: + if dur!=None: self.attributes['dur']=dur class animateTransform(SVGelement): @@ -840,15 +874,15 @@ class animateTransform(SVGelement): transform an element from and to a value. """ def __init__(self,type=None,fr=None,to=None,dur=None,**args): - SVGelement.__init__(self,'animateTransform',{'attributeName':'transform'},**args) - #As far as I know the attributeName is always transform - if type<>None: + SVGelement.__init__(self, 'animateTransform', {'attributeName':'transform'}, **args) + # As far as I know the attributeName is always transform + if type!=None: self.attributes['type']=type - if fr<>None: + if fr!=None: self.attributes['from']=fr - if to<>None: + if to!=None: self.attributes['to']=to - if dur<>None: + if dur!=None: self.attributes['dur']=dur class animateColor(SVGelement): """ac=animateColor(attribute,type,from,to,dur,**args) @@ -856,14 +890,14 @@ class animateColor(SVGelement): Animates the color of a element """ def __init__(self,attribute,type=None,fr=None,to=None,dur=None,**args): - SVGelement.__init__(self,'animateColor',{'attributeName':attribute},**args) - if type<>None: + SVGelement.__init__(self, 'animateColor', {'attributeName':attribute}, **args) + if type!=None: self.attributes['type']=type - if fr<>None: + if fr!=None: self.attributes['from']=fr - if to<>None: + if to!=None: self.attributes['to']=to - if dur<>None: + if dur!=None: self.attributes['dur']=dur class set(SVGelement): """st=set(attribute,to,during,**args) @@ -871,10 +905,10 @@ class set(SVGelement): sets an attribute to a value for a """ def __init__(self,attribute,to=None,dur=None,**args): - SVGelement.__init__(self,'set',{'attributeName':attribute},**args) - if to<>None: + SVGelement.__init__(self, 'set', {'attributeName':attribute}, **args) + if to!=None: self.attributes['to']=to - if dur<>None: + if dur!=None: self.attributes['dur']=dur @@ -895,12 +929,12 @@ class svg(SVGelement): d.toXml() """ def __init__(self,viewBox=None, width=None, height=None,**args): - SVGelement.__init__(self,'svg',**args) - if viewBox<>None: + SVGelement.__init__(self, 'svg', **args) + if viewBox!=None: self.attributes['viewBox']=_viewboxlist(viewBox) - if width<>None: + if width!=None: self.attributes['width']=width - if height<>None: + if height!=None: self.attributes['height']=height self.namespace="http://www.w3.org/2000/svg" @@ -918,27 +952,27 @@ class drawing: def __init__(self, entity={}): self.svg=None self.entity = entity - def setSVG(self,svg): + def setSVG(self, svg): self.svg=svg - #Voeg een element toe aan de grafiek toe. + # Voeg een element toe aan de grafiek toe. if use_dom_implementation==0: def toXml(self, filename='',compress=False): - import cStringIO - xml=cStringIO.StringIO() + import io + xml=io.StringIO() xml.write("<?xml version='1.0' encoding='UTF-8'?>\n") xml.write("<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.0//EN\" \"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd\"") if self.entity: xml.write(" [\n") - for item in self.entity.keys(): + for item in list(self.entity.keys()): xml.write("<!ENTITY %s \"%s\">\n" % (item, self.entity[item])) xml.write("]") xml.write(">\n") - self.svg.toXml(0,xml) + self.svg.toXml(0, xml) if not filename: if compress: import gzip - f=cStringIO.StringIO() - zf=gzip.GzipFile(fileobj=f,mode='wb') + f=io.StringIO() + zf=gzip.GzipFile(fileobj=f, mode='wb') zf.write(xml.getvalue()) zf.close() f.seek(0) @@ -948,11 +982,11 @@ class drawing: else: if filename[-4:]=='svgz': import gzip - f=gzip.GzipFile(filename=filename,mode="wb", compresslevel=9) + f=gzip.GzipFile(filename=filename, mode="wb", compresslevel=9) f.write(xml.getvalue()) f.close() else: - f=file(filename,'w') + f=file(filename, 'w') f.write(xml.getvalue()) f.close() @@ -963,40 +997,40 @@ class drawing: writes a svg drawing to the screen or to a file compresses if filename ends with svgz or if compress is true """ - doctype = implementation.createDocumentType('svg',"-//W3C//DTD SVG 1.0//EN""",'http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd ') + doctype = implementation.createDocumentType('svg', "-//W3C//DTD SVG 1.0//EN""", 'http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd ') global root - #root is defined global so it can be used by the appender. Its also possible to use it as an arugument but - #that is a bit messy. - root=implementation.createDocument(None,None,doctype) - #Create the xml document. + # root is defined global so it can be used by the appender. Its also possible to use it as an arugument but + # that is a bit messy. + root=implementation.createDocument(None, None, doctype) + # Create the xml document. global appender - def appender(element,elementroot): + def appender(element, elementroot): """This recursive function appends elements to an element and sets the attributes and type. It stops when alle elements have been appended""" if element.namespace: - e=root.createElementNS(element.namespace,element.type) + e=root.createElementNS(element.namespace, element.type) else: e=root.createElement(element.type) if element.text: textnode=root.createTextNode(element.text) e.appendChild(textnode) - for attribute in element.attributes.keys(): #in element.attributes is supported from python 2.2 - e.setAttribute(attribute,str(element.attributes[attribute])) + for attribute in list(element.attributes.keys()): #in element.attributes is supported from python 2.2 + e.setAttribute(attribute, str(element.attributes[attribute])) if element.elements: for el in element.elements: - e=appender(el,e) + e=appender(el, e) elementroot.appendChild(e) return elementroot - root=appender(self.svg,root) + root=appender(self.svg, root) if not filename: - import cStringIO - xml=cStringIO.StringIO() - PrettyPrint(root,xml) + import io + xml=io.StringIO() + PrettyPrint(root, xml) if compress: import gzip - f=cStringIO.StringIO() - zf=gzip.GzipFile(fileobj=f,mode='wb') + f=io.StringIO() + zf=gzip.GzipFile(fileobj=f, mode='wb') zf.write(xml.getvalue()) zf.close() f.seek(0) @@ -1007,23 +1041,23 @@ class drawing: try: if filename[-4:]=='svgz': import gzip - import cStringIO - xml=cStringIO.StringIO() - PrettyPrint(root,xml) - f=gzip.GzipFile(filename=filename,mode='wb',compresslevel=9) + import io + xml=io.StringIO() + PrettyPrint(root, xml) + f=gzip.GzipFile(filename=filename, mode='wb', compresslevel=9) f.write(xml.getvalue()) f.close() else: - f=open(filename,'w') - PrettyPrint(root,f) + f=open(filename, 'w') + PrettyPrint(root, f) f.close() except: - print "Cannot write SVG file: " + filename + print(("Cannot write SVG file: " + filename)) def validate(self): try: import xml.parsers.xmlproc.xmlval except: - raise exceptions.ImportError,'PyXml is required for validating SVG' + raise exceptions.ImportError('PyXml is required for validating SVG') svg=self.toXml() xv=xml.parsers.xmlproc.xmlval.XMLValidator() try: @@ -1031,38 +1065,38 @@ class drawing: except: raise Exception("SVG is not well formed, see messages above") else: - print "SVG well formed" + print("SVG well formed") if __name__=='__main__': d=drawing() - s=svg((0,0,100,100)) - r=rect(-100,-100,300,300,'cyan') + s=svg((0, 0, 100, 100)) + r=rect(-100, -100, 300, 300, 'cyan') s.addElement(r) t=title('SVGdraw Demo') s.addElement(t) g=group('animations') - e=ellipse(0,0,5,2) + e=ellipse(0, 0, 5, 2) g.addElement(e) - c=circle(0,0,1,'red') + c=circle(0, 0, 1, 'red') g.addElement(c) - pd=pathdata(0,-10) + pd=pathdata(0, -10) for i in range(6): - pd.relsmbezier(10,5,0,10) - pd.relsmbezier(-10,5,0,10) - an=animateMotion(pd,10) + pd.relsmbezier(10, 5, 0, 10) + pd.relsmbezier(-10, 5, 0, 10) + an=animateMotion(pd, 10) an.attributes['rotate']='auto-reverse' an.attributes['repeatCount']="indefinite" g.addElement(an) s.addElement(g) - for i in range(20,120,20): - u=use('#animations',i,0) + for i in range(20, 120, 20): + u=use('#animations', i, 0) s.addElement(u) - for i in range(0,120,20): - for j in range(5,105,10): - c=circle(i,j,1,'red','black',.5) + for i in range(0, 120, 20): + for j in range(5, 105, 10): + c=circle(i, j, 1, 'red', 'black', .5) s.addElement(c) d.setSVG(s) - print d.toXml() + print((d.toXml())) diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 5bf700c9..4144ae00 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -1,4 +1,3 @@ -from __future__ import print_function, division, absolute_import from redis import Redis import simplejson as json @@ -20,6 +19,6 @@ class TempData(object): if __name__ == "__main__": redis = Redis() - for key in redis.keys(): + for key in list(redis.keys()): for field in redis.hkeys(key): print("{}.{}={}".format(key, field, redis.hget(key, field))) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 77db5d53..68ef0f04 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -15,7 +15,7 @@ OVERRIDES = {} def app_set(command_id, value): """Set application wide value""" - app.config.setdefault(command_id,value) + app.config.setdefault(command_id, value) return value def get_setting(command_id,guess=None): @@ -45,7 +45,7 @@ def get_setting(command_id,guess=None): def value(command): if command: # sys.stderr.write("Found "+command+"\n") - app_set(command_id,command) + app_set(command_id, command) return command else: return None @@ -68,7 +68,7 @@ def get_setting(command_id,guess=None): def get_setting_bool(id): v = get_setting(id) - if v not in [0,False,'False','FALSE',None]: + if v not in [0, False, 'False', 'FALSE', None]: return True return False @@ -108,16 +108,16 @@ def js_path(module=None): raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)" def reaper_command(guess=None): - return get_setting("REAPER_COMMAND",guess) + return get_setting("REAPER_COMMAND", guess) def gemma_command(guess=None): - return assert_bin(get_setting("GEMMA_COMMAND",guess)) + return assert_bin(get_setting("GEMMA_COMMAND", guess)) def gemma_wrapper_command(guess=None): - return assert_bin(get_setting("GEMMA_WRAPPER_COMMAND",guess)) + return assert_bin(get_setting("GEMMA_WRAPPER_COMMAND", guess)) def plink_command(guess=None): - return assert_bin(get_setting("PLINK_COMMAND",guess)) + return assert_bin(get_setting("PLINK_COMMAND", guess)) def flat_file_exists(subdir): base = get_setting("GENENETWORK_FILES") @@ -180,7 +180,7 @@ def locate(name, subdir=None): raise Exception("Can not locate "+name+" in "+base) def locate_phewas(name, subdir=None): - return locate(name,'/phewas/'+subdir) + return locate(name, '/phewas/'+subdir) def locate_ignore_error(name, subdir=None): """ @@ -204,7 +204,7 @@ def tempdir(): """ Get UNIX TMPDIR by default """ - return valid_path(get_setting("TMPDIR","/tmp")) + return valid_path(get_setting("TMPDIR", "/tmp")) BLUE = '\033[94m' GREEN = '\033[92m' @@ -214,20 +214,20 @@ ENDC = '\033[0m' def show_settings(): from utility.tools import LOG_LEVEL - print("Set global log level to "+BLUE+LOG_LEVEL+ENDC) + print(("Set global log level to "+BLUE+LOG_LEVEL+ENDC)) log_level = getattr(logging, LOG_LEVEL.upper()) logging.basicConfig(level=log_level) logger.info(OVERRIDES) logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC) - print "runserver.py: ****** Webserver configuration - k,v pairs from app.config ******" - keylist = app.config.keys() + keylist = list(app.config.keys()) + print("runserver.py: ****** Webserver configuration - k,v pairs from app.config ******") keylist.sort() for k in keylist: try: - print("%s: %s%s%s%s" % (k,BLUE,BOLD,get_setting(k),ENDC)) + print(("%s: %s%s%s%s" % (k, BLUE, BOLD, get_setting(k), ENDC))) except: - print("%s: %s%s%s%s" % (k,GREEN,BOLD,app.config[k],ENDC)) + print(("%s: %s%s%s%s" % (k, GREEN, BOLD, app.config[k], ENDC))) # Cached values @@ -279,10 +279,10 @@ SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') SMTP_PASSWORD = get_setting('SMTP_PASSWORD') -REAPER_COMMAND = app_set("REAPER_COMMAND",reaper_command()) -GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) +REAPER_COMMAND = app_set("REAPER_COMMAND", reaper_command()) +GEMMA_COMMAND = app_set("GEMMA_COMMAND", gemma_command()) assert(GEMMA_COMMAND is not None) -PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) +PLINK_COMMAND = app_set("PLINK_COMMAND", plink_command()) GEMMA_WRAPPER_COMMAND = gemma_wrapper_command() TEMPDIR = tempdir() # defaults to UNIX TMPDIR assert_dir(TEMPDIR) @@ -295,11 +295,11 @@ assert_dir(JS_GUIX_PATH+'/cytoscape-panzoom') CSS_PATH = JS_GUIX_PATH # The CSS is bundled together with the JS # assert_dir(JS_PATH) -JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("javascript-twitter-post-fetcher")) +JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH", js_path("javascript-twitter-post-fetcher")) assert_dir(JS_TWITTER_POST_FETCHER_PATH) assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") -JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH",js_path("cytoscape")) +JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH", js_path("cytoscape")) assert_dir(JS_CYTOSCAPE_PATH) assert_file(JS_CYTOSCAPE_PATH+'/cytoscape.min.js') diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index 53661ae4..5681fadf 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -41,22 +41,22 @@ ParInfo ={ 'C57BL-6JxC57BL-6NJF2':['', '', 'C57BL/6J', 'C57BL/6NJ'], 'BXD300':['B6D2F1', 'D2B6F1', 'C57BL/6J', 'DBA/2J'], 'B6BTBRF2':['B6BTBRF1', 'BTBRB6F1', 'C57BL/6J', 'BTBRT<+>tf/J'], -'BHHBF2':['B6HF2','HB6F2','C57BL/6J','C3H/HeJ'], -'BHF2':['B6HF2','HB6F2','C57BL/6J','C3H/HeJ'], +'BHHBF2':['B6HF2', 'HB6F2', 'C57BL/6J', 'C3H/HeJ'], +'BHF2':['B6HF2', 'HB6F2', 'C57BL/6J', 'C3H/HeJ'], 'B6D2F2':['B6D2F1', 'D2B6F1', 'C57BL/6J', 'DBA/2J'], 'BDF2-1999':['B6D2F2', 'D2B6F2', 'C57BL/6J', 'DBA/2J'], 'BDF2-2005':['B6D2F1', 'D2B6F1', 'C57BL/6J', 'DBA/2J'], -'CTB6F2':['CTB6F2','B6CTF2','C57BL/6J','Castaneous'], +'CTB6F2':['CTB6F2', 'B6CTF2', 'C57BL/6J', 'Castaneous'], 'CXB':['CBF1', 'BCF1', 'C57BL/6ByJ', 'BALB/cByJ'], 'AXBXA':['ABF1', 'BAF1', 'C57BL/6J', 'A/J'], 'AXB':['ABF1', 'BAF1', 'C57BL/6J', 'A/J'], 'BXA':['BAF1', 'ABF1', 'C57BL/6J', 'A/J'], 'LXS':['LSF1', 'SLF1', 'ISS', 'ILS'], 'HXBBXH':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'], -'BayXSha':['BayXShaF1', 'ShaXBayF1', 'Bay-0','Shahdara'], -'ColXBur':['ColXBurF1', 'BurXColF1', 'Col-0','Bur-0'], -'ColXCvi':['ColXCviF1', 'CviXColF1', 'Col-0','Cvi'], -'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'], +'BayXSha':['BayXShaF1', 'ShaXBayF1', 'Bay-0', 'Shahdara'], +'ColXBur':['ColXBurF1', 'BurXColF1', 'Col-0', 'Bur-0'], +'ColXCvi':['ColXCviF1', 'CviXColF1', 'Col-0', 'Cvi'], +'SXM':['SMF1', 'MSF1', 'Steptoe', 'Morex'], 'HRDP':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'] } @@ -64,7 +64,7 @@ ParInfo ={ # Accessory Functions ######################################### -def genRandStr(prefix = "", length=8, chars=string.letters+string.digits): +def genRandStr(prefix = "", length=8, chars=string.ascii_letters+string.digits): from random import choice _str = prefix[:] for i in range(length): @@ -91,7 +91,7 @@ def readLineCSV(line): ### dcrowell July 2008 returnList[0]=returnList[0][1:] return returnList -def cmpEigenValue(A,B): +def cmpEigenValue(A, B): try: if A[0] > B[0]: return -1 @@ -107,7 +107,7 @@ def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users) if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']: access_to_confidential_phenotype_trait = 1 else: - AuthorizedUsersList=map(string.strip, string.split(authorized_users, ',')) - if AuthorizedUsersList.__contains__(userName): + AuthorizedUsersList=[x.strip() for x in authorized_users.split(',')] + if userName in AuthorizedUsersList: access_to_confidential_phenotype_trait = 1 - return access_to_confidential_phenotype_trait
\ No newline at end of file + return access_to_confidential_phenotype_trait diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 7ed9c7b8..274c3d82 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -1,8 +1,8 @@ -from __future__ import absolute_import, division, print_function - import sys +import time import jinja2 +from flask import g from flask import Flask from utility import formatting @@ -19,6 +19,13 @@ app.jinja_env.globals.update( undefined=jinja2.StrictUndefined, numify=formatting.numify) + +@app.before_request +def before_request(): + g.request_start_time = time.time() + g.request_time = lambda: "%.5fs" % (time.time() - g.request_start_time) + + from wqflask.api import router from wqflask import group_manager from wqflask import resource_manager diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index 7f5312c1..f5b50dcd 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -1,10 +1,8 @@ -from __future__ import absolute_import, division, print_function - import collections import scipy -from MySQLdb import escape_string as escape +from utility.db_tools import escape from flask import g @@ -36,7 +34,7 @@ def do_correlation(start_vars): #corr_results = collections.OrderedDict(sorted(corr_results.items(), key=lambda t: -abs(t[1][0]))) final_results = [] - for _trait_counter, trait in enumerate(corr_results.keys()[:corr_params['return_count']]): + for _trait_counter, trait in enumerate(list(corr_results.keys())[:corr_params['return_count']]): if corr_params['type'] == "tissue": [sample_r, num_overlap, sample_p, symbol] = corr_results[trait] result_dict = { @@ -76,20 +74,20 @@ def calculate_results(this_trait, this_dataset, target_dataset, corr_params): if corr_params['type'] == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") corr_results = do_tissue_correlation_for_all_traits(this_trait, trait_symbol_dict, corr_params) - sorted_results = collections.OrderedDict(sorted(corr_results.items(), + sorted_results = collections.OrderedDict(sorted(list(corr_results.items()), key=lambda t: -abs(t[1][1]))) elif corr_params['type'] == "literature" or corr_params['type'] == "lit": #ZS: Just so a user can use either "lit" or "literature" trait_geneid_dict = this_dataset.retrieve_genes("GeneId") corr_results = do_literature_correlation_for_all_traits(this_trait, this_dataset, trait_geneid_dict, corr_params) - sorted_results = collections.OrderedDict(sorted(corr_results.items(), + sorted_results = collections.OrderedDict(sorted(list(corr_results.items()), key=lambda t: -abs(t[1][1]))) else: - for target_trait, target_vals in target_dataset.trait_data.iteritems(): + for target_trait, target_vals in list(target_dataset.trait_data.items()): result = get_sample_r_and_p_values(this_trait, this_dataset, target_vals, target_dataset, corr_params['type']) if result is not None: corr_results[target_trait] = result - sorted_results = collections.OrderedDict(sorted(corr_results.items(), key=lambda t: -abs(t[1][0]))) + sorted_results = collections.OrderedDict(sorted(list(corr_results.items()), key=lambda t: -abs(t[1][0]))) return sorted_results @@ -100,10 +98,10 @@ def do_tissue_correlation_for_all_traits(this_trait, trait_symbol_dict, corr_par if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower()] - corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(symbol_list=trait_symbol_dict.values()) + corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(symbol_list=list(trait_symbol_dict.values())) tissue_corr_data = {} - for trait, symbol in trait_symbol_dict.iteritems(): + for trait, symbol in list(trait_symbol_dict.items()): if symbol and symbol.lower() in corr_result_tissue_vals_dict: this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower()] @@ -119,7 +117,7 @@ def do_literature_correlation_for_all_traits(this_trait, target_dataset, trait_g input_trait_mouse_gene_id = convert_to_mouse_gene_id(target_dataset.group.species.lower(), this_trait.geneid) lit_corr_data = {} - for trait, gene_id in trait_geneid_dict.iteritems(): + for trait, gene_id in list(trait_geneid_dict.items()): mouse_gene_id = convert_to_mouse_gene_id(target_dataset.group.species.lower(), gene_id) if mouse_gene_id and str(mouse_gene_id).find(";") == -1: @@ -234,4 +232,4 @@ def init_corr_params(start_vars): 'return_count' : return_count } - return corr_params
\ No newline at end of file + return corr_params diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index fedf3e0b..1dcafe1f 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division - from flask import g @@ -61,7 +59,7 @@ def get_types(groups): """Build types list""" types = {} - for species, group_dict in groups.iteritems(): + for species, group_dict in list(groups.items()): types[species] = {} for group_name, _group_full_name, _family_name in group_dict: if phenotypes_exist(group_name): @@ -136,9 +134,9 @@ def build_types(species, group): def get_datasets(types): """Build datasets list""" datasets = {} - for species, group_dict in types.iteritems(): + for species, group_dict in list(types.items()): datasets[species] = {} - for group, type_list in group_dict.iteritems(): + for group, type_list in list(group_dict.items()): datasets[species][group] = {} for type_name in type_list: these_datasets = build_datasets(species, group, type_name[0]) diff --git a/wqflask/wqflask/api/mapping.py b/wqflask/wqflask/api/mapping.py index 92c27c9b..d59a69df 100644 --- a/wqflask/wqflask/api/mapping.py +++ b/wqflask/wqflask/api/mapping.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import string from base import data_set diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index 6324cabe..8e59ef27 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -1,16 +1,21 @@ # GN2 API -from __future__ import absolute_import, division, print_function +import os +import io +import csv +import json +import datetime +import requests -import os, io, csv, json, datetime, requests, yaml -import zlib from zipfile import ZipFile, ZIP_DEFLATED -import StringIO import flask -from flask import g, Response, request, make_response, render_template, send_from_directory, jsonify, redirect, send_file -import sqlalchemy +from flask import g +from flask import request +from flask import make_response +from flask import send_file + from wqflask import app from wqflask.api import correlation, mapping, gen_menu @@ -308,7 +313,7 @@ def fetch_traits(dataset_name, file_format = "json"): else: filename = dataset_name + "_trait_ids.csv" - si = StringIO.StringIO() + si = io.StringIO() csv_writer = csv.writer(si) csv_writer.writerows([[trait_id] for trait_id in trait_ids]) output = make_response(si.getvalue()) @@ -322,7 +327,7 @@ def fetch_traits(dataset_name, file_format = "json"): else: filename = dataset_name + "_trait_names.csv" - si = StringIO.StringIO() + si = io.StringIO() csv_writer = csv.writer(si) csv_writer.writerows([[trait_name] for trait_name in trait_names]) output = make_response(si.getvalue()) @@ -413,7 +418,7 @@ def fetch_traits(dataset_name, file_format = "json"): for result in g.db.execute(final_query).fetchall(): results_list.append(result) - si = StringIO.StringIO() + si = io.StringIO() csv_writer = csv.writer(si) csv_writer.writerows(results_list) output = make_response(si.getvalue()) @@ -517,9 +522,9 @@ def all_sample_data(dataset_name, file_format = "csv"): line_list.append("x") results_list.append(line_list) - results_list = map(list, zip(*results_list)) + results_list = list(map(list, zip(*results_list))) - si = StringIO.StringIO() + si = io.StringIO() csv_writer = csv.writer(si) csv_writer.writerows(results_list) output = make_response(si.getvalue()) @@ -558,10 +563,10 @@ def trait_sample_data(dataset_name, trait_name, file_format = "json"): sample_list = [] for sample in sample_data: sample_dict = { - "sample_name" : sample[0], - "sample_name_2" : sample[1], - "value" : sample[2], - "data_id" : sample[3], + "sample_name": sample[0], + "sample_name_2": sample[1], + "value": sample[2], + "data_id": sample[3], } if sample[4]: sample_dict["se"] = sample[4] @@ -706,7 +711,7 @@ def get_mapping_results(): if format == "csv": filename = "mapping_" + datetime.datetime.utcnow().strftime("%b_%d_%Y_%I:%M%p") + ".csv" - si = StringIO.StringIO() + si = io.StringIO() csv_writer = csv.writer(si) csv_writer.writerows(results) output = make_response(si.getvalue()) @@ -732,7 +737,7 @@ def get_genotypes(group_name, file_format="csv", dataset_name=None): if request.args['limit_to'].isdigit(): limit_num = int(request.args['limit_to']) - si = StringIO.StringIO() + si = io.StringIO() if file_format == "csv" or file_format == "geno": filename = group_name + ".geno" @@ -966,4 +971,4 @@ def get_group_id(group_name): if group_id: return group_id[0] else: - return None
\ No newline at end of file + return None diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 4c558bfe..e074a3d8 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -1,5 +1,4 @@ -from __future__ import print_function, division, absolute_import - +import hashlib import datetime import simplejson as json @@ -27,7 +26,9 @@ Redis = get_redis_conn() def process_traits(unprocessed_traits): - if isinstance(unprocessed_traits, basestring): + if isinstance(unprocessed_traits, bytes): + unprocessed_traits = unprocessed_traits.decode('utf-8').split(",") + else: # It's a string unprocessed_traits = unprocessed_traits.split(",") traits = set() for trait in unprocessed_traits: @@ -114,7 +115,8 @@ def collections_new(): g.user_session.add_traits_to_collection(collection_id, traits) return redirect(url_for('view_collection', uc_id=collection_id)) else: - CauseAnError + # CauseAnError + pass def create_new(collection_name): params = request.args @@ -182,7 +184,7 @@ def view_collection(): params = request.args uc_id = params['uc_id'] - uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next() + uc = next((collection for collection in g.user_session.user_collections if collection["id"] == uc_id)) traits = uc["members"] trait_obs = [] diff --git a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py index 09d6b9cc..92de6073 100644 --- a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py +++ b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py @@ -18,35 +18,16 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - -import sys -# sys.path.append(".") Never do this in a webserver! - -import string -import cPickle -import os -import time -import pp -import math -import collections -import resource - - from pprint import pformat as pf from base.trait import create_trait from base import data_set from utility import webqtlUtil, helper_functions, corr_result_helpers -from db import webqtlDatabaseFunction import utility.webqtlUtil #this is for parallel computing only. from wqflask.correlation import correlation_functions -from utility.benchmark import Bench from MySQLdb import escape_string as escape -from pprint import pformat as pf - from flask import Flask, g diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index 0f3d455c..6ab8c3d8 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import math from flask import g @@ -43,13 +41,13 @@ class CorrScatterPlot(object): samples_1, samples_2, num_overlap = corr_result_helpers.normalize_values_with_samples(self.trait_1.data, self.trait_2.data) self.data = [] - self.indIDs = samples_1.keys() + self.indIDs = list(samples_1.keys()) vals_1 = [] - for sample in samples_1.keys(): + for sample in list(samples_1.keys()): vals_1.append(samples_1[sample].value) self.data.append(vals_1) vals_2 = [] - for sample in samples_2.keys(): + for sample in list(samples_2.keys()): vals_2.append(samples_2[sample].value) self.data.append(vals_2) @@ -137,4 +135,4 @@ def get_intercept_coords(slope, intercept, x_range, y_range): intercept_coords.append([x1, y1]) intercept_coords.append([x2, y2]) - return intercept_coords
\ No newline at end of file + return intercept_coords diff --git a/wqflask/wqflask/correlation/correlation_functions.py b/wqflask/wqflask/correlation/correlation_functions.py index 06dec795..b883e361 100644 --- a/wqflask/wqflask/correlation/correlation_functions.py +++ b/wqflask/wqflask/correlation/correlation_functions.py @@ -24,8 +24,6 @@ # # Last updated by NL 2011/03/23 -from __future__ import absolute_import, print_function, division - import math import rpy2.robjects import string @@ -50,12 +48,12 @@ from flask import Flask, g def cal_zero_order_corr_for_tiss (primaryValue=[], targetValue=[], method='pearson'): - R_primary = rpy2.robjects.FloatVector(range(len(primaryValue))) + R_primary = rpy2.robjects.FloatVector(list(range(len(primaryValue)))) N = len(primaryValue) for i in range(len(primaryValue)): R_primary[i] = primaryValue[i] - R_target = rpy2.robjects.FloatVector(range(len(targetValue))) + R_target = rpy2.robjects.FloatVector(list(range(len(targetValue)))) for i in range(len(targetValue)): R_target[i]=targetValue[i] @@ -114,4 +112,4 @@ def get_trait_symbol_and_tissue_values(symbol_list=None): tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list) if len(tissue_data.gene_symbols): - return tissue_data.get_symbol_values_pairs()
\ No newline at end of file + return tissue_data.get_symbol_values_pairs() diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index de7a1c0c..e710bacd 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -18,51 +18,36 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - -import sys - -import string -import cPickle -import os -import time -import pp -import math import collections -import resource import json - import scipy import numpy import rpy2.robjects as ro # R Objects -import rpy2.rinterface as ri +import utility.logger +import utility.webqtlUtil -from rpy2.robjects.packages import importr -utils = importr("utils") +from base.trait import create_trait -from pprint import pformat as pf +from rpy2.robjects.packages import importr -from base import webqtlConfig -from utility.THCell import THCell -from utility.TDCell import TDCell -from base.trait import create_trait from base import data_set -from utility import webqtlUtil, helper_functions, corr_result_helpers, hmac -from db import webqtlDatabaseFunction -import utility.webqtlUtil #this is for parallel computing only. +from utility import helper_functions +from utility import corr_result_helpers +from utility import hmac + from wqflask.correlation import correlation_functions from utility.benchmark import Bench -import utility.webqtlUtil -from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string -from MySQLdb import escape_string as escape +from utility.type_checking import is_str +from utility.type_checking import get_float +from utility.type_checking import get_int +from utility.type_checking import get_string +from utility.db_tools import escape -from pprint import pformat as pf +from flask import g -from flask import Flask, g - -import utility.logger -logger = utility.logger.getLogger(__name__ ) +utils = importr("utils") +logger = utility.logger.getLogger(__name__) METHOD_LIT = "3" METHOD_TISSUE_PEARSON = "4" @@ -72,6 +57,7 @@ TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] TISSUE_MOUSE_DB = 1 + class CorrelationResults(object): def __init__(self, start_vars): # get trait list from db (database name) @@ -108,17 +94,17 @@ class CorrelationResults(object): self.sample_data = {} self.corr_type = start_vars['corr_type'] self.corr_method = start_vars['corr_sample_method'] - self.min_expr = get_float(start_vars,'min_expr') - self.p_range_lower = get_float(start_vars,'p_range_lower',-1.0) - self.p_range_upper = get_float(start_vars,'p_range_upper',1.0) + self.min_expr = get_float(start_vars, 'min_expr') + self.p_range_lower = get_float(start_vars, 'p_range_lower', -1.0) + self.p_range_upper = get_float(start_vars, 'p_range_upper', 1.0) if ('loc_chr' in start_vars and 'min_loc_mb' in start_vars and 'max_loc_mb' in start_vars): - self.location_chr = get_string(start_vars,'loc_chr') - self.min_location_mb = get_int(start_vars,'min_loc_mb') - self.max_location_mb = get_int(start_vars,'max_loc_mb') + self.location_chr = get_string(start_vars, 'loc_chr') + self.min_location_mb = get_int(start_vars, 'min_loc_mb') + self.max_location_mb = get_int(start_vars, 'max_loc_mb') else: self.location_chr = self.min_location_mb = self.max_location_mb = None @@ -145,10 +131,10 @@ class CorrelationResults(object): if corr_samples_group == 'samples_other': primary_samples = [x for x in primary_samples if x not in ( self.dataset.group.parlist + self.dataset.group.f1list)] - self.process_samples(start_vars, self.this_trait.data.keys(), primary_samples) + self.process_samples(start_vars, list(self.this_trait.data.keys()), primary_samples) self.target_dataset = data_set.create_dataset(start_vars['corr_dataset']) - self.target_dataset.get_trait_data(self.sample_data.keys()) + self.target_dataset.get_trait_data(list(self.sample_data.keys())) self.header_fields = get_header_fields(self.target_dataset.type, self.corr_method) @@ -168,41 +154,41 @@ class CorrelationResults(object): tissue_corr_data = self.do_tissue_correlation_for_all_traits() if tissue_corr_data != None: - for trait in tissue_corr_data.keys()[:self.return_number]: + for trait in list(tissue_corr_data.keys())[:self.return_number]: self.get_sample_r_and_p_values(trait, self.target_dataset.trait_data[trait]) else: - for trait, values in self.target_dataset.trait_data.iteritems(): + for trait, values in list(self.target_dataset.trait_data.items()): self.get_sample_r_and_p_values(trait, values) elif self.corr_type == "lit": self.trait_geneid_dict = self.dataset.retrieve_genes("GeneId") lit_corr_data = self.do_lit_correlation_for_all_traits() - for trait in lit_corr_data.keys()[:self.return_number]: + for trait in list(lit_corr_data.keys())[:self.return_number]: self.get_sample_r_and_p_values(trait, self.target_dataset.trait_data[trait]) elif self.corr_type == "sample": - for trait, values in self.target_dataset.trait_data.iteritems(): + for trait, values in list(self.target_dataset.trait_data.items()): self.get_sample_r_and_p_values(trait, values) - self.correlation_data = collections.OrderedDict(sorted(self.correlation_data.items(), + self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), key=lambda t: -abs(t[1][0]))) if self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Geno": #ZS: Convert min/max chromosome to an int for the location range option range_chr_as_int = None - for order_id, chr_info in self.dataset.species.chromosomes.chromosomes.iteritems(): + for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): if 'loc_chr' in start_vars: if chr_info.name == self.location_chr: range_chr_as_int = order_id - for _trait_counter, trait in enumerate(self.correlation_data.keys()[:self.return_number]): + for _trait_counter, trait in enumerate(list(self.correlation_data.keys())[:self.return_number]): trait_object = create_trait(dataset=self.target_dataset, name=trait, get_qtl_info=True, get_sample_info=False) if self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Geno": #ZS: Convert trait chromosome to an int for the location range option chr_as_int = 0 - for order_id, chr_info in self.dataset.species.chromosomes.chromosomes.iteritems(): + for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): if chr_info.name == trait_object.chr: chr_as_int = order_id @@ -297,14 +283,14 @@ class CorrelationResults(object): #print("trait_gene_symbols: ", pf(trait_gene_symbols.values())) corr_result_tissue_vals_dict= correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=self.trait_symbol_dict.values()) + symbol_list=list(self.trait_symbol_dict.values())) #print("corr_result_tissue_vals: ", pf(corr_result_tissue_vals_dict)) #print("trait_gene_symbols: ", pf(trait_gene_symbols)) tissue_corr_data = {} - for trait, symbol in self.trait_symbol_dict.iteritems(): + for trait, symbol in list(self.trait_symbol_dict.items()): if symbol and symbol.lower() in corr_result_tissue_vals_dict: this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower()] @@ -314,7 +300,7 @@ class CorrelationResults(object): tissue_corr_data[trait] = [symbol, result[0], result[2]] - tissue_corr_data = collections.OrderedDict(sorted(tissue_corr_data.items(), + tissue_corr_data = collections.OrderedDict(sorted(list(tissue_corr_data.items()), key=lambda t: -abs(t[1][1]))) return tissue_corr_data @@ -359,7 +345,7 @@ class CorrelationResults(object): input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), self.this_trait.geneid) lit_corr_data = {} - for trait, gene_id in self.trait_geneid_dict.iteritems(): + for trait, gene_id in list(self.trait_geneid_dict.items()): mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), gene_id) if mouse_gene_id and str(mouse_gene_id).find(";") == -1: @@ -387,7 +373,7 @@ class CorrelationResults(object): else: lit_corr_data[trait] = [gene_id, 0] - lit_corr_data = collections.OrderedDict(sorted(lit_corr_data.items(), + lit_corr_data = collections.OrderedDict(sorted(list(lit_corr_data.items()), key=lambda t: -abs(t[1][1]))) return lit_corr_data @@ -648,4 +634,4 @@ def get_header_fields(data_type, corr_method): 'N', 'Sample p(r)'] - return header_fields
\ No newline at end of file + return header_fields diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 3beee84f..a394f548 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -18,52 +18,29 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - -import sys -# sys.path.append(".") Never do this in a webserver! - -import string -import cPickle -import os import datetime -import time -import pp import math -import collections -import resource - import numpy as np import scipy - -from rpy2.robjects.packages import importr import rpy2.robjects as robjects +import utility.webqtlUtil # this is for parallel computing only. +import utility.logger -from pprint import pformat as pf - -from utility.redis_tools import get_redis_conn -Redis = get_redis_conn() -THIRTY_DAYS = 60 * 60 * 24 * 30 - -from utility.THCell import THCell -from utility.TDCell import TDCell -from base.trait import GeneralTrait from base import data_set -from utility import webqtlUtil, helper_functions, corr_result_helpers - -from db import webqtlDatabaseFunction -import utility.webqtlUtil #this is for parallel computing only. -from wqflask.correlation import correlation_functions -from utility.benchmark import Bench +from functools import reduce +from functools import cmp_to_key +from rpy2.robjects.packages import importr -from MySQLdb import escape_string as escape +from utility import webqtlUtil +from utility import helper_functions +from utility import corr_result_helpers +from utility.redis_tools import get_redis_conn -from pprint import pformat as pf +logger = utility.logger.getLogger(__name__) -from flask import Flask, g, url_for +Redis = get_redis_conn() +THIRTY_DAYS = 60 * 60 * 24 * 30 -import utility.logger -logger = utility.logger.getLogger(__name__ ) class CorrelationMatrix(object): @@ -190,7 +167,7 @@ class CorrelationMatrix(object): if self.do_PCA == True: self.pca_works = "True" self.pca_trait_ids = [] - pca = self.calculate_pca(range(len(self.traits)), corr_eigen_value, corr_eigen_vectors) + pca = self.calculate_pca(list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors) self.loadings_array = self.process_loadings() else: self.pca_works = "False" @@ -199,8 +176,8 @@ class CorrelationMatrix(object): self.js_data = dict(traits = [trait.name for trait in self.traits], groups = groups, - cols = range(len(self.traits)), - rows = range(len(self.traits)), + cols = list(range(len(self.traits))), + rows = list(range(len(self.traits))), samples = self.all_sample_list, sample_data = self.sample_data,) # corr_results = [result[1] for result in result_row for result_row in self.corr_results]) @@ -271,14 +248,14 @@ def zScore(trait_data_array): i = 0 for data in trait_data_array: N = len(data) - S = reduce(lambda x,y: x+y, data, 0.) - SS = reduce(lambda x,y: x+y*y, data, 0.) + S = reduce(lambda x, y: x+y, data, 0.) + SS = reduce(lambda x, y: x+y*y, data, 0.) mean = S/N var = SS - S*S/N stdev = math.sqrt(var/(N-1)) if stdev == 0: stdev = 1e-100 - data2 = map(lambda x:(x-mean)/stdev,data) + data2 = [(x-mean)/stdev for x in data] trait_data_array[i] = data2 i += 1 return trait_data_array @@ -290,16 +267,16 @@ def sortEigenVectors(vector): combines = [] i = 0 for item in eigenValues: - combines.append([eigenValues[i],eigenVectors[i]]) + combines.append([eigenValues[i], eigenVectors[i]]) i += 1 - combines.sort(webqtlUtil.cmpEigenValue) + sorted(combines, key=cmp_to_key(webqtlUtil.cmpEigenValue)) A = [] B = [] for item in combines: A.append(item[0]) B.append(item[1]) - sum = reduce(lambda x,y: x+y, A, 0.0) - A = map(lambda x:x*100.0/sum, A) + sum = reduce(lambda x, y: x+y, A, 0.0) + A = [x*100.0/sum for x in A] return [A, B] except: - return []
\ No newline at end of file + return [] diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 35067036..72b4f3a3 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -125,7 +125,7 @@ class CTL(object): gt = create_trait(name = ts[0], dataset_name = ts[1]) gt = retrieve_sample_data(gt, dataset, individuals) for ind in individuals: - if ind in gt.data.keys(): + if ind in list(gt.data.keys()): traits.append(gt.data[ind].value) else: traits.append("-999") @@ -143,7 +143,7 @@ class CTL(object): #r_write_table(rPheno, "~/outputGN/pheno.csv") # Perform the CTL scan - res = self.r_CTLscan(rGeno, rPheno, strategy = strategy, nperm = nperm, parametric = parametric, ncores = 6) + res = self.r_CTLscan(rGeno, rPheno, strategy = strategy, nperm = nperm, parametric = parametric, nthreads=6) # Get significant interactions significant = self.r_CTLsignificant(res, significance = significance) @@ -175,7 +175,7 @@ class CTL(object): sys.stdout.flush() # Create the interactive graph for cytoscape visualization (Nodes and Edges) - if not type(significant) == ri.RNULLType: + if not isinstance(significant, ri.RNULLType): for x in range(len(significant[0])): logger.debug(significant[0][x], significant[1][x], significant[2][x]) # Debug to console tsS = significant[0][x].split(':') # Source @@ -186,15 +186,15 @@ class CTL(object): self.addNode(gtT) self.addEdge(gtS, gtT, significant, x) - significant[0][x] = gtS.symbol + " (" + gtS.name + ")" # Update the trait name for the displayed table - significant[2][x] = gtT.symbol + " (" + gtT.name + ")" # Update the trait name for the displayed table + significant[0][x] = "{} ({})".format(gtS.symbol, gtS.name) # Update the trait name for the displayed table + significant[2][x] = "{} ({})".format(gtT.symbol, gtT.name) # Update the trait name for the displayed table self.elements = json.dumps(self.nodes_list + self.edges_list) def loadImage(self, path, name): imgfile = open(self.results[path], 'rb') imgdata = imgfile.read() - imgB64 = imgdata.encode("base64") + imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) self.results[name] = bytesarray diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index f04e38bf..f420b472 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -1,127 +1,138 @@ -import httplib, urllib2
-import re
-
-from flask import Flask, g
-
-from utility.logger import getLogger
-logger = getLogger(__name__ )
-
-class InfoPage(object):
- def __init__(self, start_vars):
- self.info = None
- self.gn_accession_id = None
- if 'gn_accession_id' in start_vars:
- self.gn_accession_id = start_vars['gn_accession_id']
- self.info_page_name = start_vars['info_page_name']
-
- self.get_info()
- self.get_datasets_list()
-
- def get_info(self, create=False):
- query_base = ("SELECT InfoPageName, GN_AccesionId, Species.MenuName, Species.TaxonomyId, Tissue.Name, InbredSet.Name, " +
- "GeneChip.GeneChipName, GeneChip.GeoPlatform, AvgMethod.Name, Datasets.DatasetName, Datasets.GeoSeries, " +
- "Datasets.PublicationTitle, DatasetStatus.DatasetStatusName, Datasets.Summary, Datasets.AboutCases, " +
- "Datasets.AboutTissue, Datasets.AboutDataProcessing, Datasets.Acknowledgment, Datasets.ExperimentDesign, " +
- "Datasets.Contributors, Datasets.Citation, Datasets.Notes, Investigators.FirstName, Investigators.LastName, " +
- "Investigators.Address, Investigators.City, Investigators.State, Investigators.ZipCode, Investigators.Country, " +
- "Investigators.Phone, Investigators.Email, Investigators.Url, Organizations.OrganizationName, " +
- "InvestigatorId, DatasetId, DatasetStatusId, Datasets.AboutPlatform, InfoFileTitle, Specifics " +
- "FROM InfoFiles " +
- "LEFT JOIN Species USING (SpeciesId) " +
- "LEFT JOIN Tissue USING (TissueId) " +
- "LEFT JOIN InbredSet USING (InbredSetId) " +
- "LEFT JOIN GeneChip USING (GeneChipId) " +
- "LEFT JOIN AvgMethod USING (AvgMethodId) " +
- "LEFT JOIN Datasets USING (DatasetId) " +
- "LEFT JOIN Investigators USING (InvestigatorId) " +
- "LEFT JOIN Organizations USING (OrganizationId) " +
- "LEFT JOIN DatasetStatus USING (DatasetStatusId) WHERE ")
-
- if self.gn_accession_id:
- final_query = query_base + "GN_AccesionId = {}".format(self.gn_accession_id)
- results = g.db.execute(final_query).fetchone()
- if self.info_page_name and not results:
- final_query = query_base + "InfoPageName={}".format(self.info_page_name)
- elif self.info_page_name:
- final_query = query_base + "InfoPageName={}".format(self.info_page_name)
- results = g.db.execute(final_query).fetchone()
- else:
- raise 'No correct parameter found'
-
- if results:
- self.info = process_query_results(results)
-
- if (not results or len(results) < 1) and self.info_page_name and create:
- insert_sql = "INSERT INTO InfoFiles SET InfoFiles.InfoPageName={}".format(self.info_page_name)
- return self.get_info()
-
- if not self.gn_accession_id and self.info:
- self.gn_accession_id = self.info['accession_id']
- if not self.info_page_name and self.info:
- self.info_page_name = self.info['info_page_name']
-
- def get_datasets_list(self):
- self.filelist = []
- try:
- response = urllib2.urlopen("http://datafiles.genenetwork.org/download/GN%s" % self.gn_accession_id)
- data = response.read()
-
- matches = re.findall(r"<tr>.+?</tr>", data, re.DOTALL)
- for i, match in enumerate(matches):
- if i == 0:
- continue
- cells = re.findall(r"<td.+?>.+?</td>", match, re.DOTALL)
- full_filename = re.search(r"<a href=\"(.+?)\"", cells[1], re.DOTALL).group(1).strip()
- filename = full_filename.split("/")[-1]
- filesize = re.search(r">(.+?)<", cells[2]).group(1).strip()
- filedate = "N/A" #ZS: Since we can't get it for now
-
- self.filelist.append([filename, filedate, filesize])
- except Exception, e:
- pass
-
-def process_query_results(results):
- info_ob = {
- 'info_page_name': results[0],
- 'accession_id': results[1],
- 'menu_name': results[2],
- 'taxonomy_id': results[3],
- 'tissue_name': results[4],
- 'group_name': results[5],
- 'gene_chip_name': results[6],
- 'geo_platform': results[7],
- 'avg_method_name': results[8],
- 'dataset_name': results[9],
- 'geo_series': results[10],
- 'publication_title': results[11],
- 'dataset_status_name': results[12],
- 'dataset_summary': results[13],
- 'about_cases': results[14],
- 'about_tissue': results[15],
- 'about_data_processing': results[16],
- 'acknowledgement': results[17],
- 'experiment_design': results[18],
- 'contributors': results[19],
- 'citation': results[20],
- 'notes': results[21],
- 'investigator_firstname': results[22],
- 'investigator_lastname': results[23],
- 'investigator_address': results[24],
- 'investigator_city': results[25],
- 'investigator_state': results[26],
- 'investigator_zipcode': results[27],
- 'investigator_country': results[28],
- 'investigator_phone': results[29],
- 'investigator_email': results[30],
- 'investigator_url': results[31],
- 'organization_name': results[32],
- 'investigator_id': results[33],
- 'dataset_id': results[34],
- 'dataset_status_is': results[35],
- 'about_platform': results[36],
- 'info_file_title': results[37],
- 'specifics': results[38]
- }
-
- return info_ob
-
\ No newline at end of file +import http.client +import urllib.request +import urllib.error +import urllib.parse +import re + +from flask import Flask, g + +from utility.logger import getLogger +logger = getLogger(__name__) + + +class InfoPage(object): + def __init__(self, start_vars): + self.info = None + self.gn_accession_id = None + if 'gn_accession_id' in start_vars: + self.gn_accession_id = start_vars['gn_accession_id'] + self.info_page_name = start_vars['info_page_name'] + + self.get_info() + self.get_datasets_list() + + def get_info(self, create=False): + query_base = ("SELECT InfoPageName, GN_AccesionId, Species.MenuName, Species.TaxonomyId, Tissue.Name, InbredSet.Name, " + + "GeneChip.GeneChipName, GeneChip.GeoPlatform, AvgMethod.Name, Datasets.DatasetName, Datasets.GeoSeries, " + + "Datasets.PublicationTitle, DatasetStatus.DatasetStatusName, Datasets.Summary, Datasets.AboutCases, " + + "Datasets.AboutTissue, Datasets.AboutDataProcessing, Datasets.Acknowledgment, Datasets.ExperimentDesign, " + + "Datasets.Contributors, Datasets.Citation, Datasets.Notes, Investigators.FirstName, Investigators.LastName, " + + "Investigators.Address, Investigators.City, Investigators.State, Investigators.ZipCode, Investigators.Country, " + + "Investigators.Phone, Investigators.Email, Investigators.Url, Organizations.OrganizationName, " + + "InvestigatorId, DatasetId, DatasetStatusId, Datasets.AboutPlatform, InfoFileTitle, Specifics " + + "FROM InfoFiles " + + "LEFT JOIN Species USING (SpeciesId) " + + "LEFT JOIN Tissue USING (TissueId) " + + "LEFT JOIN InbredSet USING (InbredSetId) " + + "LEFT JOIN GeneChip USING (GeneChipId) " + + "LEFT JOIN AvgMethod USING (AvgMethodId) " + + "LEFT JOIN Datasets USING (DatasetId) " + + "LEFT JOIN Investigators USING (InvestigatorId) " + + "LEFT JOIN Organizations USING (OrganizationId) " + + "LEFT JOIN DatasetStatus USING (DatasetStatusId) WHERE ") + + if self.gn_accession_id: + final_query = query_base + \ + "GN_AccesionId = {}".format(self.gn_accession_id) + results = g.db.execute(final_query).fetchone() + if self.info_page_name and not results: + final_query = query_base + \ + "InfoPageName={}".format(self.info_page_name) + elif self.info_page_name: + final_query = query_base + \ + "InfoPageName={}".format(self.info_page_name) + results = g.db.execute(final_query).fetchone() + else: + raise 'No correct parameter found' + + if results: + self.info = process_query_results(results) + + if (not results or len(results) < 1) and self.info_page_name and create: + insert_sql = "INSERT INTO InfoFiles SET InfoFiles.InfoPageName={}".format( + self.info_page_name) + return self.get_info() + + if not self.gn_accession_id and self.info: + self.gn_accession_id = self.info['accession_id'] + if not self.info_page_name and self.info: + self.info_page_name = self.info['info_page_name'] + + def get_datasets_list(self): + self.filelist = [] + try: + response = urllib.request.urlopen( + "http://datafiles.genenetwork.org/download/GN%s" % self.gn_accession_id) + data = response.read() + + matches = re.findall(r"<tr>.+?</tr>", data, re.DOTALL) + for i, match in enumerate(matches): + if i == 0: + continue + cells = re.findall(r"<td.+?>.+?</td>", match, re.DOTALL) + full_filename = re.search( + r"<a href=\"(.+?)\"", cells[1], re.DOTALL).group(1).strip() + filename = full_filename.split("/")[-1] + filesize = re.search(r">(.+?)<", cells[2]).group(1).strip() + filedate = "N/A" # ZS: Since we can't get it for now + + self.filelist.append([filename, filedate, filesize]) + except Exception as e: + pass + +def process_query_results(results): + info_ob = { + 'info_page_name': results[0], + 'accession_id': results[1], + 'menu_name': results[2], + 'taxonomy_id': results[3], + 'tissue_name': results[4], + 'group_name': results[5], + 'gene_chip_name': results[6], + 'geo_platform': results[7], + 'avg_method_name': results[8], + 'dataset_name': results[9], + 'geo_series': results[10], + 'publication_title': results[11], + 'dataset_status_name': results[12], + 'dataset_summary': results[13], + 'about_cases': results[14], + 'about_tissue': results[15], + 'about_data_processing': results[16], + 'acknowledgement': results[17], + 'experiment_design': results[18], + 'contributors': results[19], + 'citation': results[20], + 'notes': results[21], + 'investigator_firstname': results[22], + 'investigator_lastname': results[23], + 'investigator_address': results[24], + 'investigator_city': results[25], + 'investigator_state': results[26], + 'investigator_zipcode': results[27], + 'investigator_country': results[28], + 'investigator_phone': results[29], + 'investigator_email': results[30], + 'investigator_url': results[31], + 'organization_name': results[32], + 'investigator_id': results[33], + 'dataset_id': results[34], + 'dataset_status_is': results[35], + 'about_platform': results[36], + 'info_file_title': results[37], + 'specifics': results[38] + } + + return info_ob + + diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 1e15d28f..00636563 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -1,16 +1,13 @@ -from __future__ import print_function, division - import string import requests import json from flask import Flask, g -from MySQLdb import escape_string as escape +from utility.db_tools import escape from pprint import pformat as pf import sys -# sys.path.append("..") Never in a running webserver from db import webqtlDatabaseFunction from utility.tools import GN2_BASE_URL @@ -19,6 +16,7 @@ import logging from utility.logger import getLogger logger = getLogger(__name__) + class DoSearch(object): """Parent class containing parameters/functions used for all searches""" @@ -46,8 +44,8 @@ class DoSearch(object): def handle_wildcard(self, str): keyword = str.strip() - keyword = keyword.replace("*",".*") - keyword = keyword.replace("?",".") + keyword = keyword.replace("*", ".*") + keyword = keyword.replace("?", ".") return keyword diff --git a/wqflask/wqflask/docs.py b/wqflask/wqflask/docs.py index 9fad1cf1..d653c269 100644 --- a/wqflask/wqflask/docs.py +++ b/wqflask/wqflask/docs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import codecs from flask import g @@ -42,4 +40,4 @@ def update_text(start_vars): sql = "UPDATE Docs SET content='{0}' WHERE entry='{1}';".format(content, start_vars['entry_type']) g.db.execute(sql) except: - pass
\ No newline at end of file + pass diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 3272c03d..3a886537 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -1,8 +1,6 @@ -from __future__ import print_function, division - import csv import xlsxwriter -import StringIO +import io import datetime import itertools @@ -61,9 +59,9 @@ def export_search_results_csv(targs): traits_by_group = sort_traits_by_group(trait_list) file_list = [] - for group in traits_by_group.keys(): + for group in list(traits_by_group.keys()): group_traits = traits_by_group[group] - buff = StringIO.StringIO() + buff = io.StringIO() writer = csv.writer(buff) csv_rows = [] @@ -122,7 +120,7 @@ def export_search_results_csv(targs): csv_rows.append(row_contents) - csv_rows = map(list, itertools.izip_longest(*[row for row in csv_rows])) + csv_rows = list(map(list, itertools.zip_longest(*[row for row in csv_rows]))) writer.writerows(csv_rows) csv_data = buff.getvalue() buff.close() @@ -135,9 +133,9 @@ def export_search_results_csv(targs): def sort_traits_by_group(trait_list=[]): traits_by_group = {} for trait in trait_list: - if trait.dataset.group.name not in traits_by_group.keys(): + if trait.dataset.group.name not in list(traits_by_group.keys()): traits_by_group[trait.dataset.group.name] = [] traits_by_group[trait.dataset.group.name].append(trait) - return traits_by_group
\ No newline at end of file + return traits_by_group diff --git a/wqflask/wqflask/external_tools/send_to_bnw.py b/wqflask/wqflask/external_tools/send_to_bnw.py index 68efd10d..efa17f05 100644 --- a/wqflask/wqflask/external_tools/send_to_bnw.py +++ b/wqflask/wqflask/external_tools/send_to_bnw.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - from base.trait import GeneralTrait from utility import helper_functions, corr_result_helpers @@ -69,4 +67,4 @@ class SendToBNW(object): if has_none: continue self.form_value += ",".join(str(cell) for cell in row) - self.form_value += ";"
\ No newline at end of file + self.form_value += ";" diff --git a/wqflask/wqflask/external_tools/send_to_geneweaver.py b/wqflask/wqflask/external_tools/send_to_geneweaver.py index 7a5dba73..4c958a88 100644 --- a/wqflask/wqflask/external_tools/send_to_geneweaver.py +++ b/wqflask/wqflask/external_tools/send_to_geneweaver.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - import string from flask import Flask, g @@ -54,10 +52,10 @@ class SendToGeneWeaver(object): trait_name_list = get_trait_name_list(self.trait_list) self.hidden_vars = { - 'client' : "genenetwork", - 'species' : species_name, - 'idtype' : self.chip_name, - 'list' : string.join(trait_name_list, ","), + 'client': "genenetwork", + 'species': species_name, + 'idtype': self.chip_name, + 'list': ",".join(trait_name_list), } def get_trait_name_list(trait_list): @@ -109,4 +107,4 @@ def test_chip(trait_list): chip_name = '%s_NA' % result[0] return chip_name - return chip_name
\ No newline at end of file + return chip_name diff --git a/wqflask/wqflask/external_tools/send_to_webgestalt.py b/wqflask/wqflask/external_tools/send_to_webgestalt.py index 30ca024f..2f068792 100644 --- a/wqflask/wqflask/external_tools/send_to_webgestalt.py +++ b/wqflask/wqflask/external_tools/send_to_webgestalt.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - import string from flask import Flask, g @@ -49,7 +47,7 @@ class SendToWebGestalt(object): id_type = "entrezgene" self.hidden_vars = { - 'gene_list' : string.join(gene_id_list, "\n"), + 'gene_list' : "\n".join(gene_id_list), 'id_type' : "entrezgene", 'ref_set' : "genome", 'enriched_database_category' : "geneontology", @@ -123,4 +121,4 @@ def gen_gene_id_list(trait_list): trait_name_list.append(trait.name) retrieve_trait_info(trait, trait.dataset) gene_id_list.append(str(trait.geneid)) - return trait_name_list, gene_id_list
\ No newline at end of file + return trait_name_list, gene_id_list diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index 99d5db26..69ee9623 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -1,6 +1,3 @@ - -from __future__ import print_function, division, absolute_import - import random, string from flask import (Flask, g, render_template, url_for, request, make_response, @@ -155,4 +152,4 @@ def send_group_invites(group_id, user_email_list = [], user_type="members"): save_user(user_details, user_details['user_id']) send_invitation_email(user_email, temp_password) -#@app.route()
\ No newline at end of file +#@app.route() diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index c65a1415..9e46c0c5 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import json from flask import Flask, g @@ -37,7 +35,7 @@ class GSearch(object): ProbeSetFreeze.FullName AS probesetfreeze_fullname, ProbeSet.Name AS probeset_name, ProbeSet.Symbol AS probeset_symbol, - ProbeSet.`description` AS probeset_description, + CAST(ProbeSet.`description` AS BINARY) AS probeset_description, ProbeSet.Chr AS chr, ProbeSet.Mb AS mb, ProbeSetXRef.Mean AS mean, @@ -137,8 +135,8 @@ class GSearch(object): PublishFreeze.`Name`, PublishFreeze.`FullName`, PublishXRef.`Id`, - Phenotype.`Pre_publication_description`, - Phenotype.`Post_publication_description`, + CAST(Phenotype.`Pre_publication_description` AS BINARY), + CAST(Phenotype.`Post_publication_description` AS BINARY), Publication.`Authors`, Publication.`Year`, Publication.`PubMed_ID`, diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 5098a184..cca5a4fc 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -1,46 +1,17 @@ -from __future__ import absolute_import, print_function, division - -import sys -# sys.path.append(".") Never in a running webserver - import string -import cPickle import os -import datetime -import time -import pp -import math import random -import collections -import resource - -import scipy -import numpy as np - -from pprint import pformat as pf - -from base.trait import GeneralTrait -from base import data_set from base import species from base import webqtlConfig from utility import helper_functions -from utility import Plot, Bunch -from utility import temp_data -from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR - -from MySQLdb import escape_string as escape - -import cPickle as pickle -import simplejson as json - -from pprint import pformat as pf +from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR from redis import Redis -Redis = Redis() - from flask import Flask, g - from utility.logger import getLogger + +Redis = Redis() + logger = getLogger(__name__ ) class Heatmap(object): @@ -60,7 +31,7 @@ class Heatmap(object): chrnames = [] self.species = species.TheSpecies(dataset=self.trait_list[0][1]) - for key in self.species.chromosomes.chromosomes.keys(): + for key in list(self.species.chromosomes.chromosomes.keys()): chrnames.append([self.species.chromosomes.chromosomes[key].name, self.species.chromosomes.chromosomes[key].mb_length]) for trait_db in self.trait_list: @@ -93,7 +64,7 @@ class Heatmap(object): pos = [] markernames = [] - for trait in self.trait_results.keys(): + for trait in list(self.trait_results.keys()): lodnames.append(trait) self.dataset.group.get_markers() @@ -205,4 +176,4 @@ def parse_reaper_output(gwa_filename): marker['additive'] = float(line.split("\t")[6]) marker_obs.append(marker) - return marker_obs
\ No newline at end of file + return marker_obs diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index 2c60dd70..d0dd7aea 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import string from flask import Flask, g @@ -24,7 +22,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): ##List current Species and other Species speciesId = speciesDict[species] - otherSpecies = map(lambda X: [X, speciesDict[X]], speciesDict.keys()) + otherSpecies = [[X, speciesDict[X]] for X in list(speciesDict.keys())] otherSpecies.remove([species, speciesId]) results = g.db.execute(""" @@ -33,7 +31,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): Chromosome = '%s' AND ((TxStart > %f and TxStart <= %f) OR (TxEnd > %f and TxEnd <= %f)) ORDER BY txStart - """ % (string.join(fetchFields, ", "), + """ % (", ".join(fetchFields), speciesId, chrName, startMb, endMb, startMb, endMb)).fetchall() @@ -68,7 +66,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): othSpec, othSpecId = item newdict2 = {} - resultsOther = g.db.execute("SELECT %s FROM GeneList WHERE SpeciesId = %d AND geneSymbol= '%s' LIMIT 1" % (string.join(fetchFields, ", "), + resultsOther = g.db.execute("SELECT %s FROM GeneList WHERE SpeciesId = %d AND geneSymbol= '%s' LIMIT 1" % (", ".join(fetchFields), othSpecId, newdict["GeneSymbol"])).fetchone() diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index de751907..2fc633aa 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -35,7 +35,7 @@ import json from flask import Flask, g -from htmlgen import HTMLgen2 as HT +import htmlgen as HT from base import webqtlConfig from base.GeneralObject import GeneralObject @@ -47,7 +47,11 @@ from base.webqtlConfig import GENERATED_IMAGE_DIR from utility.pillow_utils import draw_rotated_text, draw_open_polygon import utility.logger -logger = utility.logger.getLogger(__name__) +try: # Only import this for Python3 + from functools import reduce +except: + pass +logger = utility.logger.getLogger(__name__ ) RED = ImageColor.getrgb("red") BLUE = ImageColor.getrgb("blue") @@ -84,35 +88,60 @@ class HtmlGenWrapper: """Wrapper Methods for HTML gen""" @staticmethod def create_image_tag(**kwargs): - return HT.Image(**kwargs) + image = HT.Image("", "") + for key, value in list(kwargs.items()): + image.set_attribute(key, value) + return image @staticmethod def create_form_tag(**kwargs): - return HT.Form(**kwargs) + form = HT.Form("POST", "") # Default method is POST + + for key, value in list(kwargs.items()): + if key == "submit": + form.append(value) + continue + form.set_attribute(key.replace("cgi", "action"), str(value)) + return form @staticmethod def create_p_tag(**kwargs): - return HT.Paragraph(**kwargs) + paragraph = HT.Paragraph() + for key, value in list(kwargs.items()): + paragraph.set_attribute(key, value) + return paragraph @staticmethod def create_br_tag(): - return HT.BR() + return HT.VoidElement("br") @staticmethod def create_input_tag(**kwargs): - return HT.Input(**kwargs) + input_ = HT.Input() + for key, value in list(kwargs.items()): + input_.set_attribute(key.lower().replace("_", ""), value) + return input_ @staticmethod def create_area_tag(**kwargs): - return HT.Area(**kwargs) + area = HT.VoidElement("area") + for key, value in list(kwargs.items()): + area.set_attribute(key, value) + return area @staticmethod def create_link_tag(href, content, **kwargs): - return HT.Href(href, content, **kwargs) + link = HT.Link(href, content) + for key, value in list(kwargs.items()): + link.set_attribute(key, value) + return link @staticmethod def create_map_tag(**kwargs): - return HT.Map(**kwargs) + map_ = HT.Element("map") + for key, value in list(kwargs.items()): + map_.set_attribute(key, value) + return map_ class DisplayMappingResults(object): @@ -265,7 +294,7 @@ class DisplayMappingResults(object): self.manhattan_plot = start_vars['manhattan_plot'] - if 'permCheck' in start_vars.keys(): + if 'permCheck' in list(start_vars.keys()): self.permChecked = start_vars['permCheck'] else: self.permChecked = False @@ -278,46 +307,46 @@ class DisplayMappingResults(object): else: self.nperm = 0 - if 'bootCheck' in start_vars.keys(): + if 'bootCheck' in list(start_vars.keys()): self.bootChecked = start_vars['bootCheck'] else: self.bootChecked = False - if 'num_bootstrap' in start_vars.keys(): + if 'num_bootstrap' in list(start_vars.keys()): self.nboot = int(start_vars['num_bootstrap']) else: self.nboot = 0 - if 'bootstrap_results' in start_vars.keys(): + if 'bootstrap_results' in list(start_vars.keys()): self.bootResult = start_vars['bootstrap_results'] else: self.bootResult = [] - if 'do_control' in start_vars.keys(): + if 'do_control' in list(start_vars.keys()): self.doControl = start_vars['do_control'] else: self.doControl = "false" - if 'control_marker' in start_vars.keys(): + if 'control_marker' in list(start_vars.keys()): self.controlLocus = start_vars['control_marker'] else: self.controlLocus = "" - if 'covariates' in start_vars.keys(): + if 'covariates' in list(start_vars.keys()): self.covariates = start_vars['covariates'] - if 'maf' in start_vars.keys(): + if 'maf' in list(start_vars.keys()): self.maf = start_vars['maf'] else: self.maf = "" - if 'output_files' in start_vars.keys(): + if 'output_files' in list(start_vars.keys()): self.output_files = start_vars['output_files'] - if 'use_loco' in start_vars.keys() and self.mapping_method == "gemma": + if 'use_loco' in list(start_vars.keys()) and self.mapping_method == "gemma": self.use_loco = start_vars['use_loco'] - if 'reaper_version' in start_vars.keys() and self.mapping_method == "reaper": + if 'reaper_version' in list(start_vars.keys()) and self.mapping_method == "reaper": self.reaper_version = start_vars['reaper_version'] if 'output_files' in start_vars: self.output_files = ",".join(start_vars['output_files']) self.categorical_vars = "" self.perm_strata = "" - if 'perm_strata' in start_vars.keys() and 'categorical_vars' in start_vars.keys(): + if 'perm_strata' in list(start_vars.keys()) and 'categorical_vars' in list(start_vars.keys()): self.categorical_vars = start_vars['categorical_vars'] self.perm_strata = start_vars['perm_strata'] @@ -359,7 +388,7 @@ class DisplayMappingResults(object): self.graphWidth = self.MULT_GRAPH_DEFAULT_WIDTH ## BEGIN HaplotypeAnalyst - if 'haplotypeAnalystCheck' in start_vars.keys(): + if 'haplotypeAnalystCheck' in list(start_vars.keys()): self.haplotypeAnalystChecked = start_vars['haplotypeAnalystCheck'] else: self.haplotypeAnalystChecked = False @@ -367,25 +396,25 @@ class DisplayMappingResults(object): self.graphHeight = self.GRAPH_DEFAULT_HEIGHT self.dominanceChecked = False - if 'LRSCheck' in start_vars.keys(): + if 'LRSCheck' in list(start_vars.keys()): self.LRS_LOD = start_vars['LRSCheck'] else: self.LRS_LOD = start_vars['score_type'] self.intervalAnalystChecked = True self.draw2X = False - if 'additiveCheck' in start_vars.keys(): + if 'additiveCheck' in list(start_vars.keys()): self.additiveChecked = start_vars['additiveCheck'] else: self.additiveChecked = False - if 'viewLegend' in start_vars.keys(): + if 'viewLegend' in list(start_vars.keys()): self.legendChecked = start_vars['viewLegend'] else: self.legendChecked = False - if 'showSNP' in start_vars.keys(): + if 'showSNP' in list(start_vars.keys()): self.SNPChecked = start_vars['showSNP'] else: self.SNPChecked = False - if 'showGenes' in start_vars.keys(): + if 'showGenes' in list(start_vars.keys()): self.geneChecked = start_vars['showGenes'] else: self.geneChecked = False @@ -425,9 +454,9 @@ class DisplayMappingResults(object): Chr_Length.Name in (%s) Order by Chr_Length.OrderId - """ % (self.dataset.group.name, string.join(map(lambda X: "'%s'" % X[0], self.ChrList[1:]), ", "))) + """ % (self.dataset.group.name, ", ".join(["'%s'" % X[0] for X in self.ChrList[1:]]))) - self.ChrLengthMbList = map(lambda x: x[0]/1000000.0, self.ChrLengthMbList) + self.ChrLengthMbList = [x[0]/1000000.0 for x in self.ChrLengthMbList] self.ChrLengthMbSum = reduce(lambda x, y:x+y, self.ChrLengthMbList, 0.0) if self.ChrLengthMbList: self.MbGraphInterval = self.ChrLengthMbSum/(len(self.ChrLengthMbList)*12) #Empirical Mb interval @@ -458,7 +487,7 @@ class DisplayMappingResults(object): else: continue samplelist = list(self.genotype.prgy) - for j,_geno in enumerate (self.genotype[0][1].genotype): + for j, _geno in enumerate (self.genotype[0][1].genotype): for item in smd: if item.name == samplelist[j]: self.NR_INDIVIDUALS = self.NR_INDIVIDUALS + 1 @@ -550,11 +579,10 @@ class DisplayMappingResults(object): src="/image/{}.png".format(self.filename), border="0", usemap='#WebQTLImageMap' ) - self.intImg = intImg #Scales plot differently for high resolution if self.draw2X: - intCanvasX2 = Image.new("RGBA", size=(self.graphWidth*2,self.graphHeight*2)) + intCanvasX2 = Image.new("RGBA", size=(self.graphWidth*2, self.graphHeight*2)) gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2) intCanvasX2.save( "{}.png".format( @@ -571,12 +599,12 @@ class DisplayMappingResults(object): cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name=showLocusForm, - submit=HtmlGenWrapper.create_input_tag(type='hidden')) + submit=HtmlGenWrapper.create_input_tag(type_='hidden')) hddn = {'FormID':'showDatabase', 'ProbeSetID':'_','database':fd.RISet+"Geno",'CellID':'_', 'RISet':fd.RISet, 'incparentsf1':'ON'} for key in hddn.keys(): showLocusForm.append(HtmlGenWrapper.create_input_tag( - name=key, value=hddn[key], type='hidden')) + name=key, value=hddn[key], type_='hidden')) showLocusForm.append(intImg) else: showLocusForm = intImg @@ -791,17 +819,17 @@ class DisplayMappingResults(object): bootScale = bootScale[:-1] + [highestPercent] bootOffset = 50*fontZoom - bootScaleFont=ImageFont.truetype(font=VERDANA_FILE,size=13*fontZoom) + bootScaleFont=ImageFont.truetype(font=VERDANA_FILE, size=13*fontZoom) im_drawer.rectangle( xy=((canvas.size[0]-bootOffset, yZero-bootHeightThresh), - (canvas.size[0]-bootOffset-15*zoom,yZero)), + (canvas.size[0]-bootOffset-15*zoom, yZero)), fill = YELLOW, outline=BLACK) im_drawer.line( xy=((canvas.size[0]-bootOffset+4, yZero), (canvas.size[0]-bootOffset, yZero)), fill=BLACK) TEXT_Y_DISPLACEMENT = -8 - im_drawer.text(xy=(canvas.size[0]-bootOffset+10,yZero+TEXT_Y_DISPLACEMENT), text='0%', + im_drawer.text(xy=(canvas.size[0]-bootOffset+10, yZero+TEXT_Y_DISPLACEMENT), text='0%', font=bootScaleFont, fill=BLACK) for item in bootScale: @@ -809,10 +837,10 @@ class DisplayMappingResults(object): continue bootY = yZero-bootHeightThresh*item/highestPercent im_drawer.line( - xy=((canvas.size[0]-bootOffset+4,bootY), - (canvas.size[0]-bootOffset,bootY)), + xy=((canvas.size[0]-bootOffset+4, bootY), + (canvas.size[0]-bootOffset, bootY)), fill=BLACK) - im_drawer.text(xy=(canvas.size[0]-bootOffset+10,bootY+TEXT_Y_DISPLACEMENT), + im_drawer.text(xy=(canvas.size[0]-bootOffset+10, bootY+TEXT_Y_DISPLACEMENT), text='%2.1f'%item, font=bootScaleFont, fill=BLACK) if self.legendChecked: @@ -821,7 +849,7 @@ class DisplayMappingResults(object): smallLabelFont = ImageFont.truetype(font=TREBUC_FILE, size=12*fontZoom) leftOffset = xLeftOffset+(nCol-1)*200 im_drawer.rectangle( - xy=((leftOffset,startPosY-6), (leftOffset+12,startPosY+6)), + xy=((leftOffset, startPosY-6), (leftOffset+12, startPosY+6)), fill=YELLOW, outline=BLACK) im_drawer.text(xy=(leftOffset+ 20, startPosY+TEXT_Y_DISPLACEMENT), text='Frequency of the Peak LRS', @@ -918,7 +946,7 @@ class DisplayMappingResults(object): TEXT_Y_DISPLACEMENT = -8 im_drawer.text( text="Sequence Site", - xy=(leftOffset+15,startPosY+TEXT_Y_DISPLACEMENT), font=smallLabelFont, + xy=(leftOffset+15, startPosY+TEXT_Y_DISPLACEMENT), font=smallLabelFont, fill=self.TOP_RIGHT_INFO_COLOR) def drawSNPTrackNew(self, canvas, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): @@ -968,7 +996,7 @@ class DisplayMappingResults(object): def drawMultiTraitName(self, fd, canvas, gifmap, showLocusForm, offset= (40, 120, 80, 10), zoom = 1): nameWidths = [] yPaddingTop = 10 - colorFont=ImageFont.truetype(font=TREBUC_FILE,size=12) + colorFont=ImageFont.truetype(font=TREBUC_FILE, size=12) if len(self.qtlresults) >20 and self.selectedChr > -1: rightShift = 20 rightShiftStep = 60 @@ -987,21 +1015,21 @@ class DisplayMappingResults(object): rightShift += rightShiftStep name = thisTrait.displayName() - nameWidth, nameHeight = im_drawer.textsize(name,font=colorFont) + nameWidth, nameHeight = im_drawer.textsize(name, font=colorFont) nameWidths.append(nameWidth) im_drawer.rectangle( - xy=((rightShift,yPaddingTop+kstep*15), - (rectWidth+rightShift,yPaddingTop+10+kstep*15)), + xy=((rightShift, yPaddingTop+kstep*15), + (rectWidth+rightShift, yPaddingTop+10+kstep*15)), fill=thisLRSColor, outline=BLACK) im_drawer.text( - text=name,xy=(rectWidth+2+rightShift,yPaddingTop+10+kstep*15), - font=colorFont,fill=BLACK) + text=name, xy=(rectWidth+2+rightShift, yPaddingTop+10+kstep*15), + font=colorFont, fill=BLACK) if thisTrait.db: - COORDS = "%d,%d,%d,%d" %(rectWidth+2+rightShift,yPaddingTop+kstep*15,rectWidth+2+rightShift+nameWidth,yPaddingTop+10+kstep*15,) + COORDS = "%d,%d,%d,%d" %(rectWidth+2+rightShift, yPaddingTop+kstep*15, rectWidth+2+rightShift+nameWidth, yPaddingTop+10+kstep*15,) HREF= "javascript:showDatabase3('%s','%s','%s','');" % (showLocusForm, thisTrait.db.name, thisTrait.name) - Areas = HtmlGenWrapper.create_area_tag(shape='rect',coords=COORDS,href=HREF) - gifmap.areas.append(Areas) ### TODO + Areas = HtmlGenWrapper.create_area_tag(shape='rect', coords=COORDS, href=HREF) + gifmap.append(Areas) ### TODO def drawLegendPanel(self, canvas, offset= (40, 120, 80, 10), zoom = 1): im_drawer = ImageDraw.Draw(canvas) @@ -1014,80 +1042,80 @@ class DisplayMappingResults(object): if zoom == 2: fontZoom = 1.5 - labelFont=ImageFont.truetype(font=TREBUC_FILE,size=12*fontZoom) + labelFont=ImageFont.truetype(font=TREBUC_FILE, size=12*fontZoom) startPosY = 15 stepPosY = 12*fontZoom if self.manhattan_plot != True: im_drawer.line( - xy=((xLeftOffset,startPosY),(xLeftOffset+32,startPosY)), + xy=((xLeftOffset, startPosY), (xLeftOffset+32, startPosY)), fill=self.LRS_COLOR, width=2) im_drawer.text( - text=self.LRS_LOD, xy=(xLeftOffset+40,startPosY+TEXT_Y_DISPLACEMENT), - font=labelFont,fill=BLACK) + text=self.LRS_LOD, xy=(xLeftOffset+40, startPosY+TEXT_Y_DISPLACEMENT), + font=labelFont, fill=BLACK) startPosY += stepPosY if self.additiveChecked: startPosX = xLeftOffset im_drawer.line( - xy=((startPosX,startPosY),(startPosX+17,startPosY)), + xy=((startPosX, startPosY), (startPosX+17, startPosY)), fill=self.ADDITIVE_COLOR_POSITIVE, width=2) im_drawer.line( - xy=((startPosX+18,startPosY),(startPosX+32,startPosY)), + xy=((startPosX+18, startPosY), (startPosX+32, startPosY)), fill=self.ADDITIVE_COLOR_NEGATIVE, width=2) im_drawer.text( - text='Additive Effect',xy=(startPosX+40,startPosY+TEXT_Y_DISPLACEMENT), - font=labelFont,fill=BLACK) + text='Additive Effect', xy=(startPosX+40, startPosY+TEXT_Y_DISPLACEMENT), + font=labelFont, fill=BLACK) if self.genotype.type == 'intercross' and self.dominanceChecked: startPosX = xLeftOffset startPosY += stepPosY im_drawer.line( - xy=((startPosX,startPosY),(startPosX+17,startPosY)), + xy=((startPosX, startPosY), (startPosX+17, startPosY)), fill=self.DOMINANCE_COLOR_POSITIVE, width=4) im_drawer.line( - xy=((startPosX+18,startPosY),(startPosX+35,startPosY)), + xy=((startPosX+18, startPosY), (startPosX+35, startPosY)), fill=self.DOMINANCE_COLOR_NEGATIVE, width=4) im_drawer.text( - text='Dominance Effect', xy=(startPosX+42,startPosY+5), - font=labelFont,fill=BLACK) + text='Dominance Effect', xy=(startPosX+42, startPosY+5), + font=labelFont, fill=BLACK) if self.haplotypeAnalystChecked: startPosY += stepPosY startPosX = xLeftOffset im_drawer.line( - xy=((startPosX,startPosY),(startPosX+17,startPosY)), + xy=((startPosX, startPosY), (startPosX+17, startPosY)), fill=self.HAPLOTYPE_POSITIVE, width=4) im_drawer.line( - xy=((startPosX+18,startPosY),(startPosX+35,startPosY)), + xy=((startPosX+18, startPosY), (startPosX+35, startPosY)), fill=self.HAPLOTYPE_NEGATIVE, width=4) im_drawer.line( - xy=((startPosX+36,startPosY),(startPosX+53,startPosY)), + xy=((startPosX+36, startPosY), (startPosX+53, startPosY)), fill=self.HAPLOTYPE_HETEROZYGOUS, width=4) im_drawer.line( - xy=((startPosX+54,startPosY),(startPosX+67,startPosY)), + xy=((startPosX+54, startPosY), (startPosX+67, startPosY)), fill=self.HAPLOTYPE_RECOMBINATION, width=4) im_drawer.text( text='Haplotypes (Pat, Mat, Het, Unk)', - xy=(startPosX+76,startPosY+5),font=labelFont,fill=BLACK) + xy=(startPosX+76, startPosY+5), font=labelFont, fill=BLACK) if self.permChecked and self.nperm > 0: startPosY += stepPosY startPosX = xLeftOffset im_drawer.line( - xy=((startPosX, startPosY),( startPosX + 32, startPosY)), + xy=((startPosX, startPosY), ( startPosX + 32, startPosY)), fill=self.SIGNIFICANT_COLOR, width=self.SIGNIFICANT_WIDTH) im_drawer.line( - xy=((startPosX, startPosY + stepPosY),( startPosX + 32, startPosY + stepPosY)), + xy=((startPosX, startPosY + stepPosY), ( startPosX + 32, startPosY + stepPosY)), fill=self.SUGGESTIVE_COLOR, width=self.SUGGESTIVE_WIDTH) im_drawer.text( - text='Significant %s = %2.2f' % (self.LRS_LOD,self.significant), - xy=(xLeftOffset+42,startPosY+TEXT_Y_DISPLACEMENT),font=labelFont,fill=BLACK) + text='Significant %s = %2.2f' % (self.LRS_LOD, self.significant), + xy=(xLeftOffset+42, startPosY+TEXT_Y_DISPLACEMENT), font=labelFont, fill=BLACK) im_drawer.text( text='Suggestive %s = %2.2f' % (self.LRS_LOD, self.suggestive), - xy=(xLeftOffset+42,startPosY + TEXT_Y_DISPLACEMENT +stepPosY),font=labelFont, + xy=(xLeftOffset+42, startPosY + TEXT_Y_DISPLACEMENT +stepPosY), font=labelFont, fill=BLACK) - labelFont = ImageFont.truetype(font=VERDANA_FILE,size=12*fontZoom) + labelFont = ImageFont.truetype(font=VERDANA_FILE, size=12*fontZoom) labelColor = BLACK if self.dataset.type == "Publish" or self.dataset.type == "Geno": dataset_label = self.dataset.fullname @@ -1155,22 +1183,22 @@ class DisplayMappingResults(object): im_drawer.textsize(string2, font=labelFont)[0]) im_drawer.text( text=identification, - xy=(canvas.size[0] - xRightOffset-d,20*fontZoom),font=labelFont, + xy=(canvas.size[0] - xRightOffset-d, 20*fontZoom), font=labelFont, fill=labelColor) else: d = 4+ max( im_drawer.textsize(string1, font=labelFont)[0], im_drawer.textsize(string2, font=labelFont)[0]) im_drawer.text( - text=string1,xy=(canvas.size[0] - xRightOffset-d,35*fontZoom), - font=labelFont,fill=labelColor) + text=string1, xy=(canvas.size[0] - xRightOffset-d, 35*fontZoom), + font=labelFont, fill=labelColor) im_drawer.text( - text=string2,xy=(canvas.size[0] - xRightOffset-d,50*fontZoom), - font=labelFont,fill=labelColor) + text=string2, xy=(canvas.size[0] - xRightOffset-d, 50*fontZoom), + font=labelFont, fill=labelColor) if string3 != '': im_drawer.text( - text=string3,xy=(canvas.size[0] - xRightOffset-d,65*fontZoom), - font=labelFont,fill=labelColor) + text=string3, xy=(canvas.size[0] - xRightOffset-d, 65*fontZoom), + font=labelFont, fill=labelColor) def drawGeneBand(self, canvas, gifmap, plotXScale, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): @@ -1197,8 +1225,8 @@ class DisplayMappingResults(object): tenPercentLength = geneLength*0.0001 SNPdensity = theGO["snpCount"]/geneLength - exonStarts = map(float, theGO['exonStarts'].split(",")[:-1]) - exonEnds = map(float, theGO['exonEnds'].split(",")[:-1]) + exonStarts = list(map(float, theGO['exonStarts'].split(",")[:-1])) + exonEnds = list(map(float, theGO['exonEnds'].split(",")[:-1])) cdsStart = theGO['cdsStart'] cdsEnd = theGO['cdsEnd'] accession = theGO['NM_ID'] @@ -1391,7 +1419,7 @@ class DisplayMappingResults(object): labelText = "3'" im_drawer.text( text=labelText, - xy=(utrEndPix+2,geneYLocation+self.EACH_GENE_HEIGHT), + xy=(utrEndPix+2, geneYLocation+self.EACH_GENE_HEIGHT), font=ImageFont.truetype(font=ARIAL_FILE, size=2)) #draw the genes as rectangles @@ -1403,7 +1431,7 @@ class DisplayMappingResults(object): COORDS = "%d, %d, %d, %d" %(geneStartPix, geneYLocation, geneEndPix, (geneYLocation + self.EACH_GENE_HEIGHT)) # NL: 06-02-2011 Rob required to display NCBI info in a new window - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=COORDS, @@ -1544,7 +1572,7 @@ class DisplayMappingResults(object): counter = counter + 1 if item.name == samplelist[k]: ind = counter - maxind=max(ind,maxind) + maxind=max(ind, maxind) # lines if (oldgeno[k] == -1 and _geno == -1): @@ -1577,7 +1605,7 @@ class DisplayMappingResults(object): COORDS = "%d, %d, %d, %d" %(geneStartPix, geneYLocation+ind*self.EACH_GENE_HEIGHT, geneEndPix+1, (geneYLocation + ind*self.EACH_GENE_HEIGHT)) TITLE = "Strain: %s, marker (%s) \n Position %2.3f Mb." % (samplelist[k], _chr[j].name, float(txStart)) HREF = '' - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=COORDS, @@ -1698,7 +1726,7 @@ class DisplayMappingResults(object): WEBQTL_HREF = "javascript:rangeView('%s', %f, %f)" % (self.selectedChr - 1, max(0, (calBase-webqtlZoomWidth))/1000000.0, (calBase+webqtlZoomWidth)/1000000.0) WEBQTL_TITLE = "Click to view this section of the genome in WebQTL" - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=WEBQTL_COORDS, @@ -1710,7 +1738,7 @@ class DisplayMappingResults(object): outline=self.CLICKABLE_WEBQTL_REGION_COLOR, fill=self.CLICKABLE_WEBQTL_REGION_COLOR) im_drawer.line( - xy=((xBrowse1, paddingTop),( xBrowse1, (paddingTop + self.BAND_HEIGHT))), + xy=((xBrowse1, paddingTop), ( xBrowse1, (paddingTop + self.BAND_HEIGHT))), fill=self.CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR) if self.dataset.group.species == "mouse" or self.dataset.group.species == "rat": @@ -1720,7 +1748,7 @@ class DisplayMappingResults(object): else: PHENOGEN_HREF = "https://phenogen.org/gene.jsp?speciesCB=Mm&auto=Y&geneTxt=chr%s:%d-%d&genomeVer=mm10" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) PHENOGEN_TITLE = "Click to view this section of the genome in PhenoGen" - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=PHENOGEN_COORDS, @@ -1732,7 +1760,7 @@ class DisplayMappingResults(object): outline=self.CLICKABLE_PHENOGEN_REGION_COLOR, fill=self.CLICKABLE_PHENOGEN_REGION_COLOR) im_drawer.line( - xy=((xBrowse1, phenogenPaddingTop),( xBrowse1, (phenogenPaddingTop+self.BAND_HEIGHT))), + xy=((xBrowse1, phenogenPaddingTop), ( xBrowse1, (phenogenPaddingTop+self.BAND_HEIGHT))), fill=self.CLICKABLE_PHENOGEN_REGION_OUTLINE_COLOR) UCSC_COORDS = "%d, %d, %d, %d" %(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.BAND_HEIGHT)) @@ -1741,7 +1769,7 @@ class DisplayMappingResults(object): else: UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) UCSC_TITLE = "Click to view this section of the genome in the UCSC Genome Browser" - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=UCSC_COORDS, @@ -1763,7 +1791,7 @@ class DisplayMappingResults(object): else: ENSEMBL_HREF = "http://www.ensembl.org/Rattus_norvegicus/contigview?chr=%s&start=%d&end=%d" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) ENSEMBL_TITLE = "Click to view this section of the genome in the Ensembl Genome Browser" - gifmap.areas.append(HtmlGenWrapper.create_area_tag( + gifmap.append(HtmlGenWrapper.create_area_tag( shape='rect', coords=ENSEMBL_COORDS, href=ENSEMBL_HREF, @@ -1864,8 +1892,8 @@ class DisplayMappingResults(object): continue Xc = xLeftOffset + plotXScale*(_Mb - startMb) if counter % NUM_MINOR_TICKS == 0: # Draw a MAJOR mark, not just a minor tick mark - im_drawer.line(xy=((Xc,yZero), - (Xc,yZero+xMajorTickHeight)), + im_drawer.line(xy=((Xc, yZero), + (Xc, yZero+xMajorTickHeight)), fill=xAxisTickMarkColor, width=X_MAJOR_TICK_THICKNESS) # Draw the MAJOR tick mark labelStr = str(formatStr % _Mb) # What Mbase location to put on the label @@ -1875,8 +1903,8 @@ class DisplayMappingResults(object): text=labelStr, font=MBLabelFont, fill=xAxisLabelColor) else: - im_drawer.line(xy=((Xc,yZero), - (Xc,yZero+xMinorTickHeight)), + im_drawer.line(xy=((Xc, yZero), + (Xc, yZero+xMinorTickHeight)), fill=xAxisTickMarkColor, width=X_MINOR_TICK_THICKNESS) # Draw the MINOR tick mark @@ -1909,7 +1937,7 @@ class DisplayMappingResults(object): text="Megabases", xy=( xLeftOffset+(plotWidth-im_drawer.textsize( - "Megabases",font=megabaseLabelFont)[0])/2, + "Megabases", font=megabaseLabelFont)[0])/2, strYLoc+MBLabelFont.font.height+10*(zoom%2)), font=megabaseLabelFont, fill=BLACK) pass @@ -1964,7 +1992,7 @@ class DisplayMappingResults(object): for j, ChrInfo in enumerate(ChrAInfo): preLpos = -1 for i, item in enumerate(ChrInfo): - Lname,Lpos = item + Lname, Lpos = item if Lpos != preLpos: offsetA += stepA differ = 1 @@ -1978,17 +2006,17 @@ class DisplayMappingResults(object): Zorder = 0 if differ: im_drawer.line( - xy=((startPosX+Lpos,yZero),(xLeftOffset+offsetA,\ + xy=((startPosX+Lpos, yZero), (xLeftOffset+offsetA,\ yZero+25)), fill=lineColor) im_drawer.line( - xy=((xLeftOffset+offsetA,yZero+25),(xLeftOffset+offsetA,\ + xy=((xLeftOffset+offsetA, yZero+25), (xLeftOffset+offsetA,\ yZero+40+Zorder*(LRectWidth+3))), fill=lineColor) rectColor = ORANGE else: im_drawer.line( - xy=((xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3)-3),(\ + xy=((xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3)-3), (\ xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3))), fill=lineColor) rectColor = DEEPPINK @@ -1996,7 +2024,7 @@ class DisplayMappingResults(object): xy=((xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3)), (xLeftOffset+offsetA-LRectHeight, yZero+40+Zorder*(LRectWidth+3)+LRectWidth)), - outline=rectColor,fill=rectColor,width = 0) + outline=rectColor, fill=rectColor, width = 0) COORDS="%d,%d,%d,%d"%(xLeftOffset+offsetA-LRectHeight, yZero+40+Zorder*(LRectWidth+3),\ xLeftOffset+offsetA,yZero+40+Zorder*(LRectWidth+3)+LRectWidth) HREF = "/show_trait?trait_id=%s&dataset=%s" % (Lname, self.dataset.group.name+"Geno") @@ -2007,11 +2035,11 @@ class DisplayMappingResults(object): href=HREF, target="_blank", title="Locus : {}".format(Lname)) - gifmap.areas.append(Areas) + gifmap.append(Areas) ##piddle bug if j == 0: im_drawer.line( - xy=((startPosX,yZero),(startPosX,yZero+40)), + xy=((startPosX, yZero), (startPosX, yZero+40)), fill=lineColor) startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale @@ -2023,7 +2051,7 @@ class DisplayMappingResults(object): strYLoc + MBLabelFont.font.height+ 10*(zoom%2)), font=centimorganLabelFont, fill=BLACK) - im_drawer.line(xy=((xLeftOffset,yZero), (xLeftOffset+plotWidth,yZero)), + im_drawer.line(xy=((xLeftOffset, yZero), (xLeftOffset+plotWidth, yZero)), fill=BLACK, width=X_AXIS_THICKNESS) # Draw the X axis itself @@ -2125,7 +2153,7 @@ class DisplayMappingResults(object): #ZS: Convert to int if all axis values are whole numbers all_int = True for item in LRSAxisList: - if item.is_integer(): + if isinstance(item, int): continue else: all_int = False @@ -2163,7 +2191,7 @@ class DisplayMappingResults(object): LRS_LOD_Max = 0.000001 yTopOffset + 30*(zoom - 1) yLRS = yZero - (item/LRS_LOD_Max) * LRSHeightThresh - im_drawer.line(xy=((xLeftOffset,yLRS), (xLeftOffset-4,yLRS)), + im_drawer.line(xy=((xLeftOffset, yLRS), (xLeftOffset-4, yLRS)), fill=self.LRS_COLOR, width=1*zoom) if all_int: scaleStr = "%d" % item @@ -2219,8 +2247,8 @@ class DisplayMappingResults(object): shape='rect', coords=sig_coords, title=sig_title) - gifmap.areas.append(Areas1) - gifmap.areas.append(Areas2) + gifmap.append(Areas1) + gifmap.append(Areas2) start_pos_x += (chr_length_dist+self.GraphInterval)*plotXScale return start_pos_x @@ -2239,7 +2267,7 @@ class DisplayMappingResults(object): lrsEdgeWidth = 1 else: if self.additiveChecked: - additiveMax = max(map(lambda X : abs(X['additive']), self.qtlresults)) + additiveMax = max([abs(X['additive']) for X in self.qtlresults]) lrsEdgeWidth = 3 if zoom == 2: @@ -2406,7 +2434,7 @@ class DisplayMappingResults(object): im_drawer.text( text="5", xy=( - Xc-im_drawer.textsize("5",font=symbolFont)[0]/2+1, + Xc-im_drawer.textsize("5", font=symbolFont)[0]/2+1, Yc-4), fill=point_color, font=symbolFont) else: @@ -2473,8 +2501,8 @@ class DisplayMappingResults(object): ) else: im_drawer.line( - xy=((Xc0,yZero-(Yc0-yZero)), - (Xc,yZero-(Yc-yZero))), + xy=((Xc0, yZero-(Yc0-yZero)), + (Xc, yZero-(Yc-yZero))), fill=minusColor, width=lineWidth #, clipX=(xLeftOffset, xLeftOffset + plotWidth) ) @@ -2561,8 +2589,8 @@ class DisplayMappingResults(object): ###draw additive scale if not self.multipleInterval and self.additiveChecked: - additiveScaleFont=ImageFont.truetype(font=VERDANA_FILE,size=16*zoom) - additiveScale = Plot.detScaleOld(0,additiveMax) + additiveScaleFont=ImageFont.truetype(font=VERDANA_FILE, size=16*zoom) + additiveScale = Plot.detScaleOld(0, additiveMax) additiveStep = (additiveScale[1]-additiveScale[0])/additiveScale[2] additiveAxisList = Plot.frange(0, additiveScale[1], additiveStep) addPlotScale = AdditiveHeightThresh/additiveMax @@ -2572,18 +2600,18 @@ class DisplayMappingResults(object): for item in additiveAxisList: additiveY = yZero - item*addPlotScale im_drawer.line( - xy=((xLeftOffset + plotWidth,additiveY), - (xLeftOffset+4+ plotWidth,additiveY)), + xy=((xLeftOffset + plotWidth, additiveY), + (xLeftOffset+4+ plotWidth, additiveY)), fill=self.ADDITIVE_COLOR_POSITIVE, width=1*zoom) scaleStr = "%2.3f" % item im_drawer.text( text=scaleStr, - xy=(xLeftOffset + plotWidth +6,additiveY+TEXT_Y_DISPLACEMENT), - font=additiveScaleFont,fill=self.ADDITIVE_COLOR_POSITIVE) + xy=(xLeftOffset + plotWidth +6, additiveY+TEXT_Y_DISPLACEMENT), + font=additiveScaleFont, fill=self.ADDITIVE_COLOR_POSITIVE) im_drawer.line( - xy=((xLeftOffset+plotWidth,additiveY), - (xLeftOffset+plotWidth,yZero)), + xy=((xLeftOffset+plotWidth, additiveY), + (xLeftOffset+plotWidth, yZero)), fill=self.ADDITIVE_COLOR_POSITIVE, width=1*zoom) im_drawer.line( @@ -2643,7 +2671,7 @@ class DisplayMappingResults(object): chrFontZoom = 2 else: chrFontZoom = 1 - chrLabelFont=ImageFont.truetype(font=VERDANA_FILE,size=24*chrFontZoom) + chrLabelFont=ImageFont.truetype(font=VERDANA_FILE, size=24*chrFontZoom) for i, _chr in enumerate(self.genotype): if (i % 2 == 0): @@ -2665,16 +2693,16 @@ class DisplayMappingResults(object): TEXT_Y_DISPLACEMENT = 0 im_drawer.text(xy=(chrStartPix, yTopOffset + TEXT_Y_DISPLACEMENT), text=_chr.name, font=chrLabelFont, fill=BLACK) - COORDS = "%d,%d,%d,%d" %(chrStartPix, yTopOffset, chrEndPix,yTopOffset +20) + COORDS = "%d,%d,%d,%d" %(chrStartPix, yTopOffset, chrEndPix, yTopOffset +20) #add by NL 09-03-2010 - HREF = "javascript:chrView(%d,%s);" % (i,self.ChrLengthMbList) + HREF = "javascript:chrView(%d,%s);" % (i, self.ChrLengthMbList) #HREF = "javascript:changeView(%d,%s);" % (i,self.ChrLengthMbList) Areas = HtmlGenWrapper.create_area_tag( shape='rect', coords=COORDS, href=HREF) - gifmap.areas.append(Areas) + gifmap.append(Areas) startPosX += (self.ChrLengthDistList[i]+self.GraphInterval)*plotXScale return plotXScale @@ -2760,7 +2788,7 @@ class DisplayMappingResults(object): this_row = [] #container for the cells of each row selectCheck = HtmlGenWrapper.create_input_tag( - type="checkbox", + type_="checkbox", name="selectCheck", value=theGO["GeneSymbol"], Class="checkbox trait_checkbox") # checkbox for each row @@ -2774,9 +2802,17 @@ class DisplayMappingResults(object): geneIdString = 'http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s' % theGO["GeneID"] if theGO["snpCount"]: - snpString = HT.Href(url="http://genenetwork.org/webqtl/main.py?FormID=snpBrowser&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (theGO["Chromosome"], - theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffCol[0], self.diffCol[1]), - text=theGO["snpCount"], target="_blank", Class="normalsize") + snpString = HT.Link( + (f"http://genenetwork.org/webqtl/main.py?FormID=snpBrowser&" + f"chr={theGO['Chromosome']}&" + f"start={theGO['TxStart']}&" + f"end={theGO['TxEnd']}&" + f"geneName={theGO['GeneSymbol']}&" + f"s1={self.diffCol[0]}&s2=%d"), + str(theGO["snpCount"]) # The text to display + ) + snpString.set_blank_target() + snpString.set_attribute("class", "normalsize") else: snpString = 0 @@ -2817,7 +2853,7 @@ class DisplayMappingResults(object): else: chr_as_int = int(theGO["Chromosome"]) - 1 if refGene: - literatureCorrelationString = str(self.getLiteratureCorrelation(self.cursor,refGene,theGO['GeneID']) or "N/A") + literatureCorrelationString = str(self.getLiteratureCorrelation(self.cursor, refGene, theGO['GeneID']) or "N/A") this_row = [selectCheck.__str__(), str(tableIterationsCnt), @@ -2879,13 +2915,10 @@ class DisplayMappingResults(object): for gIndex, theGO in enumerate(geneCol): this_row = [] # container for the cells of each row selectCheck = str(HtmlGenWrapper.create_input_tag( - type="checkbox", + type_="checkbox", name="selectCheck", Class="checkbox trait_checkbox")) # checkbox for each row - #ZS: May want to get this working again later - #webqtlSearch = HT.Href(os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE)+"?cmd=sch&gene=%s&alias=1&species=rat" % theGO["GeneSymbol"], ">>", target="_blank").__str__() - if theGO["GeneID"] != "": geneSymbolNCBI = str(HtmlGenWrapper.create_link_tag( "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids={}".format(theGO["GeneID"]), @@ -2955,8 +2988,8 @@ class DisplayMappingResults(object): lCorr = None try: query = 'SELECT Value FROM LCorrRamin3 WHERE GeneId1 = %s and GeneId2 = %s' - for x,y in [(geneId1,geneId2),(geneId2,geneId1)]: - cursor.execute(query,(x,y)) + for x, y in [(geneId1, geneId2), (geneId2, geneId1)]: + cursor.execute(query, (x, y)) lCorr = cursor.fetchone() if lCorr: lCorr = lCorr[0] diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index b858b573..68a8d5ba 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -218,9 +218,9 @@ def parse_loco_output(this_dataset, gwa_output_filename): marker['chr'] = int(line.split("\t")[0][3:]) else: marker['chr'] = int(line.split("\t")[0]) - if marker['chr'] > previous_chr: + if marker['chr'] > int(previous_chr): previous_chr = marker['chr'] - elif marker['chr'] < previous_chr: + elif marker['chr'] < int(previous_chr): break else: marker['chr'] = line.split("\t")[0] diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index 2f327faf..fd91b6ca 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -54,7 +54,7 @@ def gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename = ''): for i, sample in enumerate(ped_sample_list): try: value = vals[i] - value = str(value).replace('value=','') + value = str(value).replace('value=', '') value = value.strip() except: value = -9999 @@ -78,13 +78,13 @@ def gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename = ''): # get strain name from ped file in order def get_samples_from_ped_file(dataset): - ped_file= open("{}{}.ped".format(flat_files('mapping'), dataset.group.name),"r") + ped_file= open("{}{}.ped".format(flat_files('mapping'), dataset.group.name), "r") line = ped_file.readline() sample_list=[] while line: - lineList = string.split(string.strip(line), '\t') - lineList = map(string.strip, lineList) + lineList = line.strip().split('\t') + lineList = [item.strip() for item in lineList] sample_name = lineList[0] sample_list.append(sample_name) @@ -111,7 +111,7 @@ def parse_plink_output(output_filename, species): line_list = build_line_list(line=line) # only keep the records whose chromosome name is in db - if species.chromosomes.chromosomes.has_key(int(line_list[0])) and line_list[-1] and line_list[-1].strip()!='NA': + if int(line_list[0]) in species.chromosomes.chromosomes and line_list[-1] and line_list[-1].strip()!='NA': chr_name = species.chromosomes.chromosomes[int(line_list[0])] snp = line_list[1] @@ -121,7 +121,7 @@ def parse_plink_output(output_filename, species): if p_value < threshold_p_value: p_value_dict[snp] = float(p_value) - if plink_results.has_key(chr_name): + if chr_name in plink_results: value_list = plink_results[chr_name] # pvalue range is [0,1] @@ -155,8 +155,8 @@ def parse_plink_output(output_filename, species): # output: lineList list ####################################################### def build_line_list(line=None): - line_list = string.split(string.strip(line),' ')# irregular number of whitespaces between columns - line_list = [item for item in line_list if item <>''] - line_list = map(string.strip, line_list) + line_list = line.strip().split(' ')# irregular number of whitespaces between columns + line_list = [item for item in line_list if item !=''] + line_list = [item.strip() for item in line_list] - return line_list
\ No newline at end of file + return line_list diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index 6b4c05ea..78b1f7b0 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -252,4 +252,4 @@ def natural_sort(marker_list): """ convert = lambda text: int(text) if text.isdigit() else text.lower() alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', str(marker_list[key]['chr'])) ] - return sorted(range(len(marker_list)), key = alphanum_key)
\ No newline at end of file + return sorted(list(range(len(marker_list))), key = alphanum_key)
\ No newline at end of file diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index c5590a85..0a5758af 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -42,7 +42,7 @@ def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permChec png = ro.r["png"] # Map the png function dev_off = ro.r["dev.off"] # Map the device off function - print(r_library("qtl")) # Load R/qtl + print((r_library("qtl"))) # Load R/qtl logger.info("QTL library loaded"); diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 8a44b3fd..fa61272f 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from base.trait import GeneralTrait from base import data_set #import create_dataset @@ -18,7 +16,7 @@ import uuid import rpy2.robjects as ro import numpy as np -import cPickle as pickle +import pickle as pickle import itertools import simplejson as json @@ -140,7 +138,7 @@ class RunMapping(object): mapping_results_filename = self.dataset.group.name + "_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) self.mapping_results_path = "{}{}.csv".format(webqtlConfig.GENERATED_IMAGE_DIR, mapping_results_filename) - if start_vars['manhattan_plot'] == "True": + if start_vars['manhattan_plot'] == "true": self.manhattan_plot = True else: self.manhattan_plot = False @@ -347,7 +345,7 @@ class RunMapping(object): if marker['chr1'] > 0 or marker['chr1'] == "X" or marker['chr1'] == "X/Y": if marker['chr1'] > highest_chr or marker['chr1'] == "X" or marker['chr1'] == "X/Y": highest_chr = marker['chr1'] - if 'lod_score' in marker.keys(): + if 'lod_score' in list(marker.keys()): self.qtl_results.append(marker) self.trimmed_markers = results @@ -411,8 +409,8 @@ class RunMapping(object): self.results_for_browser.append(browser_marker) self.annotations_for_browser.append(annot_marker) - if marker['chr'] > 0 or marker['chr'] == "X" or marker['chr'] == "X/Y": - if marker['chr'] > highest_chr or marker['chr'] == "X" or marker['chr'] == "X/Y": + if str(marker['chr']) > '0' or str(marker['chr']) == "X" or str(marker['chr']) == "X/Y": + if str(marker['chr']) > str(highest_chr) or str(marker['chr']) == "X" or str(marker['chr']) == "X/Y": highest_chr = marker['chr'] if ('lod_score' in marker.keys()) or ('lrs_value' in marker.keys()): if 'Mb' in marker.keys(): @@ -547,9 +545,9 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, output_file.write("Mb," + score_type) if 'cM' in markers[0]: output_file.write("Cm," + score_type) - if "additive" in markers[0].keys(): + if "additive" in list(markers[0].keys()): output_file.write(",Additive") - if "dominance" in markers[0].keys(): + if "dominance" in list(markers[0].keys()): output_file.write(",Dominance") output_file.write("\n") for i, marker in enumerate(markers): @@ -562,17 +560,17 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, output_file.write(str(marker['lod_score'])) else: output_file.write(str(marker['lrs_value'])) - if "additive" in marker.keys(): + if "additive" in list(marker.keys()): output_file.write("," + str(marker['additive'])) - if "dominance" in marker.keys(): + if "dominance" in list(marker.keys()): output_file.write("," + str(marker['dominance'])) if i < (len(markers) - 1): output_file.write("\n") def trim_markers_for_figure(markers): - if 'p_wald' in markers[0].keys(): + if 'p_wald' in list(markers[0].keys()): score_type = 'p_wald' - elif 'lod_score' in markers[0].keys(): + elif 'lod_score' in list(markers[0].keys()): score_type = 'lod_score' else: score_type = 'lrs_value' @@ -630,7 +628,7 @@ def trim_markers_for_figure(markers): return filtered_markers def trim_markers_for_table(markers): - if 'lod_score' in markers[0].keys(): + if 'lod_score' in list(markers[0].keys()): sorted_markers = sorted(markers, key=lambda k: k['lod_score'], reverse=True) else: sorted_markers = sorted(markers, key=lambda k: k['lrs_value'], reverse=True) @@ -714,10 +712,10 @@ def get_genofile_samplelist(dataset): def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): perm_strata_strings = [] for sample in used_samples: - if sample in sample_list.sample_attribute_values.keys(): + if sample in list(sample_list.sample_attribute_values.keys()): combined_string = "" for var in categorical_vars: - if var in sample_list.sample_attribute_values[sample].keys(): + if var in list(sample_list.sample_attribute_values[sample].keys()): combined_string += str(sample_list.sample_attribute_values[sample][var]) else: combined_string += "NA" @@ -726,8 +724,8 @@ def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): perm_strata_strings.append(combined_string) - d = dict([(y,x+1) for x,y in enumerate(sorted(set(perm_strata_strings)))]) + d = dict([(y, x+1) for x, y in enumerate(sorted(set(perm_strata_strings)))]) list_to_numbers = [d[x] for x in perm_strata_strings] perm_strata = list_to_numbers - return perm_strata
\ No newline at end of file + return perm_strata diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py index 38117a8e..772f74e4 100644 --- a/wqflask/wqflask/model.py +++ b/wqflask/wqflask/model.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import uuid import datetime @@ -18,7 +16,7 @@ from wqflask.database import Base, init_db class User(Base): __tablename__ = "user" - id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) + id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) email_address = Column(Unicode(50), unique=True, nullable=False) # Todo: Turn on strict mode for Mysql @@ -120,7 +118,7 @@ class User(Base): class Login(Base): __tablename__ = "login" - id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) + id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) user = Column(Unicode(36), ForeignKey('user.id')) timestamp = Column(DateTime(), default=lambda: datetime.datetime.utcnow()) ip_address = Column(Unicode(39)) @@ -138,7 +136,7 @@ class Login(Base): class UserCollection(Base): __tablename__ = "user_collection" - id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) + id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) user = Column(Unicode(36), ForeignKey('user.id')) # I'd prefer this to not have a length, but for the index below it needs one @@ -168,4 +166,4 @@ def display_collapsible(number): def user_uuid(): """Unique cookie for a user""" - user_uuid = request.cookies.get('user_uuid')
\ No newline at end of file + user_uuid = request.cookies.get('user_uuid') diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index f61c40b4..723a749f 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -1,4 +1,4 @@ -## Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License @@ -18,50 +18,21 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - -import sys - -import string -import cPickle -import os -import time -import pp -import math -import collections -import resource - import scipy - import simplejson as json -from rpy2.robjects.packages import importr -import rpy2.robjects as robjects - -from pprint import pformat as pf - -from utility.THCell import THCell -from utility.TDCell import TDCell from base.trait import create_trait from base import data_set -from utility import webqtlUtil, helper_functions, corr_result_helpers +from utility import helper_functions +from utility import corr_result_helpers from utility.tools import GN2_BRANCH_URL -from db import webqtlDatabaseFunction -import utility.webqtlUtil #this is for parallel computing only. -from wqflask.correlation import correlation_functions -from utility.benchmark import Bench - -from MySQLdb import escape_string as escape - -from pprint import pformat as pf - -from flask import Flask, g class NetworkGraph(object): def __init__(self, start_vars): - trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] + trait_db_list = [trait.strip() + for trait in start_vars['trait_list'].split(',')] helper_functions.get_trait_db_obs(self, trait_db_list) @@ -89,7 +60,8 @@ class NetworkGraph(object): this_trait_vals.append('') self.sample_data.append(this_trait_vals) - self.lowest_overlap = 8 #ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning) + # ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning) + self.lowest_overlap = 8 self.nodes_list = [] self.edges_list = [] @@ -101,9 +73,9 @@ class NetworkGraph(object): this_sample_data = this_trait.data corr_result_row = [] - is_spearman = False #ZS: To determine if it's above or below the diagonal + is_spearman = False # ZS: To determine if it's above or below the diagonal - max_corr = 0 #ZS: Used to determine whether node should be hidden when correlation coefficient slider is used + max_corr = 0 # ZS: Used to determine whether node should be hidden when correlation coefficient slider is used for target in self.trait_list: target_trait = target[0] @@ -122,20 +94,23 @@ class NetworkGraph(object): this_trait_vals.append(sample_value) target_vals.append(target_sample_value) - this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(this_trait_vals, target_vals) + this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values( + this_trait_vals, target_vals) if num_overlap < self.lowest_overlap: self.lowest_overlap = num_overlap if num_overlap == 0: continue else: - pearson_r, pearson_p = scipy.stats.pearsonr(this_trait_vals, target_vals) + pearson_r, pearson_p = scipy.stats.pearsonr( + this_trait_vals, target_vals) if is_spearman == False: sample_r, sample_p = pearson_r, pearson_p if sample_r == 1: continue else: - sample_r, sample_p = scipy.stats.spearmanr(this_trait_vals, target_vals) + sample_r, sample_p = scipy.stats.spearmanr( + this_trait_vals, target_vals) if -1 <= sample_r < -0.7: color = "#0000ff" @@ -153,44 +128,44 @@ class NetworkGraph(object): color = "#ffa500" width = 2 elif 0.7 <= sample_r <= 1: - color = "#ff0000" - width = 3 + color = "#ff0000" + width = 3 else: color = "#000000" - width = 0 + width = 0 if abs(sample_r) > max_corr: max_corr = abs(sample_r) - edge_data = {'id' : str(this_trait.name) + '_to_' + str(target_trait.name), - 'source' : str(this_trait.name) + ":" + str(this_trait.dataset.name), - 'target' : str(target_trait.name) + ":" + str(target_trait.dataset.name), - 'correlation' : round(sample_r, 3), - 'abs_corr' : abs(round(sample_r, 3)), - 'p_value' : round(sample_p, 3), - 'overlap' : num_overlap, - 'color' : color, - 'width' : width } + edge_data = {'id': str(this_trait.name) + '_to_' + str(target_trait.name), + 'source': str(this_trait.name) + ":" + str(this_trait.dataset.name), + 'target': str(target_trait.name) + ":" + str(target_trait.dataset.name), + 'correlation': round(sample_r, 3), + 'abs_corr': abs(round(sample_r, 3)), + 'p_value': round(sample_p, 3), + 'overlap': num_overlap, + 'color': color, + 'width': width} - edge_dict = { 'data' : edge_data } + edge_dict = {'data': edge_data} self.edges_list.append(edge_dict) if trait_db[1].type == "ProbeSet": - node_dict = { 'data' : {'id' : str(this_trait.name) + ":" + str(this_trait.dataset.name), - 'label' : this_trait.symbol, - 'symbol' : this_trait.symbol, - 'geneid' : this_trait.geneid, - 'omim' : this_trait.omim, - 'max_corr' : max_corr } } + node_dict = {'data': {'id': str(this_trait.name) + ":" + str(this_trait.dataset.name), + 'label': this_trait.symbol, + 'symbol': this_trait.symbol, + 'geneid': this_trait.geneid, + 'omim': this_trait.omim, + 'max_corr': max_corr}} elif trait_db[1].type == "Publish": - node_dict = { 'data' : {'id' : str(this_trait.name) + ":" + str(this_trait.dataset.name), - 'label' : this_trait.name, - 'max_corr' : max_corr } } + node_dict = {'data': {'id': str(this_trait.name) + ":" + str(this_trait.dataset.name), + 'label': this_trait.name, + 'max_corr': max_corr}} else: - node_dict = { 'data' : {'id' : str(this_trait.name) + ":" + str(this_trait.dataset.name), - 'label' : this_trait.name, - 'max_corr' : max_corr } } + node_dict = {'data': {'id': str(this_trait.name) + ":" + str(this_trait.dataset.name), + 'label': this_trait.name, + 'max_corr': max_corr}} self.nodes_list.append(node_dict) self.elements = json.dumps(self.nodes_list + self.edges_list) @@ -200,13 +175,13 @@ class NetworkGraph(object): for sample in self.all_sample_list: groups.append(1) - self.js_data = dict(traits = [trait.name for trait in self.traits], - groups = groups, - cols = range(len(self.traits)), - rows = range(len(self.traits)), - samples = self.all_sample_list, - sample_data = self.sample_data, - elements = self.elements,) + self.js_data = dict(traits=[trait.name for trait in self.traits], + groups=groups, + cols=list(range(len(self.traits))), + rows=list(range(len(self.traits))), + samples=self.all_sample_list, + sample_data=self.sample_data, + elements=self.elements,) def get_trait_db_obs(self, trait_db_list): self.trait_list = [] @@ -216,6 +191,6 @@ class NetworkGraph(object): trait_name, dataset_name = trait_db.split(":") dataset_ob = data_set.create_dataset(dataset_name) trait_ob = create_trait(dataset=dataset_ob, - name=trait_name, - cellid=None) - self.trait_list.append((trait_ob, dataset_ob))
\ No newline at end of file + name=trait_name, + cellid=None) + self.trait_list.append((trait_ob, dataset_ob)) diff --git a/wqflask/wqflask/news.py b/wqflask/wqflask/news.py index 8bc6b889..0675ec4b 100644 --- a/wqflask/wqflask/news.py +++ b/wqflask/wqflask/news.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import, print_function, division -import sys -reload(sys) -sys.setdefaultencoding('utf8') from flask import g class News(object): diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index 1ca5ecff..76fae54b 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -17,8 +17,6 @@ be acceptable.] """ -from __future__ import print_function, division - import re from pprint import pformat as pf @@ -78,22 +76,6 @@ def parse(pstring): logger.debug("* items are:", pf(items) + "\n") return(items) - #def encregexp(self,str): - # if not str: - # return [] - # else: - # wildcardkeyword = str.strip() - # wildcardkeyword = string.replace(wildcardkeyword,',',' ') - # wildcardkeyword = string.replace(wildcardkeyword,';',' ') - # wildcardkeyword = wildcardkeyword.split() - # NNN = len(wildcardkeyword) - # for i in range(NNN): - # keyword = wildcardkeyword[i] - # keyword = string.replace(keyword,"*",".*") - # keyword = string.replace(keyword,"?",".") - # wildcardkeyword[i] = keyword#'[[:<:]]'+ keyword+'[[:>:]]' - # return wildcardkeyword - if __name__ == '__main__': parse("foo=[3 2 1]") diff --git a/wqflask/wqflask/pbkdf2.py b/wqflask/wqflask/pbkdf2.py index f7f61a09..aea5b06c 100644 --- a/wqflask/wqflask/pbkdf2.py +++ b/wqflask/wqflask/pbkdf2.py @@ -1,140 +1,20 @@ -# -*- coding: utf-8 -*- -""" - pbkdf2 - ~~~~~~ - - This module implements pbkdf2 for Python. It also has some basic - tests that ensure that it works. The implementation is straightforward - and uses stdlib only stuff and can be easily be copy/pasted into - your favourite application. - - Use this as replacement for bcrypt that does not need a c implementation - of a modified blowfish crypto algo. - - Example usage: - - >>> pbkdf2_hex('what i want to hash', 'the random salt') - 'fa7cc8a2b0a932f8e6ea42f9787e9d36e592e0c222ada6a9' - - How to use this: - - 1. Use a constant time string compare function to compare the stored hash - with the one you're generating:: - - def safe_str_cmp(a, b): - if len(a) != len(b): - return False - rv = 0 - for x, y in izip(a, b): - rv |= ord(x) ^ ord(y) - return rv == 0 - - 2. Use `os.urandom` to generate a proper salt of at least 8 byte. - Use a unique salt per hashed password. - - 3. Store ``algorithm$salt:costfactor$hash`` in the database so that - you can upgrade later easily to a different algorithm if you need - one. For instance ``PBKDF2-256$thesalt:10000$deadbeef...``. - - - :copyright: (c) Copyright 2011 by Armin Ronacher. - :license: BSD, see LICENSE for more details. -""" -import hmac import hashlib -from struct import Struct -from operator import xor -from itertools import izip, starmap +from werkzeug.security import safe_str_cmp as ssc -_pack_int = Struct('>I').pack - -def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc=None): - """Like :func:`pbkdf2_bin` but returns a hex encoded string.""" - return pbkdf2_bin(data, salt, iterations, keylen, hashfunc).encode('hex') - - -def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None): - """Returns a binary digest for the PBKDF2 hash algorithm of `data` - with the given `salt`. It iterates `iterations` time and produces a - key of `keylen` bytes. By default SHA-1 is used as hash function, - a different hashlib `hashfunc` can be provided. +# Replace this because it just wraps around Python3's internal +# functions. Added this during migration. +def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc="sha1"): + """Wrapper function of python's hashlib.pbkdf2_hmac. """ - hashfunc = hashfunc or hashlib.sha1 - mac = hmac.new(data, None, hashfunc) - def _pseudorandom(x, mac=mac): - h = mac.copy() - h.update(x) - return map(ord, h.digest()) - buf = [] - for block in xrange(1, -(-keylen // mac.digest_size) + 1): - rv = u = _pseudorandom(salt + _pack_int(block)) - for i in xrange(iterations - 1): - u = _pseudorandom(''.join(map(chr, u))) - rv = list(starmap(xor, izip(rv, u))) - buf.extend(rv) - return ''.join(map(chr, buf))[:keylen] + dk = hashlib.pbkdf2_hmac(hashfunc, + bytes(data, "utf-8"), # password + bytes(salt, "utf-8"), # salt + iterations, + keylen) + return dk.hex() def safe_str_cmp(a, b): - if len(a) != len(b): - return False - rv = 0 - for x, y in izip(a, b): - rv |= ord(x) ^ ord(y) - return rv == 0 - - - -def test(): - failed = [] - def check(data, salt, iterations, keylen, expected): - rv = pbkdf2_hex(data, salt, iterations, keylen) - if rv != expected: - print 'Test failed:' - print ' Expected: %s' % expected - print ' Got: %s' % rv - print ' Parameters:' - print ' data=%s' % data - print ' salt=%s' % salt - print ' iterations=%d' % iterations - print - failed.append(1) - - # From RFC 6070 - check('password', 'salt', 1, 20, - '0c60c80f961f0e71f3a9b524af6012062fe037a6') - check('password', 'salt', 2, 20, - 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957') - check('password', 'salt', 4096, 20, - '4b007901b765489abead49d926f721d065a429c1') - check('passwordPASSWORDpassword', 'saltSALTsaltSALTsaltSALTsaltSALTsalt', - 4096, 25, '3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038') - check('pass\x00word', 'sa\x00lt', 4096, 16, - '56fa6aa75548099dcc37d7f03425e0c3') - # This one is from the RFC but it just takes for ages - ##check('password', 'salt', 16777216, 20, - ## 'eefe3d61cd4da4e4e9945b3d6ba2158c2634e984') - - # From Crypt-PBKDF2 - check('password', 'ATHENA.MIT.EDUraeburn', 1, 16, - 'cdedb5281bb2f801565a1122b2563515') - check('password', 'ATHENA.MIT.EDUraeburn', 1, 32, - 'cdedb5281bb2f801565a1122b25635150ad1f7a04bb9f3a333ecc0e2e1f70837') - check('password', 'ATHENA.MIT.EDUraeburn', 2, 16, - '01dbee7f4a9e243e988b62c73cda935d') - check('password', 'ATHENA.MIT.EDUraeburn', 2, 32, - '01dbee7f4a9e243e988b62c73cda935da05378b93244ec8f48a99e61ad799d86') - check('password', 'ATHENA.MIT.EDUraeburn', 1200, 32, - '5c08eb61fdf71e4e4ec3cf6ba1f5512ba7e52ddbc5e5142f708a31e2e62b1e13') - check('X' * 64, 'pass phrase equals block size', 1200, 32, - '139c30c0966bc32ba55fdbf212530ac9c5ec59f1a452f5cc9ad940fea0598ed1') - check('X' * 65, 'pass phrase exceeds block size', 1200, 32, - '9ccad6d468770cd51b10e6a68721be611a8b4d282601db3b36be9246915ec82a') - - raise SystemExit(bool(failed)) - - -if __name__ == '__main__': - test() + return ssc(a, b) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 14ff2183..85cbb2fd 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import json from flask import (Flask, g, render_template, url_for, request, make_response, @@ -125,10 +123,10 @@ def add_group_to_resource(): def get_group_names(group_masks): group_masks_with_names = {} - for group_id, group_mask in group_masks.iteritems(): + for group_id, group_mask in list(group_masks.items()): this_mask = group_mask group_name = get_group_info(group_id)['name'] this_mask['name'] = group_name group_masks_with_names[group_id] = this_mask - return group_masks_with_names
\ No newline at end of file + return group_masks_with_names diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index f63a84d1..ce836ce2 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import re import uuid from math import * @@ -53,7 +51,7 @@ views.py). search = self.search_terms self.original_search_string = self.search_terms # check for dodgy search terms - rx = re.compile(r'.*\W(href|http|sql|select|update)\W.*',re.IGNORECASE) + rx = re.compile(r'.*\W(href|http|sql|select|update)\W.*', re.IGNORECASE) if rx.match(search): logger.info("Regex failed search") self.search_term_exists = False @@ -123,7 +121,7 @@ views.py). trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) if this_trait.dataset.type == "ProbeSet": trait_dict['symbol'] = this_trait.symbol - trait_dict['description'] = this_trait.description_display.decode('utf-8', 'replace') + trait_dict['description'] = this_trait.description_display trait_dict['location'] = this_trait.location_repr trait_dict['mean'] = "N/A" trait_dict['additive'] = "N/A" @@ -151,6 +149,10 @@ views.py). trait_dict['additive'] = "N/A" if this_trait.additive != "": trait_dict['additive'] = '%.3f' % this_trait.additive + # Convert any bytes in dict to a normal utf-8 string + for key in trait_dict.keys(): + if isinstance(trait_dict[key], bytes): + trait_dict[key] = trait_dict[key].decode('utf-8') trait_list.append(trait_dict) self.trait_list = json.dumps(trait_list) @@ -272,7 +274,7 @@ def get_GO_symbols(a_search): def insert_newlines(string, every=64): """ This is because it is seemingly impossible to change the width of the description column, so I'm just manually adding line breaks """ lines = [] - for i in xrange(0, len(string), every): + for i in range(0, len(string), every): lines.append(string[i:i+every]) return '\n'.join(lines) diff --git a/wqflask/wqflask/send_mail.py b/wqflask/wqflask/send_mail.py index bf5d0dd8..86e8a558 100644 --- a/wqflask/wqflask/send_mail.py +++ b/wqflask/wqflask/send_mail.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import datetime import time diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 356db7bc..37c1d6d5 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -1,35 +1,27 @@ -from __future__ import absolute_import, print_function, division - +import re import itertools -import numpy as np -from flask import Flask, g +from flask import g +from base import webqtlCaseData from pprint import pformat as pf -from scipy import stats -from base import webqtlCaseData -from base.trait import GeneralTrait -from utility import logger -from utility import webqtlUtil from utility import Plot from utility import Bunch -logger = logger.getLogger(__name__ ) - class SampleList(object): def __init__(self, dataset, sample_names, this_trait, - sample_group_type = "primary", - header = "Samples"): + sample_group_type="primary", + header="Samples"): self.dataset = dataset self.this_trait = this_trait self.sample_group_type = sample_group_type # primary or other self.header = header - self.sample_list = [] # The actual list + self.sample_list = [] # The actual list self.sample_attribute_values = {} self.get_attributes() @@ -40,39 +32,55 @@ class SampleList(object): for counter, sample_name in enumerate(sample_names, 1): sample_name = sample_name.replace("_2nd_", "") - if type(self.this_trait) is list: # ZS: self.this_trait will be a list if it is a Temp trait - if counter <= len(self.this_trait) and str(self.this_trait[counter-1]).upper() != 'X': - sample = webqtlCaseData.webqtlCaseData(name=sample_name, value=float(self.this_trait[counter-1])) + # ZS: self.this_trait will be a list if it is a Temp trait + if isinstance(self.this_trait, list): + if (counter <= len(self.this_trait) and + str(self.this_trait[counter-1]).upper() != 'X'): + sample = webqtlCaseData.webqtlCaseData( + name=sample_name, + value=float(self.this_trait[counter-1])) else: sample = webqtlCaseData.webqtlCaseData(name=sample_name) else: - # ZS - If there's no value for the sample/strain, create the sample object (so samples with no value are still displayed in the table) + # ZS - If there's no value for the sample/strain, + # create the sample object (so samples with no value + # are still displayed in the table) try: sample = self.this_trait.data[sample_name] except KeyError: sample = webqtlCaseData.webqtlCaseData(name=sample_name) sample.extra_info = {} - if self.dataset.group.name == 'AXBXA' and sample_name in ('AXB18/19/20','AXB13/14','BXA8/17'): + if (self.dataset.group.name == 'AXBXA' and + sample_name in ('AXB18/19/20', 'AXB13/14', 'BXA8/17')): sample.extra_info['url'] = "/mouseCross.html#AXB/BXA" sample.extra_info['css_class'] = "fs12" sample.this_id = str(counter) - # ZS: For extra attribute columns; currently only used by several datasets + # ZS: For extra attribute columns; currently only used by + # several datasets if self.sample_attribute_values: - sample.extra_attributes = self.sample_attribute_values.get(sample_name, {}) + sample.extra_attributes = self.sample_attribute_values.get( + sample_name, {}) self.sample_list.append(sample) self.se_exists = any(sample.variance for sample in self.sample_list) + self.num_cases_exists = any(sample.num_cases for sample in self.sample_list) + + first_attr_col = self.get_first_attr_col() + for sample in self.sample_list: + sample.first_attr_col = first_attr_col + self.do_outliers() def __repr__(self): return "<SampleList> --> %s" % (pf(self.__dict__)) def do_outliers(self): - values = [sample.value for sample in self.sample_list if sample.value != None] + values = [sample.value for sample in self.sample_list + if sample.value is not None] upper_bound, lower_bound = Plot.find_outliers(values) for sample in self.sample_list: @@ -100,9 +108,9 @@ class SampleList(object): key, name = attr self.attributes[key] = Bunch() self.attributes[key].name = name - self.attributes[key].distinct_values = [item.Value for item in values] - self.attributes[key].distinct_values.sort(key=natural_sort_key) - + self.attributes[key].distinct_values = [ + item.Value for item in values] + natural_sort(self.attributes[key].distinct_values) all_numbers = True for value in self.attributes[key].distinct_values: try: @@ -135,7 +143,8 @@ class SampleList(object): attribute_value = item.Value # ZS: If it's an int, turn it into one for sorting - # (for example, 101 would be lower than 80 if they're strings instead of ints) + # (for example, 101 would be lower than 80 if + # they're strings instead of ints) try: attribute_value = int(attribute_value) except ValueError: @@ -144,11 +153,21 @@ class SampleList(object): attribute_values[self.attributes[item.Id].name] = attribute_value self.sample_attribute_values[sample_name] = attribute_values -def natural_sort_key(x): - """Get expected results when using as a key for sort - ints or strings are sorted properly""" - - try: - x = int(x) - except ValueError: - pass - return x + def get_first_attr_col(self): + first_attr_col = 4 + if self.se_exists: + first_attr_col += 2 + if self.num_cases_exists: + first_attr_col += 1 + + return first_attr_col + +def natural_sort(list, key=lambda s: s): + """ + Sort the list into natural alphanumeric order. + """ + def get_alphanum_key_func(key): + def convert(text): return int(text) if text.isdigit() else text + return lambda s: [convert(c) for c in re.split('([0-9]+)', key(s))] + sort_key = get_alphanum_key_func(key) + list.sort(key=sort_key)
\ No newline at end of file diff --git a/wqflask/wqflask/show_trait/export_trait_data.py b/wqflask/wqflask/show_trait/export_trait_data.py index 253c887b..379b746c 100644 --- a/wqflask/wqflask/show_trait/export_trait_data.py +++ b/wqflask/wqflask/show_trait/export_trait_data.py @@ -1,9 +1,7 @@ -from __future__ import print_function, division - import simplejson as json from pprint import pformat as pf - +from functools import cmp_to_key from base.trait import create_trait from base import data_set @@ -16,10 +14,16 @@ def export_sample_table(targs): final_sample_data = meta_data + column_headers = ["Name", "Value"] + if any(sample["se"] for sample in sample_data['primary_samples']): + column_headers.append("SE") + if any(sample["num_cases"] for sample in sample_data['primary_samples']): + column_headers.append("N") + + final_sample_data.append(column_headers) for sample_group in ['primary_samples', 'other_samples']: for row in sample_data[sample_group]: sorted_row = dict_to_sorted_list(row) - print("sorted_row is:", pf(sorted_row)) final_sample_data.append(sorted_row) return trait_name, final_sample_data @@ -37,18 +41,26 @@ def get_export_metadata(trait_id, dataset_name): metadata.append(["Phenotype URL: " + "http://genenetwork.org/show_trait?trait_id=" + trait_id + "&dataset=" + dataset_name]) metadata.append(["Group: " + dataset.group.name]) metadata.append(["Phenotype: " + this_trait.description_display.replace(",", "\",\"")]) - metadata.append(["Authors: " + this_trait.authors]) - metadata.append(["Title: " + this_trait.title]) - metadata.append(["Journal: " + this_trait.journal]) + metadata.append(["Authors: " + (this_trait.authors if this_trait.authors else "N/A")]) + metadata.append(["Title: " + (this_trait.title if this_trait.title else "N/A")]) + metadata.append(["Journal: " + (this_trait.journal if this_trait.journal else "N/A")]) metadata.append(["Dataset Link: http://gn1.genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=" + dataset.name]) - metadata.append([]) + else: + metadata.append(["Record ID: " + trait_id]) + metadata.append(["Trait URL: " + "http://genenetwork.org/show_trait?trait_id=" + trait_id + "&dataset=" + dataset_name]) + if this_trait.symbol: + metadata.append(["Symbol: " + this_trait.symbol]) + metadata.append(["Dataset: " + dataset.name]) + metadata.append(["Group: " + dataset.group.name]) + + metadata.append([]) return metadata def dict_to_sorted_list(dictionary): - sorted_list = [item for item in dictionary.iteritems()] - sorted_list = sorted(sorted_list, cmp=cmp_samples) + sorted_list = [item for item in list(dictionary.items())] + sorted_list = sorted(sorted_list, key=cmp_to_key(cmp_samples)) sorted_values = [item[1] for item in sorted_list] return sorted_values @@ -71,4 +83,4 @@ def cmp_samples(a, b): else: return 1 else: - return -1
\ No newline at end of file + return -1 diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 30b03e66..edf9638c 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -1,9 +1,7 @@ -from __future__ import absolute_import, print_function, division - import string import os import datetime -import cPickle +import pickle import uuid import requests import json as json @@ -230,8 +228,8 @@ class ShowTrait(object): hddn = OrderedDict() if self.dataset.group.allsamples: - hddn['allsamples'] = string.join(self.dataset.group.allsamples, ' ') - hddn['primary_samples'] = string.join(self.primary_sample_names, ',') + hddn['allsamples'] = ''.join(self.dataset.group.allsamples) + hddn['primary_samples'] = ''.join(self.primary_sample_names) hddn['trait_id'] = self.trait_id hddn['trait_display_name'] = self.this_trait.display_name hddn['dataset'] = self.dataset.name @@ -260,7 +258,7 @@ class ShowTrait(object): hddn['export_data'] = "" hddn['export_format'] = "excel" if len(self.scales_in_geno) < 2: - hddn['mapping_scale'] = self.scales_in_geno[self.scales_in_geno.keys()[0]][0][0] + hddn['mapping_scale'] = self.scales_in_geno[list(self.scales_in_geno.keys())[0]][0][0] # We'll need access to this_trait and hddn in the Jinja2 Template, so we put it inside self self.hddn = hddn @@ -373,7 +371,7 @@ class ShowTrait(object): this_group = self.dataset.group.name # We're checking a string here! - assert isinstance(this_group, basestring), "We need a string type thing here" + assert isinstance(this_group, str), "We need a string type thing here" if this_group[:3] == 'BXD' and this_group != "BXD-Harvested": this_group = 'BXD' @@ -409,9 +407,10 @@ class ShowTrait(object): if not self.temp_trait: other_sample_names = [] - for sample in self.this_trait.data: - if self.this_trait.data[sample].name2 != self.this_trait.data[sample].name: - if (self.this_trait.data[sample].name2 in primary_sample_names) and (self.this_trait.data[sample].name not in primary_sample_names): + for sample in list(self.this_trait.data.keys()): + if (self.this_trait.data[sample].name2 != self.this_trait.data[sample].name): + if ((self.this_trait.data[sample].name2 in primary_sample_names) and + (self.this_trait.data[sample].name not in primary_sample_names)): primary_sample_names.append(self.this_trait.data[sample].name) primary_sample_names.remove(self.this_trait.data[sample].name2) @@ -568,7 +567,7 @@ def get_table_widths(sample_groups, has_num_cases=False): def has_num_cases(this_trait): has_n = False if this_trait.dataset.type != "ProbeSet" and this_trait.dataset.type != "Geno": - for name, sample in this_trait.data.iteritems(): + for name, sample in list(this_trait.data.items()): if sample.num_cases: has_n = True break @@ -625,7 +624,7 @@ def get_categorical_variables(this_trait, sample_list): if len(sample_list.attributes) > 0: for attribute in sample_list.attributes: attribute_vals = [] - for sample_name in this_trait.data.keys(): + for sample_name in list(this_trait.data.keys()): if sample_list.attributes[attribute].name in this_trait.data[sample_name].extra_attributes: attribute_vals.append(this_trait.data[sample_name].extra_attributes[sample_list.attributes[attribute].name]) else: @@ -640,7 +639,7 @@ def get_categorical_variables(this_trait, sample_list): def get_genotype_scales(genofiles): geno_scales = {} - if type(genofiles) is list: + if isinstance(genofiles, list): for the_file in genofiles: file_location = the_file['location'] geno_scales[file_location] = get_scales_from_genofile(file_location) diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index 1d28d76a..6c3fcf53 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from flask import Flask, g, url_for import string @@ -458,8 +456,8 @@ class SnpBrowser(object): function_list = [] if function_details: - function_list = string.split(string.strip(function_details), ",") - function_list = map(string.strip, function_list) + function_list = function_details.strip().split(",") + function_list = [item.strip() for item in function_list] function_list[0] = function_list[0].title() function_details = ", ".join(item for item in function_list) function_details = function_details.replace("_", " ") @@ -477,7 +475,7 @@ class SnpBrowser(object): the_bases = [] for j, item in enumerate(allele_value_list): - if item and isinstance(item, basestring): + if item and isinstance(item, str): this_base = [str(item), base_color_dict[item]] else: this_base = "" @@ -612,7 +610,7 @@ class SnpBrowser(object): this_allele_list = [] for item in self.allele_list: - if item and isinstance(item, basestring) and (item.lower() not in this_allele_list) and (item != "-"): + if item and isinstance(item, str) and (item.lower() not in this_allele_list) and (item != "-"): this_allele_list.append(item.lower()) total_allele_count = len(this_allele_list) @@ -724,12 +722,12 @@ def get_effect_details_by_category(effect_name = None, effect_value = None): new_codon_group_list = ['Start Gained'] codon_effect_group_list = ['Start Lost', 'Stop Gained', 'Stop Lost', 'Nonsynonymous', 'Synonymous'] - effect_detail_list = string.split(string.strip(effect_value), '|') - effect_detail_list = map(string.strip, effect_detail_list) + effect_detail_list = effect_value.strip().split('|') + effect_detail_list = [item.strip() for item in effect_detail_list] for index, item in enumerate(effect_detail_list): - item_list = string.split(string.strip(item), ',') - item_list = map(string.strip, item_list) + item_list = item.strip().split(',') + item_list = [item.strip() for item in item_list] gene_id = item_list[0] gene_name = item_list[1] @@ -748,13 +746,13 @@ def get_effect_details_by_category(effect_name = None, effect_value = None): if effect_name in new_codon_group_list: new_codon = item_list[6] tmp_list = [biotype, new_codon] - function_detail_list.append(string.join(tmp_list, ", ")) + function_detail_list.append(", ".join(tmp_list)) elif effect_name in codon_effect_group_list: old_new_AA = item_list[6] old_new_codon = item_list[7] codon_num = item_list[8] tmp_list = [biotype, old_new_AA, old_new_codon, codon_num] - function_detail_list.append(string.join(tmp_list, ", ")) + function_detail_list.append(", ".join(tmp_list)) else: function_detail_list.append(biotype) @@ -854,7 +852,7 @@ def get_gene_id_name_dict(species_id, gene_name_list): if len(gene_name_list) == 0: return "" gene_name_str_list = ["'" + gene_name + "'" for gene_name in gene_name_list] - gene_name_str = string.join(gene_name_str_list, ",") + gene_name_str = ",".join(gene_name_str_list) query = """ SELECT diff --git a/wqflask/wqflask/static/new/css/main.css b/wqflask/wqflask/static/new/css/main.css index 097cd997..d5fb8a61 100644 --- a/wqflask/wqflask/static/new/css/main.css +++ b/wqflask/wqflask/static/new/css/main.css @@ -33,4 +33,8 @@ .collapsing { overflow: hidden!important; } +} + +.checkbox { + min-height: 20px; }
\ No newline at end of file diff --git a/wqflask/wqflask/static/new/css/show_trait.css b/wqflask/wqflask/static/new/css/show_trait.css index 7a7f5455..d8964f5d 100644 --- a/wqflask/wqflask/static/new/css/show_trait.css +++ b/wqflask/wqflask/static/new/css/show_trait.css @@ -1,17 +1,17 @@ tr .outlier { - background-color: #ffff99; + background-color: #ffff99; } table.dataTable tbody tr.selected { - background-color: #ffee99; + background-color: #ffee99; } #bar_chart_container { - overflow-x:scroll; + overflow-x:scroll; } div.sample_group { - overflow: auto; # needed because it contains float dataTable wrapper + overflow: auto; # needed because it contains float dataTable wrapper } .js-plotly-plot .plotly .modebar { @@ -35,8 +35,13 @@ table.dataTable thead th, table.dataTable tfoot { padding: 4px 18px 4px 10px; } +/* Header for a column with a numeric value that should be right-aligned */ +table.dataTable thead th div.numeric_header { + text-align: right; +} + table.dataTable tbody td { - padding: 4px 15px 2px 10px; + padding: 2px 15px 0px 10px; } table.dataTable.cell-border tbody th, table.dataTable.cell-border tbody td { @@ -54,8 +59,204 @@ table.dataTable.cell-border tbody tr td:first-child { .trait_value_input { text-align: right; +} .glyphicon { position: relative; top: 2px; +} + +.showtrait-main-div { + min-width: 700px; +} + +table.dataTable tbody td.column_name-Checkbox { + padding: 1px 0px 0px 9px; /* The left padding here is because text-align:center does not seem to be working properly. No idea why this is. */ + text-align: center; + vertical-align: middle; +} + +table.dataTable tbody td.column_name-Index { + padding: 2px 4px 0px 2px; + text-align: right; +} + +/* the column with +/- in it that appears in the table when there's an SE column */ +table.dataTable tbody td.column_name-PlusMinus { + padding-left: 2px; + padding-right: 2px; + text-align: center; +} + +table.dataTable tbody td.column_name-Value, table.dataTable tbody td.column_name-SE, table.dataTable tbody td.column_name-num_cases { + text-align: right; +} + +div.btn-toolbar { + margin-bottom:15px; +} + +/* div containing the form options for each segment of the trait page - correlations, statistics, mapping, etc */ +div.section-form-div { + padding: 20px; +} + +table.left-float { + float: left; +} + +div.scatterplot-btn-div { + margin-bottom:40px; +} + +div.correlation-main { + min-width: 1200px; +} + +div.correlation-options { + min-width: 700px; +} + +.min-expr-field { + width: 70px; +} + +div.p-range-slider { + width: 200px; + margin-left: 15px; +} + +span.p-range-lower { + margin-top: 5px; + font: 400 12px Arial; + color: #888; + display: inline-block; +} + +span.p-range-upper { + font: 400 12px Arial; + color: #888; + display: inline-block; + margin: 5px 0px 0px 170px; +} + +input.corr-location { + width: 50px; + display: inline; +} + +div.block-by-index-div { + margin-bottom: 10px; +} + +div.block-by-attribute-div { + margin-top:10px; + margin-bottom:10px; +} + +div.normalize-div { + margin-top:10px; +} + +div.mapping-main { + min-width: 1200px; +} +div.mapping-options { + min-width: 500px; +} + +div.covar-options { + padding-top: 7px; +} + +.control-label { + text-align: right; +} + +.chr-select, .maf-select, .scale-select, .reaper-ver-select { + width: 80px; +} + +span.covar-text { + font-size: 13px; + font-weight: 400; +} + +div.select-covar-div { + margin-bottom: 10px; +} + +.select-covar-button { + width: 80px; + padding-right: 10px; +} + +.selected-covariates { + overflow-y: scroll; + resize: none; + width: 200px; +} + +.cofactor-input { + width: 160px; + display: inline-block; +} + +.no-control-radio { + margin-left: 10px; +} + +.map-method-text { + margin-top: 20px; +} + +.rqtl-description { + padding-top: 40px; + display: none; +} + +div.sample-table-container { + padding-bottom: 10px; +} + +div.sample-table-search-container { + display: inline; + height: 40px; +} + +input.sample-table-search { + width: 200px; + display: inline; + float: left; + vertical-align: top; +} + +div.sample-table-export-container { + padding-left: 10px; + display: inline; +} + +div.export-code-container { + padding-top: 20px; + width: 600px; + display: none; +} + +.export-code-field { + padding-top: 0px; + padding-bottom: 0px; + height: 150px; +} + +table.sample-table { + float: left; + width:100%; +} + +input.trait-value-input { + text-align: right; +} + +div.inline-div { + display: inline; }
\ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js index a8a3041d..3e414034 100644 --- a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js +++ b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js @@ -104,7 +104,7 @@ submit_click = function() { //covariates_display_string = covariates_display_string.substring(0, covariates_display_string.length - 2) $("input[name=covariates]").val(covariates_string) - $(".selected_covariates").val(covariates_display_string) + $(".selected-covariates").val(covariates_display_string) return $.colorbox.close(); }; @@ -114,7 +114,7 @@ trait_click = function() { trait = $(this).parent().find('.trait').text(); dataset = $(this).parent().find('.dataset').data("dataset"); $("input[name=covariates]").val(trait + ":" + dataset) - $(".selected_covariates").text(trait) + $(".selected-covariates").text(trait) return $.colorbox.close(); }; diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 49822cd2..5a4f151c 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -75,7 +75,7 @@ build_columns = function() { ); } - if (js_data.has_num_cases === "true") { + if (js_data.has_num_cases === true) { column_list.push( { 'title': "<div style='text-align: right;'>N</div>", @@ -101,7 +101,8 @@ build_columns = function() { 'type': "natural", 'data': null, 'render': function(data, type, row, meta) { - attr_name = Object.keys(data.extra_attributes).sort((a, b) => (a > b) ? 1 : -1)[meta.col - data.first_attr_col] + attr_name = Object.keys(data.extra_attributes).sort()[meta.col - data.first_attr_col] + if (attr_name != null && attr_name != undefined){ return data.extra_attributes[attr_name] } else { @@ -111,7 +112,6 @@ build_columns = function() { } ) } - return column_list } @@ -132,23 +132,22 @@ var primary_table = $('#samples_primary').DataTable( { $('td', row).eq(3).addClass("column_name-Value") if (js_data.se_exists) { $('td', row).eq(5).addClass("column_name-SE") - if (js_data.has_num_cases === "true") { + if (js_data.has_num_cases === true) { $('td', row).eq(6).addClass("column_name-num_cases") } else { - if (js_data.has_num_cases === "true") { + if (js_data.has_num_cases === true) { $('td', row).eq(4).addClass("column_name-num_cases") } } } else { - if (js_data.has_num_cases === "true") { + if (js_data.has_num_cases === true) { $('td', row).eq(4).addClass("column_name-num_cases") } } - sorted_key_list = Object.keys(js_data.attributes).sort() - for (i=0; i < sorted_key_list.length; i++) { - $('td', row).eq(attribute_start_pos + i + 1).addClass("column_name-" + js_data.attributes[sorted_key_list[i]].name) - $('td', row).eq(attribute_start_pos + i + 1).attr("style", "text-align: " + js_data.attributes[sorted_key_list[i]].alignment + "; padding-top: 2px; padding-bottom: 0px;") + for (i=0; i < attr_keys.length; i++) { + $('td', row).eq(attribute_start_pos + i).addClass("column_name-" + js_data.attributes[attr_keys[i]].name) + $('td', row).eq(attribute_start_pos + i).attr("style", "text-align: " + js_data.attributes[attr_keys[i]].alignment + "; padding-top: 2px; padding-bottom: 0px;") } }, 'data': js_data['sample_lists'][0], @@ -190,23 +189,22 @@ if (js_data.sample_lists.length > 1){ $('td', row).eq(3).addClass("column_name-Value") if (js_data.se_exists) { $('td', row).eq(5).addClass("column_name-SE") - if (js_data.has_num_cases === "true") { + if (js_data.has_num_cases === true) { $('td', row).eq(6).addClass("column_name-num_cases") } else { - if (js_data.has_num_cases === "true") { + if (js_data.has_num_cases === true) { $('td', row).eq(4).addClass("column_name-num_cases") } } } else { - if (js_data.has_num_cases === "true") { + if (js_data.has_num_cases === true) { $('td', row).eq(4).addClass("column_name-num_cases") } } - sorted_key_list = Object.keys(js_data.attributes).sort() - for (i=0; i < sorted_key_list.length; i++) { - $('td', row).eq(attribute_start_pos + i + 1).addClass("column_name-" + js_data.attributes[sorted_key_list[i]].name) - $('td', row).eq(attribute_start_pos + i + 1).attr("style", "text-align: " + js_data.attributes[sorted_key_list[i]].alignment + "; padding-top: 2px; padding-bottom: 0px;") + for (i=0; i < attr_keys.length; i++) { + $('td', row).eq(attribute_start_pos + i + 1).addClass("column_name-" + js_data.attributes[attr_keys[i]].name) + $('td', row).eq(attribute_start_pos + i + 1).attr("style", "text-align: " + js_data.attributes[attr_keys[i]].alignment + "; padding-top: 2px; padding-bottom: 0px;") } }, 'data': js_data['sample_lists'][1], diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index eaad0a4c..a34811f8 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -107,14 +107,14 @@ d3.select("#select_covariates").on("click", (function(_this) { })(this)); $("#remove_covariates").click(function () { $("input[name=covariates]").val("") - $(".selected_covariates").val("") + $(".selected-covariates").val("") }); $(".select_covariates").click(function () { open_covariate_selection(); }); $(".remove_covariates").click(function () { $("input[name=covariates]").val("") - $(".selected_covariates").val("") + $(".selected-covariates").val("") }); d3.select("#clear_compare_trait").on("click", (function(_this) { return function() { @@ -601,7 +601,7 @@ block_by_attribute_value = function() { var row; if ($.trim($(element).text()) === exclude_by_value) { row = $(element).parent('tr'); - return $(row).find(".trait_value_input").val("x"); + return $(row).find(".trait-value-input").val("x"); } }; })(this)); @@ -641,13 +641,14 @@ block_by_index = function() { for (_k = 0, _len1 = index_list.length; _k < _len1; _k++) { index = index_list[_k]; val_nodes[index - 1].childNodes[0].value = "x"; + } }; hide_no_value = function() { return $('.value_se').each((function(_this) { return function(_index, element) { - if ($(element).find('.trait_value_input').val() === 'x') { + if ($(element).find('.trait-value-input').val() === 'x') { return $(element).hide(); } }; @@ -658,7 +659,7 @@ $('#hide_no_value').click(hide_no_value); block_outliers = function() { return $('.outlier').each((function(_this) { return function(_index, element) { - return $(element).find('.trait_value_input').val('x'); + return $(element).find('.trait-value-input').val('x'); }; })(this)); }; @@ -667,7 +668,6 @@ $('#block_outliers').click(block_outliers); reset_samples_table = function() { $('input[name="transform"]').val(""); $('span[name="transform_text"]').text("") - tables = ['samples_primary', 'samples_other']; for (_i = 0, _len = tables.length; _i < _len; _i++) { table = tables[_i]; @@ -871,7 +871,7 @@ $('#normalize').hover(function(){ $('#normalize').click(normalize_data) switch_qnorm_data = function() { - return $('.trait_value_input').each((function(_this) { + return $('.trait-value-input').each((function(_this) { return function(_index, element) { transform_val = $(element).data('transform') if (transform_val != "") { @@ -885,26 +885,65 @@ $('#qnorm').click(switch_qnorm_data); get_sample_table_data = function(table_name) { var samples; samples = []; - $('#' + table_name).find('.value_se').each((function(_this) { - return function(_index, element) { - var attribute_info, key, row_data, _ref; - row_data = {}; - row_data.name = $.trim($(element).find('.column_name-Sample').text()); - row_data.value = $(element).find('.edit_sample_value:eq(0)').val(); - if ($(element).find('.edit_sample_se').length > 0) { - row_data.se = $(element).find('.edit_sample_se').val(); + + var se_exists = false; + var n_exists = false; + + if ($('#' + table_name).length){ + table_api = $('#' + table).DataTable(); + sample_vals = []; + + name_nodes = table_api.column(2).nodes().to$(); + val_nodes = table_api.column(3).nodes().to$(); + if (js_data.se_exists){ + var_nodes = table_api.column(5).nodes().to$(); + if (js_data.has_num_cases) { + n_nodes = table_api.column(6).nodes().to$(); } - if ($(element).find('.edit_sample_num_cases').length > 0) { - row_data.num_cases = $(element).find('.edit_sample_num_cases').val(); + } else { + if (js_data.has_num_cases){ + n_nodes = table_api.column(4).nodes().to$(); } - attr_keys = Object.keys(js_data.attributes).sort() - for (i=0; i < attr_keys.length; i++) { - attribute_info = js_data.attributes[attr_keys[i]]; - row_data[attribute_info.name] = $.trim($(element).find('.column_name-' + attribute_info.name.replace(" ", "_").replace("/", "\\/")).text()); + } + + for (_j = 0; _j < val_nodes.length; _j++){ + sample_val = val_nodes[_j].childNodes[0].value + sample_name = $.trim(name_nodes[_j].childNodes[0].textContent) + if (is_number(sample_val) && sample_val !== "") { + sample_val = parseFloat(sample_val); + if (typeof var_nodes == 'undefined'){ + sample_var = null; + } else { + sample_var = var_nodes[_j].childNodes[0].value; + if (is_number(sample_var)) { + sample_var = parseFloat(sample_var); + se_exists = true; + } else { + sample_var = null; + } + } + if (typeof n_nodes == 'undefined'){ + sample_n = null; + } else { + sample_n = n_nodes[_j].childNodes[0].value; + if (is_number(sample_n)) { + n_exists = true; + sample_n = parseInt(sample_n); + } else { + sample_n = null; + } + } + row_dict = { + name: sample_name, + value: sample_val, + se: sample_var, + num_cases: sample_n + } + samples.push(row_dict) } - return samples.push(row_data); - }; - })(this)); + } + } + return samples; }; export_sample_table_data = function() { diff --git a/wqflask/wqflask/submit_bnw.py b/wqflask/wqflask/submit_bnw.py index 59e60dfd..a0e84c8c 100644 --- a/wqflask/wqflask/submit_bnw.py +++ b/wqflask/wqflask/submit_bnw.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from base.trait import GeneralTrait from base import data_set from utility import helper_functions @@ -8,4 +6,4 @@ import utility.logger logger = utility.logger.getLogger(__name__ ) def get_bnw_input(start_vars): - logger.debug("BNW VARS:", start_vars)
\ No newline at end of file + logger.debug("BNW VARS:", start_vars) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 0b12eaae..33a37594 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -65,7 +65,7 @@ </tr> </thead> <tbody> - {% for key, value in group_masks.iteritems() %} + {% for key, value in group_masks.items() %} <tr> <td>{{ value.name }}</td> <td>{{ value.data }}</td> diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index b44538cf..0f4e5ef5 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -9,6 +9,9 @@ <meta name="description" content=""> <meta name="author" content=""> + <script type="text/javascript"> + var pageLoadStart = Date.now(); + </script> <link rel="icon" type="image/png" sizes="64x64" href="/static/new/images/CITGLogo.png"> <link rel="apple-touch-icon" type="image/png" sizes="64x64" href="/static/new/images/CITGLogo.png"> <link REL="stylesheet" TYPE="text/css" href="/static/packages/bootstrap/css/bootstrap.css" /> @@ -203,6 +206,7 @@ {% if version: %} <p><small>GeneNetwork {{ version }}</small></p> {% endif %} + <p> It took the server {{ g.request_time() }} seconds to process this page.</p> </div> <div class="col-xs-2"> @@ -256,6 +260,12 @@ {% block js %} {% endblock %} - + <script type="text/javascript"> + $(window).load(function() { + let timeToLoad = document.createElement("p"); + timeToLoad.innerHTML = "It took your browser " + ((Date.now() - pageLoadStart)/1000) + " second(s) to render this page"; + document.querySelector("footer .row .col-xs-6").appendChild(timeToLoad); + }); + </script> </body> </html> diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index fb218e29..134f15be 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -170,7 +170,8 @@ {% endif %} <td data-export="{{ trait.description_display }}">{% if trait.description_display|length > 70 %}{{ trait.description_display[:70] }}...{% else %}{{ trait.description_display }}{% endif %}</td> {% if trait.authors %} - <td data-export="{{ trait.authors }}">{% if trait.authors.split(',') > 6 %}{{ trait.authors.split(',')[:6]|join(', ') }}, et al.{% else %}{{ trait.authors }}{% endif %}</td> + {% set authors_list = trait.authors.split(',') %} + <td data-export="{{ trait.authors }}">{% if authors_list|length > 6 %}{{ authors_list[:6]|join(', ') }}, et al.{% else %}{{ trait.authors }}{% endif %}</td> {% else %} <td data-export="N/A">N/A</td> {% endif %} diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 15ab4080..9b335dfe 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -1,7 +1,7 @@ <title>Loading {{ start_vars.tool_used }} Results</title> <link REL="stylesheet" TYPE="text/css" href="/static/packages/bootstrap/css/bootstrap.css" /> <form method="post" action="" name="loading_form" id="loading_form" class="form-horizontal"> - {% for key, value in start_vars.iteritems() %} + {% for key, value in start_vars.items() %} <input type="hidden" name="{{ key }}" value="{{ value }}"> {% endfor %} <div class="container"> @@ -44,4 +44,4 @@ $("#loading_form").attr("action", "{{ start_vars.form_url }}"); setTimeout(function(){ $("#loading_form").submit()}, 350); -</script>
\ No newline at end of file +</script> diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index 5ed7a90b..7b74a3f6 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -37,7 +37,7 @@ <input type="hidden" name="covariates" value=""> <input type="hidden" name="transform" value=""> - <div class="container" style="min-width: 700px;"> + <div class="container showtrait-main-div"> <div class="panel-group" id="accordion"> <div class="panel panel-default"> <div class="panel-heading" data-toggle="collapse" data-parent="#accordion" data-target="#collapseOne" aria-expanded="true"> diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html index 9420c9c6..bc1d4091 100644 --- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html +++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html @@ -1,149 +1,150 @@ -<div> - <div class="col-xs-6" style="min-width: 800px;"> - <div style="padding: 20px" class="form-horizontal"> - - <div class="form-group"> - <label for="corr_type" class="col-xs-2 control-label">Method</label> - <div class="col-xs-3 controls"> - <select name="corr_type" class="form-control"> - <option value="sample">Sample r</option> - <option value="lit">Literature r</option> - <option value="tissue">Tissue r</option> - </select> - </div> - </div> - - <div class="form-group"> - <label for="corr_dataset" class="col-xs-2 control-label">Database</label> - <div class="col-xs-10 controls"> - <select name="corr_dataset" class="form-control"> - {% for tissue in corr_tools.dataset_menu %} - {% if tissue.tissue %} - <optgroup label="{{ tissue.tissue }} ------"> - {% endif %} - {% for dataset in tissue.datasets %} - <option value="{{ dataset[1] }}" - {% if corr_tools.dataset_menu_selected == dataset[1] %} - selected - {% endif %}> - {{ dataset[0] }} - </option> - {% endfor %} - {% if tissue.tissue %} - </optgroup> - {% endif %} - {% endfor %} - </select> - </div> - </div> - - <div class="form-group"> - <label for="corr_return_results" class="col-xs-2 control-label">Return</label> - <div class="col-xs-4 controls"> - <select name="corr_return_results" class="form-control"> - {% for return_result in corr_tools.return_results_menu %} - <option value="{{ return_result }}" - {% if corr_tools.return_results_menu_selected == return_result %} - selected - {% endif %}> - Top {{ return_result }} - </option> - {% endfor %} - </select> - </div> - </div> - - <div class="form-group"> - <label for="corr_samples_group" class="col-xs-2 control-label">Samples</label> - <div class="col-xs-4 controls"> - <select name="corr_samples_group" class="form-control"> - {% for group, pretty_group in sample_group_types.items() %} - <option value="{{ group }}">{{ pretty_group }}</option> - {% endfor %} - </select> - </div> - </div> - - <div id="corr_sample_method" class="form-group"> - <label for="corr_sample_method" class="col-xs-2 control-label">Type</label> - <div class="col-xs-4 controls"> - <select name="corr_sample_method" class="form-control"> - <option value="pearson">Pearson</option> - <option value="spearman">Spearman Rank</option> - <option value="bicor">Biweight Midcorrelation</option> - </select> - </div> - </div> - {% if dataset.type != "Publish" %} - <div class="form-group"> - <label class="col-xs-2 control-label">Min Expr</label> - <div class="col-xs-4 controls"> - <input name="min_expr" value="" type="text" class="form-control" style="width: 70px;"> - </div> - </div> - <div class="form-group"> - <label class="col-xs-2 control-label">Location</label> - <div class="col-xs-5 controls"> - <span> - Chr: <input name="loc_chr" value="" type="text" class="form-control" style="width: 50px; display: inline;"> - Mb: <input name="min_loc_mb" value="" type="text" class="form-control" style="width: 50px; display: inline;"> to <input name="max_loc_mb" value="" type="text" class="form-control" style="width: 50px; display: inline;"> - </span> - <br> - </div> - </div> - {% endif %} - <div class="form-group"> - <label class="col-xs-2 control-label">Range</label> - <div class="col-xs-5 controls"> - <input name="p_range_lower" value="" type="hidden"> - <input name="p_range_upper" value="" type="hidden"> - <span style="display: inline;"> - <div id="p_range_slider" style="width: 200px;"></div> - <span style="font: 400 12px Arial; color: #888; display: inline; margin: 25px 0; width: 20px;" id="p_range_lower"></span> - <span> </span> - <span style="font: 400 12px Arial; color: #888; display: inline; margin: 15px 0; width: 20px;" id="p_range_upper"></span> - </span> - </div> - </div> - - <div class="form-group"> - <label for="corr_sample_method" class="col-xs-2 control-label"></label> - <div class="col-xs-3 controls"> - <input type="button" class="btn corr_compute submit_special btn-success" data-url="/corr_compute" title="Compute Correlation" value="Compute"> - </div> - </div> +<div class="row correlation-main"> + <div class="col-xs-3 correlation-options"> + <div class="form-horizontal section-form-div"> + + <div class="form-group"> + <label for="corr_type" class="col-xs-2 control-label">Method</label> + <div class="col-xs-3 controls"> + <select name="corr_type" class="form-control"> + <option value="sample">Sample r</option> + <option value="lit">Literature r</option> + <option value="tissue">Tissue r</option> + </select> + </div> + </div> + + <div class="form-group"> + <label for="corr_dataset" class="col-xs-2 control-label">Database</label> + <div class="col-xs-10 controls"> + <select name="corr_dataset" class="form-control"> + {% for tissue in corr_tools.dataset_menu %} + {% if tissue.tissue %} + <optgroup label="{{ tissue.tissue }} ------"> + {% endif %} + {% for dataset in tissue.datasets %} + <option value="{{ dataset[1] }}" + {% if corr_tools.dataset_menu_selected == dataset[1] %} + selected + {% endif %}> + {{ dataset[0] }} + </option> + {% endfor %} + {% if tissue.tissue %} + </optgroup> + {% endif %} + {% endfor %} + </select> + </div> + </div> + + <div class="form-group"> + <label for="corr_return_results" class="col-xs-2 control-label">Return</label> + <div class="col-xs-4 controls"> + <select name="corr_return_results" class="form-control"> + {% for return_result in corr_tools.return_results_menu %} + <option value="{{ return_result }}" + {% if corr_tools.return_results_menu_selected == return_result %} + selected + {% endif %}> + Top {{ return_result }} + </option> + {% endfor %} + </select> + </div> + </div> + + <div class="form-group"> + <label for="corr_samples_group" class="col-xs-2 control-label">Samples</label> + <div class="col-xs-4 controls"> + <select name="corr_samples_group" class="form-control"> + {% for group, pretty_group in sample_group_types.items() %} + <option value="{{ group }}">{{ pretty_group }}</option> + {% endfor %} + </select> + </div> + </div> + + <div id="corr_sample_method" class="form-group"> + <label for="corr_sample_method" class="col-xs-2 control-label">Type</label> + <div class="col-xs-4 controls"> + <select name="corr_sample_method" class="form-control"> + <option value="pearson">Pearson</option> + <option value="spearman">Spearman Rank</option> + <option value="bicor">Biweight Midcorrelation</option> + </select> + </div> + </div> + {% if dataset.type != "Publish" %} + <div class="form-group"> + <label class="col-xs-2 control-label">Min Expr</label> + <div class="col-xs-4 controls"> + <input name="min_expr" value="" type="text" class="form-control min-expr-field"> + </div> + </div> + <div class="form-group"> + <label class="col-xs-2 control-label">Location</label> + <div class="col-xs-6 controls"> + <span> + Chr: <input name="loc_chr" value="" type="text" class="form-control corr-location"> + Mb: <input name="min_loc_mb" value="" type="text" class="form-control corr-location"> to <input name="max_loc_mb" value="" type="text" class="form-control corr-location"> + </span> + <br> + </div> + </div> + {% endif %} + <div class="form-group"> + <label class="col-xs-2 control-label">Range</label> + <div class="col-xs-5 controls"> + <input name="p_range_lower" value="" type="hidden"> + <input name="p_range_upper" value="" type="hidden"> + <span class="inline-div"> + <div id="p_range_slider" class="p-range-slider"></div> + <span id="p_range_lower" class="p-range-lower"></span> + <span id="p_range_upper" class="p-range-upper"></span> + </span> + </div> + </div> + + <div class="form-group"> + <label for="corr_sample_method" class="col-xs-2 control-label"></label> + <div class="col-xs-3 controls"> + <input type="button" class="btn corr_compute submit_special btn-success" data-url="/corr_compute" title="Compute Correlation" value="Compute"> + </div> + </div> + </div> </div> - </div> - <div> - <span id="sample_r_desc" class="correlation_desc fs12"> - The <a href="http://genenetwork.org/correlationAnnotation.html#genetic_r">Sample Correlation</a> - is computed - between trait data and any - other traits in the sample database selected above. Use - <a href="http://www.genenetwork.org/glossary.html#Correlations">Spearman - Rank</a> - when the sample size is small (<20) or when there are influential outliers. - </span> - <span id="lit_r_desc" style="display: none;" class="correlation_desc fs12"> - The <a href="http://genenetwork.org/correlationAnnotation.html#literatureCorr">Literature Correlation</a> - (Lit r) between - this gene and all other genes is computed<br> - using the <a href="https://grits.eecs.utk.edu/sgo/sgo.html"> - Semantic Gene Organizer</a> - and human, rat, and mouse data from PubMed. - Values are ranked by Lit r, but Sample r and Tissue r are also displayed.<br> - <a href="http://genenetwork.org/glossary.html#Literature">More on using Lit r</a> - </span> - <span id="tissue_r_desc" style="display: none;" class="correlation_desc fs12"> - The <a href="http://genenetwork.org/webqtl/main.py?FormID=tissueCorrelation">Tissue Correlation</a> - (Tissue r) - estimates the similarity of expression of two genes - or transcripts across different cells, tissues, or organs - (<a href="http://genenetwork.org/correlationAnnotation.html#tissueCorr">glossary</a>). - Tissue correlations - are generated by analyzing expression in multiple samples usually taken from single cases.<br> - <strong>Pearson</strong> and <strong>Spearman Rank</strong> correlations have been - computed for all pairs of genes using data from mouse samples.<br> - </span> - </div> -</div>
\ No newline at end of file + <div class="col-xs-4"> + <div class="section-form-div"> + <span id="sample_r_desc" class="correlation_desc"> + The <a href="http://genenetwork.org/correlationAnnotation.html#genetic_r">Sample Correlation</a> + is computed + between trait data and any + other traits in the sample database selected above. Use + <a href="http://www.genenetwork.org/glossary.html#Correlations">Spearman + Rank</a> + when the sample size is small (<20) or when there are influential outliers. + </span> + <span id="lit_r_desc" style="display: none;" class="correlation_desc"> + The <a href="http://genenetwork.org/correlationAnnotation.html#literatureCorr">Literature Correlation</a> + (Lit r) between + this gene and all other genes is computed<br> + using the <a href="https://grits.eecs.utk.edu/sgo/sgo.html"> + Semantic Gene Organizer</a> + and human, rat, and mouse data from PubMed. + Values are ranked by Lit r, but Sample r and Tissue r are also displayed.<br> + <a href="http://genenetwork.org/glossary.html#Literature">More on using Lit r</a> + </span> + <span id="tissue_r_desc" style="display: none;" class="correlation_desc"> + The <a href="http://genenetwork.org/webqtl/main.py?FormID=tissueCorrelation">Tissue Correlation</a> + (Tissue r) + estimates the similarity of expression of two genes + or transcripts across different cells, tissues, or organs + (<a href="http://genenetwork.org/correlationAnnotation.html#tissueCorr">glossary</a>). + Tissue correlations + are generated by analyzing expression in multiple samples usually taken from single cases.<br> + <strong>Pearson</strong> and <strong>Spearman Rank</strong> correlations have been + computed for all pairs of genes using data from mouse samples.<br> + </span> + </div> + </div> + </div>
\ No newline at end of file diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 4aced50c..d2999eef 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -1,20 +1,20 @@ -<table class="table" style="max-width: 1400px;"> +<table class="table"> <tr> - <td width="150px"><b>Group</b></td> + <td><b>Group</b></td> <td>{{ this_trait.dataset.group.species[0]|upper }}{{ this_trait.dataset.group.species[1:] }}: {{ this_trait.dataset.group.name }} group</td> </tr> {% if this_trait.dataset.type == 'Publish' %} <tr> <td><b>Phenotype</b></td> - <td><div style="width:40%;">{{ this_trait.description_fmt }}</div></td> + <td><div>{{ this_trait.description_fmt }}</div></td> </tr> <tr> <td><b>Authors</b></td> - <td><div style="width:40%;">{{ this_trait.authors }}</div></td> + <td><div>{{ this_trait.authors }}</div></td> </tr> <tr> <td><b>Title</b></td> - <td><div style="width:40%;">{{ this_trait.title }}</div></td> + <td><div>{{ this_trait.title }}</div></td> </tr> <tr> <td><b>Journal</b></td> @@ -213,7 +213,7 @@ {% endif %} </table> -<div style="margin-bottom:15px;" class="btn-toolbar"> +<div class="btn-toolbar"> <div class="btn-group"> <button type="button" id="add_to_collection" class="btn btn-success" title="Add to Collection">Add</button> {% if this_trait.dataset.type == 'ProbeSet' or this_trait.dataset.type == 'Geno' %} diff --git a/wqflask/wqflask/templates/show_trait_edit_data.html b/wqflask/wqflask/templates/show_trait_edit_data.html index a8ed91b1..2e730248 100644 --- a/wqflask/wqflask/templates/show_trait_edit_data.html +++ b/wqflask/wqflask/templates/show_trait_edit_data.html @@ -1,10 +1,10 @@ <div> {% for sample_type in sample_groups %} - <div style="padding-bottom: 10px;"> - <div style="display: inline; height: 40px;"> - <input type="text" id="{{ sample_type.sample_group_type }}_searchbox" class="form-control" style="width: 200px; display: inline; float: left; vertical-align: top;" placeholder="Search This Table For ..."> + <div class="sample-table-container"> + <div class="sample-table-search-container"> + <input type="text" id="{{ sample_type.sample_group_type }}_searchbox" class="form-control sample-table-search" placeholder="Search This Table For ..."> </div> - <div style="padding-left: 10px; display: inline;"> + <div class="sample-table-export-container"> <button class="btn btn-default export">Export <span class="glyphicon glyphicon-download-alt"></span></button> <select class="select optional span2 export_format"> <option value="excel">Excel</option> @@ -12,33 +12,33 @@ </select> <input type="button" class="btn btn-success reset" value="Reset"> </div> - </div> - <div id="export_code" style="width: 600px; display: none;"> - <pre style="padding-top: 0px; padding-bottom: 0px; height: 150px;"> - <code> -# read into R -trait <- read.csv("{{ this_trait.display_name}}.csv", header = TRUE, comment.char = "#") + <div id="export_code" class="export-code-container"> + <pre class="export-code_field"> + <code> + # read into R + trait <- read.csv("{{ this_trait.display_name}}.csv", header = TRUE, comment.char = "#") -# read into python -import pandas as pd -trait = pd.read_csv("{{ this_trait.display_name}}.csv", header = 0, comment = "#") - </code> - </pre> + # read into python + import pandas as pd + trait = pd.read_csv("{{ this_trait.display_name}}.csv", header = 0, comment = "#") + </code> + </pre> + </div> </div> - {% set outer_loop = loop %} - <div class="sample_group" style="width:{{ trait_table_width }};"> - <div style="position: relative;"> - <div style="display: inline;"><h3 style="float: left;">{{ sample_type.header }}<span name="transform_text"></span></h3> - </div> - <div id="table_container"> - <table class="table-hover table-striped cell-border" id="samples_{{ sample_type.sample_group_type }}" style="float: left; width:100%;"> - <tbody> - <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td> - </tbody> - </table> - </div> + {% set outer_loop = loop %} + <div class="sample_group" style="width:{{ trait_table_width }};"> + <div style="position: relative;"> + <div class="inline-div"><h3 style="float: left;">{{ sample_type.header }}<span name="transform_text"></span></h3></div> </div> - <br> + <div id="table_container"> + <table class="table-hover table-striped cell-border sample-table" id="samples_{{ sample_type.sample_group_type }}"> + <tbody> + <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td> + </tbody> + </table> + </div> + </div> + <br> {% endfor %} <input type="hidden" name="Default_Name"> </div> diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 94388b2f..eca436c6 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -1,38 +1,35 @@ -<div> +<div class="row mapping-main"> {% if dataset.group.mapping_names|length > 0 %} - <div class="col-xs-3" style="min-width: 400px;"> + <div class="col-xs-2 mapping-options"> <div class="tabbable"> <!-- Only required for left/right tabs --> <ul class="nav nav-pills"> {% for mapping_method in dataset.group.mapping_names %} {% if mapping_method == "GEMMA" %} - <li class="gemma_tab mapping_tab {% if dataset.group.mapping_id == '1' or dataset.group.mapping_id == '2' %}active{% endif %}"> + <li class="gemma-tab mapping-tab {% if dataset.group.mapping_id == '1' or dataset.group.mapping_id == '2' %}active{% endif %}"> <a href="#gemma" data-toggle="tab">GEMMA</a> </li> {% elif mapping_method == "R/qtl" %} - <li class="rqtl_geno_tab mapping_tab {% if dataset.group.mapping_id == '3' %}active{% endif %}"> + <li class="rqtl-geno-tab mapping-tab {% if dataset.group.mapping_id == '3' %}active{% endif %}"> <a href="#rqtl_geno" data-toggle="tab">R/qtl</a> </li> {% elif mapping_method == "QTLReaper" %} - <li class="reaper_tab mapping_tab"> + <li class="reaper-tab mapping-tab"> <a href="#interval_mapping" data-toggle="tab">Haley-Knott Regression</a> </li> {% endif %} {% endfor %} - <!--<li> - <a href="#pair_scan" data-toggle="tab">Pair Scan</a> - </li>--> </ul> <div class="tab-content"> {% for mapping_method in dataset.group.mapping_names %} {% if mapping_method == "GEMMA" %} <div class="tab-pane {% if dataset.group.mapping_id == '1' or dataset.group.mapping_id == '2' %}active{% endif %}" id="gemma"> - <div style="padding-top: 20px;" class="form-horizontal"> + <div class="form-horizontal section-form-div"> <div class="mapping_method_fields form-group"> - <label for="chr_select" style="text-align: right;" class="col-xs-3 control-label">Chromosome</label> - <div style="margin-left:20px;" class="col-xs-2 controls"> - <select id="chr_gemma" class="form-control" style="width: 80px;"> + <label for="chr_select" class="col-xs-3 control-label">Chromosome</label> + <div class="col-xs-2 controls"> + <select id="chr_gemma" class="form-control chr-select"> {% for item in chr_list %} <option value="{{ item[1] }}">{{ item[0] }}</option> {% endfor %} @@ -41,8 +38,8 @@ </div> {% if genofiles and genofiles|length>0 %} <div class="mapping_method_fields form-group"> - <label for="genofiles" style="text-align: right;" class="col-xs-3 control-label">Genotypes</label> - <div style="margin-left:20px;" class="col-xs-4 controls"> + <label for="genofiles" class="col-xs-3 control-label">Genotypes</label> + <div class="col-xs-8 controls"> <select id="genofile_gemma" class="form-control"> {% for item in genofiles %} <option value="{{item['location']}}:{{item['title']}}">{{item['title']}}</option> @@ -52,14 +49,14 @@ </div> {% endif %} <div class="mapping_method_fields form-group"> - <label for="maf_gemma" style="text-align: right;" class="col-xs-3 control-label">MAF >=</label> - <div style="margin-left:20px;" class="col-xs-4 controls"> - <input name="maf_gemma" value="{{ maf }}" type="text" class="form-control" style="width: 80px;"> + <label for="maf_gemma" class="col-xs-3 control-label">MAF >=</label> + <div class="col-xs-4 controls"> + <input name="maf_gemma" value="{{ maf }}" type="text" class="form-control maf-select"> </div> </div> <div class="mapping_method_fields form-group"> - <label style="text-align: right;" class="col-xs-3 control-label">Use LOCO</label> - <div style="margin-left:20px;" class="col-xs-6 controls"> + <label class="col-xs-3 control-label">Use LOCO</label> + <div class="col-xs-6 controls"> <label class="radio-inline"> <input type="radio" name="use_loco" value="True" checked=""> Yes @@ -71,25 +68,22 @@ </div> </div> <div class="mapping_method_fields form-group"> - <div class="col-xs-3" style="text-align: right;"> - <label class="control-label">Covariates</label> - <font size="2">Select covariate(s) from a collection</font> - </div> - <div style="margin-left:20px;" class="col-xs-7"> + <label class="col-xs-3 control-label">Covariates<br><span class="covar-text">Select covariate(s) from a collection</span></label> + <div class="col-xs-8 covar-options"> {% if g.user_session.num_collections < 1 %} No collections available. Please add traits to a collection to use them as covariates. {% else %} - <div style="margin-bottom: 10px;"> - <button type="button" id="select_covariates" class="btn btn-default" style="width: 80px; padding-right: 10px;">Select</button> - <button type="button" id="remove_covariates" class="btn btn-default" style="width: 80px;">Remove</button> + <div class="select-covar-div"> + <button type="button" id="select_covariates" class="btn btn-default select-covar-button">Select</button> + <button type="button" id="remove_covariates" class="btn btn-default select-covar-button">Remove</button> </div> - <textarea rows="3" cols="50" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none; width: 200px;" class="selected_covariates"></textarea> + <textarea rows="3" cols="50" readonly placeholder="No covariates selected" class="selected-covariates"></textarea> {% endif %} </div> </div> <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> - <div style="margin-left:20px;" class="col-xs-6"> + <div class="col-xs-6"> <button id="gemma_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> </div> </div> @@ -97,20 +91,20 @@ </div> {% elif mapping_method == "QTLReaper" %} <div class="tab-pane" id="interval_mapping"> - <div style="margin-top: 20px" class="form-horizontal"> + <div class="form-horizontal section-form-div"> <div class="mapping_method_fields form-group"> - <label for="reaper_version" style="text-align: right;" class="col-xs-3 control-label">Version<sup><a href="https://github.com/chfi/rust-qtlreaper" target="_blank" title="'New' is the new qtlreaper implementation written in RUST by Christian Fischer. 'Original' corresponds to the original version written in C.">?</a></sup></label> - <div style="margin-left:20px;" class="col-xs-3 controls"> - <select name="reaper_version" class="form-control" style="width: 80px;"> + <label for="reaper_version" class="col-xs-3 control-label">Version<sup><a href="https://github.com/chfi/rust-qtlreaper" target="_blank" title="'New' is the new qtlreaper implementation written in RUST by Christian Fischer. 'Original' corresponds to the original version written in C.">?</a></sup></label> + <div class="col-xs-3 controls"> + <select name="reaper_version" class="form-control reaper-ver-select"> <option value="new">New</option> <option value="original">Original</option> </select> </div> </div> <div class="mapping_method_fields form-group"> - <label for="chr_select" style="text-align: right;" class="col-xs-3 control-label">Chromosome</label> - <div style="margin-left:20px;" class="col-xs-2 controls"> - <select id="chr_reaper" class="form-control" style="width: 80px;"> + <label for="chr_select" class="col-xs-3 control-label">Chromosome</label> + <div class="col-xs-2 controls"> + <select id="chr_reaper" class="form-control chr-select"> {% for item in chr_list %} <option value="{{ item[1] }}">{{ item[0] }}</option> {% endfor %} @@ -119,9 +113,9 @@ </div> {% if genofiles and genofiles|length>0 %} <div class="mapping_method_fields form-group"> - <label for="scale_select" style="text-align: right;" class="col-xs-3 control-label">Map Scale</label> - <div style="margin-left:20px;" class="col-xs-2 controls"> - <select id="scale_reaper" class="form-control" style="width: 80px;"> + <label for="scale_select" class="col-xs-3 control-label">Map Scale</label> + <div class="col-xs-2 controls"> + <select id="scale_reaper" class="form-control scale-select"> {% for item in scales_in_geno[genofiles[0]['location']] %} <option value="{{ item[0] }}">{{ item[1] }}</option> {% endfor %} @@ -129,8 +123,8 @@ </div> </div> <div class="mapping_method_fields form-group"> - <label style="text-align: right;" for="genofiles" class="col-xs-3 control-label">Genotypes</label> - <div style="margin-left:20px;" class="col-xs-4 controls"> + <label for="genofiles" class="col-xs-3 control-label">Genotypes</label> + <div class="col-xs-4 controls"> <select id="genofile_reaper" class="form-control"> {% for item in genofiles %} <option value="{{item['location']}}:{{item['title']}}">{{item['title']}}</option> @@ -140,9 +134,9 @@ </div> {% else %} <div class="mapping_method_fields form-group"> - <label for="scale_select" style="text-align: right;" class="col-xs-3 control-label">Map Scale</label> - <div style="margin-left:20px;" class="col-xs-2 controls"> - <select id="scale_reaper" class="form-control" style="width: 80px;"> + <label for="scale_select" class="col-xs-3 control-label">Map Scale</label> + <div class="col-xs-2 controls"> + <select id="scale_reaper" class="form-control scale-select"> {% for item in scales_in_geno[dataset.group.name + ".geno"] %} <option value="{{ item[0] }}">{{ item[1] }}</option> {% endfor %} @@ -151,79 +145,48 @@ </div> {% endif %} <div class="mapping_method_fields form-group"> - <label style="text-align: right;" for="mapping_permutations" class="col-xs-3 control-label">Permutations</label> - <div style="margin-left:20px;" class="col-xs-4 controls"> + <label for="mapping_permutations" class="col-xs-3 control-label">Permutations</label> + <div class="col-xs-4 controls"> <input name="num_perm_reaper" value="2000" type="text" class="form-control"> </div> </div> <div class="mapping_method_fields form-group"> - <label style="text-align: right;" for="mapping_bootstraps" class="col-xs-3 control-label">Bootstraps</label> - <div style="margin-left:20px;" class="col-xs-4 controls"> - <input name="num_bootstrap" value="2000" type="text" class="form-control"> - </div> - </div> - <div class="mapping_method_fields form-group"> - <label style="text-align: right;" for="control_for" class="col-xs-3 control-label">Control for<br>Cofactors</label> - <div style="margin-left:20px;" class="col-xs-6 controls"> - {% if dataset.type == 'ProbeSet' and this_trait.locus_chr != "" %} - <input name="control_reaper" value="{{ nearest_marker }}" type="text" style="width: 160px; display: inline-block;" class="form-control" /> - {% else %} - <input name="control_reaper" value="" type="text" style="display: inline-block; padding-right: 10px;" class="form-control" /> - {% endif %} - <input type="radio" name="do_control_reaper" value="true"> Yes <input type="radio" name="do_control_reaper" value="false" checked="" style="margin-left: 10px;"> No</span> - <!-- - <label class="radio-inline"> - <input type="radio" name="do_control_reaper" value="true"> - Yes - </label> - <label class="radio-inline"> - <input type="radio" name="do_control_reaper" value="false" checked=""> - No - </label> - --> - </div> - </div> - -<!-- - <div class="mapping_method_fields form-group"> - <label for="mapping_bootstraps" class="col-xs-3 control-label" title="Bootstrapping Resamples">Bootstrap Test (n=2000)</label> + <label for="mapping_bootstraps" class="col-xs-3 control-label">Bootstraps</label> <div class="col-xs-4 controls"> - <label> - <input type="checkbox" name="bootCheck" id="bootCheck"> Bootstrap Test (n=2000) - </label> + <input name="num_bootstrap" value="2000" type="text" class="form-control"> </div> </div> - <div class="mapping_method_fields form-group"> - <label style="text-align:left;" class="col-xs-12 control-label">Display Additive Effect</label> - <div class="col-xs-12 controls" id="display_additive_effect"> + <label for="control_for" class="col-xs-3 control-label">Control for<br>Cofactors</label> + <div class="col-xs-6 controls"> + <input name="control_reaper" value="{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != '' %}{{ nearest_marker }}{% endif %}" type="text" class="form-control cofactor-input" /> + <br> <label class="radio-inline"> - <input type="radio" name="display_additive" id="display_additive" value="yes" checked=""> - Yes + <input type="radio" name="do_control_reaper" value="true" checked=""> + Yes </label> <label class="radio-inline"> - <input type="radio" name="display_additive" id="display_additive" value="no"> - No + <input type="radio" name="do_control_reaper" value="false"> + No </label> </div> </div> ---> <div class="mapping_method_fields form-group"> - <label style="text-align: right;" class="col-xs-3 control-label">Interval Map</label> - <div style="margin-left:20px;" class="col-xs-6 controls"> + <label class="col-xs-3 control-label">Interval Map</label> + <div class="col-xs-6 controls"> <label class="radio-inline"> - <input type="radio" name="manhattan_plot_reaper" value="False" checked=""> + <input type="radio" name="manhattan_plot_reaper" value="false" checked=""> Yes </label> <label class="radio-inline"> - <input type="radio" name="manhattan_plot_reaper" value="True"> + <input type="radio" name="manhattan_plot_reaper" value="true"> No </label> </div> </div> <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> - <div style="margin-left:20px;" class="col-xs-6"> + <div class="col-xs-6"> <button id="interval_mapping_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Interval Mapping" value="Compute">Compute</button> </div> </div> @@ -231,11 +194,11 @@ </div> {% elif mapping_method == "R/qtl" %} <div class="tab-pane {% if dataset.group.mapping_id == '3' %}active{% endif %}" id="rqtl_geno"> - <div style="margin-top: 20px" class="form-horizontal"> + <div class="form-horizontal section-form-div"> <div class="mapping_method_fields form-group"> - <label for="chr_select" style="text-align: right;" class="col-xs-3 control-label">Chromosome</label> - <div style="margin-left:20px;" class="col-xs-2 controls"> - <select id="chr_rqtl_geno" class="form-control" style="width: 80px;"> + <label for="chr_select" class="col-xs-3 control-label">Chromosome</label> + <div class="col-xs-2 controls"> + <select id="chr_rqtl_geno" class="form-control chr-select"> {% for item in chr_list %} <option value="{{ item[1] }}">{{ item[0] }}</option> {% endfor %} @@ -244,9 +207,9 @@ </div> {% if genofiles and genofiles|length > 0 %} <div class="mapping_method_fields form-group"> - <label for="scale_select" style="text-align: right;" class="col-xs-3 control-label">Map Scale</label> - <div style="margin-left:20px;" class="col-xs-2 controls"> - <select id="scale_rqtl_geno" class="form-control" style="width: 80px;"> + <label for="scale_select" class="col-xs-3 control-label">Map Scale</label> + <div class="col-xs-2 controls"> + <select id="scale_rqtl_geno" class="form-control scale-select"> {% for item in scales_in_geno[genofiles[0]['location']] %} <option value="{{ item[0] }}">{{ item[1] }}</option> {% endfor %} @@ -254,8 +217,8 @@ </div> </div> <div class="mapping_method_fields form-group"> - <label style="text-align:right;" for="genofiles" class="col-xs-3 control-label">Genotypes</label> - <div style="margin-left:20px;" class="col-xs-4 controls"> + <label for="genofiles" class="col-xs-3 control-label">Genotypes</label> + <div class="col-xs-4 controls"> <select id="genofile_rqtl_geno" class="form-control"> {% for item in genofiles %} <option value="{{item['location']}}:{{item['title']}}">{{item['title']}}</option> @@ -265,15 +228,15 @@ </div> {% endif %} <div class="mapping_method_fields form-group"> - <label style="text-align:right;" for="mapping_permutations" class="col-xs-3 control-label">Permutations</label> - <div style="margin-left:20px;" class="col-xs-4 controls"> + <label for="mapping_permutations" class="col-xs-3 control-label">Permutations</label> + <div class="col-xs-4 controls"> <input name="num_perm_rqtl_geno" value="200" type="text" class="form-control"> </div> </div> {% if sample_groups[0].attributes|length > 0 %} <div class="mapping_method_fields form-group"> - <label style="text-align:right;" class="col-xs-3 control-label">Stratified</label> - <div style="margin-left:20px;" class="col-xs-6 controls"> + <label class="col-xs-3 control-label">Stratified</label> + <div class="col-xs-6 controls"> <label class="radio-inline"> <input type="radio" name="perm_strata" value="True" checked=""> Yes @@ -286,27 +249,23 @@ </div> {% endif %} <div class="mapping_method_fields form-group"> - <label style="text-align:right;" for="control_for" class="col-xs-3 control-label">Control for</label> - <div style="margin-left:20px;" class="col-xs-6 controls"> - {% if dataset.type == 'ProbeSet' and this_trait.locus_chr != "" %} - <input name="control_rqtl_geno" value="{{ nearest_marker }}" type="text" style="width: 160px;" class="form-control" /> - {% else %} - <input name="control_rqtl_geno" value="" type="text" class="form-control" /> - {% endif %} - <label class="radio-inline"> - <input type="radio" name="do_control_rqtl" value="true"> - Yes - </label> - <label class="radio-inline"> - <input type="radio" name="do_control_rqtl" value="false" checked=""> - No - </label> + <label for="control_for" class="col-xs-3 control-label">Control for</label> + <div class="col-xs-6 controls"> + <input name="control_rqtl_geno" value="{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != '' %}{{ nearest_marker }}{% endif %}" type="text" class="form-control cofactor-input" /> + <label class="radio-inline"> + <input type="radio" name="do_control_rqtl" value="true"> + Yes + </label> + <label class="radio-inline"> + <input type="radio" name="do_control_rqtl" value="false" checked=""> + No + </label> </div> </div> <div class="mapping_method_fields form-group"> - <label style="text-align:right;" for="mapmodel_rqtl_geno" class="col-xs-3 control-label">Model</label> - <div style="margin-left:20px;" class="col-xs-4 controls"> + <label for="mapmodel_rqtl_geno" class="col-xs-3 control-label">Model</label> + <div class="col-xs-4 controls"> <select id="mapmodel_rqtl_geno" name="mapmodel_rqtl_geno" class="form-control"> <option value="normal">Normal</option> {% if binary == "true" %}<option value="binary">Binary</option>{% endif %} @@ -317,8 +276,8 @@ </div> <div class="mapping_method_fields form-group"> - <label style="text-align:right;" for="mapmethod_rqtl_geno" class="col-xs-3 control-label">Method</label> - <div style="margin-left:20px;" class="col-xs-6 controls"> + <label for="mapmethod_rqtl_geno" class="col-xs-3 control-label">Method</label> + <div class="col-xs-6 controls"> <select id="mapmethod_rqtl_geno" name="mapmethod_rqtl_geno" class="form-control"> <option value="hk" selected>Haley-Knott</option> <option value="ehk">Extended Haley-Knott</option> @@ -329,8 +288,8 @@ </div> </div> <div id="missing_geno_div" class="mapping_method_fields form-group" style="display: none;"> - <label style="text-align:right;" for="missing_genotypes" class="col-xs-3 control-label"></label> - <div style="margin-left:20px;" class="col-xs-6 controls"> + <label for="missing_genotypes" class="col-xs-3 control-label"></label> + <div class="col-xs-6 controls"> <select id="missing_genotype" name="missing_genotypes" class="form-control"> <option value="mr">Remove Samples w/o Genotypes</option> <option value="mr-imp">Single Imputation</option> @@ -338,25 +297,9 @@ </select> </div> </div> - - <!-- - <div class="mapping_method_fields form-group"> - <label style="text-align:left;" class="col-xs-12 control-label">Pair Scan</label> - <div class="col-xs-12 controls"> - <label class="radio-inline"> - <input type="radio" name="pair_scan" value="true"> - Yes - </label> - <label class="radio-inline"> - <input type="radio" name="pair_scan" value="false" checked=""> - No - </label> - </div> - </div> - --> <div class="mapping_method_fields form-group"> - <label style="text-align:right;" class="col-xs-3 control-label">Manhattan<br>Plot</label> - <div style="margin-left:20px;" class="col-xs-6 controls"> + <label class="col-xs-3 control-label">Manhattan<br>Plot</label> + <div class="col-xs-6 controls"> <label class="radio-inline"> <input type="radio" name="manhattan_plot_rqtl" value="True"> Yes @@ -368,25 +311,22 @@ </div> </div> <div class="mapping_method_fields form-group"> - <div class="col-xs-3" style="text-align: right;"> - <label class="control-label">Covariates</label> - <font size="2">Select covariate(s) from a collection</font> - </div> - <div style="margin-left:20px;" class="col-xs-7"> - {% if g.user_session.num_collections < 1 %} - No collections available. Please add traits to a collection to use them as covariates. - {% else %} - <div style="margin-bottom: 10px;"> - <button type="button" class="btn btn-default select_covariates" style="width: 80px; padding-right: 10px;">Select</button> - <button type="button" class="btn btn-default remove_covariates" style="width: 80px;">Remove</button> - </div> - <textarea rows="3" cols="50" readonly placeholder="No covariates selected" style="overflow-y: scroll; resize: none; width: 200px;" class="selected_covariates"></textarea> - {% endif %} - </div> + <label class="col-xs-3 control-label">Covariates<br><span class="covar-text">Select covariate(s) from a collection</span></label> + <div class="col-xs-8 covar-options"> + {% if g.user_session.num_collections < 1 %} + No collections available. Please add traits to a collection to use them as covariates. + {% else %} + <div class="select-covar-div"> + <button type="button" id="select_covariates" class="btn btn-default select-covar-button">Select</button> + <button type="button" id="remove_covariates" class="btn btn-default select-covar-button">Remove</button> + </div> + <textarea rows="3" cols="50" readonly placeholder="No covariates selected" class="selected-covariates"></textarea> + {% endif %} + </div> </div> <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> - <div style="margin-left:20px;" class="col-xs-6"> + <div class="col-xs-6 controls"> <button id="rqtl_geno_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> </div> </div> @@ -397,23 +337,25 @@ </div> </div> </div> - <div> - <dl> - {% for mapping_method in dataset.group.mapping_names %} - {% if mapping_method == "GEMMA" %} - <dt style="padding-top: 20px;">GEMMA</dt> - <dd>Maps traits with correction for kinship among samples using a linear mixed model method, and also allows users to fit multiple covariates such as sex, age, treatment, and genetic markers (<a href="https://www.ncbi.nlm.nih.gov/pubmed/24531419">PMID: 2453419</a>, and <a href="https://github.com/genetics-statistics/GEMMA"> GitHub code</a>). GEMMA incorporates the Leave One Chromosome Out (LOCO) method to ensure that the correction for kinship does not remove useful genetic variance near each marker. Markers can be filtered to include only those with minor allele frequencies (MAF) above a threshold. The default MAF is 0.05.</dd> - {% elif mapping_method == "R/qtl" %} - <dt style="margin-top: 20px;">R/qtl (version 1.44.9</dt> - <dd>The original R/qtl mapping package that supports classic experimental crosses including 4-parent F2 intercrosses (e.g., NIA ITP UM-HET3). R/qtl is ideal for populations that do not have complex kinship or admixture (<a href="https://www.ncbi.nlm.nih.gov/pubmed/12724300">PMID: 12724300</a>). Both R/qtl as implemented here, and R/qtl2 (<a href="https://www.ncbi.nlm.nih.gov/pubmed/30591514">PMID: 30591514</a>) are available as <a href="https://kbroman.org/pages/software.html">R suites</a>.</dd> - {% elif mapping_method == "QTLReaper" %} - <dt style="margin-top: 20px;">Haley-Knott Regression</dt> - <dd>Fast linear mapping method (<a href="https://www.ncbi.nlm.nih.gov/pubmed/16718932">PMID 16718932</a>) works well with F2 intercrosses and backcrosses, but that is not recommended for complex or admixed populations (e.g., GWAS or heterogeneous stock studies) or for advanced intercrosses, recombinant inbred families, or diallel crosses. Interactive plots in GeneNetwork have relied on the fast HK mapping for two decades and we still use this method for mapping omics data sets and computing genome-wide permutation threshold (<a href="https://github.com/pjotrp/QTLReaper">QTL Reaper code</a>).</dd> - {% endif %} - {% endfor %} - </dl> - <div class="rqtl_description" style="padding-top: 40px; display: none;"> - More information on R/qtl mapping models and methods can be found <a href="http://www.rqtl.org/tutorials/rqtltour.pdf">here</a>. + <div class="col-xs-6"> + <div class="section-form-div"> + <dl> + {% for mapping_method in dataset.group.mapping_names %} + {% if mapping_method == "GEMMA" %} + <dt>GEMMA</dt> + <dd>Maps traits with correction for kinship among samples using a linear mixed model method, and also allows users to fit multiple covariates such as sex, age, treatment, and genetic markers (<a href="https://www.ncbi.nlm.nih.gov/pubmed/24531419">PMID: 2453419</a>, and <a href="https://github.com/genetics-statistics/GEMMA"> GitHub code</a>). GEMMA incorporates the Leave One Chromosome Out (LOCO) method to ensure that the correction for kinship does not remove useful genetic variance near each marker. Markers can be filtered to include only those with minor allele frequencies (MAF) above a threshold. The default MAF is 0.05.</dd> + {% elif mapping_method == "R/qtl" %} + <dt class="map-method-text">R/qtl (version 1.44.9</dt> + <dd>The original R/qtl mapping package that supports classic experimental crosses including 4-parent F2 intercrosses (e.g., NIA ITP UM-HET3). R/qtl is ideal for populations that do not have complex kinship or admixture (<a href="https://www.ncbi.nlm.nih.gov/pubmed/12724300">PMID: 12724300</a>). Both R/qtl as implemented here, and R/qtl2 (<a href="https://www.ncbi.nlm.nih.gov/pubmed/30591514">PMID: 30591514</a>) are available as <a href="https://kbroman.org/pages/software.html">R suites</a>.</dd> + {% elif mapping_method == "QTLReaper" %} + <dt class="map-method-text">Haley-Knott Regression</dt> + <dd>Fast linear mapping method (<a href="https://www.ncbi.nlm.nih.gov/pubmed/16718932">PMID 16718932</a>) works well with F2 intercrosses and backcrosses, but that is not recommended for complex or admixed populations (e.g., GWAS or heterogeneous stock studies) or for advanced intercrosses, recombinant inbred families, or diallel crosses. Interactive plots in GeneNetwork have relied on the fast HK mapping for two decades and we still use this method for mapping omics data sets and computing genome-wide permutation threshold (<a href="https://github.com/pjotrp/QTLReaper">QTL Reaper code</a>).</dd> + {% endif %} + {% endfor %} + </dl> + <div class="rqtl-description"> + More information on R/qtl mapping models and methods can be found <a href="http://www.rqtl.org/tutorials/rqtltour.pdf">here</a>. + </div> </div> </div> <div id="mapping_result_holder_wrapper" style="display:none;"> diff --git a/wqflask/wqflask/templates/show_trait_statistics.html b/wqflask/wqflask/templates/show_trait_statistics.html index 7bdb3ef9..4f347d4e 100644 --- a/wqflask/wqflask/templates/show_trait_statistics.html +++ b/wqflask/wqflask/templates/show_trait_statistics.html @@ -1,148 +1,137 @@ <div> - <div class="tabbable"> <!-- Only required for left/right tabs --> - <ul class="nav nav-pills"> - <li class="active"> - <a href="#stats_tab" data-toggle="tab">Basic Statistics</a> - </li> - <li> - <a href="#histogram_tab" class="histogram_tab" data-toggle="tab">Histogram</a> - </li> - {% if num_values < 256 %} - <li> - <a href="#bar_chart_tab" class="bar_chart_tab" data-toggle="tab">Bar Chart</a> - </li> - {% endif %} - <li> - <a href="#probability_plot" class="prob_plot_tab" data-toggle="tab">Probability Plot</a> - </li> - <li> - <a href="#box_plot_tab" class="box_plot_tab" data-toggle="tab">Box Plot</a> - </li> - <li> - <a href="#violin_plot_tab" class="violin_plot_tab" data-toggle="tab">Violin Plot</a> - </li> - <li> - <a href="#scatterplot_matrix" data-toggle="tab">Scatterplot Matrix</a> - </li> - </ul> - - <div class="tab-content"> - <div class="tab-pane active" id="stats_tab"> - <div style="padding: 20px;" class="form-horizontal"> - <table id="stats_table" style="width: {{ stats_table_width }}px; float: left;" class="table table-hover table-striped table-bordered"></table> - </div> - </div> - <div class="tab-pane" id="histogram_tab"> - <div style="padding: 20px" class="form-horizontal"> - {% if sample_groups|length != 1 %} - <select class="histogram_samples_group"> - {% for group, pretty_group in sample_group_types.items() %} - <option value="{{ group }}">{{ pretty_group }}</option> - {% endfor %} - </select> - <br><br> - {% endif %} - <div id="histogram_container"> - <div id="histogram" class="barchart"></div> - </div> - </div> + <div class="tabbable"> <!-- Only required for left/right tabs --> + <ul class="nav nav-pills"> + <li class="active"> + <a href="#stats_tab" data-toggle="tab">Basic Statistics</a> + </li> + <li> + <a href="#histogram_tab" class="histogram_tab" data-toggle="tab">Histogram</a> + </li> + {% if num_values < 256 %} + <li> + <a href="#bar_chart_tab" class="bar_chart_tab" data-toggle="tab">Bar Chart</a> + </li> + {% endif %} + <li> + <a href="#probability_plot" class="prob_plot_tab" data-toggle="tab">Probability Plot</a> + </li> + <li> + <a href="#box_plot_tab" class="box_plot_tab" data-toggle="tab">Box Plot</a> + </li> + <li> + <a href="#violin_plot_tab" class="violin_plot_tab" data-toggle="tab">Violin Plot</a> + </li> + <li> + <a href="#scatterplot_matrix" data-toggle="tab">Scatterplot Matrix</a> + </li> + </ul> + <div class="tab-content"> + <div class="tab-pane active" id="stats_tab"> + <div class="form-horizontal section-form-div"> + <table id="stats_table" style="width: {{ stats_table_width }}px;" class="table table-hover table-striped table-bordered left-float"></table> </div> - {% if num_values < 256 %} - <div class="tab-pane" id="bar_chart_tab"> - <div style="padding: 20px" class="form-horizontal"> - {% if sample_groups|length != 1 %} - <select class="bar_chart_samples_group"> - {% for group, pretty_group in sample_group_types.items() %} - <option value="{{ group }}">{{ pretty_group }}</option> - {% endfor %} - </select> - {% endif %} - {% if sample_groups[0].attributes %} - <div class="input-append"> - <select id="color_attribute" size="1"> - <option value="None">None</option> - {% for attribute in sample_groups[0].attributes %} - <option value="{{ sample_groups[0].attributes[attribute].name.replace(' ', '_') }}"> - {{ sample_groups[0].attributes[attribute].name }}</option> - {% endfor %} - </select> - </div> - {% endif %} - <div id="update_bar_chart" class="btn-group" style="margin-bottom: 5px;"> - <button type="button" class="btn btn-default sort_by_name" value="name"> - <i class="icon-resize-horizontal"></i> Sort By Name - </button> - <button type="button" class="btn btn-default sort_by_value" value="value"> - <i class="icon-signal"></i> Sort By Value - </button> - </div> - <!-- - <div class="btn-group"> - <button type="button" class="btn btn-default" id="color_by_trait" style="margin-bottom: 5px;"> - <i class="icon-tint"></i> Color by Trait - </button> - </div> - --> - <div id="bar_chart_container"> - <div id="bar_chart"></div> - </div> + </div> + <div class="tab-pane" id="histogram_tab"> + <div class="form-horizontal section-form-div"> + {% if sample_groups|length != 1 %} + Select Group: + <select class="histogram_samples_group"> + {% for group, pretty_group in sample_group_types.items() %} + <option value="{{ group }}">{{ pretty_group }}</option> + {% endfor %} + </select> + <br><br> + {% endif %} + <div id="histogram_container"> + <div id="histogram" class="barchart"></div> </div> </div> - {% endif %} - <div class="tab-pane" id="probability_plot"> - <div style="padding: 20px" class="form-horizontal"> - {% if sample_groups|length != 1 %} - Select Group: - <select class="prob_plot_samples_group"> - {% for group, pretty_group in sample_group_types.items() %} - <option value="{{ group }}">{{ pretty_group }}</option> + </div> + {% if num_values < 256 %} + <div class="tab-pane" id="bar_chart_tab"> + <div class="form-horizontal section-form-div"> + {% if sample_groups|length != 1 %} + Select Group: + <select class="bar_chart_samples_group"> + {% for group, pretty_group in sample_group_types.items() %} + <option value="{{ group }}">{{ pretty_group }}</option> + {% endfor %} + </select> + {% endif %} + {% if sample_groups[0].attributes %} + <div class="input-append"> + <select id="color_attribute" size="1"> + <option value="None">None</option> + {% for attribute in sample_groups[0].attributes %} + <option value="{{ sample_groups[0].attributes[attribute].name.replace(' ', '_') }}"> + {{ sample_groups[0].attributes[attribute].name }}</option> {% endfor %} </select> - <br> - <br> - {% endif %} - - <div id="prob_plot_container"> - <div id="prob_plot_div"></div> - </div> - <div id="shapiro_wilk_text"></div> - <div> - More about <a href="http://en.wikipedia.org/wiki/Normal_probability_plot">Normal Probability Plots</a> and more - about interpreting these plots from the <a href="http://genenetwork.org/glossary.html#normal_probability">glossary</a> - </div> - </div> - - </div> - <div class="tab-pane" id="box_plot_tab"> - <div id="box_plot_container"> - <div id="box_plot"></div> + {% endif %} + <div id="update_bar_chart" class="btn-group"> + <button type="button" class="btn btn-default sort_by_name" value="name"> + <i class="icon-resize-horizontal"></i> Sort By Name + </button> + <button type="button" class="btn btn-default sort_by_value" value="value"> + <i class="icon-signal"></i> Sort By Value + </button> </div> - </div> - <div class="tab-pane" id="violin_plot_tab"> - <div id="violin_plot_container"> - <div id="violin_plot"></div> + <div id="bar_chart_container"> + <div id="bar_chart"></div> </div> </div> - <div class="tab-pane" id="scatterplot_matrix"> - - <div style="margin-bottom:40px;" class="btn-group"> - <button type="button" class="btn btn-default" id="select_compare_trait"> - <i class="icon-th-large"></i> Select Traits - </button> - <button type="button" class="btn btn-default" id="clear_compare_trait"> - <i class="icon-trash"></i> Clear - </button> + </div> + {% endif %} + <div class="tab-pane" id="probability_plot"> + <div class="form-horizontal section-form-div"> + {% if sample_groups|length != 1 %} + Select Group: + <select class="prob_plot_samples_group"> + {% for group, pretty_group in sample_group_types.items() %} + <option value="{{ group }}">{{ pretty_group }}</option> + {% endfor %} + </select> + <br> + <br> + {% endif %} + + <div id="prob_plot_container"> + <div id="prob_plot_div"></div> </div> - <div id="scatterplot_container"> - <div id="comparison_scatterplot" class="qtlcharts"></div> + <div id="shapiro_wilk_text"></div> + <div> + More about <a href="http://en.wikipedia.org/wiki/Normal_probability_plot">Normal Probability Plots</a> and more + about interpreting these plots from the <a href="http://genenetwork.org/glossary.html#normal_probability">glossary</a> </div> </div> </div> + <div class="tab-pane" id="box_plot_tab"> + <div id="box_plot_container"> + <div id="box_plot"></div> + </div> + </div> + <div class="tab-pane" id="violin_plot_tab"> + <div id="violin_plot_container"> + <div id="violin_plot"></div> + </div> + </div> + <div class="tab-pane" id="scatterplot_matrix"> + <div class="btn-group scatterplot-btn-div"> + <button type="button" class="btn btn-default" id="select_compare_trait"> + <i class="icon-th-large"></i> Select Traits + </button> + <button type="button" class="btn btn-default" id="clear_compare_trait"> + <i class="icon-trash"></i> Clear + </button> + </div> + <div id="scatterplot_container"> + <div id="comparison_scatterplot" class="qtlcharts"></div> + </div> + </div> </div> - <!--</div>--> + </div> <div id="collections_holder_wrapper" style="display:none;"> <div id="collections_holder"></div> </div> - </div> diff --git a/wqflask/wqflask/templates/show_trait_transform_and_filter.html b/wqflask/wqflask/templates/show_trait_transform_and_filter.html index 9fb560a0..0418d972 100644 --- a/wqflask/wqflask/templates/show_trait_transform_and_filter.html +++ b/wqflask/wqflask/templates/show_trait_transform_and_filter.html @@ -1,78 +1,74 @@ <div> <div class="form-horizontal"> - <p>Edit or delete values in the Trait Data boxes, and use the - <strong>Reset</strong> option as - needed. - </p> - - <div id="blockMenuSpan" class="input-append" style="margin-bottom: 10px;"> - <label for="remove_samples_field">Block samples by index:</label> - <input type="text" id="remove_samples_field" placeholder="Example: 3, 5-10, 12"> - <select id="block_group" size="1"> - <option value="primary"> - {{ sample_group_types['samples_primary'] }} - </option> - <option value="other"> - {{ sample_group_types['samples_other'] }} - </option> - </select> - <input type="button" id="block_by_index" class="btn btn-danger" value="Block"> - </div> - <div id="remove_samples_invalid" class="alert alert-error" style="display:none;"> - Please check that your input is formatted correctly, e.g. <strong>3, 5-10, 12</strong> - </div> - {% if sample_groups[0].attributes %} - <div class="input-append" style="margin-top:10px; margin-bottom:10px;"> - <label for="exclude_menu">Block samples by group:</label> - <select id="exclude_menu" size=1> - {% for attribute in sample_groups[0].attributes %} - <option value="{{ sample_groups[0].attributes[attribute].name.replace(' ', '_') }}"> - {{ sample_groups[0].attributes[attribute].name }}</option> - {% endfor %} - </select> - <select id="attribute_values" size=1> - </select> - <input type="button" id="exclude_group" class="btn" value="Block"> - </div> - {% endif %} - <div> - <input type="button" id="hide_no_value" class="btn btn-default" value="Hide No Value"> - <input type="button" id="block_outliers" class="btn btn-default" value="Block Outliers"> - <input type="button" class="btn btn-success reset" value="Reset"> - <span class="input-append"> - <input type="button" class="btn btn-default export" value="Export"> - <select class="select optional span2 export_format"> - <option value="excel">Excel</option> - <option value="csv">CSV</option> - </select> - </span> - <br> - <div style="margin-top:10px;"> - <input type="button" id="normalize" class="btn btn-success" value="Normalize"> - <select id="norm_method" class="select optional span2"> - {% if negative_vals_exist == "false" %} - <option value="log2">Log2</option> - <option value="log10">Log10</option> - <option value="sqrt">Square Root</option> - {% endif %} - <option value="zscore">Z-score</option> - <option value="qnorm">Quantile</option> - <option value="invert">Invert +/-</option> - </select> - </div> - </div> + <p>Edit or delete values in the Trait Data boxes, and use the + <strong>Reset</strong> option as + needed. + </p> + <div id="blockMenuSpan" class="input-append block-by-index-div"> + <label for="remove_samples_field">Block samples by index:</label> + <input type="text" id="remove_samples_field" placeholder="Example: 3, 5-10, 12"> + <select id="block_group" size="1"> + <option value="primary"> + {{ sample_group_types['samples_primary'] }} + </option> + <option value="other"> + {{ sample_group_types['samples_other'] }} + </option> + </select> + <input type="button" id="block_by_index" class="btn btn-danger" value="Block"> + </div> + <div id="remove_samples_invalid" class="alert alert-error" style="display:none;"> + Please check that your input is formatted correctly, e.g. <strong>3, 5-10, 12</strong> + </div> + {% if sample_groups[0].attributes %} + <div class="input-append block-by-attribute-div"> + <label for="exclude_menu">Block samples by group:</label> + <select id="exclude_menu" size=1> + {% for attribute in sample_groups[0].attributes %} + <option value="{{ sample_groups[0].attributes[attribute].name.replace(' ', '_') }}"> + {{ sample_groups[0].attributes[attribute].name }}</option> + {% endfor %} + </select> + <select id="attribute_values" size=1> + </select> + <input type="button" id="exclude_group" class="btn" value="Block"> + </div> + {% endif %} + <div> + <input type="button" id="hide_no_value" class="btn btn-default" value="Hide No Value"> + <input type="button" id="block_outliers" class="btn btn-default" value="Block Outliers"> + <input type="button" class="btn btn-success reset" value="Reset"> + <span class="input-append"> + <input type="button" class="btn btn-default export" value="Export"> + <select class="select optional span2 export_format"> + <option value="excel">Excel</option> + <option value="csv">CSV</option> + </select> + </span> <br> - - <div> - <p>Outliers highlighted in - <strong style="background-color:orange;">orange</strong> - can be hidden using - the <strong>Hide Outliers</strong> button. - </p> - - <p>Samples with no value (x) can be hidden by clicking - <strong>Hide No Value</strong> button. - </p> + <div class="normalize-div"> + <input type="button" id="normalize" class="btn btn-success" value="Normalize"> + <select id="norm_method" class="select optional span2"> + {% if negative_vals_exist == "false" %} + <option value="log2">Log2</option> + <option value="log10">Log10</option> + <option value="sqrt">Square Root</option> + {% endif %} + <option value="zscore">Z-score</option> + <option value="qnorm">Quantile</option> + <option value="invert">Invert +/-</option> + </select> </div> + </div> + </div> + <br> + <div> + <p>Outliers highlighted in + <strong style="background-color:orange;">orange</strong> + can be hidden using + the <strong>Hide Outliers</strong> button. + </p> + + <p>Samples with no value (x) can be hidden by clicking<strong>Hide No Value</strong> button.</p> </div> </div>
\ No newline at end of file diff --git a/wqflask/wqflask/update_search_results.py b/wqflask/wqflask/update_search_results.py index 68bea9d6..672f95b1 100644 --- a/wqflask/wqflask/update_search_results.py +++ b/wqflask/wqflask/update_search_results.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import json from flask import Flask, g diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index 077a799b..f25ebc32 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import os import hashlib import datetime @@ -39,15 +37,13 @@ def basic_info(): ip_address = request.remote_addr, user_agent = request.headers.get('User-Agent')) -def encode_password(pass_gen_fields, unencrypted_password): - hashfunc = getattr(hashlib, pass_gen_fields['hashfunc']) - salt = base64.b64decode(pass_gen_fields['salt']) +def encode_password(pass_gen_fields, unencrypted_password): encrypted_password = pbkdf2.pbkdf2_hex(str(unencrypted_password), - pass_gen_fields['salt'], - pass_gen_fields['iterations'], - pass_gen_fields['keylength'], - hashfunc) + pass_gen_fields['salt'], + pass_gen_fields['iterations'], + pass_gen_fields['keylength'], + pass_gen_fields['hashfunc']) pass_gen_fields.pop("unencrypted_password", None) pass_gen_fields["password"] = encrypted_password @@ -199,7 +195,7 @@ def login(): if user_details: submitted_password = params['password'] pwfields = user_details['password'] - if type(pwfields) is str: + if isinstance(pwfields, str): pwfields = json.loads(pwfields) encrypted_pass_fields = encode_password(pwfields, submitted_password) password_match = pbkdf2.safe_str_cmp(encrypted_pass_fields['password'], pwfields['password']) @@ -478,4 +474,4 @@ def register(): @app.errorhandler(401) def unauthorized(error): - return redirect(url_for('login'))
\ No newline at end of file + return redirect(url_for('login')) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index a871e91a..7b25b68e 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -1,53 +1,63 @@ -from __future__ import print_function, division, absolute_import - import os import hashlib import datetime import time -import logging import uuid -import hashlib import hmac import base64 -import urlparse - +import redis # used for collections import simplejson as json +import requests -#from redis import StrictRedis -import redis # used for collections -Redis = redis.StrictRedis() +from base.data_set import create_datasets_list -from flask import (Flask, g, render_template, url_for, request, make_response, - redirect, flash, abort) +from flask import g +from flask import render_template +from flask import url_for +from flask import request +from flask import make_response +from flask import redirect +from flask import flash from wqflask import app -from pprint import pformat as pf - -from wqflask import pbkdf2 # password hashing +from wqflask import pbkdf2 # password hashing from wqflask.database import db_session from wqflask import model -from utility import Bunch, Struct, after +from smtplib import SMTP + +from pprint import pformat as pf -import logging +from utility import Bunch +from utility import Struct from utility.logger import getLogger -logger = getLogger(__name__) -from base.data_set import create_datasets_list +from utility.redis_tools import get_user_id +from utility.redis_tools import get_user_by_unique_column +from utility.redis_tools import set_user_attribute +from utility.redis_tools import save_user +from utility.redis_tools import save_verification_code +from utility.redis_tools import check_verification_code +from utility.redis_tools import get_user_collections +from utility.redis_tools import save_collections -import requests +from utility.tools import SMTP_CONNECT +from utility.tools import SMTP_USERNAME +from utility.tools import SMTP_PASSWORD -from utility.redis_tools import get_user_id, get_user_by_unique_column, set_user_attribute, save_user, save_verification_code, check_verification_code, get_user_collections, save_collections -from smtplib import SMTP -from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_ALCHEMY +logger = getLogger(__name__) + + +Redis = redis.StrictRedis() THREE_DAYS = 60 * 60 * 24 * 3 -#THREE_DAYS = 45 + def timestamp(): return datetime.datetime.utcnow().isoformat() + class AnonUser(object): """Anonymous user handling""" cookie_name = 'anon_user_v1' @@ -230,29 +240,10 @@ class UserSession(object): return len(self.user_collections) -### -# ZS: This is currently not used, but I'm leaving it here commented out because the old "set superuser" code (at the bottom of this file) used it -### -# @property -# def user_ob(self): -# """Actual sqlalchemy record""" -# # Only look it up once if needed, then store it -# # raise "OBSOLETE: use ElasticSearch instead" -# try: -# if LOG_SQL_ALCHEMY: -# logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG) -# -# # Already did this before -# return self.db_object -# except AttributeError: -# # Doesn't exist so we'll create it -# self.db_object = model.User.query.get(self.user_id) -# return self.db_object - def add_collection(self, collection_name, traits): """Add collection into ElasticSearch""" - collection_dict = {'id': unicode(uuid.uuid4()), + collection_dict = {'id': str(uuid.uuid4()), 'name': collection_name, 'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), 'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), @@ -867,7 +858,7 @@ def forgot_password_submit(): email_address = params['email_address'] next_page = None if email_address != "": - logger.debug("Wants to send password E-mail to ",email_address) + logger.debug("Wants to send password E-mail to ", email_address) user_details = get_user_by_unique_column("email_address", email_address) if user_details: ForgotPasswordEmail(user_details["email_address"]) diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 3aa2c151..9f4c5458 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import datetime import time import uuid @@ -129,10 +127,10 @@ class UserSession(object): @property def user_id(self): """Shortcut to the user_id""" - if 'user_id' not in self.record: - self.record['user_id'] = str(uuid.uuid4()) + if b'user_id' not in self.record: + self.record[b'user_id'] = str(uuid.uuid4()) - return self.record['user_id'] + return self.record[b'user_id'] @property def redis_user_id(self): @@ -184,7 +182,7 @@ class UserSession(object): def add_collection(self, collection_name, traits): """Add collection into Redis""" - collection_dict = {'id': unicode(uuid.uuid4()), + collection_dict = {'id': str(uuid.uuid4()), 'name': collection_name, 'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), 'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 94ec7137..08673f79 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -2,8 +2,6 @@ # # Main routing table for GN2 -from __future__ import absolute_import, division, print_function - import traceback # for error page import os # for error gifs import random # for random error gif @@ -14,13 +12,13 @@ import csv import simplejson as json import yaml import xlsxwriter -import StringIO # Todo: Use cStringIO? +import io # Todo: Use cStringIO? from zipfile import ZipFile, ZIP_DEFLATED import gc import numpy as np -import cPickle as pickle +import pickle as pickle import uuid import flask @@ -54,7 +52,7 @@ from wqflask.docs import Docs, update_text from wqflask.db_info import InfoPage from utility import temp_data -from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,JS_TWITTER_POST_FETCHER_PATH,JS_GUIX_PATH, CSS_PATH +from utility.tools import SQL_URI, TEMPDIR, USE_REDIS, USE_GN_SERVER, GN_SERVER_URL, GN_VERSION, JS_TWITTER_POST_FETCHER_PATH, JS_GUIX_PATH, CSS_PATH from utility.helper_functions import get_species_groups from utility.authentication_tools import check_resource_availability from utility.redis_tools import get_redis_conn @@ -129,10 +127,10 @@ def handle_bad_request(e): list = [fn for fn in os.listdir("./wqflask/static/gif/error") if fn.endswith(".gif") ] animation = random.choice(list) - resp = make_response(render_template("error.html",message=err_msg,stack=formatted_lines,error_image=animation,version=GN_VERSION)) + resp = make_response(render_template("error.html", message=err_msg, stack=formatted_lines, error_image=animation, version=GN_VERSION)) # logger.error("Set cookie %s with %s" % (err_msg, animation)) - resp.set_cookie(err_msg[:32],animation) + resp.set_cookie(err_msg[:32], animation) return resp @app.route("/authentication_needed") @@ -166,7 +164,7 @@ def tmp_page(img_path): logger.info("initial_start_vars:", initial_start_vars) imgfile = open(GENERATED_IMAGE_DIR + img_path, 'rb') imgdata = imgfile.read() - imgB64 = imgdata.encode("base64") + imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) return render_template("show_image.html", img_base64 = bytesarray ) @@ -215,8 +213,6 @@ def search_page(): result = the_search.__dict__ valid_search = result['search_term_exists'] - logger.debugf("result", result) - if USE_REDIS and valid_search: Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) Redis.expire(key, 60*60) @@ -264,7 +260,7 @@ def docedit(): @app.route('/generated/<filename>') def generated_file(filename): logger.info(request.url) - return send_from_directory(GENERATED_IMAGE_DIR,filename) + return send_from_directory(GENERATED_IMAGE_DIR, filename) @app.route("/help") def help(): @@ -380,7 +376,7 @@ def export_trait_excel(): logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) - buff = StringIO.StringIO() + buff = io.BytesIO() workbook = xlsxwriter.Workbook(buff, {'in_memory': True}) worksheet = workbook.add_worksheet() for i, row in enumerate(sample_data): @@ -404,7 +400,7 @@ def export_trait_csv(): logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) - buff = StringIO.StringIO() + buff = io.StringIO() writer = csv.writer(buff) for row in sample_data: writer.writerow(row) @@ -427,7 +423,7 @@ def export_traits_csv(): now = datetime.datetime.now() time_str = now.strftime('%H:%M_%d%B%Y') filename = "export_{}".format(time_str) - memory_file = StringIO.StringIO() + memory_file = io.StringIO() with ZipFile(memory_file, mode='w', compression=ZIP_DEFLATED) as zf: for the_file in file_list: zf.writestr(the_file[0], the_file[1]) @@ -470,7 +466,7 @@ def export_perm_data(): ["#Comment: Results sorted from low to high peak linkage"] ] - buff = StringIO.StringIO() + buff = io.StringIO() writer = csv.writer(buff) writer.writerows(the_rows) for item in perm_info['perm_data']: @@ -543,7 +539,7 @@ def heatmap_page(): result = template_vars.__dict__ - for item in template_vars.__dict__.keys(): + for item in list(template_vars.__dict__.keys()): logger.info(" ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) @@ -647,7 +643,7 @@ def loading_page(): if 'wanted_inputs' in initial_start_vars: wanted = initial_start_vars['wanted_inputs'].split(",") start_vars = {} - for key, value in initial_start_vars.iteritems(): + for key, value in list(initial_start_vars.items()): if key in wanted or key.startswith(('value:')): start_vars[key] = value @@ -747,7 +743,7 @@ def mapping_results_page(): 'transform' ) start_vars = {} - for key, value in initial_start_vars.iteritems(): + for key, value in list(initial_start_vars.items()): if key in wanted or key.startswith(('value:')): start_vars[key] = value #logger.debug("Mapping called with start_vars:", start_vars) @@ -794,7 +790,7 @@ def mapping_results_page(): logger.info("initial_start_vars:", initial_start_vars) imgfile = open(TEMPDIR + img_path, 'rb') imgdata = imgfile.read() - imgB64 = imgdata.encode("base64") + imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) result['pair_scan_array'] = bytesarray rendered_template = render_template("pair_scan_results.html", **result) @@ -954,8 +950,8 @@ def json_default_handler(obj): if hasattr(obj, 'isoformat'): return obj.isoformat() # Handle integer keys for dictionaries - elif isinstance(obj, int): - return str(int) + elif isinstance(obj, int) or isinstance(obj, uuid.UUID): + return str(obj) # Handle custom objects if hasattr(obj, '__dict__'): return obj.__dict__ @@ -963,5 +959,5 @@ def json_default_handler(obj): # logger.info("Not going to serialize Dataset") # return None else: - raise TypeError, 'Object of type %s with value of %s is not JSON serializable' % ( - type(obj), repr(obj)) + raise TypeError('Object of type %s with value of %s is not JSON serializable' % ( + type(obj), repr(obj))) diff --git a/wqflask/wqflask/wgcna/wgcna_analysis.py b/wqflask/wqflask/wgcna/wgcna_analysis.py index 70077703..6bf75216 100644 --- a/wqflask/wqflask/wgcna/wgcna_analysis.py +++ b/wqflask/wqflask/wgcna/wgcna_analysis.py @@ -3,21 +3,19 @@ WGCNA analysis for GN2 Author / Maintainer: Danny Arends <Danny.Arends@gmail.com> """ +import base64 import sys -from numpy import * -import scipy as sp # SciPy import rpy2.robjects as ro # R Objects import rpy2.rinterface as ri +from array import array as arr +from numpy import * from base.webqtlConfig import GENERATED_IMAGE_DIR -from utility import webqtlUtil # Random number for the image - -import base64 -import array +from rpy2.robjects.packages import importr +from utility import webqtlUtil # Random number for the image from utility import helper_functions -from rpy2.robjects.packages import importr utils = importr("utils") # Get pointers to some common R functions @@ -71,8 +69,8 @@ class WGCNA(object): self.r_enableWGCNAThreads() self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] - print("Retrieved phenotype data from database", - requestform['trait_list']) + print(("Retrieved phenotype data from database", + requestform['trait_list'])) helper_functions.get_trait_db_obs(self, self.trait_db_list) # self.input contains the phenotype values we need to send to R @@ -127,14 +125,14 @@ class WGCNA(object): powers = [int(threshold.strip()) for threshold in requestform['SoftThresholds'].rstrip().split(",")] rpow = r_unlist(r_c(powers)) - print("SoftThresholds: {} == {}".format(powers, rpow)) + print(("SoftThresholds: {} == {}".format(powers, rpow))) self.sft = self.r_pickSoftThreshold( rM, powerVector=rpow, verbose=5) - print("PowerEstimate: {}".format(self.sft[0])) + print(("PowerEstimate: {}".format(self.sft[0]))) self.results['PowerEstimate'] = self.sft[0] if self.sft[0][0] is ri.NA_Integer: - print "No power is suitable for the analysis, just use 1" + print("No power is suitable for the analysis, just use 1") # No power could be estimated self.results['Power'] = 1 else: @@ -157,7 +155,7 @@ class WGCNA(object): self.results['network'] = network # How many modules and how many gene per module ? - print "WGCNA found {} modules".format(r_table(network[1])) + print(("WGCNA found {} modules".format(r_table(network[1])))) self.results['nmod'] = r_length(r_table(network[1]))[0] # The iconic WCGNA plot of the modules in the hanging tree @@ -172,11 +170,11 @@ class WGCNA(object): sys.stdout.flush() def render_image(self, results): - print("pre-loading imgage results:", self.results['imgloc']) + print(("pre-loading imgage results:", self.results['imgloc'])) imgfile = open(self.results['imgloc'], 'rb') imgdata = imgfile.read() - imgB64 = imgdata.encode("base64") - bytesarray = array.array('B', imgB64) + imgB64 = base64.b64encode(imgdata) + bytesarray = arr('B', imgB64) self.results['imgdata'] = bytesarray def process_results(self, results): |