From 5c5aefc79d45d66ab30bdbbc5dab24b3f5f79dc2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 17 Aug 2020 20:35:53 +0300 Subject: Apply pep8 * scripts/maintenance/readProbeSetSE_v7.py: Apply pep8 to file to replace tabs with spaces and use correct indentation. --- scripts/maintenance/readProbeSetSE_v7.py | 258 ++++++++++++++++--------------- 1 file changed, 130 insertions(+), 128 deletions(-) diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index fd6f0bb8..2a1d44ff 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -13,12 +13,12 @@ import time def translateAlias(str): - if str == "B6": - return "C57BL/6J" - elif str == "D2": - return "DBA/2J" - else: - return str + if str == "B6": + return "C57BL/6J" + elif str == "D2": + return "DBA/2J" + else: + return str ######################################################################## # @@ -26,25 +26,27 @@ def translateAlias(str): # ######################################################################## + dataStart = 1 -GeneChipId = int( raw_input("Enter GeneChipId:") ) -ProbeSetFreezeId = int( raw_input("Enter ProbeSetFreezeId:") ) +GeneChipId = int(raw_input("Enter GeneChipId:")) +ProbeSetFreezeId = int(raw_input("Enter ProbeSetFreezeId:")) input_file_name = raw_input("Enter file name with suffix:") fp = open("%s" % input_file_name, 'rb') try: - passwd = getpass.getpass('Please enter mysql password here : ') - con = MySQLdb.Connect(db='db_webqtl',host='localhost', user='username',passwd=passwd) + passwd = getpass.getpass('Please enter mysql password here : ') + con = MySQLdb.Connect(db='db_webqtl', host='localhost', + user='username', passwd=passwd) - db = con.cursor() - print "You have successfully connected to mysql.\n" + db = con.cursor() + print("You have successfully connected to mysql.\n") except: - print "You entered incorrect password.\n" - sys.exit(0) - + print("You entered incorrect password.\n") + sys.exit(0) + time0 = time.time() ######################################################################## # @@ -52,9 +54,9 @@ time0 = time.time() # ######################################################################## -#GeneChipId = 4 -#dataStart = 1 -#ProbeSetFreezeId = 359 #JAX Liver 6C Affy M430 2.0 (Jul11) MDP +#GeneChipId = 4 +#dataStart = 1 +# ProbeSetFreezeId = 359 #JAX Liver 6C Affy M430 2.0 (Jul11) MDP #fp = open("GSE10493_AllSamples_6C_Z_AvgSE.txt", 'rb') @@ -64,191 +66,191 @@ time0 = time.time() # generate the gene list of expression data here # ######################################################################### -print 'Checking if each line have same number of members' +print('Checking if each line have same number of members') GeneList = [] isCont = 1 header = fp.readline() -header = string.split(string.strip(header),'\t') +header = string.split(string.strip(header), '\t') header = map(string.strip, header) nfield = len(header) line = fp.readline() -kj=0 +kj = 0 while line: - line2 = string.split(string.strip(line),'\t') - line2 = map(string.strip, line2) - if len(line2) != nfield: - print "Error : " + line - isCont = 0 + line2 = string.split(string.strip(line), '\t') + line2 = map(string.strip, line2) + if len(line2) != nfield: + isCont = 0 + print("Error : " + line) - GeneList.append(line2[0]) - line = fp.readline() + GeneList.append(line2[0]) + line = fp.readline() - kj+=1 - if kj%100000 == 0: - print 'checked ',kj,' lines' + kj += 1 + if kj % 100000 == 0: + print('checked ', kj, ' lines') GeneList = map(string.lower, GeneList) GeneList.sort() - -if isCont==0: - sys.exit(0) +if isCont == 0: + sys.exit(0) -print 'used ',time.time()-time0,' seconds' + +print('used ', time.time()-time0, ' seconds') ######################################################################### # # Check if each strain exist in database # generate the string id list of expression data here # ######################################################################### -print 'Checking if each strain exist in database' +print('Checking if each strain exist in database') isCont = 1 fp.seek(0) header = fp.readline() -header = string.split(string.strip(header),'\t') +header = string.split(string.strip(header), '\t') header = map(string.strip, header) header = map(translateAlias, header) header = header[dataStart:] Ids = [] for item in header: - try: - db.execute('select Id from Strain where Name = "%s"' % item) - Ids.append(db.fetchall()[0][0]) - except: - print item,'does not exist, check the if the strain name is correct' - isCont=0 + try: + db.execute('select Id from Strain where Name = "%s"' % item) + Ids.append(db.fetchall()[0][0]) + except: + isCont = 0 + print(item, 'does not exist, check the if the strain name is correct') -if isCont==0: - sys.exit(0) +if isCont == 0: + sys.exit(0) -print 'used ',time.time()-time0,' seconds' +print('used ', time.time()-time0, ' seconds') ######################################################################## # # Check if each ProbeSet exist in database # ######################################################################## -print 'Check if each ProbeSet exist in database' +print('Check if each ProbeSet exist in database') ##---- find PID is name or target ----## line = fp.readline() line = fp.readline() -line2 = string.split(string.strip(line),'\t') +line2 = string.split(string.strip(line), '\t') line2 = map(string.strip, line2) PId = line2[0] -db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % (PId, GeneChipId)) +db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % + (PId, GeneChipId)) results = db.fetchall() IdStr = 'TargetId' -if len(results)>0: - IdStr = 'Name' +if len(results) > 0: + IdStr = 'Name' ##---- get Name/TargetId list from database ----## -db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % (IdStr, GeneChipId, IdStr)) +db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % ( + IdStr, GeneChipId, IdStr)) results = db.fetchall() - + Names = [] for item in results: - Names.append(item[0]) -Names = map(string.lower, Names) -Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# + Names.append(item[0]) + Names = map(string.lower, Names) + Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# ##---- compare genelist with names ----## -x=y=0 -x1=-1 -GeneList2=[] -while xNames[y]: - y += 1 - - if x%100000==0: - print 'check Name, checked %d lines'%x - -while x Names[y]: + y += 1 + + if x % 100000 == 0: + print('check Name, checked %d lines' % x) + +while x < len(GeneList): + GeneList2.append(GeneList[x]) + x += 1 + +isCont = 1 ferror = open("ProbeSetError.txt", "wb") for item in GeneList2: - ferror.write(item + " doesn't exist \n") - print item, " doesn't exist" - isCont = 0 - -if isCont==0: - sys.exit(0) + ferror.write(item + " doesn't exist \n") + isCont = 0 + + print(item, " doesn't exist") +if isCont == 0: + sys.exit(0) -print 'used ',time.time()-time0,' seconds' +print('used ', time.time()-time0, ' seconds') ############################# -#Insert new Data into SE +# Insert new Data into SE ############################ db.execute(""" - select ProbeSet.%s, ProbeSetXRef.DataId from ProbeSet, ProbeSetXRef - where ProbeSet.Id=ProbeSetXRef.ProbeSetId and ProbeSetXRef.ProbeSetFreezeId=%d""" - % (IdStr, ProbeSetFreezeId)) + select ProbeSet.%s, ProbeSetXRef.DataId from ProbeSet, ProbeSetXRef + where ProbeSet.Id=ProbeSetXRef.ProbeSetId and ProbeSetXRef.ProbeSetFreezeId=%d""" + % (IdStr, ProbeSetFreezeId)) results = db.fetchall() ProbeNameId = {} for Name, Id in results: - ProbeNameId[Name] = Id + ProbeNameId[Name] = Id ferror = open("ProbeError.txt", "wb") DataValues = [] -fp.seek(0) #XZ add this line -line = fp.readline() #XZ add this line +fp.seek(0) # XZ add this line +line = fp.readline() # XZ add this line line = fp.readline() kj = 0 while line: - line2 = string.split(string.strip(line),'\t') - line2 = map(string.strip, line2) - - CellId = line2[0] - if not ProbeNameId.has_key(CellId): - ferror.write(CellId + " doesn't exist\n") - print CellId, " doesn't exist" - else: - DataId = ProbeNameId[CellId] - datasorig = line2[dataStart:] - - i = 0 - for item in datasorig: - if item != '': - value = '('+str(DataId)+','+str(Ids[i])+','+str(item)+')' - DataValues.append(value) - i += 1 - - kj += 1 - if kj % 100 == 0: - Dataitems = ','.join(DataValues) - cmd = 'insert ProbeSetSE values %s' % Dataitems - db.execute(cmd) - - DataValues = [] - print 'inserted ',kj,' lines' - print 'used ',time.time()-time0,' seconds' - line = fp.readline() - -if len(DataValues)>0: - DataValues = ','.join(DataValues) - cmd = 'insert ProbeSetSE values %s' % DataValues - db.execute(cmd) + line2 = string.split(string.strip(line), '\t') + line2 = map(string.strip, line2) + + CellId = line2[0] + if not ProbeNameId.has_key(CellId): + ferror.write(CellId + " doesn't exist\n") + else: + DataId = ProbeNameId[CellId] + datasorig = line2[dataStart:] + + i = 0 + for item in datasorig: + if item != '': + value = '('+str(DataId)+','+str(Ids[i])+','+str(item)+')' + DataValues.append(value) + i += 1 + + kj += 1 + if kj % 100 == 0: + Dataitems = ','.join(DataValues) + cmd = 'insert ProbeSetSE values %s' % Dataitems + db.execute(cmd) + + DataValues = [] + line = fp.readline() + print(CellId, " doesn't exist") + print('inserted ', kj, ' lines') + print('used ', time.time()-time0, ' seconds') + +if len(DataValues) > 0: + DataValues = ','.join(DataValues) + cmd = 'insert ProbeSetSE values %s' % DataValues + db.execute(cmd) con.close() - - -- cgit v1.2.3 From 6c50dff7552df87318253d77ca93e4cc8e26f283 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 17 Aug 2020 20:37:57 +0300 Subject: Replace DOS style line endings with UNIX style ones * scripts/maintenance/readProbeSetSE_v7.py: Run *dos2unix* against file --- scripts/maintenance/readProbeSetSE_v7.py | 512 +++++++++++++++---------------- 1 file changed, 256 insertions(+), 256 deletions(-) diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index 2a1d44ff..0b15ce09 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -1,256 +1,256 @@ -#!/usr/bin/python2 -"""This script use the nearest marker to the transcript as control, increasing permutation rounds according to the p-value""" -######################################################################## -# Last Updated Sep 27, 2011 by Xiaodong -# This version fix the bug that incorrectly exclude the first 2 probesetIDs -######################################################################## - -import string -import sys -import MySQLdb -import getpass -import time - - -def translateAlias(str): - if str == "B6": - return "C57BL/6J" - elif str == "D2": - return "DBA/2J" - else: - return str - -######################################################################## -# -# Indicate Data Start Position, ProbeFreezeId, GeneChipId, DataFile -# -######################################################################## - - -dataStart = 1 - -GeneChipId = int(raw_input("Enter GeneChipId:")) -ProbeSetFreezeId = int(raw_input("Enter ProbeSetFreezeId:")) -input_file_name = raw_input("Enter file name with suffix:") - -fp = open("%s" % input_file_name, 'rb') - - -try: - passwd = getpass.getpass('Please enter mysql password here : ') - con = MySQLdb.Connect(db='db_webqtl', host='localhost', - user='username', passwd=passwd) - - db = con.cursor() - print("You have successfully connected to mysql.\n") -except: - print("You entered incorrect password.\n") - sys.exit(0) - -time0 = time.time() -######################################################################## -# -# Indicate Data Start Position, ProbeFreezeId, GeneChipId, DataFile -# -######################################################################## - -#GeneChipId = 4 -#dataStart = 1 -# ProbeSetFreezeId = 359 #JAX Liver 6C Affy M430 2.0 (Jul11) MDP -#fp = open("GSE10493_AllSamples_6C_Z_AvgSE.txt", 'rb') - - -######################################################################### -# -# Check if each line have same number of members -# generate the gene list of expression data here -# -######################################################################### -print('Checking if each line have same number of members') - -GeneList = [] -isCont = 1 -header = fp.readline() -header = string.split(string.strip(header), '\t') -header = map(string.strip, header) -nfield = len(header) -line = fp.readline() - -kj = 0 -while line: - line2 = string.split(string.strip(line), '\t') - line2 = map(string.strip, line2) - if len(line2) != nfield: - isCont = 0 - print("Error : " + line) - - GeneList.append(line2[0]) - line = fp.readline() - - kj += 1 - if kj % 100000 == 0: - print('checked ', kj, ' lines') - -GeneList = map(string.lower, GeneList) -GeneList.sort() - -if isCont == 0: - sys.exit(0) - - -print('used ', time.time()-time0, ' seconds') -######################################################################### -# -# Check if each strain exist in database -# generate the string id list of expression data here -# -######################################################################### -print('Checking if each strain exist in database') - -isCont = 1 -fp.seek(0) -header = fp.readline() -header = string.split(string.strip(header), '\t') -header = map(string.strip, header) -header = map(translateAlias, header) -header = header[dataStart:] -Ids = [] -for item in header: - try: - db.execute('select Id from Strain where Name = "%s"' % item) - Ids.append(db.fetchall()[0][0]) - except: - isCont = 0 - print(item, 'does not exist, check the if the strain name is correct') - -if isCont == 0: - sys.exit(0) - - -print('used ', time.time()-time0, ' seconds') -######################################################################## -# -# Check if each ProbeSet exist in database -# -######################################################################## -print('Check if each ProbeSet exist in database') - -##---- find PID is name or target ----## -line = fp.readline() -line = fp.readline() -line2 = string.split(string.strip(line), '\t') -line2 = map(string.strip, line2) -PId = line2[0] - -db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % - (PId, GeneChipId)) -results = db.fetchall() -IdStr = 'TargetId' -if len(results) > 0: - IdStr = 'Name' - - -##---- get Name/TargetId list from database ----## -db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % ( - IdStr, GeneChipId, IdStr)) -results = db.fetchall() - -Names = [] -for item in results: - Names.append(item[0]) - Names = map(string.lower, Names) - Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# - -##---- compare genelist with names ----## -x = y = 0 -x1 = -1 -GeneList2 = [] -while x < len(GeneList) and y < len(Names): - if GeneList[x] == Names[y]: - x += 1 - y += 1 - elif GeneList[x] < Names[y]: - if x != x1: - GeneList2.append(GeneList[x]) - x1 = x - x += 1 - elif GeneList[x] > Names[y]: - y += 1 - - if x % 100000 == 0: - print('check Name, checked %d lines' % x) - -while x < len(GeneList): - GeneList2.append(GeneList[x]) - x += 1 - -isCont = 1 -ferror = open("ProbeSetError.txt", "wb") -for item in GeneList2: - ferror.write(item + " doesn't exist \n") - isCont = 0 - - print(item, " doesn't exist") -if isCont == 0: - sys.exit(0) - - -print('used ', time.time()-time0, ' seconds') -############################# -# Insert new Data into SE -############################ -db.execute(""" - select ProbeSet.%s, ProbeSetXRef.DataId from ProbeSet, ProbeSetXRef - where ProbeSet.Id=ProbeSetXRef.ProbeSetId and ProbeSetXRef.ProbeSetFreezeId=%d""" - % (IdStr, ProbeSetFreezeId)) -results = db.fetchall() - -ProbeNameId = {} -for Name, Id in results: - ProbeNameId[Name] = Id - -ferror = open("ProbeError.txt", "wb") - -DataValues = [] - -fp.seek(0) # XZ add this line -line = fp.readline() # XZ add this line -line = fp.readline() - -kj = 0 -while line: - line2 = string.split(string.strip(line), '\t') - line2 = map(string.strip, line2) - - CellId = line2[0] - if not ProbeNameId.has_key(CellId): - ferror.write(CellId + " doesn't exist\n") - else: - DataId = ProbeNameId[CellId] - datasorig = line2[dataStart:] - - i = 0 - for item in datasorig: - if item != '': - value = '('+str(DataId)+','+str(Ids[i])+','+str(item)+')' - DataValues.append(value) - i += 1 - - kj += 1 - if kj % 100 == 0: - Dataitems = ','.join(DataValues) - cmd = 'insert ProbeSetSE values %s' % Dataitems - db.execute(cmd) - - DataValues = [] - line = fp.readline() - print(CellId, " doesn't exist") - print('inserted ', kj, ' lines') - print('used ', time.time()-time0, ' seconds') - -if len(DataValues) > 0: - DataValues = ','.join(DataValues) - cmd = 'insert ProbeSetSE values %s' % DataValues - db.execute(cmd) - -con.close() +#!/usr/bin/python2 +"""This script use the nearest marker to the transcript as control, increasing permutation rounds according to the p-value""" +######################################################################## +# Last Updated Sep 27, 2011 by Xiaodong +# This version fix the bug that incorrectly exclude the first 2 probesetIDs +######################################################################## + +import string +import sys +import MySQLdb +import getpass +import time + + +def translateAlias(str): + if str == "B6": + return "C57BL/6J" + elif str == "D2": + return "DBA/2J" + else: + return str + +######################################################################## +# +# Indicate Data Start Position, ProbeFreezeId, GeneChipId, DataFile +# +######################################################################## + + +dataStart = 1 + +GeneChipId = int(raw_input("Enter GeneChipId:")) +ProbeSetFreezeId = int(raw_input("Enter ProbeSetFreezeId:")) +input_file_name = raw_input("Enter file name with suffix:") + +fp = open("%s" % input_file_name, 'rb') + + +try: + passwd = getpass.getpass('Please enter mysql password here : ') + con = MySQLdb.Connect(db='db_webqtl', host='localhost', + user='username', passwd=passwd) + + db = con.cursor() + print("You have successfully connected to mysql.\n") +except: + print("You entered incorrect password.\n") + sys.exit(0) + +time0 = time.time() +######################################################################## +# +# Indicate Data Start Position, ProbeFreezeId, GeneChipId, DataFile +# +######################################################################## + +#GeneChipId = 4 +#dataStart = 1 +# ProbeSetFreezeId = 359 #JAX Liver 6C Affy M430 2.0 (Jul11) MDP +#fp = open("GSE10493_AllSamples_6C_Z_AvgSE.txt", 'rb') + + +######################################################################### +# +# Check if each line have same number of members +# generate the gene list of expression data here +# +######################################################################### +print('Checking if each line have same number of members') + +GeneList = [] +isCont = 1 +header = fp.readline() +header = string.split(string.strip(header), '\t') +header = map(string.strip, header) +nfield = len(header) +line = fp.readline() + +kj = 0 +while line: + line2 = string.split(string.strip(line), '\t') + line2 = map(string.strip, line2) + if len(line2) != nfield: + isCont = 0 + print("Error : " + line) + + GeneList.append(line2[0]) + line = fp.readline() + + kj += 1 + if kj % 100000 == 0: + print('checked ', kj, ' lines') + +GeneList = map(string.lower, GeneList) +GeneList.sort() + +if isCont == 0: + sys.exit(0) + + +print('used ', time.time()-time0, ' seconds') +######################################################################### +# +# Check if each strain exist in database +# generate the string id list of expression data here +# +######################################################################### +print('Checking if each strain exist in database') + +isCont = 1 +fp.seek(0) +header = fp.readline() +header = string.split(string.strip(header), '\t') +header = map(string.strip, header) +header = map(translateAlias, header) +header = header[dataStart:] +Ids = [] +for item in header: + try: + db.execute('select Id from Strain where Name = "%s"' % item) + Ids.append(db.fetchall()[0][0]) + except: + isCont = 0 + print(item, 'does not exist, check the if the strain name is correct') + +if isCont == 0: + sys.exit(0) + + +print('used ', time.time()-time0, ' seconds') +######################################################################## +# +# Check if each ProbeSet exist in database +# +######################################################################## +print('Check if each ProbeSet exist in database') + +##---- find PID is name or target ----## +line = fp.readline() +line = fp.readline() +line2 = string.split(string.strip(line), '\t') +line2 = map(string.strip, line2) +PId = line2[0] + +db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % + (PId, GeneChipId)) +results = db.fetchall() +IdStr = 'TargetId' +if len(results) > 0: + IdStr = 'Name' + + +##---- get Name/TargetId list from database ----## +db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % ( + IdStr, GeneChipId, IdStr)) +results = db.fetchall() + +Names = [] +for item in results: + Names.append(item[0]) + Names = map(string.lower, Names) + Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# + +##---- compare genelist with names ----## +x = y = 0 +x1 = -1 +GeneList2 = [] +while x < len(GeneList) and y < len(Names): + if GeneList[x] == Names[y]: + x += 1 + y += 1 + elif GeneList[x] < Names[y]: + if x != x1: + GeneList2.append(GeneList[x]) + x1 = x + x += 1 + elif GeneList[x] > Names[y]: + y += 1 + + if x % 100000 == 0: + print('check Name, checked %d lines' % x) + +while x < len(GeneList): + GeneList2.append(GeneList[x]) + x += 1 + +isCont = 1 +ferror = open("ProbeSetError.txt", "wb") +for item in GeneList2: + ferror.write(item + " doesn't exist \n") + isCont = 0 + + print(item, " doesn't exist") +if isCont == 0: + sys.exit(0) + + +print('used ', time.time()-time0, ' seconds') +############################# +# Insert new Data into SE +############################ +db.execute(""" + select ProbeSet.%s, ProbeSetXRef.DataId from ProbeSet, ProbeSetXRef + where ProbeSet.Id=ProbeSetXRef.ProbeSetId and ProbeSetXRef.ProbeSetFreezeId=%d""" + % (IdStr, ProbeSetFreezeId)) +results = db.fetchall() + +ProbeNameId = {} +for Name, Id in results: + ProbeNameId[Name] = Id + +ferror = open("ProbeError.txt", "wb") + +DataValues = [] + +fp.seek(0) # XZ add this line +line = fp.readline() # XZ add this line +line = fp.readline() + +kj = 0 +while line: + line2 = string.split(string.strip(line), '\t') + line2 = map(string.strip, line2) + + CellId = line2[0] + if not ProbeNameId.has_key(CellId): + ferror.write(CellId + " doesn't exist\n") + else: + DataId = ProbeNameId[CellId] + datasorig = line2[dataStart:] + + i = 0 + for item in datasorig: + if item != '': + value = '('+str(DataId)+','+str(Ids[i])+','+str(item)+')' + DataValues.append(value) + i += 1 + + kj += 1 + if kj % 100 == 0: + Dataitems = ','.join(DataValues) + cmd = 'insert ProbeSetSE values %s' % Dataitems + db.execute(cmd) + + DataValues = [] + line = fp.readline() + print(CellId, " doesn't exist") + print('inserted ', kj, ' lines') + print('used ', time.time()-time0, ' seconds') + +if len(DataValues) > 0: + DataValues = ','.join(DataValues) + cmd = 'insert ProbeSetSE values %s' % DataValues + db.execute(cmd) + +con.close() -- cgit v1.2.3 From e49ef0954e111ace2044cced0a83b4d9dc00bc72 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 17 Aug 2020 20:45:08 +0300 Subject: Fix malformed docstring * wqflask/utility/Plot.py: Update docstring. --- wqflask/utility/Plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index d4373412..82bf6070 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -97,7 +97,7 @@ def find_outliers(vals): >>> find_outliers([3.504, 5.234, 6.123, 7.234, 3.542, 5.341, 7.852, 4.555, 12.537]) (11.252500000000001, 0.5364999999999993) - >>> >>> find_outliers([9,12,15,17,31,50,7,5,6,8]) + >>> find_outliers([9,12,15,17,31,50,7,5,6,8]) (32.0, -8.0) If there are no vals, returns None for the upper and lower bounds, -- cgit v1.2.3 From af330a2aa7b36fd0cf8505eb20fa06d2ed58b86b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 17 Aug 2020 17:41:58 +0300 Subject: Wrap print statements in parentheses --- scripts/maintenance/QTL_Reaper_v6.py | 6 ++-- .../Update_Case_Attributes_MySQL_tab.py | 2 +- scripts/maintenance/delete_genotypes.py | 14 ++++---- scripts/maintenance/delete_phenotypes.py | 14 ++++---- scripts/maintenance/load_genotypes.py | 16 ++++----- scripts/maintenance/load_phenotypes.py | 30 ++++++++-------- scripts/maintenance/readProbeSetMean_v7.py | 40 +++++++++++----------- scripts/maintenance/readProbeSetSE_v7.py | 22 ++++++------ wqflask/run_gunicorn.py | 2 +- wqflask/utility/startup_config.py | 4 +-- wqflask/utility/svg.py | 10 +++--- wqflask/utility/tools.py | 8 ++--- wqflask/wqflask/marker_regression/rqtl_mapping.py | 2 +- wqflask/wqflask/pbkdf2.py | 16 ++++----- wqflask/wqflask/views.py | 4 +-- wqflask/wqflask/wgcna/wgcna_analysis.py | 12 +++---- 16 files changed, 101 insertions(+), 101 deletions(-) diff --git a/scripts/maintenance/QTL_Reaper_v6.py b/scripts/maintenance/QTL_Reaper_v6.py index e50dbd40..7fb56eca 100755 --- a/scripts/maintenance/QTL_Reaper_v6.py +++ b/scripts/maintenance/QTL_Reaper_v6.py @@ -53,7 +53,7 @@ for ProbeSetFreezeId in ProbeSetFreezeIds: #if InbredSetId==12: # InbredSetId=2 - print ProbeSetFreezeId, InbredSets[InbredSetId] + print((ProbeSetFreezeId, InbredSets[InbredSetId])) genotype_1.read(InbredSets[InbredSetId]) locuses = [] @@ -102,7 +102,7 @@ for ProbeSetFreezeId in ProbeSetFreezeIds: kj += 1 if kj%1000==0: - print ProbeSetFreezeId, InbredSets[InbredSetId],kj + print((ProbeSetFreezeId, InbredSets[InbredSetId],kj)) - print ProbeSetFreezeIds + print(ProbeSetFreezeIds) diff --git a/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py b/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py index 0f8602c9..bf796df4 100644 --- a/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py +++ b/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py @@ -24,4 +24,4 @@ for row in csv_data: #close the connection to the database. mydb.commit() cursor.close() -print "Done" \ No newline at end of file +print("Done") \ No newline at end of file diff --git a/scripts/maintenance/delete_genotypes.py b/scripts/maintenance/delete_genotypes.py index fa693f0f..060640e1 100755 --- a/scripts/maintenance/delete_genotypes.py +++ b/scripts/maintenance/delete_genotypes.py @@ -8,13 +8,13 @@ import genotypes def main(argv): # config config = utilities.get_config(argv[1]) - print "config:" + print("config:") for item in config.items('config'): - print "\t%s" % (str(item)) + print(("\t%s" % (str(item)))) # var - print "variable:" + print("variable:") inbredsetid = config.get('config', 'inbredsetid') - print "\tinbredsetid: %s" % inbredsetid + print(("\tinbredsetid: %s" % inbredsetid)) # datafile datafile = open(config.get('config', 'datafile'), 'r') datafile = csv.reader(datafile, delimiter='\t', quotechar='"') @@ -25,9 +25,9 @@ def main(argv): continue genoname = row[0] delrowcount += genotypes.delete(genoname, inbredsetid) - print "deleted %d genotypes" % (delrowcount) + print(("deleted %d genotypes" % (delrowcount))) if __name__ == "__main__": - print "command line arguments:\n\t%s" % sys.argv + print(("command line arguments:\n\t%s" % sys.argv)) main(sys.argv) - print "exit successfully" + print("exit successfully") diff --git a/scripts/maintenance/delete_phenotypes.py b/scripts/maintenance/delete_phenotypes.py index 326c466e..60dbec61 100755 --- a/scripts/maintenance/delete_phenotypes.py +++ b/scripts/maintenance/delete_phenotypes.py @@ -8,13 +8,13 @@ import phenotypes def main(argv): # config config = utilities.get_config(argv[1]) - print "config:" + print("config:") for item in config.items('config'): - print "\t%s" % (str(item)) + print(("\t%s" % (str(item)))) # var - print "variable:" + print("variable:") inbredsetid = config.get('config', 'inbredsetid') - print "\tinbredsetid: %s" % inbredsetid + print(("\tinbredsetid: %s" % inbredsetid)) # datafile datafile = open(config.get('config', 'datafile'), 'r') datafile = csv.reader(datafile, delimiter='\t', quotechar='"') @@ -27,9 +27,9 @@ def main(argv): except: continue delrowcount += phenotypes.delete(publishxrefid=publishxrefid, inbredsetid=inbredsetid) - print "deleted %d phenotypes" % (delrowcount) + print(("deleted %d phenotypes" % (delrowcount))) if __name__ == "__main__": - print "command line arguments:\n\t%s" % sys.argv + print(("command line arguments:\n\t%s" % sys.argv)) main(sys.argv) - print "exit successfully" + print("exit successfully") diff --git a/scripts/maintenance/load_genotypes.py b/scripts/maintenance/load_genotypes.py index 338483f4..c235a31f 100755 --- a/scripts/maintenance/load_genotypes.py +++ b/scripts/maintenance/load_genotypes.py @@ -8,7 +8,7 @@ def main(argv): config = utilities.get_config(argv[1]) print("config file:") for item in config.items('config'): - print("\t%s" % str(item)) + print(("\t%s" % str(item))) parse_genofile(config, fetch_parameters(config)) def fetch_parameters(config): @@ -20,7 +20,7 @@ def fetch_parameters(config): config_dic['genofile'] = config.get('config', 'genofile') print("config dictionary:") for k, v in config_dic.items(): - print("\t%s: %s" % (k, v)) + print(("\t%s: %s" % (k, v))) return config_dic def parse_genofile(config, config_dic): @@ -43,9 +43,9 @@ def parse_genofile(config, config_dic): # print("geno file meta dictionary:") for k, v in meta_dic.items(): - print("\t%s: %s" % (k, v)) + print(("\t%s: %s" % (k, v))) # - print("geno file head:\n\t%s" % line) + print(("geno file head:\n\t%s" % line)) strainnames = line.split()[4:] config_dic['strains'] = datastructure.get_strains_bynames(inbredsetid=config_dic['inbredsetid'], strainnames=strainnames, updatestrainxref="yes") continue @@ -81,7 +81,7 @@ def check_or_insert_geno(config_dic, marker_dic): result = cursor.fetchone() if result: genoid = result[0] - print("get geno record: %d" % genoid) + print(("get geno record: %d" % genoid)) else: sql = """ INSERT INTO Geno @@ -95,7 +95,7 @@ def check_or_insert_geno(config_dic, marker_dic): cursor.execute(sql, (config_dic['speciesid'], marker_dic['locus'], marker_dic['locus'], marker_dic['chromosome'], marker_dic['mb'])) rowcount = cursor.rowcount genoid = con.insert_id() - print("INSERT INTO Geno: %d record: %d" % (rowcount, genoid)) + print(("INSERT INTO Geno: %d record: %d" % (rowcount, genoid))) return genoid def check_genoxref(config_dic, marker_dic): @@ -146,9 +146,9 @@ def insert_genoxref(config_dic, marker_dic): """ cursor.execute(sql, (config_dic['genofreezeid'], marker_dic['genoid'], config_dic['dataid'], marker_dic['cm'], 'N')) rowcount = cursor.rowcount - print("INSERT INTO GenoXRef: %d record" % (rowcount)) + print(("INSERT INTO GenoXRef: %d record" % (rowcount))) if __name__ == "__main__": - print("command line arguments:\n\t%s" % sys.argv) + print(("command line arguments:\n\t%s" % sys.argv)) main(sys.argv) print("exit successfully") diff --git a/scripts/maintenance/load_phenotypes.py b/scripts/maintenance/load_phenotypes.py index c3c6570b..61d527d4 100755 --- a/scripts/maintenance/load_phenotypes.py +++ b/scripts/maintenance/load_phenotypes.py @@ -7,31 +7,31 @@ import datastructure def main(argv): # config config = utilities.get_config(argv[1]) - print "config:" + print("config:") for item in config.items('config'): - print "\t%s" % (str(item)) + print(("\t%s" % (str(item)))) # var inbredsetid = config.get('config', 'inbredsetid') - print "inbredsetid: %s" % inbredsetid + print(("inbredsetid: %s" % inbredsetid)) species = datastructure.get_species(inbredsetid) speciesid = species[0] - print "speciesid: %s" % speciesid + print(("speciesid: %s" % speciesid)) dataid = datastructure.get_nextdataid_phenotype() - print "next data id: %s" % dataid + print(("next data id: %s" % dataid)) cursor, con = utilities.get_cursor() # datafile datafile = open(config.get('config', 'datafile'), 'r') phenotypedata = csv.reader(datafile, delimiter='\t', quotechar='"') phenotypedata_head = phenotypedata.next() - print "phenotypedata head:\n\t%s" % phenotypedata_head + print(("phenotypedata head:\n\t%s" % phenotypedata_head)) strainnames = phenotypedata_head[1:] strains = datastructure.get_strains_bynames(inbredsetid=inbredsetid, strainnames=strainnames, updatestrainxref="yes") # metafile metafile = open(config.get('config', 'metafile'), 'r') phenotypemeta = csv.reader(metafile, delimiter='\t', quotechar='"') phenotypemeta_head = phenotypemeta.next() - print "phenotypemeta head:\n\t%s" % phenotypemeta_head - print + print(("phenotypemeta head:\n\t%s" % phenotypemeta_head)) + print() # load for metarow in phenotypemeta: # @@ -67,7 +67,7 @@ def main(argv): )) rowcount = cursor.rowcount phenotypeid = con.insert_id() - print "INSERT INTO Phenotype: %d record: %d" % (rowcount, phenotypeid) + print(("INSERT INTO Phenotype: %d record: %d" % (rowcount, phenotypeid))) # Publication publicationid = None # reset pubmed_id = utilities.to_db_string(metarow[0], None) @@ -81,7 +81,7 @@ def main(argv): re = cursor.fetchone() if re: publicationid = re[0] - print "get Publication record: %d" % publicationid + print(("get Publication record: %d" % publicationid)) if not publicationid: sql = """ INSERT INTO Publication @@ -109,7 +109,7 @@ def main(argv): )) rowcount = cursor.rowcount publicationid = con.insert_id() - print "INSERT INTO Publication: %d record: %d" % (rowcount, publicationid) + print(("INSERT INTO Publication: %d record: %d" % (rowcount, publicationid))) # data for index, strain in enumerate(strains): # @@ -158,14 +158,14 @@ def main(argv): cursor.execute(sql, (inbredsetid, phenotypeid, publicationid, dataid, "")) rowcount = cursor.rowcount publishxrefid = con.insert_id() - print "INSERT INTO PublishXRef: %d record: %d" % (rowcount, publishxrefid) + print(("INSERT INTO PublishXRef: %d record: %d" % (rowcount, publishxrefid))) # for loop next dataid += 1 - print + print() # release con.close() if __name__ == "__main__": - print "command line arguments:\n\t%s" % sys.argv + print(("command line arguments:\n\t%s" % sys.argv)) main(sys.argv) - print "exit successfully" + print("exit successfully") diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index e9c8f25c..e7a4c826 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -42,9 +42,9 @@ try: con = MySQLdb.Connect(db='db_webqtl',host='localhost', user='username',passwd=passwd) db = con.cursor() - print "You have successfully connected to mysql.\n" + print("You have successfully connected to mysql.\n") except: - print "You entered incorrect password.\n" + print("You entered incorrect password.\n") sys.exit(0) time0 = time.time() @@ -55,7 +55,7 @@ time0 = time.time() # generate the gene list of expression data here # ######################################################################### -print 'Checking if each line have same number of members' +print('Checking if each line have same number of members') GeneList = [] isCont = 1 @@ -70,7 +70,7 @@ while line: line2 = string.split(string.strip(line),'\t') line2 = map(string.strip, line2) if len(line2) != nfield: - print "Error : " + line + print(("Error : " + line)) isCont = 0 GeneList.append(line2[0]) @@ -78,7 +78,7 @@ while line: kj+=1 if kj%100000 == 0: - print 'checked ',kj,' lines' + print(('checked ',kj,' lines')) GeneList = map(string.lower, GeneList) GeneList.sort() @@ -87,14 +87,14 @@ if isCont==0: sys.exit(0) -print 'used ',time.time()-time0,' seconds' +print(('used ',time.time()-time0,' seconds')) ######################################################################### # # Check if each strain exist in database # generate the string id list of expression data here # ######################################################################### -print 'Checking if each strain exist in database' +print('Checking if each strain exist in database') isCont = 1 fp.seek(0) @@ -109,20 +109,20 @@ for item in header: db.execute('select Id from Strain where Name = "%s"' % item) Ids.append(db.fetchall()[0][0]) except: - print item,'does not exist, check the if the strain name is correct' + print((item,'does not exist, check the if the strain name is correct')) isCont=0 if isCont==0: sys.exit(0) -print 'used ',time.time()-time0,' seconds' +print(('used ',time.time()-time0,' seconds')) ######################################################################## # # Check if each ProbeSet exist in database # ######################################################################## -print 'Check if each ProbeSet exist in database' +print('Check if each ProbeSet exist in database') ##---- find PID is name or target ----## line = fp.readline() @@ -146,7 +146,7 @@ Names = [] for item in results: Names.append(item[0]) -print Names +print(Names) Names = map(string.lower, Names) @@ -170,7 +170,7 @@ while x 0: DataValues = ','.join(DataValues) diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index adffdca3..58108e03 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -7,7 +7,7 @@ # from flask import Flask # application = Flask(__name__) -print "===> Starting up Gunicorn process" +print("===> Starting up Gunicorn process") from wqflask import app from utility.startup_config import app_config diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 817284dd..42ead709 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -27,7 +27,7 @@ def app_config(): port = get_setting_int("SERVER_PORT") if get_setting_bool("USE_GN_SERVER"): - print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]") + print(("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]")) import requests page = requests.get(get_setting("GN_SERVER_URL")) if page.status_code != 200: @@ -36,4 +36,4 @@ def app_config(): # import utility.elasticsearch_tools as es # es.test_elasticsearch_connection() - print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) + print(("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL")))) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index db13b9d1..6285ea63 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -102,7 +102,7 @@ if use_dom_implementation<>0: from xml.dom import implementation from xml.dom.ext import PrettyPrint except: - raise exceptions.ImportError, "PyXML is required for using the dom implementation" + raise exceptions.ImportError("PyXML is required for using the dom implementation") #The implementation is used for the creating the XML document. #The prettyprint module is used for converting the xml document object to a xml file @@ -1018,12 +1018,12 @@ class drawing: PrettyPrint(root,f) f.close() except: - print "Cannot write SVG file: " + filename + print(("Cannot write SVG file: " + filename)) def validate(self): try: import xml.parsers.xmlproc.xmlval except: - raise exceptions.ImportError,'PyXml is required for validating SVG' + raise exceptions.ImportError('PyXml is required for validating SVG') svg=self.toXml() xv=xml.parsers.xmlproc.xmlval.XMLValidator() try: @@ -1031,7 +1031,7 @@ class drawing: except: raise Exception("SVG is not well formed, see messages above") else: - print "SVG well formed" + print("SVG well formed") if __name__=='__main__': @@ -1065,4 +1065,4 @@ if __name__=='__main__': s.addElement(c) d.setSVG(s) - print d.toXml() + print((d.toXml())) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 77db5d53..f790d424 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -214,20 +214,20 @@ ENDC = '\033[0m' def show_settings(): from utility.tools import LOG_LEVEL - print("Set global log level to "+BLUE+LOG_LEVEL+ENDC) + print(("Set global log level to "+BLUE+LOG_LEVEL+ENDC)) log_level = getattr(logging, LOG_LEVEL.upper()) logging.basicConfig(level=log_level) logger.info(OVERRIDES) logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC) - print "runserver.py: ****** Webserver configuration - k,v pairs from app.config ******" keylist = app.config.keys() + print("runserver.py: ****** Webserver configuration - k,v pairs from app.config ******") keylist.sort() for k in keylist: try: - print("%s: %s%s%s%s" % (k,BLUE,BOLD,get_setting(k),ENDC)) + print(("%s: %s%s%s%s" % (k,BLUE,BOLD,get_setting(k),ENDC))) except: - print("%s: %s%s%s%s" % (k,GREEN,BOLD,app.config[k],ENDC)) + print(("%s: %s%s%s%s" % (k,GREEN,BOLD,app.config[k],ENDC))) # Cached values diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index c5590a85..0a5758af 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -42,7 +42,7 @@ def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permChec png = ro.r["png"] # Map the png function dev_off = ro.r["dev.off"] # Map the device off function - print(r_library("qtl")) # Load R/qtl + print((r_library("qtl"))) # Load R/qtl logger.info("QTL library loaded"); diff --git a/wqflask/wqflask/pbkdf2.py b/wqflask/wqflask/pbkdf2.py index f7f61a09..811c83b0 100644 --- a/wqflask/wqflask/pbkdf2.py +++ b/wqflask/wqflask/pbkdf2.py @@ -92,14 +92,14 @@ def test(): def check(data, salt, iterations, keylen, expected): rv = pbkdf2_hex(data, salt, iterations, keylen) if rv != expected: - print 'Test failed:' - print ' Expected: %s' % expected - print ' Got: %s' % rv - print ' Parameters:' - print ' data=%s' % data - print ' salt=%s' % salt - print ' iterations=%d' % iterations - print + print('Test failed:') + print((' Expected: %s' % expected)) + print((' Got: %s' % rv)) + print(' Parameters:') + print((' data=%s' % data)) + print((' salt=%s' % salt)) + print((' iterations=%d' % iterations)) + print() failed.append(1) # From RFC 6070 diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index dde22bf7..d67f1a2e 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -943,5 +943,5 @@ def json_default_handler(obj): # logger.info("Not going to serialize Dataset") # return None else: - raise TypeError, 'Object of type %s with value of %s is not JSON serializable' % ( - type(obj), repr(obj)) + raise TypeError('Object of type %s with value of %s is not JSON serializable' % ( + type(obj), repr(obj))) diff --git a/wqflask/wqflask/wgcna/wgcna_analysis.py b/wqflask/wqflask/wgcna/wgcna_analysis.py index 880a1cb2..d79ad6df 100644 --- a/wqflask/wqflask/wgcna/wgcna_analysis.py +++ b/wqflask/wqflask/wgcna/wgcna_analysis.py @@ -60,7 +60,7 @@ class WGCNA(object): print("Starting WGCNA analysis on dataset") self.r_enableWGCNAThreads() # Enable multi threading self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] - print("Retrieved phenotype data from database", requestform['trait_list']) + print(("Retrieved phenotype data from database", requestform['trait_list'])) helper_functions.get_trait_db_obs(self, self.trait_db_list) self.input = {} # self.input contains the phenotype values we need to send to R @@ -101,13 +101,13 @@ class WGCNA(object): if requestform.get('SoftThresholds') is not None: powers = [int(threshold.strip()) for threshold in requestform['SoftThresholds'].rstrip().split(",")] rpow = r_unlist(r_c(powers)) - print "SoftThresholds: {} == {}".format(powers, rpow) + print(("SoftThresholds: {} == {}".format(powers, rpow))) self.sft = self.r_pickSoftThreshold(rM, powerVector = rpow, verbose = 5) - print "PowerEstimate: {}".format(self.sft[0]) + print(("PowerEstimate: {}".format(self.sft[0]))) self.results['PowerEstimate'] = self.sft[0] if self.sft[0][0] is ri.NA_Integer: - print "No power is suitable for the analysis, just use 1" + print("No power is suitable for the analysis, just use 1") self.results['Power'] = 1 # No power could be estimated else: self.results['Power'] = self.sft[0][0] # Use the estimated power @@ -122,7 +122,7 @@ class WGCNA(object): self.results['network'] = network # How many modules and how many gene per module ? - print "WGCNA found {} modules".format(r_table(network[1])) + print(("WGCNA found {} modules".format(r_table(network[1])))) self.results['nmod'] = r_length(r_table(network[1]))[0] # The iconic WCGNA plot of the modules in the hanging tree @@ -135,7 +135,7 @@ class WGCNA(object): sys.stdout.flush() def render_image(self, results): - print("pre-loading imgage results:", self.results['imgloc']) + print(("pre-loading imgage results:", self.results['imgloc'])) imgfile = open(self.results['imgloc'], 'rb') imgdata = imgfile.read() imgB64 = imgdata.encode("base64") -- cgit v1.2.3 From acdc611247c8eaccbba3dec00ad640e10a4c99b1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 17 Aug 2020 21:45:04 +0300 Subject: Replace "<>" with "!=" * wqflask/utility/svg.py: Use "!=". The operator, "<>" is removed in Python3 --- wqflask/utility/svg.py | 196 ++++++++++++++++++++++++------------------------- 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 6285ea63..f737d2f2 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -97,7 +97,7 @@ use_dom_implementation=0 import exceptions -if use_dom_implementation<>0: +if use_dom_implementation!=0: try: from xml.dom import implementation from xml.dom.ext import PrettyPrint @@ -398,22 +398,22 @@ class rect(SVGelement): """ def __init__(self,x=None,y=None,width=None,height=None,fill=None,stroke=None,stroke_width=None,**args): if width==None or height==None: - if width<>None: raise ValueError, 'height is required' - if height<>None: raise ValueError, 'width is required' + if width!=None: + if height!=None: else: raise ValueError, 'both height and width are required' SVGelement.__init__(self,'rect',{'width':width,'height':height},**args) - if x<>None: + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width class ellipse(SVGelement): @@ -423,22 +423,22 @@ class ellipse(SVGelement): """ def __init__(self,cx=None,cy=None,rx=None,ry=None,fill=None,stroke=None,stroke_width=None,**args): if rx==None or ry== None: - if rx<>None: raise ValueError, 'rx is required' - if ry<>None: raise ValueError, 'ry is required' + if rx!=None: + if ry!=None: else: raise ValueError, 'both rx and ry are required' SVGelement.__init__(self,'ellipse',{'rx':rx,'ry':ry},**args) - if cx<>None: + if cx!=None: self.attributes['cx']=cx - if cy<>None: + if cy!=None: self.attributes['cy']=cy - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width @@ -451,15 +451,15 @@ class circle(SVGelement): if r==None: raise ValueError, 'r is required' SVGelement.__init__(self,'circle',{'r':r},**args) - if cx<>None: + if cx!=None: self.attributes['cx']=cx - if cy<>None: + if cy!=None: self.attributes['cy']=cy - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width class point(circle): @@ -478,17 +478,17 @@ class line(SVGelement): """ def __init__(self,x1=None,y1=None,x2=None,y2=None,stroke=None,stroke_width=None,**args): SVGelement.__init__(self,'line',**args) - if x1<>None: + if x1!=None: self.attributes['x1']=x1 - if y1<>None: + if y1!=None: self.attributes['y1']=y1 - if x2<>None: + if x2!=None: self.attributes['x2']=x2 - if y2<>None: + if y2!=None: self.attributes['y2']=y2 - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke class polyline(SVGelement): @@ -498,11 +498,11 @@ class polyline(SVGelement): """ def __init__(self,points,fill=None,stroke=None,stroke_width=None,**args): SVGelement.__init__(self,'polyline',{'points':_xypointlist(points)},**args) - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke class polygon(SVGelement): @@ -512,11 +512,11 @@ class polygon(SVGelement): """ def __init__(self,points,fill=None,stroke=None,stroke_width=None,**args): SVGelement.__init__(self,'polygon',{'points':_xypointlist(points)},**args) - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke class path(SVGelement): @@ -526,13 +526,13 @@ class path(SVGelement): """ def __init__(self,pathdata,fill=None,stroke=None,stroke_width=None,id=None,**args): SVGelement.__init__(self,'path',{'d':str(pathdata)},**args) - if stroke<>None: + if stroke!=None: self.attributes['stroke']=stroke - if fill<>None: + if fill!=None: self.attributes['fill']=fill - if stroke_width<>None: + if stroke_width!=None: self.attributes['stroke-width']=stroke_width - if id<>None: + if id!=None: self.attributes['id']=id @@ -543,17 +543,17 @@ class text(SVGelement): """ def __init__(self,x=None,y=None,text=None,font_size=None,font_family=None,text_anchor=None,**args): SVGelement.__init__(self,'text',**args) - if x<>None: + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y - if font_size<>None: + if font_size!=None: self.attributes['font-size']=font_size - if font_family<>None: + if font_family!=None: self.attributes['font-family']=font_family - if text<>None: + if text!=None: self.text=text - if text_anchor<>None: + if text_anchor!=None: self.attributes['text-anchor']=text_anchor @@ -564,7 +564,7 @@ class textpath(SVGelement): """ def __init__(self,link,text=None,**args): SVGelement.__init__(self,'textPath',{'xlink:href':link},**args) - if text<>None: + if text!=None: self.text=text class pattern(SVGelement): @@ -576,15 +576,15 @@ class pattern(SVGelement): """ def __init__(self,x=None,y=None,width=None,height=None,patternUnits=None,**args): SVGelement.__init__(self,'pattern',**args) - if x<>None: + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y - if width<>None: + if width!=None: self.attributes['width']=width - if height<>None: + if height!=None: self.attributes['height']=height - if patternUnits<>None: + if patternUnits!=None: self.attributes['patternUnits']=patternUnits class title(SVGelement): @@ -595,7 +595,7 @@ class title(SVGelement): """ def __init__(self,text=None,**args): SVGelement.__init__(self,'title',**args) - if text<>None: + if text!=None: self.text=text class description(SVGelement): @@ -606,7 +606,7 @@ class description(SVGelement): """ def __init__(self,text=None,**args): SVGelement.__init__(self,'desc',**args) - if text<>None: + if text!=None: self.text=text class lineargradient(SVGelement): @@ -617,15 +617,15 @@ class lineargradient(SVGelement): """ def __init__(self,x1=None,y1=None,x2=None,y2=None,id=None,**args): SVGelement.__init__(self,'linearGradient',**args) - if x1<>None: + if x1!=None: self.attributes['x1']=x1 - if y1<>None: + if y1!=None: self.attributes['y1']=y1 - if x2<>None: + if x2!=None: self.attributes['x2']=x2 - if y2<>None: + if y2!=None: self.attributes['y2']=y2 - if id<>None: + if id!=None: self.attributes['id']=id class radialgradient(SVGelement): @@ -636,17 +636,17 @@ class radialgradient(SVGelement): """ def __init__(self,cx=None,cy=None,r=None,fx=None,fy=None,id=None,**args): SVGelement.__init__(self,'radialGradient',**args) - if cx<>None: + if cx!=None: self.attributes['cx']=cx - if cy<>None: + if cy!=None: self.attributes['cy']=cy - if r<>None: + if r!=None: self.attributes['r']=r - if fx<>None: + if fx!=None: self.attributes['fx']=fx - if fy<>None: + if fy!=None: self.attributes['fy']=fy - if id<>None: + if id!=None: self.attributes['id']=id class stop(SVGelement): @@ -656,7 +656,7 @@ class stop(SVGelement): """ def __init__(self,offset,stop_color=None,**args): SVGelement.__init__(self,'stop',{'offset':offset},**args) - if stop_color<>None: + if stop_color!=None: self.attributes['stop-color']=stop_color class style(SVGelement): @@ -675,16 +675,16 @@ class image(SVGelement): """ def __init__(self,url,x=None,y=None,width=None,height=None,**args): if width==None or height==None: - if width<>None: raise ValueError, 'height is required' - if height<>None: raise ValueError, 'width is required' + if width!=None: + if height!=None: else: raise ValueError, 'both height and width are required' SVGelement.__init__(self,'image',{'xlink:href':url,'width':width,'height':height},**args) - if x<>None: + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y class cursor(SVGelement): @@ -704,17 +704,17 @@ class marker(SVGelement): """ def __init__(self,id=None,viewBox=None,refx=None,refy=None,markerWidth=None,markerHeight=None,**args): SVGelement.__init__(self,'marker',**args) - if id<>None: + if id!=None: self.attributes['id']=id - if viewBox<>None: + if viewBox!=None: self.attributes['viewBox']=_viewboxlist(viewBox) - if refx<>None: + if refx!=None: self.attributes['refX']=refx - if refy<>None: + if refy!=None: self.attributes['refY']=refy - if markerWidth<>None: + if markerWidth!=None: self.attributes['markerWidth']=markerWidth - if markerHeight<>None: + if markerHeight!=None: self.attributes['markerHeight']=markerHeight class group(SVGelement): @@ -725,7 +725,7 @@ class group(SVGelement): """ def __init__(self,id=None,**args): SVGelement.__init__(self,'g',**args) - if id<>None: + if id!=None: self.attributes['id']=id class symbol(SVGelement): @@ -739,9 +739,9 @@ class symbol(SVGelement): def __init__(self,id=None,viewBox=None,**args): SVGelement.__init__(self,'symbol',**args) - if id<>None: + if id!=None: self.attributes['id']=id - if viewBox<>None: + if viewBox!=None: self.attributes['viewBox']=_viewboxlist(viewBox) class defs(SVGelement): @@ -770,14 +770,14 @@ class use(SVGelement): """ def __init__(self,link,x=None,y=None,width=None,height=None,**args): SVGelement.__init__(self,'use',{'xlink:href':link},**args) - if x<>None: + if x!=None: self.attributes['x']=x - if y<>None: + if y!=None: self.attributes['y']=y - if width<>None: + if width!=None: self.attributes['width']=width - if height<>None: + if height!=None: self.attributes['height']=height @@ -796,7 +796,7 @@ class view(SVGelement): a view can be used to create a view with different attributes""" def __init__(self,id=None,**args): SVGelement.__init__(self,'view',**args) - if id<>None: + if id!=None: self.attributes['id']=id class script(SVGelement): @@ -815,11 +815,11 @@ class animate(SVGelement): """ def __init__(self,attribute,fr=None,to=None,dur=None,**args): SVGelement.__init__(self,'animate',{'attributeName':attribute},**args) - if fr<>None: + if fr!=None: self.attributes['from']=fr - if to<>None: + if to!=None: self.attributes['to']=to - if dur<>None: + if dur!=None: self.attributes['dur']=dur class animateMotion(SVGelement): @@ -829,9 +829,9 @@ class animateMotion(SVGelement): """ def __init__(self,pathdata,dur,**args): SVGelement.__init__(self,'animateMotion',**args) - if pathdata<>None: + if pathdata!=None: self.attributes['path']=str(pathdata) - if dur<>None: + if dur!=None: self.attributes['dur']=dur class animateTransform(SVGelement): @@ -842,13 +842,13 @@ class animateTransform(SVGelement): def __init__(self,type=None,fr=None,to=None,dur=None,**args): SVGelement.__init__(self,'animateTransform',{'attributeName':'transform'},**args) #As far as I know the attributeName is always transform - if type<>None: + if type!=None: self.attributes['type']=type - if fr<>None: + if fr!=None: self.attributes['from']=fr - if to<>None: + if to!=None: self.attributes['to']=to - if dur<>None: + if dur!=None: self.attributes['dur']=dur class animateColor(SVGelement): """ac=animateColor(attribute,type,from,to,dur,**args) @@ -857,13 +857,13 @@ class animateColor(SVGelement): """ def __init__(self,attribute,type=None,fr=None,to=None,dur=None,**args): SVGelement.__init__(self,'animateColor',{'attributeName':attribute},**args) - if type<>None: + if type!=None: self.attributes['type']=type - if fr<>None: + if fr!=None: self.attributes['from']=fr - if to<>None: + if to!=None: self.attributes['to']=to - if dur<>None: + if dur!=None: self.attributes['dur']=dur class set(SVGelement): """st=set(attribute,to,during,**args) @@ -872,9 +872,9 @@ class set(SVGelement): """ def __init__(self,attribute,to=None,dur=None,**args): SVGelement.__init__(self,'set',{'attributeName':attribute},**args) - if to<>None: + if to!=None: self.attributes['to']=to - if dur<>None: + if dur!=None: self.attributes['dur']=dur @@ -896,11 +896,11 @@ class svg(SVGelement): """ def __init__(self,viewBox=None, width=None, height=None,**args): SVGelement.__init__(self,'svg',**args) - if viewBox<>None: + if viewBox!=None: self.attributes['viewBox']=_viewboxlist(viewBox) - if width<>None: + if width!=None: self.attributes['width']=width - if height<>None: + if height!=None: self.attributes['height']=height self.namespace="http://www.w3.org/2000/svg" -- cgit v1.2.3 From b0f87da9cbb0a8a462d38472264ed34af5dd8912 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 17 Aug 2020 21:47:49 +0300 Subject: Replace "raw_input" with "input" * scripts/maintenance/readProbeSetMean_v7.py: Use "input". See https://docs.python.org/3/whatsnew/3.0.html#builtins --- scripts/maintenance/readProbeSetMean_v7.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index e7a4c826..fea26731 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -31,9 +31,9 @@ def translateAlias(str): dataStart = 1 -GeneChipId = int( raw_input("Enter GeneChipId:") ) -ProbeSetFreezeId = int( raw_input("Enter ProbeSetFreezeId:") ) -input_file_name = raw_input("Enter file name with suffix:") +GeneChipId = int( input("Enter GeneChipId:") ) +ProbeSetFreezeId = int( input("Enter ProbeSetFreezeId:") ) +input_file_name = input("Enter file name with suffix:") fp = open("%s" % input_file_name, 'rb') -- cgit v1.2.3 From 4e10f4bd8fb902810ee033abb8d509ab641308e1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 18 Aug 2020 17:09:51 +0300 Subject: Apply pep8 * wqflask/utility/svg.py: Apply pep8 to fix indentation error when running `2to3-3.8 -w .`: ```` RefactoringTool: Can't parse ./wqflask/utility/svg.py: IndentationError: unindent does not match any outer indentation level (, line 403) ```` --- wqflask/utility/svg.py | 321 ++++++++++++++++++++++++++++--------------------- 1 file changed, 184 insertions(+), 137 deletions(-) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index f737d2f2..c850feb8 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -25,54 +25,56 @@ # Last updated by GeneNetwork Core Team 2010/10/20 #!/usr/bin/env python -##Copyright (c) 2002, Fedor Baart & Hans de Wit (Stichting Farmaceutische Kengetallen) -##All rights reserved. +# Copyright (c) 2002, Fedor Baart & Hans de Wit (Stichting Farmaceutische Kengetallen) +# All rights reserved. ## -##Redistribution and use in source and binary forms, with or without modification, -##are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: ## -##Redistributions of source code must retain the above copyright notice, this -##list of conditions and the following disclaimer. +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. ## -##Redistributions in binary form must reproduce the above copyright notice, -##this list of conditions and the following disclaimer in the documentation and/or -##other materials provided with the distribution. +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. ## -##Neither the name of the Stichting Farmaceutische Kengetallen nor the names of -##its contributors may be used to endorse or promote products derived from this -##software without specific prior written permission. +# Neither the name of the Stichting Farmaceutische Kengetallen nor the names of +# its contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. ## -##THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -##AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -##IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -##DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -##FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -##DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -##SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -##CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -##OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -##OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Thanks to Gerald Rosennfellner for his help and useful comments. -##Thanks to Gerald Rosennfellner for his help and useful comments. - -__doc__="""Use SVGdraw to generate your SVGdrawings. +import sys +import exceptions +__doc__ = """Use SVGdraw to generate your SVGdrawings. SVGdraw uses an object model drawing and a method toXML to create SVG graphics by using easy to use classes and methods usualy you start by creating a drawing eg d=drawing() - #then you create a SVG root element + # then you create a SVG root element s=svg() - #then you add some elements eg a circle and add it to the svg root element + # then you add some elements eg a circle and add it to the svg root element c=circle() - #you can supply attributes by using named arguments. + # you can supply attributes by using named arguments. c=circle(fill='red',stroke='blue') - #or by updating the attributes attribute: + # or by updating the attributes attribute: c.attributes['stroke-width']=1 s.addElement(c) - #then you add the svg root element to the drawing + # then you add the svg root element to the drawing d.setSVG(s) - #and finaly you xmlify the drawing + # and finaly you xmlify the drawing d.toXml() @@ -82,7 +84,7 @@ This module was created using the SVG specification of www.w3c.org and the O'Reilly (www.oreilly.com) python books as information sources. A svg viewer is available from www.adobe.com""" -__version__="1.0" +__version__ = "1.0" # there are two possibilities to generate svg: # via a dom implementation and directly using text strings @@ -93,33 +95,34 @@ __version__="1.0" # Note that PyXML is required for using the dom implementation. # It is also possible to use the standard minidom. But I didn't try that one. # Anyway the text based approach is about 60 times faster than using the full dom implementation. -use_dom_implementation=0 +use_dom_implementation = 0 -import exceptions -if use_dom_implementation!=0: +if use_dom_implementation != 0: try: from xml.dom import implementation from xml.dom.ext import PrettyPrint except: - raise exceptions.ImportError("PyXML is required for using the dom implementation") -#The implementation is used for the creating the XML document. -#The prettyprint module is used for converting the xml document object to a xml file + raise exceptions.ImportError( + "PyXML is required for using the dom implementation") +# The implementation is used for the creating the XML document. +# The prettyprint module is used for converting the xml document object to a xml file + +assert sys.version_info[0] >= 2 +if sys.version_info[1] < 2: + True = 1 + False = 0 + file = open + +sys.setrecursionlimit = 50 +# The recursion limit is set conservative so mistakes like s=svg() s.addElement(s) +# won't eat up too much processor time. + +# the following code is pasted form xml.sax.saxutils +# it makes it possible to run the code without the xml sax package installed +# To make it possible to have in your text elements, it is necessary to escape the texts + -import sys -assert sys.version_info[0]>=2 -if sys.version_info[1]<2: - True=1 - False=0 - file=open - -sys.setrecursionlimit=50 -#The recursion limit is set conservative so mistakes like s=svg() s.addElement(s) -#won't eat up too much processor time. - -#the following code is pasted form xml.sax.saxutils -#it makes it possible to run the code without the xml sax package installed -#To make it possible to have in your text elements, it is necessary to escape the texts def _escape(data, entities={}): """Escape &, <, and > in a string of data. @@ -127,13 +130,14 @@ def _escape(data, entities={}): the optional entities parameter. The keys and values must all be strings; each key will be replaced with its corresponding value. """ - #data = data.replace("&", "&") + # data = data.replace("&", "&") data = data.replace("<", "<") data = data.replace(">", ">") for chars, entity in entities.items(): data = data.replace(chars, entity) return data + def _quoteattr(data, entities={}): """Escape and quote an attribute value. @@ -156,96 +160,121 @@ def _quoteattr(data, entities={}): return data - def _xypointlist(a): """formats a list of xy pairs""" - s='' - for e in a: #this could be done more elegant - s+=str(e)[1:-1] +' ' + s = '' + for e in a: # this could be done more elegant + s += str(e)[1:-1] + ' ' return s + def _viewboxlist(a): """formats a tuple""" - s='' + s = '' for e in a: - s+=str(e)+' ' + s += str(e)+' ' return s + def _pointlist(a): """formats a list of numbers""" return str(a)[1:-1] + class pathdata: """class used to create a pathdata object which can be used for a path. although most methods are pretty straightforward it might be useful to look at the SVG specification.""" - #I didn't test the methods below. - def __init__(self,x=None,y=None): - self.path=[] + # I didn't test the methods below. + + def __init__(self, x=None, y=None): + self.path = [] if x is not None and y is not None: self.path.append('M '+str(x)+' '+str(y)) + def closepath(self): """ends the path""" self.path.append('z') - def move(self,x,y): + + def move(self, x, y): """move to absolute""" self.path.append('M '+str(x)+' '+str(y)) - def relmove(self,x,y): + + def relmove(self, x, y): """move to relative""" self.path.append('m '+str(x)+' '+str(y)) - def line(self,x,y): + + def line(self, x, y): """line to absolute""" self.path.append('L '+str(x)+' '+str(y)) - def relline(self,x,y): + + def relline(self, x, y): """line to relative""" self.path.append('l '+str(x)+' '+str(y)) - def hline(self,x): + + def hline(self, x): """horizontal line to absolute""" self.path.append('H'+str(x)) - def relhline(self,x): + + def relhline(self, x): """horizontal line to relative""" self.path.append('h'+str(x)) - def vline(self,y): + + def vline(self, y): """verical line to absolute""" self.path.append('V'+str(y)) - def relvline(self,y): + + def relvline(self, y): """vertical line to relative""" self.path.append('v'+str(y)) - def bezier(self,x1,y1,x2,y2,x,y): + + def bezier(self, x1, y1, x2, y2, x, y): """bezier with xy1 and xy2 to xy absolut""" - self.path.append('C'+str(x1)+','+str(y1)+' '+str(x2)+','+str(y2)+' '+str(x)+','+str(y)) - def relbezier(self,x1,y1,x2,y2,x,y): + self.path.append('C'+str(x1)+','+str(y1)+' '+str(x2) + + ','+str(y2)+' '+str(x)+','+str(y)) + + def relbezier(self, x1, y1, x2, y2, x, y): """bezier with xy1 and xy2 to xy relative""" - self.path.append('c'+str(x1)+','+str(y1)+' '+str(x2)+','+str(y2)+' '+str(x)+','+str(y)) - def smbezier(self,x2,y2,x,y): + self.path.append('c'+str(x1)+','+str(y1)+' '+str(x2) + + ','+str(y2)+' '+str(x)+','+str(y)) + + def smbezier(self, x2, y2, x, y): """smooth bezier with xy2 to xy absolut""" self.path.append('S'+str(x2)+','+str(y2)+' '+str(x)+','+str(y)) - def relsmbezier(self,x2,y2,x,y): + + def relsmbezier(self, x2, y2, x, y): """smooth bezier with xy2 to xy relative""" self.path.append('s'+str(x2)+','+str(y2)+' '+str(x)+','+str(y)) - def qbezier(self,x1,y1,x,y): + + def qbezier(self, x1, y1, x, y): """quadratic bezier with xy1 to xy absolut""" self.path.append('Q'+str(x1)+','+str(y1)+' '+str(x)+','+str(y)) - def relqbezier(self,x1,y1,x,y): + + def relqbezier(self, x1, y1, x, y): """quadratic bezier with xy1 to xy relative""" self.path.append('q'+str(x1)+','+str(y1)+' '+str(x)+','+str(y)) - def smqbezier(self,x,y): + + def smqbezier(self, x, y): """smooth quadratic bezier to xy absolut""" self.path.append('T'+str(x)+','+str(y)) - def relsmqbezier(self,x,y): + + def relsmqbezier(self, x, y): """smooth quadratic bezier to xy relative""" self.path.append('t'+str(x)+','+str(y)) - def ellarc(self,rx,ry,xrot,laf,sf,x,y): + + def ellarc(self, rx, ry, xrot, laf, sf, x, y): """elliptival arc with rx and ry rotating with xrot using large-arc-flag and sweep-flag to xy absolut""" - self.path.append('A'+str(rx)+','+str(ry)+' '+str(xrot)+' '+str(laf)+' '+str(sf)+' '+str(x)+' '+str(y)) - def relellarc(self,rx,ry,xrot,laf,sf,x,y): + self.path.append('A'+str(rx)+','+str(ry)+' '+str(xrot) + + ' '+str(laf)+' '+str(sf)+' '+str(x)+' '+str(y)) + + def relellarc(self, rx, ry, xrot, laf, sf, x, y): """elliptival arc with rx and ry rotating with xrot using large-arc-flag and sweep-flag to xy relative""" - self.path.append('a'+str(rx)+','+str(ry)+' '+str(xrot)+' '+str(laf)+' '+str(sf)+' '+str(x)+' '+str(y)) + self.path.append('a'+str(rx)+','+str(ry)+' '+str(xrot) + + ' '+str(laf)+' '+str(sf)+' '+str(x)+' '+str(y)) + def __repr__(self): return ' '.join(self.path) - - class SVGelement: """SVGelement(type,attributes,elements,text,namespace,**args) Creates a arbitrary svg element and is intended to be subclassed not used on its own. @@ -256,52 +285,56 @@ class SVGelement: namespace. Note the elements==None, if elements = None:self.elements=[] construction. This is done because if you default to elements=[] every object has a reference to the same empty list.""" - def __init__(self,type='',attributes=None,elements=None,text='',namespace='',cdata=None, **args): - self.type=type - if attributes==None: - self.attributes={} + + def __init__(self, type='', attributes=None, elements=None, text='', namespace='', cdata=None, **args): + self.type = type + if attributes == None: + self.attributes = {} else: - self.attributes=attributes - if elements==None: - self.elements=[] + self.attributes = attributes + if elements == None: + self.elements = [] else: - self.elements=elements - self.text=text - self.namespace=namespace - self.cdata=cdata + self.elements = elements + self.text = text + self.namespace = namespace + self.cdata = cdata for arg in args.keys(): arg2 = arg.replace("__", ":") arg2 = arg2.replace("_", "-") - self.attributes[arg2]=args[arg] - def addElement(self,SVGelement): + self.attributes[arg2] = args[arg] + + def addElement(self, SVGelement): """adds an element to a SVGelement SVGelement.addElement(SVGelement) """ self.elements.append(SVGelement) - def toXml(self,level,f): + def toXml(self, level, f): f.write('\t'*level) f.write('<'+self.type) for attkey in self.attributes.keys(): - f.write(' '+_escape(str(attkey))+'='+_quoteattr(str(self.attributes[attkey]))) + f.write(' '+_escape(str(attkey))+'=' + + _quoteattr(str(self.attributes[attkey]))) if self.namespace: - f.write(' xmlns="'+ _escape(str(self.namespace))+'" xmlns:xlink="http://www.w3.org/1999/xlink"') + f.write(' xmlns="' + _escape(str(self.namespace)) + + '" xmlns:xlink="http://www.w3.org/1999/xlink"') if self.elements or self.text or self.cdata: f.write('>') if self.elements: f.write('\n') for element in self.elements: - element.toXml(level+1,f) + element.toXml(level+1, f) if self.cdata: f.write('\n'+'\t'*(level+1)+'\n') if self.text: - if type(self.text)==type(''): #If the text is only text + if type(self.text) == type(''): # If the text is only text f.write(_escape(str(self.text))) - else: #If the text is a spannedtext class + else: # If the text is a spannedtext class f.write(str(self.text)) if self.elements: f.write('\t'*level+'\n') @@ -312,6 +345,7 @@ class SVGelement: else: f.write('/>\n') + class tspan(SVGelement): """ts=tspan(text='',**args) @@ -323,19 +357,22 @@ class tspan(SVGelement): st.addtspan(ts) t=text(3,5,st) """ - def __init__(self,text=None,**args): - SVGelement.__init__(self,'tspan',**args) - if self.text<>None: - self.text=text + + def __init__(self, text=None, **args): + SVGelement.__init__(self, 'tspan', **args) + if self.text <> None: + self.text = text + def __repr__(self): - s=" --- scripts/maintenance/QTL_Reaper_v6.py | 2 +- scripts/maintenance/readProbeSetMean_v7.py | 16 ++++++++-------- scripts/maintenance/readProbeSetSE_v7.py | 16 ++++++++-------- test/requests/link_checker.py | 3 +-- wqflask/utility/webqtlUtil.py | 2 +- wqflask/wqflask/api/router.py | 2 +- wqflask/wqflask/correlation_matrix/show_corr_matrix.py | 4 ++-- wqflask/wqflask/export_traits.py | 2 +- wqflask/wqflask/interval_analyst/GeneUtil.py | 2 +- .../wqflask/marker_regression/display_mapping_results.py | 10 +++++----- wqflask/wqflask/marker_regression/plink_mapping.py | 4 ++-- wqflask/wqflask/pbkdf2.py | 2 +- wqflask/wqflask/snp_browser/snp_browser.py | 6 +++--- 13 files changed, 35 insertions(+), 36 deletions(-) diff --git a/scripts/maintenance/QTL_Reaper_v6.py b/scripts/maintenance/QTL_Reaper_v6.py index 7fb56eca..2fbeb53b 100755 --- a/scripts/maintenance/QTL_Reaper_v6.py +++ b/scripts/maintenance/QTL_Reaper_v6.py @@ -23,7 +23,7 @@ for item in results: ProbeSetFreezeIds=sys.argv[1:] if ProbeSetFreezeIds: #####convert the Ids to integer - ProbeSetFreezeIds=map(int, ProbeSetFreezeIds) + ProbeSetFreezeIds=list(map(int, ProbeSetFreezeIds)) else: #####get all of the dataset that need be updated diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index fea26731..97767715 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -61,14 +61,14 @@ GeneList = [] isCont = 1 header = fp.readline() header = string.split(string.strip(header),'\t') -header = map(string.strip, header) +header = list(map(string.strip, header)) nfield = len(header) line = fp.readline() kj=0 while line: line2 = string.split(string.strip(line),'\t') - line2 = map(string.strip, line2) + line2 = list(map(string.strip, line2)) if len(line2) != nfield: print(("Error : " + line)) isCont = 0 @@ -80,7 +80,7 @@ while line: if kj%100000 == 0: print(('checked ',kj,' lines')) -GeneList = map(string.lower, GeneList) +GeneList = list(map(string.lower, GeneList)) GeneList.sort() if isCont==0: @@ -100,8 +100,8 @@ isCont = 1 fp.seek(0) header = fp.readline() header = string.split(string.strip(header),'\t') -header = map(string.strip, header) -header = map(translateAlias, header) +header = list(map(string.strip, header)) +header = list(map(translateAlias, header)) header = header[dataStart:] Ids = [] for item in header: @@ -128,7 +128,7 @@ print('Check if each ProbeSet exist in database') line = fp.readline() line = fp.readline() line2 = string.split(string.strip(line),'\t') -line2 = map(string.strip, line2) +line2 = list(map(string.strip, line2)) PId = line2[0] db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % (PId, GeneChipId) ) @@ -148,7 +148,7 @@ for item in results: print(Names) -Names = map(string.lower, Names) +Names = list(map(string.lower, Names)) Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# @@ -223,7 +223,7 @@ values1 = [] values2 = [] while line: line2 = string.split(string.strip(line),'\t') - line2 = map(string.strip, line2) + line2 = list(map(string.strip, line2)) PId = line2[0] recordId = NameIds[PId] diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index 79ed455f..7b2fee87 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -72,14 +72,14 @@ GeneList = [] isCont = 1 header = fp.readline() header = string.split(string.strip(header), '\t') -header = map(string.strip, header) +header = list(map(string.strip, header)) nfield = len(header) line = fp.readline() kj = 0 while line: line2 = string.split(string.strip(line), '\t') - line2 = map(string.strip, line2) + line2 = list(map(string.strip, line2)) if len(line2) != nfield: isCont = 0 print(("Error : " + line)) @@ -91,7 +91,7 @@ while line: if kj % 100000 == 0: print(('checked ', kj, ' lines')) -GeneList = map(string.lower, GeneList) +GeneList = list(map(string.lower, GeneList)) GeneList.sort() if isCont == 0: @@ -111,8 +111,8 @@ isCont = 1 fp.seek(0) header = fp.readline() header = string.split(string.strip(header), '\t') -header = map(string.strip, header) -header = map(translateAlias, header) +header = list(map(string.strip, header)) +header = list(map(translateAlias, header)) header = header[dataStart:] Ids = [] for item in header: @@ -139,7 +139,7 @@ print('Check if each ProbeSet exist in database') line = fp.readline() line = fp.readline() line2 = string.split(string.strip(line), '\t') -line2 = map(string.strip, line2) +line2 = list(map(string.strip, line2)) PId = line2[0] db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % @@ -158,7 +158,7 @@ results = db.fetchall() Names = [] for item in results: Names.append(item[0]) - Names = map(string.lower, Names) + Names = list(map(string.lower, Names)) Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# ##---- compare genelist with names ----## @@ -220,7 +220,7 @@ line = fp.readline() kj = 0 while line: line2 = string.split(string.strip(line), '\t') - line2 = map(string.strip, line2) + line2 = list(map(string.strip, line2)) CellId = line2[0] if not ProbeNameId.has_key(CellId): diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index 715f330c..df4d32d8 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -27,8 +27,7 @@ def get_links(doc): lambda x: not ( is_root_link(x) or is_mailto_link(x)) - , map(lambda y: y.get("href") - , doc.cssselect("a"))) + , [y.get("href") for y in doc.cssselect("a")]) def verify_link(link): if link[0] == "#": diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index 53661ae4..79991149 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -107,7 +107,7 @@ def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users) if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']: access_to_confidential_phenotype_trait = 1 else: - AuthorizedUsersList=map(string.strip, string.split(authorized_users, ',')) + AuthorizedUsersList=list(map(string.strip, string.split(authorized_users, ','))) if AuthorizedUsersList.__contains__(userName): access_to_confidential_phenotype_trait = 1 return access_to_confidential_phenotype_trait \ No newline at end of file diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index 6324cabe..3fa1d5ba 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -517,7 +517,7 @@ def all_sample_data(dataset_name, file_format = "csv"): line_list.append("x") results_list.append(line_list) - results_list = map(list, zip(*results_list)) + results_list = list(map(list, zip(*results_list))) si = StringIO.StringIO() csv_writer = csv.writer(si) diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 0ac94139..a912344f 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -278,7 +278,7 @@ def zScore(trait_data_array): stdev = math.sqrt(var/(N-1)) if stdev == 0: stdev = 1e-100 - data2 = map(lambda x:(x-mean)/stdev,data) + data2 = [(x-mean)/stdev for x in data] trait_data_array[i] = data2 i += 1 return trait_data_array @@ -299,7 +299,7 @@ def sortEigenVectors(vector): A.append(item[0]) B.append(item[1]) sum = reduce(lambda x,y: x+y, A, 0.0) - A = map(lambda x:x*100.0/sum, A) + A = [x*100.0/sum for x in A] return [A, B] except: return [] \ No newline at end of file diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 3272c03d..6646cc36 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -122,7 +122,7 @@ def export_search_results_csv(targs): csv_rows.append(row_contents) - csv_rows = map(list, itertools.izip_longest(*[row for row in csv_rows])) + csv_rows = list(map(list, itertools.izip_longest(*[row for row in csv_rows]))) writer.writerows(csv_rows) csv_data = buff.getvalue() buff.close() diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index 2c60dd70..273168a8 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -24,7 +24,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): ##List current Species and other Species speciesId = speciesDict[species] - otherSpecies = map(lambda X: [X, speciesDict[X]], speciesDict.keys()) + otherSpecies = [[X, speciesDict[X]] for X in speciesDict.keys()] otherSpecies.remove([species, speciesId]) results = g.db.execute(""" diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index bda899fb..7b6e70d2 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -389,9 +389,9 @@ class DisplayMappingResults(object): Chr_Length.Name in (%s) Order by Chr_Length.OrderId - """ % (self.dataset.group.name, string.join(map(lambda X: "'%s'" % X[0], self.ChrList[1:]), ", "))) + """ % (self.dataset.group.name, string.join(["'%s'" % X[0] for X in self.ChrList[1:]], ", "))) - self.ChrLengthMbList = map(lambda x: x[0]/1000000.0, self.ChrLengthMbList) + self.ChrLengthMbList = [x[0]/1000000.0 for x in self.ChrLengthMbList] self.ChrLengthMbSum = reduce(lambda x, y:x+y, self.ChrLengthMbList, 0.0) if self.ChrLengthMbList: self.MbGraphInterval = self.ChrLengthMbSum/(len(self.ChrLengthMbList)*12) #Empirical Mb interval @@ -1147,8 +1147,8 @@ class DisplayMappingResults(object): tenPercentLength = geneLength*0.0001 SNPdensity = theGO["snpCount"]/geneLength - exonStarts = map(float, theGO['exonStarts'].split(",")[:-1]) - exonEnds = map(float, theGO['exonEnds'].split(",")[:-1]) + exonStarts = list(map(float, theGO['exonStarts'].split(",")[:-1])) + exonEnds = list(map(float, theGO['exonEnds'].split(",")[:-1])) cdsStart = theGO['cdsStart'] cdsEnd = theGO['cdsEnd'] accession = theGO['NM_ID'] @@ -2145,7 +2145,7 @@ class DisplayMappingResults(object): lrsEdgeWidth = 1 else: if self.additiveChecked: - additiveMax = max(map(lambda X : abs(X['additive']), self.qtlresults)) + additiveMax = max([abs(X['additive']) for X in self.qtlresults]) lrsEdgeWidth = 3 if zoom == 2: diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index 2f327faf..9571015e 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -84,7 +84,7 @@ def get_samples_from_ped_file(dataset): while line: lineList = string.split(string.strip(line), '\t') - lineList = map(string.strip, lineList) + lineList = list(map(string.strip, lineList)) sample_name = lineList[0] sample_list.append(sample_name) @@ -157,6 +157,6 @@ def parse_plink_output(output_filename, species): def build_line_list(line=None): line_list = string.split(string.strip(line),' ')# irregular number of whitespaces between columns line_list = [item for item in line_list if item <>''] - line_list = map(string.strip, line_list) + line_list = list(map(string.strip, line_list)) return line_list \ No newline at end of file diff --git a/wqflask/wqflask/pbkdf2.py b/wqflask/wqflask/pbkdf2.py index 811c83b0..731c8843 100644 --- a/wqflask/wqflask/pbkdf2.py +++ b/wqflask/wqflask/pbkdf2.py @@ -66,7 +66,7 @@ def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None): def _pseudorandom(x, mac=mac): h = mac.copy() h.update(x) - return map(ord, h.digest()) + return list(map(ord, h.digest())) buf = [] for block in xrange(1, -(-keylen // mac.digest_size) + 1): rv = u = _pseudorandom(salt + _pack_int(block)) diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index 1d28d76a..b18bfc62 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -459,7 +459,7 @@ class SnpBrowser(object): function_list = [] if function_details: function_list = string.split(string.strip(function_details), ",") - function_list = map(string.strip, function_list) + function_list = list(map(string.strip, function_list)) function_list[0] = function_list[0].title() function_details = ", ".join(item for item in function_list) function_details = function_details.replace("_", " ") @@ -725,11 +725,11 @@ def get_effect_details_by_category(effect_name = None, effect_value = None): codon_effect_group_list = ['Start Lost', 'Stop Gained', 'Stop Lost', 'Nonsynonymous', 'Synonymous'] effect_detail_list = string.split(string.strip(effect_value), '|') - effect_detail_list = map(string.strip, effect_detail_list) + effect_detail_list = list(map(string.strip, effect_detail_list)) for index, item in enumerate(effect_detail_list): item_list = string.split(string.strip(item), ',') - item_list = map(string.strip, item_list) + item_list = list(map(string.strip, item_list)) gene_id = item_list[0] gene_name = item_list[1] -- cgit v1.2.3 From bafbb5b7a4b7db2ca230f292eb45be7e67985259 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 02:05:05 +0300 Subject: Remove erroneous `if .. else` branch * wqflask/utility/svg.py [roct, ellipse, SVGelement]: Raise only a single value error if either height or width is not defined. Fixes parsing error when running `2to3-3.8 -f apply -w .` --- wqflask/utility/svg.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index c850feb8..d66c954e 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -445,12 +445,8 @@ class rect(SVGelement): def __init__(self, x=None, y=None, width=None, height=None, fill=None, stroke=None, stroke_width=None, **args): if width == None or height == None: - raise ValueError, 'height is required' - raise ValueError, 'width is required' - if width!=None: - if height!=None: - else: - raise ValueError, 'both height and width are required' + raise ValueError, 'both height and width are required' + SVGelement.__init__(self,'rect',{'width':width,'height':height},**args) if x!=None: self.attributes['x']=x @@ -470,12 +466,8 @@ class ellipse(SVGelement): """ def __init__(self,cx=None,cy=None,rx=None,ry=None,fill=None,stroke=None,stroke_width=None,**args): if rx==None or ry== None: - raise ValueError, 'rx is required' - raise ValueError, 'ry is required' - if rx!=None: - if ry!=None: - else: - raise ValueError, 'both rx and ry are required' + raise ValueError, 'both rx and ry are required' + SVGelement.__init__(self,'ellipse',{'rx':rx,'ry':ry},**args) if cx!=None: self.attributes['cx']=cx @@ -722,12 +714,7 @@ class image(SVGelement): """ def __init__(self,url,x=None,y=None,width=None,height=None,**args): if width==None or height==None: - raise ValueError, 'height is required' - raise ValueError, 'width is required' - if width!=None: - if height!=None: - else: - raise ValueError, 'both height and width are required' + raise ValueError, 'both height and width are required' SVGelement.__init__(self,'image',{'xlink:href':url,'width':width,'height':height},**args) if x!=None: self.attributes['x']=x -- cgit v1.2.3 From ba123e1e0fe693f9778993c3f8e5a70a28658a4c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 02:31:31 +0300 Subject: Fix dictionary iteration methods Run `2to3-3.8 -f dict -w .` See: and --- scripts/maintenance/load_genotypes.py | 4 +-- wqflask/base/GeneralObject.py | 8 ++--- wqflask/base/trait.py | 4 +-- wqflask/maintenance/gen_select_dataset.py | 6 ++-- .../maintenance/generate_probesetfreeze_file.py | 2 +- wqflask/utility/__init__.py | 4 +-- wqflask/utility/benchmark.py | 4 +-- wqflask/utility/gen_geno_ob.py | 2 +- wqflask/utility/helper_functions.py | 2 +- wqflask/utility/svg.py | 14 ++++---- wqflask/utility/temp_data.py | 2 +- wqflask/utility/tools.py | 2 +- wqflask/wqflask/api/correlation.py | 16 ++++----- wqflask/wqflask/api/gen_menu.py | 6 ++-- wqflask/wqflask/correlation/corr_scatter_plot.py | 6 ++-- wqflask/wqflask/correlation/show_corr_results.py | 30 ++++++++--------- wqflask/wqflask/ctl/ctl_analysis.py | 2 +- wqflask/wqflask/export_traits.py | 4 +-- wqflask/wqflask/heatmap/heatmap.py | 4 +-- wqflask/wqflask/interval_analyst/GeneUtil.py | 2 +- .../marker_regression/display_mapping_results.py | 38 +++++++++++----------- wqflask/wqflask/marker_regression/run_mapping.py | 24 +++++++------- wqflask/wqflask/resource_manager.py | 2 +- wqflask/wqflask/show_trait/export_trait_data.py | 2 +- wqflask/wqflask/show_trait/show_trait.py | 8 ++--- wqflask/wqflask/views.py | 6 ++-- 26 files changed, 102 insertions(+), 102 deletions(-) diff --git a/scripts/maintenance/load_genotypes.py b/scripts/maintenance/load_genotypes.py index c235a31f..51278d48 100755 --- a/scripts/maintenance/load_genotypes.py +++ b/scripts/maintenance/load_genotypes.py @@ -19,7 +19,7 @@ def fetch_parameters(config): config_dic['dataid'] = datastructure.get_nextdataid_genotype() config_dic['genofile'] = config.get('config', 'genofile') print("config dictionary:") - for k, v in config_dic.items(): + for k, v in list(config_dic.items()): print(("\t%s: %s" % (k, v))) return config_dic @@ -42,7 +42,7 @@ def parse_genofile(config, config_dic): if line.lower().startswith("chr"): # print("geno file meta dictionary:") - for k, v in meta_dic.items(): + for k, v in list(meta_dic.items()): print(("\t%s: %s" % (k, v))) # print(("geno file head:\n\t%s" % line)) diff --git a/wqflask/base/GeneralObject.py b/wqflask/base/GeneralObject.py index 0fccaab3..707569db 100644 --- a/wqflask/base/GeneralObject.py +++ b/wqflask/base/GeneralObject.py @@ -33,7 +33,7 @@ class GeneralObject: def __init__(self, *args, **kw): self.contents = list(args) - for name, value in kw.items(): + for name, value in list(kw.items()): setattr(self, name, value) def __setitem__(self, key, value): @@ -50,16 +50,16 @@ class GeneralObject: def __str__(self): s = '' - for key in self.__dict__.keys(): + for key in list(self.__dict__.keys()): if key != 'contents': s += '%s = %s\n' % (key, self.__dict__[key]) return s def __repr__(self): s = '' - for key in self.__dict__.keys(): + for key in list(self.__dict__.keys()): s += '%s = %s\n' % (key, self.__dict__[key]) return s def __cmp__(self, other): - return len(self.__dict__.keys()).__cmp__(len(other.__dict__.keys())) + return len(list(self.__dict__.keys())).__cmp__(len(list(other.__dict__.keys()))) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 7666348e..e82df226 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -118,7 +118,7 @@ class GeneralTrait(object): vals = [] the_vars = [] sample_aliases = [] - for sample_name, sample_data in self.data.items(): + for sample_name, sample_data in list(self.data.items()): if sample_data.value != None: if not include_variance or sample_data.variance != None: samples.append(sample_name) @@ -260,7 +260,7 @@ def get_sample_data(): trait_dict['pubmed_link'] = trait_ob.pubmed_link trait_dict['pubmed_text'] = trait_ob.pubmed_text - return json.dumps([trait_dict, {key: value.value for key, value in trait_ob.data.iteritems() }]) + return json.dumps([trait_dict, {key: value.value for key, value in list(trait_ob.data.items()) }]) else: return None diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index 647e58a2..78217587 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -108,7 +108,7 @@ def get_types(groups): """Build types list""" types = {} #print("Groups: ", pf(groups)) - for species, group_dict in groups.iteritems(): + for species, group_dict in list(groups.items()): types[species] = {} for group_name, _group_full_name in group_dict: # make group an alias to shorten the code @@ -195,9 +195,9 @@ def build_types(species, group): def get_datasets(types): """Build datasets list""" datasets = {} - for species, group_dict in types.iteritems(): + for species, group_dict in list(types.items()): datasets[species] = {} - for group, type_list in group_dict.iteritems(): + for group, type_list in list(group_dict.items()): datasets[species][group] = {} for type_name in type_list: these_datasets = build_datasets(species, group, type_name[0]) diff --git a/wqflask/maintenance/generate_probesetfreeze_file.py b/wqflask/maintenance/generate_probesetfreeze_file.py index b7b2dc8e..4231cc7c 100644 --- a/wqflask/maintenance/generate_probesetfreeze_file.py +++ b/wqflask/maintenance/generate_probesetfreeze_file.py @@ -82,7 +82,7 @@ def get_probeset_vals(cursor, dataset_name): def trim_strains(strains, probeset_vals): trimmed_strains = [] #print("probeset_vals is:", pf(probeset_vals)) - first_probeset = list(probeset_vals.itervalues())[0] + first_probeset = list(probeset_vals.values())[0] print("\n**** first_probeset is:", pf(first_probeset)) for strain in strains: print("\n**** strain is:", pf(strain)) diff --git a/wqflask/utility/__init__.py b/wqflask/utility/__init__.py index d9856eed..204ff59a 100644 --- a/wqflask/utility/__init__.py +++ b/wqflask/utility/__init__.py @@ -19,7 +19,7 @@ class Struct(object): ''' def __init__(self, obj): - for k, v in obj.iteritems(): + for k, v in list(obj.items()): if isinstance(v, dict): setattr(self, k, Struct(v)) else: @@ -30,6 +30,6 @@ class Struct(object): def __repr__(self): return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for - (k, v) in self.__dict__.iteritems())) + (k, v) in list(self.__dict__.items()))) diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py index 8f1c916b..221e5151 100644 --- a/wqflask/utility/benchmark.py +++ b/wqflask/utility/benchmark.py @@ -38,9 +38,9 @@ class Bench(object): @classmethod def report(cls): - total_time = sum((time_taken for time_taken in cls.entries.itervalues())) + total_time = sum((time_taken for time_taken in list(cls.entries.values()))) print("\nTiming report\n") - for name, time_taken in cls.entries.iteritems(): + for name, time_taken in list(cls.entries.items()): percent = int(round((time_taken/total_time) * 100)) print("[{}%] {}: {}".format(percent, name, time_taken)) print() diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index 23b0b650..ae42f834 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -175,7 +175,7 @@ class Locus(object): start_pos = 3 for allele in marker_row[start_pos:]: - if allele in geno_table.keys(): + if allele in list(geno_table.keys()): self.genotype.append(geno_table[allele]) else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown self.genotype.append("U") \ No newline at end of file diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 9ce809b6..9a4a235a 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__ ) def get_species_dataset_trait(self, start_vars): #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype" - if "temp_trait" in start_vars.keys(): + if "temp_trait" in list(start_vars.keys()): if start_vars['temp_trait'] == "True": self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) else: diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index d66c954e..c6a5c260 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -133,7 +133,7 @@ def _escape(data, entities={}): # data = data.replace("&", "&") data = data.replace("<", "<") data = data.replace(">", ">") - for chars, entity in entities.items(): + for chars, entity in list(entities.items()): data = data.replace(chars, entity) return data @@ -299,7 +299,7 @@ class SVGelement: self.text = text self.namespace = namespace self.cdata = cdata - for arg in args.keys(): + for arg in list(args.keys()): arg2 = arg.replace("__", ":") arg2 = arg2.replace("_", "-") self.attributes[arg2] = args[arg] @@ -314,7 +314,7 @@ class SVGelement: def toXml(self, level, f): f.write('\t'*level) f.write('<'+self.type) - for attkey in self.attributes.keys(): + for attkey in list(self.attributes.keys()): f.write(' '+_escape(str(attkey))+'=' + _quoteattr(str(self.attributes[attkey]))) if self.namespace: @@ -365,7 +365,7 @@ class tspan(SVGelement): def __repr__(self): s = "\n" % (item, self.entity[item])) xml.write("]") xml.write(">\n") @@ -1015,7 +1015,7 @@ class drawing: if element.text: textnode=root.createTextNode(element.text) e.appendChild(textnode) - for attribute in element.attributes.keys(): #in element.attributes is supported from python 2.2 + for attribute in list(element.attributes.keys()): #in element.attributes is supported from python 2.2 e.setAttribute(attribute,str(element.attributes[attribute])) if element.elements: for el in element.elements: diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 5bf700c9..2f2726c6 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -20,6 +20,6 @@ class TempData(object): if __name__ == "__main__": redis = Redis() - for key in redis.keys(): + for key in list(redis.keys()): for field in redis.hkeys(key): print("{}.{}={}".format(key, field, redis.hget(key, field))) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index f790d424..51a87fe1 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -220,7 +220,7 @@ def show_settings(): logger.info(OVERRIDES) logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC) - keylist = app.config.keys() + keylist = list(app.config.keys()) print("runserver.py: ****** Webserver configuration - k,v pairs from app.config ******") keylist.sort() for k in keylist: diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index 7f5312c1..eb05645e 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -36,7 +36,7 @@ def do_correlation(start_vars): #corr_results = collections.OrderedDict(sorted(corr_results.items(), key=lambda t: -abs(t[1][0]))) final_results = [] - for _trait_counter, trait in enumerate(corr_results.keys()[:corr_params['return_count']]): + for _trait_counter, trait in enumerate(list(corr_results.keys())[:corr_params['return_count']]): if corr_params['type'] == "tissue": [sample_r, num_overlap, sample_p, symbol] = corr_results[trait] result_dict = { @@ -76,20 +76,20 @@ def calculate_results(this_trait, this_dataset, target_dataset, corr_params): if corr_params['type'] == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") corr_results = do_tissue_correlation_for_all_traits(this_trait, trait_symbol_dict, corr_params) - sorted_results = collections.OrderedDict(sorted(corr_results.items(), + sorted_results = collections.OrderedDict(sorted(list(corr_results.items()), key=lambda t: -abs(t[1][1]))) elif corr_params['type'] == "literature" or corr_params['type'] == "lit": #ZS: Just so a user can use either "lit" or "literature" trait_geneid_dict = this_dataset.retrieve_genes("GeneId") corr_results = do_literature_correlation_for_all_traits(this_trait, this_dataset, trait_geneid_dict, corr_params) - sorted_results = collections.OrderedDict(sorted(corr_results.items(), + sorted_results = collections.OrderedDict(sorted(list(corr_results.items()), key=lambda t: -abs(t[1][1]))) else: - for target_trait, target_vals in target_dataset.trait_data.iteritems(): + for target_trait, target_vals in list(target_dataset.trait_data.items()): result = get_sample_r_and_p_values(this_trait, this_dataset, target_vals, target_dataset, corr_params['type']) if result is not None: corr_results[target_trait] = result - sorted_results = collections.OrderedDict(sorted(corr_results.items(), key=lambda t: -abs(t[1][0]))) + sorted_results = collections.OrderedDict(sorted(list(corr_results.items()), key=lambda t: -abs(t[1][0]))) return sorted_results @@ -100,10 +100,10 @@ def do_tissue_correlation_for_all_traits(this_trait, trait_symbol_dict, corr_par if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower()] - corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(symbol_list=trait_symbol_dict.values()) + corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(symbol_list=list(trait_symbol_dict.values())) tissue_corr_data = {} - for trait, symbol in trait_symbol_dict.iteritems(): + for trait, symbol in list(trait_symbol_dict.items()): if symbol and symbol.lower() in corr_result_tissue_vals_dict: this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower()] @@ -119,7 +119,7 @@ def do_literature_correlation_for_all_traits(this_trait, target_dataset, trait_g input_trait_mouse_gene_id = convert_to_mouse_gene_id(target_dataset.group.species.lower(), this_trait.geneid) lit_corr_data = {} - for trait, gene_id in trait_geneid_dict.iteritems(): + for trait, gene_id in list(trait_geneid_dict.items()): mouse_gene_id = convert_to_mouse_gene_id(target_dataset.group.species.lower(), gene_id) if mouse_gene_id and str(mouse_gene_id).find(";") == -1: diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index cc11e14b..71d9ee03 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -61,7 +61,7 @@ def get_types(groups): """Build types list""" types = {} - for species, group_dict in groups.iteritems(): + for species, group_dict in list(groups.items()): types[species] = {} for group_name, _group_full_name, _family_name in group_dict: if phenotypes_exist(group_name): @@ -136,9 +136,9 @@ def build_types(species, group): def get_datasets(types): """Build datasets list""" datasets = {} - for species, group_dict in types.iteritems(): + for species, group_dict in list(types.items()): datasets[species] = {} - for group, type_list in group_dict.iteritems(): + for group, type_list in list(group_dict.items()): datasets[species][group] = {} for type_name in type_list: these_datasets = build_datasets(species, group, type_name[0]) diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index 819836b1..57a8d85f 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -36,13 +36,13 @@ class CorrScatterPlot(object): samples_1, samples_2, num_overlap = corr_result_helpers.normalize_values_with_samples(self.trait_1.data, self.trait_2.data) self.data = [] - self.indIDs = samples_1.keys() + self.indIDs = list(samples_1.keys()) vals_1 = [] - for sample in samples_1.keys(): + for sample in list(samples_1.keys()): vals_1.append(samples_1[sample].value) self.data.append(vals_1) vals_2 = [] - for sample in samples_2.keys(): + for sample in list(samples_2.keys()): vals_2.append(samples_2[sample].value) self.data.append(vals_2) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index de7a1c0c..15a21ee6 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -145,10 +145,10 @@ class CorrelationResults(object): if corr_samples_group == 'samples_other': primary_samples = [x for x in primary_samples if x not in ( self.dataset.group.parlist + self.dataset.group.f1list)] - self.process_samples(start_vars, self.this_trait.data.keys(), primary_samples) + self.process_samples(start_vars, list(self.this_trait.data.keys()), primary_samples) self.target_dataset = data_set.create_dataset(start_vars['corr_dataset']) - self.target_dataset.get_trait_data(self.sample_data.keys()) + self.target_dataset.get_trait_data(list(self.sample_data.keys())) self.header_fields = get_header_fields(self.target_dataset.type, self.corr_method) @@ -168,41 +168,41 @@ class CorrelationResults(object): tissue_corr_data = self.do_tissue_correlation_for_all_traits() if tissue_corr_data != None: - for trait in tissue_corr_data.keys()[:self.return_number]: + for trait in list(tissue_corr_data.keys())[:self.return_number]: self.get_sample_r_and_p_values(trait, self.target_dataset.trait_data[trait]) else: - for trait, values in self.target_dataset.trait_data.iteritems(): + for trait, values in list(self.target_dataset.trait_data.items()): self.get_sample_r_and_p_values(trait, values) elif self.corr_type == "lit": self.trait_geneid_dict = self.dataset.retrieve_genes("GeneId") lit_corr_data = self.do_lit_correlation_for_all_traits() - for trait in lit_corr_data.keys()[:self.return_number]: + for trait in list(lit_corr_data.keys())[:self.return_number]: self.get_sample_r_and_p_values(trait, self.target_dataset.trait_data[trait]) elif self.corr_type == "sample": - for trait, values in self.target_dataset.trait_data.iteritems(): + for trait, values in list(self.target_dataset.trait_data.items()): self.get_sample_r_and_p_values(trait, values) - self.correlation_data = collections.OrderedDict(sorted(self.correlation_data.items(), + self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), key=lambda t: -abs(t[1][0]))) if self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Geno": #ZS: Convert min/max chromosome to an int for the location range option range_chr_as_int = None - for order_id, chr_info in self.dataset.species.chromosomes.chromosomes.iteritems(): + for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): if 'loc_chr' in start_vars: if chr_info.name == self.location_chr: range_chr_as_int = order_id - for _trait_counter, trait in enumerate(self.correlation_data.keys()[:self.return_number]): + for _trait_counter, trait in enumerate(list(self.correlation_data.keys())[:self.return_number]): trait_object = create_trait(dataset=self.target_dataset, name=trait, get_qtl_info=True, get_sample_info=False) if self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Geno": #ZS: Convert trait chromosome to an int for the location range option chr_as_int = 0 - for order_id, chr_info in self.dataset.species.chromosomes.chromosomes.iteritems(): + for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): if chr_info.name == trait_object.chr: chr_as_int = order_id @@ -297,14 +297,14 @@ class CorrelationResults(object): #print("trait_gene_symbols: ", pf(trait_gene_symbols.values())) corr_result_tissue_vals_dict= correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=self.trait_symbol_dict.values()) + symbol_list=list(self.trait_symbol_dict.values())) #print("corr_result_tissue_vals: ", pf(corr_result_tissue_vals_dict)) #print("trait_gene_symbols: ", pf(trait_gene_symbols)) tissue_corr_data = {} - for trait, symbol in self.trait_symbol_dict.iteritems(): + for trait, symbol in list(self.trait_symbol_dict.items()): if symbol and symbol.lower() in corr_result_tissue_vals_dict: this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower()] @@ -314,7 +314,7 @@ class CorrelationResults(object): tissue_corr_data[trait] = [symbol, result[0], result[2]] - tissue_corr_data = collections.OrderedDict(sorted(tissue_corr_data.items(), + tissue_corr_data = collections.OrderedDict(sorted(list(tissue_corr_data.items()), key=lambda t: -abs(t[1][1]))) return tissue_corr_data @@ -359,7 +359,7 @@ class CorrelationResults(object): input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), self.this_trait.geneid) lit_corr_data = {} - for trait, gene_id in self.trait_geneid_dict.iteritems(): + for trait, gene_id in list(self.trait_geneid_dict.items()): mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), gene_id) if mouse_gene_id and str(mouse_gene_id).find(";") == -1: @@ -387,7 +387,7 @@ class CorrelationResults(object): else: lit_corr_data[trait] = [gene_id, 0] - lit_corr_data = collections.OrderedDict(sorted(lit_corr_data.items(), + lit_corr_data = collections.OrderedDict(sorted(list(lit_corr_data.items()), key=lambda t: -abs(t[1][1]))) return lit_corr_data diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 35067036..f0be7a98 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -125,7 +125,7 @@ class CTL(object): gt = create_trait(name = ts[0], dataset_name = ts[1]) gt = retrieve_sample_data(gt, dataset, individuals) for ind in individuals: - if ind in gt.data.keys(): + if ind in list(gt.data.keys()): traits.append(gt.data[ind].value) else: traits.append("-999") diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 6646cc36..28c6593d 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -61,7 +61,7 @@ def export_search_results_csv(targs): traits_by_group = sort_traits_by_group(trait_list) file_list = [] - for group in traits_by_group.keys(): + for group in list(traits_by_group.keys()): group_traits = traits_by_group[group] buff = StringIO.StringIO() writer = csv.writer(buff) @@ -135,7 +135,7 @@ def export_search_results_csv(targs): def sort_traits_by_group(trait_list=[]): traits_by_group = {} for trait in trait_list: - if trait.dataset.group.name not in traits_by_group.keys(): + if trait.dataset.group.name not in list(traits_by_group.keys()): traits_by_group[trait.dataset.group.name] = [] traits_by_group[trait.dataset.group.name].append(trait) diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 5098a184..577426b0 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -60,7 +60,7 @@ class Heatmap(object): chrnames = [] self.species = species.TheSpecies(dataset=self.trait_list[0][1]) - for key in self.species.chromosomes.chromosomes.keys(): + for key in list(self.species.chromosomes.chromosomes.keys()): chrnames.append([self.species.chromosomes.chromosomes[key].name, self.species.chromosomes.chromosomes[key].mb_length]) for trait_db in self.trait_list: @@ -93,7 +93,7 @@ class Heatmap(object): pos = [] markernames = [] - for trait in self.trait_results.keys(): + for trait in list(self.trait_results.keys()): lodnames.append(trait) self.dataset.group.get_markers() diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index 273168a8..a39e5d0f 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -24,7 +24,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): ##List current Species and other Species speciesId = speciesDict[species] - otherSpecies = [[X, speciesDict[X]] for X in speciesDict.keys()] + otherSpecies = [[X, speciesDict[X]] for X in list(speciesDict.keys())] otherSpecies.remove([species, speciesId]) results = g.db.execute(""" diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 7b6e70d2..0328ce85 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -229,7 +229,7 @@ class DisplayMappingResults(object): self.manhattan_plot = start_vars['manhattan_plot'] - if 'permCheck' in start_vars.keys(): + if 'permCheck' in list(start_vars.keys()): self.permChecked = start_vars['permCheck'] else: self.permChecked = False @@ -242,46 +242,46 @@ class DisplayMappingResults(object): else: self.nperm = 0 - if 'bootCheck' in start_vars.keys(): + if 'bootCheck' in list(start_vars.keys()): self.bootChecked = start_vars['bootCheck'] else: self.bootChecked = False - if 'num_bootstrap' in start_vars.keys(): + if 'num_bootstrap' in list(start_vars.keys()): self.nboot = int(start_vars['num_bootstrap']) else: self.nboot = 0 - if 'bootstrap_results' in start_vars.keys(): + if 'bootstrap_results' in list(start_vars.keys()): self.bootResult = start_vars['bootstrap_results'] else: self.bootResult = [] - if 'do_control' in start_vars.keys(): + if 'do_control' in list(start_vars.keys()): self.doControl = start_vars['do_control'] else: self.doControl = "false" - if 'control_marker' in start_vars.keys(): + if 'control_marker' in list(start_vars.keys()): self.controlLocus = start_vars['control_marker'] else: self.controlLocus = "" - if 'covariates' in start_vars.keys(): + if 'covariates' in list(start_vars.keys()): self.covariates = start_vars['covariates'] - if 'maf' in start_vars.keys(): + if 'maf' in list(start_vars.keys()): self.maf = start_vars['maf'] else: self.maf = "" - if 'output_files' in start_vars.keys(): + if 'output_files' in list(start_vars.keys()): self.output_files = start_vars['output_files'] - if 'use_loco' in start_vars.keys() and self.mapping_method == "gemma": + if 'use_loco' in list(start_vars.keys()) and self.mapping_method == "gemma": self.use_loco = start_vars['use_loco'] - if 'reaper_version' in start_vars.keys() and self.mapping_method == "reaper": + if 'reaper_version' in list(start_vars.keys()) and self.mapping_method == "reaper": self.reaper_version = start_vars['reaper_version'] if 'output_files' in start_vars: self.output_files = ",".join(start_vars['output_files']) self.categorical_vars = "" self.perm_strata = "" - if 'perm_strata' in start_vars.keys() and 'categorical_vars' in start_vars.keys(): + if 'perm_strata' in list(start_vars.keys()) and 'categorical_vars' in list(start_vars.keys()): self.categorical_vars = start_vars['categorical_vars'] self.perm_strata = start_vars['perm_strata'] @@ -323,7 +323,7 @@ class DisplayMappingResults(object): self.graphWidth = self.MULT_GRAPH_DEFAULT_WIDTH ## BEGIN HaplotypeAnalyst - if 'haplotypeAnalystCheck' in start_vars.keys(): + if 'haplotypeAnalystCheck' in list(start_vars.keys()): self.haplotypeAnalystChecked = start_vars['haplotypeAnalystCheck'] else: self.haplotypeAnalystChecked = False @@ -331,25 +331,25 @@ class DisplayMappingResults(object): self.graphHeight = self.GRAPH_DEFAULT_HEIGHT self.dominanceChecked = False - if 'LRSCheck' in start_vars.keys(): + if 'LRSCheck' in list(start_vars.keys()): self.LRS_LOD = start_vars['LRSCheck'] else: self.LRS_LOD = start_vars['score_type'] self.intervalAnalystChecked = True self.draw2X = False - if 'additiveCheck' in start_vars.keys(): + if 'additiveCheck' in list(start_vars.keys()): self.additiveChecked = start_vars['additiveCheck'] else: self.additiveChecked = False - if 'viewLegend' in start_vars.keys(): + if 'viewLegend' in list(start_vars.keys()): self.legendChecked = start_vars['viewLegend'] else: self.legendChecked = False - if 'showSNP' in start_vars.keys(): + if 'showSNP' in list(start_vars.keys()): self.SNPChecked = start_vars['showSNP'] else: self.SNPChecked = False - if 'showGenes' in start_vars.keys(): + if 'showGenes' in list(start_vars.keys()): self.geneChecked = start_vars['showGenes'] else: self.geneChecked = False @@ -530,7 +530,7 @@ class DisplayMappingResults(object): showLocusForm = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name=showLocusForm, submit=HT.Input(type='hidden')) hddn = {'FormID':'showDatabase', 'ProbeSetID':'_','database':fd.RISet+"Geno",'CellID':'_', 'RISet':fd.RISet, 'incparentsf1':'ON'} - for key in hddn.keys(): + for key in list(hddn.keys()): showLocusForm.append(HT.Input(name=key, value=hddn[key], type='hidden')) showLocusForm.append(intImg) else: diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index c9d10f7c..145dbc77 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -347,7 +347,7 @@ class RunMapping(object): if marker['chr1'] > 0 or marker['chr1'] == "X" or marker['chr1'] == "X/Y": if marker['chr1'] > highest_chr or marker['chr1'] == "X" or marker['chr1'] == "X/Y": highest_chr = marker['chr1'] - if 'lod_score' in marker.keys(): + if 'lod_score' in list(marker.keys()): self.qtl_results.append(marker) self.trimmed_markers = results @@ -411,7 +411,7 @@ class RunMapping(object): if marker['chr'] > 0 or marker['chr'] == "X" or marker['chr'] == "X/Y": if marker['chr'] > highest_chr or marker['chr'] == "X" or marker['chr'] == "X/Y": highest_chr = marker['chr'] - if ('lod_score' in marker.keys()) or ('lrs_value' in marker.keys()): + if ('lod_score' in list(marker.keys())) or ('lrs_value' in list(marker.keys())): self.qtl_results.append(marker) with Bench("Exporting Results"): @@ -538,28 +538,28 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, output_file.write("Mb," + score_type) else: output_file.write("Cm," + score_type) - if "additive" in markers[0].keys(): + if "additive" in list(markers[0].keys()): output_file.write(",Additive") - if "dominance" in markers[0].keys(): + if "dominance" in list(markers[0].keys()): output_file.write(",Dominance") output_file.write("\n") for i, marker in enumerate(markers): output_file.write(marker['name'] + "," + str(marker['chr']) + "," + str(marker['Mb']) + ",") - if "lod_score" in marker.keys(): + if "lod_score" in list(marker.keys()): output_file.write(str(marker['lod_score'])) else: output_file.write(str(marker['lrs_value'])) - if "additive" in marker.keys(): + if "additive" in list(marker.keys()): output_file.write("," + str(marker['additive'])) - if "dominance" in marker.keys(): + if "dominance" in list(marker.keys()): output_file.write("," + str(marker['dominance'])) if i < (len(markers) - 1): output_file.write("\n") def trim_markers_for_figure(markers): - if 'p_wald' in markers[0].keys(): + if 'p_wald' in list(markers[0].keys()): score_type = 'p_wald' - elif 'lod_score' in markers[0].keys(): + elif 'lod_score' in list(markers[0].keys()): score_type = 'lod_score' else: score_type = 'lrs_value' @@ -617,7 +617,7 @@ def trim_markers_for_figure(markers): return filtered_markers def trim_markers_for_table(markers): - if 'lod_score' in markers[0].keys(): + if 'lod_score' in list(markers[0].keys()): sorted_markers = sorted(markers, key=lambda k: k['lod_score'], reverse=True) else: sorted_markers = sorted(markers, key=lambda k: k['lrs_value'], reverse=True) @@ -695,10 +695,10 @@ def get_genofile_samplelist(dataset): def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): perm_strata_strings = [] for sample in used_samples: - if sample in sample_list.sample_attribute_values.keys(): + if sample in list(sample_list.sample_attribute_values.keys()): combined_string = "" for var in categorical_vars: - if var in sample_list.sample_attribute_values[sample].keys(): + if var in list(sample_list.sample_attribute_values[sample].keys()): combined_string += str(sample_list.sample_attribute_values[sample][var]) else: combined_string += "NA" diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 39a07310..6b3e00fb 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -125,7 +125,7 @@ def add_group_to_resource(): def get_group_names(group_masks): group_masks_with_names = {} - for group_id, group_mask in group_masks.iteritems(): + for group_id, group_mask in list(group_masks.items()): this_mask = group_mask group_name = get_group_info(group_id)['name'] this_mask['name'] = group_name diff --git a/wqflask/wqflask/show_trait/export_trait_data.py b/wqflask/wqflask/show_trait/export_trait_data.py index 253c887b..68c3ad7d 100644 --- a/wqflask/wqflask/show_trait/export_trait_data.py +++ b/wqflask/wqflask/show_trait/export_trait_data.py @@ -47,7 +47,7 @@ def get_export_metadata(trait_id, dataset_name): def dict_to_sorted_list(dictionary): - sorted_list = [item for item in dictionary.iteritems()] + sorted_list = [item for item in list(dictionary.items())] sorted_list = sorted(sorted_list, cmp=cmp_samples) sorted_values = [item[1] for item in sorted_list] return sorted_values diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index f188fd9d..c156e61b 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -261,7 +261,7 @@ class ShowTrait(object): hddn['export_data'] = "" hddn['export_format'] = "excel" if len(self.scales_in_geno) < 2: - hddn['mapping_scale'] = self.scales_in_geno[self.scales_in_geno.keys()[0]][0][0] + hddn['mapping_scale'] = self.scales_in_geno[list(self.scales_in_geno.keys())[0]][0][0] # We'll need access to this_trait and hddn in the Jinja2 Template, so we put it inside self self.hddn = hddn @@ -405,7 +405,7 @@ class ShowTrait(object): if not self.temp_trait: other_sample_names = [] - for sample in self.this_trait.data.keys(): + for sample in list(self.this_trait.data.keys()): if (self.this_trait.data[sample].name2 in primary_sample_names) and (self.this_trait.data[sample].name not in primary_sample_names): primary_sample_names.append(self.this_trait.data[sample].name) primary_sample_names.remove(self.this_trait.data[sample].name2) @@ -558,7 +558,7 @@ def get_table_widths(sample_groups, has_num_cases=False): def has_num_cases(this_trait): has_n = False if this_trait.dataset.type != "ProbeSet" and this_trait.dataset.type != "Geno": - for name, sample in this_trait.data.iteritems(): + for name, sample in list(this_trait.data.items()): if sample.num_cases: has_n = True break @@ -611,7 +611,7 @@ def get_categorical_variables(this_trait, sample_list): if len(sample_list.attributes) > 0: for attribute in sample_list.attributes: attribute_vals = [] - for sample_name in this_trait.data.keys(): + for sample_name in list(this_trait.data.keys()): if sample_list.attributes[attribute].name in this_trait.data[sample_name].extra_attributes: attribute_vals.append(this_trait.data[sample_name].extra_attributes[sample_list.attributes[attribute].name]) else: diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index d67f1a2e..394a9e28 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -533,7 +533,7 @@ def heatmap_page(): result = template_vars.__dict__ - for item in template_vars.__dict__.keys(): + for item in list(template_vars.__dict__.keys()): logger.info(" ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) @@ -637,7 +637,7 @@ def loading_page(): if 'wanted_inputs' in initial_start_vars: wanted = initial_start_vars['wanted_inputs'].split(",") start_vars = {} - for key, value in initial_start_vars.iteritems(): + for key, value in list(initial_start_vars.items()): if key in wanted or key.startswith(('value:')): start_vars[key] = value @@ -737,7 +737,7 @@ def mapping_results_page(): 'transform' ) start_vars = {} - for key, value in initial_start_vars.iteritems(): + for key, value in list(initial_start_vars.items()): if key in wanted or key.startswith(('value:')): start_vars[key] = value #logger.debug("Mapping called with start_vars:", start_vars) -- cgit v1.2.3 From 28e75b1c96819eab1e8052a3045ece26e5f35c42 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 02:38:15 +0300 Subject: Wrap `filter()` usage in a `list` call Run `2to3-3.8 -f filter -w .` See --- test/requests/link_checker.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index df4d32d8..fa70747d 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -23,11 +23,9 @@ def is_in_page_link(link): return pattern.match(link) def get_links(doc): - return filter( - lambda x: not ( + return [x for x in [y.get("href") for y in doc.cssselect("a")] if not ( is_root_link(x) - or is_mailto_link(x)) - , [y.get("href") for y in doc.cssselect("a")]) + or is_mailto_link(x))] def verify_link(link): if link[0] == "#": @@ -56,9 +54,9 @@ def check_page(host, start_url): print("Checking links host "+host+" in page `"+start_url+"`") doc = parse(start_url).getroot() links = get_links(doc) - in_page_links = filter(is_in_page_link, links) - internal_links = filter(is_internal_link, links) - external_links = filter(lambda x: not (is_internal_link(x) or is_in_page_link(x)), links) + in_page_links = list(filter(is_in_page_link, links)) + internal_links = list(filter(is_internal_link, links)) + external_links = [x for x in links if not (is_internal_link(x) or is_in_page_link(x))] for link in internal_links: verify_link(host+link) -- cgit v1.2.3 From e55b1502340cc99cd8a5d705261a5ff3c87f3718 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 02:41:46 +0300 Subject: Change `dict.has_key(key)` to `key in dict` Run `2to3-3.8 -f has_key -w .` See: --- scripts/maintenance/readProbeSetSE_v7.py | 2 +- wqflask/wqflask/marker_regression/plink_mapping.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index 7b2fee87..2700a8ef 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -223,7 +223,7 @@ while line: line2 = list(map(string.strip, line2)) CellId = line2[0] - if not ProbeNameId.has_key(CellId): + if CellId not in ProbeNameId: ferror.write(CellId + " doesn't exist\n") else: DataId = ProbeNameId[CellId] diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index 9571015e..38ef7190 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -111,7 +111,7 @@ def parse_plink_output(output_filename, species): line_list = build_line_list(line=line) # only keep the records whose chromosome name is in db - if species.chromosomes.chromosomes.has_key(int(line_list[0])) and line_list[-1] and line_list[-1].strip()!='NA': + if int(line_list[0]) in species.chromosomes.chromosomes and line_list[-1] and line_list[-1].strip()!='NA': chr_name = species.chromosomes.chromosomes[int(line_list[0])] snp = line_list[1] @@ -121,7 +121,7 @@ def parse_plink_output(output_filename, species): if p_value < threshold_p_value: p_value_dict[snp] = float(p_value) - if plink_results.has_key(chr_name): + if chr_name in plink_results: value_list = plink_results[chr_name] # pvalue range is [0,1] -- cgit v1.2.3 From e63331da5ed0a11f2a558d799dd570bf44ad584e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:01:33 +0300 Subject: Replace `izip` with python's built-in equivalent Run `2to3-3.8 -f itertools -w . && 2to3-3.8 -f itertools_imports -w .` See: and --- wqflask/base/data_set.py | 2 +- wqflask/maintenance/quantile_normalize.py | 4 ++-- wqflask/wqflask/export_traits.py | 2 +- wqflask/wqflask/pbkdf2.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index cfba9104..b0119b58 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -259,7 +259,7 @@ class Markers(object): # if len(self.markers) > len(p_values): # self.markers = self.markers[:len(p_values)] - for marker, p_value in itertools.izip(self.markers, p_values): + for marker, p_value in zip(self.markers, p_values): if not p_value: continue marker['p_value'] = float(p_value) diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 41a3aad8..34886f44 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, print_function, division import sys sys.path.insert(0,'./') -from itertools import izip + import MySQLdb import urlparse @@ -60,7 +60,7 @@ def set_data(dataset_name): sample_list = [] with open(orig_file, 'r') as orig_fh, open('/home/zas1024/cfw_data/quant_norm.csv', 'r') as quant_fh: - for i, (line1, line2) in enumerate(izip(orig_fh, quant_fh)): + for i, (line1, line2) in enumerate(zip(orig_fh, quant_fh)): trait_dict = {} sample_list = [] if i == 0: diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 28c6593d..a8b49829 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -122,7 +122,7 @@ def export_search_results_csv(targs): csv_rows.append(row_contents) - csv_rows = list(map(list, itertools.izip_longest(*[row for row in csv_rows]))) + csv_rows = list(map(list, itertools.zip_longest(*[row for row in csv_rows]))) writer.writerows(csv_rows) csv_data = buff.getvalue() buff.close() diff --git a/wqflask/wqflask/pbkdf2.py b/wqflask/wqflask/pbkdf2.py index 731c8843..0ed50790 100644 --- a/wqflask/wqflask/pbkdf2.py +++ b/wqflask/wqflask/pbkdf2.py @@ -44,7 +44,7 @@ import hmac import hashlib from struct import Struct from operator import xor -from itertools import izip, starmap +from itertools import starmap _pack_int = Struct('>I').pack @@ -72,7 +72,7 @@ def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None): rv = u = _pseudorandom(salt + _pack_int(block)) for i in xrange(iterations - 1): u = _pseudorandom(''.join(map(chr, u))) - rv = list(starmap(xor, izip(rv, u))) + rv = list(starmap(xor, zip(rv, u))) buf.extend(rv) return ''.join(map(chr, buf))[:keylen] @@ -81,7 +81,7 @@ def safe_str_cmp(a, b): if len(a) != len(b): return False rv = 0 - for x, y in izip(a, b): + for x, y in zip(a, b): rv |= ord(x) ^ ord(y) return rv == 0 -- cgit v1.2.3 From caec08fa1e738fa9bc1b0b6bf626d8325f798712 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:08:08 +0300 Subject: Convert the old not-equal syntax, <>, to != Run `2to3-3.8 -f ne -w .` See: --- wqflask/utility/svg.py | 2 +- wqflask/wqflask/marker_regression/plink_mapping.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index c6a5c260..c7356e57 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -360,7 +360,7 @@ class tspan(SVGelement): def __init__(self, text=None, **args): SVGelement.__init__(self, 'tspan', **args) - if self.text <> None: + if self.text != None: self.text = text def __repr__(self): diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index 38ef7190..d4ee6fe6 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -156,7 +156,7 @@ def parse_plink_output(output_filename, species): ####################################################### def build_line_list(line=None): line_list = string.split(string.strip(line),' ')# irregular number of whitespaces between columns - line_list = [item for item in line_list if item <>''] + line_list = [item for item in line_list if item !=''] line_list = list(map(string.strip, line_list)) return line_list \ No newline at end of file -- cgit v1.2.3 From 8be6ecf3d6b70b40be97d4abebb59eabcce8c8f8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:10:33 +0300 Subject: Convert the use of iterator’s next() methods to the next() function Run `2to3-3.8 -f next -w .` See: --- scripts/maintenance/delete_genotypes.py | 2 +- scripts/maintenance/load_phenotypes.py | 10 +++++----- wqflask/wqflask/collect.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/maintenance/delete_genotypes.py b/scripts/maintenance/delete_genotypes.py index 060640e1..b7f83758 100755 --- a/scripts/maintenance/delete_genotypes.py +++ b/scripts/maintenance/delete_genotypes.py @@ -18,7 +18,7 @@ def main(argv): # datafile datafile = open(config.get('config', 'datafile'), 'r') datafile = csv.reader(datafile, delimiter='\t', quotechar='"') - datafile.next() + next(datafile) delrowcount = 0 for row in datafile: if len(row) == 0: diff --git a/scripts/maintenance/load_phenotypes.py b/scripts/maintenance/load_phenotypes.py index 61d527d4..759d2eec 100755 --- a/scripts/maintenance/load_phenotypes.py +++ b/scripts/maintenance/load_phenotypes.py @@ -22,22 +22,22 @@ def main(argv): # datafile datafile = open(config.get('config', 'datafile'), 'r') phenotypedata = csv.reader(datafile, delimiter='\t', quotechar='"') - phenotypedata_head = phenotypedata.next() + phenotypedata_head = next(phenotypedata) print(("phenotypedata head:\n\t%s" % phenotypedata_head)) strainnames = phenotypedata_head[1:] strains = datastructure.get_strains_bynames(inbredsetid=inbredsetid, strainnames=strainnames, updatestrainxref="yes") # metafile metafile = open(config.get('config', 'metafile'), 'r') phenotypemeta = csv.reader(metafile, delimiter='\t', quotechar='"') - phenotypemeta_head = phenotypemeta.next() + phenotypemeta_head = next(phenotypemeta) print(("phenotypemeta head:\n\t%s" % phenotypemeta_head)) print() # load for metarow in phenotypemeta: # - datarow_value = phenotypedata.next() - datarow_se = phenotypedata.next() - datarow_n = phenotypedata.next() + datarow_value = next(phenotypedata) + datarow_se = next(phenotypedata) + datarow_n = next(phenotypedata) # Phenotype sql = """ INSERT INTO Phenotype diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 42a09fed..4c6e38e6 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -193,7 +193,7 @@ def view_collection(): params = request.args uc_id = params['uc_id'] - uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next() + uc = next((collection for collection in g.user_session.user_collections if collection["id"] == uc_id)) traits = uc["members"] trait_obs = [] -- cgit v1.2.3 From 7e60647223017220747d248ed1c986cc8374435e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:13:53 +0300 Subject: Wrap `raise` statements in parenthesis Run `2to3-3.8 -f raise -w .` See: --- wqflask/base/trait.py | 2 +- wqflask/utility/svg.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index e82df226..c2d260e3 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -605,6 +605,6 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if trait.lrs != "": trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs else: - raise KeyError, `trait.name`+' information is not found in the database.' + raise KeyError(`trait.name`+' information is not found in the database.') return trait \ No newline at end of file diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index c7356e57..872f22fe 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -445,7 +445,7 @@ class rect(SVGelement): def __init__(self, x=None, y=None, width=None, height=None, fill=None, stroke=None, stroke_width=None, **args): if width == None or height == None: - raise ValueError, 'both height and width are required' + raise ValueError('both height and width are required') SVGelement.__init__(self,'rect',{'width':width,'height':height},**args) if x!=None: @@ -466,7 +466,7 @@ class ellipse(SVGelement): """ def __init__(self,cx=None,cy=None,rx=None,ry=None,fill=None,stroke=None,stroke_width=None,**args): if rx==None or ry== None: - raise ValueError, 'both rx and ry are required' + raise ValueError('both rx and ry are required') SVGelement.__init__(self,'ellipse',{'rx':rx,'ry':ry},**args) if cx!=None: @@ -488,7 +488,7 @@ class circle(SVGelement): """ def __init__(self,cx=None,cy=None,r=None,fill=None,stroke=None,stroke_width=None,**args): if r==None: - raise ValueError, 'r is required' + raise ValueError('r is required') SVGelement.__init__(self,'circle',{'r':r},**args) if cx!=None: self.attributes['cx']=cx @@ -714,7 +714,7 @@ class image(SVGelement): """ def __init__(self,url,x=None,y=None,width=None,height=None,**args): if width==None or height==None: - raise ValueError, 'both height and width are required' + raise ValueError('both height and width are required') SVGelement.__init__(self,'image',{'xlink:href':url,'width':width,'height':height},**args) if x!=None: self.attributes['x']=x -- cgit v1.2.3 From e8e7f835861b6e4eb940a9f55b7769d225f2c87d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:16:29 +0300 Subject: Convert `raw_input` to `input` Run `2to3-3.8 -f raw_input -w .` See: --- scripts/maintenance/readProbeSetSE_v7.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index 2700a8ef..833c3f5f 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -29,9 +29,9 @@ def translateAlias(str): dataStart = 1 -GeneChipId = int(raw_input("Enter GeneChipId:")) -ProbeSetFreezeId = int(raw_input("Enter ProbeSetFreezeId:")) -input_file_name = raw_input("Enter file name with suffix:") +GeneChipId = int(input("Enter GeneChipId:")) +ProbeSetFreezeId = int(input("Enter ProbeSetFreezeId:")) +input_file_name = input("Enter file name with suffix:") fp = open("%s" % input_file_name, 'rb') -- cgit v1.2.3 From 17a72093f829666cc6e8df722771d31066dd71d0 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:24:27 +0300 Subject: Handle the move of reduce() to functools.reduce() Run `2to3-3.8 -f reduce -w .` See: --- wqflask/wqflask/correlation_matrix/show_corr_matrix.py | 6 +++++- wqflask/wqflask/marker_regression/display_mapping_results.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index a912344f..e6c817e7 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -42,6 +42,10 @@ import rpy2.robjects as robjects from pprint import pformat as pf from utility.redis_tools import get_redis_conn +try: # Only import this for Python3 + from functools import reduce +except: + pass Redis = get_redis_conn() THIRTY_DAYS = 60 * 60 * 24 * 30 @@ -302,4 +306,4 @@ def sortEigenVectors(vector): A = [x*100.0/sum for x in A] return [A, B] except: - return [] \ No newline at end of file + return [] diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 0328ce85..c8b9c405 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -48,6 +48,10 @@ from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR, GENERATED_IMAGE_DIR from utility.pillow_utils import draw_rotated_text, draw_open_polygon import utility.logger +try: # Only import this for Python3 + from functools import reduce +except: + pass logger = utility.logger.getLogger(__name__ ) RED = ImageColor.getrgb("red") -- cgit v1.2.3 From 1c3ade2f739608563cc1659dd93e5b2abc446046 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:27:21 +0300 Subject: Replace backtick repr with the repr() function * wqflask/base/trait.py: See --- wqflask/base/trait.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index c2d260e3..a513b589 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -605,6 +605,6 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if trait.lrs != "": trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs else: - raise KeyError(`trait.name`+' information is not found in the database.') + raise KeyError(repr(trait.name)+' information is not found in the database.') return trait \ No newline at end of file -- cgit v1.2.3 From 606ec8b9b3bbb8b2269ab546ebafa454f1753176 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:31:53 +0300 Subject: Handle the rename of urllib and urllib2 to the urllib package See: --- wqflask/base/trait.py | 2 +- wqflask/db/call.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index a513b589..2b8f2e72 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -6,7 +6,7 @@ import resource import codecs import requests import random -import urllib +import urllib.request, urllib.parse, urllib.error from base import webqtlConfig from base.webqtlCaseData import webqtlCaseData diff --git a/wqflask/db/call.py b/wqflask/db/call.py index 1a1b3adc..82cfebb4 100644 --- a/wqflask/db/call.py +++ b/wqflask/db/call.py @@ -3,7 +3,7 @@ from flask import g import string -import urllib2 +import urllib.request, urllib.error, urllib.parse import json from utility.tools import USE_GN_SERVER, LOG_SQL, GN_SERVER_URL from utility.benchmark import Bench @@ -59,7 +59,7 @@ def gn_server(path): """ with Bench("GN_SERVER",LOG_SQL): - res = urllib2.urlopen(GN_SERVER_URL+path) + res = urllib.request.urlopen(GN_SERVER_URL+path) rest = res.read() res2 = json.loads(rest) logger.debug(res2) -- cgit v1.2.3 From 9d5dff44fb4d07f926659dde0c6205bf12a1ca5b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:36:24 +0300 Subject: Rename xrange() to range() and wrap existing range() calls with list See: --- wqflask/maintenance/quantile_normalize.py | 2 +- wqflask/utility/Plot.py | 2 +- wqflask/wqflask/correlation/correlation_functions.py | 4 ++-- wqflask/wqflask/correlation_matrix/show_corr_matrix.py | 6 +++--- wqflask/wqflask/marker_regression/qtlreaper_mapping.py | 2 +- wqflask/wqflask/network_graph/network_graph.py | 4 ++-- wqflask/wqflask/pbkdf2.py | 4 ++-- wqflask/wqflask/search_results.py | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 34886f44..82b695f4 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -37,7 +37,7 @@ def create_dataframe(input_file): with open(input_file) as f: ncols = len(f.readline().split("\t")) - input_array = np.loadtxt(open(input_file, "rb"), delimiter="\t", skiprows=1, usecols=range(1, ncols)) + input_array = np.loadtxt(open(input_file, "rb"), delimiter="\t", skiprows=1, usecols=list(range(1, ncols))) return pd.DataFrame(input_array) #This function taken from https://github.com/ShawnLYU/Quantile_Normalize diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 82bf6070..c9053dde 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -86,7 +86,7 @@ def frange(start, end=None, inc=1.0): # Need to adjust the count. AFAICT, it always comes up one short. count += 1 L = [start] * count - for i in xrange(1, count): + for i in range(1, count): L[i] = start + i * inc return L diff --git a/wqflask/wqflask/correlation/correlation_functions.py b/wqflask/wqflask/correlation/correlation_functions.py index 06dec795..abaa212f 100644 --- a/wqflask/wqflask/correlation/correlation_functions.py +++ b/wqflask/wqflask/correlation/correlation_functions.py @@ -50,12 +50,12 @@ from flask import Flask, g def cal_zero_order_corr_for_tiss (primaryValue=[], targetValue=[], method='pearson'): - R_primary = rpy2.robjects.FloatVector(range(len(primaryValue))) + R_primary = rpy2.robjects.FloatVector(list(range(len(primaryValue)))) N = len(primaryValue) for i in range(len(primaryValue)): R_primary[i] = primaryValue[i] - R_target = rpy2.robjects.FloatVector(range(len(targetValue))) + R_target = rpy2.robjects.FloatVector(list(range(len(targetValue)))) for i in range(len(targetValue)): R_target[i]=targetValue[i] diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index e6c817e7..832746bb 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -194,7 +194,7 @@ class CorrelationMatrix(object): if self.do_PCA == True: self.pca_works = "True" self.pca_trait_ids = [] - pca = self.calculate_pca(range(len(self.traits)), corr_eigen_value, corr_eigen_vectors) + pca = self.calculate_pca(list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors) self.loadings_array = self.process_loadings() else: self.pca_works = "False" @@ -203,8 +203,8 @@ class CorrelationMatrix(object): self.js_data = dict(traits = [trait.name for trait in self.traits], groups = groups, - cols = range(len(self.traits)), - rows = range(len(self.traits)), + cols = list(range(len(self.traits))), + rows = list(range(len(self.traits))), samples = self.all_sample_list, sample_data = self.sample_data,) # corr_results = [result[1] for result in result_row for result_row in self.corr_results]) diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index 0c560582..189c1985 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -228,4 +228,4 @@ def natural_sort(marker_list): """ convert = lambda text: int(text) if text.isdigit() else text.lower() alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', str(marker_list[key]['chr'])) ] - return sorted(range(len(marker_list)), key = alphanum_key) \ No newline at end of file + return sorted(list(range(len(marker_list))), key = alphanum_key) \ No newline at end of file diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index f61c40b4..ac2ff017 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -202,8 +202,8 @@ class NetworkGraph(object): self.js_data = dict(traits = [trait.name for trait in self.traits], groups = groups, - cols = range(len(self.traits)), - rows = range(len(self.traits)), + cols = list(range(len(self.traits))), + rows = list(range(len(self.traits))), samples = self.all_sample_list, sample_data = self.sample_data, elements = self.elements,) diff --git a/wqflask/wqflask/pbkdf2.py b/wqflask/wqflask/pbkdf2.py index 0ed50790..917b9d31 100644 --- a/wqflask/wqflask/pbkdf2.py +++ b/wqflask/wqflask/pbkdf2.py @@ -68,9 +68,9 @@ def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None): h.update(x) return list(map(ord, h.digest())) buf = [] - for block in xrange(1, -(-keylen // mac.digest_size) + 1): + for block in range(1, -(-keylen // mac.digest_size) + 1): rv = u = _pseudorandom(salt + _pack_int(block)) - for i in xrange(iterations - 1): + for i in range(iterations - 1): u = _pseudorandom(''.join(map(chr, u))) rv = list(starmap(xor, zip(rv, u))) buf.extend(rv) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index de4b01eb..5b3946e3 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -266,7 +266,7 @@ def get_GO_symbols(a_search): def insert_newlines(string, every=64): """ This is because it is seemingly impossible to change the width of the description column, so I'm just manually adding line breaks """ lines = [] - for i in xrange(0, len(string), every): + for i in range(0, len(string), every): lines.append(string[i:i+every]) return '\n'.join(lines) -- cgit v1.2.3 From 3aaa28ea762c496eeb84e09e45194e3fd2a51673 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:45:49 +0300 Subject: Make Python more idiomatic Run `2to3-3.8 -f idioms -w .` See: --- scripts/maintenance/readProbeSetMean_v7.py | 6 ++---- scripts/maintenance/readProbeSetSE_v7.py | 6 ++---- wqflask/base/data_set.py | 4 ++-- wqflask/utility/authentication_tools.py | 2 +- wqflask/utility/svg.py | 2 +- wqflask/wqflask/ctl/ctl_analysis.py | 2 +- wqflask/wqflask/show_trait/SampleList.py | 2 +- wqflask/wqflask/show_trait/show_trait.py | 2 +- wqflask/wqflask/user_login.py | 2 +- 9 files changed, 12 insertions(+), 16 deletions(-) diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index 97767715..864b4e08 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -80,8 +80,7 @@ while line: if kj%100000 == 0: print(('checked ',kj,' lines')) -GeneList = list(map(string.lower, GeneList)) -GeneList.sort() +GeneList = sorted(map(string.lower, GeneList)) if isCont==0: sys.exit(0) @@ -148,9 +147,8 @@ for item in results: print(Names) -Names = list(map(string.lower, Names)) +Names = sorted(map(string.lower, Names)) -Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# ##---- compare genelist with names ----## diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index 833c3f5f..20a846a4 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -91,8 +91,7 @@ while line: if kj % 100000 == 0: print(('checked ', kj, ' lines')) -GeneList = list(map(string.lower, GeneList)) -GeneList.sort() +GeneList = sorted(map(string.lower, GeneList)) if isCont == 0: sys.exit(0) @@ -158,8 +157,7 @@ results = db.fetchall() Names = [] for item in results: Names.append(item[0]) - Names = list(map(string.lower, Names)) - Names.sort() # -- Fixed the lower case problem of ProbeSets affx-mur_b2_at doesn't exist --# + Names = sorted(map(string.lower, Names)) ##---- compare genelist with names ----## x = y = 0 diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index b0119b58..06e1c551 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -254,7 +254,7 @@ class Markers(object): logger.debug("length of self.markers:", len(self.markers)) logger.debug("length of p_values:", len(p_values)) - if type(p_values) is list: + if isinstance(p_values, list): # THIS IS only needed for the case when we are limiting the number of p-values calculated # if len(self.markers) > len(p_values): # self.markers = self.markers[:len(p_values)] @@ -270,7 +270,7 @@ class Markers(object): marker['lod_score'] = -math.log10(marker['p_value']) # Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 - elif type(p_values) is dict: + elif isinstance(p_values, dict): filtered_markers = [] for marker in self.markers: #logger.debug("marker[name]", marker['name']) diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index ece7022c..bc03eb55 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__ ) def check_resource_availability(dataset, trait_id=None): #At least for now assume temporary entered traits are accessible - if type(dataset) == str: + if isinstance(dataset, str): return webqtlConfig.DEFAULT_PRIVILEGES if dataset.type == "Temp": return webqtlConfig.DEFAULT_PRIVILEGES diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 872f22fe..874ada9d 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -332,7 +332,7 @@ class SVGelement: f.write('\n'+'\t'*(level+2)+line) f.write('\n'+'\t'*(level+1)+']]>\n') if self.text: - if type(self.text) == type(''): # If the text is only text + if isinstance(self.text, type('')): # If the text is only text f.write(_escape(str(self.text))) else: # If the text is a spannedtext class f.write(str(self.text)) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index f0be7a98..e58a7b87 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -175,7 +175,7 @@ class CTL(object): sys.stdout.flush() # Create the interactive graph for cytoscape visualization (Nodes and Edges) - if not type(significant) == ri.RNULLType: + if not isinstance(significant, ri.RNULLType): for x in range(len(significant[0])): logger.debug(significant[0][x], significant[1][x], significant[2][x]) # Debug to console tsS = significant[0][x].split(':') # Source diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index ad78ebcc..21ba7f63 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -43,7 +43,7 @@ class SampleList(object): for counter, sample_name in enumerate(sample_names, 1): sample_name = sample_name.replace("_2nd_", "") - if type(self.this_trait) is list: #ZS: self.this_trait will be a list if it is a Temp trait + if isinstance(self.this_trait, list): #ZS: self.this_trait will be a list if it is a Temp trait if counter <= len(self.this_trait) and str(self.this_trait[counter-1]).upper() != 'X': sample = webqtlCaseData.webqtlCaseData(name=sample_name, value=float(self.this_trait[counter-1])) else: diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index c156e61b..65899ab2 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -625,7 +625,7 @@ def get_categorical_variables(this_trait, sample_list): def get_genotype_scales(genofiles): geno_scales = {} - if type(genofiles) is list: + if isinstance(genofiles, list): for the_file in genofiles: file_location = the_file['location'] geno_scales[file_location] = get_scales_from_genofile(file_location) diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index cfee0079..04672b45 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -193,7 +193,7 @@ def login(): if user_details: submitted_password = params['password'] pwfields = user_details['password'] - if type(pwfields) is str: + if isinstance(pwfields, str): pwfields = json.loads(pwfields) encrypted_pass_fields = encode_password(pwfields, submitted_password) password_match = pbkdf2.safe_str_cmp(encrypted_pass_fields['password'], pwfields['password']) -- cgit v1.2.3 From db41cd49b6d8ccd2c3318209118ffe098bc9293e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 Aug 2020 03:57:05 +0300 Subject: Remove extra whitespace(or add it) from comma separated items See: --- etc/default_settings.py | 2 +- scripts/maintenance/QTL_Reaper_v6.py | 4 +- scripts/maintenance/readProbeSetMean_v7.py | 28 ++-- setup.py | 2 +- wqflask/base/trait.py | 6 +- wqflask/db/call.py | 10 +- wqflask/db/webqtlDatabaseFunction.py | 8 +- wqflask/maintenance/gen_select_dataset.py | 2 +- wqflask/maintenance/quantile_normalize.py | 2 +- wqflask/maintenance/set_resource_defaults.py | 2 +- wqflask/utility/Plot.py | 48 +++--- wqflask/utility/elasticsearch_tools.py | 2 +- wqflask/utility/logger.py | 26 +-- wqflask/utility/pillow_utils.py | 4 +- wqflask/utility/startup_config.py | 2 +- wqflask/utility/svg.py | 130 +++++++------- wqflask/utility/tools.py | 32 ++-- wqflask/utility/webqtlUtil.py | 16 +- wqflask/wqflask/api/router.py | 8 +- wqflask/wqflask/correlation/show_corr_results.py | 12 +- .../wqflask/correlation_matrix/show_corr_matrix.py | 8 +- wqflask/wqflask/do_search.py | 4 +- .../wqflask/external_tools/send_to_geneweaver.py | 8 +- .../marker_regression/display_mapping_results.py | 186 ++++++++++----------- wqflask/wqflask/marker_regression/plink_mapping.py | 6 +- wqflask/wqflask/marker_regression/run_mapping.py | 2 +- wqflask/wqflask/search_results.py | 2 +- wqflask/wqflask/show_trait/SampleList.py | 2 +- wqflask/wqflask/user_manager.py | 2 +- wqflask/wqflask/views.py | 8 +- 30 files changed, 287 insertions(+), 287 deletions(-) diff --git a/etc/default_settings.py b/etc/default_settings.py index f368237b..82c605da 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -24,7 +24,7 @@ import os import sys -GN_VERSION = open("../etc/VERSION","r").read() +GN_VERSION = open("../etc/VERSION", "r").read() GN_SERVER_URL = "http://localhost:8880/" # REST API server # ---- MySQL diff --git a/scripts/maintenance/QTL_Reaper_v6.py b/scripts/maintenance/QTL_Reaper_v6.py index 2fbeb53b..35f2d1a1 100755 --- a/scripts/maintenance/QTL_Reaper_v6.py +++ b/scripts/maintenance/QTL_Reaper_v6.py @@ -7,7 +7,7 @@ import reaper import MySQLdb import time -con = MySQLdb.Connect(db='db_webqtl',user='username',passwd='', host="localhost") +con = MySQLdb.Connect(db='db_webqtl', user='username', passwd='', host="localhost") cursor = con.cursor() genotypeDir = '/gnshare/gn/web/genotypes/' @@ -102,7 +102,7 @@ for ProbeSetFreezeId in ProbeSetFreezeIds: kj += 1 if kj%1000==0: - print((ProbeSetFreezeId, InbredSets[InbredSetId],kj)) + print((ProbeSetFreezeId, InbredSets[InbredSetId], kj)) print(ProbeSetFreezeIds) diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index 864b4e08..59a51cf9 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -39,7 +39,7 @@ fp = open("%s" % input_file_name, 'rb') try: passwd = getpass.getpass('Please enter mysql password here : ') - con = MySQLdb.Connect(db='db_webqtl',host='localhost', user='username',passwd=passwd) + con = MySQLdb.Connect(db='db_webqtl', host='localhost', user='username', passwd=passwd) db = con.cursor() print("You have successfully connected to mysql.\n") @@ -60,14 +60,14 @@ print('Checking if each line have same number of members') GeneList = [] isCont = 1 header = fp.readline() -header = string.split(string.strip(header),'\t') +header = string.split(string.strip(header), '\t') header = list(map(string.strip, header)) nfield = len(header) line = fp.readline() kj=0 while line: - line2 = string.split(string.strip(line),'\t') + line2 = string.split(string.strip(line), '\t') line2 = list(map(string.strip, line2)) if len(line2) != nfield: print(("Error : " + line)) @@ -78,7 +78,7 @@ while line: kj+=1 if kj%100000 == 0: - print(('checked ',kj,' lines')) + print(('checked ', kj, ' lines')) GeneList = sorted(map(string.lower, GeneList)) @@ -86,7 +86,7 @@ if isCont==0: sys.exit(0) -print(('used ',time.time()-time0,' seconds')) +print(('used ', time.time()-time0, ' seconds')) ######################################################################### # # Check if each strain exist in database @@ -98,7 +98,7 @@ print('Checking if each strain exist in database') isCont = 1 fp.seek(0) header = fp.readline() -header = string.split(string.strip(header),'\t') +header = string.split(string.strip(header), '\t') header = list(map(string.strip, header)) header = list(map(translateAlias, header)) header = header[dataStart:] @@ -108,14 +108,14 @@ for item in header: db.execute('select Id from Strain where Name = "%s"' % item) Ids.append(db.fetchall()[0][0]) except: - print((item,'does not exist, check the if the strain name is correct')) + print((item, 'does not exist, check the if the strain name is correct')) isCont=0 if isCont==0: sys.exit(0) -print(('used ',time.time()-time0,' seconds')) +print(('used ', time.time()-time0, ' seconds')) ######################################################################## # # Check if each ProbeSet exist in database @@ -126,7 +126,7 @@ print('Check if each ProbeSet exist in database') ##---- find PID is name or target ----## line = fp.readline() line = fp.readline() -line2 = string.split(string.strip(line),'\t') +line2 = string.split(string.strip(line), '\t') line2 = list(map(string.strip, line2)) PId = line2[0] @@ -185,7 +185,7 @@ if isCont==0: sys.exit(0) -print(('used ',time.time()-time0,' seconds')) +print(('used ', time.time()-time0, ' seconds')) ######################################################################### # # Insert data into database @@ -200,7 +200,7 @@ results = db.fetchall() NameIds = {} for item in results: NameIds[item[0]] = item[1] -print(('used ',time.time()-time0,' seconds')) +print(('used ', time.time()-time0, ' seconds')) print('inserting data') @@ -220,7 +220,7 @@ kj = 0 values1 = [] values2 = [] while line: - line2 = string.split(string.strip(line),'\t') + line2 = string.split(string.strip(line), '\t') line2 = list(map(string.strip, line2)) PId = line2[0] recordId = NameIds[PId] @@ -253,8 +253,8 @@ while line: values1=[] values2=[] - print(('Inserted ', kj,' lines')) - print(('used ',time.time()-time0,' seconds')) + print(('Inserted ', kj, ' lines')) + print(('used ', time.time()-time0, ' seconds')) line = fp.readline() diff --git a/setup.py b/setup.py index a9b71fab..8436dcd3 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup(name='genenetwork2', url = "https://github.com/genenetwork/genenetwork2/blob/master/README.md", description = 'Website and tools for genetics.', include_package_data=True, - packages=['wqflask','etc'], + packages=['wqflask', 'etc'], scripts=['bin/genenetwork2'], # package_data = { # 'etc': ['*.py'] diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 2b8f2e72..05b272c3 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -193,7 +193,7 @@ class GeneralTrait(object): ''' if self.chr and self.mb: - self.location = 'Chr %s @ %s Mb' % (self.chr,self.mb) + self.location = 'Chr %s @ %s Mb' % (self.chr, self.mb) elif self.chr: self.location = 'Chr %s @ Unknown position' % (self.chr) else: @@ -440,7 +440,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): #XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name # to avoid the problem of same marker name from different species. elif dataset.type == 'Geno': - display_fields_string = string.join(dataset.display_fields,',Geno.') + display_fields_string = string.join(dataset.display_fields, ',Geno.') display_fields_string = 'Geno.' + display_fields_string query = """ SELECT %s @@ -459,7 +459,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): query = """SELECT %s FROM %s WHERE Name = %s""" logger.sql(query) trait_info = g.db.execute(query, - (string.join(dataset.display_fields,','), + (string.join(dataset.display_fields, ','), dataset.type, trait.name)).fetchone() if trait_info: diff --git a/wqflask/db/call.py b/wqflask/db/call.py index 82cfebb4..3b8f782e 100644 --- a/wqflask/db/call.py +++ b/wqflask/db/call.py @@ -26,8 +26,8 @@ GN_SERVER result when set (which should return a Tuple) else: res2 = result, if LOG_SQL: - logger.debug("Replaced SQL call",query) - logger.debug(path,res2) + logger.debug("Replaced SQL call", query) + logger.debug(path, res2) return res2 else: return fetchone(query) @@ -37,7 +37,7 @@ def fetchone(query): original fetchone, but with logging) """ - with Bench("SQL",LOG_SQL): + with Bench("SQL", LOG_SQL): def helper(query): res = g.db.execute(query) return res.fetchone() @@ -48,7 +48,7 @@ def fetchall(query): original fetchall, but with logging) """ - with Bench("SQL",LOG_SQL): + with Bench("SQL", LOG_SQL): def helper(query): res = g.db.execute(query) return res.fetchall() @@ -58,7 +58,7 @@ def gn_server(path): """Return JSON record by calling GN_SERVER """ - with Bench("GN_SERVER",LOG_SQL): + with Bench("GN_SERVER", LOG_SQL): res = urllib.request.urlopen(GN_SERVER_URL+path) rest = res.read() res2 = json.loads(rest) diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index 8a9dc79d..2805febd 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -35,13 +35,13 @@ def retrieve_species(group): """Get the species of a group (e.g. returns string "mouse" on "BXD" """ - result = fetch1("select Species.Name from Species, InbredSet where InbredSet.Name = '%s' and InbredSet.SpeciesId = Species.Id" % (group),"/cross/"+group+".json",lambda r: (r["species"],))[0] - logger.debug("retrieve_species result:",result) + result = fetch1("select Species.Name from Species, InbredSet where InbredSet.Name = '%s' and InbredSet.SpeciesId = Species.Id" % (group), "/cross/"+group+".json", lambda r: (r["species"],))[0] + logger.debug("retrieve_species result:", result) return result def retrieve_species_id(group): - result = fetch1("select SpeciesId from InbredSet where Name = '%s'" % (group),"/cross/"+group+".json",lambda r: (r["species_id"],))[0] - logger.debug("retrieve_species_id result:",result) + result = fetch1("select SpeciesId from InbredSet where Name = '%s'" % (group), "/cross/"+group+".json", lambda r: (r["species_id"],))[0] + logger.debug("retrieve_species_id result:", result) return result diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index 78217587..d12b328f 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -41,7 +41,7 @@ from __future__ import print_function, division import sys # NEW: Note we prepend the current path - otherwise a guix instance of GN2 may be used instead -sys.path.insert(0,'./') +sys.path.insert(0, './') # NEW: import app to avoid a circular dependency on utility.tools from wqflask import app diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 82b695f4..43edfd13 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, print_function, division import sys -sys.path.insert(0,'./') +sys.path.insert(0, './') diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index 54fd8e7e..d53a255b 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -22,7 +22,7 @@ import sys import json # NEW: Note we prepend the current path - otherwise a guix instance of GN2 may be used instead -sys.path.insert(0,'./') +sys.path.insert(0, './') # NEW: import app to avoid a circular dependency on utility.tools from wqflask import app diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index c9053dde..4f6e694e 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -58,7 +58,7 @@ def cformat(d, rank=0): strD = "%2.6f" % d if rank == 0: - while strD[-1] in ('0','.'): + while strD[-1] in ('0', '.'): if strD[-1] == '0' and strD[-2] == '.' and len(strD) <= 4: break elif strD[-1] == '.': @@ -162,7 +162,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab j = int((item-xLow)/step) Count[j] += 1 - yLow, yTop, stepY=detScale(0,max(Count)) + yLow, yTop, stepY=detScale(0, max(Count)) #draw data xScale = plotWidth/(xTop-xLow) @@ -174,7 +174,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab xc = (dataXY[i]-xLow)*xScale+xLeftOffset yc =-(count-yLow)*yScale+yTopOffset+plotHeight im_drawer.rectangle( - xy=((xc+2,yc),(xc+barWidth-2,yTopOffset+plotHeight)), + xy=((xc+2, yc), (xc+barWidth-2, yTopOffset+plotHeight)), outline=barColor, fill=barColor) #draw drawing region @@ -183,81 +183,81 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab ) #draw scale - scaleFont=ImageFont.truetype(font=COUR_FILE,size=11) + scaleFont=ImageFont.truetype(font=COUR_FILE, size=11) x=xLow for i in range(int(stepX)+1): xc=xLeftOffset+(x-xLow)*xScale im_drawer.line( - xy=((xc,yTopOffset+plotHeight),(xc,yTopOffset+plotHeight+5)), + xy=((xc, yTopOffset+plotHeight), (xc, yTopOffset+plotHeight+5)), fill=axesColor) strX = cformat(d=x, rank=0) im_drawer.text( text=strX, - xy=(xc-im_drawer.textsize(strX,font=scaleFont)[0]/2, - yTopOffset+plotHeight+14),font=scaleFont) + xy=(xc-im_drawer.textsize(strX, font=scaleFont)[0]/2, + yTopOffset+plotHeight+14), font=scaleFont) x+= (xTop - xLow)/stepX y=yLow for i in range(int(stepY)+1): yc=yTopOffset+plotHeight-(y-yLow)*yScale - im_drawer.line(xy=((xLeftOffset,yc),(xLeftOffset-5,yc)), fill=axesColor) + im_drawer.line(xy=((xLeftOffset, yc), (xLeftOffset-5, yc)), fill=axesColor) strY = "%d" %y im_drawer.text( text=strY, - xy=(xLeftOffset-im_drawer.textsize(strY,font=scaleFont)[0]-6,yc+5), + xy=(xLeftOffset-im_drawer.textsize(strY, font=scaleFont)[0]-6, yc+5), font=scaleFont) y+= (yTop - yLow)/stepY #draw label - labelFont=ImageFont.truetype(font=TAHOMA_FILE,size=17) + labelFont=ImageFont.truetype(font=TAHOMA_FILE, size=17) if XLabel: im_drawer.text( text=XLabel, xy=(xLeftOffset+( - plotWidth-im_drawer.textsize(XLabel,font=labelFont)[0])/2.0, + plotWidth-im_drawer.textsize(XLabel, font=labelFont)[0])/2.0, yTopOffset+plotHeight+yBottomOffset-10), - font=labelFont,fill=labelColor) + font=labelFont, fill=labelColor) if YLabel: draw_rotated_text(canvas, text=YLabel, xy=(19, yTopOffset+plotHeight-( plotHeight-im_drawer.textsize( - YLabel,font=labelFont)[0])/2.0), + YLabel, font=labelFont)[0])/2.0), font=labelFont, fill=labelColor, angle=90) - labelFont=ImageFont.truetype(font=VERDANA_FILE,size=16) + labelFont=ImageFont.truetype(font=VERDANA_FILE, size=16) if title: im_drawer.text( text=title, xy=(xLeftOffset+(plotWidth-im_drawer.textsize( - title,font=labelFont)[0])/2.0, + title, font=labelFont)[0])/2.0, 20), - font=labelFont,fill=labelColor) + font=labelFont, fill=labelColor) # This function determines the scale of the plot -def detScaleOld(min,max): +def detScaleOld(min, max): if min>=max: return None elif min == -1.0 and max == 1.0: - return [-1.2,1.2,12] + return [-1.2, 1.2, 12] else: a=max-min b=floor(log10(a)) - c=pow(10.0,b) + c=pow(10.0, b) if a < c*5.0: c/=2.0 #print a,b,c low=c*floor(min/c) high=c*ceil(max/c) - return [low,high,round((high-low)/c)] + return [low, high, round((high-low)/c)] def detScale(min=0,max=0): if min>=max: return None elif min == -1.0 and max == 1.0: - return [-1.2,1.2,12] + return [-1.2, 1.2, 12] else: a=max-min if max != 0: @@ -269,7 +269,7 @@ def detScale(min=0,max=0): min -= 0.1*a a=max-min b=floor(log10(a)) - c=pow(10.0,b) + c=pow(10.0, b) low=c*floor(min/c) high=c*ceil(max/c) n = round((high-low)/c) @@ -287,7 +287,7 @@ def detScale(min=0,max=0): high=c*ceil(max/c) n = round((high-low)/c) - return [low,high,n] + return [low, high, n] def bluefunc(x): return 1.0 / (1.0 + exp(-10*(x-0.6))) @@ -296,7 +296,7 @@ def redfunc(x): return 1.0 / (1.0 + exp(10*(x-0.5))) def greenfunc(x): - return 1 - pow(redfunc(x+0.2),2) - bluefunc(x-0.3) + return 1 - pow(redfunc(x+0.2), 2) - bluefunc(x-0.3) def colorSpectrum(n=100): multiple = 10 diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index 15cdd0bc..a5580811 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -59,7 +59,7 @@ def get_elasticsearch_connection(for_user=True): try: assert(ELASTICSEARCH_HOST) assert(ELASTICSEARCH_PORT) - logger.info("ES HOST",ELASTICSEARCH_HOST) + logger.info("ES HOST", ELASTICSEARCH_HOST) es = Elasticsearch([{ "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py index 510b1041..e904eb94 100644 --- a/wqflask/utility/logger.py +++ b/wqflask/utility/logger.py @@ -42,10 +42,10 @@ class GNLogger: """ - def __init__(self,name): + def __init__(self, name): self.logger = logging.getLogger(name) - def setLevel(self,value): + def setLevel(self, value): """Set the undelying log level""" self.logger.setLevel(value) @@ -54,7 +54,7 @@ class GNLogger: level=num to filter on LOG_LEVEL_DEBUG. """ - self.collect(self.logger.debug,*args) + self.collect(self.logger.debug, *args) def debug20(self,*args): """Call logging.debug for multiple args. Use level=num to filter on @@ -63,15 +63,15 @@ LOG_LEVEL_DEBUG (NYI). """ if level <= LOG_LEVEL_DEBUG: if self.logger.getEffectiveLevel() < 20: - self.collect(self.logger.debug,*args) + self.collect(self.logger.debug, *args) def info(self,*args): """Call logging.info for multiple args""" - self.collect(self.logger.info,*args) + self.collect(self.logger.info, *args) def warning(self,*args): """Call logging.warning for multiple args""" - self.collect(self.logger.warning,*args) + self.collect(self.logger.warning, *args) # self.logger.warning(self.collect(*args)) def error(self,*args): @@ -79,13 +79,13 @@ LOG_LEVEL_DEBUG (NYI). now = datetime.datetime.utcnow() time_str = now.strftime('%H:%M:%S UTC %Y%m%d') l = [time_str]+list(args) - self.collect(self.logger.error,*l) + self.collect(self.logger.error, *l) def infof(self,*args): """Call logging.info for multiple args lazily""" # only evaluate function when logging if self.logger.getEffectiveLevel() < 30: - self.collectf(self.logger.debug,*args) + self.collectf(self.logger.debug, *args) def debugf(self,level=0,*args): """Call logging.debug for multiple args lazily and handle @@ -95,15 +95,15 @@ LOG_LEVEL_DEBUG (NYI). # only evaluate function when logging if level <= LOG_LEVEL_DEBUG: if self.logger.getEffectiveLevel() < 20: - self.collectf(self.logger.debug,*args) + self.collectf(self.logger.debug, *args) def sql(self, sqlcommand, fun = None): """Log SQL command, optionally invoking a timed fun""" if LOG_SQL: caller = stack()[1][3] - if caller in ['fetchone','fetch1','fetchall']: + if caller in ['fetchone', 'fetch1', 'fetchall']: caller = stack()[2][3] - self.info(caller,sqlcommand) + self.info(caller, sqlcommand) if fun: result = fun(sqlcommand) if LOG_SQL: @@ -119,7 +119,7 @@ LOG_LEVEL_DEBUG (NYI). if isinstance(a, str): out = out + a else: - out = out + pf(a,width=160) + out = out + pf(a, width=160) fun(out) def collectf(self,fun,*args): @@ -134,7 +134,7 @@ LOG_LEVEL_DEBUG (NYI). if isinstance(a, str): out = out + a else: - out = out + pf(a,width=160) + out = out + pf(a, width=160) fun(out) # Get the module logger. You can override log levels at the diff --git a/wqflask/utility/pillow_utils.py b/wqflask/utility/pillow_utils.py index dfbf3e19..1e2ed075 100644 --- a/wqflask/utility/pillow_utils.py +++ b/wqflask/utility/pillow_utils.py @@ -10,9 +10,9 @@ WHITE = ImageColor.getrgb("white") def draw_rotated_text(canvas, text, font, xy, fill=BLACK, angle=-90): # type: (Image, str, ImageFont, tuple, ImageColor, int) """Utility function draw rotated text""" - tmp_img = Image.new("RGBA", font.getsize(text), color=(0,0,0,0)) + tmp_img = Image.new("RGBA", font.getsize(text), color=(0, 0, 0, 0)) draw_text = ImageDraw.Draw(tmp_img) - draw_text.text(text=text, xy=(0,0), font=font, fill=fill) + draw_text.text(text=text, xy=(0, 0), font=font, fill=fill) tmp_img2 = tmp_img.rotate(angle, expand=1) tmp_img2.save("/tmp/{}.png".format(text), format="png") canvas.paste(im=tmp_img2, box=tuple([int(i) for i in xy])) diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 42ead709..f1aaebb6 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -36,4 +36,4 @@ def app_config(): # import utility.elasticsearch_tools as es # es.test_elasticsearch_connection() - print(("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL")))) + print(("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE, str(port), ENDC, get_setting("WEBSERVER_URL")))) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 874ada9d..19eda0ce 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -447,7 +447,7 @@ class rect(SVGelement): if width == None or height == None: raise ValueError('both height and width are required') - SVGelement.__init__(self,'rect',{'width':width,'height':height},**args) + SVGelement.__init__(self, 'rect', {'width':width,'height':height}, **args) if x!=None: self.attributes['x']=x if y!=None: @@ -468,7 +468,7 @@ class ellipse(SVGelement): if rx==None or ry== None: raise ValueError('both rx and ry are required') - SVGelement.__init__(self,'ellipse',{'rx':rx,'ry':ry},**args) + SVGelement.__init__(self, 'ellipse', {'rx':rx,'ry':ry}, **args) if cx!=None: self.attributes['cx']=cx if cy!=None: @@ -489,7 +489,7 @@ class circle(SVGelement): def __init__(self,cx=None,cy=None,r=None,fill=None,stroke=None,stroke_width=None,**args): if r==None: raise ValueError('r is required') - SVGelement.__init__(self,'circle',{'r':r},**args) + SVGelement.__init__(self, 'circle', {'r':r}, **args) if cx!=None: self.attributes['cx']=cx if cy!=None: @@ -508,7 +508,7 @@ class point(circle): very small rectangle if you use many points because a circle is difficult to render. """ def __init__(self,x,y,fill='black',**args): - circle.__init__(self,x,y,1,fill,**args) + circle.__init__(self, x, y, 1, fill, **args) class line(SVGelement): """l=line(x1,y1,x2,y2,stroke,stroke_width,**args) @@ -516,7 +516,7 @@ class line(SVGelement): A line is defined by a begin x,y pair and an end x,y pair """ def __init__(self,x1=None,y1=None,x2=None,y2=None,stroke=None,stroke_width=None,**args): - SVGelement.__init__(self,'line',**args) + SVGelement.__init__(self, 'line', **args) if x1!=None: self.attributes['x1']=x1 if y1!=None: @@ -536,7 +536,7 @@ class polyline(SVGelement): a polyline is defined by a list of xy pairs """ def __init__(self,points,fill=None,stroke=None,stroke_width=None,**args): - SVGelement.__init__(self,'polyline',{'points':_xypointlist(points)},**args) + SVGelement.__init__(self, 'polyline', {'points':_xypointlist(points)}, **args) if fill!=None: self.attributes['fill']=fill if stroke_width!=None: @@ -550,7 +550,7 @@ class polygon(SVGelement): a polygon is defined by a list of xy pairs """ def __init__(self,points,fill=None,stroke=None,stroke_width=None,**args): - SVGelement.__init__(self,'polygon',{'points':_xypointlist(points)},**args) + SVGelement.__init__(self, 'polygon', {'points':_xypointlist(points)}, **args) if fill!=None: self.attributes['fill']=fill if stroke_width!=None: @@ -564,7 +564,7 @@ class path(SVGelement): a path is defined by a path object and optional width, stroke and fillcolor """ def __init__(self,pathdata,fill=None,stroke=None,stroke_width=None,id=None,**args): - SVGelement.__init__(self,'path',{'d':str(pathdata)},**args) + SVGelement.__init__(self, 'path', {'d':str(pathdata)}, **args) if stroke!=None: self.attributes['stroke']=stroke if fill!=None: @@ -581,7 +581,7 @@ class text(SVGelement): a text element can bge used for displaying text on the screen """ def __init__(self,x=None,y=None,text=None,font_size=None,font_family=None,text_anchor=None,**args): - SVGelement.__init__(self,'text',**args) + SVGelement.__init__(self, 'text', **args) if x!=None: self.attributes['x']=x if y!=None: @@ -602,7 +602,7 @@ class textpath(SVGelement): a textpath places a text on a path which is referenced by a link. """ def __init__(self,link,text=None,**args): - SVGelement.__init__(self,'textPath',{'xlink:href':link},**args) + SVGelement.__init__(self, 'textPath', {'xlink:href':link}, **args) if text!=None: self.text=text @@ -614,7 +614,7 @@ class pattern(SVGelement): in x and y to cover the areas to be painted. """ def __init__(self,x=None,y=None,width=None,height=None,patternUnits=None,**args): - SVGelement.__init__(self,'pattern',**args) + SVGelement.__init__(self, 'pattern', **args) if x!=None: self.attributes['x']=x if y!=None: @@ -633,7 +633,7 @@ class title(SVGelement): add at least one to the root svg element """ def __init__(self,text=None,**args): - SVGelement.__init__(self,'title',**args) + SVGelement.__init__(self, 'title', **args) if text!=None: self.text=text @@ -644,7 +644,7 @@ class description(SVGelement): Add this element before adding other elements. """ def __init__(self,text=None,**args): - SVGelement.__init__(self,'desc',**args) + SVGelement.__init__(self, 'desc', **args) if text!=None: self.text=text @@ -655,7 +655,7 @@ class lineargradient(SVGelement): stop elements van be added to define the gradient colors. """ def __init__(self,x1=None,y1=None,x2=None,y2=None,id=None,**args): - SVGelement.__init__(self,'linearGradient',**args) + SVGelement.__init__(self, 'linearGradient', **args) if x1!=None: self.attributes['x1']=x1 if y1!=None: @@ -674,7 +674,7 @@ class radialgradient(SVGelement): stop elements van be added to define the gradient colors. """ def __init__(self,cx=None,cy=None,r=None,fx=None,fy=None,id=None,**args): - SVGelement.__init__(self,'radialGradient',**args) + SVGelement.__init__(self, 'radialGradient', **args) if cx!=None: self.attributes['cx']=cx if cy!=None: @@ -694,7 +694,7 @@ class stop(SVGelement): Puts a stop color at the specified radius """ def __init__(self,offset,stop_color=None,**args): - SVGelement.__init__(self,'stop',{'offset':offset},**args) + SVGelement.__init__(self, 'stop', {'offset':offset}, **args) if stop_color!=None: self.attributes['stop-color']=stop_color @@ -704,7 +704,7 @@ class style(SVGelement): Add a CDATA element to this element for defing in line stylesheets etc.. """ def __init__(self,type,cdata=None,**args): - SVGelement.__init__(self,'style',{'type':type},cdata=cdata, **args) + SVGelement.__init__(self, 'style', {'type':type}, cdata=cdata, **args) class image(SVGelement): @@ -715,7 +715,7 @@ class image(SVGelement): def __init__(self,url,x=None,y=None,width=None,height=None,**args): if width==None or height==None: raise ValueError('both height and width are required') - SVGelement.__init__(self,'image',{'xlink:href':url,'width':width,'height':height},**args) + SVGelement.__init__(self, 'image', {'xlink:href':url,'width':width,'height':height}, **args) if x!=None: self.attributes['x']=x if y!=None: @@ -727,7 +727,7 @@ class cursor(SVGelement): defines a custom cursor for a element or a drawing """ def __init__(self,url,**args): - SVGelement.__init__(self,'cursor',{'xlink:href':url},**args) + SVGelement.__init__(self, 'cursor', {'xlink:href':url}, **args) class marker(SVGelement): @@ -737,7 +737,7 @@ class marker(SVGelement): add an element to it which should be used as a marker. """ def __init__(self,id=None,viewBox=None,refx=None,refy=None,markerWidth=None,markerHeight=None,**args): - SVGelement.__init__(self,'marker',**args) + SVGelement.__init__(self, 'marker', **args) if id!=None: self.attributes['id']=id if viewBox!=None: @@ -758,7 +758,7 @@ class group(SVGelement): g.addElement(SVGelement) """ def __init__(self,id=None,**args): - SVGelement.__init__(self,'g',**args) + SVGelement.__init__(self, 'g', **args) if id!=None: self.attributes['id']=id @@ -772,7 +772,7 @@ class symbol(SVGelement): """ def __init__(self,id=None,viewBox=None,**args): - SVGelement.__init__(self,'symbol',**args) + SVGelement.__init__(self, 'symbol', **args) if id!=None: self.attributes['id']=id if viewBox!=None: @@ -784,7 +784,7 @@ class defs(SVGelement): container for defining elements """ def __init__(self,**args): - SVGelement.__init__(self,'defs',**args) + SVGelement.__init__(self, 'defs', **args) class switch(SVGelement): """sw=switch(**args) @@ -794,7 +794,7 @@ class switch(SVGelement): Refer to the SVG specification for details. """ def __init__(self,**args): - SVGelement.__init__(self,'switch',**args) + SVGelement.__init__(self, 'switch', **args) class use(SVGelement): @@ -803,7 +803,7 @@ class use(SVGelement): references a symbol by linking to its id and its position, height and width """ def __init__(self,link,x=None,y=None,width=None,height=None,**args): - SVGelement.__init__(self,'use',{'xlink:href':link},**args) + SVGelement.__init__(self, 'use', {'xlink:href':link}, **args) if x!=None: self.attributes['x']=x if y!=None: @@ -822,14 +822,14 @@ class link(SVGelement): a.addElement(SVGelement) """ def __init__(self,link='',**args): - SVGelement.__init__(self,'a',{'xlink:href':link},**args) + SVGelement.__init__(self, 'a', {'xlink:href':link}, **args) class view(SVGelement): """v=view(id,**args) a view can be used to create a view with different attributes""" def __init__(self,id=None,**args): - SVGelement.__init__(self,'view',**args) + SVGelement.__init__(self, 'view', **args) if id!=None: self.attributes['id']=id @@ -840,7 +840,7 @@ class script(SVGelement): """ def __init__(self,type,cdata=None,**args): - SVGelement.__init__(self,'script',{'type':type},cdata=cdata,**args) + SVGelement.__init__(self, 'script', {'type':type}, cdata=cdata, **args) class animate(SVGelement): """an=animate(attribute,from,to,during,**args) @@ -848,7 +848,7 @@ class animate(SVGelement): animates an attribute. """ def __init__(self,attribute,fr=None,to=None,dur=None,**args): - SVGelement.__init__(self,'animate',{'attributeName':attribute},**args) + SVGelement.__init__(self, 'animate', {'attributeName':attribute}, **args) if fr!=None: self.attributes['from']=fr if to!=None: @@ -862,7 +862,7 @@ class animateMotion(SVGelement): animates a SVGelement over the given path in dur seconds """ def __init__(self,pathdata,dur,**args): - SVGelement.__init__(self,'animateMotion',**args) + SVGelement.__init__(self, 'animateMotion', **args) if pathdata!=None: self.attributes['path']=str(pathdata) if dur!=None: @@ -874,7 +874,7 @@ class animateTransform(SVGelement): transform an element from and to a value. """ def __init__(self,type=None,fr=None,to=None,dur=None,**args): - SVGelement.__init__(self,'animateTransform',{'attributeName':'transform'},**args) + SVGelement.__init__(self, 'animateTransform', {'attributeName':'transform'}, **args) # As far as I know the attributeName is always transform if type!=None: self.attributes['type']=type @@ -890,7 +890,7 @@ class animateColor(SVGelement): Animates the color of a element """ def __init__(self,attribute,type=None,fr=None,to=None,dur=None,**args): - SVGelement.__init__(self,'animateColor',{'attributeName':attribute},**args) + SVGelement.__init__(self, 'animateColor', {'attributeName':attribute}, **args) if type!=None: self.attributes['type']=type if fr!=None: @@ -905,7 +905,7 @@ class set(SVGelement): sets an attribute to a value for a """ def __init__(self,attribute,to=None,dur=None,**args): - SVGelement.__init__(self,'set',{'attributeName':attribute},**args) + SVGelement.__init__(self, 'set', {'attributeName':attribute}, **args) if to!=None: self.attributes['to']=to if dur!=None: @@ -929,7 +929,7 @@ class svg(SVGelement): d.toXml() """ def __init__(self,viewBox=None, width=None, height=None,**args): - SVGelement.__init__(self,'svg',**args) + SVGelement.__init__(self, 'svg', **args) if viewBox!=None: self.attributes['viewBox']=_viewboxlist(viewBox) if width!=None: @@ -952,7 +952,7 @@ class drawing: def __init__(self, entity={}): self.svg=None self.entity = entity - def setSVG(self,svg): + def setSVG(self, svg): self.svg=svg # Voeg een element toe aan de grafiek toe. if use_dom_implementation==0: @@ -967,12 +967,12 @@ class drawing: xml.write("\n" % (item, self.entity[item])) xml.write("]") xml.write(">\n") - self.svg.toXml(0,xml) + self.svg.toXml(0, xml) if not filename: if compress: import gzip f=cStringIO.StringIO() - zf=gzip.GzipFile(fileobj=f,mode='wb') + zf=gzip.GzipFile(fileobj=f, mode='wb') zf.write(xml.getvalue()) zf.close() f.seek(0) @@ -982,11 +982,11 @@ class drawing: else: if filename[-4:]=='svgz': import gzip - f=gzip.GzipFile(filename=filename,mode="wb", compresslevel=9) + f=gzip.GzipFile(filename=filename, mode="wb", compresslevel=9) f.write(xml.getvalue()) f.close() else: - f=file(filename,'w') + f=file(filename, 'w') f.write(xml.getvalue()) f.close() @@ -997,40 +997,40 @@ class drawing: writes a svg drawing to the screen or to a file compresses if filename ends with svgz or if compress is true """ - doctype = implementation.createDocumentType('svg',"-//W3C//DTD SVG 1.0//EN""",'http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd ') + doctype = implementation.createDocumentType('svg', "-//W3C//DTD SVG 1.0//EN""", 'http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd ') global root # root is defined global so it can be used by the appender. Its also possible to use it as an arugument but # that is a bit messy. - root=implementation.createDocument(None,None,doctype) + root=implementation.createDocument(None, None, doctype) # Create the xml document. global appender - def appender(element,elementroot): + def appender(element, elementroot): """This recursive function appends elements to an element and sets the attributes and type. It stops when alle elements have been appended""" if element.namespace: - e=root.createElementNS(element.namespace,element.type) + e=root.createElementNS(element.namespace, element.type) else: e=root.createElement(element.type) if element.text: textnode=root.createTextNode(element.text) e.appendChild(textnode) for attribute in list(element.attributes.keys()): #in element.attributes is supported from python 2.2 - e.setAttribute(attribute,str(element.attributes[attribute])) + e.setAttribute(attribute, str(element.attributes[attribute])) if element.elements: for el in element.elements: - e=appender(el,e) + e=appender(el, e) elementroot.appendChild(e) return elementroot - root=appender(self.svg,root) + root=appender(self.svg, root) if not filename: import cStringIO xml=cStringIO.StringIO() - PrettyPrint(root,xml) + PrettyPrint(root, xml) if compress: import gzip f=cStringIO.StringIO() - zf=gzip.GzipFile(fileobj=f,mode='wb') + zf=gzip.GzipFile(fileobj=f, mode='wb') zf.write(xml.getvalue()) zf.close() f.seek(0) @@ -1043,13 +1043,13 @@ class drawing: import gzip import cStringIO xml=cStringIO.StringIO() - PrettyPrint(root,xml) - f=gzip.GzipFile(filename=filename,mode='wb',compresslevel=9) + PrettyPrint(root, xml) + f=gzip.GzipFile(filename=filename, mode='wb', compresslevel=9) f.write(xml.getvalue()) f.close() else: - f=open(filename,'w') - PrettyPrint(root,f) + f=open(filename, 'w') + PrettyPrint(root, f) f.close() except: print(("Cannot write SVG file: " + filename)) @@ -1070,32 +1070,32 @@ if __name__=='__main__': d=drawing() - s=svg((0,0,100,100)) - r=rect(-100,-100,300,300,'cyan') + s=svg((0, 0, 100, 100)) + r=rect(-100, -100, 300, 300, 'cyan') s.addElement(r) t=title('SVGdraw Demo') s.addElement(t) g=group('animations') - e=ellipse(0,0,5,2) + e=ellipse(0, 0, 5, 2) g.addElement(e) - c=circle(0,0,1,'red') + c=circle(0, 0, 1, 'red') g.addElement(c) - pd=pathdata(0,-10) + pd=pathdata(0, -10) for i in range(6): - pd.relsmbezier(10,5,0,10) - pd.relsmbezier(-10,5,0,10) - an=animateMotion(pd,10) + pd.relsmbezier(10, 5, 0, 10) + pd.relsmbezier(-10, 5, 0, 10) + an=animateMotion(pd, 10) an.attributes['rotate']='auto-reverse' an.attributes['repeatCount']="indefinite" g.addElement(an) s.addElement(g) - for i in range(20,120,20): - u=use('#animations',i,0) + for i in range(20, 120, 20): + u=use('#animations', i, 0) s.addElement(u) - for i in range(0,120,20): - for j in range(5,105,10): - c=circle(i,j,1,'red','black',.5) + for i in range(0, 120, 20): + for j in range(5, 105, 10): + c=circle(i, j, 1, 'red', 'black', .5) s.addElement(c) d.setSVG(s) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 51a87fe1..68ef0f04 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -15,7 +15,7 @@ OVERRIDES = {} def app_set(command_id, value): """Set application wide value""" - app.config.setdefault(command_id,value) + app.config.setdefault(command_id, value) return value def get_setting(command_id,guess=None): @@ -45,7 +45,7 @@ def get_setting(command_id,guess=None): def value(command): if command: # sys.stderr.write("Found "+command+"\n") - app_set(command_id,command) + app_set(command_id, command) return command else: return None @@ -68,7 +68,7 @@ def get_setting(command_id,guess=None): def get_setting_bool(id): v = get_setting(id) - if v not in [0,False,'False','FALSE',None]: + if v not in [0, False, 'False', 'FALSE', None]: return True return False @@ -108,16 +108,16 @@ def js_path(module=None): raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)" def reaper_command(guess=None): - return get_setting("REAPER_COMMAND",guess) + return get_setting("REAPER_COMMAND", guess) def gemma_command(guess=None): - return assert_bin(get_setting("GEMMA_COMMAND",guess)) + return assert_bin(get_setting("GEMMA_COMMAND", guess)) def gemma_wrapper_command(guess=None): - return assert_bin(get_setting("GEMMA_WRAPPER_COMMAND",guess)) + return assert_bin(get_setting("GEMMA_WRAPPER_COMMAND", guess)) def plink_command(guess=None): - return assert_bin(get_setting("PLINK_COMMAND",guess)) + return assert_bin(get_setting("PLINK_COMMAND", guess)) def flat_file_exists(subdir): base = get_setting("GENENETWORK_FILES") @@ -180,7 +180,7 @@ def locate(name, subdir=None): raise Exception("Can not locate "+name+" in "+base) def locate_phewas(name, subdir=None): - return locate(name,'/phewas/'+subdir) + return locate(name, '/phewas/'+subdir) def locate_ignore_error(name, subdir=None): """ @@ -204,7 +204,7 @@ def tempdir(): """ Get UNIX TMPDIR by default """ - return valid_path(get_setting("TMPDIR","/tmp")) + return valid_path(get_setting("TMPDIR", "/tmp")) BLUE = '\033[94m' GREEN = '\033[92m' @@ -225,9 +225,9 @@ def show_settings(): keylist.sort() for k in keylist: try: - print(("%s: %s%s%s%s" % (k,BLUE,BOLD,get_setting(k),ENDC))) + print(("%s: %s%s%s%s" % (k, BLUE, BOLD, get_setting(k), ENDC))) except: - print(("%s: %s%s%s%s" % (k,GREEN,BOLD,app.config[k],ENDC))) + print(("%s: %s%s%s%s" % (k, GREEN, BOLD, app.config[k], ENDC))) # Cached values @@ -279,10 +279,10 @@ SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') SMTP_PASSWORD = get_setting('SMTP_PASSWORD') -REAPER_COMMAND = app_set("REAPER_COMMAND",reaper_command()) -GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) +REAPER_COMMAND = app_set("REAPER_COMMAND", reaper_command()) +GEMMA_COMMAND = app_set("GEMMA_COMMAND", gemma_command()) assert(GEMMA_COMMAND is not None) -PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) +PLINK_COMMAND = app_set("PLINK_COMMAND", plink_command()) GEMMA_WRAPPER_COMMAND = gemma_wrapper_command() TEMPDIR = tempdir() # defaults to UNIX TMPDIR assert_dir(TEMPDIR) @@ -295,11 +295,11 @@ assert_dir(JS_GUIX_PATH+'/cytoscape-panzoom') CSS_PATH = JS_GUIX_PATH # The CSS is bundled together with the JS # assert_dir(JS_PATH) -JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("javascript-twitter-post-fetcher")) +JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH", js_path("javascript-twitter-post-fetcher")) assert_dir(JS_TWITTER_POST_FETCHER_PATH) assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") -JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH",js_path("cytoscape")) +JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH", js_path("cytoscape")) assert_dir(JS_CYTOSCAPE_PATH) assert_file(JS_CYTOSCAPE_PATH+'/cytoscape.min.js') diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index 79991149..77cc3416 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -41,22 +41,22 @@ ParInfo ={ 'C57BL-6JxC57BL-6NJF2':['', '', 'C57BL/6J', 'C57BL/6NJ'], 'BXD300':['B6D2F1', 'D2B6F1', 'C57BL/6J', 'DBA/2J'], 'B6BTBRF2':['B6BTBRF1', 'BTBRB6F1', 'C57BL/6J', 'BTBRT<+>tf/J'], -'BHHBF2':['B6HF2','HB6F2','C57BL/6J','C3H/HeJ'], -'BHF2':['B6HF2','HB6F2','C57BL/6J','C3H/HeJ'], +'BHHBF2':['B6HF2', 'HB6F2', 'C57BL/6J', 'C3H/HeJ'], +'BHF2':['B6HF2', 'HB6F2', 'C57BL/6J', 'C3H/HeJ'], 'B6D2F2':['B6D2F1', 'D2B6F1', 'C57BL/6J', 'DBA/2J'], 'BDF2-1999':['B6D2F2', 'D2B6F2', 'C57BL/6J', 'DBA/2J'], 'BDF2-2005':['B6D2F1', 'D2B6F1', 'C57BL/6J', 'DBA/2J'], -'CTB6F2':['CTB6F2','B6CTF2','C57BL/6J','Castaneous'], +'CTB6F2':['CTB6F2', 'B6CTF2', 'C57BL/6J', 'Castaneous'], 'CXB':['CBF1', 'BCF1', 'C57BL/6ByJ', 'BALB/cByJ'], 'AXBXA':['ABF1', 'BAF1', 'C57BL/6J', 'A/J'], 'AXB':['ABF1', 'BAF1', 'C57BL/6J', 'A/J'], 'BXA':['BAF1', 'ABF1', 'C57BL/6J', 'A/J'], 'LXS':['LSF1', 'SLF1', 'ISS', 'ILS'], 'HXBBXH':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'], -'BayXSha':['BayXShaF1', 'ShaXBayF1', 'Bay-0','Shahdara'], -'ColXBur':['ColXBurF1', 'BurXColF1', 'Col-0','Bur-0'], -'ColXCvi':['ColXCviF1', 'CviXColF1', 'Col-0','Cvi'], -'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'], +'BayXSha':['BayXShaF1', 'ShaXBayF1', 'Bay-0', 'Shahdara'], +'ColXBur':['ColXBurF1', 'BurXColF1', 'Col-0', 'Bur-0'], +'ColXCvi':['ColXCviF1', 'CviXColF1', 'Col-0', 'Cvi'], +'SXM':['SMF1', 'MSF1', 'Steptoe', 'Morex'], 'HRDP':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'] } @@ -91,7 +91,7 @@ def readLineCSV(line): ### dcrowell July 2008 returnList[0]=returnList[0][1:] return returnList -def cmpEigenValue(A,B): +def cmpEigenValue(A, B): try: if A[0] > B[0]: return -1 diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index 3fa1d5ba..b81da0dc 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -558,10 +558,10 @@ def trait_sample_data(dataset_name, trait_name, file_format = "json"): sample_list = [] for sample in sample_data: sample_dict = { - "sample_name" : sample[0], - "sample_name_2" : sample[1], - "value" : sample[2], - "data_id" : sample[3], + "sample_name": sample[0], + "sample_name_2": sample[1], + "value": sample[2], + "data_id": sample[3], } if sample[4]: sample_dict["se"] = sample[4] diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 15a21ee6..4116e2df 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -108,17 +108,17 @@ class CorrelationResults(object): self.sample_data = {} self.corr_type = start_vars['corr_type'] self.corr_method = start_vars['corr_sample_method'] - self.min_expr = get_float(start_vars,'min_expr') - self.p_range_lower = get_float(start_vars,'p_range_lower',-1.0) - self.p_range_upper = get_float(start_vars,'p_range_upper',1.0) + self.min_expr = get_float(start_vars, 'min_expr') + self.p_range_lower = get_float(start_vars, 'p_range_lower', -1.0) + self.p_range_upper = get_float(start_vars, 'p_range_upper', 1.0) if ('loc_chr' in start_vars and 'min_loc_mb' in start_vars and 'max_loc_mb' in start_vars): - self.location_chr = get_string(start_vars,'loc_chr') - self.min_location_mb = get_int(start_vars,'min_loc_mb') - self.max_location_mb = get_int(start_vars,'max_loc_mb') + self.location_chr = get_string(start_vars, 'loc_chr') + self.min_location_mb = get_int(start_vars, 'min_loc_mb') + self.max_location_mb = get_int(start_vars, 'max_loc_mb') else: self.location_chr = self.min_location_mb = self.max_location_mb = None diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 832746bb..b582cd23 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -275,8 +275,8 @@ def zScore(trait_data_array): i = 0 for data in trait_data_array: N = len(data) - S = reduce(lambda x,y: x+y, data, 0.) - SS = reduce(lambda x,y: x+y*y, data, 0.) + S = reduce(lambda x, y: x+y, data, 0.) + SS = reduce(lambda x, y: x+y*y, data, 0.) mean = S/N var = SS - S*S/N stdev = math.sqrt(var/(N-1)) @@ -294,7 +294,7 @@ def sortEigenVectors(vector): combines = [] i = 0 for item in eigenValues: - combines.append([eigenValues[i],eigenVectors[i]]) + combines.append([eigenValues[i], eigenVectors[i]]) i += 1 combines.sort(webqtlUtil.cmpEigenValue) A = [] @@ -302,7 +302,7 @@ def sortEigenVectors(vector): for item in combines: A.append(item[0]) B.append(item[1]) - sum = reduce(lambda x,y: x+y, A, 0.0) + sum = reduce(lambda x, y: x+y, A, 0.0) A = [x*100.0/sum for x in A] return [A, B] except: diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 1e15d28f..cc9c1860 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -46,8 +46,8 @@ class DoSearch(object): def handle_wildcard(self, str): keyword = str.strip() - keyword = keyword.replace("*",".*") - keyword = keyword.replace("?",".") + keyword = keyword.replace("*", ".*") + keyword = keyword.replace("?", ".") return keyword diff --git a/wqflask/wqflask/external_tools/send_to_geneweaver.py b/wqflask/wqflask/external_tools/send_to_geneweaver.py index 7a5dba73..9844bab4 100644 --- a/wqflask/wqflask/external_tools/send_to_geneweaver.py +++ b/wqflask/wqflask/external_tools/send_to_geneweaver.py @@ -54,10 +54,10 @@ class SendToGeneWeaver(object): trait_name_list = get_trait_name_list(self.trait_list) self.hidden_vars = { - 'client' : "genenetwork", - 'species' : species_name, - 'idtype' : self.chip_name, - 'list' : string.join(trait_name_list, ","), + 'client': "genenetwork", + 'species': species_name, + 'idtype': self.chip_name, + 'list': string.join(trait_name_list, ","), } def get_trait_name_list(trait_list): diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index c8b9c405..f282b010 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -27,8 +27,8 @@ import datetime import string from math import * -from PIL import (Image,ImageDraw,ImageFont,ImageColor) -import sys,os +from PIL import (Image, ImageDraw, ImageFont, ImageColor) +import sys, os import cPickle import httplib import json @@ -426,7 +426,7 @@ class DisplayMappingResults(object): else: continue samplelist = list(self.genotype.prgy) - for j,_geno in enumerate (self.genotype[0][1].genotype): + for j, _geno in enumerate (self.genotype[0][1].genotype): for item in smd: if item.name == samplelist[j]: self.NR_INDIVIDUALS = self.NR_INDIVIDUALS + 1 @@ -518,7 +518,7 @@ class DisplayMappingResults(object): #Scales plot differently for high resolution if self.draw2X: - intCanvasX2 = Image.new("RGBA", size=(self.graphWidth*2,self.graphHeight*2)) + intCanvasX2 = Image.new("RGBA", size=(self.graphWidth*2, self.graphHeight*2)) gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2) intCanvasX2.save( "{}.png".format( @@ -745,17 +745,17 @@ class DisplayMappingResults(object): bootScale = bootScale[:-1] + [highestPercent] bootOffset = 50*fontZoom - bootScaleFont=ImageFont.truetype(font=VERDANA_FILE,size=13*fontZoom) + bootScaleFont=ImageFont.truetype(font=VERDANA_FILE, size=13*fontZoom) im_drawer.rectangle( xy=((canvas.size[0]-bootOffset, yZero-bootHeightThresh), - (canvas.size[0]-bootOffset-15*zoom,yZero)), + (canvas.size[0]-bootOffset-15*zoom, yZero)), fill = YELLOW, outline=BLACK) im_drawer.line( xy=((canvas.size[0]-bootOffset+4, yZero), (canvas.size[0]-bootOffset, yZero)), fill=BLACK) TEXT_Y_DISPLACEMENT = -8 - im_drawer.text(xy=(canvas.size[0]-bootOffset+10,yZero+TEXT_Y_DISPLACEMENT), text='0%', + im_drawer.text(xy=(canvas.size[0]-bootOffset+10, yZero+TEXT_Y_DISPLACEMENT), text='0%', font=bootScaleFont, fill=BLACK) for item in bootScale: @@ -763,10 +763,10 @@ class DisplayMappingResults(object): continue bootY = yZero-bootHeightThresh*item/highestPercent im_drawer.line( - xy=((canvas.size[0]-bootOffset+4,bootY), - (canvas.size[0]-bootOffset,bootY)), + xy=((canvas.size[0]-bootOffset+4, bootY), + (canvas.size[0]-bootOffset, bootY)), fill=BLACK) - im_drawer.text(xy=(canvas.size[0]-bootOffset+10,bootY+TEXT_Y_DISPLACEMENT), + im_drawer.text(xy=(canvas.size[0]-bootOffset+10, bootY+TEXT_Y_DISPLACEMENT), text='%2.1f'%item, font=bootScaleFont, fill=BLACK) if self.legendChecked: @@ -775,7 +775,7 @@ class DisplayMappingResults(object): smallLabelFont = ImageFont.truetype(font=TREBUC_FILE, size=12*fontZoom) leftOffset = xLeftOffset+(nCol-1)*200 im_drawer.rectangle( - xy=((leftOffset,startPosY-6), (leftOffset+12,startPosY+6)), + xy=((leftOffset, startPosY-6), (leftOffset+12, startPosY+6)), fill=YELLOW, outline=BLACK) im_drawer.text(xy=(leftOffset+ 20, startPosY+TEXT_Y_DISPLACEMENT), text='Frequency of the Peak LRS', @@ -872,7 +872,7 @@ class DisplayMappingResults(object): TEXT_Y_DISPLACEMENT = -8 im_drawer.text( text="Sequence Site", - xy=(leftOffset+15,startPosY+TEXT_Y_DISPLACEMENT), font=smallLabelFont, + xy=(leftOffset+15, startPosY+TEXT_Y_DISPLACEMENT), font=smallLabelFont, fill=self.TOP_RIGHT_INFO_COLOR) def drawSNPTrackNew(self, canvas, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): @@ -922,7 +922,7 @@ class DisplayMappingResults(object): def drawMultiTraitName(self, fd, canvas, gifmap, showLocusForm, offset= (40, 120, 80, 10), zoom = 1): nameWidths = [] yPaddingTop = 10 - colorFont=ImageFont.truetype(font=TREBUC_FILE,size=12) + colorFont=ImageFont.truetype(font=TREBUC_FILE, size=12) if len(self.qtlresults) >20 and self.selectedChr > -1: rightShift = 20 rightShiftStep = 60 @@ -941,20 +941,20 @@ class DisplayMappingResults(object): rightShift += rightShiftStep name = thisTrait.displayName() - nameWidth, nameHeight = im_drawer.textsize(name,font=colorFont) + nameWidth, nameHeight = im_drawer.textsize(name, font=colorFont) nameWidths.append(nameWidth) im_drawer.rectangle( - xy=((rightShift,yPaddingTop+kstep*15), - (rectWidth+rightShift,yPaddingTop+10+kstep*15)), + xy=((rightShift, yPaddingTop+kstep*15), + (rectWidth+rightShift, yPaddingTop+10+kstep*15)), fill=thisLRSColor, outline=BLACK) im_drawer.text( - text=name,xy=(rectWidth+2+rightShift,yPaddingTop+10+kstep*15), - font=colorFont,fill=BLACK) + text=name, xy=(rectWidth+2+rightShift, yPaddingTop+10+kstep*15), + font=colorFont, fill=BLACK) if thisTrait.db: - COORDS = "%d,%d,%d,%d" %(rectWidth+2+rightShift,yPaddingTop+kstep*15,rectWidth+2+rightShift+nameWidth,yPaddingTop+10+kstep*15,) + COORDS = "%d,%d,%d,%d" %(rectWidth+2+rightShift, yPaddingTop+kstep*15, rectWidth+2+rightShift+nameWidth, yPaddingTop+10+kstep*15,) HREF= "javascript:showDatabase3('%s','%s','%s','');" % (showLocusForm, thisTrait.db.name, thisTrait.name) - Areas = HT.Area(shape='rect',coords=COORDS,href=HREF) + Areas = HT.Area(shape='rect', coords=COORDS, href=HREF) gifmap.areas.append(Areas) def drawLegendPanel(self, canvas, offset= (40, 120, 80, 10), zoom = 1): @@ -968,80 +968,80 @@ class DisplayMappingResults(object): if zoom == 2: fontZoom = 1.5 - labelFont=ImageFont.truetype(font=TREBUC_FILE,size=12*fontZoom) + labelFont=ImageFont.truetype(font=TREBUC_FILE, size=12*fontZoom) startPosY = 15 stepPosY = 12*fontZoom if self.manhattan_plot != True: im_drawer.line( - xy=((xLeftOffset,startPosY),(xLeftOffset+32,startPosY)), + xy=((xLeftOffset, startPosY), (xLeftOffset+32, startPosY)), fill=self.LRS_COLOR, width=2) im_drawer.text( - text=self.LRS_LOD, xy=(xLeftOffset+40,startPosY+TEXT_Y_DISPLACEMENT), - font=labelFont,fill=BLACK) + text=self.LRS_LOD, xy=(xLeftOffset+40, startPosY+TEXT_Y_DISPLACEMENT), + font=labelFont, fill=BLACK) startPosY += stepPosY if self.additiveChecked: startPosX = xLeftOffset im_drawer.line( - xy=((startPosX,startPosY),(startPosX+17,startPosY)), + xy=((startPosX, startPosY), (startPosX+17, startPosY)), fill=self.ADDITIVE_COLOR_POSITIVE, width=2) im_drawer.line( - xy=((startPosX+18,startPosY),(startPosX+32,startPosY)), + xy=((startPosX+18, startPosY), (startPosX+32, startPosY)), fill=self.ADDITIVE_COLOR_NEGATIVE, width=2) im_drawer.text( - text='Additive Effect',xy=(startPosX+40,startPosY+TEXT_Y_DISPLACEMENT), - font=labelFont,fill=BLACK) + text='Additive Effect', xy=(startPosX+40, startPosY+TEXT_Y_DISPLACEMENT), + font=labelFont, fill=BLACK) if self.genotype.type == 'intercross' and self.dominanceChecked: startPosX = xLeftOffset startPosY += stepPosY im_drawer.line( - xy=((startPosX,startPosY),(startPosX+17,startPosY)), + xy=((startPosX, startPosY), (startPosX+17, startPosY)), fill=self.DOMINANCE_COLOR_POSITIVE, width=4) im_drawer.line( - xy=((startPosX+18,startPosY),(startPosX+35,startPosY)), + xy=((startPosX+18, startPosY), (startPosX+35, startPosY)), fill=self.DOMINANCE_COLOR_NEGATIVE, width=4) im_drawer.text( - text='Dominance Effect', xy=(startPosX+42,startPosY+5), - font=labelFont,fill=BLACK) + text='Dominance Effect', xy=(startPosX+42, startPosY+5), + font=labelFont, fill=BLACK) if self.haplotypeAnalystChecked: startPosY += stepPosY startPosX = xLeftOffset im_drawer.line( - xy=((startPosX,startPosY),(startPosX+17,startPosY)), + xy=((startPosX, startPosY), (startPosX+17, startPosY)), fill=self.HAPLOTYPE_POSITIVE, width=4) im_drawer.line( - xy=((startPosX+18,startPosY),(startPosX+35,startPosY)), + xy=((startPosX+18, startPosY), (startPosX+35, startPosY)), fill=self.HAPLOTYPE_NEGATIVE, width=4) im_drawer.line( - xy=((startPosX+36,startPosY),(startPosX+53,startPosY)), + xy=((startPosX+36, startPosY), (startPosX+53, startPosY)), fill=self.HAPLOTYPE_HETEROZYGOUS, width=4) im_drawer.line( - xy=((startPosX+54,startPosY),(startPosX+67,startPosY)), + xy=((startPosX+54, startPosY), (startPosX+67, startPosY)), fill=self.HAPLOTYPE_RECOMBINATION, width=4) im_drawer.text( text='Haplotypes (Pat, Mat, Het, Unk)', - xy=(startPosX+76,startPosY+5),font=labelFont,fill=BLACK) + xy=(startPosX+76, startPosY+5), font=labelFont, fill=BLACK) if self.permChecked and self.nperm > 0: startPosY += stepPosY startPosX = xLeftOffset im_drawer.line( - xy=((startPosX, startPosY),( startPosX + 32, startPosY)), + xy=((startPosX, startPosY), ( startPosX + 32, startPosY)), fill=self.SIGNIFICANT_COLOR, width=self.SIGNIFICANT_WIDTH) im_drawer.line( - xy=((startPosX, startPosY + stepPosY),( startPosX + 32, startPosY + stepPosY)), + xy=((startPosX, startPosY + stepPosY), ( startPosX + 32, startPosY + stepPosY)), fill=self.SUGGESTIVE_COLOR, width=self.SUGGESTIVE_WIDTH) im_drawer.text( - text='Significant %s = %2.2f' % (self.LRS_LOD,self.significant), - xy=(xLeftOffset+42,startPosY+TEXT_Y_DISPLACEMENT),font=labelFont,fill=BLACK) + text='Significant %s = %2.2f' % (self.LRS_LOD, self.significant), + xy=(xLeftOffset+42, startPosY+TEXT_Y_DISPLACEMENT), font=labelFont, fill=BLACK) im_drawer.text( text='Suggestive %s = %2.2f' % (self.LRS_LOD, self.suggestive), - xy=(xLeftOffset+42,startPosY + TEXT_Y_DISPLACEMENT +stepPosY),font=labelFont, + xy=(xLeftOffset+42, startPosY + TEXT_Y_DISPLACEMENT +stepPosY), font=labelFont, fill=BLACK) - labelFont = ImageFont.truetype(font=VERDANA_FILE,size=12*fontZoom) + labelFont = ImageFont.truetype(font=VERDANA_FILE, size=12*fontZoom) labelColor = BLACK if self.dataset.type == "Publish" or self.dataset.type == "Geno": dataset_label = self.dataset.fullname @@ -1109,22 +1109,22 @@ class DisplayMappingResults(object): im_drawer.textsize(string2, font=labelFont)[0]) im_drawer.text( text=identification, - xy=(canvas.size[0] - xRightOffset-d,20*fontZoom),font=labelFont, + xy=(canvas.size[0] - xRightOffset-d, 20*fontZoom), font=labelFont, fill=labelColor) else: d = 4+ max( im_drawer.textsize(string1, font=labelFont)[0], im_drawer.textsize(string2, font=labelFont)[0]) im_drawer.text( - text=string1,xy=(canvas.size[0] - xRightOffset-d,35*fontZoom), - font=labelFont,fill=labelColor) + text=string1, xy=(canvas.size[0] - xRightOffset-d, 35*fontZoom), + font=labelFont, fill=labelColor) im_drawer.text( - text=string2,xy=(canvas.size[0] - xRightOffset-d,50*fontZoom), - font=labelFont,fill=labelColor) + text=string2, xy=(canvas.size[0] - xRightOffset-d, 50*fontZoom), + font=labelFont, fill=labelColor) if string3 != '': im_drawer.text( - text=string3,xy=(canvas.size[0] - xRightOffset-d,65*fontZoom), - font=labelFont,fill=labelColor) + text=string3, xy=(canvas.size[0] - xRightOffset-d, 65*fontZoom), + font=labelFont, fill=labelColor) def drawGeneBand(self, canvas, gifmap, plotXScale, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): @@ -1345,7 +1345,7 @@ class DisplayMappingResults(object): labelText = "3'" im_drawer.text( text=labelText, - xy=(utrEndPix+2,geneYLocation+self.EACH_GENE_HEIGHT), + xy=(utrEndPix+2, geneYLocation+self.EACH_GENE_HEIGHT), font=ImageFont.truetype(font=ARIAL_FILE, size=2)) #draw the genes as rectangles @@ -1357,7 +1357,7 @@ class DisplayMappingResults(object): COORDS = "%d, %d, %d, %d" %(geneStartPix, geneYLocation, geneEndPix, (geneYLocation + self.EACH_GENE_HEIGHT)) # NL: 06-02-2011 Rob required to display NCBI info in a new window - gifmap.areas.append(HT.Area(shape='rect',coords=COORDS,href=HREF, title=TITLE,target="_blank")) + gifmap.areas.append(HT.Area(shape='rect', coords=COORDS, href=HREF, title=TITLE, target="_blank")) ## BEGIN HaplotypeAnalyst def drawHaplotypeBand(self, canvas, gifmap, plotXScale, offset= (40, 120, 80, 10), zoom = 1, startMb = None, endMb = None): @@ -1490,7 +1490,7 @@ class DisplayMappingResults(object): counter = counter + 1 if item.name == samplelist[k]: ind = counter - maxind=max(ind,maxind) + maxind=max(ind, maxind) # lines if (oldgeno[k] == -1 and _geno == -1): @@ -1523,7 +1523,7 @@ class DisplayMappingResults(object): COORDS = "%d, %d, %d, %d" %(geneStartPix, geneYLocation+ind*self.EACH_GENE_HEIGHT, geneEndPix+1, (geneYLocation + ind*self.EACH_GENE_HEIGHT)) TITLE = "Strain: %s, marker (%s) \n Position %2.3f Mb." % (samplelist[k], _chr[j].name, float(txStart)) HREF = '' - gifmap.areas.append(HT.Area(shape='rect',coords=COORDS,href=HREF, title=TITLE)) + gifmap.areas.append(HT.Area(shape='rect', coords=COORDS, href=HREF, title=TITLE)) # if there are no more markers in a chromosome, the plotRight value calculated above will be before the plotWidth # resulting in some empty space on the right side of the plot area. This draws an "unknown" bar from plotRight to the edge. @@ -1642,14 +1642,14 @@ class DisplayMappingResults(object): WEBQTL_HREF = "javascript:rangeView('%s', %f, %f)" % (self.selectedChr - 1, max(0, (calBase-webqtlZoomWidth))/1000000.0, (calBase+webqtlZoomWidth)/1000000.0) WEBQTL_TITLE = "Click to view this section of the genome in WebQTL" - gifmap.areas.append(HT.Area(shape='rect',coords=WEBQTL_COORDS,href=WEBQTL_HREF, title=WEBQTL_TITLE)) + gifmap.areas.append(HT.Area(shape='rect', coords=WEBQTL_COORDS, href=WEBQTL_HREF, title=WEBQTL_TITLE)) im_drawer.rectangle( xy=((xBrowse1, paddingTop), (xBrowse2, (paddingTop + self.BAND_HEIGHT))), outline=self.CLICKABLE_WEBQTL_REGION_COLOR, fill=self.CLICKABLE_WEBQTL_REGION_COLOR) im_drawer.line( - xy=((xBrowse1, paddingTop),( xBrowse1, (paddingTop + self.BAND_HEIGHT))), + xy=((xBrowse1, paddingTop), ( xBrowse1, (paddingTop + self.BAND_HEIGHT))), fill=self.CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR) if self.dataset.group.species == "mouse" or self.dataset.group.species == "rat": @@ -1659,14 +1659,14 @@ class DisplayMappingResults(object): else: PHENOGEN_HREF = "https://phenogen.org/gene.jsp?speciesCB=Mm&auto=Y&geneTxt=chr%s:%d-%d&genomeVer=mm10" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) PHENOGEN_TITLE = "Click to view this section of the genome in PhenoGen" - gifmap.areas.append(HT.Area(shape='rect',coords=PHENOGEN_COORDS,href=PHENOGEN_HREF, title=PHENOGEN_TITLE)) + gifmap.areas.append(HT.Area(shape='rect', coords=PHENOGEN_COORDS, href=PHENOGEN_HREF, title=PHENOGEN_TITLE)) im_drawer.rectangle( xy=((xBrowse1, phenogenPaddingTop), (xBrowse2, (phenogenPaddingTop+self.BAND_HEIGHT))), outline=self.CLICKABLE_PHENOGEN_REGION_COLOR, fill=self.CLICKABLE_PHENOGEN_REGION_COLOR) im_drawer.line( - xy=((xBrowse1, phenogenPaddingTop),( xBrowse1, (phenogenPaddingTop+self.BAND_HEIGHT))), + xy=((xBrowse1, phenogenPaddingTop), ( xBrowse1, (phenogenPaddingTop+self.BAND_HEIGHT))), fill=self.CLICKABLE_PHENOGEN_REGION_OUTLINE_COLOR) UCSC_COORDS = "%d, %d, %d, %d" %(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.BAND_HEIGHT)) @@ -1675,7 +1675,7 @@ class DisplayMappingResults(object): else: UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) UCSC_TITLE = "Click to view this section of the genome in the UCSC Genome Browser" - gifmap.areas.append(HT.Area(shape='rect',coords=UCSC_COORDS,href=UCSC_HREF, title=UCSC_TITLE)) + gifmap.areas.append(HT.Area(shape='rect', coords=UCSC_COORDS, href=UCSC_HREF, title=UCSC_TITLE)) im_drawer.rectangle( xy=((xBrowse1, ucscPaddingTop), (xBrowse2, (ucscPaddingTop+self.BAND_HEIGHT))), @@ -1692,7 +1692,7 @@ class DisplayMappingResults(object): else: ENSEMBL_HREF = "http://www.ensembl.org/Rattus_norvegicus/contigview?chr=%s&start=%d&end=%d" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) ENSEMBL_TITLE = "Click to view this section of the genome in the Ensembl Genome Browser" - gifmap.areas.append(HT.Area(shape='rect',coords=ENSEMBL_COORDS,href=ENSEMBL_HREF, title=ENSEMBL_TITLE)) + gifmap.areas.append(HT.Area(shape='rect', coords=ENSEMBL_COORDS, href=ENSEMBL_HREF, title=ENSEMBL_TITLE)) im_drawer.rectangle( xy=((xBrowse1, ensemblPaddingTop), (xBrowse2, (ensemblPaddingTop+self.BAND_HEIGHT))), @@ -1789,8 +1789,8 @@ class DisplayMappingResults(object): continue Xc = xLeftOffset + plotXScale*(_Mb - startMb) if counter % NUM_MINOR_TICKS == 0: # Draw a MAJOR mark, not just a minor tick mark - im_drawer.line(xy=((Xc,yZero), - (Xc,yZero+xMajorTickHeight)), + im_drawer.line(xy=((Xc, yZero), + (Xc, yZero+xMajorTickHeight)), fill=xAxisTickMarkColor, width=X_MAJOR_TICK_THICKNESS) # Draw the MAJOR tick mark labelStr = str(formatStr % _Mb) # What Mbase location to put on the label @@ -1800,8 +1800,8 @@ class DisplayMappingResults(object): text=labelStr, font=MBLabelFont, fill=xAxisLabelColor) else: - im_drawer.line(xy=((Xc,yZero), - (Xc,yZero+xMinorTickHeight)), + im_drawer.line(xy=((Xc, yZero), + (Xc, yZero+xMinorTickHeight)), fill=xAxisTickMarkColor, width=X_MINOR_TICK_THICKNESS) # Draw the MINOR tick mark @@ -1834,7 +1834,7 @@ class DisplayMappingResults(object): text="Megabases", xy=( xLeftOffset+(plotWidth-im_drawer.textsize( - "Megabases",font=megabaseLabelFont)[0])/2, + "Megabases", font=megabaseLabelFont)[0])/2, strYLoc+MBLabelFont.font.height+10*(zoom%2)), font=megabaseLabelFont, fill=BLACK) pass @@ -1889,7 +1889,7 @@ class DisplayMappingResults(object): for j, ChrInfo in enumerate(ChrAInfo): preLpos = -1 for i, item in enumerate(ChrInfo): - Lname,Lpos = item + Lname, Lpos = item if Lpos != preLpos: offsetA += stepA differ = 1 @@ -1903,17 +1903,17 @@ class DisplayMappingResults(object): Zorder = 0 if differ: im_drawer.line( - xy=((startPosX+Lpos,yZero),(xLeftOffset+offsetA,\ + xy=((startPosX+Lpos, yZero), (xLeftOffset+offsetA,\ yZero+25)), fill=lineColor) im_drawer.line( - xy=((xLeftOffset+offsetA,yZero+25),(xLeftOffset+offsetA,\ + xy=((xLeftOffset+offsetA, yZero+25), (xLeftOffset+offsetA,\ yZero+40+Zorder*(LRectWidth+3))), fill=lineColor) rectColor = ORANGE else: im_drawer.line( - xy=((xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3)-3),(\ + xy=((xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3)-3), (\ xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3))), fill=lineColor) rectColor = DEEPPINK @@ -1921,9 +1921,9 @@ class DisplayMappingResults(object): xy=((xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3)), (xLeftOffset+offsetA-LRectHeight, yZero+40+Zorder*(LRectWidth+3)+LRectWidth)), - outline=rectColor,fill=rectColor,width = 0) + outline=rectColor, fill=rectColor, width = 0) COORDS="%d,%d,%d,%d"%(xLeftOffset+offsetA-LRectHeight, yZero+40+Zorder*(LRectWidth+3),\ - xLeftOffset+offsetA,yZero+40+Zorder*(LRectWidth+3)+LRectWidth) + xLeftOffset+offsetA, yZero+40+Zorder*(LRectWidth+3)+LRectWidth) HREF="/show_trait?trait_id=%s&dataset=%s" % (Lname, self.dataset.group.name+"Geno") #HREF="javascript:showDatabase3('%s','%s','%s','');" % (showLocusForm,fd.RISet+"Geno", Lname) Areas=HT.Area(shape='rect', coords=COORDS, href=HREF, target="_blank", title="Locus : " + Lname) @@ -1931,7 +1931,7 @@ class DisplayMappingResults(object): ##piddle bug if j == 0: im_drawer.line( - xy=((startPosX,yZero),(startPosX,yZero+40)), + xy=((startPosX, yZero), (startPosX, yZero+40)), fill=lineColor) startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale @@ -1943,7 +1943,7 @@ class DisplayMappingResults(object): strYLoc + MBLabelFont.font.height+ 10*(zoom%2)), font=centimorganLabelFont, fill=BLACK) - im_drawer.line(xy=((xLeftOffset,yZero), (xLeftOffset+plotWidth,yZero)), + im_drawer.line(xy=((xLeftOffset, yZero), (xLeftOffset+plotWidth, yZero)), fill=BLACK, width=X_AXIS_THICKNESS) # Draw the X axis itself @@ -2079,7 +2079,7 @@ class DisplayMappingResults(object): LRS_LOD_Max = 0.000001 yTopOffset + 30*(zoom - 1) yLRS = yZero - (item/LRS_LOD_Max) * LRSHeightThresh - im_drawer.line(xy=((xLeftOffset,yLRS), (xLeftOffset-4,yLRS)), + im_drawer.line(xy=((xLeftOffset, yLRS), (xLeftOffset-4, yLRS)), fill=self.LRS_COLOR, width=1*zoom) if all_int: scaleStr = "%d" % item @@ -2127,8 +2127,8 @@ class DisplayMappingResults(object): else: sugg_title = "Suggestive LOD = %0.2f" % (self.suggestive/4.61) sig_title = "Significant LOD = %0.2f" % (self.significant/4.61) - Areas1 = HT.Area(shape='rect',coords=sugg_coords,title=sugg_title) - Areas2 = HT.Area(shape='rect',coords=sig_coords,title=sig_title) + Areas1 = HT.Area(shape='rect', coords=sugg_coords, title=sugg_title) + Areas2 = HT.Area(shape='rect', coords=sig_coords, title=sig_title) gifmap.areas.append(Areas1) gifmap.areas.append(Areas2) @@ -2316,7 +2316,7 @@ class DisplayMappingResults(object): im_drawer.text( text="5", xy=( - Xc-im_drawer.textsize("5",font=symbolFont)[0]/2+1, + Xc-im_drawer.textsize("5", font=symbolFont)[0]/2+1, Yc-4), fill=point_color, font=symbolFont) else: @@ -2383,8 +2383,8 @@ class DisplayMappingResults(object): ) else: im_drawer.line( - xy=((Xc0,yZero-(Yc0-yZero)), - (Xc,yZero-(Yc-yZero))), + xy=((Xc0, yZero-(Yc0-yZero)), + (Xc, yZero-(Yc-yZero))), fill=minusColor, width=lineWidth #, clipX=(xLeftOffset, xLeftOffset + plotWidth) ) @@ -2471,8 +2471,8 @@ class DisplayMappingResults(object): ###draw additive scale if not self.multipleInterval and self.additiveChecked: - additiveScaleFont=ImageFont.truetype(font=VERDANA_FILE,size=16*zoom) - additiveScale = Plot.detScaleOld(0,additiveMax) + additiveScaleFont=ImageFont.truetype(font=VERDANA_FILE, size=16*zoom) + additiveScale = Plot.detScaleOld(0, additiveMax) additiveStep = (additiveScale[1]-additiveScale[0])/additiveScale[2] additiveAxisList = Plot.frange(0, additiveScale[1], additiveStep) addPlotScale = AdditiveHeightThresh/additiveMax @@ -2482,18 +2482,18 @@ class DisplayMappingResults(object): for item in additiveAxisList: additiveY = yZero - item*addPlotScale im_drawer.line( - xy=((xLeftOffset + plotWidth,additiveY), - (xLeftOffset+4+ plotWidth,additiveY)), + xy=((xLeftOffset + plotWidth, additiveY), + (xLeftOffset+4+ plotWidth, additiveY)), fill=self.ADDITIVE_COLOR_POSITIVE, width=1*zoom) scaleStr = "%2.3f" % item im_drawer.text( text=scaleStr, - xy=(xLeftOffset + plotWidth +6,additiveY+TEXT_Y_DISPLACEMENT), - font=additiveScaleFont,fill=self.ADDITIVE_COLOR_POSITIVE) + xy=(xLeftOffset + plotWidth +6, additiveY+TEXT_Y_DISPLACEMENT), + font=additiveScaleFont, fill=self.ADDITIVE_COLOR_POSITIVE) im_drawer.line( - xy=((xLeftOffset+plotWidth,additiveY), - (xLeftOffset+plotWidth,yZero)), + xy=((xLeftOffset+plotWidth, additiveY), + (xLeftOffset+plotWidth, yZero)), fill=self.ADDITIVE_COLOR_POSITIVE, width=1*zoom) im_drawer.line( @@ -2553,7 +2553,7 @@ class DisplayMappingResults(object): chrFontZoom = 2 else: chrFontZoom = 1 - chrLabelFont=ImageFont.truetype(font=VERDANA_FILE,size=24*chrFontZoom) + chrLabelFont=ImageFont.truetype(font=VERDANA_FILE, size=24*chrFontZoom) for i, _chr in enumerate(self.genotype): if (i % 2 == 0): @@ -2575,10 +2575,10 @@ class DisplayMappingResults(object): TEXT_Y_DISPLACEMENT = 0 im_drawer.text(xy=(chrStartPix, yTopOffset + TEXT_Y_DISPLACEMENT), text=_chr.name, font=chrLabelFont, fill=BLACK) - COORDS = "%d,%d,%d,%d" %(chrStartPix, yTopOffset, chrEndPix,yTopOffset +20) + COORDS = "%d,%d,%d,%d" %(chrStartPix, yTopOffset, chrEndPix, yTopOffset +20) #add by NL 09-03-2010 - HREF = "javascript:chrView(%d,%s);" % (i,self.ChrLengthMbList) + HREF = "javascript:chrView(%d,%s);" % (i, self.ChrLengthMbList) #HREF = "javascript:changeView(%d,%s);" % (i,self.ChrLengthMbList) Areas = HT.Area(shape='rect', coords=COORDS, href=HREF) gifmap.areas.append(Areas) @@ -2720,7 +2720,7 @@ class DisplayMappingResults(object): else: chr_as_int = int(theGO["Chromosome"]) - 1 if refGene: - literatureCorrelationString = str(self.getLiteratureCorrelation(self.cursor,refGene,theGO['GeneID']) or "N/A") + literatureCorrelationString = str(self.getLiteratureCorrelation(self.cursor, refGene, theGO['GeneID']) or "N/A") this_row = [selectCheck.__str__(), str(tableIterationsCnt), @@ -2820,8 +2820,8 @@ class DisplayMappingResults(object): lCorr = None try: query = 'SELECT Value FROM LCorrRamin3 WHERE GeneId1 = %s and GeneId2 = %s' - for x,y in [(geneId1,geneId2),(geneId2,geneId1)]: - cursor.execute(query,(x,y)) + for x, y in [(geneId1, geneId2), (geneId2, geneId1)]: + cursor.execute(query, (x, y)) lCorr = cursor.fetchone() if lCorr: lCorr = lCorr[0] diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index d4ee6fe6..2f282adc 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -54,7 +54,7 @@ def gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename = ''): for i, sample in enumerate(ped_sample_list): try: value = vals[i] - value = str(value).replace('value=','') + value = str(value).replace('value=', '') value = value.strip() except: value = -9999 @@ -78,7 +78,7 @@ def gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename = ''): # get strain name from ped file in order def get_samples_from_ped_file(dataset): - ped_file= open("{}{}.ped".format(flat_files('mapping'), dataset.group.name),"r") + ped_file= open("{}{}.ped".format(flat_files('mapping'), dataset.group.name), "r") line = ped_file.readline() sample_list=[] @@ -155,7 +155,7 @@ def parse_plink_output(output_filename, species): # output: lineList list ####################################################### def build_line_list(line=None): - line_list = string.split(string.strip(line),' ')# irregular number of whitespaces between columns + line_list = string.split(string.strip(line), ' ')# irregular number of whitespaces between columns line_list = [item for item in line_list if item !=''] line_list = list(map(string.strip, line_list)) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 145dbc77..1e6dff57 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -707,7 +707,7 @@ def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): perm_strata_strings.append(combined_string) - d = dict([(y,x+1) for x,y in enumerate(sorted(set(perm_strata_strings)))]) + d = dict([(y, x+1) for x, y in enumerate(sorted(set(perm_strata_strings)))]) list_to_numbers = [d[x] for x in perm_strata_strings] perm_strata = list_to_numbers diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 5b3946e3..c07a7670 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -53,7 +53,7 @@ views.py). search = self.search_terms self.original_search_string = self.search_terms # check for dodgy search terms - rx = re.compile(r'.*\W(href|http|sql|select|update)\W.*',re.IGNORECASE) + rx = re.compile(r'.*\W(href|http|sql|select|update)\W.*', re.IGNORECASE) if rx.match(search): logger.info("Regex failed search") self.search_term_exists = False diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 21ba7f63..f17e825e 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -57,7 +57,7 @@ class SampleList(object): sample = webqtlCaseData.webqtlCaseData(name=sample_name) sample.extra_info = {} - if self.dataset.group.name == 'AXBXA' and sample_name in ('AXB18/19/20','AXB13/14','BXA8/17'): + if self.dataset.group.name == 'AXBXA' and sample_name in ('AXB18/19/20', 'AXB13/14', 'BXA8/17'): sample.extra_info['url'] = "/mouseCross.html#AXB/BXA" sample.extra_info['css_class'] = "fs12" diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index a871e91a..232cb8da 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -867,7 +867,7 @@ def forgot_password_submit(): email_address = params['email_address'] next_page = None if email_address != "": - logger.debug("Wants to send password E-mail to ",email_address) + logger.debug("Wants to send password E-mail to ", email_address) user_details = get_user_by_unique_column("email_address", email_address) if user_details: ForgotPasswordEmail(user_details["email_address"]) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 394a9e28..92c20fc7 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -55,7 +55,7 @@ from wqflask.ctl import ctl_analysis from wqflask.snp_browser import snp_browser from utility import temp_data -from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,JS_TWITTER_POST_FETCHER_PATH,JS_GUIX_PATH, CSS_PATH +from utility.tools import SQL_URI, TEMPDIR, USE_REDIS, USE_GN_SERVER, GN_SERVER_URL, GN_VERSION, JS_TWITTER_POST_FETCHER_PATH, JS_GUIX_PATH, CSS_PATH from utility.helper_functions import get_species_groups from utility.authentication_tools import check_resource_availability from utility.redis_tools import get_redis_conn @@ -133,10 +133,10 @@ def handle_bad_request(e): list = [fn for fn in os.listdir("./wqflask/static/gif/error") if fn.endswith(".gif") ] animation = random.choice(list) - resp = make_response(render_template("error.html",message=err_msg,stack=formatted_lines,error_image=animation,version=GN_VERSION)) + resp = make_response(render_template("error.html", message=err_msg, stack=formatted_lines, error_image=animation, version=GN_VERSION)) # logger.error("Set cookie %s with %s" % (err_msg, animation)) - resp.set_cookie(err_msg[:32],animation) + resp.set_cookie(err_msg[:32], animation) return resp @app.route("/authentication_needed") @@ -257,7 +257,7 @@ def docedit(): @app.route('/generated/') def generated_file(filename): logger.info(request.url) - return send_from_directory(GENERATED_IMAGE_DIR,filename) + return send_from_directory(GENERATED_IMAGE_DIR, filename) @app.route("/help") def help(): -- cgit v1.2.3 From 09cb4a05526c73229428401f356251bbfe8bf1cd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 20 Aug 2020 15:39:50 +0300 Subject: Add backward support for urllib * wqflask/base/trait.py: Remove unused imports * wqflask/db/call.py (Bench): Add urllib2 fallback for python2 --- wqflask/base/trait.py | 3 +-- wqflask/db/call.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 05b272c3..a19b66f7 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -6,7 +6,6 @@ import resource import codecs import requests import random -import urllib.request, urllib.parse, urllib.error from base import webqtlConfig from base.webqtlCaseData import webqtlCaseData @@ -607,4 +606,4 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): else: raise KeyError(repr(trait.name)+' information is not found in the database.') - return trait \ No newline at end of file + return trait diff --git a/wqflask/db/call.py b/wqflask/db/call.py index 3b8f782e..0971d2a2 100644 --- a/wqflask/db/call.py +++ b/wqflask/db/call.py @@ -3,7 +3,10 @@ from flask import g import string -import urllib.request, urllib.error, urllib.parse +try: # Python2 support + import urllib.request, urllib.error, urllib.parse +except: + import urllib2 import json from utility.tools import USE_GN_SERVER, LOG_SQL, GN_SERVER_URL from utility.benchmark import Bench @@ -59,7 +62,11 @@ def gn_server(path): """ with Bench("GN_SERVER", LOG_SQL): - res = urllib.request.urlopen(GN_SERVER_URL+path) + res = () + try: + res = urllib.request.urlopen(GN_SERVER_URL+path) + except: + res = urllib2.urlopen(GN_SERVER_URL+path) rest = res.read() res2 = json.loads(rest) logger.debug(res2) -- cgit v1.2.3 From 7f623509a4afded47eb8580451c146487d2ef662 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 20 Aug 2020 15:41:18 +0300 Subject: Disable python2 QTLReaper * wqflask/base/data_set.py: Remove usage of "reaper" to load datasets --- wqflask/base/data_set.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 06e1c551..43beec26 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -34,7 +34,6 @@ from utility import webqtlUtil from db import webqtlDatabaseFunction from base import species from base import webqtlConfig -import reaper from flask import Flask, g import os import math @@ -456,12 +455,7 @@ class DatasetGroup(object): full_filename = str(locate(self.genofile, 'genotype')) else: full_filename = str(locate(self.name + '.geno', 'genotype')) - - if use_reaper: - genotype_1 = reaper.Dataset() - genotype_1.read(full_filename) - else: - genotype_1 = gen_geno_ob.genotype(full_filename) + genotype_1 = gen_geno_ob.genotype(full_filename) if genotype_1.type == "group" and self.parlist: genotype_2 = genotype_1.add( -- cgit v1.2.3 From b32004bc517d9a58be52662297e541c23f60355b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 20 Aug 2020 16:01:54 +0300 Subject: Replace all references to python2.7 with python3.8 Run: ``` find . -type f -print0 | xargs -0 sed -i \ "s|lib/python2.7/site-packages|lib/python3.8/site-packages|g" ``` --- bin/genenetwork2 | 6 +++--- doc/GUIX-Reproducible-from-source.org | 6 +++--- doc/README.org | 2 +- doc/development.org | 4 ++-- etc/default_settings.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index dd6db7d6..5f4e0f9a 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -60,7 +60,7 @@ GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) echo GN2_BASE_DIR=$GN2_BASE_DIR -GUIX_SITE=$GN2_BASE_DIR/lib/python2.7/site-packages +GUIX_SITE=$GN2_BASE_DIR/lib/python3.8/site-packages if [ -d $GUIX_SITE ]; then echo INFO: GN2 is running from GNU Guix GN2_BASE_DIR=$GUIX_SITE @@ -105,7 +105,7 @@ echo GN2_SETTINGS=$settings # if [ -z $ELASTICSEARCH_PROFILE ]; then # echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; # else -# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python3.8/site-packages" # fi if [ -z $GN2_PROFILE ] ; then @@ -121,7 +121,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$PYTHON_GN_PATH:$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!! + export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! export R_LIBS_SITE=$GN2_PROFILE/site-library export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index 19e4d14f..fffa9571 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -167,7 +167,7 @@ the Guix suggested environment vars. Check the output of #+begin_src bash guix package --search-paths -export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages" +export PYTHONPATH="/root/.guix-profile/lib/python3.8/site-packages" export R_LIBS_SITE="/root/.guix-profile/site-library/" #+end_src @@ -265,7 +265,7 @@ software. If something is not working, take a hint from the settings file that comes in the Guix installation. It sits in something like -: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py +: cat ~/.guix-profile/lib/python3.8/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py ** Set up nginx port forwarding @@ -380,7 +380,7 @@ After setting the paths for the server #+begin_src bash export PATH=~/.guix-profile/bin:$PATH -export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +export PYTHONPATH="$HOME/.guix-profile/lib/python3.8/site-packages" export R_LIBS_SITE="$HOME/.guix-profile/site-library/" export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0" export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0" diff --git a/doc/README.org b/doc/README.org index 46df03c7..43c92e3c 100644 --- a/doc/README.org +++ b/doc/README.org @@ -218,7 +218,7 @@ information given by guix: On one system: -: export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +: export PYTHONPATH="$HOME/.guix-profile/lib/python3.8/site-packages" : export R_LIBS_SITE="$HOME/.guix-profile/site-library/" : export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0" diff --git a/doc/development.org b/doc/development.org index e65ccd58..cd3beea3 100644 --- a/doc/development.org +++ b/doc/development.org @@ -67,11 +67,11 @@ You can install a Python package locally with pip, e.g. pip install hjson #+END_SRC -This installed in ~$HOME/.local/lib/python2.7/site-packages~. To add +This installed in ~$HOME/.local/lib/python3.8/site-packages~. To add the search path for GeneNetwork use the environment variable #+BEGIN_SRC sh -export PYTHON_GN_PATH=$HOME/.local/lib/python2.7/site-packages +export PYTHON_GN_PATH=$HOME/.local/lib/python3.8/site-packages #+END_SRC Now you should be able to do diff --git a/etc/default_settings.py b/etc/default_settings.py index 82c605da..27522187 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -19,7 +19,7 @@ # # For GNU Guix deployment also check the paths in # -# ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py +# ~/.guix-profile/lib/python3.8/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py import os import sys -- cgit v1.2.3 From 1f4fb6b24f3508d80be1f07cd62e38ac9385ef41 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 20 Aug 2020 16:35:17 +0300 Subject: Handle module renames in the standard library Run: ``` 2to3-3.8 -f imports -w . && \ 2to3-3.8 -f imports2 -w . ``` See: and --- wqflask/base/data_set.py | 2 +- wqflask/maintenance/gen_select_dataset.py | 4 ++-- wqflask/maintenance/quantile_normalize.py | 4 ++-- wqflask/maintenance/set_resource_defaults.py | 4 ++-- wqflask/utility/svg.py | 16 ++++++++-------- wqflask/wqflask/api/router.py | 14 +++++++------- wqflask/wqflask/collect.py | 2 +- .../wqflask/comparison_bar_chart/comparison_bar_chart.py | 2 +- wqflask/wqflask/correlation/show_corr_results.py | 2 +- wqflask/wqflask/correlation_matrix/show_corr_matrix.py | 2 +- wqflask/wqflask/export_traits.py | 4 ++-- wqflask/wqflask/heatmap/heatmap.py | 4 ++-- .../wqflask/marker_regression/display_mapping_results.py | 4 ++-- wqflask/wqflask/marker_regression/run_mapping.py | 2 +- wqflask/wqflask/network_graph/network_graph.py | 2 +- wqflask/wqflask/show_trait/show_trait.py | 2 +- wqflask/wqflask/user_manager.py | 2 +- wqflask/wqflask/views.py | 12 ++++++------ 18 files changed, 42 insertions(+), 42 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 43beec26..8151a29d 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,7 +44,7 @@ import codecs import json import requests import gzip -import cPickle as pickle +import pickle as pickle import itertools from redis import Redis diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index d12b328f..fd65a52a 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -50,7 +50,7 @@ from utility.tools import locate, locate_ignore_error, TEMPDIR, SQL_URI import MySQLdb import simplejson as json -import urlparse +import urllib.parse #import sqlalchemy as sa @@ -66,7 +66,7 @@ from pprint import pformat as pf def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" - parsed_uri = urlparse.urlparse(SQL_URI) + parsed_uri = urllib.parse.urlparse(SQL_URI) db_conn_info = dict( db = parsed_uri.path[1:], diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 43edfd13..4d6e03bf 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -6,7 +6,7 @@ sys.path.insert(0, './') import MySQLdb -import urlparse +import urllib.parse import numpy as np import pandas as pd @@ -22,7 +22,7 @@ from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" - parsed_uri = urlparse.urlparse(SQL_URI) + parsed_uri = urllib.parse.urlparse(SQL_URI) db_conn_info = dict( db = parsed_uri.path[1:], diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index d53a255b..abd5416c 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -34,7 +34,7 @@ Redis = get_redis_conn() import MySQLdb -import urlparse +import urllib.parse from utility.logger import getLogger logger = getLogger(__name__) @@ -42,7 +42,7 @@ logger = getLogger(__name__) def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" - parsed_uri = urlparse.urlparse(SQL_URI) + parsed_uri = urllib.parse.urlparse(SQL_URI) db_conn_info = dict( db = parsed_uri.path[1:], diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 19eda0ce..b92cc2d1 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -957,8 +957,8 @@ class drawing: # Voeg een element toe aan de grafiek toe. if use_dom_implementation==0: def toXml(self, filename='',compress=False): - import cStringIO - xml=cStringIO.StringIO() + import io + xml=io.StringIO() xml.write("\n") xml.write(" 1: - memory_file = StringIO.StringIO() + memory_file = io.StringIO() with ZipFile(memory_file, mode='w', compression=ZIP_DEFLATED) as zf: for the_file in file_list: zf.writestr(the_file[0], the_file[1]) @@ -460,7 +460,7 @@ def export_perm_data(): ["#Comment: Results sorted from low to high peak linkage"] ] - buff = StringIO.StringIO() + buff = io.StringIO() writer = csv.writer(buff) writer.writerows(the_rows) for item in perm_info['perm_data']: -- cgit v1.2.3 From ab78dd757a7d11375e32637158ddcffc029a3113 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sun, 23 Aug 2020 15:28:10 +0300 Subject: Remove unused imports --- wqflask/wqflask/correlation/show_corr_results.py | 13 +------------ wqflask/wqflask/news.py | 3 --- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 65360040..233a3a03 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -20,22 +20,11 @@ from __future__ import absolute_import, print_function, division -import sys - -import string -import pickle -import os -import time -import pp -import math import collections -import resource import json - import scipy import numpy import rpy2.robjects as ro # R Objects -import rpy2.rinterface as ri from rpy2.robjects.packages import importr utils = importr("utils") @@ -648,4 +637,4 @@ def get_header_fields(data_type, corr_method): 'N', 'Sample p(r)'] - return header_fields \ No newline at end of file + return header_fields diff --git a/wqflask/wqflask/news.py b/wqflask/wqflask/news.py index 8bc6b889..20319756 100644 --- a/wqflask/wqflask/news.py +++ b/wqflask/wqflask/news.py @@ -1,7 +1,4 @@ from __future__ import absolute_import, print_function, division -import sys -reload(sys) -sys.setdefaultencoding('utf8') from flask import g class News(object): -- cgit v1.2.3 From 666a423b0b2f8cf98b9e70ed3dcf1c6dc3896b1d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sun, 23 Aug 2020 15:28:41 +0300 Subject: Use Python3 string methods * wqflask/utility/webqtlUtil.py (genRandStr): Replace "string.letters" with "string.ascii_letters". --- wqflask/utility/webqtlUtil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index 77cc3416..d4979011 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -64,7 +64,7 @@ ParInfo ={ # Accessory Functions ######################################### -def genRandStr(prefix = "", length=8, chars=string.letters+string.digits): +def genRandStr(prefix = "", length=8, chars=string.ascii_letters+string.digits): from random import choice _str = prefix[:] for i in range(length): @@ -110,4 +110,4 @@ def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users) AuthorizedUsersList=list(map(string.strip, string.split(authorized_users, ','))) if AuthorizedUsersList.__contains__(userName): access_to_confidential_phenotype_trait = 1 - return access_to_confidential_phenotype_trait \ No newline at end of file + return access_to_confidential_phenotype_trait -- cgit v1.2.3 From 239fc74198f4dd0388d01b41bc3a3278aed5973b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 01:33:04 +0300 Subject: Remove unused imports --- .../comparison_bar_chart/comparison_bar_chart.py | 18 ----------- wqflask/wqflask/correlation/show_corr_results.py | 12 ++----- .../wqflask/correlation_matrix/show_corr_matrix.py | 11 ------- wqflask/wqflask/heatmap/heatmap.py | 37 +++------------------- wqflask/wqflask/network_graph/network_graph.py | 20 ++---------- 5 files changed, 9 insertions(+), 89 deletions(-) diff --git a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py index 3e5daa50..16832621 100644 --- a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py +++ b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py @@ -19,34 +19,16 @@ # This module is used by GeneNetwork project (www.genenetwork.org) from __future__ import absolute_import, print_function, division - -import sys -# sys.path.append(".") Never do this in a webserver! - -import string -import pickle -import os -import time -import pp -import math -import collections -import resource - - from pprint import pformat as pf from base.trait import create_trait from base import data_set from utility import webqtlUtil, helper_functions, corr_result_helpers -from db import webqtlDatabaseFunction import utility.webqtlUtil #this is for parallel computing only. from wqflask.correlation import correlation_functions -from utility.benchmark import Bench from MySQLdb import escape_string as escape -from pprint import pformat as pf - from flask import Flask, g diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 233a3a03..5ced30b6 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -29,24 +29,16 @@ import rpy2.robjects as ro # R Objects from rpy2.robjects.packages import importr utils = importr("utils") -from pprint import pformat as pf - -from base import webqtlConfig -from utility.THCell import THCell -from utility.TDCell import TDCell -from base.trait import create_trait from base import data_set from utility import webqtlUtil, helper_functions, corr_result_helpers, hmac from db import webqtlDatabaseFunction -import utility.webqtlUtil #this is for parallel computing only. +import utility.webqtlUtil #this is for parallel computing only. from wqflask.correlation import correlation_functions from utility.benchmark import Bench import utility.webqtlUtil -from utility.type_checking import is_float, is_int, is_str, get_float, get_int, get_string - +from utility.type_checking import is_str, get_float, get_int, get_string from MySQLdb import escape_string as escape -from pprint import pformat as pf from flask import Flask, g diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index ea786289..bd5aca1f 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -20,19 +20,8 @@ from __future__ import absolute_import, print_function, division -import sys -# sys.path.append(".") Never do this in a webserver! - -import string -import pickle -import os import datetime -import time -import pp import math -import collections -import resource - import numpy as np import scipy diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 4fdc1f13..1f1cdb90 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -1,46 +1,19 @@ from __future__ import absolute_import, print_function, division -import sys -# sys.path.append(".") Never in a running webserver - import string -import pickle import os -import datetime -import time -import pp -import math import random -import collections -import resource - -import scipy -import numpy as np - -from pprint import pformat as pf - -from base.trait import GeneralTrait -from base import data_set from base import species from base import webqtlConfig from utility import helper_functions -from utility import Plot, Bunch -from utility import temp_data -from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR - -from MySQLdb import escape_string as escape - -import pickle as pickle -import simplejson as json - -from pprint import pformat as pf +from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR from redis import Redis -Redis = Redis() - from flask import Flask, g - from utility.logger import getLogger + +Redis = Redis() + logger = getLogger(__name__ ) class Heatmap(object): @@ -205,4 +178,4 @@ def parse_reaper_output(gwa_filename): marker['additive'] = float(line.split("\t")[6]) marker_obs.append(marker) - return marker_obs \ No newline at end of file + return marker_obs diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index 7916651f..2a4c4a93 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -20,27 +20,11 @@ from __future__ import absolute_import, print_function, division -import sys - -import string -import pickle -import os -import time -import pp -import math -import collections -import resource - import scipy - import simplejson as json - -from rpy2.robjects.packages import importr -import rpy2.robjects as robjects - from pprint import pformat as pf -from utility.THCell import THCell + from utility.TDCell import TDCell from base.trait import create_trait from base import data_set @@ -218,4 +202,4 @@ class NetworkGraph(object): trait_ob = create_trait(dataset=dataset_ob, name=trait_name, cellid=None) - self.trait_list.append((trait_ob, dataset_ob)) \ No newline at end of file + self.trait_list.append((trait_ob, dataset_ob)) -- cgit v1.2.3 From 3335bb967e59da17d104624be971e4a3e98e07c8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 01:33:43 +0300 Subject: Update corestats import * wqflask/utility/Plot.py: import corestats from the utility module. --- wqflask/utility/Plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 7d9d7649..48a5c7ba 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -32,7 +32,7 @@ from PIL import ImageFont from math import * -import corestats +import utility.corestats as corestats from base import webqtlConfig from utility.pillow_utils import draw_rotated_text import utility.logger -- cgit v1.2.3 From 5305c6b0080b36c3ad0865677b613dd970991d7f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 17:30:29 +0300 Subject: Replace python2-htmlgen with python-htmlgen * wqflask/wqflask/marker_regression/display_mapping_results.py: Use python-htmlgen --- .../marker_regression/display_mapping_results.py | 74 ++++++++++++++-------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 255e3b74..ccdafa14 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -35,7 +35,7 @@ import json from flask import Flask, g -from htmlgen import HTMLgen2 as HT +import htmlgen as HT from base import webqtlConfig from base.GeneralObject import GeneralObject @@ -88,35 +88,60 @@ class HtmlGenWrapper: """Wrapper Methods for HTML gen""" @staticmethod def create_image_tag(**kwargs): - return HT.Image(**kwargs) + image = HT.Image("", "") + for key, value in list(kwargs.items()): + image.set_attribute(key, value) + return image @staticmethod def create_form_tag(**kwargs): - return HT.Form(**kwargs) + form = HT.Form("POST", "") # Default method is POST + + for key, value in list(kwargs.items()): + if key == "submit": + form.append(value) + continue + form.set_attribute(key.replace("cgi", "action"), str(value)) + return form @staticmethod def create_p_tag(**kwargs): - return HT.Paragraph(**kwargs) + paragraph = HT.Paragraph() + for key, value in list(kwargs.items()): + paragraph.set_attribute(key, value) + return paragraph @staticmethod def create_br_tag(): - return HT.BR() + return HT.VoidElement("br") @staticmethod def create_input_tag(**kwargs): - return HT.Input(**kwargs) + input_ = HT.Input() + for key, value in list(kwargs.items()): + input_.set_attribute(key.lower().replace("_", ""), value) + return input_ @staticmethod def create_area_tag(**kwargs): - return HT.Area(**kwargs) + area = HT.VoidElement("area") + for key, value in list(kwargs.items()): + area.set_attribute(key, value) + return area @staticmethod def create_link_tag(href, content, **kwargs): - return HT.Href(href, content, **kwargs) + link = HT.Link(href, content) + for key, value in list(kwargs.items()): + link.set_attribute(key, value) + return link @staticmethod def create_map_tag(**kwargs): - return HT.Map(**kwargs) + map_ = HT.Element("map") + for key, value in list(kwargs.items()): + map_.set_attribute(key, value) + return map_ class DisplayMappingResults(object): @@ -554,7 +579,6 @@ class DisplayMappingResults(object): src="/image/{}.png".format(self.filename), border="0", usemap='#WebQTLImageMap' ) - self.intImg = intImg #Scales plot differently for high resolution if self.draw2X: @@ -575,12 +599,12 @@ class DisplayMappingResults(object): cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name=showLocusForm, - submit=HtmlGenWrapper.create_input_tag(type='hidden')) + submit=HtmlGenWrapper.create_input_tag(type_='hidden')) hddn = {'FormID':'showDatabase', 'ProbeSetID':'_','database':fd.RISet+"Geno",'CellID':'_', 'RISet':fd.RISet, 'incparentsf1':'ON'} for key in hddn.keys(): showLocusForm.append(HtmlGenWrapper.create_input_tag( - name=key, value=hddn[key], type='hidden')) + name=key, value=hddn[key], type_='hidden')) showLocusForm.append(intImg) else: showLocusForm = intImg @@ -1002,7 +1026,7 @@ class DisplayMappingResults(object): COORDS = "%d,%d,%d,%d" %(rectWidth+2+rightShift, yPaddingTop+kstep*15, rectWidth+2+rightShift+nameWidth, yPaddingTop+10+kstep*15,) HREF= "javascript:showDatabase3('%s','%s','%s','');" % (showLocusForm, thisTrait.db.name, thisTrait.name) Areas = HtmlGenWrapper.create_area_tag(shape='rect', coords=COORDS, href=HREF) - gifmap.areas.append(Areas) ### TODO + gifmap.append(Areas) ### TODO def drawLegendPanel(self, canvas, offset= (40, 120, 80, 10), zoom = 1): im_drawer = ImageDraw.Draw(canvas) @@ -1404,7 +1428,7 @@ class DisplayMappingResults(object): COORDS = "%d, %d, %d, %d" %(geneStartPix, geneYLocation, geneEndPix, (geneYLocation + self.EACH_GENE_HEIGHT)) # NL: 06-02-2011 Rob required to display NCBI info in a new window - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=COORDS, @@ -1578,7 +1602,7 @@ class DisplayMappingResults(object): COORDS = "%d, %d, %d, %d" %(geneStartPix, geneYLocation+ind*self.EACH_GENE_HEIGHT, geneEndPix+1, (geneYLocation + ind*self.EACH_GENE_HEIGHT)) TITLE = "Strain: %s, marker (%s) \n Position %2.3f Mb." % (samplelist[k], _chr[j].name, float(txStart)) HREF = '' - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=COORDS, @@ -1702,7 +1726,7 @@ class DisplayMappingResults(object): WEBQTL_HREF = "javascript:rangeView('%s', %f, %f)" % (self.selectedChr - 1, max(0, (calBase-webqtlZoomWidth))/1000000.0, (calBase+webqtlZoomWidth)/1000000.0) WEBQTL_TITLE = "Click to view this section of the genome in WebQTL" - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=WEBQTL_COORDS, @@ -1724,7 +1748,7 @@ class DisplayMappingResults(object): else: PHENOGEN_HREF = "https://phenogen.org/gene.jsp?speciesCB=Mm&auto=Y&geneTxt=chr%s:%d-%d&genomeVer=mm10" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) PHENOGEN_TITLE = "Click to view this section of the genome in PhenoGen" - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=PHENOGEN_COORDS, @@ -1745,7 +1769,7 @@ class DisplayMappingResults(object): else: UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) UCSC_TITLE = "Click to view this section of the genome in the UCSC Genome Browser" - gifmap.areas.append( + gifmap.append( HtmlGenWrapper.create_area_tag( shape='rect', coords=UCSC_COORDS, @@ -1767,7 +1791,7 @@ class DisplayMappingResults(object): else: ENSEMBL_HREF = "http://www.ensembl.org/Rattus_norvegicus/contigview?chr=%s&start=%d&end=%d" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) ENSEMBL_TITLE = "Click to view this section of the genome in the Ensembl Genome Browser" - gifmap.areas.append(HtmlGenWrapper.create_area_tag( + gifmap.append(HtmlGenWrapper.create_area_tag( shape='rect', coords=ENSEMBL_COORDS, href=ENSEMBL_HREF, @@ -2011,7 +2035,7 @@ class DisplayMappingResults(object): href=HREF, target="_blank", title="Locus : {}".format(Lname)) - gifmap.areas.append(Areas) + gifmap.append(Areas) ##piddle bug if j == 0: im_drawer.line( @@ -2227,8 +2251,8 @@ class DisplayMappingResults(object): shape='rect', coords=sig_coords, title=sig_title) - gifmap.areas.append(Areas1) - gifmap.areas.append(Areas2) + gifmap.append(Areas1) + gifmap.append(Areas2) start_pos_x += (chr_length_dist+self.GraphInterval)*plotXScale return start_pos_x @@ -2682,7 +2706,7 @@ class DisplayMappingResults(object): shape='rect', coords=COORDS, href=HREF) - gifmap.areas.append(Areas) + gifmap.append(Areas) startPosX += (self.ChrLengthDistList[i]+self.GraphInterval)*plotXScale return plotXScale @@ -2768,7 +2792,7 @@ class DisplayMappingResults(object): this_row = [] #container for the cells of each row selectCheck = HtmlGenWrapper.create_input_tag( - type="checkbox", + type_="checkbox", name="selectCheck", value=theGO["GeneSymbol"], Class="checkbox trait_checkbox") # checkbox for each row @@ -2887,7 +2911,7 @@ class DisplayMappingResults(object): for gIndex, theGO in enumerate(geneCol): this_row = [] # container for the cells of each row selectCheck = str(HtmlGenWrapper.create_input_tag( - type="checkbox", + type_="checkbox", name="selectCheck", Class="checkbox trait_checkbox")) # checkbox for each row -- cgit v1.2.3 From 6486bc57651f074245ae0bf786f9e92460d73fef Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 17:31:17 +0300 Subject: Fix failing tests that use python2-htmlgen * wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py: Fix tests. --- .../test_display_mapping_results.py | 91 ++++++++++++---------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py b/wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py index 6f791df1..8ae0f09f 100644 --- a/wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py +++ b/wqflask/tests/wqflask/marker_regression/test_display_mapping_results.py @@ -1,6 +1,6 @@ import unittest -from htmlgen import HTMLgen2 as HT +import htmlgen as HT from wqflask.marker_regression.display_mapping_results import ( DisplayMappingResults, HtmlGenWrapper @@ -26,9 +26,9 @@ class TestHtmlGenWrapper(unittest.TestCase): width="10", height="13", usemap="#webqtlmap")), - ("""""") + ("""random""") ) def test_create_form(self): @@ -37,7 +37,7 @@ class TestHtmlGenWrapper(unittest.TestCase): cgi="/testing/", enctype='multipart/form-data', name="formName", - submit=HT.Input(type='hidden') + submit=HtmlGenWrapper.create_input_tag(type_='hidden', name='Default_Name') ) test_image = HtmlGenWrapper.create_image_tag( src="test.png", @@ -49,10 +49,10 @@ class TestHtmlGenWrapper(unittest.TestCase): ) self.assertEqual( str(test_form).replace("\n", ""), - ("""
""")) + ("""
""")) hddn = { 'FormID': 'showDatabase', 'ProbeSetID': '_', @@ -62,21 +62,26 @@ class TestHtmlGenWrapper(unittest.TestCase): 'incparentsf1': 'ON' } for key in hddn.keys(): - test_form.append(HT.Input(name=key, value=hddn[key], - type='hidden')) + test_form.append( + HtmlGenWrapper.create_input_tag( + name=key, + value=hddn[key], + type_='hidden')) test_form.append(test_image) + self.assertEqual(str(test_form).replace("\n", ""), ( - """
""" - """""" - """""" - """""" - """""" - """""" - """""" - """random""" - """
""")) + """
""" + """""" + """""" + """""" + """""" + """""" + """""" + """""" + """random""" + """
""")) def test_create_paragraph(self): """Test HT.Paragraph method""" @@ -89,48 +94,48 @@ class TestHtmlGenWrapper(unittest.TestCase): ) self.assertEqual( str(test_p_element), - """

""" + """

""" ) - test_p_element.append(HT.BR()) + test_p_element.append(HtmlGenWrapper.create_br_tag()) test_p_element.append(par_text) self.assertEqual( str(test_p_element), - """


{}

""".format(par_text) + """


{}

""".format(par_text) ) def test_create_br_tag(self): """Test HT.BR() method""" self.assertEqual(str(HtmlGenWrapper.create_br_tag()), - "
") + "
") def test_create_input_tag(self): """Test HT.Input method""" self.assertEqual( str(HtmlGenWrapper.create_input_tag( - type="hidden", + type_="hidden", name="name", value="key", Class="trait trait_")).replace("\n", ""), - ("""""")) + ("""""")) def test_create_map_tag(self): """Test HT.Map method""" self.assertEqual(str(HtmlGenWrapper.create_map_tag( name="WebqTLImageMap")).replace("\n", ""), - """""") - gifmap = HtmlGenWrapper.create_map_tag(areas=[]) - gifmap.areas.append(HT.Area(shape="rect", - coords='1 2 3', href='#area1')) - gifmap.areas.append(HT.Area(shape="rect", - coords='1 2 3', href='#area2')) + """""") + gifmap = HtmlGenWrapper.create_map_tag(name="test") + gifmap.append(HtmlGenWrapper.create_area_tag(shape="rect", + coords='1 2 3', href='#area1')) + gifmap.append(HtmlGenWrapper.create_area_tag(shape="rect", + coords='1 2 3', href='#area2')) self.assertEqual( str(gifmap).replace("\n", ""), - ("""""" - """""" - """""" - """""")) + ("""""" + """""" + """""" + """""")) def test_create_area_tag(self): """Test HT.Area method""" @@ -140,12 +145,12 @@ class TestHtmlGenWrapper(unittest.TestCase): coords="1 2", href="http://test.com", title="Some Title")).replace("\n", ""), - ("""""")) + ("""""")) def test_create_link_tag(self): """Test HT.HREF method""" self.assertEqual( str(HtmlGenWrapper.create_link_tag( "www.test.com", "test", target="_blank")).replace("\n", ""), - """test""") + """test""") -- cgit v1.2.3 From 133db0e8dbc5a0812dc1f06402f2b788aecaef20 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 17:36:23 +0300 Subject: Replace calls to "basestring with "str"" See: --- wqflask/utility/helper_functions.py | 2 +- wqflask/wqflask/collect.py | 2 +- wqflask/wqflask/show_trait/show_trait.py | 2 +- wqflask/wqflask/snp_browser/snp_browser.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 9a4a235a..107c9ec6 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -34,7 +34,7 @@ def get_species_dataset_trait(self, start_vars): #self.genotype = self.dataset.group.genotype def get_trait_db_obs(self, trait_db_list): - if isinstance(trait_db_list, basestring): + if isinstance(trait_db_list, str): trait_db_list = trait_db_list.split(",") self.trait_list = [] diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index d77567f8..06c00930 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -38,7 +38,7 @@ from utility.logger import getLogger logger = getLogger(__name__) def process_traits(unprocessed_traits): - if isinstance(unprocessed_traits, basestring): + if isinstance(unprocessed_traits, str): unprocessed_traits = unprocessed_traits.split(",") traits = set() for trait in unprocessed_traits: diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 738aa28a..6a74cada 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -372,7 +372,7 @@ class ShowTrait(object): this_group = self.dataset.group.name # We're checking a string here! - assert isinstance(this_group, basestring), "We need a string type thing here" + assert isinstance(this_group, str), "We need a string type thing here" if this_group[:3] == 'BXD' and this_group != "BXD-Harvested": this_group = 'BXD' diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index b18bfc62..b3d26caf 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -477,7 +477,7 @@ class SnpBrowser(object): the_bases = [] for j, item in enumerate(allele_value_list): - if item and isinstance(item, basestring): + if item and isinstance(item, str): this_base = [str(item), base_color_dict[item]] else: this_base = "" @@ -612,7 +612,7 @@ class SnpBrowser(object): this_allele_list = [] for item in self.allele_list: - if item and isinstance(item, basestring) and (item.lower() not in this_allele_list) and (item != "-"): + if item and isinstance(item, str) and (item.lower() not in this_allele_list) and (item != "-"): this_allele_list.append(item.lower()) total_allele_count = len(this_allele_list) -- cgit v1.2.3 From 94f97127a9e451c7adb36613d090a612601b9d01 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 17:38:13 +0300 Subject: Rename "unicode" to "str" See: --- wqflask/base/data_set.py | 4 ++-- wqflask/base/trait.py | 10 +++++----- wqflask/wqflask/model.py | 6 +++--- wqflask/wqflask/user_manager.py | 2 +- wqflask/wqflask/user_session.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8151a29d..39a8a2ed 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1044,9 +1044,9 @@ class MrnaAssayDataSet(DataSet): # XZ, 12/08/2008: description # XZ, 06/05/2009: Rob asked to add probe target description - description_string = unicode( + description_string = str( str(this_trait.description).strip(codecs.BOM_UTF8), 'utf-8') - target_string = unicode( + target_string = str( str(this_trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') if len(description_string) > 1 and description_string != 'None': diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index a19b66f7..09c41fa7 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -490,9 +490,9 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): else: trait.description_display = "" - trait.abbreviation = unicode(str(trait.abbreviation).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") - trait.description_display = unicode(str(trait.description_display).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") - trait.authors = unicode(str(trait.authors).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + trait.abbreviation = str(str(trait.abbreviation).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + trait.description_display = str(str(trait.description_display).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + trait.authors = str(str(trait.authors).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") if not trait.year.isdigit(): trait.pubmed_text = "N/A" @@ -503,8 +503,8 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id if dataset.type == 'ProbeSet' and dataset.group: - description_string = unicode(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') - target_string = unicode(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') + description_string = str(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') + target_string = str(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') if len(description_string) > 1 and description_string != 'None': description_display = description_string diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py index 38117a8e..a135761c 100644 --- a/wqflask/wqflask/model.py +++ b/wqflask/wqflask/model.py @@ -18,7 +18,7 @@ from wqflask.database import Base, init_db class User(Base): __tablename__ = "user" - id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) + id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) email_address = Column(Unicode(50), unique=True, nullable=False) # Todo: Turn on strict mode for Mysql @@ -120,7 +120,7 @@ class User(Base): class Login(Base): __tablename__ = "login" - id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) + id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) user = Column(Unicode(36), ForeignKey('user.id')) timestamp = Column(DateTime(), default=lambda: datetime.datetime.utcnow()) ip_address = Column(Unicode(39)) @@ -138,7 +138,7 @@ class Login(Base): class UserCollection(Base): __tablename__ = "user_collection" - id = Column(Unicode(36), primary_key=True, default=lambda: unicode(uuid.uuid4())) + id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) user = Column(Unicode(36), ForeignKey('user.id')) # I'd prefer this to not have a length, but for the index below it needs one diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 56d1c084..3c41e2b8 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -252,7 +252,7 @@ class UserSession(object): def add_collection(self, collection_name, traits): """Add collection into ElasticSearch""" - collection_dict = {'id': unicode(uuid.uuid4()), + collection_dict = {'id': str(uuid.uuid4()), 'name': collection_name, 'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), 'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 3aa2c151..71c31c57 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -184,7 +184,7 @@ class UserSession(object): def add_collection(self, collection_name, traits): """Add collection into Redis""" - collection_dict = {'id': unicode(uuid.uuid4()), + collection_dict = {'id': str(uuid.uuid4()), 'name': collection_name, 'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), 'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), -- cgit v1.2.3 From 7717f0d3d4802d115a8b5ace4d379864ba5b1188 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 18:51:54 +0300 Subject: Replace "__cmp__" with "__eq__" magic method * wqflask/base/GeneralObject.py: "__cmp__" is no longer used in python3. See: --- wqflask/base/GeneralObject.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/base/GeneralObject.py b/wqflask/base/GeneralObject.py index 707569db..0122ee32 100644 --- a/wqflask/base/GeneralObject.py +++ b/wqflask/base/GeneralObject.py @@ -61,5 +61,6 @@ class GeneralObject: s += '%s = %s\n' % (key, self.__dict__[key]) return s - def __cmp__(self, other): - return len(list(self.__dict__.keys())).__cmp__(len(list(other.__dict__.keys()))) + def __eq__(self, other): + return (len(list(self.__dict__.keys())) == + len(list(other.__dict__.keys()))) -- cgit v1.2.3 From 303e4b71c2172da5be19c84d4be5a062329ac013 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 19:12:33 +0300 Subject: Remove "from __future__ import new_feature" statements See: --- test/requests/link_checker.py | 1 - test/requests/main_web_functionality.py | 22 ++++++++++++---------- test/requests/mapping_tests.py | 1 - test/requests/navigation_tests.py | 1 - test/requests/test-website.py | 2 +- webtests/browser_run.py | 4 +--- webtests/correlation_matrix_test.py | 2 -- webtests/correlation_test.py | 2 -- webtests/marker_regression_test.py | 2 -- webtests/show_trait_js_test.py | 2 -- webtests/test_runner.py | 4 +--- wqflask/base/data_set.py | 1 - wqflask/base/mrna_assay_tissue_data.py | 4 +--- wqflask/base/species.py | 4 +--- wqflask/base/trait.py | 2 -- wqflask/maintenance/convert_dryad_to_bimbam.py | 3 +-- wqflask/maintenance/convert_geno_to_bimbam.py | 3 +-- wqflask/maintenance/gen_select_dataset.py | 10 +--------- .../maintenance/generate_kinship_from_bimbam.py | 3 +-- .../maintenance/generate_probesetfreeze_file.py | 2 -- wqflask/maintenance/geno_to_json.py | 3 +-- wqflask/maintenance/get_group_samplelists.py | 2 -- wqflask/maintenance/print_benchmark.py | 4 +--- wqflask/maintenance/quantile_normalize.py | 7 +------ wqflask/maintenance/set_resource_defaults.py | 4 +--- wqflask/utility/Plot.py | 2 -- wqflask/utility/after.py | 4 +--- wqflask/utility/authentication_tools.py | 2 -- wqflask/utility/benchmark.py | 2 -- wqflask/utility/chunks.py | 2 -- wqflask/utility/db_tools.py | 2 -- wqflask/utility/gen_geno_ob.py | 4 +--- wqflask/utility/genofile_parser.py | 1 - wqflask/utility/helper_functions.py | 2 -- wqflask/utility/hmac.py | 4 +--- wqflask/utility/redis_tools.py | 4 +--- wqflask/utility/temp_data.py | 1 - wqflask/wqflask/__init__.py | 2 -- wqflask/wqflask/api/correlation.py | 4 +--- wqflask/wqflask/api/gen_menu.py | 2 -- wqflask/wqflask/api/mapping.py | 2 -- wqflask/wqflask/api/router.py | 4 +--- wqflask/wqflask/collect.py | 3 --- .../comparison_bar_chart/comparison_bar_chart.py | 1 - wqflask/wqflask/correlation/corr_scatter_plot.py | 4 +--- .../wqflask/correlation/correlation_functions.py | 4 +--- wqflask/wqflask/correlation/show_corr_results.py | 2 -- .../wqflask/correlation_matrix/show_corr_matrix.py | 2 -- wqflask/wqflask/do_search.py | 2 -- wqflask/wqflask/docs.py | 4 +--- wqflask/wqflask/export_traits.py | 4 +--- wqflask/wqflask/external_tools/send_to_bnw.py | 4 +--- .../wqflask/external_tools/send_to_geneweaver.py | 4 +--- .../wqflask/external_tools/send_to_webgestalt.py | 4 +--- wqflask/wqflask/group_manager.py | 5 +---- wqflask/wqflask/gsearch.py | 2 -- wqflask/wqflask/heatmap/heatmap.py | 2 -- wqflask/wqflask/interval_analyst/GeneUtil.py | 2 -- wqflask/wqflask/marker_regression/run_mapping.py | 4 +--- wqflask/wqflask/model.py | 4 +--- wqflask/wqflask/network_graph/network_graph.py | 2 -- wqflask/wqflask/news.py | 1 - wqflask/wqflask/parser.py | 2 -- wqflask/wqflask/resource_manager.py | 4 +--- wqflask/wqflask/search_results.py | 2 -- wqflask/wqflask/send_mail.py | 2 -- wqflask/wqflask/show_trait/SampleList.py | 2 -- wqflask/wqflask/show_trait/export_trait_data.py | 4 +--- wqflask/wqflask/show_trait/show_trait.py | 2 -- wqflask/wqflask/snp_browser/snp_browser.py | 2 -- wqflask/wqflask/submit_bnw.py | 4 +--- wqflask/wqflask/update_search_results.py | 2 -- wqflask/wqflask/user_login.py | 4 +--- wqflask/wqflask/user_manager.py | 2 -- wqflask/wqflask/user_session.py | 2 -- wqflask/wqflask/views.py | 2 -- 76 files changed, 45 insertions(+), 190 deletions(-) diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index 5e16a5cd..949a33c8 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import requests from lxml.html import parse diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py index d070dab9..78030307 100644 --- a/test/requests/main_web_functionality.py +++ b/test/requests/main_web_functionality.py @@ -1,9 +1,7 @@ -from __future__ import print_function -import re import requests from lxml.html import parse from link_checker import check_page -from requests.exceptions import ConnectionError + def check_home(url): doc = parse(url).getroot() @@ -11,20 +9,23 @@ def check_home(url): assert(search_button[0].value == "Search") print("OK") + def check_search_page(host): data = dict( - species="mouse" - , group="BXD" - , type="Hippocampus mRNA" - , dataset="HC_M2_0606_P" - , search_terms_or="" - , search_terms_and="MEAN=(15 16) LRS=(23 46)") + species="mouse", + group="BXD", + type="Hippocampus mRNA", + dataset="HC_M2_0606_P", + search_terms_or="", + search_terms_and="MEAN=(15 16) LRS=(23 46)") result = requests.get(host+"/search", params=data) found = result.text.find("records are shown below") assert(found >= 0) assert(result.status_code == 200) print("OK") - check_traits_page(host, "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P") + check_traits_page(host, ("/show_trait?trait_id=1435395_" + "s_at&dataset=HC_M2_0606_P")) + def check_traits_page(host, traits_url): doc = parse(host+traits_url).getroot() @@ -33,6 +34,7 @@ def check_traits_page(host, traits_url): print("OK") check_page(host, host+traits_url) + def check_main_web_functionality(args_obj, parser): print("") print("Checking main web functionality...") diff --git a/test/requests/mapping_tests.py b/test/requests/mapping_tests.py index 5748a2a3..19b22c21 100644 --- a/test/requests/mapping_tests.py +++ b/test/requests/mapping_tests.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import copy import json diff --git a/test/requests/navigation_tests.py b/test/requests/navigation_tests.py index eda27324..6b91c1fd 100644 --- a/test/requests/navigation_tests.py +++ b/test/requests/navigation_tests.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import requests from lxml.html import parse diff --git a/test/requests/test-website.py b/test/requests/test-website.py index f90d1843..8bfb47c2 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -3,7 +3,7 @@ # env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 # # Mostly to pick up the Guix GN2_PROFILE and python modules -from __future__ import print_function + import argparse from link_checker import check_links from link_checker import check_packaged_js_files diff --git a/webtests/browser_run.py b/webtests/browser_run.py index 2ec299c5..7ee540b7 100644 --- a/webtests/browser_run.py +++ b/webtests/browser_run.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - __all__ = ('sleep', 'testmod', 'test') from doctest import testmod @@ -71,4 +69,4 @@ class Test(object): -test = Test() \ No newline at end of file +test = Test() diff --git a/webtests/correlation_matrix_test.py b/webtests/correlation_matrix_test.py index 8529c265..97114890 100644 --- a/webtests/correlation_matrix_test.py +++ b/webtests/correlation_matrix_test.py @@ -65,8 +65,6 @@ text: 0.608\n71 """ -from __future__ import absolute_import, division, print_function - from browser_run import * testmod() diff --git a/webtests/correlation_test.py b/webtests/correlation_test.py index aad3a69f..311bb847 100644 --- a/webtests/correlation_test.py +++ b/webtests/correlation_test.py @@ -44,8 +44,6 @@ text: 1.000 """ -from __future__ import absolute_import, division, print_function - from browser_run import * testmod() diff --git a/webtests/marker_regression_test.py b/webtests/marker_regression_test.py index c4f76f53..9b4a4acb 100644 --- a/webtests/marker_regression_test.py +++ b/webtests/marker_regression_test.py @@ -48,8 +48,6 @@ text: 11.511 """ -from __future__ import absolute_import, division, print_function - from browser_run import * testmod() diff --git a/webtests/show_trait_js_test.py b/webtests/show_trait_js_test.py index 0fd2c16c..34ffd3b7 100644 --- a/webtests/show_trait_js_test.py +++ b/webtests/show_trait_js_test.py @@ -35,8 +35,6 @@ style: display: none; """ -from __future__ import absolute_import, division, print_function - from browser_run import * testmod() diff --git a/webtests/test_runner.py b/webtests/test_runner.py index ef6d0d69..b5b590a6 100644 --- a/webtests/test_runner.py +++ b/webtests/test_runner.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import unittest import doctest import glob @@ -25,4 +23,4 @@ def main(): runner.run(suite) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 39a8a2ed..8ac7a279 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -18,7 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division from db.call import fetchall, fetchone, fetch1 from utility.logger import getLogger from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index 6fec5dcd..33ce12bd 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import collections from flask import g @@ -92,4 +90,4 @@ class MrnaAssayTissueData(object): else: symbol_values_dict[result.Symbol.lower()].append(result.value) - return symbol_values_dict \ No newline at end of file + return symbol_values_dict diff --git a/wqflask/base/species.py b/wqflask/base/species.py index 6d99af65..e3f1bc4a 100644 --- a/wqflask/base/species.py +++ b/wqflask/base/species.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import collections from flask import Flask, g @@ -59,4 +57,4 @@ class Chromosomes(object): results = g.db.execute(query).fetchall() for item in results: - self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length) \ No newline at end of file + self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 09c41fa7..b20efd2a 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import os import string import resource diff --git a/wqflask/maintenance/convert_dryad_to_bimbam.py b/wqflask/maintenance/convert_dryad_to_bimbam.py index e833b395..12ce35e9 100644 --- a/wqflask/maintenance/convert_dryad_to_bimbam.py +++ b/wqflask/maintenance/convert_dryad_to_bimbam.py @@ -6,7 +6,6 @@ Convert data dryad files to a BIMBAM _geno and _snps file """ -from __future__ import print_function, division, absolute_import import sys sys.path.append("..") @@ -67,4 +66,4 @@ def convert_dryad_to_bimbam(filename): if __name__=="__main__": input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" - convert_dryad_to_bimbam(input_filename) \ No newline at end of file + convert_dryad_to_bimbam(input_filename) diff --git a/wqflask/maintenance/convert_geno_to_bimbam.py b/wqflask/maintenance/convert_geno_to_bimbam.py index 528b98cf..d49742f2 100644 --- a/wqflask/maintenance/convert_geno_to_bimbam.py +++ b/wqflask/maintenance/convert_geno_to_bimbam.py @@ -9,7 +9,6 @@ code """ -from __future__ import print_function, division, absolute_import import sys sys.path.append("..") import os @@ -187,4 +186,4 @@ if __name__=="__main__": #convertob = ConvertGenoFile("/home/zas1024/gene/genotype_files/genotypes/SRxSHRSPF2.geno", "/home/zas1024/gene/genotype_files/new_genotypes/SRxSHRSPF2.json") #convertob.convert() ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory) - #ConvertGenoFiles(Geno_Directory) \ No newline at end of file + #ConvertGenoFiles(Geno_Directory) diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index fd65a52a..544e2fd1 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -30,14 +30,6 @@ It needs to be run manually when database has been changed. Run it as # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import print_function, division - -#from flask import config -# -#cdict = {} -#config = config.Config(cdict).from_envvar('WQFLASK_SETTINGS') -#print("cdict is:", cdict) - import sys # NEW: Note we prepend the current path - otherwise a guix instance of GN2 may be used instead @@ -319,4 +311,4 @@ def _test_it(): if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() - main() \ No newline at end of file + main() diff --git a/wqflask/maintenance/generate_kinship_from_bimbam.py b/wqflask/maintenance/generate_kinship_from_bimbam.py index b53f5dda..60257b28 100644 --- a/wqflask/maintenance/generate_kinship_from_bimbam.py +++ b/wqflask/maintenance/generate_kinship_from_bimbam.py @@ -8,7 +8,6 @@ and uses GEMMA to generate their corresponding kinship/relatedness matrix file """ -from __future__ import print_function, division, absolute_import import sys sys.path.append("..") import os @@ -58,4 +57,4 @@ if __name__=="__main__": Bimbam_Directory = """/export/local/home/zas1024/genotype_files/genotype/bimbam/""" GenerateKinshipMatrices.process_all(Geno_Directory, Bimbam_Directory) - #./gemma -g /home/zas1024/genotype_files/genotype/bimbam/BXD_geno.txt -p /home/zas1024/genotype_files/genotype/bimbam/BXD_pheno.txt -gk 1 -o BXD \ No newline at end of file + #./gemma -g /home/zas1024/genotype_files/genotype/bimbam/BXD_geno.txt -p /home/zas1024/genotype_files/genotype/bimbam/BXD_pheno.txt -gk 1 -o BXD diff --git a/wqflask/maintenance/generate_probesetfreeze_file.py b/wqflask/maintenance/generate_probesetfreeze_file.py index 4231cc7c..b1e41e9a 100644 --- a/wqflask/maintenance/generate_probesetfreeze_file.py +++ b/wqflask/maintenance/generate_probesetfreeze_file.py @@ -1,7 +1,5 @@ #!/usr/bin/python -from __future__ import absolute_import, print_function, division - import sys # sys.path.insert(0, "..") - why? diff --git a/wqflask/maintenance/geno_to_json.py b/wqflask/maintenance/geno_to_json.py index 9579812a..7e7fd241 100644 --- a/wqflask/maintenance/geno_to_json.py +++ b/wqflask/maintenance/geno_to_json.py @@ -9,7 +9,6 @@ code """ -from __future__ import print_function, division, absolute_import import sys sys.path.append("..") import os @@ -194,4 +193,4 @@ if __name__=="__main__": ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory) #ConvertGenoFiles(Geno_Directory) - #process_csv(Input_File, Output_File) \ No newline at end of file + #process_csv(Input_File, Output_File) diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py index fb22898a..3f9d0278 100644 --- a/wqflask/maintenance/get_group_samplelists.py +++ b/wqflask/maintenance/get_group_samplelists.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import os import glob import gzip diff --git a/wqflask/maintenance/print_benchmark.py b/wqflask/maintenance/print_benchmark.py index ae327cf3..b24ce4f1 100644 --- a/wqflask/maintenance/print_benchmark.py +++ b/wqflask/maintenance/print_benchmark.py @@ -1,7 +1,5 @@ #!/usr/bin/python -from __future__ import absolute_import, print_function, division - import time from pprint import pformat as pf @@ -40,4 +38,4 @@ def new_main(): print(pf(TheCounter.Counters)) if __name__ == '__main__': - new_main() \ No newline at end of file + new_main() diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 4d6e03bf..701b2b50 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -1,10 +1,5 @@ -from __future__ import absolute_import, print_function, division - import sys sys.path.insert(0, './') - - - import MySQLdb import urllib.parse @@ -126,4 +121,4 @@ if __name__ == '__main__': } ) - print(response) \ No newline at end of file + print(response) diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index abd5416c..4177c124 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -16,8 +16,6 @@ To run: """ -from __future__ import print_function, division - import sys import json @@ -163,4 +161,4 @@ def main(): if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() - main() \ No newline at end of file + main() diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 48a5c7ba..61f408d2 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -24,8 +24,6 @@ # # Last updated by GeneNetwork Core Team 2010/10/20 -from __future__ import print_function - from PIL import ImageColor from PIL import ImageDraw from PIL import ImageFont diff --git a/wqflask/utility/after.py b/wqflask/utility/after.py index b628a0a4..06091ecb 100644 --- a/wqflask/utility/after.py +++ b/wqflask/utility/after.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - """ See: http://flask.pocoo.org/docs/patterns/deferredcallbacks/#deferred-callbacks @@ -13,4 +11,4 @@ def after_this_request(f): if not hasattr(g, 'after_request_callbacks'): g.after_request_callbacks = [] g.after_request_callbacks.append(f) - return f \ No newline at end of file + return f diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index bc03eb55..0e499180 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import json import requests diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py index 221e5151..ea5a0ab6 100644 --- a/wqflask/utility/benchmark.py +++ b/wqflask/utility/benchmark.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import collections import inspect import time diff --git a/wqflask/utility/chunks.py b/wqflask/utility/chunks.py index d91b9bf4..9a7db102 100644 --- a/wqflask/utility/chunks.py +++ b/wqflask/utility/chunks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import math import time diff --git a/wqflask/utility/db_tools.py b/wqflask/utility/db_tools.py index 4034f39c..92bde505 100644 --- a/wqflask/utility/db_tools.py +++ b/wqflask/utility/db_tools.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from MySQLdb import escape_string as escape def create_in_clause(items): diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index ae42f834..81085ffe 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import utility.logger logger = utility.logger.getLogger(__name__ ) @@ -178,4 +176,4 @@ class Locus(object): if allele in list(geno_table.keys()): self.genotype.append(geno_table[allele]) else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown - self.genotype.append("U") \ No newline at end of file + self.genotype.append("U") diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py index af306731..0b736176 100644 --- a/wqflask/utility/genofile_parser.py +++ b/wqflask/utility/genofile_parser.py @@ -1,7 +1,6 @@ # CTL analysis for GN2 # Author / Maintainer: Danny Arends -from __future__ import print_function, division, absolute_import import sys import os import glob diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 107c9ec6..7eb7f013 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from base import data_set from base.trait import create_trait from base.species import TheSpecies diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py index b08be97e..73e28790 100644 --- a/wqflask/utility/hmac.py +++ b/wqflask/utility/hmac.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import hmac import hashlib @@ -37,4 +35,4 @@ def url_for_hmac(endpoint, **values): return url + combiner + "hm=" + hm app.jinja_env.globals.update(url_for_hmac=url_for_hmac, - data_hmac=data_hmac) \ No newline at end of file + data_hmac=data_hmac) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index 1377a564..13ac5cfe 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import uuid import simplejson as json import datetime @@ -306,4 +304,4 @@ def change_resource_owner(resource_id, new_owner_id): the_resource['owner_id'] = new_owner_id Redis.delete("resource") - Redis.hset("resources", resource_id, json.dumps(the_resource)) \ No newline at end of file + Redis.hset("resources", resource_id, json.dumps(the_resource)) diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 2f2726c6..4144ae00 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -1,4 +1,3 @@ -from __future__ import print_function, division, absolute_import from redis import Redis import simplejson as json diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index d729aef5..e73f833f 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import sys import jinja2 diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index eb05645e..7da13121 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import collections import scipy @@ -234,4 +232,4 @@ def init_corr_params(start_vars): 'return_count' : return_count } - return corr_params \ No newline at end of file + return corr_params diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index 41966f78..1dcafe1f 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division - from flask import g diff --git a/wqflask/wqflask/api/mapping.py b/wqflask/wqflask/api/mapping.py index 92c27c9b..d59a69df 100644 --- a/wqflask/wqflask/api/mapping.py +++ b/wqflask/wqflask/api/mapping.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import string from base import data_set diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index 5f9b81b8..acf7ce4a 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -1,7 +1,5 @@ # GN2 API -from __future__ import absolute_import, division, print_function - import os, io, csv, json, datetime, requests, yaml import zlib from zipfile import ZipFile, ZIP_DEFLATED @@ -966,4 +964,4 @@ def get_group_id(group_name): if group_id: return group_id[0] else: - return None \ No newline at end of file + return None diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 06c00930..15383603 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -1,6 +1,3 @@ -from __future__ import print_function, division, absolute_import - - import os import hashlib import datetime diff --git a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py index 16832621..92de6073 100644 --- a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py +++ b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py @@ -18,7 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division from pprint import pformat as pf from base.trait import create_trait diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index 57a8d85f..929cd2c9 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import math from flask import g @@ -130,4 +128,4 @@ def get_intercept_coords(slope, intercept, x_range, y_range): intercept_coords.append([x1, y1]) intercept_coords.append([x2, y2]) - return intercept_coords \ No newline at end of file + return intercept_coords diff --git a/wqflask/wqflask/correlation/correlation_functions.py b/wqflask/wqflask/correlation/correlation_functions.py index abaa212f..b883e361 100644 --- a/wqflask/wqflask/correlation/correlation_functions.py +++ b/wqflask/wqflask/correlation/correlation_functions.py @@ -24,8 +24,6 @@ # # Last updated by NL 2011/03/23 -from __future__ import absolute_import, print_function, division - import math import rpy2.robjects import string @@ -114,4 +112,4 @@ def get_trait_symbol_and_tissue_values(symbol_list=None): tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list) if len(tissue_data.gene_symbols): - return tissue_data.get_symbol_values_pairs() \ No newline at end of file + return tissue_data.get_symbol_values_pairs() diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 5ced30b6..e5b87c6a 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - import collections import json import scipy diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index bd5aca1f..7b4bda31 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - import datetime import math import numpy as np diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index cc9c1860..de8e1e78 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division - import string import requests import json diff --git a/wqflask/wqflask/docs.py b/wqflask/wqflask/docs.py index 78407e22..8628b81d 100644 --- a/wqflask/wqflask/docs.py +++ b/wqflask/wqflask/docs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import codecs from flask import g @@ -42,4 +40,4 @@ def update_text(start_vars): sql = "UPDATE Docs SET content='{0}' WHERE entry='{1}';".format(content, start_vars['entry_type']) g.db.execute(sql) except: - pass \ No newline at end of file + pass diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 2f4e9aac..3a886537 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division - import csv import xlsxwriter import io @@ -140,4 +138,4 @@ def sort_traits_by_group(trait_list=[]): traits_by_group[trait.dataset.group.name].append(trait) - return traits_by_group \ No newline at end of file + return traits_by_group diff --git a/wqflask/wqflask/external_tools/send_to_bnw.py b/wqflask/wqflask/external_tools/send_to_bnw.py index 68efd10d..efa17f05 100644 --- a/wqflask/wqflask/external_tools/send_to_bnw.py +++ b/wqflask/wqflask/external_tools/send_to_bnw.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - from base.trait import GeneralTrait from utility import helper_functions, corr_result_helpers @@ -69,4 +67,4 @@ class SendToBNW(object): if has_none: continue self.form_value += ",".join(str(cell) for cell in row) - self.form_value += ";" \ No newline at end of file + self.form_value += ";" diff --git a/wqflask/wqflask/external_tools/send_to_geneweaver.py b/wqflask/wqflask/external_tools/send_to_geneweaver.py index 9844bab4..93164233 100644 --- a/wqflask/wqflask/external_tools/send_to_geneweaver.py +++ b/wqflask/wqflask/external_tools/send_to_geneweaver.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - import string from flask import Flask, g @@ -109,4 +107,4 @@ def test_chip(trait_list): chip_name = '%s_NA' % result[0] return chip_name - return chip_name \ No newline at end of file + return chip_name diff --git a/wqflask/wqflask/external_tools/send_to_webgestalt.py b/wqflask/wqflask/external_tools/send_to_webgestalt.py index 30ca024f..b255ba95 100644 --- a/wqflask/wqflask/external_tools/send_to_webgestalt.py +++ b/wqflask/wqflask/external_tools/send_to_webgestalt.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - import string from flask import Flask, g @@ -123,4 +121,4 @@ def gen_gene_id_list(trait_list): trait_name_list.append(trait.name) retrieve_trait_info(trait, trait.dataset) gene_id_list.append(str(trait.geneid)) - return trait_name_list, gene_id_list \ No newline at end of file + return trait_name_list, gene_id_list diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index 24848ed8..e6079c3e 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -1,6 +1,3 @@ - -from __future__ import print_function, division, absolute_import - from flask import (Flask, g, render_template, url_for, request, make_response, redirect, flash) @@ -142,4 +139,4 @@ def send_group_invites(group_id, user_email_list = [], user_type="members"): else: send_verification_email(user_details, template_name = "email/group_verification.txt", key_prefix = "verification_code", subject = "You've been invited to join a GeneNetwork user group") -#@app.route() \ No newline at end of file +#@app.route() diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index c65a1415..6d797a29 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import json from flask import Flask, g diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 1f1cdb90..cca5a4fc 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import string import os import random diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index a39e5d0f..17c8ccbf 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import string from flask import Flask, g diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 67512bc6..7ae84b16 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from base.trait import GeneralTrait from base import data_set #import create_dataset @@ -711,4 +709,4 @@ def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): list_to_numbers = [d[x] for x in perm_strata_strings] perm_strata = list_to_numbers - return perm_strata \ No newline at end of file + return perm_strata diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py index a135761c..772f74e4 100644 --- a/wqflask/wqflask/model.py +++ b/wqflask/wqflask/model.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import uuid import datetime @@ -168,4 +166,4 @@ def display_collapsible(number): def user_uuid(): """Unique cookie for a user""" - user_uuid = request.cookies.get('user_uuid') \ No newline at end of file + user_uuid = request.cookies.get('user_uuid') diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index 2a4c4a93..70e5dd58 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -18,8 +18,6 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -from __future__ import absolute_import, print_function, division - import scipy import simplejson as json from pprint import pformat as pf diff --git a/wqflask/wqflask/news.py b/wqflask/wqflask/news.py index 20319756..0675ec4b 100644 --- a/wqflask/wqflask/news.py +++ b/wqflask/wqflask/news.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import, print_function, division from flask import g class News(object): diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index 1ca5ecff..472dd30e 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -17,8 +17,6 @@ be acceptable.] """ -from __future__ import print_function, division - import re from pprint import pformat as pf diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 6b3e00fb..e883d5da 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import json from flask import (Flask, g, render_template, url_for, request, make_response, @@ -131,4 +129,4 @@ def get_group_names(group_masks): this_mask['name'] = group_name group_masks_with_names[group_id] = this_mask - return group_masks_with_names \ No newline at end of file + return group_masks_with_names diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index c07a7670..f6c677a8 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import re import uuid from math import * diff --git a/wqflask/wqflask/send_mail.py b/wqflask/wqflask/send_mail.py index bf5d0dd8..86e8a558 100644 --- a/wqflask/wqflask/send_mail.py +++ b/wqflask/wqflask/send_mail.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import datetime import time diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index f17e825e..6fcf7cec 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from flask import Flask, g from base import webqtlCaseData diff --git a/wqflask/wqflask/show_trait/export_trait_data.py b/wqflask/wqflask/show_trait/export_trait_data.py index 68c3ad7d..2d76b935 100644 --- a/wqflask/wqflask/show_trait/export_trait_data.py +++ b/wqflask/wqflask/show_trait/export_trait_data.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division - import simplejson as json from pprint import pformat as pf @@ -71,4 +69,4 @@ def cmp_samples(a, b): else: return 1 else: - return -1 \ No newline at end of file + return -1 diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 6a74cada..e93b0289 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import string import os import datetime diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index b3d26caf..0db7e1fe 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from flask import Flask, g, url_for import string diff --git a/wqflask/wqflask/submit_bnw.py b/wqflask/wqflask/submit_bnw.py index 59e60dfd..a0e84c8c 100644 --- a/wqflask/wqflask/submit_bnw.py +++ b/wqflask/wqflask/submit_bnw.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - from base.trait import GeneralTrait from base import data_set from utility import helper_functions @@ -8,4 +6,4 @@ import utility.logger logger = utility.logger.getLogger(__name__ ) def get_bnw_input(start_vars): - logger.debug("BNW VARS:", start_vars) \ No newline at end of file + logger.debug("BNW VARS:", start_vars) diff --git a/wqflask/wqflask/update_search_results.py b/wqflask/wqflask/update_search_results.py index 68bea9d6..672f95b1 100644 --- a/wqflask/wqflask/update_search_results.py +++ b/wqflask/wqflask/update_search_results.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function, division - import json from flask import Flask, g diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index 04672b45..10cb7319 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import os import hashlib import datetime @@ -470,4 +468,4 @@ def register(): @app.errorhandler(401) def unauthorized(error): - return redirect(url_for('login')) \ No newline at end of file + return redirect(url_for('login')) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 3c41e2b8..24191a15 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import os import hashlib import datetime diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 71c31c57..c1f38396 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import datetime import time import uuid diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 83d5202e..b0489e64 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -2,8 +2,6 @@ # # Main routing table for GN2 -from __future__ import absolute_import, division, print_function - import traceback # for error page import os # for error gifs import random # for random error gif -- cgit v1.2.3 From 24bed3fc4f3a69c24074e9c5829a46b66702cbbf Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 22:01:03 +0300 Subject: Remove debug statement * wqflask/wqflask/views.py(search_page): "debugf" raises an error since the first arg is expected to be an int --- wqflask/wqflask/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b0489e64..0224da4a 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -206,8 +206,6 @@ def search_page(): result = the_search.__dict__ valid_search = result['search_term_exists'] - logger.debugf("result", result) - if USE_REDIS and valid_search: Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) Redis.expire(key, 60*60) -- cgit v1.2.3 From 768de71f09b116cad0f42c792c22a7dc410ea2f4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Aug 2020 22:23:40 +0300 Subject: Replace tabs with spaces * scripts/maintenance/readProbeSetMean_v7.py: Run python-autopep8 on this file. --- scripts/maintenance/readProbeSetMean_v7.py | 250 ++++++++++++++--------------- 1 file changed, 125 insertions(+), 125 deletions(-) diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index 59a51cf9..a540796a 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -9,19 +9,17 @@ import sys import MySQLdb import getpass import time -#import pdb -#pdb.set_trace() ######################################################################## def translateAlias(str): - if str == "B6": - return "C57BL/6J" - elif str == "D2": - return "DBA/2J" - else: - return str + if str == "B6": + return "C57BL/6J" + elif str == "D2": + return "DBA/2J" + else: + return str ######################################################################## # @@ -29,23 +27,25 @@ def translateAlias(str): # ######################################################################## + dataStart = 1 -GeneChipId = int( input("Enter GeneChipId:") ) -ProbeSetFreezeId = int( input("Enter ProbeSetFreezeId:") ) +GeneChipId = int(input("Enter GeneChipId:")) +ProbeSetFreezeId = int(input("Enter ProbeSetFreezeId:")) input_file_name = input("Enter file name with suffix:") fp = open("%s" % input_file_name, 'rb') try: - passwd = getpass.getpass('Please enter mysql password here : ') - con = MySQLdb.Connect(db='db_webqtl', host='localhost', user='username', passwd=passwd) + passwd = getpass.getpass('Please enter mysql password here : ') + con = MySQLdb.Connect(db='db_webqtl', host='localhost', + user='username', passwd=passwd) - db = con.cursor() - print("You have successfully connected to mysql.\n") + db = con.cursor() + print("You have successfully connected to mysql.\n") except: - print("You entered incorrect password.\n") - sys.exit(0) + print("You entered incorrect password.\n") + sys.exit(0) time0 = time.time() @@ -65,25 +65,25 @@ header = list(map(string.strip, header)) nfield = len(header) line = fp.readline() -kj=0 +kj = 0 while line: - line2 = string.split(string.strip(line), '\t') - line2 = list(map(string.strip, line2)) - if len(line2) != nfield: - print(("Error : " + line)) - isCont = 0 + line2 = string.split(string.strip(line), '\t') + line2 = list(map(string.strip, line2)) + if len(line2) != nfield: + print(("Error : " + line)) + isCont = 0 - GeneList.append(line2[0]) - line = fp.readline() + GeneList.append(line2[0]) + line = fp.readline() - kj+=1 - if kj%100000 == 0: - print(('checked ', kj, ' lines')) + kj += 1 + if kj % 100000 == 0: + print(('checked ', kj, ' lines')) GeneList = sorted(map(string.lower, GeneList)) - -if isCont==0: - sys.exit(0) + +if isCont == 0: + sys.exit(0) print(('used ', time.time()-time0, ' seconds')) @@ -104,15 +104,15 @@ header = list(map(translateAlias, header)) header = header[dataStart:] Ids = [] for item in header: - try: - db.execute('select Id from Strain where Name = "%s"' % item) - Ids.append(db.fetchall()[0][0]) - except: - print((item, 'does not exist, check the if the strain name is correct')) - isCont=0 + try: + db.execute('select Id from Strain where Name = "%s"' % item) + Ids.append(db.fetchall()[0][0]) + except: + print((item, 'does not exist, check the if the strain name is correct')) + isCont = 0 -if isCont==0: - sys.exit(0) +if isCont == 0: + sys.exit(0) print(('used ', time.time()-time0, ' seconds')) @@ -130,59 +130,60 @@ line2 = string.split(string.strip(line), '\t') line2 = list(map(string.strip, line2)) PId = line2[0] -db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % (PId, GeneChipId) ) +db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % + (PId, GeneChipId)) results = db.fetchall() IdStr = 'TargetId' -if len(results)>0: - IdStr = 'Name' +if len(results) > 0: + IdStr = 'Name' ##---- get Name/TargetId list from database ----## -db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % (IdStr, GeneChipId, IdStr)) +db.execute('select distinct(%s) from ProbeSet where ChipId=%d order by %s' % ( + IdStr, GeneChipId, IdStr)) results = db.fetchall() - + Names = [] for item in results: - Names.append(item[0]) - + Names.append(item[0]) + print(Names) Names = sorted(map(string.lower, Names)) - ##---- compare genelist with names ----## -x=y=0 -x1=-1 -GeneList2=[] -while xNames[y]: - y += 1 - - if x%100000==0: - print(('check Name, checked %d lines'%x)) - -while x Names[y]: + y += 1 + + if x % 100000 == 0: + print(('check Name, checked %d lines' % x)) + +while x < len(GeneList): + GeneList2.append(GeneList[x]) + x += 1 + +isCont = 1 ferror = open("ProbeSetError.txt", "wb") for item in GeneList2: - ferror.write(item + " doesn't exist \n") - print((item, " doesn't exist, check if the ProbeSet name is correct")) - isCont = 0 - -if isCont==0: - sys.exit(0) + ferror.write(item + " doesn't exist \n") + print((item, " doesn't exist, check if the ProbeSet name is correct")) + isCont = 0 + +if isCont == 0: + sys.exit(0) print(('used ', time.time()-time0, ' seconds')) @@ -195,11 +196,12 @@ print('getting ProbeSet/Id') #---- get Name/Id map ----# -db.execute('select %s, Id from ProbeSet where ChipId=%d order by %s' % (IdStr, GeneChipId, IdStr)) +db.execute('select %s, Id from ProbeSet where ChipId=%d order by %s' % + (IdStr, GeneChipId, IdStr)) results = db.fetchall() NameIds = {} for item in results: - NameIds[item[0]] = item[1] + NameIds[item[0]] = item[1] print(('used ', time.time()-time0, ' seconds')) @@ -220,53 +222,51 @@ kj = 0 values1 = [] values2 = [] while line: - line2 = string.split(string.strip(line), '\t') - line2 = list(map(string.strip, line2)) - PId = line2[0] - recordId = NameIds[PId] - - maxDataId += 1 - datasorig = line2[dataStart:] - - ###### Data Table items ###### - i=0 - for item in datasorig: - try: - values1.append('(%d,%d,%s)' % (maxDataId, Ids[i], float(item))) - except: - pass - i += 1 - - values2.append("(%d,%d,%d)" % (ProbeSetFreezeId, recordId, maxDataId)) - - - ##---- insert into table ----## - kj += 1 - if kj % 100 == 0: - cmd = ','.join(values1) - cmd = 'insert into ProbeSetData values %s' % cmd - db.execute(cmd) - - cmd = ','.join(values2) - cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd - db.execute(cmd) - - values1=[] - values2=[] - print(('Inserted ', kj, ' lines')) - print(('used ', time.time()-time0, ' seconds')) - - line = fp.readline() - - - -if len(values1)>0: - cmd = ','.join(values1) - cmd = 'insert into ProbeSetData values %s' % cmd - db.execute(cmd) - - cmd = ','.join(values2) - cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd - db.execute(cmd) + line2 = string.split(string.strip(line), '\t') + line2 = list(map(string.strip, line2)) + PId = line2[0] + recordId = NameIds[PId] + + maxDataId += 1 + datasorig = line2[dataStart:] + + ###### Data Table items ###### + i = 0 + for item in datasorig: + try: + values1.append('(%d,%d,%s)' % (maxDataId, Ids[i], float(item))) + except: + pass + i += 1 + + values2.append("(%d,%d,%d)" % (ProbeSetFreezeId, recordId, maxDataId)) + + ##---- insert into table ----## + kj += 1 + if kj % 100 == 0: + cmd = ','.join(values1) + cmd = 'insert into ProbeSetData values %s' % cmd + db.execute(cmd) + + cmd = ','.join(values2) + cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd + db.execute(cmd) + + values1 = [] + values2 = [] + print(('Inserted ', kj, ' lines')) + print(('used ', time.time()-time0, ' seconds')) + + line = fp.readline() + + +if len(values1) > 0: + cmd = ','.join(values1) + cmd = 'insert into ProbeSetData values %s' % cmd + db.execute(cmd) + + cmd = ','.join(values2) + cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd + db.execute(cmd) con.close() -- cgit v1.2.3 From 357ca458695fbc60c97de3d1cdf89034a8722bc5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:18:11 +0300 Subject: Replace "string.split" & "string.join" with python's inbuilt methods --- scripts/maintenance/readProbeSetMean_v7.py | 20 ++++++++++---------- scripts/maintenance/readProbeSetSE_v7.py | 14 +++++++------- wqflask/base/data_set.py | 2 +- wqflask/base/trait.py | 14 +++++++------- wqflask/utility/webqtlUtil.py | 4 ++-- wqflask/wqflask/external_tools/send_to_geneweaver.py | 2 +- wqflask/wqflask/external_tools/send_to_webgestalt.py | 2 +- wqflask/wqflask/interval_analyst/GeneUtil.py | 4 ++-- .../marker_regression/display_mapping_results.py | 2 +- wqflask/wqflask/marker_regression/plink_mapping.py | 6 +++--- wqflask/wqflask/show_trait/show_trait.py | 4 ++-- wqflask/wqflask/snp_browser/snp_browser.py | 12 ++++++------ 12 files changed, 43 insertions(+), 43 deletions(-) diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index a540796a..43f084f4 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -60,15 +60,15 @@ print('Checking if each line have same number of members') GeneList = [] isCont = 1 header = fp.readline() -header = string.split(string.strip(header), '\t') -header = list(map(string.strip, header)) +header = header.strip().split('\t') +header = [x.strip() for x in header] nfield = len(header) line = fp.readline() kj = 0 while line: - line2 = string.split(string.strip(line), '\t') - line2 = list(map(string.strip, line2)) + line2 = line.strip().split('\t') + line2 = [x.strip() for x in line2] if len(line2) != nfield: print(("Error : " + line)) isCont = 0 @@ -98,8 +98,8 @@ print('Checking if each strain exist in database') isCont = 1 fp.seek(0) header = fp.readline() -header = string.split(string.strip(header), '\t') -header = list(map(string.strip, header)) +header = header.strip().split('\t') +header = [x.strip() for x in header] header = list(map(translateAlias, header)) header = header[dataStart:] Ids = [] @@ -126,8 +126,8 @@ print('Check if each ProbeSet exist in database') ##---- find PID is name or target ----## line = fp.readline() line = fp.readline() -line2 = string.split(string.strip(line), '\t') -line2 = list(map(string.strip, line2)) +line2 = line.strip().split('\t') +line2 = [x.strip() for x in line2] PId = line2[0] db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % @@ -222,8 +222,8 @@ kj = 0 values1 = [] values2 = [] while line: - line2 = string.split(string.strip(line), '\t') - line2 = list(map(string.strip, line2)) + line2 = line.strip().split('\t') + line2 = [x.strip() for x in line2] PId = line2[0] recordId = NameIds[PId] diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index 20a846a4..edd9e7b0 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -71,14 +71,14 @@ print('Checking if each line have same number of members') GeneList = [] isCont = 1 header = fp.readline() -header = string.split(string.strip(header), '\t') +header = header.strip().split('\t') header = list(map(string.strip, header)) nfield = len(header) line = fp.readline() kj = 0 while line: - line2 = string.split(string.strip(line), '\t') + line2 = line.strip().split('\t') line2 = list(map(string.strip, line2)) if len(line2) != nfield: isCont = 0 @@ -109,7 +109,7 @@ print('Checking if each strain exist in database') isCont = 1 fp.seek(0) header = fp.readline() -header = string.split(string.strip(header), '\t') +header = header.strip().split('\t') header = list(map(string.strip, header)) header = list(map(translateAlias, header)) header = header[dataStart:] @@ -137,8 +137,8 @@ print('Check if each ProbeSet exist in database') ##---- find PID is name or target ----## line = fp.readline() line = fp.readline() -line2 = string.split(string.strip(line), '\t') -line2 = list(map(string.strip, line2)) +line2 = line.strip().split('\t') +line2 = [x.strip() for x in line2] PId = line2[0] db.execute('select Id from ProbeSet where Name="%s" and ChipId=%d' % @@ -217,8 +217,8 @@ line = fp.readline() kj = 0 while line: - line2 = string.split(string.strip(line), '\t') - line2 = list(map(string.strip, line2)) + line2 = line.strip().split('\t') + line2 = [x.strip() for x in line2] CellId = line2[0] if CellId not in ProbeNameId: diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8ac7a279..ce017fb4 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -697,7 +697,7 @@ class DataSet(object): else: query = "SELECT {}.Name,".format(escape(dataset_type)) data_start_pos = 1 - query += string.join(temp, ', ') + query += ', '.join(temp) query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(dataset_type, self.type, self.type)) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index b20efd2a..2fd5d725 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -150,8 +150,8 @@ class GeneralTrait(object): alias = 'Not available' if getattr(self, "alias", None): - alias = string.replace(self.alias, ";", " ") - alias = string.join(string.split(alias), ", ") + alias = self.alias.replace(";", " ") + alias = ", ".join(alias.split()) return alias @@ -437,7 +437,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): #XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name # to avoid the problem of same marker name from different species. elif dataset.type == 'Geno': - display_fields_string = string.join(dataset.display_fields, ',Geno.') + display_fields_string = ',Geno.'.join(dataset.display_fields) display_fields_string = 'Geno.' + display_fields_string query = """ SELECT %s @@ -456,8 +456,8 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): query = """SELECT %s FROM %s WHERE Name = %s""" logger.sql(query) trait_info = g.db.execute(query, - (string.join(dataset.display_fields, ','), - dataset.type, trait.name)).fetchone() + ','.join(dataset.display_fields), + dataset.type, trait.name).fetchone() if trait_info: trait.haveinfo = True @@ -501,8 +501,8 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id if dataset.type == 'ProbeSet' and dataset.group: - description_string = str(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') - target_string = str(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') + description_string = trait.description + target_string = trait.probe_target_description if len(description_string) > 1 and description_string != 'None': description_display = description_string diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index d4979011..5681fadf 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -107,7 +107,7 @@ def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users) if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']: access_to_confidential_phenotype_trait = 1 else: - AuthorizedUsersList=list(map(string.strip, string.split(authorized_users, ','))) - if AuthorizedUsersList.__contains__(userName): + AuthorizedUsersList=[x.strip() for x in authorized_users.split(',')] + if userName in AuthorizedUsersList: access_to_confidential_phenotype_trait = 1 return access_to_confidential_phenotype_trait diff --git a/wqflask/wqflask/external_tools/send_to_geneweaver.py b/wqflask/wqflask/external_tools/send_to_geneweaver.py index 93164233..4c958a88 100644 --- a/wqflask/wqflask/external_tools/send_to_geneweaver.py +++ b/wqflask/wqflask/external_tools/send_to_geneweaver.py @@ -55,7 +55,7 @@ class SendToGeneWeaver(object): 'client': "genenetwork", 'species': species_name, 'idtype': self.chip_name, - 'list': string.join(trait_name_list, ","), + 'list': ",".join(trait_name_list), } def get_trait_name_list(trait_list): diff --git a/wqflask/wqflask/external_tools/send_to_webgestalt.py b/wqflask/wqflask/external_tools/send_to_webgestalt.py index b255ba95..2f068792 100644 --- a/wqflask/wqflask/external_tools/send_to_webgestalt.py +++ b/wqflask/wqflask/external_tools/send_to_webgestalt.py @@ -47,7 +47,7 @@ class SendToWebGestalt(object): id_type = "entrezgene" self.hidden_vars = { - 'gene_list' : string.join(gene_id_list, "\n"), + 'gene_list' : "\n".join(gene_id_list), 'id_type' : "entrezgene", 'ref_set' : "genome", 'enriched_database_category' : "geneontology", diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index 17c8ccbf..d0dd7aea 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -31,7 +31,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): Chromosome = '%s' AND ((TxStart > %f and TxStart <= %f) OR (TxEnd > %f and TxEnd <= %f)) ORDER BY txStart - """ % (string.join(fetchFields, ", "), + """ % (", ".join(fetchFields), speciesId, chrName, startMb, endMb, startMb, endMb)).fetchall() @@ -66,7 +66,7 @@ def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): othSpec, othSpecId = item newdict2 = {} - resultsOther = g.db.execute("SELECT %s FROM GeneList WHERE SpeciesId = %d AND geneSymbol= '%s' LIMIT 1" % (string.join(fetchFields, ", "), + resultsOther = g.db.execute("SELECT %s FROM GeneList WHERE SpeciesId = %d AND geneSymbol= '%s' LIMIT 1" % (", ".join(fetchFields), othSpecId, newdict["GeneSymbol"])).fetchone() diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index ccdafa14..dfaa1562 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -454,7 +454,7 @@ class DisplayMappingResults(object): Chr_Length.Name in (%s) Order by Chr_Length.OrderId - """ % (self.dataset.group.name, string.join(["'%s'" % X[0] for X in self.ChrList[1:]], ", "))) + """ % (self.dataset.group.name, ", ".join(["'%s'" % X[0] for X in self.ChrList[1:]]))) self.ChrLengthMbList = [x[0]/1000000.0 for x in self.ChrLengthMbList] self.ChrLengthMbSum = reduce(lambda x, y:x+y, self.ChrLengthMbList, 0.0) diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index 2f282adc..6c38c34f 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -83,7 +83,7 @@ def get_samples_from_ped_file(dataset): sample_list=[] while line: - lineList = string.split(string.strip(line), '\t') + lineList = line.strip().split('\t') lineList = list(map(string.strip, lineList)) sample_name = lineList[0] @@ -155,8 +155,8 @@ def parse_plink_output(output_filename, species): # output: lineList list ####################################################### def build_line_list(line=None): - line_list = string.split(string.strip(line), ' ')# irregular number of whitespaces between columns + line_list = line.strip().split(' ')# irregular number of whitespaces between columns line_list = [item for item in line_list if item !=''] line_list = list(map(string.strip, line_list)) - return line_list \ No newline at end of file + return line_list diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index e93b0289..88cd7dca 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -229,8 +229,8 @@ class ShowTrait(object): hddn = OrderedDict() if self.dataset.group.allsamples: - hddn['allsamples'] = string.join(self.dataset.group.allsamples, ' ') - hddn['primary_samples'] = string.join(self.primary_sample_names, ',') + hddn['allsamples'] = ''.join(self.dataset.group.allsamples) + hddn['primary_samples'] = ''.join(self.primary_sample_names) hddn['trait_id'] = self.trait_id hddn['trait_display_name'] = self.this_trait.display_name hddn['dataset'] = self.dataset.name diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index 0db7e1fe..2df71b12 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -456,7 +456,7 @@ class SnpBrowser(object): function_list = [] if function_details: - function_list = string.split(string.strip(function_details), ",") + function_list = function_details.strip().split(",") function_list = list(map(string.strip, function_list)) function_list[0] = function_list[0].title() function_details = ", ".join(item for item in function_list) @@ -722,11 +722,11 @@ def get_effect_details_by_category(effect_name = None, effect_value = None): new_codon_group_list = ['Start Gained'] codon_effect_group_list = ['Start Lost', 'Stop Gained', 'Stop Lost', 'Nonsynonymous', 'Synonymous'] - effect_detail_list = string.split(string.strip(effect_value), '|') + effect_detail_list = effect_value.strip().split('|') effect_detail_list = list(map(string.strip, effect_detail_list)) for index, item in enumerate(effect_detail_list): - item_list = string.split(string.strip(item), ',') + item_list = item.strip().split(',') item_list = list(map(string.strip, item_list)) gene_id = item_list[0] @@ -746,13 +746,13 @@ def get_effect_details_by_category(effect_name = None, effect_value = None): if effect_name in new_codon_group_list: new_codon = item_list[6] tmp_list = [biotype, new_codon] - function_detail_list.append(string.join(tmp_list, ", ")) + function_detail_list.append(", ".join(tmp_list)) elif effect_name in codon_effect_group_list: old_new_AA = item_list[6] old_new_codon = item_list[7] codon_num = item_list[8] tmp_list = [biotype, old_new_AA, old_new_codon, codon_num] - function_detail_list.append(string.join(tmp_list, ", ")) + function_detail_list.append(", ".join(tmp_list)) else: function_detail_list.append(biotype) @@ -852,7 +852,7 @@ def get_gene_id_name_dict(species_id, gene_name_list): if len(gene_name_list) == 0: return "" gene_name_str_list = ["'" + gene_name + "'" for gene_name in gene_name_list] - gene_name_str = string.join(gene_name_str_list, ",") + gene_name_str = ",".join(gene_name_str_list) query = """ SELECT -- cgit v1.2.3 From 958972339e6c244ebedaf494bf43ca1eecabe6c4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:22:14 +0300 Subject: Remove unused imports --- wqflask/base/mrna_assay_tissue_data.py | 1 - wqflask/base/species.py | 5 ----- wqflask/runserver.py | 9 --------- wqflask/wqflask/correlation_matrix/show_corr_matrix.py | 4 ---- wqflask/wqflask/do_search.py | 2 +- wqflask/wqflask/network_graph/network_graph.py | 5 ----- 6 files changed, 1 insertion(+), 25 deletions(-) diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index 33ce12bd..12f18f71 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -7,7 +7,6 @@ from utility import Bunch from MySQLdb import escape_string as escape -from pprint import pformat as pf from utility.logger import getLogger logger = getLogger(__name__ ) diff --git a/wqflask/base/species.py b/wqflask/base/species.py index e3f1bc4a..2771d116 100644 --- a/wqflask/base/species.py +++ b/wqflask/base/species.py @@ -2,11 +2,6 @@ import collections from flask import Flask, g -#from MySQLdb import escape_string as escape - -from utility import Bunch - -from pprint import pformat as pf from utility.logger import getLogger logger = getLogger(__name__ ) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 15572d97..58d79589 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -11,25 +11,16 @@ from wqflask import app import logging -import utility.logger -logger = utility.logger.getLogger(__name__ ) - -import signal -signal.signal(signal.SIGPIPE, signal.SIG_DFL) BLUE = '\033[94m' GREEN = '\033[92m' BOLD = '\033[1m' ENDC = '\033[0m' -from utility.startup_config import app_config - app_config() werkzeug_logger = logging.getLogger('werkzeug') -from utility.tools import WEBSERVER_MODE, SERVER_PORT - if WEBSERVER_MODE == 'DEBUG': app.run(host='0.0.0.0', port=SERVER_PORT, diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 7b4bda31..dac4037e 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -47,10 +47,6 @@ import utility.webqtlUtil #this is for parallel computing only. from wqflask.correlation import correlation_functions from utility.benchmark import Bench -from MySQLdb import escape_string as escape - -from pprint import pformat as pf - from flask import Flask, g, url_for import utility.logger diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index de8e1e78..7e71f7d7 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -8,7 +8,6 @@ from MySQLdb import escape_string as escape from pprint import pformat as pf import sys -# sys.path.append("..") Never in a running webserver from db import webqtlDatabaseFunction from utility.tools import GN2_BASE_URL @@ -17,6 +16,7 @@ import logging from utility.logger import getLogger logger = getLogger(__name__) + class DoSearch(object): """Parent class containing parameters/functions used for all searches""" diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index 70e5dd58..cfefe4ec 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -31,11 +31,6 @@ from utility.tools import GN2_BRANCH_URL from db import webqtlDatabaseFunction import utility.webqtlUtil #this is for parallel computing only. from wqflask.correlation import correlation_functions -from utility.benchmark import Bench - -from MySQLdb import escape_string as escape - -from pprint import pformat as pf from flask import Flask, g -- cgit v1.2.3 From 391c1681eaeabfdbe65a64a1bb8b05beca30141e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:23:29 +0300 Subject: Add global method to convert binary string to plain string * wqflask/utility/db_tools.py: escape_string returns a binary string which introduces a bug when composing sql query string. The escaped strings have to be converted to plain text. --- wqflask/utility/db_tools.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/wqflask/utility/db_tools.py b/wqflask/utility/db_tools.py index 92bde505..6e19778f 100644 --- a/wqflask/utility/db_tools.py +++ b/wqflask/utility/db_tools.py @@ -1,4 +1,5 @@ -from MySQLdb import escape_string as escape +from MySQLdb import escape_string as escape_ + def create_in_clause(items): """Create an in clause for mysql""" @@ -6,8 +7,11 @@ def create_in_clause(items): in_clause = '( {} )'.format(in_clause) return in_clause + def mescape(*items): """Multiple escape""" - escaped = [escape(str(item)) for item in items] - #print("escaped is:", escaped) - return escaped + return [escape_(str(item)).decode('utf8') for item in items] + + +def escape(string_): + return escape_(string_).decode('utf8') -- cgit v1.2.3 From 5992562296aabdd51bc3575f36d16d1c245d9d1f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:26:20 +0300 Subject: Remove redundant methods which are defined elsewhere * wqflask/base/data_set.py: Delete "create_in_clause", and "mescape" which are defined in "utility.db_tools" --- wqflask/base/data_set.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index ce017fb4..07704e4c 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -23,7 +23,9 @@ from utility.logger import getLogger from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL from db.gn_server import menu_main from pprint import pformat as pf -from MySQLdb import escape_string as escape +from utility.db_tools import escape +from utility.db_tools import mescape +from utility.db_tools import create_in_clause from maintenance import get_group_samplelists from utility.tools import locate, locate_ignore_error, flat_files from utility import gen_geno_ob @@ -204,20 +206,6 @@ def create_datasets_list(): return datasets -def create_in_clause(items): - """Create an in clause for mysql""" - in_clause = ', '.join("'{}'".format(x) for x in mescape(*items)) - in_clause = '( {} )'.format(in_clause) - return in_clause - - -def mescape(*items): - """Multiple escape""" - escaped = [escape(str(item)) for item in items] - #logger.debug("escaped is:", escaped) - return escaped - - class Markers(object): """Todo: Build in cacheing so it saves us reading the same file more than once""" -- cgit v1.2.3 From 2697301045266b40f571eeb2f9c7d8eb220552b4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:28:08 +0300 Subject: Replace mysqldb's "escape" with the one from utils.db_tools Mysqldb's escape returns a binary string. Use utils.db_tools which adds an extra step of converting the binary string to a string. --- wqflask/base/mrna_assay_tissue_data.py | 2 +- wqflask/base/trait.py | 2 +- wqflask/wqflask/api/correlation.py | 2 +- wqflask/wqflask/correlation/show_corr_results.py | 2 +- wqflask/wqflask/do_search.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index 12f18f71..f1929518 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -5,7 +5,7 @@ from flask import g from utility import db_tools from utility import Bunch -from MySQLdb import escape_string as escape +from utility.db_tools import escape from utility.logger import getLogger diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 2fd5d725..e3507ae1 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -19,7 +19,7 @@ Redis = get_redis_conn() from wqflask import app import simplejson as json -from MySQLdb import escape_string as escape +from utility.db_tools import escape from pprint import pformat as pf from flask import Flask, g, request, url_for, redirect, make_response, render_template diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index 7da13121..f5b50dcd 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -2,7 +2,7 @@ import collections import scipy -from MySQLdb import escape_string as escape +from utility.db_tools import escape from flask import g diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index e5b87c6a..91146e5b 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -35,7 +35,7 @@ from wqflask.correlation import correlation_functions from utility.benchmark import Bench import utility.webqtlUtil from utility.type_checking import is_str, get_float, get_int, get_string -from MySQLdb import escape_string as escape +from utility.db_tools import escape from flask import Flask, g diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 7e71f7d7..00636563 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -4,7 +4,7 @@ import json from flask import Flask, g -from MySQLdb import escape_string as escape +from utility.db_tools import escape from pprint import pformat as pf import sys -- cgit v1.2.3 From 3095a738e883d1526d7f9cb35d50ace062616a06 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:29:23 +0300 Subject: Remove comment block --- wqflask/wqflask/parser.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index 472dd30e..76fae54b 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -76,22 +76,6 @@ def parse(pstring): logger.debug("* items are:", pf(items) + "\n") return(items) - #def encregexp(self,str): - # if not str: - # return [] - # else: - # wildcardkeyword = str.strip() - # wildcardkeyword = string.replace(wildcardkeyword,',',' ') - # wildcardkeyword = string.replace(wildcardkeyword,';',' ') - # wildcardkeyword = wildcardkeyword.split() - # NNN = len(wildcardkeyword) - # for i in range(NNN): - # keyword = wildcardkeyword[i] - # keyword = string.replace(keyword,"*",".*") - # keyword = string.replace(keyword,"?",".") - # wildcardkeyword[i] = keyword#'[[:<:]]'+ keyword+'[[:>:]]' - # return wildcardkeyword - if __name__ == '__main__': parse("foo=[3 2 1]") -- cgit v1.2.3 From c030e128508b17896dbef3ba9794698195cafc04 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:30:09 +0300 Subject: Apply pep-8 formatting --- wqflask/runserver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 58d79589..b48ceee1 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -12,10 +12,10 @@ from wqflask import app import logging -BLUE = '\033[94m' +BLUE = '\033[94m' GREEN = '\033[92m' -BOLD = '\033[1m' -ENDC = '\033[0m' +BOLD = '\033[1m' +ENDC = '\033[0m' app_config() @@ -38,7 +38,7 @@ elif WEBSERVER_MODE == 'DEV': threaded=False, processes=0, use_reloader=True) -else: # staging/production modes +else: # staging/production modes app.run(host='0.0.0.0', port=SERVER_PORT, debug=False, -- cgit v1.2.3 From 869da13b92e70fa0769cd1a49e15b03e2c9d0550 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:31:33 +0300 Subject: Fix failing tests --- wqflask/tests/base/test_data_set.py | 71 +++++++++++++++++++++++------- wqflask/tests/base/test_general_object.py | 9 ++-- wqflask/tests/wqflask/api/test_gen_menu.py | 6 +++ 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/wqflask/tests/base/test_data_set.py b/wqflask/tests/base/test_data_set.py index 94780a5d..e6000e56 100644 --- a/wqflask/tests/base/test_data_set.py +++ b/wqflask/tests/base/test_data_set.py @@ -4,7 +4,7 @@ import unittest import mock from wqflask import app -from data import gen_menu_json +from .data import gen_menu_json from base.data_set import DatasetType @@ -59,9 +59,14 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("BXDGeno"), "Geno") self.assertEqual(data_set("BXDPublish"), "Publish") self.assertEqual(data_set("HLC_0311"), "ProbeSet") + redis_mock.set.assert_called_once_with( "dataset_structure", - '{"BXDGeno": "Geno", "BXDPublish": "Publish", "HLCPublish": "Publish", "HLC_0311": "ProbeSet", "HC_M2_0606_P": "ProbeSet"}') + ('{"HLC_0311": "ProbeSet", ' + '"HLCPublish": "Publish", ' + '"BXDGeno": "Geno", ' + '"HC_M2_0606_P": "ProbeSet", ' + '"BXDPublish": "Publish"}')) @mock.patch('base.data_set.g') def test_set_dataset_key_mrna(self, db_mock): @@ -74,8 +79,17 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("Test"), "ProbeSet") redis_mock.set.assert_called_once_with( "dataset_structure", - '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "ProbeSet", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') - expected_db_call = """""" + ('{"AD-cases-controls-MyersGeno": "Geno", ' + '"AD-cases-controls-MyersPublish": "Publish", ' + '"AKXDGeno": "Geno", ' + '"AXBXAGeno": "Geno", ' + '"AXBXAPublish": "Publish", ' + '"Aging-Brain-UCIPublish": "Publish", ' + '"All Phenotypes": "Publish", ' + '"B139_K_1206_M": "ProbeSet", ' + '"B139_K_1206_R": "ProbeSet", ' + '"Test": "ProbeSet"}')) + db_mock.db.execute.assert_called_with( ("SELECT ProbeSetFreeze.Id FROM ProbeSetFreeze " + "WHERE ProbeSetFreeze.Name = \"Test\" ") @@ -92,13 +106,21 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("Test"), "Publish") redis_mock.set.assert_called_once_with( "dataset_structure", - '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Publish", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') - expected_db_call = """""" + ('{"AD-cases-controls-MyersGeno": "Geno", ' + '"AD-cases-controls-MyersPublish": "Publish", ' + '"AKXDGeno": "Geno", ' + '"AXBXAGeno": "Geno", ' + '"AXBXAPublish": "Publish", ' + '"Aging-Brain-UCIPublish": "Publish", ' + '"All Phenotypes": "Publish", ' + '"B139_K_1206_M": "ProbeSet", ' + '"B139_K_1206_R": "ProbeSet", ' + '"Test": "Publish"}')) db_mock.db.execute.assert_called_with( - ("SELECT InfoFiles.GN_AccesionId " + - "FROM InfoFiles, PublishFreeze, InbredSet " + + ("SELECT InfoFiles.GN_AccesionId " + "FROM InfoFiles, PublishFreeze, InbredSet " "WHERE InbredSet.Name = 'Test' AND " - "PublishFreeze.InbredSetId = InbredSet.Id AND " + + "PublishFreeze.InbredSetId = InbredSet.Id AND " "InfoFiles.InfoPageName = PublishFreeze.Name") ) @@ -111,10 +133,20 @@ class TestDataSetTypes(unittest.TestCase): data_set = DatasetType(redis_mock) data_set.set_dataset_key("other_pheno", "Test") self.assertEqual(data_set("Test"), "Publish") + redis_mock.set.assert_called_once_with( "dataset_structure", - '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Publish", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') - expected_db_call = """""" + ('{"AD-cases-controls-MyersGeno": "Geno", ' + '"AD-cases-controls-MyersPublish": "Publish", ' + '"AKXDGeno": "Geno", ' + '"AXBXAGeno": "Geno", ' + '"AXBXAPublish": "Publish", ' + '"Aging-Brain-UCIPublish": "Publish", ' + '"All Phenotypes": "Publish", ' + '"B139_K_1206_M": "ProbeSet", ' + '"B139_K_1206_R": "ProbeSet", ' + '"Test": "Publish"}')) + db_mock.db.execute.assert_called_with( ("SELECT PublishFreeze.Name " + "FROM PublishFreeze, InbredSet " + @@ -133,8 +165,17 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("Test"), "Geno") redis_mock.set.assert_called_once_with( "dataset_structure", - '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Geno", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') - expected_db_call = """""" + ('{"AD-cases-controls-MyersGeno": "Geno", ' + '"AD-cases-controls-MyersPublish": "Publish", ' + '"AKXDGeno": "Geno", ' + '"AXBXAGeno": "Geno", ' + '"AXBXAPublish": "Publish", ' + '"Aging-Brain-UCIPublish": "Publish", ' + '"All Phenotypes": "Publish", ' + '"B139_K_1206_M": "ProbeSet", ' + '"B139_K_1206_R": "ProbeSet", ' + '"Test": "Geno"}')) + db_mock.db.execute.assert_called_with( - ("SELECT GenoFreeze.Id FROM GenoFreeze WHERE GenoFreeze.Name = \"Test\" ") - ) + ("SELECT GenoFreeze.Id FROM " + "GenoFreeze WHERE GenoFreeze.Name = \"Test\" ")) diff --git a/wqflask/tests/base/test_general_object.py b/wqflask/tests/base/test_general_object.py index c7701021..00fd3c72 100644 --- a/wqflask/tests/base/test_general_object.py +++ b/wqflask/tests/base/test_general_object.py @@ -17,9 +17,9 @@ class TestGeneralObjectTests(unittest.TestCase): def test_object_dict(self): """Test whether the base class is printed properly""" test_obj = GeneralObject("a", name="test", value=1) - self.assertEqual(str(test_obj), "value = 1\nname = test\n") + self.assertEqual(str(test_obj), "name = test\nvalue = 1\n") self.assertEqual( - repr(test_obj), "value = 1\nname = test\ncontents = ['a']\n") + repr(test_obj), "contents = ['a']\nname = test\nvalue = 1\n") self.assertEqual(len(test_obj), 2) self.assertEqual(test_obj["value"], 1) test_obj["test"] = 1 @@ -36,6 +36,5 @@ class TestGeneralObjectTests(unittest.TestCase): test_obj1 = GeneralObject("a", name="test", value=1) test_obj2 = GeneralObject("b", name="test2", value=2) test_obj3 = GeneralObject("a", name="test", x=1, y=2) - self.assertTrue(test_obj1 == test_obj2 ) - self.assertFalse(test_obj1 == test_obj3 ) - + self.assertTrue(test_obj1 == test_obj2) + self.assertFalse(test_obj1 == test_obj3) diff --git a/wqflask/tests/wqflask/api/test_gen_menu.py b/wqflask/tests/wqflask/api/test_gen_menu.py index 239484aa..bf41054d 100644 --- a/wqflask/tests/wqflask/api/test_gen_menu.py +++ b/wqflask/tests/wqflask/api/test_gen_menu.py @@ -2,6 +2,7 @@ import unittest import mock +from wqflask import app from wqflask.api.gen_menu import gen_dropdown_json from wqflask.api.gen_menu import get_species from wqflask.api.gen_menu import get_groups @@ -17,6 +18,8 @@ class TestGenMenu(unittest.TestCase): """Tests for the gen_menu module""" def setUp(self): + self.app_context = app.app_context() + self.app_context.push() self.test_group = { 'mouse': [ ['H_T1', @@ -67,6 +70,9 @@ class TestGenMenu(unittest.TestCase): } } + def tearDown(self): + self.app_context.pop() + @mock.patch('wqflask.api.gen_menu.g') def test_get_species(self, db_mock): """Test that assertion is raised when dataset and dataset_name -- cgit v1.2.3 From 46443ec8d2cdfd7c60358a889d90a90e4f7daaf4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:32:34 +0300 Subject: Replace string arguments to "hmac.new" with bytearray See: --- wqflask/utility/hmac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py index 73e28790..aa21c741 100644 --- a/wqflask/utility/hmac.py +++ b/wqflask/utility/hmac.py @@ -10,7 +10,7 @@ def hmac_creation(stringy): secret = app.config['SECRET_HMAC_CODE'] - hmaced = hmac.new(secret, stringy, hashlib.sha1) + hmaced = hmac.new(bytearray(secret, 'utf8'), bytearray(stringy, 'utf8'), hashlib.sha1) hm = hmaced.hexdigest() # ZS: Leaving the below comment here to ask Pjotr about # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." -- cgit v1.2.3 From d5fc56b802f2195dfdad3f6c3e20a3832aa95567 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:34:09 +0300 Subject: Import "reduce" without a try block * wqflask/wqflask/correlation_matrix/show_corr_matrix.py: Import 'reduce' --- wqflask/wqflask/correlation_matrix/show_corr_matrix.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index dac4037e..49ba9e5d 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -29,10 +29,8 @@ import rpy2.robjects as robjects from pprint import pformat as pf from utility.redis_tools import get_redis_conn -try: # Only import this for Python3 - from functools import reduce -except: - pass +from functools import reduce + Redis = get_redis_conn() THIRTY_DAYS = 60 * 60 * 24 * 30 -- cgit v1.2.3 From 83b8880088572eb11110b3b873c04148112be3b9 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 01:35:30 +0300 Subject: Convert sibling imports to relative imports See: --- wqflask/runserver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index b48ceee1..a81d62b8 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -7,8 +7,9 @@ # # /sbin/iptables -A INPUT -p tcp -i eth0 -s ! 71.236.239.43 --dport 5003 -j DROP -from wqflask import app - +from .wqflask import app +from .utility.startup_config import app_config +from .utility.tools import WEBSERVER_MODE, SERVER_PORT import logging -- cgit v1.2.3 From 3d5cc150fb5927edb759c52a52cae8533bc22252 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 03:13:43 +0300 Subject: Replace dict.iteritems() with dict.items() "iteritems()" got omitted in python3.X --- wqflask/wqflask/templates/admin/manage_resource.html | 2 +- wqflask/wqflask/templates/loading.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 0b12eaae..33a37594 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -65,7 +65,7 @@ - {% for key, value in group_masks.iteritems() %} + {% for key, value in group_masks.items() %} {{ value.name }} {{ value.data }} diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 15ab4080..9b335dfe 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -1,7 +1,7 @@ Loading {{ start_vars.tool_used }} Results
- {% for key, value in start_vars.iteritems() %} + {% for key, value in start_vars.items() %} {% endfor %}
@@ -44,4 +44,4 @@ $("#loading_form").attr("action", "{{ start_vars.form_url }}"); setTimeout(function(){ $("#loading_form").submit()}, 350); - \ No newline at end of file + -- cgit v1.2.3 From e102c7c0a7d1ad3c305ed8db0d19fa12cfa5cb38 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 03:19:23 +0300 Subject: Remove relative imports * wqflask/runserver.py: Relative imports leads to import errors. --- wqflask/runserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index a81d62b8..df957bd9 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -7,9 +7,9 @@ # # /sbin/iptables -A INPUT -p tcp -i eth0 -s ! 71.236.239.43 --dport 5003 -j DROP -from .wqflask import app -from .utility.startup_config import app_config -from .utility.tools import WEBSERVER_MODE, SERVER_PORT +from wqflask import app +from utility.startup_config import app_config +from utility.tools import WEBSERVER_MODE, SERVER_PORT import logging -- cgit v1.2.3 From 1e030b2543c22a8148f39657b5921724d9512d40 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 03:22:06 +0300 Subject: Remove python2 string processing to utf-8 * wqflask/base/trait.py: Remove python2 codecs.BOM_UTF8 methods. * wqflask/wqflask/search_results.py: Remove utf-8 decode method on string. --- wqflask/base/trait.py | 4 ---- wqflask/wqflask/search_results.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index e3507ae1..6950cf11 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -488,10 +488,6 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): else: trait.description_display = "" - trait.abbreviation = str(str(trait.abbreviation).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") - trait.description_display = str(str(trait.description_display).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") - trait.authors = str(str(trait.authors).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") - if not trait.year.isdigit(): trait.pubmed_text = "N/A" else: diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index f6c677a8..0de14f15 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -115,7 +115,7 @@ views.py). trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) if this_trait.dataset.type == "ProbeSet": trait_dict['symbol'] = this_trait.symbol - trait_dict['description'] = this_trait.description_display.decode('utf-8', 'replace') + trait_dict['description'] = this_trait.description_display trait_dict['location'] = this_trait.location_repr trait_dict['mean'] = "N/A" trait_dict['additive'] = "N/A" -- cgit v1.2.3 From c6c80dde5cfb03c73b61523950af0ca6d208aea4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 03:25:06 +0300 Subject: Update imports * wqflask/wqflask/api/router.py: Break imports into separate lines and remove unused ones. --- wqflask/wqflask/api/router.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index acf7ce4a..8e59ef27 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -1,14 +1,21 @@ # GN2 API -import os, io, csv, json, datetime, requests, yaml -import zlib +import os +import io +import csv +import json +import datetime +import requests + from zipfile import ZipFile, ZIP_DEFLATED -import io import flask -from flask import g, Response, request, make_response, render_template, send_from_directory, jsonify, redirect, send_file -import sqlalchemy +from flask import g +from flask import request +from flask import make_response +from flask import send_file + from wqflask import app from wqflask.api import correlation, mapping, gen_menu -- cgit v1.2.3 From b46faf479e129801f4bec2457444b5341e8aeabf Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 03:26:36 +0300 Subject: Add a check for a "uuid" object * wqflask/wqflask/views.py(json_default_handler): Check for "uuid.UUID" instance to avoid TypeError. --- wqflask/wqflask/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 0224da4a..557ff523 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -930,8 +930,8 @@ def json_default_handler(obj): if hasattr(obj, 'isoformat'): return obj.isoformat() # Handle integer keys for dictionaries - elif isinstance(obj, int): - return str(int) + elif isinstance(obj, int) or isinstance(obj, uuid.UUID): + return str(obj) # Handle custom objects if hasattr(obj, '__dict__'): return obj.__dict__ -- cgit v1.2.3 From 70dbeeb5832711ed5271434e482c18bc7ea095b8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 27 Aug 2020 03:28:43 +0300 Subject: Add check for empty group_code when adding a new resource * wqflask/utility/authentication_tools.py(add_new_resource): If group_code is "None", an error is thrown when you try to: `group_code + "_" + str(trait_id)` --- wqflask/utility/authentication_tools.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 0e499180..73c39399 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -49,7 +49,10 @@ def add_new_resource(dataset, trait_id=None): } if dataset.type == "Publish": - resource_ob['name'] = get_group_code(dataset) + "_" + str(trait_id) + group_code = get_group_code(dataset) + if group_code is None: + group_code = "" + resource_ob['name'] = group_code + "_" + str(trait_id) resource_ob['data'] = { 'dataset': dataset.id, 'trait' : trait_id @@ -74,7 +77,6 @@ def add_new_resource(dataset, trait_id=None): def get_group_code(dataset): results = g.db.execute("SELECT InbredSetCode from InbredSet where Name='{}'".format(dataset.group.name)).fetchone() - return results[0] def check_admin(resource_id=None): -- cgit v1.2.3 From 90475fed0b2d1bd192a641bd417f6dfef79653d0 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 31 Aug 2020 20:34:31 +0300 Subject: Remove typo at beginning of line * wqflask/utility/authentication_tools.py: Remove accidental char at beginning of line. --- wqflask/utility/authentication_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 68bbb276..c52ebafa 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -88,7 +88,7 @@ def check_admin(resource_id=None): response = json.loads(requests.get(the_url).content)['admin'] except: resource_info = get_resource_info(resource_id) -l response = resource_info['default_mask']['admin'] + response = resource_info['default_mask']['admin'] if 'edit-admins' in response: return "edit-admins" -- cgit v1.2.3 From f4a8789a5f28e1527d4c801b40176f47aa44146c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 17 Sep 2020 17:16:28 +0300 Subject: Apply pep8 * wqflask/base/trait.py: Apply pep8. * wqflask/utility/authentication_tools.py: Ditto. --- wqflask/base/trait.py | 192 +++++++++++++++++++------------- wqflask/utility/authentication_tools.py | 32 ++++-- 2 files changed, 134 insertions(+), 90 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 6950cf11..548ccc4c 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -1,36 +1,30 @@ -import os -import string -import resource -import codecs import requests -import random +import simplejson as json +from wqflask import app from base import webqtlConfig from base.webqtlCaseData import webqtlCaseData from base.data_set import create_dataset -from db import webqtlDatabaseFunction -from utility import webqtlUtil from utility import hmac from utility.authentication_tools import check_resource_availability -from utility.tools import GN2_BASE_URL, GN_VERSION -from utility.redis_tools import get_redis_conn, get_resource_id, get_resource_info -Redis = get_redis_conn() +from utility.tools import GN2_BASE_URL +from utility.redis_tools import get_redis_conn, get_resource_id -from wqflask import app - -import simplejson as json from utility.db_tools import escape -from pprint import pformat as pf -from flask import Flask, g, request, url_for, redirect, make_response, render_template +from flask import g, request, url_for from utility.logger import getLogger -logger = getLogger(__name__ ) + +logger = getLogger(__name__) + +Redis = get_redis_conn() + def create_trait(**kw): - assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name"; + assert bool(kw.get('dataset')) != bool( + kw.get('dataset_name')), "Needs dataset ob. or name" - permitted = True if kw.get('name'): if kw.get('dataset_name'): if kw.get('dataset_name') != "Temp": @@ -40,18 +34,23 @@ def create_trait(**kw): if kw.get('dataset_name') != "Temp": if dataset.type == 'Publish': - permissions = check_resource_availability(dataset, kw.get('name')) + permissions = check_resource_availability( + dataset, kw.get('name')) else: permissions = check_resource_availability(dataset) if "view" in permissions['data']: the_trait = GeneralTrait(**kw) if the_trait.dataset.type != "Temp": - the_trait = retrieve_trait_info(the_trait, the_trait.dataset, get_qtl_info=kw.get('get_qtl_info')) + the_trait = retrieve_trait_info( + the_trait, + the_trait.dataset, + get_qtl_info=kw.get('get_qtl_info')) return the_trait else: return None + class GeneralTrait(object): """ Trait class defines a trait in webqtl, can be either Microarray, @@ -61,12 +60,17 @@ class GeneralTrait(object): def __init__(self, get_qtl_info=False, get_sample_info=True, **kw): # xor assertion - assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name"; - self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. + assert bool(kw.get('dataset')) != bool( + kw.get('dataset_name')), "Needs dataset ob. or name" + # Trait ID, ProbeSet ID, Published ID, etc. + self.name = kw.get('name') if kw.get('dataset_name'): if kw.get('dataset_name') == "Temp": temp_group = self.name.split("_")[2] - self.dataset = create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = temp_group) + self.dataset = create_dataset( + dataset_name="Temp", + dataset_type="Temp", + group_name=temp_group) else: self.dataset = create_dataset(kw.get('dataset_name')) else: @@ -74,7 +78,8 @@ class GeneralTrait(object): self.cellid = kw.get('cellid') self.identification = kw.get('identification', 'un-named trait') self.haveinfo = kw.get('haveinfo', False) - self.sequence = kw.get('sequence') # Blat sequence, available for ProbeSet + # Blat sequence, available for ProbeSet + self.sequence = kw.get('sequence') self.data = kw.get('data', {}) self.view = True @@ -100,9 +105,10 @@ class GeneralTrait(object): elif len(name2) == 3: self.dataset, self.name, self.cellid = name2 - # Todo: These two lines are necessary most of the time, but perhaps not all of the time - # So we could add a simple if statement to short-circuit this if necessary - if get_sample_info != False: + # Todo: These two lines are necessary most of the time, but + # perhaps not all of the time So we could add a simple if + # statement to short-circuit this if necessary + if get_sample_info is not False: self = retrieve_sample_data(self, self.dataset) def export_informative(self, include_variance=0): @@ -116,13 +122,13 @@ class GeneralTrait(object): the_vars = [] sample_aliases = [] for sample_name, sample_data in list(self.data.items()): - if sample_data.value != None: - if not include_variance or sample_data.variance != None: + if sample_data.value is not None: + if not include_variance or sample_data.variance is not None: samples.append(sample_name) vals.append(sample_data.value) the_vars.append(sample_data.variance) sample_aliases.append(sample_data.name2) - return samples, vals, the_vars, sample_aliases + return samples, vals, the_vars, sample_aliases @property def description_fmt(self): @@ -161,12 +167,17 @@ class GeneralTrait(object): alias = 'Not available' if self.symbol: - human_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.upper()) - mouse_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.capitalize()) - other_response = requests.get(GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.lower()) + human_response = requests.get( + GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.upper()) + mouse_response = requests.get( + GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.capitalize()) + other_response = requests.get( + GN2_BASE_URL + "gn3/gene/aliases/" + self.symbol.lower()) if human_response and mouse_response and other_response: - alias_list = json.loads(human_response.content) + json.loads(mouse_response.content) + json.loads(other_response.content) + alias_list = json.loads(human_response.content) + json.loads( + mouse_response.content) + \ + json.loads(other_response.content) filtered_aliases = [] seen = set() @@ -180,33 +191,34 @@ class GeneralTrait(object): return alias - @property def location_fmt(self): '''Return a text formatted location - While we're at it we set self.location in case we need it later (do we?) + While we're at it we set self.location in case we need it + later (do we?) ''' if self.chr and self.mb: - self.location = 'Chr %s @ %s Mb' % (self.chr, self.mb) + self.location = 'Chr %s @ %s Mb' % (self.chr, self.mb) elif self.chr: self.location = 'Chr %s @ Unknown position' % (self.chr) else: self.location = 'Not available' fmt = self.location - ##XZ: deal with direction + # XZ: deal with direction if self.strand_probe == '+': fmt += (' on the plus strand ') elif self.strand_probe == '-': fmt += (' on the minus strand ') return fmt - + + def retrieve_sample_data(trait, dataset, samplelist=None): - if samplelist == None: + if samplelist is None: samplelist = [] if dataset.type == "Temp": @@ -222,16 +234,19 @@ def retrieve_sample_data(trait, dataset, samplelist=None): all_samples_ordered = dataset.group.all_samples_ordered() for i, item in enumerate(results): try: - trait.data[all_samples_ordered[i]] = webqtlCaseData(all_samples_ordered[i], float(item)) + trait.data[all_samples_ordered[i]] = webqtlCaseData( + all_samples_ordered[i], float(item)) except: pass else: for item in results: name, value, variance, num_cases, name2 = item if not samplelist or (samplelist and name in samplelist): - trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) + # name, value, variance, num_cases) + trait.data[name] = webqtlCaseData(*item) return trait + @app.route("/trait/get_sample_data") def get_sample_data(): params = request.args @@ -247,7 +262,8 @@ def get_sample_data(): trait_dict['group'] = trait_ob.dataset.group.name trait_dict['tissue'] = trait_ob.dataset.tissue trait_dict['species'] = trait_ob.dataset.group.species - trait_dict['url'] = url_for('show_trait_page', trait_id = trait, dataset = dataset) + trait_dict['url'] = url_for( + 'show_trait_page', trait_id=trait, dataset=dataset) trait_dict['description'] = trait_ob.description_display if trait_ob.dataset.type == "ProbeSet": trait_dict['symbol'] = trait_ob.symbol @@ -257,22 +273,27 @@ def get_sample_data(): trait_dict['pubmed_link'] = trait_ob.pubmed_link trait_dict['pubmed_text'] = trait_ob.pubmed_text - return json.dumps([trait_dict, {key: value.value for key, value in list(trait_ob.data.items()) }]) + return json.dumps([trait_dict, {key: value.value for + key, value in list( + trait_ob.data.items())}]) else: return None - + + def jsonable(trait): """Return a dict suitable for using as json Actual turning into json doesn't happen here though""" - dataset = create_dataset(dataset_name = trait.dataset.name, dataset_type = trait.dataset.type, group_name = trait.dataset.group.name) - + dataset = create_dataset(dataset_name=trait.dataset.name, + dataset_type=trait.dataset.type, + group_name=trait.dataset.group.name) + if dataset.type == "ProbeSet": return dict(name=trait.name, symbol=trait.symbol, dataset=dataset.name, - dataset_name = dataset.shortname, + dataset_name=dataset.shortname, description=trait.description_display, mean=trait.mean, location=trait.location_repr, @@ -284,7 +305,7 @@ def jsonable(trait): if trait.pubmed_id: return dict(name=trait.name, dataset=dataset.name, - dataset_name = dataset.shortname, + dataset_name=dataset.shortname, description=trait.description_display, abbreviation=trait.abbreviation, authors=trait.authors, @@ -297,7 +318,7 @@ def jsonable(trait): else: return dict(name=trait.name, dataset=dataset.name, - dataset_name = dataset.shortname, + dataset_name=dataset.shortname, description=trait.description_display, abbreviation=trait.abbreviation, authors=trait.authors, @@ -309,19 +330,20 @@ def jsonable(trait): elif dataset.type == "Geno": return dict(name=trait.name, dataset=dataset.name, - dataset_name = dataset.shortname, + dataset_name=dataset.shortname, location=trait.location_repr ) else: return dict() + def jsonable_table_row(trait, dataset_name, index): """Return a list suitable for json and intended to be displayed in a table Actual turning into json doesn't happen here though""" dataset = create_dataset(dataset_name) - + if dataset.type == "ProbeSet": if trait.mean == "": mean = "N/A" @@ -333,11 +355,13 @@ def jsonable_table_row(trait, dataset_name, index): additive = "%.3f" % round(float(trait.additive), 2) return ['', index, - ''+str(trait.name)+'', + ''+str(trait.name)+'', trait.symbol, trait.description_display, trait.location_repr, - mean, + mean, trait.LRS_score_repr, trait.LRS_location_repr, additive] @@ -349,7 +373,9 @@ def jsonable_table_row(trait, dataset_name, index): if trait.pubmed_id: return ['', index, - ''+str(trait.name)+'', + ''+str(trait.name)+'', trait.description_display, trait.authors, '' + trait.pubmed_text + '', @@ -359,7 +385,9 @@ def jsonable_table_row(trait, dataset_name, index): else: return ['', index, - ''+str(trait.name)+'', + ''+str(trait.name)+'', trait.description_display, trait.authors, trait.pubmed_text, @@ -369,7 +397,9 @@ def jsonable_table_row(trait, dataset_name, index): elif dataset.type == "Geno": return ['', index, - ''+str(trait.name)+'', + ''+str(trait.name)+'', trait.location_repr] else: return dict() @@ -380,14 +410,16 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): resource_id = get_resource_id(dataset, trait.name) if dataset.type == 'Publish': - the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view".format(resource_id, g.user_session.user_id) + the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view".format( + resource_id, g.user_session.user_id) else: - the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view&trait={}".format(resource_id, g.user_session.user_id, trait.name) + the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view&trait={}".format( + resource_id, g.user_session.user_id, trait.name) try: response = requests.get(the_url).content trait_info = json.loads(response) - except: #ZS: I'm assuming the trait is viewable if the try fails for some reason; it should never reach this point unless the user has privileges, since that's dealt with in create_trait + except: # ZS: I'm assuming the trait is viewable if the try fails for some reason; it should never reach this point unless the user has privileges, since that's dealt with in create_trait if dataset.type == 'Publish': query = """ SELECT @@ -416,8 +448,8 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): logger.sql(query) trait_info = g.db.execute(query).fetchone() - #XZ, 05/08/2009: Xiaodong add this block to use ProbeSet.Id to find the probeset instead of just using ProbeSet.Name - #XZ, 05/08/2009: to avoid the problem of same probeset name from different platforms. + # XZ, 05/08/2009: Xiaodong add this block to use ProbeSet.Id to find the probeset instead of just using ProbeSet.Name + # XZ, 05/08/2009: to avoid the problem of same probeset name from different platforms. elif dataset.type == 'ProbeSet': display_fields_string = ', ProbeSet.'.join(dataset.display_fields) display_fields_string = 'ProbeSet.' + display_fields_string @@ -430,11 +462,11 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): ProbeSetFreeze.Name = '%s' AND ProbeSet.Name = '%s' """ % (escape(display_fields_string), - escape(dataset.name), - escape(str(trait.name))) + escape(dataset.name), + escape(str(trait.name))) logger.sql(query) trait_info = g.db.execute(query).fetchone() - #XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name + # XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name # to avoid the problem of same marker name from different species. elif dataset.type == 'Geno': display_fields_string = ',Geno.'.join(dataset.display_fields) @@ -448,11 +480,11 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): GenoFreeze.Name = '%s' AND Geno.Name = '%s' """ % (escape(display_fields_string), - escape(dataset.name), - escape(trait.name)) + escape(dataset.name), + escape(trait.name)) logger.sql(query) trait_info = g.db.execute(query).fetchone() - else: #Temp type + else: # Temp type query = """SELECT %s FROM %s WHERE Name = %s""" logger.sql(query) trait_info = g.db.execute(query, @@ -462,7 +494,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if trait_info: trait.haveinfo = True for i, field in enumerate(dataset.display_fields): - holder = trait_info[i] + holder = trait_info[i] setattr(trait, field, holder) if dataset.type == 'Publish': @@ -475,9 +507,9 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): description = trait.post_publication_description - #If the dataset is confidential and the user has access to confidential - #phenotype traits, then display the pre-publication description instead - #of the post-publication description + # If the dataset is confidential and the user has access to confidential + # phenotype traits, then display the pre-publication description instead + # of the post-publication description if trait.confidential: trait.abbreviation = trait.pre_publication_abbreviation trait.description_display = trait.pre_publication_description @@ -514,15 +546,17 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.location_repr = 'N/A' if trait.chr and trait.mb: - trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) + trait.location_repr = 'Chr%s: %.6f' % ( + trait.chr, float(trait.mb)) elif dataset.type == "Geno": trait.location_repr = 'N/A' if trait.chr and trait.mb: - trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) + trait.location_repr = 'Chr%s: %.6f' % ( + trait.chr, float(trait.mb)) if get_qtl_info: - #LRS and its location + # LRS and its location trait.LRS_score_repr = "N/A" trait.LRS_location_repr = "N/A" trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.additive = "" @@ -594,10 +628,12 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.locus = trait.lrs = trait.additive = "" if (dataset.type == 'Publish' or dataset.type == "ProbeSet") and trait.locus_chr != "" and trait.locus_mb != "": - trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (trait.locus_chr, float(trait.locus_mb)) + trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % ( + trait.locus_chr, float(trait.locus_mb)) if trait.lrs != "": trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs else: - raise KeyError(repr(trait.name)+' information is not found in the database.') - + raise KeyError(repr(trait.name) + + ' information is not found in the database.') + return trait diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 390ad75a..ce0c0749 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -1,30 +1,37 @@ import json import requests -from base import data_set, webqtlConfig - -from utility import hmac -from utility.redis_tools import get_redis_conn, get_resource_info, get_resource_id, add_resource -Redis = get_redis_conn() +from flask import g +from base import webqtlConfig -logger = logging.getLogger(__name__) +from utility.redis_tools import (get_redis_conn, + get_resource_info, + get_resource_id, + add_resource) +Redis = get_redis_conn() def check_resource_availability(dataset, trait_id=None): - # At least for now assume temporary entered traits are accessible if type(dataset) == str or dataset.type == "Temp": return webqtlConfig.DEFAULT_PRIVILEGES resource_id = get_resource_id(dataset, trait_id) - if resource_id: # ZS: This should never be false, but it's technically possible if a non-Temp dataset somehow had a type other than Publish/ProbeSet/Geno + # ZS: This should never be false, but it's technically possible if + # a non-Temp dataset somehow had a type other than + # Publish/ProbeSet/Geno + if resource_id: resource_info = get_resource_info(resource_id) - if not resource_info: # ZS: If resource isn't already in redis, add it with default privileges + + # ZS: If resource isn't already in redis, add it with default + # privileges + if not resource_info: resource_info = add_new_resource(dataset, trait_id) - # ZS: Check if super-user - we should probably come up with some way to integrate this into the proxy + # ZS: Check if super-user - we should probably come up with some + # way to integrate this into the proxy if g.user_session.user_id in Redis.smembers("super_users"): return webqtlConfig.SUPER_PRIVILEGES @@ -76,8 +83,9 @@ def add_new_resource(dataset, trait_id=None): def get_group_code(dataset): - results = g.db.execute("SELECT InbredSetCode from InbredSet where Name='{}'".format( - dataset.group.name)).fetchone() + results = g.db.execute( + "SELECT InbredSetCode from InbredSet where Name='{}'".format( + dataset.group.name)).fetchone() if results[0]: return results[0] else: -- cgit v1.2.3 From 16faa26e52b1f0191595e16550d553907d2f9d67 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 17 Sep 2020 17:21:30 +0300 Subject: Fix errors in tests * wqflask/tests/utility/test_authentication_tools.py test_check_resource_availability_non_default_mask): Mock flask's global 'g' variable properly. * wqflask/tests/base/test_trait.py: Ditto. * wqflask/tests/utility/test_authentication_tools.py: Ditto. --- wqflask/tests/base/test_trait.py | 9 +++------ wqflask/tests/utility/test_authentication_tools.py | 8 ++------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/wqflask/tests/base/test_trait.py b/wqflask/tests/base/test_trait.py index 53b0d440..d333458a 100644 --- a/wqflask/tests/base/test_trait.py +++ b/wqflask/tests/base/test_trait.py @@ -38,9 +38,8 @@ class TestRetrieveTraitInfo(unittest.TestCase): dataset={}) @mock.patch('base.trait.requests.get') - @mock.patch('base.trait.g') + @mock.patch('base.trait.g', mock.Mock()) def test_retrieve_trait_info_with_empty_trait_info(self, - g_mock, requests_mock): """Empty trait info""" requests_mock.return_value = TestNilResponse() @@ -49,9 +48,8 @@ class TestRetrieveTraitInfo(unittest.TestCase): dataset=mock.MagicMock()) @mock.patch('base.trait.requests.get') - @mock.patch('base.trait.g') + @mock.patch('base.trait.g', mock.Mock()) def test_retrieve_trait_info_with_non_empty_trait_info(self, - g_mock, requests_mock): """Test that attributes are set""" mock_dataset = mock.MagicMock() @@ -66,9 +64,8 @@ class TestRetrieveTraitInfo(unittest.TestCase): self.assertEqual(test_trait.d, 4) @mock.patch('base.trait.requests.get') - @mock.patch('base.trait.g') + @mock.patch('base.trait.g', mock.Mock()) def test_retrieve_trait_info_utf8_parsing(self, - g_mock, requests_mock): """Test that utf-8 strings are parsed correctly""" utf_8_string = "test_string" diff --git a/wqflask/tests/utility/test_authentication_tools.py b/wqflask/tests/utility/test_authentication_tools.py index 99c74245..ef94eabc 100644 --- a/wqflask/tests/utility/test_authentication_tools.py +++ b/wqflask/tests/utility/test_authentication_tools.py @@ -38,17 +38,15 @@ class TestCheckResourceAvailability(unittest.TestCase): """Test methods related to checking the resource availability""" @mock.patch('utility.authentication_tools.add_new_resource') @mock.patch('utility.authentication_tools.Redis') - @mock.patch('utility.authentication_tools.g') + @mock.patch('utility.authentication_tools.g', mock.Mock()) @mock.patch('utility.authentication_tools.get_resource_id') def test_check_resource_availability_default_mask( self, resource_id_mock, - g_mock, redis_mock, add_new_resource_mock): """Test the resource availability with default mask""" resource_id_mock.return_value = 1 - g_mock.return_value = mock.Mock() redis_mock.smembers.return_value = [] test_dataset = mock.MagicMock() type(test_dataset).type = mock.PropertyMock(return_value="Test") @@ -58,18 +56,16 @@ class TestCheckResourceAvailability(unittest.TestCase): @mock.patch('utility.authentication_tools.requests.get') @mock.patch('utility.authentication_tools.add_new_resource') @mock.patch('utility.authentication_tools.Redis') - @mock.patch('utility.authentication_tools.g') + @mock.patch('utility.authentication_tools.g', TestUserSession()) @mock.patch('utility.authentication_tools.get_resource_id') def test_check_resource_availability_non_default_mask( self, resource_id_mock, - g_mock, redis_mock, add_new_resource_mock, requests_mock): """Test the resource availability with default mask""" resource_id_mock.return_value = 1 - g_mock.return_value = mock.Mock() redis_mock.smembers.return_value = [] add_new_resource_mock.return_value = {"default_mask": 2} requests_mock.return_value = TestResponse() -- cgit v1.2.3 From 6064148eb2b723a308f0d29595a75ab64f47e1e2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 17 Sep 2020 17:34:11 +0300 Subject: Replace string arguments to "hmac.new" with bytearray Same as: https://github.com/genenetwork/genenetwork2/pull/422/commits/46443ec8d2cdfd7c60358a889d90a90e4f7daaf4 --- wqflask/utility/hmac.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py index 10387bb0..6623f69a 100644 --- a/wqflask/utility/hmac.py +++ b/wqflask/utility/hmac.py @@ -10,7 +10,9 @@ def hmac_creation(stringy): """Helper function to create the actual hmac""" secret = app.config['SECRET_HMAC_CODE'] - hmaced = hmac.new(secret, stringy, hashlib.sha1) + hmaced = hmac.new(bytearray(secret, "utf-8"), + bytearray(stringy, "utf-8"), + hashlib.sha1) hm = hmaced.hexdigest() # ZS: Leaving the below comment here to ask Pjotr about # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." -- cgit v1.2.3 From bfb5fab5ac57934f7a6bd54938188fa0d2669ad7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 17 Sep 2020 21:16:28 +0300 Subject: Convert "target_string" to a str type * wqflask/base/trait.py (retrieve_trait_info): Convert "target_string" to a str, otherwise an exception will be thrown if it's None. --- wqflask/base/trait.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 548ccc4c..c37a8325 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -538,8 +538,8 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): description_display = trait.symbol if (len(description_display) > 1 and description_display != 'N/A' and - len(target_string) > 1 and target_string != 'None'): - description_display = description_display + '; ' + target_string.strip() + len(str(target_string)) > 1 and target_string != 'None'): + description_display = description_display + '; ' + str(target_string).strip() # Save it for the jinja2 template trait.description_display = description_display -- cgit v1.2.3 From d52cc25d120173de3fd16bbd42ea38498d458e35 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 17 Sep 2020 21:19:00 +0300 Subject: Update Mechanical Rob * test/requests/link_checker.py (verify_static_file): Cast result.content.find's args to bytes. * test/requests/main_web_functionality.py (check_search_page): Update text to search for in a successful result. This value changed in 2aac6a06. --- test/requests/link_checker.py | 2 +- test/requests/main_web_functionality.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/requests/link_checker.py b/test/requests/link_checker.py index e9943492..6ac26ba7 100644 --- a/test/requests/link_checker.py +++ b/test/requests/link_checker.py @@ -54,7 +54,7 @@ def verify_static_file(link): try: result = requests.get(link, timeout=20, verify=False) if (result.status_code == 200 and - result.content.find("Error: 404 Not Found") <= 0): + result.content.find(bytes("Error: 404 Not Found", "utf-8")) <= 0): print(link+" ==> OK") else: print("ERROR: link {}".format(link)) diff --git a/test/requests/main_web_functionality.py b/test/requests/main_web_functionality.py index 78030307..cefb8b8c 100644 --- a/test/requests/main_web_functionality.py +++ b/test/requests/main_web_functionality.py @@ -9,7 +9,6 @@ def check_home(url): assert(search_button[0].value == "Search") print("OK") - def check_search_page(host): data = dict( species="mouse", @@ -19,7 +18,7 @@ def check_search_page(host): search_terms_or="", search_terms_and="MEAN=(15 16) LRS=(23 46)") result = requests.get(host+"/search", params=data) - found = result.text.find("records are shown below") + found = result.text.find("records were found.") assert(found >= 0) assert(result.status_code == 200) print("OK") -- cgit v1.2.3 From 7f94f657e51bbf9a7a1064b12a7d67b53a14eb11 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 26 Sep 2020 01:15:09 +0300 Subject: Fix inconsistent line ending in file * wqflask/wqflask/db_info.py: Replace ^M with proper unix line endings. See --- wqflask/wqflask/db_info.py | 254 ++++++++++++++++++++++----------------------- 1 file changed, 127 insertions(+), 127 deletions(-) diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index f04e38bf..5f3f5e3f 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -1,127 +1,127 @@ -import httplib, urllib2 -import re - -from flask import Flask, g - -from utility.logger import getLogger -logger = getLogger(__name__ ) - -class InfoPage(object): - def __init__(self, start_vars): - self.info = None - self.gn_accession_id = None - if 'gn_accession_id' in start_vars: - self.gn_accession_id = start_vars['gn_accession_id'] - self.info_page_name = start_vars['info_page_name'] - - self.get_info() - self.get_datasets_list() - - def get_info(self, create=False): - query_base = ("SELECT InfoPageName, GN_AccesionId, Species.MenuName, Species.TaxonomyId, Tissue.Name, InbredSet.Name, " + - "GeneChip.GeneChipName, GeneChip.GeoPlatform, AvgMethod.Name, Datasets.DatasetName, Datasets.GeoSeries, " + - "Datasets.PublicationTitle, DatasetStatus.DatasetStatusName, Datasets.Summary, Datasets.AboutCases, " + - "Datasets.AboutTissue, Datasets.AboutDataProcessing, Datasets.Acknowledgment, Datasets.ExperimentDesign, " + - "Datasets.Contributors, Datasets.Citation, Datasets.Notes, Investigators.FirstName, Investigators.LastName, " + - "Investigators.Address, Investigators.City, Investigators.State, Investigators.ZipCode, Investigators.Country, " + - "Investigators.Phone, Investigators.Email, Investigators.Url, Organizations.OrganizationName, " + - "InvestigatorId, DatasetId, DatasetStatusId, Datasets.AboutPlatform, InfoFileTitle, Specifics " + - "FROM InfoFiles " + - "LEFT JOIN Species USING (SpeciesId) " + - "LEFT JOIN Tissue USING (TissueId) " + - "LEFT JOIN InbredSet USING (InbredSetId) " + - "LEFT JOIN GeneChip USING (GeneChipId) " + - "LEFT JOIN AvgMethod USING (AvgMethodId) " + - "LEFT JOIN Datasets USING (DatasetId) " + - "LEFT JOIN Investigators USING (InvestigatorId) " + - "LEFT JOIN Organizations USING (OrganizationId) " + - "LEFT JOIN DatasetStatus USING (DatasetStatusId) WHERE ") - - if self.gn_accession_id: - final_query = query_base + "GN_AccesionId = {}".format(self.gn_accession_id) - results = g.db.execute(final_query).fetchone() - if self.info_page_name and not results: - final_query = query_base + "InfoPageName={}".format(self.info_page_name) - elif self.info_page_name: - final_query = query_base + "InfoPageName={}".format(self.info_page_name) - results = g.db.execute(final_query).fetchone() - else: - raise 'No correct parameter found' - - if results: - self.info = process_query_results(results) - - if (not results or len(results) < 1) and self.info_page_name and create: - insert_sql = "INSERT INTO InfoFiles SET InfoFiles.InfoPageName={}".format(self.info_page_name) - return self.get_info() - - if not self.gn_accession_id and self.info: - self.gn_accession_id = self.info['accession_id'] - if not self.info_page_name and self.info: - self.info_page_name = self.info['info_page_name'] - - def get_datasets_list(self): - self.filelist = [] - try: - response = urllib2.urlopen("http://datafiles.genenetwork.org/download/GN%s" % self.gn_accession_id) - data = response.read() - - matches = re.findall(r".+?", data, re.DOTALL) - for i, match in enumerate(matches): - if i == 0: - continue - cells = re.findall(r".+?", match, re.DOTALL) - full_filename = re.search(r"(.+?)<", cells[2]).group(1).strip() - filedate = "N/A" #ZS: Since we can't get it for now - - self.filelist.append([filename, filedate, filesize]) - except Exception, e: - pass - -def process_query_results(results): - info_ob = { - 'info_page_name': results[0], - 'accession_id': results[1], - 'menu_name': results[2], - 'taxonomy_id': results[3], - 'tissue_name': results[4], - 'group_name': results[5], - 'gene_chip_name': results[6], - 'geo_platform': results[7], - 'avg_method_name': results[8], - 'dataset_name': results[9], - 'geo_series': results[10], - 'publication_title': results[11], - 'dataset_status_name': results[12], - 'dataset_summary': results[13], - 'about_cases': results[14], - 'about_tissue': results[15], - 'about_data_processing': results[16], - 'acknowledgement': results[17], - 'experiment_design': results[18], - 'contributors': results[19], - 'citation': results[20], - 'notes': results[21], - 'investigator_firstname': results[22], - 'investigator_lastname': results[23], - 'investigator_address': results[24], - 'investigator_city': results[25], - 'investigator_state': results[26], - 'investigator_zipcode': results[27], - 'investigator_country': results[28], - 'investigator_phone': results[29], - 'investigator_email': results[30], - 'investigator_url': results[31], - 'organization_name': results[32], - 'investigator_id': results[33], - 'dataset_id': results[34], - 'dataset_status_is': results[35], - 'about_platform': results[36], - 'info_file_title': results[37], - 'specifics': results[38] - } - - return info_ob - \ No newline at end of file +import httplib, urllib2 +import re + +from flask import Flask, g + +from utility.logger import getLogger +logger = getLogger(__name__ ) + +class InfoPage(object): + def __init__(self, start_vars): + self.info = None + self.gn_accession_id = None + if 'gn_accession_id' in start_vars: + self.gn_accession_id = start_vars['gn_accession_id'] + self.info_page_name = start_vars['info_page_name'] + + self.get_info() + self.get_datasets_list() + + def get_info(self, create=False): + query_base = ("SELECT InfoPageName, GN_AccesionId, Species.MenuName, Species.TaxonomyId, Tissue.Name, InbredSet.Name, " + + "GeneChip.GeneChipName, GeneChip.GeoPlatform, AvgMethod.Name, Datasets.DatasetName, Datasets.GeoSeries, " + + "Datasets.PublicationTitle, DatasetStatus.DatasetStatusName, Datasets.Summary, Datasets.AboutCases, " + + "Datasets.AboutTissue, Datasets.AboutDataProcessing, Datasets.Acknowledgment, Datasets.ExperimentDesign, " + + "Datasets.Contributors, Datasets.Citation, Datasets.Notes, Investigators.FirstName, Investigators.LastName, " + + "Investigators.Address, Investigators.City, Investigators.State, Investigators.ZipCode, Investigators.Country, " + + "Investigators.Phone, Investigators.Email, Investigators.Url, Organizations.OrganizationName, " + + "InvestigatorId, DatasetId, DatasetStatusId, Datasets.AboutPlatform, InfoFileTitle, Specifics " + + "FROM InfoFiles " + + "LEFT JOIN Species USING (SpeciesId) " + + "LEFT JOIN Tissue USING (TissueId) " + + "LEFT JOIN InbredSet USING (InbredSetId) " + + "LEFT JOIN GeneChip USING (GeneChipId) " + + "LEFT JOIN AvgMethod USING (AvgMethodId) " + + "LEFT JOIN Datasets USING (DatasetId) " + + "LEFT JOIN Investigators USING (InvestigatorId) " + + "LEFT JOIN Organizations USING (OrganizationId) " + + "LEFT JOIN DatasetStatus USING (DatasetStatusId) WHERE ") + + if self.gn_accession_id: + final_query = query_base + "GN_AccesionId = {}".format(self.gn_accession_id) + results = g.db.execute(final_query).fetchone() + if self.info_page_name and not results: + final_query = query_base + "InfoPageName={}".format(self.info_page_name) + elif self.info_page_name: + final_query = query_base + "InfoPageName={}".format(self.info_page_name) + results = g.db.execute(final_query).fetchone() + else: + raise 'No correct parameter found' + + if results: + self.info = process_query_results(results) + + if (not results or len(results) < 1) and self.info_page_name and create: + insert_sql = "INSERT INTO InfoFiles SET InfoFiles.InfoPageName={}".format(self.info_page_name) + return self.get_info() + + if not self.gn_accession_id and self.info: + self.gn_accession_id = self.info['accession_id'] + if not self.info_page_name and self.info: + self.info_page_name = self.info['info_page_name'] + + def get_datasets_list(self): + self.filelist = [] + try: + response = urllib2.urlopen("http://datafiles.genenetwork.org/download/GN%s" % self.gn_accession_id) + data = response.read() + + matches = re.findall(r".+?", data, re.DOTALL) + for i, match in enumerate(matches): + if i == 0: + continue + cells = re.findall(r".+?", match, re.DOTALL) + full_filename = re.search(r"(.+?)<", cells[2]).group(1).strip() + filedate = "N/A" #ZS: Since we can't get it for now + + self.filelist.append([filename, filedate, filesize]) + except Exception, e: + pass + +def process_query_results(results): + info_ob = { + 'info_page_name': results[0], + 'accession_id': results[1], + 'menu_name': results[2], + 'taxonomy_id': results[3], + 'tissue_name': results[4], + 'group_name': results[5], + 'gene_chip_name': results[6], + 'geo_platform': results[7], + 'avg_method_name': results[8], + 'dataset_name': results[9], + 'geo_series': results[10], + 'publication_title': results[11], + 'dataset_status_name': results[12], + 'dataset_summary': results[13], + 'about_cases': results[14], + 'about_tissue': results[15], + 'about_data_processing': results[16], + 'acknowledgement': results[17], + 'experiment_design': results[18], + 'contributors': results[19], + 'citation': results[20], + 'notes': results[21], + 'investigator_firstname': results[22], + 'investigator_lastname': results[23], + 'investigator_address': results[24], + 'investigator_city': results[25], + 'investigator_state': results[26], + 'investigator_zipcode': results[27], + 'investigator_country': results[28], + 'investigator_phone': results[29], + 'investigator_email': results[30], + 'investigator_url': results[31], + 'organization_name': results[32], + 'investigator_id': results[33], + 'dataset_id': results[34], + 'dataset_status_is': results[35], + 'about_platform': results[36], + 'info_file_title': results[37], + 'specifics': results[38] + } + + return info_ob + -- cgit v1.2.3 From 0dc2401b9f66456315808f4c781dd88417941b01 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 26 Sep 2020 01:16:58 +0300 Subject: Run 2to3-8 tool through file * wqflask/wqflask/db_info.py: Run: 2to3-3.8 -w wqflask/wqflask/db_info.py --- wqflask/wqflask/db_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index 5f3f5e3f..45a4c96e 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -1,4 +1,4 @@ -import httplib, urllib2 +import http.client, urllib.request, urllib.error, urllib.parse import re from flask import Flask, g @@ -63,7 +63,7 @@ class InfoPage(object): def get_datasets_list(self): self.filelist = [] try: - response = urllib2.urlopen("http://datafiles.genenetwork.org/download/GN%s" % self.gn_accession_id) + response = urllib.request.urlopen("http://datafiles.genenetwork.org/download/GN%s" % self.gn_accession_id) data = response.read() matches = re.findall(r".+?", data, re.DOTALL) @@ -77,7 +77,7 @@ class InfoPage(object): filedate = "N/A" #ZS: Since we can't get it for now self.filelist.append([filename, filedate, filesize]) - except Exception, e: + except Exception as e: pass def process_query_results(results): -- cgit v1.2.3 From 1ac412ce0273081fc8df8296b4d03d46e4922fd8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 26 Sep 2020 01:18:40 +0300 Subject: Apply PEP 8 to buffer * wqflask/wqflask/db_info.py: Run pep8 through buffers. Fixes inconsistent use of tabs and spaces in indentation which trips up unittests. --- wqflask/wqflask/db_info.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index 45a4c96e..f420b472 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -1,10 +1,14 @@ -import http.client, urllib.request, urllib.error, urllib.parse +import http.client +import urllib.request +import urllib.error +import urllib.parse import re from flask import Flask, g from utility.logger import getLogger -logger = getLogger(__name__ ) +logger = getLogger(__name__) + class InfoPage(object): def __init__(self, start_vars): @@ -38,12 +42,15 @@ class InfoPage(object): "LEFT JOIN DatasetStatus USING (DatasetStatusId) WHERE ") if self.gn_accession_id: - final_query = query_base + "GN_AccesionId = {}".format(self.gn_accession_id) + final_query = query_base + \ + "GN_AccesionId = {}".format(self.gn_accession_id) results = g.db.execute(final_query).fetchone() if self.info_page_name and not results: - final_query = query_base + "InfoPageName={}".format(self.info_page_name) + final_query = query_base + \ + "InfoPageName={}".format(self.info_page_name) elif self.info_page_name: - final_query = query_base + "InfoPageName={}".format(self.info_page_name) + final_query = query_base + \ + "InfoPageName={}".format(self.info_page_name) results = g.db.execute(final_query).fetchone() else: raise 'No correct parameter found' @@ -52,18 +59,20 @@ class InfoPage(object): self.info = process_query_results(results) if (not results or len(results) < 1) and self.info_page_name and create: - insert_sql = "INSERT INTO InfoFiles SET InfoFiles.InfoPageName={}".format(self.info_page_name) + insert_sql = "INSERT INTO InfoFiles SET InfoFiles.InfoPageName={}".format( + self.info_page_name) return self.get_info() if not self.gn_accession_id and self.info: self.gn_accession_id = self.info['accession_id'] if not self.info_page_name and self.info: - self.info_page_name = self.info['info_page_name'] + self.info_page_name = self.info['info_page_name'] def get_datasets_list(self): self.filelist = [] try: - response = urllib.request.urlopen("http://datafiles.genenetwork.org/download/GN%s" % self.gn_accession_id) + response = urllib.request.urlopen( + "http://datafiles.genenetwork.org/download/GN%s" % self.gn_accession_id) data = response.read() matches = re.findall(r".+?", data, re.DOTALL) @@ -71,10 +80,11 @@ class InfoPage(object): if i == 0: continue cells = re.findall(r".+?", match, re.DOTALL) - full_filename = re.search(r"(.+?)<", cells[2]).group(1).strip() - filedate = "N/A" #ZS: Since we can't get it for now + filedate = "N/A" # ZS: Since we can't get it for now self.filelist.append([filename, filedate, filesize]) except Exception as e: @@ -125,3 +135,4 @@ def process_query_results(results): return info_ob + -- cgit v1.2.3 From f67b0c92af93e3153fe9375e98a139386a4f2fb0 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 26 Sep 2020 01:21:10 +0300 Subject: Fix failing tests * wqflask/tests/base/test_trait.py: Remove decode function call which is no longer supported in python3. --- wqflask/tests/base/test_trait.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/tests/base/test_trait.py b/wqflask/tests/base/test_trait.py index 9c3154f3..d333458a 100644 --- a/wqflask/tests/base/test_trait.py +++ b/wqflask/tests/base/test_trait.py @@ -93,6 +93,6 @@ class TestRetrieveTraitInfo(unittest.TestCase): test_trait = retrieve_trait_info(trait=mock_trait, dataset=mock_dataset) self.assertEqual(test_trait.abbreviation, - "ファイルを画面毎に見て行くには、次のコマンドを使います。".decode('utf-8')) + "ファイルを画面毎に見て行くには、次のコマンドを使います。") self.assertEqual(test_trait.authors, - "Jane Doe かいと".decode('utf-8')) + "Jane Doe かいと") -- cgit v1.2.3 From cde844b7de1875bda37c6672acf712c4a8bf2b79 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 26 Sep 2020 01:22:47 +0300 Subject: Use Python3 image to run tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fd9a886..f27feb5f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ on: jobs: unittest: runs-on: ubuntu-latest - container: bonfacekilz/python2-genenetwork2:latest + container: bonfacekilz/python3-genenetwork2:latest steps: # First start with mariadb set then checkout. The checkout gives -- cgit v1.2.3 From bbcecf8e7d8389b8466fe51dde6538387fdce6b5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 26 Sep 2020 01:33:52 +0300 Subject: Fix failing tests by replacing mock with unittest.mock * wqflask/tests/base/test_data_set.py (imports): Run: `find . -type f -name "*py" -print0 | xargs -0 sed -i \ "s|import mock|from unittest import mock|g"` * wqflask/tests/base/test_trait.py(imports): Ditto. * wqflask/tests/utility/test_authentication_tools.py(imports): Ditto. * wqflask/tests/utility/test_hmac.py(imports). Ditto. * wqflask/tests/wqflask/api/test_gen_menu.py: Ditto --- wqflask/tests/base/test_data_set.py | 2 +- wqflask/tests/base/test_trait.py | 2 +- wqflask/tests/utility/test_authentication_tools.py | 2 +- wqflask/tests/utility/test_hmac.py | 2 +- wqflask/tests/wqflask/api/test_gen_menu.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/tests/base/test_data_set.py b/wqflask/tests/base/test_data_set.py index b1de4932..96563a16 100644 --- a/wqflask/tests/base/test_data_set.py +++ b/wqflask/tests/base/test_data_set.py @@ -1,7 +1,7 @@ """Tests for wqflask/base/data_set.py""" import unittest -import mock +from unittest import mock from wqflask import app from .data import gen_menu_json diff --git a/wqflask/tests/base/test_trait.py b/wqflask/tests/base/test_trait.py index d333458a..bf4e88e0 100644 --- a/wqflask/tests/base/test_trait.py +++ b/wqflask/tests/base/test_trait.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Tests wqflask/base/trait.py""" import unittest -import mock +from unittest import mock from base.trait import GeneralTrait from base.trait import retrieve_trait_info diff --git a/wqflask/tests/utility/test_authentication_tools.py b/wqflask/tests/utility/test_authentication_tools.py index ef94eabc..5c391be5 100644 --- a/wqflask/tests/utility/test_authentication_tools.py +++ b/wqflask/tests/utility/test_authentication_tools.py @@ -1,6 +1,6 @@ """Tests for authentication tools""" import unittest -import mock +from unittest import mock from utility.authentication_tools import check_resource_availability from utility.authentication_tools import add_new_resource diff --git a/wqflask/tests/utility/test_hmac.py b/wqflask/tests/utility/test_hmac.py index 16b50771..7c61c0a6 100644 --- a/wqflask/tests/utility/test_hmac.py +++ b/wqflask/tests/utility/test_hmac.py @@ -2,7 +2,7 @@ """Test hmac utility functions""" import unittest -import mock +from unittest import mock from utility.hmac import data_hmac from utility.hmac import url_for_hmac diff --git a/wqflask/tests/wqflask/api/test_gen_menu.py b/wqflask/tests/wqflask/api/test_gen_menu.py index bf41054d..84898bd1 100644 --- a/wqflask/tests/wqflask/api/test_gen_menu.py +++ b/wqflask/tests/wqflask/api/test_gen_menu.py @@ -1,6 +1,6 @@ """Test cases for wqflask.api.gen_menu""" import unittest -import mock +from unittest import mock from wqflask import app from wqflask.api.gen_menu import gen_dropdown_json -- cgit v1.2.3 From a8c983a69b12ff32a30d488d7f326915f1aa7d63 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 28 Sep 2020 18:00:00 +0300 Subject: Fix casting error * wqflask/base/trait.py (retrieve_trait_info): If the description_string or discription_display value return a None object, evaluate to an empty string. Fixes errors related to *len(None)*. --- wqflask/base/trait.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 7666348e..03c4b079 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -507,13 +507,14 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): description_string = unicode(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') target_string = unicode(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') - if len(description_string) > 1 and description_string != 'None': + if str(description_string or "") != "" and description_string != 'None': description_display = description_string else: description_display = trait.symbol - if (len(description_display) > 1 and description_display != 'N/A' and - len(target_string) > 1 and target_string != 'None'): + if (str(description_display or "") != "" and + description_display != 'N/A' and + str(target_string or "") != "" and target_string != 'None'): description_display = description_display + '; ' + target_string.strip() # Save it for the jinja2 template @@ -607,4 +608,4 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): else: raise KeyError, `trait.name`+' information is not found in the database.' - return trait \ No newline at end of file + return trait -- cgit v1.2.3 From 21ec6863d7f8380d66c39f1ed3b8d9cc0dc86e49 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 28 Sep 2020 18:34:53 +0300 Subject: Fix failing test * wqflask/tests/base/test_trait.py: Add app_context to test class, otherwise an error related to working outside the app_context is generated. Adding the app_context is required when mocking "Flask.g". --- wqflask/tests/base/test_trait.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/wqflask/tests/base/test_trait.py b/wqflask/tests/base/test_trait.py index 60ebaee0..826ccefd 100644 --- a/wqflask/tests/base/test_trait.py +++ b/wqflask/tests/base/test_trait.py @@ -3,6 +3,7 @@ import unittest from unittest import mock +from wqflask import app from base.trait import GeneralTrait from base.trait import retrieve_trait_info @@ -31,6 +32,14 @@ class MockTrait(GeneralTrait): class TestRetrieveTraitInfo(unittest.TestCase): """Tests for 'retrieve_trait_info'""" + + def setUp(self): + self.app_context = app.app_context() + self.app_context.push() + + def tearDown(self): + self.app_context.pop() + def test_retrieve_trait_info_with_empty_dataset(self): """Test that an exception is raised when dataset is empty""" with self.assertRaises(AssertionError): @@ -104,7 +113,7 @@ class TestRetrieveTraitInfo(unittest.TestCase): resource_id_mock, g_mock, requests_mock): - """Test """ + """Test retrieve trait info when lrs has a value""" resource_id_mock.return_value = 1 g_mock.db.execute.return_value.fetchone = mock.Mock() g_mock.db.execute.return_value.fetchone.side_effect = [ -- cgit v1.2.3 From ce728fa92e02725202bca486e8336b2e1158f4ef Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 29 Sep 2020 15:57:01 +0300 Subject: Add test for process_traits from collections.py * wqflask/tests/wqflask/test_collect.py: Add it. --- wqflask/tests/wqflask/test_collect.py | 73 +++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 wqflask/tests/wqflask/test_collect.py diff --git a/wqflask/tests/wqflask/test_collect.py b/wqflask/tests/wqflask/test_collect.py new file mode 100644 index 00000000..06609b4c --- /dev/null +++ b/wqflask/tests/wqflask/test_collect.py @@ -0,0 +1,73 @@ +"""Test cases for some methods in collect.py""" + +import unittest +import mock + +from flask import Flask +from wqflask.collect import process_traits + +app = Flask(__name__) + + +class MockSession: + """Helper class for mocking wqflask.collect.g.user_session.logged_in""" + def __init__(self, is_logged_in=False): + self.is_logged_in = is_logged_in + + @property + def logged_in(self): + return self.is_logged_in + + +class MockFlaskG: + """Helper class for mocking wqflask.collect.g.user_session""" + def __init__(self, is_logged_in=False): + self.is_logged_in = is_logged_in + + @property + def user_session(self): + if self.is_logged_in: + return MockSession(is_logged_in=True) + return MockSession() + + +class TestCollect(unittest.TestCase): + + def setUp(self): + self.app_context = app.app_context() + self.app_context.push() + + def tearDown(self): + self.app_context.pop() + + @mock.patch("wqflask.collect.g", MockFlaskG()) + def test_process_traits_with_bytestring(self): + """ + Test that the correct traits are returned when the user is logged + out and bytes are used. + """ + self.assertEqual(process_traits( + b'1452452_at:HC_M2_0606_P:163d04f7db7c9e110de6,' + b'1452447_at:HC_M2_0606_P:eeece8fceb67072debea,' + b'1451401_a_at:HC_M2_0606_P:a043d23b3b3906d8318e,' + b'1429252_at:HC_M2_0606_P:6fa378b349bc9180e8f5'), + set(['1429252_at:HC_M2_0606_P', + '1451401_a_at:HC_M2_0606_P', + '1452447_at:HC_M2_0606_P', + '1452452_at:HC_M2_0606_P'])) + + @mock.patch("wqflask.collect.g", MockFlaskG()) + def test_process_traits_with_normal_string(self): + """ + Test that the correct traits are returned when the user is logged + out and a normal string is used. + """ + self.assertEqual(process_traits( + '1452452_at:HC_M2_0606_P:163d04f7db7c9e110de6,' + '1452447_at:HC_M2_0606_P:eeece8fceb67072debea,' + '1451401_a_at:HC_M2_0606_P:a043d23b3b3906d8318e,' + '1429252_at:HC_M2_0606_P:6fa378b349bc9180e8f5'), + set(['1429252_at:HC_M2_0606_P', + '1451401_a_at:HC_M2_0606_P', + '1452447_at:HC_M2_0606_P', + '1452452_at:HC_M2_0606_P'])) -- cgit v1.2.3 From d6fa2ebf9b3c53daec63e2ccd23feb87335a1e81 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 29 Sep 2020 17:09:11 +0300 Subject: Add check for bytes * wqflask/wqflask/collect.py (process_traits): Check whether the passed "unprocessed_traits" is a byte otherwise an error is generated when bytes(the default) are passed. --- wqflask/wqflask/collect.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 15383603..e7dac72a 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -35,7 +35,9 @@ from utility.logger import getLogger logger = getLogger(__name__) def process_traits(unprocessed_traits): - if isinstance(unprocessed_traits, str): + if isinstance(unprocessed_traits, bytes): + unprocessed_traits = unprocessed_traits.decode('utf-8').split(",") + else: # It's a string unprocessed_traits = unprocessed_traits.split(",") traits = set() for trait in unprocessed_traits: -- cgit v1.2.3 From 744d0cd6cd5e2bca98acbec1b8a1eed6986f824e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 29 Sep 2020 22:59:38 +0300 Subject: Apply pep 8 to buffer * wqflask/wqflask/user_manager.py: Remove some stale comments. (imports): Apply pep-8 and remove unused imports. --- wqflask/wqflask/user_manager.py | 73 +++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 24191a15..7b25b68e 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -2,50 +2,62 @@ import os import hashlib import datetime import time -import logging import uuid -import hashlib import hmac import base64 -import urllib.parse - +import redis # used for collections import simplejson as json +import requests -#from redis import StrictRedis -import redis # used for collections -Redis = redis.StrictRedis() +from base.data_set import create_datasets_list -from flask import (Flask, g, render_template, url_for, request, make_response, - redirect, flash, abort) +from flask import g +from flask import render_template +from flask import url_for +from flask import request +from flask import make_response +from flask import redirect +from flask import flash from wqflask import app -from pprint import pformat as pf - -from wqflask import pbkdf2 # password hashing +from wqflask import pbkdf2 # password hashing from wqflask.database import db_session from wqflask import model -from utility import Bunch, Struct, after +from smtplib import SMTP + +from pprint import pformat as pf -import logging +from utility import Bunch +from utility import Struct from utility.logger import getLogger -logger = getLogger(__name__) -from base.data_set import create_datasets_list +from utility.redis_tools import get_user_id +from utility.redis_tools import get_user_by_unique_column +from utility.redis_tools import set_user_attribute +from utility.redis_tools import save_user +from utility.redis_tools import save_verification_code +from utility.redis_tools import check_verification_code +from utility.redis_tools import get_user_collections +from utility.redis_tools import save_collections -import requests +from utility.tools import SMTP_CONNECT +from utility.tools import SMTP_USERNAME +from utility.tools import SMTP_PASSWORD -from utility.redis_tools import get_user_id, get_user_by_unique_column, set_user_attribute, save_user, save_verification_code, check_verification_code, get_user_collections, save_collections -from smtplib import SMTP -from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_ALCHEMY +logger = getLogger(__name__) + + +Redis = redis.StrictRedis() THREE_DAYS = 60 * 60 * 24 * 3 -#THREE_DAYS = 45 + def timestamp(): return datetime.datetime.utcnow().isoformat() + class AnonUser(object): """Anonymous user handling""" cookie_name = 'anon_user_v1' @@ -228,25 +240,6 @@ class UserSession(object): return len(self.user_collections) -### -# ZS: This is currently not used, but I'm leaving it here commented out because the old "set superuser" code (at the bottom of this file) used it -### -# @property -# def user_ob(self): -# """Actual sqlalchemy record""" -# # Only look it up once if needed, then store it -# # raise "OBSOLETE: use ElasticSearch instead" -# try: -# if LOG_SQL_ALCHEMY: -# logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG) -# -# # Already did this before -# return self.db_object -# except AttributeError: -# # Doesn't exist so we'll create it -# self.db_object = model.User.query.get(self.user_id) -# return self.db_object - def add_collection(self, collection_name, traits): """Add collection into ElasticSearch""" -- cgit v1.2.3 From 52ce47b7e1417bfa877ea68d2bcb119fc951d1d1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 29 Sep 2020 23:13:16 +0300 Subject: Remove undefined variable --- wqflask/wqflask/collect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index e7dac72a..9752c8b5 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -124,7 +124,8 @@ def collections_new(): g.user_session.add_traits_to_collection(collection_id, traits) return redirect(url_for('view_collection', uc_id=collection_id)) else: - CauseAnError + # CauseAnError + pass def create_new(collection_name): params = request.args -- cgit v1.2.3 From 475b6e333e6356103592b0983d5efb8e84eb73e7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 30 Sep 2020 02:09:27 +0300 Subject: Fix import in tests --- wqflask/tests/wqflask/test_collect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/tests/wqflask/test_collect.py b/wqflask/tests/wqflask/test_collect.py index 06609b4c..9a36132d 100644 --- a/wqflask/tests/wqflask/test_collect.py +++ b/wqflask/tests/wqflask/test_collect.py @@ -1,7 +1,7 @@ """Test cases for some methods in collect.py""" import unittest -import mock +from unittest import mock from flask import Flask from wqflask.collect import process_traits -- cgit v1.2.3 From dce0cd8725e68e6984c8b54a2cc108e4f8c9df7a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 30 Sep 2020 16:55:25 +0300 Subject: Fix StopIteration error thrown in view collections page * wqflask/wqflask/user_session.py (user_id): Use byte-string when fetching user_id. See: https://github.com/genenetwork/genenetwork2/pull/422#issuecomment-701082458 --- wqflask/wqflask/user_session.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index c1f38396..9f4c5458 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -127,10 +127,10 @@ class UserSession(object): @property def user_id(self): """Shortcut to the user_id""" - if 'user_id' not in self.record: - self.record['user_id'] = str(uuid.uuid4()) + if b'user_id' not in self.record: + self.record[b'user_id'] = str(uuid.uuid4()) - return self.record['user_id'] + return self.record[b'user_id'] @property def redis_user_id(self): -- cgit v1.2.3 From bbc1a53dd0a8a60e8c22f0576be0a2624cb5a3ad Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 30 Sep 2020 17:10:17 +0300 Subject: Apply pep 8 formatting * wqflask/wqflask/correlation/show_corr_results.py [imports]: Apply pep8 and remove unused imports. --- wqflask/wqflask/correlation/show_corr_results.py | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 91146e5b..8e0aa5df 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -23,25 +23,29 @@ import json import scipy import numpy import rpy2.robjects as ro # R Objects +import utility.logger +import utility.webqtlUtil from rpy2.robjects.packages import importr -utils = importr("utils") from base import data_set -from utility import webqtlUtil, helper_functions, corr_result_helpers, hmac -from db import webqtlDatabaseFunction -import utility.webqtlUtil #this is for parallel computing only. +from utility import helper_functions +from utility import corr_result_helpers +from utility import hmac + from wqflask.correlation import correlation_functions from utility.benchmark import Bench -import utility.webqtlUtil -from utility.type_checking import is_str, get_float, get_int, get_string -from utility.db_tools import escape +from utility.type_checking import is_str +from utility.type_checking import get_float +from utility.type_checking import get_int +from utility.type_checking import get_string +from utility.db_tools import escape -from flask import Flask, g +from flask import g -import utility.logger -logger = utility.logger.getLogger(__name__ ) +utils = importr("utils") +logger = utility.logger.getLogger(__name__) METHOD_LIT = "3" METHOD_TISSUE_PEARSON = "4" @@ -51,6 +55,7 @@ TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] TISSUE_MOUSE_DB = 1 + class CorrelationResults(object): def __init__(self, start_vars): # get trait list from db (database name) -- cgit v1.2.3 From f0aa59dc9413daceb580938c4d706e8831bf4daa Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 30 Sep 2020 17:11:06 +0300 Subject: Add missing import * wqflask/wqflask/correlation/show_corr_results.py: Add it. --- wqflask/wqflask/correlation/show_corr_results.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 8e0aa5df..e710bacd 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -26,6 +26,8 @@ import rpy2.robjects as ro # R Objects import utility.logger import utility.webqtlUtil +from base.trait import create_trait + from rpy2.robjects.packages import importr from base import data_set -- cgit v1.2.3 From 968bc7d468b2a24ded48b4deeb76b216ddb9a350 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 1 Oct 2020 16:10:41 +0300 Subject: Make object JSON serializable * wqflask/wqflask/search_results.py (SearchResultPage.gen_search_result): Some of the items is self.trait_list are in bytes, and *json.loads* can't serialize bytes into JSON. --- wqflask/wqflask/search_results.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index aa8f9e8f..ce836ce2 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -149,6 +149,10 @@ views.py). trait_dict['additive'] = "N/A" if this_trait.additive != "": trait_dict['additive'] = '%.3f' % this_trait.additive + # Convert any bytes in dict to a normal utf-8 string + for key in trait_dict.keys(): + if isinstance(trait_dict[key], bytes): + trait_dict[key] = trait_dict[key].decode('utf-8') trait_list.append(trait_dict) self.trait_list = json.dumps(trait_list) -- cgit v1.2.3 From 4aa6c2dc7d6e74316e793f219a6a44411fbef5ac Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 1 Oct 2020 18:16:45 +0300 Subject: Convert trait attributes to utf-8 strings if they were bytes --- wqflask/base/trait.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index d0c79119..24d0476d 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -147,7 +147,8 @@ class GeneralTrait(object): formatted = self.post_publication_description else: formatted = "Not available" - + if isinstance(formatted, bytes): + formatted = formatted.decode("utf-8") return formatted @property @@ -495,6 +496,8 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.haveinfo = True for i, field in enumerate(dataset.display_fields): holder = trait_info[i] + if isinstance(holder, bytes): + holder = holder.decode('utf-8') setattr(trait, field, holder) if dataset.type == 'Publish': -- cgit v1.2.3 From 94c36c8c93bea837a6e277d1dc3a2bc9120f9044 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 2 Oct 2020 00:39:06 +0300 Subject: Cast "previous_chr" to an int * wqflask/wqflask/marker_regression/gemma_mapping.py (parse_loco_output): Cast "previous_chr" to an int which prevents the error "Python typeerror: ‘>’ not supported between instances of ‘str’ and ‘int’" from being thrown. The error is **only** thrown in Python3. --- wqflask/wqflask/marker_regression/gemma_mapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index b858b573..68a8d5ba 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -218,9 +218,9 @@ def parse_loco_output(this_dataset, gwa_output_filename): marker['chr'] = int(line.split("\t")[0][3:]) else: marker['chr'] = int(line.split("\t")[0]) - if marker['chr'] > previous_chr: + if marker['chr'] > int(previous_chr): previous_chr = marker['chr'] - elif marker['chr'] < previous_chr: + elif marker['chr'] < int(previous_chr): break else: marker['chr'] = line.split("\t")[0] -- cgit v1.2.3 From 509bd3b90dcb6ab71f42385d1feccb154a6d5108 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 2 Oct 2020 00:46:36 +0300 Subject: Remove unnecessary commented out blocks * wqflask/wqflask/show_trait/SampleList.py: See above. --- wqflask/wqflask/show_trait/SampleList.py | 44 -------------------------------- 1 file changed, 44 deletions(-) diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 6fcf7cec..7a6f1664 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -102,7 +102,6 @@ class SampleList(object): self.attributes = {} for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name)): key, name = attr - #logger.debug("radish: %s - %s" % (key, name)) self.attributes[key] = Bunch() self.attributes[key].name = name self.attributes[key].distinct_values = [item.Value for item in values] @@ -153,49 +152,6 @@ class SampleList(object): """Returns true if SE values exist for any samples, otherwise false""" return any(sample.variance for sample in self.sample_list) - -# def get_transform_vals(dataset, trait): -# es = get_elasticsearch_connection(for_user=False) - -# logger.info("DATASET NAME:", dataset.name) - -# query = '{"bool": {"must": [{"match": {"name": "%s"}}, {"match": {"dataset": "%s"}}]}}' % (trait.name, dataset.name) - -# es_body = { -# "query": { -# "bool": { -# "must": [ -# { -# "match": { -# "name": "%s" % (trait.name) -# } -# }, -# { -# "match": { -# "dataset": "%s" % (dataset.name) -# } -# } -# ] -# } -# } -# } - -# response = es.search( index = "traits", doc_type = "trait", body = es_body ) -# logger.info("THE RESPONSE:", response) -# results = response['hits']['hits'] - -# if len(results) > 0: -# samples = results[0]['_source']['samples'] - -# sample_dict = {} -# for sample in samples: -# sample_dict[sample['name']] = sample['qnorm'] - -# #logger.info("SAMPLE DICT:", sample_dict) -# return sample_dict -# else: -# return None - def natural_sort_key(x): """Get expected results when using as a key for sort - ints or strings are sorted properly""" -- cgit v1.2.3 From 5aee7c35ef27dbfa590dfd30364000840dd0006a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 2 Oct 2020 01:04:23 +0300 Subject: Update natural_sort function * wqflask/wqflask/show_trait/SampleList.py (natural_sort): Add new function. (SampleList.get_attributes): Use new sort function to do natural_sort. --- wqflask/wqflask/show_trait/SampleList.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 7a6f1664..b43b57dc 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -1,3 +1,4 @@ +import re from flask import Flask, g from base import webqtlCaseData @@ -105,8 +106,7 @@ class SampleList(object): self.attributes[key] = Bunch() self.attributes[key].name = name self.attributes[key].distinct_values = [item.Value for item in values] - self.attributes[key].distinct_values.sort(key=natural_sort_key) - + natural_sort(self.attributes[key].distinct_values) all_numbers = True for value in self.attributes[key].distinct_values: try: @@ -152,11 +152,15 @@ class SampleList(object): """Returns true if SE values exist for any samples, otherwise false""" return any(sample.variance for sample in self.sample_list) -def natural_sort_key(x): - """Get expected results when using as a key for sort - ints or strings are sorted properly""" - - try: - x = int(x) - except ValueError: - pass - return x + + +def natural_sort(list, key=lambda s:s): + """ + Sort the list into natural alphanumeric order. + """ + def get_alphanum_key_func(key): + convert = lambda text: int(text) if text.isdigit() else text + return lambda s: [convert(c) for c in re.split('([0-9]+)', key(s))] + sort_key = get_alphanum_key_func(key) + list.sort(key=sort_key) + -- cgit v1.2.3 From 5559aeca7989cfe0b454dff018eada6fb69a73a6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 2 Oct 2020 02:36:09 +0300 Subject: Replace is_integer with isinstance when checking for an int value --- wqflask/wqflask/marker_regression/display_mapping_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 5a7a4614..745b8e78 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2157,7 +2157,7 @@ class DisplayMappingResults(object): #ZS: Convert to int if all axis values are whole numbers all_int = True for item in LRSAxisList: - if item.is_integer(): + if isinstance(item, int): continue else: all_int = False -- cgit v1.2.3 From a3b9372cc24857ac38a1938bc2a94b2dc96471bb Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 2 Oct 2020 02:36:56 +0300 Subject: Cast variable properly to avoid type exceptions * wqflask/wqflask/marker_regression/run_mapping.py: See 94c36c8c. --- wqflask/wqflask/marker_regression/run_mapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index f42d2315..995923a7 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -409,8 +409,8 @@ class RunMapping(object): self.results_for_browser.append(browser_marker) self.annotations_for_browser.append(annot_marker) - if marker['chr'] > 0 or marker['chr'] == "X" or marker['chr'] == "X/Y": - if marker['chr'] > highest_chr or marker['chr'] == "X" or marker['chr'] == "X/Y": + if str(marker['chr']) > '0' or str(marker['chr']) == "X" or str(marker['chr']) == "X/Y": + if str(marker['chr']) > str(highest_chr) or str(marker['chr']) == "X" or str(marker['chr']) == "X/Y": highest_chr = marker['chr'] if ('lod_score' in marker.keys()) or ('lrs_value' in marker.keys()): if 'Mb' in marker.keys(): -- cgit v1.2.3 From ea3876a87f5f6ac6e5990514f485213181456bb2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 2 Oct 2020 02:41:41 +0300 Subject: Apply PEP 8 formatting --- wqflask/wqflask/show_trait/SampleList.py | 70 ++++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index b43b57dc..dccd1ba8 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -1,83 +1,82 @@ import re -from flask import Flask, g +import itertools +import utility.logger +from flask import g from base import webqtlCaseData -from utility import webqtlUtil, Plot, Bunch -from base.trait import GeneralTrait -import numpy as np -from scipy import stats -from pprint import pformat as pf +from utility import Plot +from utility import Bunch -import simplejson as json +from pprint import pformat as pf -import itertools +logger = utility.logger.getLogger(__name__) -import utility.logger -logger = utility.logger.getLogger(__name__ ) class SampleList(object): def __init__(self, dataset, sample_names, this_trait, - sample_group_type = "primary", - header = "Samples"): + sample_group_type="primary", + header="Samples"): self.dataset = dataset self.this_trait = this_trait self.sample_group_type = sample_group_type # primary or other self.header = header - self.sample_list = [] # The actual list + self.sample_list = [] # The actual list self.sample_attribute_values = {} self.get_attributes() - #self.sample_qnorm = get_transform_vals(self.dataset, this_trait) - if self.this_trait and self.dataset: self.get_extra_attribute_values() for counter, sample_name in enumerate(sample_names, 1): sample_name = sample_name.replace("_2nd_", "") - if isinstance(self.this_trait, list): #ZS: self.this_trait will be a list if it is a Temp trait - if counter <= len(self.this_trait) and str(self.this_trait[counter-1]).upper() != 'X': - sample = webqtlCaseData.webqtlCaseData(name=sample_name, value=float(self.this_trait[counter-1])) + # ZS: self.this_trait will be a list if it is a Temp trait + if isinstance(self.this_trait, list): + if (counter <= len(self.this_trait) and + str(self.this_trait[counter-1]).upper() != 'X'): + sample = webqtlCaseData.webqtlCaseData( + name=sample_name, + value=float(self.this_trait[counter-1])) else: sample = webqtlCaseData.webqtlCaseData(name=sample_name) else: - #ZS - If there's no value for the sample/strain, create the sample object (so samples with no value are still displayed in the table) + # ZS - If there's no value for the sample/strain, + # create the sample object (so samples with no value + # are still displayed in the table) try: sample = self.this_trait.data[sample_name] except KeyError: - #logger.debug("No sample %s, let's create it now" % sample_name) sample = webqtlCaseData.webqtlCaseData(name=sample_name) sample.extra_info = {} - if self.dataset.group.name == 'AXBXA' and sample_name in ('AXB18/19/20', 'AXB13/14', 'BXA8/17'): + if (self.dataset.group.name == 'AXBXA' and + sample_name in ('AXB18/19/20', 'AXB13/14', 'BXA8/17')): sample.extra_info['url'] = "/mouseCross.html#AXB/BXA" sample.extra_info['css_class'] = "fs12" sample.this_id = str(counter) - #### For extra attribute columns; currently only used by several datasets - Zach + # For extra attribute columns; currently only used by + # several datasets - Zach if self.sample_attribute_values: - sample.extra_attributes = self.sample_attribute_values.get(sample_name, {}) - #logger.debug("sample.extra_attributes is", pf(sample.extra_attributes)) - + sample.extra_attributes = self.sample_attribute_values.get( + sample_name, {}) self.sample_list.append(sample) - - #logger.debug("attribute vals are", pf(self.sample_attribute_values)) - self.do_outliers() def __repr__(self): return " --> %s" % (pf(self.__dict__)) def do_outliers(self): - values = [sample.value for sample in self.sample_list if sample.value != None] + values = [sample.value for sample in self.sample_list + if sample.value is not None] upper_bound, lower_bound = Plot.find_outliers(values) for sample in self.sample_list: @@ -105,7 +104,8 @@ class SampleList(object): key, name = attr self.attributes[key] = Bunch() self.attributes[key].name = name - self.attributes[key].distinct_values = [item.Value for item in values] + self.attributes[key].distinct_values = [ + item.Value for item in values] natural_sort(self.attributes[key].distinct_values) all_numbers = True for value in self.attributes[key].distinct_values: @@ -138,8 +138,9 @@ class SampleList(object): for item in items: attribute_value = item.Value - #ZS: If it's an int, turn it into one for sorting - #(for example, 101 would be lower than 80 if they're strings instead of ints) + # ZS: If it's an int, turn it into one for sorting + # (for example, 101 would be lower than 80 if + # they're strings instead of ints) try: attribute_value = int(attribute_value) except ValueError: @@ -154,13 +155,12 @@ class SampleList(object): return any(sample.variance for sample in self.sample_list) -def natural_sort(list, key=lambda s:s): +def natural_sort(list, key=lambda s: s): """ Sort the list into natural alphanumeric order. """ def get_alphanum_key_func(key): - convert = lambda text: int(text) if text.isdigit() else text + def convert(text): return int(text) if text.isdigit() else text return lambda s: [convert(c) for c in re.split('([0-9]+)', key(s))] sort_key = get_alphanum_key_func(key) list.sort(key=sort_key) - -- cgit v1.2.3 From 11180abbba8af29499e6eda2c7c408fc4e0eb80a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 2 Oct 2020 03:16:18 +0300 Subject: Split author's list into it's own variable * wqflask/wqflask/templates/correlation_page.html: Ditto. Prevents jinja from throwing an error due to compaing a list type with an int. --- wqflask/wqflask/templates/correlation_page.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index bb2a697f..435c0b37 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -170,7 +170,8 @@ {% endif %} {% if trait.description_display|length > 70 %}{{ trait.description_display[:70] }}...{% else %}{{ trait.description_display }}{% endif %} {% if trait.authors %} - {% if trait.authors.split(',') > 6 %}{{ trait.authors.split(',')[:6]|join(', ') }}, et al.{% else %}{{ trait.authors }}{% endif %} + {% set authors_list = trait.authors.split(',') %} + {% if authors_list|length > 6 %}{{ authors_list[:6]|join(', ') }}, et al.{% else %}{{ trait.authors }}{% endif %} {% else %} N/A {% endif %} -- cgit v1.2.3 From f0dd46e668a55bd024f5dee4e99cf1215a107c26 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 2 Oct 2020 03:48:23 +0300 Subject: Ignore utf-8 encoding errors * wqflask/base/trait.py (retrieve_trait_info): Add extra param to ignore errors such as: "utf-8 codec can't decode byte 0xb5 in position 0: invalid start byte" thrown in Python3. --- wqflask/base/trait.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 24d0476d..a493cfbf 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -497,7 +497,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): for i, field in enumerate(dataset.display_fields): holder = trait_info[i] if isinstance(holder, bytes): - holder = holder.decode('utf-8') + holder = holder.decode("utf-8", errors="ignore") setattr(trait, field, holder) if dataset.type == 'Publish': -- cgit v1.2.3 From 4aa138360fd41f9d496e00aa744700cd4d8a53a6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 7 Oct 2020 15:08:05 +0300 Subject: Apply PEP-8 formatting --- .../wqflask/correlation_matrix/show_corr_matrix.py | 29 ++-- wqflask/wqflask/network_graph/network_graph.py | 100 ++++++------ wqflask/wqflask/wgcna/wgcna_analysis.py | 169 ++++++++++++--------- 3 files changed, 156 insertions(+), 142 deletions(-) diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 49ba9e5d..c0d84aa2 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -22,33 +22,24 @@ import datetime import math import numpy as np import scipy - -from rpy2.robjects.packages import importr import rpy2.robjects as robjects +import utility.webqtlUtil # this is for parallel computing only. +import utility.logger -from pprint import pformat as pf +from base import data_set +from functools import reduce +from rpy2.robjects.packages import importr +from utility import webqtlUtil +from utility import helper_functions +from utility import corr_result_helpers from utility.redis_tools import get_redis_conn -from functools import reduce + +logger = utility.logger.getLogger(__name__) Redis = get_redis_conn() THIRTY_DAYS = 60 * 60 * 24 * 30 -from utility.THCell import THCell -from utility.TDCell import TDCell -from base.trait import GeneralTrait -from base import data_set -from utility import webqtlUtil, helper_functions, corr_result_helpers - -from db import webqtlDatabaseFunction -import utility.webqtlUtil #this is for parallel computing only. -from wqflask.correlation import correlation_functions -from utility.benchmark import Bench - -from flask import Flask, g, url_for - -import utility.logger -logger = utility.logger.getLogger(__name__ ) class CorrelationMatrix(object): diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index cfefe4ec..723a749f 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -1,4 +1,4 @@ -## Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License @@ -20,25 +20,19 @@ import scipy import simplejson as json -from pprint import pformat as pf - -from utility.TDCell import TDCell from base.trait import create_trait from base import data_set -from utility import webqtlUtil, helper_functions, corr_result_helpers +from utility import helper_functions +from utility import corr_result_helpers from utility.tools import GN2_BRANCH_URL -from db import webqtlDatabaseFunction -import utility.webqtlUtil #this is for parallel computing only. -from wqflask.correlation import correlation_functions - -from flask import Flask, g class NetworkGraph(object): def __init__(self, start_vars): - trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] + trait_db_list = [trait.strip() + for trait in start_vars['trait_list'].split(',')] helper_functions.get_trait_db_obs(self, trait_db_list) @@ -66,7 +60,8 @@ class NetworkGraph(object): this_trait_vals.append('') self.sample_data.append(this_trait_vals) - self.lowest_overlap = 8 #ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning) + # ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning) + self.lowest_overlap = 8 self.nodes_list = [] self.edges_list = [] @@ -78,9 +73,9 @@ class NetworkGraph(object): this_sample_data = this_trait.data corr_result_row = [] - is_spearman = False #ZS: To determine if it's above or below the diagonal + is_spearman = False # ZS: To determine if it's above or below the diagonal - max_corr = 0 #ZS: Used to determine whether node should be hidden when correlation coefficient slider is used + max_corr = 0 # ZS: Used to determine whether node should be hidden when correlation coefficient slider is used for target in self.trait_list: target_trait = target[0] @@ -99,20 +94,23 @@ class NetworkGraph(object): this_trait_vals.append(sample_value) target_vals.append(target_sample_value) - this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(this_trait_vals, target_vals) + this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values( + this_trait_vals, target_vals) if num_overlap < self.lowest_overlap: self.lowest_overlap = num_overlap if num_overlap == 0: continue else: - pearson_r, pearson_p = scipy.stats.pearsonr(this_trait_vals, target_vals) + pearson_r, pearson_p = scipy.stats.pearsonr( + this_trait_vals, target_vals) if is_spearman == False: sample_r, sample_p = pearson_r, pearson_p if sample_r == 1: continue else: - sample_r, sample_p = scipy.stats.spearmanr(this_trait_vals, target_vals) + sample_r, sample_p = scipy.stats.spearmanr( + this_trait_vals, target_vals) if -1 <= sample_r < -0.7: color = "#0000ff" @@ -130,44 +128,44 @@ class NetworkGraph(object): color = "#ffa500" width = 2 elif 0.7 <= sample_r <= 1: - color = "#ff0000" - width = 3 + color = "#ff0000" + width = 3 else: color = "#000000" - width = 0 + width = 0 if abs(sample_r) > max_corr: max_corr = abs(sample_r) - edge_data = {'id' : str(this_trait.name) + '_to_' + str(target_trait.name), - 'source' : str(this_trait.name) + ":" + str(this_trait.dataset.name), - 'target' : str(target_trait.name) + ":" + str(target_trait.dataset.name), - 'correlation' : round(sample_r, 3), - 'abs_corr' : abs(round(sample_r, 3)), - 'p_value' : round(sample_p, 3), - 'overlap' : num_overlap, - 'color' : color, - 'width' : width } + edge_data = {'id': str(this_trait.name) + '_to_' + str(target_trait.name), + 'source': str(this_trait.name) + ":" + str(this_trait.dataset.name), + 'target': str(target_trait.name) + ":" + str(target_trait.dataset.name), + 'correlation': round(sample_r, 3), + 'abs_corr': abs(round(sample_r, 3)), + 'p_value': round(sample_p, 3), + 'overlap': num_overlap, + 'color': color, + 'width': width} - edge_dict = { 'data' : edge_data } + edge_dict = {'data': edge_data} self.edges_list.append(edge_dict) if trait_db[1].type == "ProbeSet": - node_dict = { 'data' : {'id' : str(this_trait.name) + ":" + str(this_trait.dataset.name), - 'label' : this_trait.symbol, - 'symbol' : this_trait.symbol, - 'geneid' : this_trait.geneid, - 'omim' : this_trait.omim, - 'max_corr' : max_corr } } + node_dict = {'data': {'id': str(this_trait.name) + ":" + str(this_trait.dataset.name), + 'label': this_trait.symbol, + 'symbol': this_trait.symbol, + 'geneid': this_trait.geneid, + 'omim': this_trait.omim, + 'max_corr': max_corr}} elif trait_db[1].type == "Publish": - node_dict = { 'data' : {'id' : str(this_trait.name) + ":" + str(this_trait.dataset.name), - 'label' : this_trait.name, - 'max_corr' : max_corr } } + node_dict = {'data': {'id': str(this_trait.name) + ":" + str(this_trait.dataset.name), + 'label': this_trait.name, + 'max_corr': max_corr}} else: - node_dict = { 'data' : {'id' : str(this_trait.name) + ":" + str(this_trait.dataset.name), - 'label' : this_trait.name, - 'max_corr' : max_corr } } + node_dict = {'data': {'id': str(this_trait.name) + ":" + str(this_trait.dataset.name), + 'label': this_trait.name, + 'max_corr': max_corr}} self.nodes_list.append(node_dict) self.elements = json.dumps(self.nodes_list + self.edges_list) @@ -177,13 +175,13 @@ class NetworkGraph(object): for sample in self.all_sample_list: groups.append(1) - self.js_data = dict(traits = [trait.name for trait in self.traits], - groups = groups, - cols = list(range(len(self.traits))), - rows = list(range(len(self.traits))), - samples = self.all_sample_list, - sample_data = self.sample_data, - elements = self.elements,) + self.js_data = dict(traits=[trait.name for trait in self.traits], + groups=groups, + cols=list(range(len(self.traits))), + rows=list(range(len(self.traits))), + samples=self.all_sample_list, + sample_data=self.sample_data, + elements=self.elements,) def get_trait_db_obs(self, trait_db_list): self.trait_list = [] @@ -193,6 +191,6 @@ class NetworkGraph(object): trait_name, dataset_name = trait_db.split(":") dataset_ob = data_set.create_dataset(dataset_name) trait_ob = create_trait(dataset=dataset_ob, - name=trait_name, - cellid=None) + name=trait_name, + cellid=None) self.trait_list.append((trait_ob, dataset_ob)) diff --git a/wqflask/wqflask/wgcna/wgcna_analysis.py b/wqflask/wqflask/wgcna/wgcna_analysis.py index d79ad6df..0afe7e82 100644 --- a/wqflask/wqflask/wgcna/wgcna_analysis.py +++ b/wqflask/wqflask/wgcna/wgcna_analysis.py @@ -1,122 +1,146 @@ # WGCNA analysis for GN2 # Author / Maintainer: Danny Arends import sys -from numpy import * -import scipy as sp # SciPy import rpy2.robjects as ro # R Objects import rpy2.rinterface as ri +import array +from numpy import * from base.webqtlConfig import GENERATED_IMAGE_DIR -from utility import webqtlUtil # Random number for the image - -import base64 -import array +from rpy2.robjects.packages import importr +from utility import webqtlUtil # Random number for the image from utility import helper_functions -from rpy2.robjects.packages import importr utils = importr("utils") -## Get pointers to some common R functions -r_library = ro.r["library"] # Map the library function -r_options = ro.r["options"] # Map the options function -r_read_csv = ro.r["read.csv"] # Map the read.csv function -r_dim = ro.r["dim"] # Map the dim function -r_c = ro.r["c"] # Map the c function -r_cat = ro.r["cat"] # Map the cat function -r_paste = ro.r["paste"] # Map the paste function -r_unlist = ro.r["unlist"] # Map the unlist function -r_unique = ro.r["unique"] # Map the unique function -r_length = ro.r["length"] # Map the length function -r_unlist = ro.r["unlist"] # Map the unlist function -r_list = ro.r.list # Map the list function -r_matrix = ro.r.matrix # Map the matrix function -r_seq = ro.r["seq"] # Map the seq function -r_table = ro.r["table"] # Map the table function -r_names = ro.r["names"] # Map the names function -r_sink = ro.r["sink"] # Map the sink function -r_is_NA = ro.r["is.na"] # Map the is.na function -r_file = ro.r["file"] # Map the file function -r_png = ro.r["png"] # Map the png function for plotting -r_dev_off = ro.r["dev.off"] # Map the dev.off function +# Get pointers to some common R functions +r_library = ro.r["library"] # Map the library function +r_options = ro.r["options"] # Map the options function +r_read_csv = ro.r["read.csv"] # Map the read.csv function +r_dim = ro.r["dim"] # Map the dim function +r_c = ro.r["c"] # Map the c function +r_cat = ro.r["cat"] # Map the cat function +r_paste = ro.r["paste"] # Map the paste function +r_unlist = ro.r["unlist"] # Map the unlist function +r_unique = ro.r["unique"] # Map the unique function +r_length = ro.r["length"] # Map the length function +r_unlist = ro.r["unlist"] # Map the unlist function +r_list = ro.r.list # Map the list function +r_matrix = ro.r.matrix # Map the matrix function +r_seq = ro.r["seq"] # Map the seq function +r_table = ro.r["table"] # Map the table function +r_names = ro.r["names"] # Map the names function +r_sink = ro.r["sink"] # Map the sink function +r_is_NA = ro.r["is.na"] # Map the is.na function +r_file = ro.r["file"] # Map the file function +r_png = ro.r["png"] # Map the png function for plotting +r_dev_off = ro.r["dev.off"] # Map the dev.off function + class WGCNA(object): def __init__(self): print("Initialization of WGCNA") #log = r_file("/tmp/genenetwork_wcgna.log", open = "wt") - #r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file + # r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file #r_sink(log, type = "message") - r_library("WGCNA") # Load WGCNA - Should only be done once, since it is quite expensive - r_options(stringsAsFactors = False) + # Load WGCNA - Should only be done once, since it is quite expensive + r_library("WGCNA") + r_options(stringsAsFactors=False) print("Initialization of WGCNA done, package loaded in R session") - self.r_enableWGCNAThreads = ro.r["enableWGCNAThreads"] # Map the enableWGCNAThreads function - self.r_pickSoftThreshold = ro.r["pickSoftThreshold"] # Map the pickSoftThreshold function - self.r_blockwiseModules = ro.r["blockwiseModules"] # Map the blockwiseModules function - self.r_labels2colors = ro.r["labels2colors"] # Map the labels2colors function - self.r_plotDendroAndColors = ro.r["plotDendroAndColors"] # Map the plotDendroAndColors function + # Map the enableWGCNAThreads function + self.r_enableWGCNAThreads = ro.r["enableWGCNAThreads"] + # Map the pickSoftThreshold function + self.r_pickSoftThreshold = ro.r["pickSoftThreshold"] + # Map the blockwiseModules function + self.r_blockwiseModules = ro.r["blockwiseModules"] + # Map the labels2colors function + self.r_labels2colors = ro.r["labels2colors"] + # Map the plotDendroAndColors function + self.r_plotDendroAndColors = ro.r["plotDendroAndColors"] print("Obtained pointers to WGCNA functions") def run_analysis(self, requestform): print("Starting WGCNA analysis on dataset") - self.r_enableWGCNAThreads() # Enable multi threading - self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] - print(("Retrieved phenotype data from database", requestform['trait_list'])) + # Enable multi threading + self.r_enableWGCNAThreads() + self.trait_db_list = [trait.strip() + for trait in requestform['trait_list'].split(',')] + print(("Retrieved phenotype data from database", + requestform['trait_list'])) helper_functions.get_trait_db_obs(self, self.trait_db_list) self.input = {} # self.input contains the phenotype values we need to send to R - strains = [] # All the strains we have data for (contains duplicates) - traits = [] # All the traits we have data for (should not contain duplicates) + # All the strains we have data for (contains duplicates) + strains = [] + # All the traits we have data for (should not contain duplicates) + traits = [] for trait in self.trait_list: traits.append(trait[0].name) self.input[trait[0].name] = {} for strain in trait[0].data: strains.append(strain) - self.input[trait[0].name][strain] = trait[0].data[strain].value + self.input[trait[0].name][strain] = trait[0].data[strain].value # Transfer the load data from python to R - uStrainsR = r_unique(ro.Vector(strains)) # Unique strains in R vector + # Unique strains in R vector + uStrainsR = r_unique(ro.Vector(strains)) uTraitsR = r_unique(ro.Vector(traits)) # Unique traits in R vector r_cat("The number of unique strains:", r_length(uStrainsR), "\n") r_cat("The number of unique traits:", r_length(uTraitsR), "\n") # rM is the datamatrix holding all the data in R /rows = strains columns = traits - rM = ro.r.matrix(ri.NA_Real, nrow=r_length(uStrainsR), ncol=r_length(uTraitsR), dimnames = r_list(uStrainsR, uTraitsR)) + rM = ro.r.matrix(ri.NA_Real, nrow=r_length(uStrainsR), ncol=r_length( + uTraitsR), dimnames=r_list(uStrainsR, uTraitsR)) for t in uTraitsR: - trait = t[0] # R uses vectors every single element is a vector + # R uses vectors every single element is a vector + trait = t[0] for s in uStrainsR: - strain = s[0] # R uses vectors every single element is a vector + # R uses vectors every single element is a vector + strain = s[0] #DEBUG: print(trait, strain, " in python: ", self.input[trait].get(strain), "in R:", rM.rx(strain,trait)[0]) - rM.rx[strain, trait] = self.input[trait].get(strain) # Update the matrix location + rM.rx[strain, trait] = self.input[trait].get( + strain) # Update the matrix location sys.stdout.flush() self.results = {} - self.results['nphe'] = r_length(uTraitsR)[0] # Number of phenotypes/traits - self.results['nstr'] = r_length(uStrainsR)[0] # Number of strains + # Number of phenotypes/traits + self.results['nphe'] = r_length(uTraitsR)[0] + self.results['nstr'] = r_length( + uStrainsR)[0] # Number of strains self.results['phenotypes'] = uTraitsR # Traits used - self.results['strains'] = uStrainsR # Strains used in the analysis - self.results['requestform'] = requestform # Store the user specified parameters for the output page + # Strains used in the analysis + self.results['strains'] = uStrainsR + # Store the user specified parameters for the output page + self.results['requestform'] = requestform # Calculate soft threshold if the user specified the SoftThreshold variable if requestform.get('SoftThresholds') is not None: - powers = [int(threshold.strip()) for threshold in requestform['SoftThresholds'].rstrip().split(",")] - rpow = r_unlist(r_c(powers)) - print(("SoftThresholds: {} == {}".format(powers, rpow))) - self.sft = self.r_pickSoftThreshold(rM, powerVector = rpow, verbose = 5) - - print(("PowerEstimate: {}".format(self.sft[0]))) - self.results['PowerEstimate'] = self.sft[0] - if self.sft[0][0] is ri.NA_Integer: - print("No power is suitable for the analysis, just use 1") - self.results['Power'] = 1 # No power could be estimated - else: - self.results['Power'] = self.sft[0][0] # Use the estimated power + powers = [int(threshold.strip()) + for threshold in requestform['SoftThresholds'].rstrip().split(",")] + rpow = r_unlist(r_c(powers)) + print(("SoftThresholds: {} == {}".format(powers, rpow))) + self.sft = self.r_pickSoftThreshold( + rM, powerVector=rpow, verbose=5) + + print(("PowerEstimate: {}".format(self.sft[0]))) + self.results['PowerEstimate'] = self.sft[0] + if self.sft[0][0] is ri.NA_Integer: + print("No power is suitable for the analysis, just use 1") + # No power could be estimated + self.results['Power'] = 1 + else: + # Use the estimated power + self.results['Power'] = self.sft[0][0] else: - # The user clicked a button, so no soft threshold selection - self.results['Power'] = requestform.get('Power') # Use the power value the user gives + # The user clicked a button, so no soft threshold selection + # Use the power value the user gives + self.results['Power'] = requestform.get('Power') # Create the block wise modules using WGCNA - network = self.r_blockwiseModules(rM, power = self.results['Power'], TOMType = requestform['TOMtype'], minModuleSize = requestform['MinModuleSize'], verbose = 3) + network = self.r_blockwiseModules( + rM, power=self.results['Power'], TOMType=requestform['TOMtype'], minModuleSize=requestform['MinModuleSize'], verbose=3) # Save the network for the GUI self.results['network'] = network @@ -130,7 +154,8 @@ class WGCNA(object): self.results['imgloc'] = GENERATED_IMAGE_DIR + self.results['imgurl'] r_png(self.results['imgloc'], width=1000, height=600, type='cairo-png') mergedColors = self.r_labels2colors(network[1]) - self.r_plotDendroAndColors(network[5][0], mergedColors, "Module colors", dendroLabels = False, hang = 0.03, addGuide = True, guideHang = 0.05) + self.r_plotDendroAndColors(network[5][0], mergedColors, "Module colors", + dendroLabels=False, hang=0.03, addGuide=True, guideHang=0.05) r_dev_off() sys.stdout.flush() @@ -146,11 +171,11 @@ class WGCNA(object): print("Processing WGCNA output") template_vars = {} template_vars["input"] = self.input - template_vars["powers"] = self.sft[1:] # Results from the soft threshold analysis + # Results from the soft threshold analysis + template_vars["powers"] = self.sft[1:] template_vars["results"] = self.results self.render_image(results) sys.stdout.flush() - #r_sink(type = "message") # This restores R output to the stdout/stderr - #r_sink() # We should end the Rpy session more or less + # r_sink(type = "message") # This restores R output to the stdout/stderr + # r_sink() # We should end the Rpy session more or less return(dict(template_vars)) - -- cgit v1.2.3 From ef11f5e40ec0b8ef90a00b9d98d81cca2ff5d8d9 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 7 Oct 2020 15:58:14 +0300 Subject: Convert cmp function into a key function * wqflask/wqflask/correlation_matrix/show_corr_matrix.py (sortEigenVectors): Convert the comparison function into a key function. See: https://docs.python.org/3/howto/sorting.html#the-old-way-using-the-cmp-parameter --- wqflask/wqflask/correlation_matrix/show_corr_matrix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index c0d84aa2..f24b9f7f 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -28,6 +28,7 @@ import utility.logger from base import data_set from functools import reduce +from functools import cmp_to_key from rpy2.robjects.packages import importr from utility import webqtlUtil @@ -268,7 +269,7 @@ def sortEigenVectors(vector): for item in eigenValues: combines.append([eigenValues[i], eigenVectors[i]]) i += 1 - combines.sort(webqtlUtil.cmpEigenValue) + sorted(combines, key=cmp_to_key(webqtlUtil.cmpEigenValue)) A = [] B = [] for item in combines: -- cgit v1.2.3 From 65846e5b05841b7c31a37298b49b99bb77d48a31 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 8 Oct 2020 22:37:07 +0300 Subject: Update base64 conversion * wqflask/wqflask/wgcna/wgcna_analysis.py (WGCNA.render_image): Use Python3's base64 library from the stdlib to perform conversion. * wqflask/wqflask/ctl/ctl_analysis.py (CTL.loadImage): Ditto. --- wqflask/wqflask/ctl/ctl_analysis.py | 2 +- wqflask/wqflask/wgcna/wgcna_analysis.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index e58a7b87..2a2124ca 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -194,7 +194,7 @@ class CTL(object): def loadImage(self, path, name): imgfile = open(self.results[path], 'rb') imgdata = imgfile.read() - imgB64 = imgdata.encode("base64") + imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) self.results[name] = bytesarray diff --git a/wqflask/wqflask/wgcna/wgcna_analysis.py b/wqflask/wqflask/wgcna/wgcna_analysis.py index 0afe7e82..35fb3ec0 100644 --- a/wqflask/wqflask/wgcna/wgcna_analysis.py +++ b/wqflask/wqflask/wgcna/wgcna_analysis.py @@ -1,5 +1,6 @@ # WGCNA analysis for GN2 # Author / Maintainer: Danny Arends +import base64 import sys import rpy2.robjects as ro # R Objects import rpy2.rinterface as ri @@ -163,8 +164,8 @@ class WGCNA(object): print(("pre-loading imgage results:", self.results['imgloc'])) imgfile = open(self.results['imgloc'], 'rb') imgdata = imgfile.read() - imgB64 = imgdata.encode("base64") bytesarray = array.array('B', imgB64) + imgB64 = base64.b64encode(imgdata) self.results['imgdata'] = bytesarray def process_results(self, results): -- cgit v1.2.3 From dca0dab44a40faecd39ad7a176674fd4389b1903 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 8 Oct 2020 22:39:52 +0300 Subject: Use built-in array as a named import * wqflask/wqflask/wgcna/wgcna_analysis.py (WGCNA.render_image): Use named import method. This is a workaround for using explicitly using "array" which conflicts with something else in the namespace. --- wqflask/wqflask/wgcna/wgcna_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/wgcna/wgcna_analysis.py b/wqflask/wqflask/wgcna/wgcna_analysis.py index 35fb3ec0..93742daf 100644 --- a/wqflask/wqflask/wgcna/wgcna_analysis.py +++ b/wqflask/wqflask/wgcna/wgcna_analysis.py @@ -4,8 +4,8 @@ import base64 import sys import rpy2.robjects as ro # R Objects import rpy2.rinterface as ri -import array +from array import array as arr from numpy import * from base.webqtlConfig import GENERATED_IMAGE_DIR from rpy2.robjects.packages import importr @@ -164,8 +164,8 @@ class WGCNA(object): print(("pre-loading imgage results:", self.results['imgloc'])) imgfile = open(self.results['imgloc'], 'rb') imgdata = imgfile.read() - bytesarray = array.array('B', imgB64) imgB64 = base64.b64encode(imgdata) + bytesarray = arr('B', imgB64) self.results['imgdata'] = bytesarray def process_results(self, results): -- cgit v1.2.3 From 8db56f2472aa8d2549ff9fb2045f257c0388f6cc Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 9 Oct 2020 01:03:15 +0300 Subject: Replace "ncores" with "nthreads" param to r_CTLscan --- wqflask/wqflask/ctl/ctl_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 2a2124ca..6ffba28f 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -143,7 +143,7 @@ class CTL(object): #r_write_table(rPheno, "~/outputGN/pheno.csv") # Perform the CTL scan - res = self.r_CTLscan(rGeno, rPheno, strategy = strategy, nperm = nperm, parametric = parametric, ncores = 6) + res = self.r_CTLscan(rGeno, rPheno, strategy = strategy, nperm = nperm, parametric = parametric, nthreads=6) # Get significant interactions significant = self.r_CTLsignificant(res, significance = significance) -- cgit v1.2.3 From f604c81a5cf501c39bf195159a5b1c70a97195dc Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 9 Oct 2020 01:04:03 +0300 Subject: Replace string concat("+") with python3's f-string --- wqflask/wqflask/ctl/ctl_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 6ffba28f..72b4f3a3 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -186,8 +186,8 @@ class CTL(object): self.addNode(gtT) self.addEdge(gtS, gtT, significant, x) - significant[0][x] = gtS.symbol + " (" + gtS.name + ")" # Update the trait name for the displayed table - significant[2][x] = gtT.symbol + " (" + gtT.name + ")" # Update the trait name for the displayed table + significant[0][x] = "{} ({})".format(gtS.symbol, gtS.name) # Update the trait name for the displayed table + significant[2][x] = "{} ({})".format(gtT.symbol, gtT.name) # Update the trait name for the displayed table self.elements = json.dumps(self.nodes_list + self.edges_list) -- cgit v1.2.3 From 4e786c530d231718773cb629d3cac7449314c151 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 9 Oct 2020 01:38:10 +0300 Subject: Update base64 conversion --- wqflask/wqflask/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 7fdc62e5..e8d17613 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -164,7 +164,7 @@ def tmp_page(img_path): logger.info("initial_start_vars:", initial_start_vars) imgfile = open(GENERATED_IMAGE_DIR + img_path, 'rb') imgdata = imgfile.read() - imgB64 = imgdata.encode("base64") + imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) return render_template("show_image.html", img_base64 = bytesarray ) @@ -790,7 +790,7 @@ def mapping_results_page(): logger.info("initial_start_vars:", initial_start_vars) imgfile = open(TEMPDIR + img_path, 'rb') imgdata = imgfile.read() - imgB64 = imgdata.encode("base64") + imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) result['pair_scan_array'] = bytesarray rendered_template = render_template("pair_scan_results.html", **result) -- cgit v1.2.3 From 16cd94dfa0e892576f1c566948b775b954f7f508 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 9 Oct 2020 03:04:30 +0300 Subject: Fix export function on traits and analysis page * wqflask/wqflask/views.py (export_trait_excel): Write data as bytes instead of strings. See: https://stackoverflow.com/questions/32075135/python-3-in-memory-zipfile-error-string-argument-expected-got-bytes * wqflask/wqflask/show_trait/export_trait_data.py: Convert cmp function into a key function. --- wqflask/wqflask/show_trait/export_trait_data.py | 4 ++-- wqflask/wqflask/views.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/show_trait/export_trait_data.py b/wqflask/wqflask/show_trait/export_trait_data.py index 2d76b935..f2132966 100644 --- a/wqflask/wqflask/show_trait/export_trait_data.py +++ b/wqflask/wqflask/show_trait/export_trait_data.py @@ -1,7 +1,7 @@ import simplejson as json from pprint import pformat as pf - +from functools import cmp_to_key from base.trait import create_trait from base import data_set @@ -46,7 +46,7 @@ def get_export_metadata(trait_id, dataset_name): def dict_to_sorted_list(dictionary): sorted_list = [item for item in list(dictionary.items())] - sorted_list = sorted(sorted_list, cmp=cmp_samples) + sorted_list = sorted(sorted_list, key=cmp_to_key(cmp_samples)) sorted_values = [item[1] for item in sorted_list] return sorted_values diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index e8d17613..c37be418 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -376,7 +376,7 @@ def export_trait_excel(): logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) - buff = io.StringIO() + buff = io.BytesIO() workbook = xlsxwriter.Workbook(buff, {'in_memory': True}) worksheet = workbook.add_worksheet() for i, row in enumerate(sample_data): -- cgit v1.2.3 From 1c987139736be4f855c49f4a2f9ce4abc497d9e2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 12 Oct 2020 14:29:00 +0300 Subject: Wrap args of print statements correctly using braces * wqflask/wqflask/wgcna/wgcna_analysis.py: Update print statements by wrapping then in "()". Run: "2to3-3.8 -w wqflask/wqflask/wgcna/wgcna_analysis.py" --- wqflask/wqflask/wgcna/wgcna_analysis.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/wgcna/wgcna_analysis.py b/wqflask/wqflask/wgcna/wgcna_analysis.py index c6e03ec5..6bf75216 100644 --- a/wqflask/wqflask/wgcna/wgcna_analysis.py +++ b/wqflask/wqflask/wgcna/wgcna_analysis.py @@ -69,8 +69,8 @@ class WGCNA(object): self.r_enableWGCNAThreads() self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] - print("Retrieved phenotype data from database", - requestform['trait_list']) + print(("Retrieved phenotype data from database", + requestform['trait_list'])) helper_functions.get_trait_db_obs(self, self.trait_db_list) # self.input contains the phenotype values we need to send to R @@ -125,14 +125,14 @@ class WGCNA(object): powers = [int(threshold.strip()) for threshold in requestform['SoftThresholds'].rstrip().split(",")] rpow = r_unlist(r_c(powers)) - print("SoftThresholds: {} == {}".format(powers, rpow)) + print(("SoftThresholds: {} == {}".format(powers, rpow))) self.sft = self.r_pickSoftThreshold( rM, powerVector=rpow, verbose=5) - print("PowerEstimate: {}".format(self.sft[0])) + print(("PowerEstimate: {}".format(self.sft[0]))) self.results['PowerEstimate'] = self.sft[0] if self.sft[0][0] is ri.NA_Integer: - print "No power is suitable for the analysis, just use 1" + print("No power is suitable for the analysis, just use 1") # No power could be estimated self.results['Power'] = 1 else: -- cgit v1.2.3 From 4d1cc4be5ee49d9eaffb0114dad876d7cb2b9bb1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 19 Oct 2020 16:24:29 +0300 Subject: Delete stale comment --- wqflask/wqflask/marker_regression/display_mapping_results.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 745b8e78..377f58b6 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2915,9 +2915,6 @@ class DisplayMappingResults(object): name="selectCheck", Class="checkbox trait_checkbox")) # checkbox for each row - #ZS: May want to get this working again later - #webqtlSearch = HT.Href(os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE)+"?cmd=sch&gene=%s&alias=1&species=rat" % theGO["GeneSymbol"], ">>", target="_blank").__str__() - if theGO["GeneID"] != "": geneSymbolNCBI = str(HtmlGenWrapper.create_link_tag( "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids={}".format(theGO["GeneID"]), -- cgit v1.2.3 From 3b5355dec882a636b2e02e41333e9dbf0dce641a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 19 Oct 2020 16:24:56 +0300 Subject: Update snpString link * wqflask/wqflask/marker_regression/display_mapping_results.py: Replace "%" strings with f-strings. Also, replace old "Href" with new htmlgen "Link" method. --- .../wqflask/marker_regression/display_mapping_results.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 377f58b6..ca430e8d 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2806,9 +2806,17 @@ class DisplayMappingResults(object): geneIdString = 'http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s' % theGO["GeneID"] if theGO["snpCount"]: - snpString = HT.Href(url="http://genenetwork.org/webqtl/main.py?FormID=snpBrowser&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (theGO["Chromosome"], - theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffCol[0], self.diffCol[1]), - text=theGO["snpCount"], target="_blank", Class="normalsize") + snpString = HT.Link( + (f"http://genenetwork.org/webqtl/main.py?FormID=snpBrowser&" + f"chr={theGO['Chromosome']}&" + f"start={theGO['TxStart']}&" + f"end={theGO['TxEnd']}&" + f"geneName={theGO['GeneSymbol']}&" + f"s1={self.diffCol[0]}&s2=%d"), + theGO["snpCount"] # The text to display + ) + snpString.set_blank_target() + snpString.set_attribute("class", "normalsize") else: snpString = 0 -- cgit v1.2.3 From b2b7bacdbc8dc28dd062ebdb94698fca88f9b0cb Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 20 Oct 2020 22:19:56 +0300 Subject: Add pbkdf2 tests * wqflask/tests/wqflask/test_pbkdf2.py: New tests. --- wqflask/tests/wqflask/test_pbkdf2.py | 61 ++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 wqflask/tests/wqflask/test_pbkdf2.py diff --git a/wqflask/tests/wqflask/test_pbkdf2.py b/wqflask/tests/wqflask/test_pbkdf2.py new file mode 100644 index 00000000..a33fbd4f --- /dev/null +++ b/wqflask/tests/wqflask/test_pbkdf2.py @@ -0,0 +1,61 @@ +"""Test cases pbkdf2""" + +import unittest +from wqflask.pbkdf2 import pbkdf2_hex + + +class TestPbkdf2(unittest.TestCase): + def test_pbkdf2_hex(self): + """ + Test pbkdf2_hex function + """ + + for password, salt, iterations, keylen, expected_value in [ + ('password', 'salt', 1, 20, + '0c60c80f961f0e71f3a9b524af6012062fe037a6'), + ('password', 'salt', 2, 20, + 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'), + ('password', 'salt', 4096, 20, + '4b007901b765489abead49d926f721d065a429c1'), + ('passwordPASSWORDpassword', + 'saltSALTsaltSALTsaltSALTsaltSALTsalt', + 4096, 25, + '3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038'), + ('pass\x00word', 'sa\x00lt', 4096, 16, + '56fa6aa75548099dcc37d7f03425e0c3'), + ('password', 'ATHENA.MIT.EDUraeburn', 1, 16, + 'cdedb5281bb2f801565a1122b2563515'), + ('password', 'ATHENA.MIT.EDUraeburn', 1, 32, + ('cdedb5281bb2f80' + '1565a1122b256351' + '50ad1f7a04bb9f3a33' + '3ecc0e2e1f70837')), + ('password', 'ATHENA.MIT.EDUraeburn', 2, 16, + '01dbee7f4a9e243e988b62c73cda935d'), + ('password', 'ATHENA.MIT.EDUraeburn', 2, 32, + ('01dbee7f4a9e243e9' + '88b62c73cda935da05' + '378b93244ec8f48a99' + 'e61ad799d86')), + ('password', 'ATHENA.MIT.EDUraeburn', 1200, 32, + ('5c08eb61fdf71e' + '4e4ec3cf6ba1f55' + '12ba7e52ddbc5e51' + '42f708a31e2e62b1e13')), + ('X' * 64, 'pass phrase equals block size', 1200, 32, + ('139c30c0966bc32ba' + '55fdbf212530ac9c5' + 'ec59f1a452f5cc9ad' + '940fea0598ed1')), + ('X' * 65, 'pass phrase exceeds block size', 1200, 32, + ('9ccad6d468770cd' + '51b10e6a68721be6' + '11a8b4d282601db3' + 'b36be9246915ec82a')) + ]: + self.assertEqual( + pbkdf2_hex(data=password, + salt=salt, + iterations=iterations, + keylen=keylen), + expected_value) -- cgit v1.2.3 From d08d2322c5f88752ff062569ce6110cbc3815b62 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 20 Oct 2020 22:21:07 +0300 Subject: Update pbkdf2 * wqflask/wqflask/pbkdf2.py: Remove unnecessary pbkdf2 usage docs--- it already exists in python3 stdlib--- and "__main__" section which runs the test. (imports): Delete unused imports. Add safe_str_cmp. (pbkdf2_hex): Use hashlib.pbkdf2_hmac. (safe_str_cmp): Replace with werkzeug.security.safe_str_cmp (test): Delete it. Tests moved to test_pbkdf2.py. --- wqflask/wqflask/pbkdf2.py | 144 ++++------------------------------------------ 1 file changed, 12 insertions(+), 132 deletions(-) diff --git a/wqflask/wqflask/pbkdf2.py b/wqflask/wqflask/pbkdf2.py index 917b9d31..aea5b06c 100644 --- a/wqflask/wqflask/pbkdf2.py +++ b/wqflask/wqflask/pbkdf2.py @@ -1,140 +1,20 @@ -# -*- coding: utf-8 -*- -""" - pbkdf2 - ~~~~~~ - - This module implements pbkdf2 for Python. It also has some basic - tests that ensure that it works. The implementation is straightforward - and uses stdlib only stuff and can be easily be copy/pasted into - your favourite application. - - Use this as replacement for bcrypt that does not need a c implementation - of a modified blowfish crypto algo. - - Example usage: - - >>> pbkdf2_hex('what i want to hash', 'the random salt') - 'fa7cc8a2b0a932f8e6ea42f9787e9d36e592e0c222ada6a9' - - How to use this: - - 1. Use a constant time string compare function to compare the stored hash - with the one you're generating:: - - def safe_str_cmp(a, b): - if len(a) != len(b): - return False - rv = 0 - for x, y in izip(a, b): - rv |= ord(x) ^ ord(y) - return rv == 0 - - 2. Use `os.urandom` to generate a proper salt of at least 8 byte. - Use a unique salt per hashed password. - - 3. Store ``algorithm$salt:costfactor$hash`` in the database so that - you can upgrade later easily to a different algorithm if you need - one. For instance ``PBKDF2-256$thesalt:10000$deadbeef...``. - - - :copyright: (c) Copyright 2011 by Armin Ronacher. - :license: BSD, see LICENSE for more details. -""" -import hmac import hashlib -from struct import Struct -from operator import xor -from itertools import starmap +from werkzeug.security import safe_str_cmp as ssc -_pack_int = Struct('>I').pack - -def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc=None): - """Like :func:`pbkdf2_bin` but returns a hex encoded string.""" - return pbkdf2_bin(data, salt, iterations, keylen, hashfunc).encode('hex') - - -def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None): - """Returns a binary digest for the PBKDF2 hash algorithm of `data` - with the given `salt`. It iterates `iterations` time and produces a - key of `keylen` bytes. By default SHA-1 is used as hash function, - a different hashlib `hashfunc` can be provided. +# Replace this because it just wraps around Python3's internal +# functions. Added this during migration. +def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc="sha1"): + """Wrapper function of python's hashlib.pbkdf2_hmac. """ - hashfunc = hashfunc or hashlib.sha1 - mac = hmac.new(data, None, hashfunc) - def _pseudorandom(x, mac=mac): - h = mac.copy() - h.update(x) - return list(map(ord, h.digest())) - buf = [] - for block in range(1, -(-keylen // mac.digest_size) + 1): - rv = u = _pseudorandom(salt + _pack_int(block)) - for i in range(iterations - 1): - u = _pseudorandom(''.join(map(chr, u))) - rv = list(starmap(xor, zip(rv, u))) - buf.extend(rv) - return ''.join(map(chr, buf))[:keylen] + dk = hashlib.pbkdf2_hmac(hashfunc, + bytes(data, "utf-8"), # password + bytes(salt, "utf-8"), # salt + iterations, + keylen) + return dk.hex() def safe_str_cmp(a, b): - if len(a) != len(b): - return False - rv = 0 - for x, y in zip(a, b): - rv |= ord(x) ^ ord(y) - return rv == 0 - - - -def test(): - failed = [] - def check(data, salt, iterations, keylen, expected): - rv = pbkdf2_hex(data, salt, iterations, keylen) - if rv != expected: - print('Test failed:') - print((' Expected: %s' % expected)) - print((' Got: %s' % rv)) - print(' Parameters:') - print((' data=%s' % data)) - print((' salt=%s' % salt)) - print((' iterations=%d' % iterations)) - print() - failed.append(1) - - # From RFC 6070 - check('password', 'salt', 1, 20, - '0c60c80f961f0e71f3a9b524af6012062fe037a6') - check('password', 'salt', 2, 20, - 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957') - check('password', 'salt', 4096, 20, - '4b007901b765489abead49d926f721d065a429c1') - check('passwordPASSWORDpassword', 'saltSALTsaltSALTsaltSALTsaltSALTsalt', - 4096, 25, '3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038') - check('pass\x00word', 'sa\x00lt', 4096, 16, - '56fa6aa75548099dcc37d7f03425e0c3') - # This one is from the RFC but it just takes for ages - ##check('password', 'salt', 16777216, 20, - ## 'eefe3d61cd4da4e4e9945b3d6ba2158c2634e984') - - # From Crypt-PBKDF2 - check('password', 'ATHENA.MIT.EDUraeburn', 1, 16, - 'cdedb5281bb2f801565a1122b2563515') - check('password', 'ATHENA.MIT.EDUraeburn', 1, 32, - 'cdedb5281bb2f801565a1122b25635150ad1f7a04bb9f3a333ecc0e2e1f70837') - check('password', 'ATHENA.MIT.EDUraeburn', 2, 16, - '01dbee7f4a9e243e988b62c73cda935d') - check('password', 'ATHENA.MIT.EDUraeburn', 2, 32, - '01dbee7f4a9e243e988b62c73cda935da05378b93244ec8f48a99e61ad799d86') - check('password', 'ATHENA.MIT.EDUraeburn', 1200, 32, - '5c08eb61fdf71e4e4ec3cf6ba1f5512ba7e52ddbc5e5142f708a31e2e62b1e13') - check('X' * 64, 'pass phrase equals block size', 1200, 32, - '139c30c0966bc32ba55fdbf212530ac9c5ec59f1a452f5cc9ad940fea0598ed1') - check('X' * 65, 'pass phrase exceeds block size', 1200, 32, - '9ccad6d468770cd51b10e6a68721be611a8b4d282601db3b36be9246915ec82a') - - raise SystemExit(bool(failed)) - - -if __name__ == '__main__': - test() + return ssc(a, b) -- cgit v1.2.3 From 5d06524c8465064248cc3605c69dd32687ea7565 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 20 Oct 2020 22:55:19 +0300 Subject: Add tests for encoding password * wqflask/tests/wqflask/test_user_login.py: New tests. --- wqflask/tests/wqflask/test_user_login.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 wqflask/tests/wqflask/test_user_login.py diff --git a/wqflask/tests/wqflask/test_user_login.py b/wqflask/tests/wqflask/test_user_login.py new file mode 100644 index 00000000..61cd9ab9 --- /dev/null +++ b/wqflask/tests/wqflask/test_user_login.py @@ -0,0 +1,21 @@ +"""Test cases for some methods in login.py""" + +import unittest +from wqflask.user_login import encode_password + + +class TestUserLogin(unittest.TestCase): + def test_encode_password(self): + """ + Test encode password + """ + pass_gen_fields = { + "salt": "salt", + "hashfunc": "sha1", + "iterations": 4096, + "keylength": 20, + } + self.assertEqual( + encode_password(pass_gen_fields, + "password").get("password"), + '4b007901b765489abead49d926f721d065a429c1') -- cgit v1.2.3 From b94060ef1fa07cb072593f7085b8cd0be5874dca Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 20 Oct 2020 22:55:38 +0300 Subject: Update how pbkdf2.pbkdf2_hex is used when encrypting pass * wqflask/wqflask/user_login.py (encode_password): - Remove unused local variable "salt". - Pass "hashfunc" as a string. - Apply Pep-8. --- wqflask/wqflask/user_login.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index cb2edbc5..f25ebc32 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -37,15 +37,13 @@ def basic_info(): ip_address = request.remote_addr, user_agent = request.headers.get('User-Agent')) -def encode_password(pass_gen_fields, unencrypted_password): - hashfunc = getattr(hashlib, pass_gen_fields['hashfunc']) - salt = base64.b64decode(pass_gen_fields['salt']) +def encode_password(pass_gen_fields, unencrypted_password): encrypted_password = pbkdf2.pbkdf2_hex(str(unencrypted_password), - pass_gen_fields['salt'], - pass_gen_fields['iterations'], - pass_gen_fields['keylength'], - hashfunc) + pass_gen_fields['salt'], + pass_gen_fields['iterations'], + pass_gen_fields['keylength'], + pass_gen_fields['hashfunc']) pass_gen_fields.pop("unencrypted_password", None) pass_gen_fields["password"] = encrypted_password -- cgit v1.2.3