Merge pull request #1 from genenetwork/testing

Updating my testing branch

10 files changed, 313 insertions, 1190 deletions

diff --git a/wqflask/base/anon_collection.py b/wqflask/base/anon_collection.py
deleted file mode 100644
index 8ee73296..00000000
--- a/wqflask/base/anon_collection.py
+++ /dev/null
@@ -1,21 +0,0 @@
-class AnonCollection(TraitCollection):

-    

-    def __init__(self, anon_id)

-        self.anon_id = anon_id

-        self.collection_members = Redis.smembers(self.anon_id)

-        print("self.collection_members is:", self.collection_members)

-        self.num_members = len(self.collection_members)

-        

-

-    @app.route("/collections/remove", methods=('POST',))

-    def remove_traits(traits_to_remove):

-        print("traits_to_remove:", traits_to_remove)

-        for trait in traits_to_remove:

-            Redis.srem(self.anon_id, trait)

-        members_now = self.collection_members - traits_to_remove

-        print("members_now:", members_now)

-        print("Went from {} to {} members in set.".format(len(self.collection_members), len(members_now)))

-

-        # We need to return something so we'll return this...maybe in the future

-        # we can use it to check the results

-        return str(len(members_now))

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 54dd3c4b..ebf3f021 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -26,10 +26,10 @@ import collections
 import codecs
 
 import json
+import requests
 import gzip
 import cPickle as pickle
 import itertools
-from operator import itemgetter
 
 from redis import Redis
 Redis = Redis()
@@ -44,8 +44,11 @@ from db import webqtlDatabaseFunction
 from utility import webqtlUtil
 from utility.benchmark import Bench
 from utility import chunks
+from utility import gen_geno_ob
 from utility.tools import locate, locate_ignore_error, flat_files
 
+from wqflask.api import gen_menu
+
 from maintenance import get_group_samplelists
 
 from MySQLdb import escape_string as escape
@@ -61,18 +64,21 @@ logger = getLogger(__name__ )
 # Each subclass will add to this
 DS_NAME_MAP = {}
 
-def create_dataset(dataset_name, dataset_type = None, get_samplelist = True):
+def create_dataset(dataset_name, rebuild=True, dataset_type = None, get_samplelist = True, group_name = None):
     if not dataset_type:
         dataset_type = Dataset_Getter(dataset_name)
         logger.debug("dataset_type", dataset_type)
 
     dataset_ob = DS_NAME_MAP[dataset_type]
     dataset_class = globals()[dataset_ob]
-    return dataset_class(dataset_name, get_samplelist)
+    if dataset_type == "Temp":
+        return dataset_class(dataset_name, get_samplelist, group_name)
+    else:
+        return dataset_class(dataset_name, get_samplelist)
 
 class Dataset_Types(object):
 
-    def __init__(self):
+    def __init__(self, rebuild=False):
         """Create a dictionary of samples where the value is set to Geno,
 Publish or ProbeSet. E.g.
 
@@ -88,8 +94,10 @@ Publish or ProbeSet. E.g.
 
         """
         self.datasets = {}
-        if USE_GN_SERVER:
-            data = menu_main()
+        if rebuild: #ZS: May make this the only option
+            data = json.loads(requests.get("http://gn2.genenetwork.org/api/v_pre1/gen_dropdown").content)
+            logger.debug("THE DATA:", data)
+            #data = gen_menu.gen_dropdown_json()
         else:
             file_name = "wqflask/static/new/javascript/dataset_menu_structure.json"
             with open(file_name, 'r') as fh:
@@ -107,8 +115,9 @@ Publish or ProbeSet. E.g.
                         else:
                             new_type = "ProbeSet"
                         self.datasets[short_dataset_name] = new_type
+
         # Set LOG_LEVEL_DEBUG=5 to see the following:
-        logger.debugf(5,"datasets",self.datasets)
+        logger.debugf(5, "datasets",self.datasets)
 
     def __call__(self, name):
         return self.datasets[name]
@@ -166,20 +175,31 @@ def mescape(*items):
 class Markers(object):
     """Todo: Build in cacheing so it saves us reading the same file more than once"""
     def __init__(self, name):
-        json_data_fh = open(locate(name + '.json','genotype/json'))
-        try:
-            markers = json.load(json_data_fh)
-        except:
-            markers = []
+        json_data_fh = open(locate(name + ".json",'genotype/json'))
+
+        markers = []
+        with open("%s/%s_snps.txt" % (flat_files('genotype/bimbam'), name), 'r') as bimbam_fh:
+            if len(bimbam_fh.readline().split(", ")) > 2:
+                delimiter = ", "
+            elif len(bimbam_fh.readline().split(",")) > 2:
+                delimiter = ","
+            elif len(bimbam_fh.readline().split("\t")) > 2:
+                delimiter = "\t"
+            else:
+                delimiter = " "
+            for line in bimbam_fh:
+                marker = {}
+                marker['name'] = line.split(delimiter)[0].rstrip()
+                marker['Mb'] = float(line.split(delimiter)[1].rstrip())/1000000
+                marker['chr'] = line.split(delimiter)[2].rstrip()
+                markers.append(marker)
 
         for marker in markers:
-            if (marker['chr'] != "X") and (marker['chr'] != "Y"):
+            if (marker['chr'] != "X") and (marker['chr'] != "Y") and (marker['chr'] != "M"):
                 marker['chr'] = int(marker['chr'])
             marker['Mb'] = float(marker['Mb'])
 
         self.markers = markers
-        #logger.debug("self.markers:", self.markers)
-
 
     def add_pvalues(self, p_values):
         logger.debug("length of self.markers:", len(self.markers))
@@ -261,10 +281,12 @@ class DatasetGroup(object):
     has multiple datasets associated with it.
 
     """
-    def __init__(self, dataset):
+    def __init__(self, dataset, name=None):
         """This sets self.group and self.group_id"""
-        #logger.debug("DATASET NAME2:", dataset.name)
-        self.name, self.id, self.genetic_type = fetchone(dataset.query_for_group)
+        if name == None:
+            self.name, self.id, self.genetic_type = fetchone(dataset.query_for_group)
+        else:
+            self.name, self.id, self.genetic_type = fetchone("SELECT InbredSet.Name, InbredSet.Id, InbredSet.GeneticType FROM InbredSet where Name='%s'" % name)
         if self.name == 'BXD300':
             self.name = "BXD"
 
@@ -272,7 +294,6 @@ class DatasetGroup(object):
         self.parlist = None
         self.get_f1_parent_strains()
 
-        self.accession_id = self.get_accession_id()
         self.mapping_id, self.mapping_names = self.get_mapping_methods()
 
         self.species = webqtlDatabaseFunction.retrieve_species(self.name)
@@ -282,55 +303,39 @@ class DatasetGroup(object):
         self._datasets = None
         self.genofile = None
 
-    def get_accession_id(self):
-        results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where
-                    InbredSet.Name = %s and
-                    PublishFreeze.InbredSetId = InbredSet.Id and
-                    InfoFiles.InfoPageName = PublishFreeze.Name and
-                    PublishFreeze.public > 0 and
-                    PublishFreeze.confidentiality < 1 order by
-                    PublishFreeze.CreateTime desc""", (self.name)).fetchone()
-
-        if results != None:
-            return str(results[0])
-        else:
-            return "None"
-
     def get_mapping_methods(self):
 
         mapping_id = g.db.execute("select MappingMethodId from InbredSet where Name= '%s'" % self.name).fetchone()[0]
         if mapping_id == "1":
-            mapping_names = ["QTLReaper", "PYLMM", "R/qtl"]
+            mapping_names = ["GEMMA", "QTLReaper", "R/qtl"]
         elif mapping_id == "2":
             mapping_names = ["GEMMA"]
+        elif mapping_id == "3":
+            mapping_names = ["R/qtl"]
         elif mapping_id == "4":
-            mapping_names = ["PLINK"]
+            mapping_names = ["GEMMA", "PLINK"]
         else:
             mapping_names = []
 
         return mapping_id, mapping_names
 
-    def get_specified_markers(self, markers = []):
-        self.markers = HumanMarkers(self.name, markers)
-
     def get_markers(self):
-        logger.debug("self.species is:", self.species)
-
         def check_plink_gemma():
             if flat_file_exists("mapping"):
                 MAPPING_PATH = flat_files("mapping")+"/"
-                if (os.path.isfile(MAPPING_PATH+self.name+".bed") and
-                    (os.path.isfile(MAPPING_PATH+self.name+".map") or
-                     os.path.isfile(MAPPING_PATH+self.name+".bim"))):
+                if os.path.isfile(MAPPING_PATH+self.name+".bed"):
                     return True
             return False
 
         if check_plink_gemma():
             marker_class = HumanMarkers
         else:
-            marker_class = Markers            
+            marker_class = Markers
 
-        self.markers = marker_class(self.name)
+        if self.genofile:
+            self.markers = marker_class(self.genofile[:-5])
+        else:
+            self.markers = marker_class(self.name)
 
     def get_f1_parent_strains(self):
         try:
@@ -344,30 +349,32 @@ class DatasetGroup(object):
         if maternal and paternal:
             self.parlist = [maternal, paternal]
 
+    def get_genofiles(self):
+        jsonfile = "%s/%s.json" % (webqtlConfig.GENODIR, self.name)
+        try:
+            f = open(jsonfile)
+        except:
+            return None
+        jsondata = json.load(f)
+        return jsondata['genofile']
+
     def get_samplelist(self):
         result = None
-        key = "samplelist:v2:" + self.name
+        key = "samplelist:v3:" + self.name
         if USE_REDIS:
             result = Redis.get(key)
 
         if result is not None:
-            #logger.debug("Sample List Cache hit!!!")
-            #logger.debug("Before unjsonifying {}: {}".format(type(result), result))
             self.samplelist = json.loads(result)
-            #logger.debug("  type: ", type(self.samplelist))
-            #logger.debug("  self.samplelist: ", self.samplelist)
         else:
             logger.debug("Cache not hit")
 
             genotype_fn = locate_ignore_error(self.name+".geno",'genotype')
-            mapping_fn = locate_ignore_error(self.name+".fam",'mapping')
-            if mapping_fn:
-                self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn)
-            elif genotype_fn:
+            if genotype_fn:
                 self.samplelist = get_group_samplelists.get_samplelist("geno", genotype_fn)
             else:
                 self.samplelist = None
-            logger.debug("Sample list: ",self.samplelist)
+
             if USE_REDIS:
                 Redis.set(key, json.dumps(self.samplelist))
                 Redis.expire(key, 60*5)
@@ -378,19 +385,27 @@ class DatasetGroup(object):
         [result.extend(l) for l in lists if l]
         return result
 
-    def read_genotype_file(self):
+    def read_genotype_file(self, use_reaper=False):
         '''Read genotype from .geno file instead of database'''
         #genotype_1 is Dataset Object without parents and f1
         #genotype_2 is Dataset Object with parents and f1 (not for intercross)
 
-        genotype_1 = reaper.Dataset()
+        #genotype_1 = reaper.Dataset()
 
         # reaper barfs on unicode filenames, so here we ensure it's a string
         if self.genofile:
-            full_filename = str(locate(self.genofile, 'genotype'))
+            if "RData" in self.genofile: #ZS: This is a temporary fix; I need to change the way the JSON files that point to multiple genotype files are structured to point to other file types like RData
+                full_filename = str(locate(self.genofile.split(".")[0] + ".geno", 'genotype'))
+            else:
+                full_filename = str(locate(self.genofile, 'genotype'))
         else:
             full_filename = str(locate(self.name + '.geno', 'genotype'))
-        genotype_1.read(full_filename)
+
+        if use_reaper:
+            genotype_1 = reaper.Dataset()
+            genotype_1.read(full_filename)
+        else:
+            genotype_1 = gen_geno_ob.genotype(full_filename)
 
         if genotype_1.type == "group" and self.parlist:
             genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1])       #, F1=_f1)
@@ -419,13 +434,16 @@ def datasets(group_name, this_group = None):
           FROM PublishFreeze,InbredSet
           WHERE PublishFreeze.InbredSetId = InbredSet.Id
             and InbredSet.Name = '%s'
-            and PublishFreeze.public > %s)
+            and PublishFreeze.public > %s
+            and PublishFreeze.confidentiality < 1
+          ORDER BY PublishFreeze.Id ASC)
          UNION
          (SELECT '#GenoFreeze',GenoFreeze.FullName,GenoFreeze.Name
           FROM GenoFreeze, InbredSet
           WHERE GenoFreeze.InbredSetId = InbredSet.Id
             and InbredSet.Name = '%s'
-            and GenoFreeze.public > %s)
+            and GenoFreeze.public > %s
+            and GenoFreeze.confidentiality < 1)
          UNION
          (SELECT Tissue.Name, ProbeSetFreeze.FullName,ProbeSetFreeze.Name
           FROM ProbeSetFreeze, ProbeFreeze, InbredSet, Tissue
@@ -434,27 +452,34 @@ def datasets(group_name, this_group = None):
             and ProbeFreeze.InbredSetId = InbredSet.Id
             and InbredSet.Name like %s
             and ProbeSetFreeze.public > %s
-          ORDER BY Tissue.Name, ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId)
+            and ProbeSetFreeze.confidentiality < 1
+          ORDER BY Tissue.Name, ProbeSetFreeze.OrderList DESC)
         ''' % (group_name, webqtlConfig.PUBLICTHRESH,
               group_name, webqtlConfig.PUBLICTHRESH,
               "'" + group_name + "'", webqtlConfig.PUBLICTHRESH))
 
-    #for tissue_name, dataset in itertools.groupby(the_results, itemgetter(0)):
-    for dataset_item in the_results:
+    sorted_results = sorted(the_results, key=lambda kv: kv[0])
+
+    pheno_inserted = False #ZS: This is kind of awkward, but need to ensure Phenotypes show up before Genotypes in dropdown
+    geno_inserted = False
+    for dataset_item in sorted_results:
         tissue_name = dataset_item[0]
         dataset = dataset_item[1]
         dataset_short = dataset_item[2]
         if tissue_name in ['#PublishFreeze', '#GenoFreeze']:
-            dataset_menu.append(dict(tissue=None, datasets=[(dataset, dataset_short)]))
+            if tissue_name == '#PublishFreeze' and (dataset_short == group_name + 'Publish'):
+                dataset_menu.insert(0, dict(tissue=None, datasets=[(dataset, dataset_short)]))
+                pheno_inserted = True
+            elif pheno_inserted and tissue_name == '#GenoFreeze':
+                dataset_menu.insert(1, dict(tissue=None, datasets=[(dataset, dataset_short)]))
+                geno_inserted = True
+            else:
+                dataset_menu.append(dict(tissue=None, datasets=[(dataset, dataset_short)]))
         else:
-            dataset_sub_menu = [item[1:] for item in dataset]
-
             tissue_already_exists = False
-            tissue_position = None
             for i, tissue_dict in enumerate(dataset_menu):
                 if tissue_dict['tissue'] == tissue_name:
                     tissue_already_exists = True
-                    tissue_position = i
                     break
 
             if tissue_already_exists:
@@ -481,7 +506,7 @@ class DataSet(object):
 
     """
 
-    def __init__(self, name, get_samplelist = True):
+    def __init__(self, name, get_samplelist = True, group_name = None):
 
         assert name, "Need a name"
         self.name = name
@@ -493,11 +518,13 @@ class DataSet(object):
 
         self.setup()
 
-        self.check_confidentiality()
-
-        self.retrieve_other_names()
-
-        self.group = DatasetGroup(self)   # sets self.group and self.group_id and gets genotype
+        if self.type == "Temp": #Need to supply group name as input if temp trait
+            self.group = DatasetGroup(self, name=group_name)   # sets self.group and self.group_id and gets genotype
+        else:
+            self.check_confidentiality()
+            self.retrieve_other_names()
+            self.group = DatasetGroup(self)   # sets self.group and self.group_id and gets genotype
+            self.accession_id = self.get_accession_id()
         if get_samplelist == True:
              self.group.get_samplelist()
         self.species = species.TheSpecies(self)
@@ -512,6 +539,31 @@ class DataSet(object):
     def riset():
         Weve_Renamed_This_As_Group
 
+    def get_accession_id(self):
+        if self.type == "Publish":
+            results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where
+                        InbredSet.Name = %s and
+                        PublishFreeze.InbredSetId = InbredSet.Id and
+                        InfoFiles.InfoPageName = PublishFreeze.Name and
+                        PublishFreeze.public > 0 and
+                        PublishFreeze.confidentiality < 1 order by
+                        PublishFreeze.CreateTime desc""", (self.group.name)).fetchone()
+        elif self.type == "Geno":
+            results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, GenoFreeze, InbredSet where
+                        InbredSet.Name = %s and
+                        GenoFreeze.InbredSetId = InbredSet.Id and
+                        InfoFiles.InfoPageName = GenoFreeze.ShortName and
+                        GenoFreeze.public > 0 and
+                        GenoFreeze.confidentiality < 1 order by
+                        GenoFreeze.CreateTime desc""", (self.group.name)).fetchone()
+        else:
+            results = None
+
+        if results != None:
+            return str(results[0])
+        else:
+            return "None"
+
     def retrieve_other_names(self):
         """This method fetches the the dataset names in search_result.
 
@@ -658,6 +710,7 @@ class PhenotypeDataSet(DataSet):
                             'Phenotype.Pre_publication_description',
                             'Phenotype.Pre_publication_abbreviation',
                             'Phenotype.Post_publication_abbreviation',
+                            'PublishXRef.mean',
                             'Phenotype.Lab_code',
                             'Publication.PubMed_ID',
                             'Publication.Abstract',
@@ -666,13 +719,14 @@ class PhenotypeDataSet(DataSet):
                             'PublishXRef.Id']
 
         # Figure out what display_fields is
-        self.display_fields = ['name',
+        self.display_fields = ['name', 'group_code',
                                'pubmed_id',
                                'pre_publication_description',
                                'post_publication_description',
                                'original_description',
                                'pre_publication_abbreviation',
                                'post_publication_abbreviation',
+                               'mean',
                                'lab_code',
                                'submitter', 'owner',
                                'authorized_users',
@@ -708,20 +762,6 @@ class PhenotypeDataSet(DataSet):
         # (Urgently?) Need to write this
         pass
 
-    def get_trait_list(self):
-        query = """
-            select PublishXRef.Id
-            from PublishXRef, PublishFreeze
-            where PublishFreeze.InbredSetId=PublishXRef.InbredSetId
-            and PublishFreeze.Id = {}
-            """.format(escape(str(self.id)))
-        logger.sql(query)
-        results = g.db.execute(query).fetchall()
-        trait_data = {}
-        for trait in results:
-            trait_data[trait[0]] = self.retrieve_sample_data(trait[0])
-        return trait_data
-
     def get_trait_info(self, trait_list, species = ''):
         for this_trait in trait_list:
 
@@ -735,7 +775,7 @@ class PhenotypeDataSet(DataSet):
             #of the post-publication description
             if this_trait.confidential:
                 this_trait.description_display = ""
-                continue   # for now
+                continue   # for now, because no authorization features
 
                 if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(
                         privilege=self.privilege,
@@ -759,9 +799,7 @@ class PhenotypeDataSet(DataSet):
 
             #LRS and its location
             this_trait.LRS_score_repr = "N/A"
-            this_trait.LRS_score_value = 0
             this_trait.LRS_location_repr = "N/A"
-            this_trait.LRS_location_value = 1000000
 
             if this_trait.lrs:
                 query = """
@@ -778,17 +816,7 @@ class PhenotypeDataSet(DataSet):
                         LRS_Chr = result[0]
                         LRS_Mb = result[1]
 
-                        #XZ: LRS_location_value is used for sorting
-                        try:
-                            LRS_location_value = int(LRS_Chr)*1000 + float(LRS_Mb)
-                        except:
-                            if LRS_Chr.upper() == 'X':
-                                LRS_location_value = 20*1000 + float(LRS_Mb)
-                            else:
-                                LRS_location_value = ord(str(LRS_chr).upper()[0])*1000 + float(LRS_Mb)
-
                         this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs
-                        this_trait.LRS_score_value = LRS_score_value = this_trait.lrs
                         this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb))
 
     def retrieve_sample_data(self, trait):
@@ -850,45 +878,18 @@ class GenotypeDataSet(DataSet):
     def check_confidentiality(self):
         return geno_mrna_confidentiality(self)
 
-    def get_trait_list(self):
-        query = """
-            select Geno.Name
-            from Geno, GenoXRef
-            where GenoXRef.GenoId = Geno.Id
-            and GenoFreezeId = {}
-            """.format(escape(str(self.id)))
-        logger.sql(query)
-        results = g.db.execute(query).fetchall()
-        trait_data = {}
-        for trait in results:
-            trait_data[trait[0]] = self.retrieve_sample_data(trait[0])
-        return trait_data
-
     def get_trait_info(self, trait_list, species=None):
         for this_trait in trait_list:
             if not this_trait.haveinfo:
                 this_trait.retrieveInfo()
 
-            #XZ: trait_location_value is used for sorting
-            trait_location_repr = 'N/A'
-            trait_location_value = 1000000
-
             if this_trait.chr and this_trait.mb:
-                try:
-                    trait_location_value = int(this_trait.chr)*1000 + this_trait.mb
-                except:
-                    if this_trait.chr.upper() == 'X':
-                        trait_location_value = 20*1000 + this_trait.mb
-                    else:
-                        trait_location_value = ord(str(this_trait.chr).upper()[0])*1000 + this_trait.mb
-
                 this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb) )
-                this_trait.location_value = trait_location_value
 
     def retrieve_sample_data(self, trait):
         query = """
                     SELECT
-                            Strain.Name, GenoData.value, GenoSE.error, GenoData.Id, Sample.Name2
+                            Strain.Name, GenoData.value, GenoSE.error, GenoData.Id, Strain.Name2
                     FROM
                             (GenoData, GenoFreeze, Strain, Geno, GenoXRef)
                     left join GenoSE on
@@ -940,6 +941,7 @@ class MrnaAssayDataSet(DataSet):
                                'blatseq', 'targetseq',
                                'chipid', 'comments',
                                'strand_probe', 'strand_gene',
+                               'proteinid', 'uniprotid',
                                'probe_set_target_region',
                                'probe_set_specificity',
                                'probe_set_blat_score',
@@ -978,20 +980,6 @@ class MrnaAssayDataSet(DataSet):
     def check_confidentiality(self):
         return geno_mrna_confidentiality(self)
 
-    def get_trait_list_1(self):
-        query = """
-            select ProbeSet.Name
-            from ProbeSet, ProbeSetXRef
-            where ProbeSetXRef.ProbeSetId = ProbeSet.Id
-            and ProbeSetFreezeId = {}
-            """.format(escape(str(self.id)))
-        logger.sql(query)
-        results = g.db.execute(query).fetchall()
-        trait_data = {}
-        for trait in results:
-            trait_data[trait[0]] = self.retrieve_sample_data(trait[0])
-        return trait_data
-
     def get_trait_info(self, trait_list=None, species=''):
 
         #  Note: setting trait_list to [] is probably not a great idea.
@@ -1023,27 +1011,8 @@ class MrnaAssayDataSet(DataSet):
             # Save it for the jinja2 template
             this_trait.description_display = description_display
 
-            #XZ: trait_location_value is used for sorting
-            trait_location_repr = 'N/A'
-            trait_location_value = 1000000
-
             if this_trait.chr and this_trait.mb:
-                #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y")
-                #This is so we can convert the location to a number used for sorting
-                trait_location_value = self.convert_location_to_value(this_trait.chr, this_trait.mb)
-                #try:
-                #    trait_location_value = int(this_trait.chr)*1000 + this_trait.mb
-                #except ValueError:
-                #    if this_trait.chr.upper() == 'X':
-                #        trait_location_value = 20*1000 + this_trait.mb
-                #    else:
-                #        trait_location_value = (ord(str(this_trait.chr).upper()[0])*1000 +
-                #                               this_trait.mb)
-
-                #ZS: Put this in function currently called "convert_location_to_value"
-                this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr,
-                                                                float(this_trait.mb))
-                this_trait.location_value = trait_location_value
+                this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb))
 
             #Get mean expression value
             query = (
@@ -1065,9 +1034,7 @@ class MrnaAssayDataSet(DataSet):
 
             #LRS and its location
             this_trait.LRS_score_repr = 'N/A'
-            this_trait.LRS_score_value = 0
             this_trait.LRS_location_repr = 'N/A'
-            this_trait.LRS_location_value = 1000000
 
             #Max LRS and its Locus location
             if this_trait.lrs and this_trait.locus:
@@ -1082,40 +1049,10 @@ class MrnaAssayDataSet(DataSet):
 
                 if result:
                     lrs_chr, lrs_mb = result
-                    #XZ: LRS_location_value is used for sorting
-                    lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb)
                     this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs
-                    this_trait.LRS_score_value = this_trait.lrs
                     this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb))
 
-
-    def convert_location_to_value(self, chromosome, mb):
-        try:
-            location_value = int(chromosome)*1000 + float(mb)
-        except ValueError:
-            if chromosome.upper() == 'X':
-                location_value = 20*1000 + float(mb)
-            else:
-                location_value = (ord(str(chromosome).upper()[0])*1000 +
-                                  float(mb))
-
-        return location_value
-
-    def get_sequence(self):
-        query = """
-                    SELECT
-                            ProbeSet.BlatSeq
-                    FROM
-                            ProbeSet, ProbeSetFreeze, ProbeSetXRef
-                    WHERE
-                            ProbeSet.Id=ProbeSetXRef.ProbeSetId and
-                            ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and
-                            ProbeSet.Name = %s
-                            ProbeSetFreeze.Name = %s
-                """ % (escape(self.name), escape(self.dataset.name))
-        logger.sql(query)
-        results = g.db.execute(query).fetchone()
-        return results[0]
+        return trait_list
 
     def retrieve_sample_data(self, trait):
         query = """
@@ -1139,7 +1076,6 @@ class MrnaAssayDataSet(DataSet):
         #logger.debug("RETRIEVED RESULTS HERE:", results)
         return results
 
-
     def retrieve_genes(self, column_name):
         query = """
                     select ProbeSet.Name, ProbeSet.%s
@@ -1156,6 +1092,8 @@ class MrnaAssayDataSet(DataSet):
 class TempDataSet(DataSet):
     '''Temporary user-generated data set'''
 
+    DS_NAME_MAP['Temp'] = 'TempDataSet'
+
     def setup(self):
         self.search_fields = ['name',
                               'description']
@@ -1191,19 +1129,6 @@ class TempDataSet(DataSet):
         desc = self.handle_pca(desc)
         return desc
 
-    def get_group(self):
-        query = """
-                    SELECT
-                            InbredSet.Name, InbredSet.Id
-                    FROM
-                            InbredSet, Temp
-                    WHERE
-                            Temp.InbredSetId = InbredSet.Id AND
-                            Temp.Name = "%s"
-            """ % self.name
-        logger.sql(query)
-        self.group, self.group_id = g.db.execute(query).fetchone()
-
     def retrieve_sample_data(self, trait):
         query = """
                 SELECT
diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py
index eb836e6c..6fec5dcd 100644
--- a/wqflask/base/mrna_assay_tissue_data.py
+++ b/wqflask/base/mrna_assay_tissue_data.py
@@ -18,21 +18,11 @@ class MrnaAssayTissueData(object):
 
     def __init__(self, gene_symbols=None):
         self.gene_symbols = gene_symbols
-        self.have_data = False
         if self.gene_symbols == None:
             self.gene_symbols = []
 
-        #print("self.gene_symbols:", self.gene_symbols)
-
         self.data = collections.defaultdict(Bunch)
 
-        #self.gene_id_dict ={}
-        #self.data_id_dict = {}
-        #self.chr_dict = {}
-        #self.mb_dict = {}
-        #self.desc_dict = {}
-        #self.probe_target_desc_dict = {}
-
         query =  '''select t.Symbol, t.GeneId, t.DataId, t.Chr, t.Mb, t.description, t.Probe_Target_Description
                         from (
                         select Symbol, max(Mean) as maxmean
@@ -52,7 +42,6 @@ class MrnaAssayTissueData(object):
             in_clause = db_tools.create_in_clause(gene_symbols)
 
             #ZS: This was in the query, not sure why: http://docs.python.org/2/library/string.html?highlight=lower#string.lower
-
             query += ''' Symbol in {} group by Symbol)
                 as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol
                 and t.Mean = x.maxmean;
@@ -67,9 +56,7 @@ class MrnaAssayTissueData(object):
 
         for result in results:
             symbol = result[0]
-            #if symbol.lower() in [gene_symbol.lower() for gene_symbol in gene_symbols]:
             if symbol.lower() in lower_symbols:
-                #gene_symbols.append(symbol)
                 symbol = symbol.lower()
 
                 self.data[symbol].gene_id = result.GeneId
@@ -79,8 +66,6 @@ class MrnaAssayTissueData(object):
                 self.data[symbol].description = result.description
                 self.data[symbol].probe_target_description = result.Probe_Target_Description
 
-        print("self.data: ", pf(self.data))
-
     ###########################################################################
     #Input: cursor, symbolList (list), dataIdDict(Dict)
     #output: symbolValuepairDict (dictionary):one dictionary of Symbol and Value Pair,
@@ -107,53 +92,4 @@ class MrnaAssayTissueData(object):
                 else:
                     symbol_values_dict[result.Symbol.lower()].append(result.value)
 
-        #for symbol in self.data:
-        #    data_id = self.data[symbol].data_id
-        #    symbol_values_dict[symbol] = self.get_tissue_values(data_id)
-
-
-        return symbol_values_dict
-
-
-    #def get_tissue_values(self, data_id):
-    #    """Gets the tissue values for a particular gene"""
-    #
-    #    tissue_values=[]
-    #
-    #    query = """SELECT value, id
-    #               FROM TissueProbeSetData
-    #               WHERE Id IN {}""".format(db_tools.create_in_clause(data_id))
-    #
-    #    #try :
-    #    results = g.db.execute(query).fetchall()
-    #    for result in results:
-    #        tissue_values.append(result.value)
-    #    #symbol_values_dict[symbol] = value_list
-    #    #except:
-    #    #    symbol_values_pairs[symbol] = None
-    #
-    #    return tissue_values
-
-########################################################################################################
-#input: cursor, symbolList (list), dataIdDict(Dict): key is symbol
-#output: SymbolValuePairDict(dictionary):one dictionary of Symbol and Value Pair.
-#        key is symbol, value is one list of expression values of one probeSet.
-#function: wrapper function for getSymbolValuePairDict function
-#          build gene symbol list if necessary, cut it into small lists if necessary,
-#          then call getSymbolValuePairDict function and merge the results.
-########################################################################################################
-
-#def get_trait_symbol_and_tissue_values(symbol_list=None):
-#    tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list)
-#
-#    #symbolList,
-#    #geneIdDict,
-#    #dataIdDict,
-#    #ChrDict,
-#    #MbDict,
-#    #descDict,
-#    #pTargetDescDict = getTissueProbeSetXRefInfo(
-#    #                    GeneNameLst=GeneNameLst,TissueProbeSetFreezeId=TissueProbeSetFreezeId)
-#
-#    if len(tissue_data.gene_symbols):
-#        return get_symbol_values_pairs(tissue_data)
+        return symbol_values_dict
\ No newline at end of file
diff --git a/wqflask/base/species.py b/wqflask/base/species.py
index ce763fc3..6d99af65 100644
--- a/wqflask/base/species.py
+++ b/wqflask/base/species.py
@@ -14,23 +14,13 @@ from utility.logger import getLogger
 logger = getLogger(__name__ )
 
 class TheSpecies(object):
-    def __init__(self, dataset):
-        self.dataset = dataset
-        #print("self.dataset is:", pf(self.dataset.__dict__))
-        self.chromosomes = Chromosomes(self.dataset)
-        self.genome_mb_length = self.chromosomes.get_genome_mb_length()
-
-    #@property
-    #def chromosomes(self):
-    #    chromosomes = [("All", -1)]
-    #
-    #    for counter, genotype in enumerate(self.dataset.group.genotype):
-    #        if len(genotype) > 1:
-    #            chromosomes.append((genotype.name, counter))
-    #
-    #    print("chromosomes is: ", pf(chromosomes))
-    #
-    #    return chromosomes
+    def __init__(self, dataset=None, species_name=None):
+        if species_name != None:
+            self.name = species_name
+            self.chromosomes = Chromosomes(species=self.name)
+        else:
+            self.dataset = dataset
+            self.chromosomes = Chromosomes(dataset=self.dataset)
 
 class IndChromosome(object):
     def __init__(self, name, length):
@@ -42,17 +32,22 @@ class IndChromosome(object):
         """Chromosome length in megabases"""
         return self.length / 1000000
 
-    def set_cm_length(self, genofile_chr):
-        self.cm_length = genofile_chr[-1].cM - genofile_chr[0].cM
-
-
 class Chromosomes(object):
-    def __init__(self, dataset):
-        self.dataset = dataset
+    def __init__(self, dataset=None, species=None):
         self.chromosomes = collections.OrderedDict()
+        if species != None:
+            query = """
+                Select
+                        Chr_Length.Name, Chr_Length.OrderId, Length from Chr_Length, Species
+                where
+                        Chr_Length.SpeciesId = Species.SpeciesId AND
+                        Species.Name = '%s'
+                Order by OrderId
+                """ % species.capitalize()
+        else:
+            self.dataset = dataset
 
-
-        query = """
+            query = """
                 Select
                         Chr_Length.Name, Chr_Length.OrderId, Length from Chr_Length, InbredSet
                 where
@@ -64,64 +59,4 @@ class Chromosomes(object):
         results = g.db.execute(query).fetchall()
 
         for item in results:
-            self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length)
-
-        self.set_mb_graph_interval()
-        #self.get_cm_length_list()
-
-
-    def set_mb_graph_interval(self):
-        """Empirical megabase interval"""
-
-        if self.chromosomes:
-            self.mb_graph_interval = self.get_genome_mb_length()/(len(self.chromosomes)*12)
-        else:
-            self.mb_graph_interval = 1
-
-        #if self.chromosomes:
-        #assert self.chromosomes, "Have to add some code back in apparently to set it to 1"
-        #self.mb_graph_interval = self.get_genome_mb_length()/(len(self.chromosomes)*12)
-        #else:
-            #self.mb_graph_interval = 1
-
-
-    def get_genome_mb_length(self):
-        """Gets the sum of each chromosome's length in megabases"""
-
-        return sum([ind_chromosome.mb_length for ind_chromosome in self.chromosomes.values()])
-
-
-    def get_genome_cm_length(self):
-        """Gets the sum of each chromosome's length in centimorgans"""
-
-        return sum([ind_chromosome.cm_length for ind_chromosome in self.chromosomes.values()])
-
-    def get_cm_length_list(self):
-        """Chromosome length in centimorgans
-
-        Calculates the length in centimorgans by subtracting the centimorgan position
-        of the last marker in a chromosome by the position of the first marker
-
-        """
-
-        self.dataset.group.read_genotype_file()
-
-        self.cm_length_list = []
-
-        for chromosome in self.dataset.group.genotype:
-            self.cm_length_list.append(chromosome[-1].cM - chromosome[0].cM)
-
-        print("self.cm_length_list:", pf(self.cm_length_list))
-
-        assert len(self.cm_length_list) == len(self.chromosomes), "Uh-oh lengths should be equal!"
-        for counter, chromosome in enumerate(self.chromosomes.values()):
-            chromosome.cm_length = self.cm_length_list[counter]
-            #self.chromosomes[counter].cm_length = item
-
-        for key, value in self.chromosomes.items():
-            print("bread - %s: %s" % (key, pf(vars(value))))
-
-
-# Testing
-#if __name__ == '__main__':
-#    foo = dict(bar=dict(length))
+            self.chromosomes[item.OrderId] = IndChromosome(item.Name, item.Length)
\ No newline at end of file
diff --git a/wqflask/base/template.py b/wqflask/base/template.py
deleted file mode 100644
index aa8f90dc..00000000
--- a/wqflask/base/template.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# Copyright (C) University of Tennessee Health Science Center, Memphis, TN.
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Affero General Public License for more details.
-#
-# This program is available from Source Forge: at GeneNetwork Project
-# (sourceforge.net/projects/genenetwork/).
-#
-# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010)
-# at rwilliams@uthsc.edu and xzhou15@uthsc.edu
-#
-#
-#
-# This module is used by GeneNetwork project (www.genenetwork.org)
-#
-# Created by GeneNetwork Core Team 2010/08/10
-#
-# Last updated by GeneNetwork Core Team 2010/10/20
-
-template = """
-<?XML VERSION="1.0" ENCODING="UTF-8">
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<HTML>
-<HEAD>
-<TITLE>%s</TITLE>
-
-<META http-equiv=Content-Type content="text/html; charset=iso-8859-1">
-<META NAME="keywords" CONTENT="genetics, bioinformatics, genome, phenome, gene expression, complex trait analysis, gene mapping, SNP, quantitative trait locus QTL, expression eQTL, WebQTL, Traitnet, Traitnetwork, personalized medicine">
-<META NAME="description" CONTENT ="GeneNetwork is a free scientific web resource used to study relationships between differences in genes, environmental factors, phenotypes, and disease risk." >
-<META NAME="author" CONTENT ="GeneNetwork developers" >
-<META NAME="geo.placename" CONTENT ="Memphis, TN" >
-<META NAME="geo.region" CONTENT="US-TN">
-%s
-<LINK REL="stylesheet" TYPE="text/css" HREF='/css/general.css'>
-<LINK REL="stylesheet" TYPE="text/css" HREF='/css/menu.css'>
-<link rel="stylesheet" media="all" type="text/css" href="/css/tabbed_pages.css" />
-<LINK REL="apple-touch-icon" href="/images/ipad_icon3.png" />
-<link type="text/css" href='/css/custom-theme/jquery-ui-1.8.12.custom.css' rel='Stylesheet' />
-<link type="text/css" href='/css/tab_style.css' rel='Stylesheet' />
-
-<script type="text/javascript" src="/javascript/jquery-1.5.2.min.js"></script>
-<SCRIPT SRC="/javascript/webqtl.js"></SCRIPT>
-<SCRIPT SRC="/javascript/dhtml.js"></SCRIPT>
-<SCRIPT SRC="/javascript/tablesorter.js"></SCRIPT>
-<SCRIPT SRC="/javascript/jqueryFunction.js"></SCRIPT>
-<script src="/javascript/tabbed_pages.js" type="text/javascript"></script>
-<script src="/javascript/jquery-ui-1.8.12.custom.min.js" type="text/javascript"></script>
-%s
-
-<script type="text/javascript">
-  var _gaq = _gaq || [];
-  _gaq.push(['_setAccount', 'UA-3782271-1']);
-  _gaq.push(['_trackPageview']);
-  (function() {
-    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
-    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
-    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
-  })();
-</script>
-</HEAD>
-<BODY  bottommargin="2" leftmargin="2" rightmargin="2" topmargin="2" text=#000000 bgColor=#ffffff %s>
-%s
-<TABLE cellSpacing=5 cellPadding=4 width="100%%" border=0>
-        <TBODY>
-        <!-- Start of header -->
-        <TR>
-                %s
-        </TR>
-        <!-- End of header -->
-
-        <!-- Start of body -->
-        <TR>
-                <TD  bgColor=#eeeeee class="solidBorder">
-                <Table width= "100%%" cellSpacing=0 cellPadding=5>
-                <TR>
-                %s
-                </TR>
-                </TABLE>
-                </TD>
-        </TR>
-        <!-- End of body -->
-
-        <!-- Start of footer -->
-        <TR>
-                <TD align=center bgColor=#ddddff class="solidBorder">
-                        <TABLE width="90%%">%s</table>
-                </td>
-        </TR>
-        <!-- End of footer -->
-</TABLE>
-
-<!-- menu script itself. you should not modify this file -->
-<script language="JavaScript" src="/javascript/menu_new.js"></script>
-<!-- items structure. menu hierarchy and links are stored there -->
-<script language="JavaScript" src="/javascript/menu_items.js"></script>
-<!-- files with geometry and styles structures -->
-<script language="JavaScript" src="/javascript/menu_tpl.js"></script>
-<script language="JavaScript">
-        <!--//
-        // Note where menu initialization block is located in HTML document.
-        // Don't try to position menu locating menu initialization block in
-        // some table cell or other HTML element. Always put it before </body>
-        // each menu gets two parameters (see demo files)
-        // 1. items structure
-        // 2. geometry structure
-        new menu (MENU_ITEMS, MENU_POS);
-        // make sure files containing definitions for these variables are linked to the document
-        // if you got some javascript error like "MENU_POS is not defined", then you've made syntax
-        // error in menu_tpl.js file or that file isn't linked properly.
-
-        // also take a look at stylesheets loaded in header in order to set styles
-        //-->
-</script>
-</BODY>
-</HTML>
-"""
diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index bf87e879..5525472e 100644
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -3,14 +3,17 @@ from __future__ import absolute_import, division, print_function
 import string
 import resource
 import codecs
+import requests
 
-from htmlgen import HTMLgen2 as HT
+import redis
+Redis = redis.StrictRedis()
 
 from base import webqtlConfig
 from base.webqtlCaseData import webqtlCaseData
 from base.data_set import create_dataset
 from db import webqtlDatabaseFunction
 from utility import webqtlUtil
+from utility import hmac
 
 from wqflask import app
 
@@ -18,17 +21,11 @@ import simplejson as json
 from MySQLdb import escape_string as escape
 from pprint import pformat as pf
 
-from flask import Flask, g, request
+from flask import Flask, g, request, url_for
 
 from utility.logger import getLogger
 logger = getLogger(__name__ )
 
-from wqflask import user_manager
-
-def print_mem(stage=""):
-    mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
-    print("{}: {}".format(stage, mem/1024))
-
 class GeneralTrait(object):
     """
     Trait class defines a trait in webqtl, can be either Microarray,
@@ -39,13 +36,15 @@ class GeneralTrait(object):
     def __init__(self, get_qtl_info=False, get_sample_info=True, **kw):
         # xor assertion
         assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name";
+        self.name = kw.get('name')                 # Trait ID, ProbeSet ID, Published ID, etc.
         if kw.get('dataset_name'):
-            self.dataset = create_dataset(kw.get('dataset_name'))
-            #print(" in GeneralTrait created dataset:", self.dataset)
+            if kw.get('dataset_name') == "Temp":
+                temp_group = self.name.split("_")[2]
+                self.dataset = create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = temp_group)
+            else:
+                self.dataset = create_dataset(kw.get('dataset_name'))
         else:
             self.dataset = kw.get('dataset')
-        self.name = kw.get('name')                 # Trait ID, ProbeSet ID, Published ID, etc.
-        #print("THE NAME IS:", self.name)
         self.cellid = kw.get('cellid')
         self.identification = kw.get('identification', 'un-named trait')
         self.haveinfo = kw.get('haveinfo', False)
@@ -61,6 +60,10 @@ class GeneralTrait(object):
         self.num_overlap = None
         self.strand_probe = None
         self.symbol = None
+        self.display_name = self.name
+
+        self.LRS_score_repr = "N/A"
+        self.LRS_location_repr = "N/A"
 
         if kw.get('fullname'):
             name2 = value.split("::")
@@ -72,94 +75,11 @@ class GeneralTrait(object):
 
         # Todo: These two lines are necessary most of the time, but perhaps not all of the time
         # So we could add a simple if statement to short-circuit this if necessary
-        self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info)
+        if self.dataset.type != "Temp":
+            self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info)
         if get_sample_info != False:
             self = retrieve_sample_data(self, self.dataset)
 
-
-    def get_name(self):
-        stringy = ""
-        if self.dataset and self.name:
-            stringy = "%s::%s" % (self.dataset, self.name)
-            if self.cellid:
-                stringy += "::" + self.cellid
-        else:
-            stringy = self.description
-        return stringy
-
-
-    def get_given_name(self):
-        """
-         when user enter a trait or GN generate a trait, user want show the name
-         not the name that generated by GN randomly, the two follow function are
-         used to give the real name and the database. displayName() will show the
-         database also, getGivenName() just show the name.
-         For other trait, displayName() as same as getName(), getGivenName() as
-         same as self.name
-
-         Hongqiang 11/29/07
-
-        """
-        stringy = self.name
-        if self.dataset and self.name:
-            desc = self.dataset.get_desc()
-            if desc:
-                #desc = self.handle_pca(desc)
-                stringy = desc
-        return stringy
-
-
-    def display_name(self):
-        stringy = ""
-        if self.dataset and self.name:
-            desc = self.dataset.get_desc()
-            #desc = self.handle_pca(desc)
-            if desc:
-                #desc = self.handle_pca(desc)
-                #stringy = desc
-                #if desc.__contains__('PCA'):
-                #    desc = desc[desc.rindex(':')+1:].strip()
-                #else:
-                #    desc = desc[:desc.index('entered')].strip()
-                #desc = self.handle_pca(desc)
-                stringy = "%s::%s" % (self.dataset, desc)
-            else:
-                stringy = "%s::%s" % (self.dataset, self.name)
-                if self.cellid:
-                    stringy += "::" + self.cellid
-        else:
-            stringy = self.description
-
-        return stringy
-
-
-    #def __str__(self):
-    #       #return "%s %s" % (self.getName(), self.group)
-    #       return self.getName()
-    #__str__ = getName
-    #__repr__ = __str__
-
-    def export_data(self, samplelist, the_type="val"):
-        """
-        export data according to samplelist
-        mostly used in calculating correlation
-
-        """
-        result = []
-        for sample in samplelist:
-            if self.data.has_key(sample):
-                if the_type=='val':
-                    result.append(self.data[sample].val)
-                elif the_type=='var':
-                    result.append(self.data[sample].var)
-                elif the_type=='N':
-                    result.append(self.data[sample].N)
-                else:
-                    raise KeyError, `the_type`+' the_type is incorrect.'
-            else:
-                result.append(None)
-        return result
-
     def export_informative(self, include_variance=0):
         """
         export informative sample
@@ -179,19 +99,6 @@ class GeneralTrait(object):
                     sample_aliases.append(sample_data.name2)
         return  samples, vals, the_vars, sample_aliases
 
-
-    @property
-    def name_header_fmt(self):
-        '''Return a human-readable name for use in page header'''
-        if self.dataset.type == 'ProbeSet':
-            return self.symbol
-        elif self.dataset.type == 'Geno':
-            return self.name
-        elif self.dataset.type == 'Publish':
-            return self.post_publication_abbreviation
-        else:
-            return "unnamed"
-
     @property
     def description_fmt(self):
         '''Return a text formated description'''
@@ -200,6 +107,8 @@ class GeneralTrait(object):
                 formatted = self.description
                 if self.probe_target_description:
                     formatted += "; " + self.probe_target_description
+            else:
+                formatted = "Not available"
         elif self.dataset.type == 'Publish':
             if self.confidential:
                 formatted = self.pre_publication_description
@@ -212,11 +121,36 @@ class GeneralTrait(object):
     @property
     def alias_fmt(self):
         '''Return a text formatted alias'''
+
+        alias = 'Not available'
         if self.alias:
             alias = string.replace(self.alias, ";", " ")
             alias = string.join(string.split(alias), ", ")
-        else:
-            alias = 'Not available'
+
+        return alias
+
+    @property
+    def wikidata_alias_fmt(self):
+        '''Return a text formatted alias'''
+
+        alias = 'Not available'
+        if self.symbol:
+            human_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.upper())
+            mouse_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.capitalize())
+            other_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.lower())
+
+            if human_response and mouse_response and other_response:
+                alias_list = json.loads(human_response.content) + json.loads(mouse_response.content) + json.loads(other_response.content)
+
+                filtered_aliases = []
+                seen = set()
+                for item in alias_list:
+                    if item in seen:
+                        continue
+                    else:
+                        filtered_aliases.append(item)
+                        seen.add(item)
+                alias = "; ".join(filtered_aliases)
 
         return alias
 
@@ -244,60 +178,33 @@ class GeneralTrait(object):
             fmt += (' on the minus strand ')
 
         return fmt
-
-# In ProbeSet, there are maybe several annotations match one sequence
-# so we need use sequence(BlatSeq) as the identification, when we update
-# one annotation, we update the others who match the sequence also.
-#
-# Hongqiang Li, 3/3/2008
-def getSequence(trait, dataset_name):
-    dataset = create_dataset(dataset_name)
-     
-    if dataset.type == 'ProbeSet':
-        results = g.db.execute('''
-                       SELECT
-                               ProbeSet.BlatSeq
-                       FROM
-                               ProbeSet, ProbeSetFreeze, ProbeSetXRef
-                       WHERE
-                               ProbeSet.Id=ProbeSetXRef.ProbeSetId and
-                               ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and
-                               ProbeSet.Name = %s
-                               ProbeSetFreeze.Name = %s
-               ''', trait.name, dataset.name).fetchone()
-
-        return results[0]        
         
 def retrieve_sample_data(trait, dataset, samplelist=None):
     if samplelist == None:
         samplelist = []
 
-    results = dataset.retrieve_sample_data(trait.name)
+    if dataset.type == "Temp":
+        results = Redis.get(trait.name).split()
+    else:
+        results = dataset.retrieve_sample_data(trait.name)
 
     # Todo: is this necessary? If not remove
     trait.data.clear()
 
-    all_samples_ordered = dataset.group.all_samples_ordered()
-
     if results:
-        for item in results:
-            name, value, variance, num_cases, name2 = item
-            if not samplelist or (samplelist and name in samplelist):
-                trait.data[name] = webqtlCaseData(*item)   #name, value, variance, num_cases)
-                
-    return trait
-        
-def convert_location_to_value(chromosome, mb):
-    try:
-        location_value = int(chromosome)*1000 + float(mb)
-    except ValueError:
-        if chromosome.upper() == 'X':
-            location_value = 20*1000 + float(mb)
+        if dataset.type == "Temp":
+            all_samples_ordered = dataset.group.all_samples_ordered()
+            for i, item in enumerate(results):
+                try:
+                    trait.data[all_samples_ordered[i]] = webqtlCaseData(all_samples_ordered[i], float(item))
+                except:
+                    pass
         else:
-            location_value = (ord(str(chromosome).upper()[0])*1000 +
-                              float(mb))
-
-    return location_value
+            for item in results:
+                name, value, variance, num_cases, name2 = item
+                if not samplelist or (samplelist and name in samplelist):
+                    trait.data[name] = webqtlCaseData(*item)   #name, value, variance, num_cases)
+    return trait
 
 @app.route("/trait/get_sample_data")
 def get_sample_data():
@@ -307,25 +214,37 @@ def get_sample_data():
 
     trait_ob = GeneralTrait(name=trait, dataset_name=dataset)
 
-    return json.dumps([trait, {key: value.value for key, value in trait_ob.data.iteritems() }])
-
-    #jsonable_sample_data = {}
-    #for sample in trait_ob.data.iteritems():
-    #    jsonable_sample_data[sample] = trait_ob.data[sample].value
-    #
-    #return jsonable_sample_data
+    trait_dict = {}
+    trait_dict['name'] = trait
+    trait_dict['db'] = dataset
+    trait_dict['type'] = trait_ob.dataset.type
+    trait_dict['group'] = trait_ob.dataset.group.name
+    trait_dict['tissue'] = trait_ob.dataset.tissue
+    trait_dict['species'] = trait_ob.dataset.group.species
+    trait_dict['url'] = url_for('show_trait_page', trait_id = trait, dataset = dataset)
+    trait_dict['description'] = trait_ob.description_display
+    if trait_ob.dataset.type == "ProbeSet":
+        trait_dict['symbol'] = trait_ob.symbol
+        trait_dict['location'] = trait_ob.location_repr
+    elif trait_ob.dataset.type == "Publish":
+        if trait_ob.pubmed_id:
+            trait_dict['pubmed_link'] = trait_ob.pubmed_link
+        trait_dict['pubmed_text'] = trait_ob.pubmed_text
+
+    return json.dumps([trait_dict, {key: value.value for key, value in trait_ob.data.iteritems() }])
     
-def jsonable(trait, dataset_name):
+def jsonable(trait):
     """Return a dict suitable for using as json
 
     Actual turning into json doesn't happen here though"""
 
-    dataset = create_dataset(dataset_name)
+    dataset = create_dataset(dataset_name = trait.dataset.name, dataset_type = trait.dataset.type, group_name = trait.dataset.group.name)
     
     if dataset.type == "ProbeSet":
         return dict(name=trait.name,
                     symbol=trait.symbol,
                     dataset=dataset.name,
+                    dataset_name = dataset.shortname,
                     description=trait.description_display,
                     mean=trait.mean,
                     location=trait.location_repr,
@@ -337,7 +256,9 @@ def jsonable(trait, dataset_name):
         if trait.pubmed_id:
             return dict(name=trait.name,
                         dataset=dataset.name,
+                        dataset_name = dataset.shortname,
                         description=trait.description_display,
+                        abbreviation=trait.abbreviation,
                         authors=trait.authors,
                         pubmed_text=trait.pubmed_text,
                         pubmed_link=trait.pubmed_link,
@@ -348,7 +269,9 @@ def jsonable(trait, dataset_name):
         else:
             return dict(name=trait.name,
                         dataset=dataset.name,
+                        dataset_name = dataset.shortname,
                         description=trait.description_display,
+                        abbreviation=trait.abbreviation,
                         authors=trait.authors,
                         pubmed_text=trait.pubmed_text,
                         lrs_score=trait.LRS_score_repr,
@@ -358,6 +281,7 @@ def jsonable(trait, dataset_name):
     elif dataset.type == "Geno":
         return dict(name=trait.name,
                     dataset=dataset.name,
+                    dataset_name = dataset.shortname,
                     location=trait.location_repr
                     )
     else:
@@ -379,7 +303,7 @@ def jsonable_table_row(trait, dataset_name, index):
             additive = "N/A"
         else:
             additive = "%.3f" % round(float(trait.additive), 2)
-        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
+        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                 index,
                 '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                 trait.symbol,
@@ -395,7 +319,7 @@ def jsonable_table_row(trait, dataset_name, index):
         else:
             additive = "%.2f" % round(float(trait.additive), 2)
         if trait.pubmed_id:
-            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
+            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                     index,
                     '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                     trait.description_display,
@@ -405,7 +329,7 @@ def jsonable_table_row(trait, dataset_name, index):
                     trait.LRS_location_repr,
                     additive]
         else:
-            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
+            return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                     index,
                     '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                     trait.description_display,
@@ -415,7 +339,7 @@ def jsonable_table_row(trait, dataset_name, index):
                     trait.LRS_location_repr,
                     additive]
     elif dataset.type == "Geno":
-        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
+        return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">',
                 index,
                 '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>',
                 trait.location_repr]
@@ -428,21 +352,22 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False):
     if dataset.type == 'Publish':
         query = """
                 SELECT
-                        PublishXRef.Id, Publication.PubMed_ID,
+                        PublishXRef.Id, InbredSet.InbredSetCode, Publication.PubMed_ID,
                         Phenotype.Pre_publication_description, Phenotype.Post_publication_description, Phenotype.Original_description,
-                        Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation,
+                        Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation, PublishXRef.mean,
                         Phenotype.Lab_code, Phenotype.Submitter, Phenotype.Owner, Phenotype.Authorized_Users,
                         Publication.Authors, Publication.Title, Publication.Abstract,
                         Publication.Journal, Publication.Volume, Publication.Pages,
                         Publication.Month, Publication.Year, PublishXRef.Sequence,
                         Phenotype.Units, PublishXRef.comments
                 FROM
-                        PublishXRef, Publication, Phenotype, PublishFreeze
+                        PublishXRef, Publication, Phenotype, PublishFreeze, InbredSet
                 WHERE
                         PublishXRef.Id = %s AND
                         Phenotype.Id = PublishXRef.PhenotypeId AND
                         Publication.Id = PublishXRef.PublicationId AND
                         PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND
+                        PublishXRef.InbredSetId = InbredSet.Id AND
                         PublishFreeze.Id = %s
                 """ % (trait.name, dataset.id)
 
@@ -492,17 +417,25 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False):
         trait_info = g.db.execute(query,
                                   (string.join(dataset.display_fields,','),
                                                dataset.type, trait.name)).fetchone()
+
     if trait_info:
         trait.haveinfo = True
 
         #XZ: assign SQL query result to trait attributes.
         for i, field in enumerate(dataset.display_fields):
             holder = trait_info[i]
+            # if isinstance(trait_info[i], basestring):
+            #     logger.debug("HOLDER:", holder)
+            #     logger.debug("HOLDER2:", holder.decode(encoding='latin1'))
+            #     holder = unicode(trait_info[i], "utf-8", "ignore")
             if isinstance(trait_info[i], basestring):
-                holder = unicode(trait_info[i], "utf-8", "ignore")
+                holder = holder.encode('latin1')
             setattr(trait, field, holder)
 
         if dataset.type == 'Publish':
+            if trait.group_code:
+                trait.display_name = trait.group_code + "_" + str(trait.name)
+
             trait.confidential = 0
             if trait.pre_publication_description and not trait.pubmed_id:
                 trait.confidential = 1
@@ -513,6 +446,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False):
             #phenotype traits, then display the pre-publication description instead
             #of the post-publication description
             if trait.confidential:
+                trait.abbreviation = trait.pre_publication_abbreviation
                 trait.description_display = trait.pre_publication_description
 
                 #if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(
@@ -522,11 +456,16 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False):
                 #
                 #    description = self.pre_publication_description
             else:
+                trait.abbreviation = trait.post_publication_abbreviation
                 if description:
                     trait.description_display = description.strip()
                 else:
                     trait.description_display = ""
 
+            trait.abbreviation = unicode(str(trait.abbreviation).strip(codecs.BOM_UTF8), 'utf-8', errors="replace")
+            trait.description_display = unicode(str(trait.description_display).strip(codecs.BOM_UTF8), 'utf-8', errors="replace")
+            trait.authors = unicode(str(trait.authors).strip(codecs.BOM_UTF8), 'utf-8', errors="replace")
+
             if not trait.year.isdigit():
                 trait.pubmed_text = "N/A"
             else:
@@ -535,38 +474,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False):
             if trait.pubmed_id:
                 trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id
 
-
-        trait.homologeneid = None
         if dataset.type == 'ProbeSet' and dataset.group:
-            if trait.geneid:
-                #XZ, 05/26/2010: From time to time, this query get error message because some geneid values in database are not number.
-                #XZ: So I have to test if geneid is number before execute the query.
-                #XZ: The geneid values in database should be cleaned up.
-                #try:
-                #    float(self.geneid)
-                #    geneidIsNumber = True
-                #except ValueError:
-                #    geneidIsNumber = False
-                #if geneidIsNumber:
-                query = """
-                        SELECT
-                                HomologeneId
-                        FROM
-                                Homologene, Species, InbredSet
-                        WHERE
-                                Homologene.GeneId ='%s' AND
-                                InbredSet.Name = '%s' AND
-                                InbredSet.SpeciesId = Species.Id AND
-                                Species.TaxonomyId = Homologene.TaxonomyId
-                        """ % (escape(str(trait.geneid)), escape(dataset.group.name))
-                logger.sql(query)
-                result = g.db.execute(query).fetchone()
-                #else:
-                #    result = None
-
-                if result:
-                    trait.homologeneid = result[0]
-
             description_string = unicode(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8')
             target_string = unicode(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8')
 
@@ -582,47 +490,22 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False):
             # Save it for the jinja2 template
             trait.description_display = description_display
 
-            #XZ: trait_location_value is used for sorting
             trait.location_repr = 'N/A'
-            trait.location_value = 1000000
-
             if trait.chr and trait.mb:
-                #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y")
-                #This is so we can convert the location to a number used for sorting
-                trait_location_value = convert_location_to_value(trait.chr, trait.mb)
-                 #try:
-                #    trait_location_value = int(self.chr)*1000 + self.mb
-                #except ValueError:
-                #    if self.chr.upper() == 'X':
-                #        trait_location_value = 20*1000 + self.mb
-                #    else:
-                #        trait_location_value = (ord(str(self.chr).upper()[0])*1000 +
-                #                               self.mb)
-
-                #ZS: Put this in function currently called "convert_location_to_value"
                 trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb))
-                trait.location_value = trait_location_value
 
         elif dataset.type == "Geno":
             trait.location_repr = 'N/A'
-            trait.location_value = 1000000
-
             if trait.chr and trait.mb:
-                #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y")
-                #This is so we can convert the location to a number used for sorting
-                trait_location_value = convert_location_to_value(trait.chr, trait.mb)
-
-                #ZS: Put this in function currently called "convert_location_to_value"
                 trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb))
-                trait.location_value = trait_location_value
 
         if get_qtl_info:
             #LRS and its location
             trait.LRS_score_repr = "N/A"
-            trait.LRS_score_value = 0
             trait.LRS_location_repr = "N/A"
-            trait.LRS_location_value = 1000000
+            trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.additive = ""
             if dataset.type == 'ProbeSet' and not trait.cellid:
+                trait.mean = ""
                 query = """
                         SELECT
                                 ProbeSetXRef.Locus, ProbeSetXRef.LRS, ProbeSetXRef.pValue, ProbeSetXRef.mean, ProbeSetXRef.additive
@@ -653,9 +536,6 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False):
                             trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = ""
                     else:
                         trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = ""
-                else:
-                    trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.mean = trait.additive = ""
-
 
             if dataset.type == 'Publish':
                 query = """
@@ -692,19 +572,9 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False):
                     trait.locus = trait.lrs = trait.additive = ""
 
             if (dataset.type == 'Publish' or dataset.type == "ProbeSet") and trait.locus_chr != "" and trait.locus_mb != "":
-                #XZ: LRS_location_value is used for sorting
-                try:
-                    LRS_location_value = int(trait.locus_chr)*1000 + float(trait.locus_mb)
-                except:
-                    if trait.locus_chr.upper() == 'X':
-                        LRS_location_value = 20*1000 + float(trait.locus_mb)
-                    else:
-                        LRS_location_value = ord(str(trait.locus_chr).upper()[0])*1000 + float(trait.locus_mb)
-
                 trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (trait.locus_chr, float(trait.locus_mb))
                 if trait.lrs != "":
                     trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs
-                    trait.LRS_score_value = LRS_score_value = trait.lrs
     else:
         raise KeyError, `trait.name`+' information is not found in the database.'
         
diff --git a/wqflask/base/trait_collection.py b/wqflask/base/trait_collection.py
deleted file mode 100644
index d388a3af..00000000
--- a/wqflask/base/trait_collection.py
+++ /dev/null
@@ -1,53 +0,0 @@
-class TraitCollection(object):

-    

-    def __init__(self, is_anon=False):

-        self.is_anon = is_anon

-        

-        

-    @app.route("/collections/remove", methods=('POST',))

-    def remove_traits():

-        if is_anon:

-            AnonCollection.remove_traits()

-        else:

-            UserCollection.remove_traits()

-            

-        params = request.form

-        print("params are:", params)

-        uc_id = params['uc_id']

-        uc = model.UserCollection.query.get(uc_id)

-        traits_to_remove = params.getlist('traits[]')

-        print("traits_to_remove are:", traits_to_remove)

-        traits_to_remove = process_traits(traits_to_remove)

-        print("\n\n  after processing, traits_to_remove:", traits_to_remove)

-        all_traits = uc.members_as_set()

-        print("  all_traits:", all_traits)

-        members_now = all_traits - traits_to_remove

-        print("  members_now:", members_now)

-        print("Went from {} to {} members in set.".format(len(all_traits), len(members_now)))

-        uc.members = json.dumps(list(members_now))

-        uc.changed_timestamp = datetime.datetime.utcnow()

-        db_session.commit()

-    

-        # We need to return something so we'll return this...maybe in the future

-        # we can use it to check the results

-        return str(len(members_now))

-

-    def __init__(self, anon_id)

-        self.anon_key = anon_key

-        self.collection_members = Redis.smembers(self.anon_id)

-        print("self.collection_members is:", self.collection_members)

-        self.num_members = len(self.collection_members)

-        

-

-    @app.route("/collections/remove", methods=('POST',))

-    def remove_traits(traits_to_remove):

-        print("traits_to_remove:", traits_to_remove)

-        for trait in traits_to_remove:

-            Redis.srem(self.anon_id, trait)

-        members_now = self.collection_members - traits_to_remove

-        print("members_now:", members_now)

-        print("Went from {} to {} members in set.".format(len(self.collection_members), len(members_now)))

-

-        # We need to return something so we'll return this...maybe in the future

-        # we can use it to check the results

-        return str(len(members_now))

diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py
index 845a7224..d8487f01 100644
--- a/wqflask/base/webqtlCaseData.py
+++ b/wqflask/base/webqtlCaseData.py
@@ -44,12 +44,12 @@ class webqtlCaseData(object):
 
     def __repr__(self):
         str = "<webqtlCaseData> "
-        if self.value:
+        if self.value != None:
             str += "value=%2.3f" % self.value
-        if self.variance:
+        if self.variance != None:
             str += " variance=%2.3f" % self.variance
         if self.num_cases:
-            str += " ndata=%d" % self.num_cases
+            str += " ndata=%s" % self.num_cases
         if self.name:
             str += " name=%s" % self.name
         if self.name2:
@@ -66,14 +66,21 @@ class webqtlCaseData(object):
 
     @property
     def display_value(self):
-        if self.value:
+        if self.value != None:
             return "%2.3f" % self.value
         else:
             return "x"
 
     @property
     def display_variance(self):
-        if self.variance:
+        if self.variance != None:
             return "%2.3f" % self.variance
         else:
             return "x"
+
+    @property
+    def display_num_cases(self):
+        if self.num_cases != None:
+            return "%s" % self.num_cases
+        else:
+            return "x"
diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index e5f10edf..018d5d54 100644
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -20,45 +20,44 @@ USERDICT = {'guest':1,'user':2, 'admin':3, 'root':4}
 #minimum number of informative strains
 KMININFORMATIVE = 5
 
-#maximum number of traits for interval mapping
-MULTIPLEMAPPINGLIMIT = 11
-
-#maximum number of traits for correlation
-MAXCORR = 100
-
 #Daily download limit from one IP
 DAILYMAXIMUM = 1000
 
 #maximum LRS value
 MAXLRS = 460.0
 
-#temporary data life span
-MAXLIFE = 86400
-
 #MINIMUM Database public value
 PUBLICTHRESH = 0
 
-#NBCI address
-NCBI_LOCUSID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s"
-UCSC_REFSEQ = "http://genome.cse.ucsc.edu/cgi-bin/hgGene?db=%s&hgg_gene=%s&hgg_chrom=chr%s&hgg_start=%s&hgg_end=%s"
-GENBANK_ID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=Nucleotide&cmd=search&doptcmdl=DocSum&term=%s"
-OMIM_ID = "http://www.ncbi.nlm.nih.gov/omim/%s"
-UNIGEN_ID = "http://www.ncbi.nlm.nih.gov/UniGene/clust.cgi?ORG=%s&CID=%s";
-HOMOLOGENE_ID = "http://www.ncbi.nlm.nih.gov/sites/entrez?Db=homologene&Cmd=DetailsSearch&Term=%s"
+#EXTERNAL LINK ADDRESSES
 PUBMEDLINK_URL = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids=%s&dopt=Abstract"
-UCSC_POS = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=mammal&org=%s&db=%s&position=chr%s:%s-%s&pix=800&Submit=submit"
 UCSC_BLAT = 'http://genome.ucsc.edu/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s'
 UTHSC_BLAT = 'http://ucscbrowser.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s'
-UCSC_GENOME = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=http://web2qtl.utmem.edu:88/snp/chr%s"
-ENSEMBLE_BLAT = 'http://www.ensembl.org/Mus_musculus/featureview?type=AffyProbe&id=%s'
-DBSNP = 'http://www.ncbi.nlm.nih.gov/SNP/snp_ref.cgi?type=rs&rs=%s'
-UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s&hgt.customText=http://gbic.biol.rug.nl/~ralberts/tracks/%s/%s"
-GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit"
-ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s"
-
-# The following paths are no longer in use!
-#   HTMLPATH is replaced by GENODIR
-#   IMGDIR is replaced by GENERATED_IMAGE_DIR
+UTHSC_BLAT2 = 'http://ucscbrowserbeta.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s'
+GENOMEBROWSER_URL="https://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=%s"
+NCBI_LOCUSID = "http://www.ncbi.nlm.nih.gov/gene?cmd=Retrieve&dopt=Graphics&list_uids=%s"
+GENBANK_ID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=Nucleotide&cmd=search&doptcmdl=DocSum&term=%s"
+OMIM_ID = "http://www.ncbi.nlm.nih.gov/omim/%s"
+UNIGEN_ID = "http://www.ncbi.nlm.nih.gov/UniGene/clust.cgi?ORG=%s&CID=%s"
+HOMOLOGENE_ID = "http://www.ncbi.nlm.nih.gov/homologene/?term=%s"
+GENOTATION_URL = "http://www.genotation.org/Getd2g.pl?gene_list=%s"
+GTEX_URL = "https://www.gtexportal.org/home/gene/%s"
+GENEBRIDGE_URL = "https://www.systems-genetics.org/modules_by_gene/%s?organism=%s"
+GENEMANIA_URL = "https://genemania.org/search/%s/%s"
+UCSC_REFSEQ = "http://genome.cse.ucsc.edu/cgi-bin/hgTracks?db=%s&hgg_gene=%s&hgg_chrom=chr%s&hgg_start=%s&hgg_end=%s"
+BIOGPS_URL = "http://biogps.org/?org=%s#goto=genereport&id=%s"
+STRING_URL = "http://string-db.org/newstring_cgi/show_network_section.pl?identifier=%s"
+PANTHER_URL = "http://www.pantherdb.org/genes/geneList.do?searchType=basic&fieldName=all&organism=all&listType=1&fieldValue=%s"
+GEMMA_URL = "http://www.chibi.ubc.ca/Gemma/gene/showGene.html?ncbiid=%s"
+ABA_URL = "http://mouse.brain-map.org/search/show?search_type=gene&search_term=%s"
+EBIGWAS_URL = "https://www.ebi.ac.uk/gwas/search?query=%s"
+WIKI_PI_URL = "http://severus.dbmi.pitt.edu/wiki-pi/index.php/search?q=%s"
+ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Transcript/Idhistory?t=%s"
+DBSNP = 'http://ensembl.org/Mus_musculus/Variation/Population?v=%s'
+PROTEIN_ATLAS_URL = "http://www.proteinatlas.org/search/%s"
+OPEN_TARGETS_URL = "https://genetics.opentargets.org/gene/%s"
+UNIPROT_URL = "https://www.uniprot.org/uniprot/%s"
+RGD_URL = "https://rgd.mcw.edu/rgdweb/elasticResults.html?term=%s&category=Gene&species=%s"
 
 # Temporary storage (note that this TMPDIR can be set as an
 # environment variable - use utility.tools.TEMPDIR when you
@@ -70,8 +69,8 @@ assert_writable_dir(TMPDIR)
 
 CACHEDIR             = mk_dir(TMPDIR+'/cache/')
 # We can no longer write into the git tree:
-GENERATED_IMAGE_DIR  = mk_dir(TMPDIR+'/generated/')
-GENERATED_TEXT_DIR   = mk_dir(TMPDIR+'/generated_text/')
+GENERATED_IMAGE_DIR  = mk_dir(TMPDIR+'generated/')
+GENERATED_TEXT_DIR   = mk_dir(TMPDIR+'generated_text/')
 
 # Make sure we have permissions to access these
 assert_writable_dir(CACHEDIR)
@@ -81,6 +80,7 @@ assert_writable_dir(GENERATED_TEXT_DIR)
 # Flat file directories
 GENODIR              = flat_files('genotype')+'/'
 assert_dir(GENODIR)
+assert_dir(GENODIR+'bimbam') # for gemma
 
 # JSON genotypes are OBSOLETE
 JSON_GENODIR         = flat_files('genotype/json')+'/'
@@ -92,4 +92,3 @@ if not valid_path(JSON_GENODIR):
 PORTADDR = "http://50.16.251.170"
 INFOPAGEHREF = '/dbdoc/%s.html'
 CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR'
-SCRIPTFILE = 'main.py'
diff --git a/wqflask/base/webqtlFormData.py b/wqflask/base/webqtlFormData.py
deleted file mode 100644
index 10251756..00000000
--- a/wqflask/base/webqtlFormData.py
+++ /dev/null
@@ -1,352 +0,0 @@
-# Copyright (C) University of Tennessee Health Science Center, Memphis, TN.
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Affero General Public License for more details.
-#
-# This program is available from Source Forge: at GeneNetwork Project
-# (sourceforge.net/projects/genenetwork/).
-#
-# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010)
-# at rwilliams@uthsc.edu and xzhou15@uthsc.edu
-#
-#
-#
-# This module is used by GeneNetwork project (www.genenetwork.org)
-#
-# Created by GeneNetwork Core Team 2010/08/10
-#
-# Last updated by GeneNetwork Core Team 2010/10/20
-
-#from mod_python import Cookie
-
-from __future__ import print_function
-from pprint import pformat as pf
-
-import string
-import os
-
-import reaper
-
-import webqtlConfig
-from webqtlCaseData import webqtlCaseData
-from utility import webqtlUtil
-
-class webqtlFormData(object):
-    'Represents data from a WebQTL form page, needed to generate the next page'
-
-    attrs = ('formID','group','genotype','samplelist','allsamplelist', 'display_variance'
-                'suggestive','significance','submitID','identification', 'enablevariance',
-                'nperm','nboot','email','incparentsf1','genotype_1','genotype_2','traitInfo')
-
-    #XZ: Attention! All attribute values must be picklable!
-
-    def __init__(self,
-                 start_vars = None,
-                 req = None,
-                 mod_python_session=None,
-                 FieldStorage_formdata=None):
-        # Todo: rework this whole thing
-        print("in webqtlFormData start_vars are:", pf(start_vars))
-        for item in webqtlFormData.attrs:
-            self.__dict__[item] = None
-
-        #ZS: This is only used in DataEditingPage.py (as far as I know)
-        self.varianceDispName = None
-
-        for item in start_vars:
-            self.__dict__[item] = start_vars[item]
-        print("  Now self.dict is:", pf(self.__dict__))
-
-        #Todo: This can't be good below...rework
-        try:
-            self.remote_ip = req.connection.remote_ip
-        except:
-            self.remote_ip = '1.2.3.4'
-
-        if req and req.headers_in.has_key('referer'):
-            self.refURL = req.headers_in['referer']
-        else:
-            self.refURL = None
-
-        # For now let's just comment all this out - Sam
-
-        #self.cookies = cookieData.cookieData(Cookie.get_cookies(req)) #XZ: dictionary type. To hold values transfered from mod_python Cookie.
-        #
-        ##XZ: dictionary type. To hold values transfered from mod_python Session object. We assume that it is always picklable.
-        #self.input_session_data = sessionData.sessionData( mod_python_session )
-        #
-        ##XZ: FieldStorage_formdata may contain item that can't be pickled. Must convert to picklable data.
-        #self.formdata = cgiData( FieldStorage_formdata )
-        #
-        ##get Form ID
-        #self.formID = self.formdata.getfirst('FormID')
-        #
-        ##get rest of the attributes
-        #if self.formID:
-        #       for item in self.attrs:
-        #               value = self.formdata.getfirst(item)
-        #               if value != None:
-        #                       setattr(self,item,string.strip(value))
-
-        self.ppolar = None
-        self.mpolar = None
-
-        print("[yellow] self.group is:", self.group)
-        if self.group:
-            #try:
-            #    # NL, 07/27/2010. ParInfo has been moved from webqtlForm.py to webqtlUtil.py;
-            _f1, _f12, self.mpolar, self.ppolar = webqtlUtil.ParInfo[self.group]
-            #except:
-            #    f1 = f12 = self.mpolar = self.ppolar = None
-
-
-        def set_number(stringy):
-            return int(stringy) if stringy else 2000 # Rob asked to change the default value to 2000
-
-        self.nperm = set_number(self.nperm)
-        self.nboot = set_number(self.nboot)
-
-
-        #if self.allsamplelist:
-        #    self.allsamplelist = map(string.strip, string.split(self.allsamplelist))
-        print("self.allsamplelist is:", self.allsamplelist)
-        if self.allsamplelist:
-            self.allsamplelist = self.allsamplelist.split()
-        print("now self.allsamplelist is:", self.allsamplelist)
-        #self.readGenotype()
-        #self.readData()
-
-        if self.group == 'BXD300':
-            self.group = 'BXD'
-
-
-    def __getitem__(self, key):
-        print("in __getitem__")
-        return self.__dict__[key]
-
-    def get(self, key, default=None):
-        if key in self.__dict__:
-            return self.__dict__[key]
-        else:
-            return default
-
-    def __str__(self):
-        rstr = ''
-        for item in self.attrs:
-            if item != 'genotype':
-                rstr += '%s:%s\n' % (item,str(getattr(self,item)))
-        return rstr
-
-
-    def readGenotype(self):
-        '''read genotype from .geno file'''
-        if self.group == 'BXD300':
-            self.group = 'BXD'
-
-        assert self.group, "self.group needs to be set"
-
-        #genotype_1 is Dataset Object without parents and f1
-        #genotype_2 is Dataset Object with parents and f1 (not for intercross)
-
-        self.genotype_1 = reaper.Dataset()
-
-        full_filename = locate(self.group + '.geno','genotype')
-
-        # reaper barfs on unicode filenames, so here we ensure it's a string
-        full_filename = str(full_filename)
-        self.genotype_1.read(full_filename)
-
-        print("Got to after read")
-
-        try:
-            # NL, 07/27/2010. ParInfo has been moved from webqtlForm.py to webqtlUtil.py;
-            _f1, _f12, _mat, _pat = webqtlUtil.ParInfo[self.group]
-        except KeyError:
-            _f1 = _f12 = _mat = _pat = None
-
-        self.genotype_2 = self.genotype_1
-        if self.genotype_1.type == "group" and _mat and _pat:
-            self.genotype_2 = self.genotype_1.add(Mat=_mat, Pat=_pat)       #, F1=_f1)
-
-        #determine default genotype object
-        if self.incparentsf1 and self.genotype_1.type != "intercross":
-            self.genotype = self.genotype_2
-        else:
-            self.incparentsf1 = 0
-            self.genotype = self.genotype_1
-
-        self.samplelist = list(self.genotype.prgy)
-        self.f1list = []
-        self.parlist = []
-
-        if _f1 and _f12:
-            self.f1list = [_f1, _f12]
-        if _mat and _pat:
-            self.parlist = [_mat, _pat]
-
-
-    def readData(self, samplelist, incf1=None):
-        '''read user input data or from trait data and analysis form'''
-
-        if incf1 == None:
-            incf1 = []
-
-        if not self.genotype:
-            self.readGenotype()
-        if not samplelist:
-            if incf1:
-                samplelist = self.f1list + self.samplelist
-            else:
-                samplelist = self.samplelist
-
-        #print("before traitfiledata self.traitfile is:", pf(self.traitfile))
-
-        traitfiledata = getattr(self, "traitfile", None)
-        traitpastedata = getattr(self, "traitpaste", None)
-        variancefiledata = getattr(self, "variancefile", None)
-        variancepastedata = getattr(self, "variancepaste", None)
-        Nfiledata = getattr(self, "Nfile", None)
-
-        #### Todo: Rewrite below when we get to someone submitting their own trait #####
-
-        def to_float(item):
-            try:
-                return float(item)
-            except ValueError:
-                return None
-
-        print("bottle samplelist is:", samplelist)
-        if traitfiledata:
-            tt = traitfiledata.split()
-            values = map(webqtlUtil.StringAsFloat, tt)
-        elif traitpastedata:
-            tt = traitpastedata.split()
-            values = map(webqtlUtil.StringAsFloat, tt)
-        else:
-            print("mapping formdataasfloat")
-            #values = map(self.FormDataAsFloat, samplelist)
-            values = [to_float(getattr(self, key)) for key in samplelist]
-        print("rocket values is:", values)
-
-
-        if len(values) < len(samplelist):
-            values += [None] * (len(samplelist) - len(values))
-        elif len(values) > len(samplelist):
-            values = values[:len(samplelist)]
-        print("now values is:", values)
-
-
-        if variancefiledata:
-            tt = variancefiledata.split()
-            variances = map(webqtlUtil.StringAsFloat, tt)
-        elif variancepastedata:
-            tt = variancepastedata.split()
-            variances = map(webqtlUtil.StringAsFloat, tt)
-        else:
-            variances = map(self.FormVarianceAsFloat, samplelist)
-
-        if len(variances) < len(samplelist):
-            variances += [None]*(len(samplelist) - len(variances))
-        elif len(variances) > len(samplelist):
-            variances = variances[:len(samplelist)]
-
-        if Nfiledata:
-            tt = string.split(Nfiledata)
-            nsamples = map(webqtlUtil.IntAsFloat, tt)
-            if len(nsamples) < len(samplelist):
-                nsamples += [None]*(len(samplelist) - len(nsamples))
-        else:
-            nsamples = map(self.FormNAsFloat, samplelist)
-
-        ##values, variances, nsamples is obsolete
-        self.allTraitData = {}
-        for i, _sample in enumerate(samplelist):
-            if values[i] != None:
-                self.allTraitData[_sample] = webqtlCaseData(
-                    _sample, values[i], variances[i], nsamples[i])
-        print("allTraitData is:", pf(self.allTraitData))
-
-
-
-    def informativeStrains(self, samplelist=None, include_variances = None):
-        '''if readData was called, use this to output informative samples (sample with values)'''
-
-        if not samplelist:
-            samplelist = self.samplelist
-
-        samples = []
-        values = []
-        variances = []
-
-        #print("self.allTraitData is:", pf(self.allTraitData))
-
-        for sample in samplelist:
-            if sample in self.allTraitData:
-                _val, _var = self.allTraitData[sample].value, self.allTraitData[sample].variance
-                if _val != None:
-                    if include_variances:
-                        if _var != None:
-                            samples.append(sample)
-                            values.append(_val)
-                            variances.append(_var)
-                    else:
-                        samples.append(sample)
-                        values.append(_val)
-                        variances.append(None)
-
-        return samples, values, variances, len(samples)
-
-
-
-    #def FormDataAsFloat(self, key):
-    #
-    #    #try:
-    #    #    return float(self.key)
-    #    #except:
-    #    #    return None
-
-
-    def FormVarianceAsFloat(self, key):
-        try:
-            return float(self.formdata.getfirst('V' + key))
-        except:
-            return None
-
-    def FormNAsFloat(self, key):
-        try:
-            return int(self.formdata.getfirst('N' + key))
-        except:
-            return None
-
-    def Sample(self):
-        'Create some dummy data for testing'
-        self.group = 'BXD'
-        self.incparentsf1 = 'on'
-        #self.display = 9.2
-        #self.significance = 16.1
-        self.readGenotype()
-        self.identification = 'BXD : Coat color example by Lu Lu, et al'
-        #self.readGenotype()
-        #self.genotype.ReadMM('AXBXAforQTL')
-        #self.samplelist = map((lambda x, y='': '%s%s' % (y,x)), self.genotype.prgy)
-        #self.samplelist.sort()
-        self.allTraitData = {'BXD29': webqtlCaseData(3), 'BXD28': webqtlCaseData(2),
-        'BXD25': webqtlCaseData(2), 'BXD24': webqtlCaseData(2), 'BXD27': webqtlCaseData(2),
-        'BXD21': webqtlCaseData(1), 'BXD20': webqtlCaseData(4), 'BXD23': webqtlCaseData(4),
-        'BXD22': webqtlCaseData(3), 'BXD14': webqtlCaseData(4), 'BXD15': webqtlCaseData(2),
-        'BXD16': webqtlCaseData(3), 'BXD11': webqtlCaseData(4), 'BXD12': webqtlCaseData(3),
-        'BXD13': webqtlCaseData(2), 'BXD18': webqtlCaseData(3), 'BXD19': webqtlCaseData(3),
-        'BXD38': webqtlCaseData(3), 'BXD39': webqtlCaseData(3), 'BXD36': webqtlCaseData(2),
-        'BXD34': webqtlCaseData(4), 'BXD35': webqtlCaseData(4), 'BXD32': webqtlCaseData(4),
-        'BXD33': webqtlCaseData(3), 'BXD30': webqtlCaseData(1), 'BXD31': webqtlCaseData(4),
-        'DBA/2J': webqtlCaseData(1), 'BXD8': webqtlCaseData(3), 'BXD9': webqtlCaseData(1),
-        'BXD6': webqtlCaseData(3), 'BXD5': webqtlCaseData(3), 'BXD2': webqtlCaseData(4),
-        'BXD1': webqtlCaseData(1), 'C57BL/6J': webqtlCaseData(4), 'B6D2F1': webqtlCaseData(4),
-        'BXD42': webqtlCaseData(4), 'BXD40': webqtlCaseData(3)}