Merge branch 'testing' of https://github.com/genenetwork/genenetwork2 into HEAD

18 files changed, 280 insertions, 138 deletions

diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py
index db40f6ea..23b0b650 100644
--- a/wqflask/utility/gen_geno_ob.py
+++ b/wqflask/utility/gen_geno_ob.py
@@ -156,7 +156,10 @@ class Locus(object):
                 self.cM = float(marker_row[geno_ob.cm_column])
             except:
                 self.cM = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else 0
-            self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
+            try:
+                self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None
+            except:
+                self.Mb = self.cM
 
             geno_table = {
                 geno_ob.mat: -1,
diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py
index 74eb869f..fa6e03b4 100644
--- a/wqflask/wqflask/collect.py
+++ b/wqflask/wqflask/collect.py
@@ -75,7 +75,7 @@ def collections_add():
 
     collections = g.user_session.user_collections
     if len(collections) < 1:
-        collection_name = "Default Collection"
+        collection_name = "Your Default Collection"
         uc_id = g.user_session.add_collection(collection_name, set())
         collections = g.user_session.user_collections
 
@@ -113,7 +113,7 @@ def collections_new():
                     collection_name = collection["name"]
                     default_collection_exists = True
             if not default_collection_exists:
-                return create_new("Default Collection")
+                return create_new("Your Default Collection")
         else:
             collection_id = params['existing_collection'].split(":")[0]
             collection_name = params['existing_collection'].split(":")[1]
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index b8f84721..a7e11738 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -267,8 +267,8 @@ class DisplayMappingResults(object):
         else:
             self.genotype = self.dataset.group.read_genotype_file()
 
-        if self.mapping_method == "rqtl_geno" and self.genotype.filler == True:
-            self.genotype = self.genotype.read_rdata_output(self.qtlresults)
+        #if self.mapping_method == "rqtl_geno" and self.genotype.filler == True:
+        #    self.genotype = self.genotype.read_rdata_output(self.qtlresults)
 
         #Darwing Options
         try:
@@ -935,10 +935,14 @@ class DisplayMappingResults(object):
                 string3 += 'no cofactors'
         elif self.mapping_method == "rqtl_plink" or self.mapping_method == "rqtl_geno":
             string3 = 'Using R/qtl mapping method with '
-            if self.controlLocus and self.doControl != "false":
+            if self.covariates != "":
+                string3 += 'the cofactors below:'
+                cofactor_names = ", ".join([covar.split(":")[0] for covar in self.covariates.split(",")])
+                string4 = cofactor_names
+            elif self.controlLocus and self.doControl != "false":
                 string3 += '%s as control' % self.controlLocus
             else:
-                string3 += 'no control for other QTLs'
+                string3 += 'no cofactors'
         else:
             string3 = 'Using Haldane mapping function with '
             if self.controlLocus and self.doControl != "false":
diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py
index d76f3812..aae8e602 100644
--- a/wqflask/wqflask/marker_regression/rqtl_mapping.py
+++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py
@@ -11,7 +11,7 @@ from utility.tools import locate, TEMPDIR
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
 
-def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan, samples, cofactors):
+def run_rqtl_geno(vals, samples, dataset, method, model, permCheck, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, pair_scan, cofactors):
     ## Get pointers to some common R functions
     r_library     = ro.r["library"]                 # Map the library function
     r_c           = ro.r["c"]                       # Map the c function
@@ -27,26 +27,26 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
     calc_genoprob              = ro.r["calc.genoprob"]         # Map the calc.genoprob function
 
     crossname = dataset.group.name
-    try:
-        generate_cross_from_rdata(dataset)
-        read_cross_from_rdata      = ro.r["generate_cross_from_rdata"] # Map the local read_cross_from_rdata function
-        genofilelocation  = locate(crossname + ".RData", "genotype/rdata")
-        cross_object = read_cross_from_rdata(genofilelocation)  # Map the local GENOtoCSVR function
-    except:
-        generate_cross_from_geno(dataset)
-        GENOtoCSVR                 = ro.r["GENOtoCSVR"]            # Map the local GENOtoCSVR function
-        crossfilelocation = TMPDIR + crossname + ".cross"
-        genofilelocation  = locate(crossname + ".geno", "genotype")
-
-        GENOtoCSVR      = ro.r["GENOtoCSVR"]                                # Map the local GENOtoCSVR function
-        cross_object = GENOtoCSVR(genofilelocation, crossfilelocation)      # TODO: Add the SEX if that is available
+    #try:
+    #    generate_cross_from_rdata(dataset)
+    #    read_cross_from_rdata      = ro.r["generate_cross_from_rdata"] # Map the local read_cross_from_rdata function
+    #    genofilelocation  = locate(crossname + ".RData", "genotype/rdata")
+    #    cross_object = read_cross_from_rdata(genofilelocation)  # Map the local GENOtoCSVR function
+    #except:
+    generate_cross_from_geno(dataset)
+    GENOtoCSVR                 = ro.r["GENOtoCSVR"]            # Map the local GENOtoCSVR function
+    crossfilelocation = TMPDIR + crossname + ".cross"
+    genofilelocation  = locate(dataset.group.genofile, "genotype")
+    cross_object = GENOtoCSVR(genofilelocation, crossfilelocation)      # TODO: Add the SEX if that is available
 
     if manhattan_plot:
         cross_object = calc_genoprob(cross_object)
     else:
         cross_object = calc_genoprob(cross_object, step=1, stepwidth="max")
 
-    cross_object = add_phenotype(cross_object, sanitize_rqtl_phenotype(vals), "the_pheno")                 # Add the phenotype
+    pheno_string = sanitize_rqtl_phenotype(vals)
+
+    cross_object = add_phenotype(cross_object, pheno_string, "the_pheno")                 # Add the phenotype
 
     # Scan for QTLs
     marker_covars = create_marker_covariates(control_marker, cross_object)  # Create the additive covariate markers
@@ -78,15 +78,22 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
             logger.info("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method)
 
         if num_perm > 0 and permCheck == "ON":                                                                   # Do permutation (if requested by user)
-            if do_control == "true" or cofactors != "":
-                perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = num_perm, model=model, method=method)
+            if len(perm_strata_list) > 0: #ZS: The strata list would only be populated if "Stratified" was checked on before mapping
+                cross_object, strata_ob = add_perm_strata(cross_object, perm_strata_list)
+                if do_control == "true" or cofactors != "":
+                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = int(num_perm), perm_strata = strata_ob, model=model, method=method)
+                else:
+                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, perm_strata = strata_ob, model=model, method=method)
             else:
-                perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method)
+                if do_control == "true" or cofactors != "":
+                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = int(num_perm), model=model, method=method)
+                else:
+                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method)
 
             perm_output, suggestive, significant = process_rqtl_perm_results(num_perm, perm_data_frame)          # Functions that sets the thresholds for the webinterface
-            return perm_output, suggestive, significant, process_rqtl_results(result_data_frame)
+            return perm_output, suggestive, significant, process_rqtl_results(result_data_frame, dataset.group.species)
         else:
-            return process_rqtl_results(result_data_frame)
+            return process_rqtl_results(result_data_frame, dataset.group.species)
 
 def generate_cross_from_rdata(dataset):
     rdata_location  = locate(dataset.group.name + ".RData", "genotype/rdata")
@@ -112,8 +119,12 @@ def generate_cross_from_geno(dataset):        # TODO: Need to figure out why som
          header = readLines(genotypes, 40)                                                                                 # Assume a geno header is not longer than 40 lines
          toskip = which(unlist(lapply(header, function(x){ length(grep("Chr\t", x)) })) == 1)-1                            # Major hack to skip the geno headers
 
-         genocodes <- c(getGenoCode(header, 'mat'), getGenoCode(header, 'het'), getGenoCode(header, 'pat'))                # Get the genotype codes
          type <- getGenoCode(header, 'type')
+         if(type == '4-way'){
+            genocodes <- c('1','2','3','4')
+         } else {
+            genocodes <- c(getGenoCode(header, 'mat'), getGenoCode(header, 'het'), getGenoCode(header, 'pat'))                # Get the genotype codes
+         }
          genodata <- read.csv(genotypes, sep='\t', skip=toskip, header=TRUE, na.strings=getGenoCode(header,'unk'), colClasses='character', comment.char = '#')
          cat('Genodata:', toskip, " ", dim(genodata), genocodes, '\n')
          if(is.null(phenotype)) phenotype <- runif((ncol(genodata)-4))                                                     # If there isn't a phenotype, generate a random one
@@ -127,7 +138,21 @@ def generate_cross_from_geno(dataset):        # TODO: Need to figure out why som
          if(type == 'riset') cross <- convert2riself(cross)                                                                # If its a RIL, convert to a RIL in R/qtl
          return(cross)
       }
-    """ % (dataset.group.name + ".geno"))
+    """ % (dataset.group.genofile))
+
+def add_perm_strata(cross, perm_strata):
+    col_string = 'c("the_strata")'
+    perm_strata_string = "c("
+    for item in perm_strata:
+        perm_strata_string += str(item) + ","
+
+    perm_strata_string = perm_strata_string[:-1] + ")"
+
+    cross = add_phenotype(cross, perm_strata_string, "the_strata")
+
+    strata_ob = pull_var("perm_strata", cross, col_string)
+
+    return cross, strata_ob
 
 def sanitize_rqtl_phenotype(vals):
     pheno_as_string = "c("
@@ -143,6 +168,7 @@ def sanitize_rqtl_phenotype(vals):
             else:
                 pheno_as_string += str(val)
     pheno_as_string += ")"
+
     return pheno_as_string
 
 def add_phenotype(cross, pheno_as_string, col_name):
@@ -150,11 +176,11 @@ def add_phenotype(cross, pheno_as_string, col_name):
     ro.r('the_cross$pheno <- cbind(pull.pheno(the_cross), ' + col_name + ' = '+ pheno_as_string +')')
     return ro.r["the_cross"]
 
-def pull_covar(cross, covar_name_string):
+def pull_var(var_name, cross, var_string):
     ro.globalenv["the_cross"] = cross
-    ro.r('trait_covars <- pull.pheno(the_cross, ' + covar_name_string + ')')
+    ro.r(var_name +' <- pull.pheno(the_cross, ' + var_string + ')')
 
-    return ro.r["trait_covars"]
+    return ro.r[var_name]
 
 def add_cofactors(cross, this_dataset, covariates, samples):
     ro.numpy2ri.activate()
@@ -190,19 +216,18 @@ def add_cofactors(cross, this_dataset, covariates, samples):
         covar_as_string += ")"
 
         col_name = "covar_" + str(i)
+        cross = add_phenotype(cross, covar_as_string, col_name)
 
         if i < (len(covariate_list) - 1):
             covar_name_string += '"' + col_name + '", '
         else:
             covar_name_string += '"' + col_name + '"'
 
-        cross = add_phenotype(cross, covar_as_string, col_name)
-
     covar_name_string += ")"
 
-    covars = pull_covar(cross, covar_name_string)
+    covars_ob = pull_var("trait_covars", cross, covar_name_string)
 
-    return cross, covars
+    return cross, covars_ob
 
 def create_marker_covariates(control_marker, cross):
     ro.globalenv["the_cross"] = cross
@@ -245,14 +270,17 @@ def process_rqtl_perm_results(num_perm, results):
 
     return perm_output, suggestive, significant
 
-def process_rqtl_results(result):        # TODO: how to make this a one liner and not copy the stuff in a loop
+def process_rqtl_results(result, species_name):        # TODO: how to make this a one liner and not copy the stuff in a loop
     qtl_results = []
     output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)]
 
     for i, line in enumerate(result.iter_row()):
         marker = {}
         marker['name'] = result.rownames[i]
-        marker['chr'] = output[i][0]
+        if species_name == "mouse" and output[i][0] == 20: #ZS: This is awkward, but I'm not sure how to change the 20s to Xs in the RData file
+            marker['chr'] = "X"
+        else:
+            marker['chr'] = output[i][0]
         marker['cM'] = output[i][1]
         marker['Mb'] = output[i][1]
         marker['lod_score'] = output[i][2]
diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py
index 3006c4ff..e191902c 100644
--- a/wqflask/wqflask/marker_regression/run_mapping.py
+++ b/wqflask/wqflask/marker_regression/run_mapping.py
@@ -38,6 +38,7 @@ from utility import Plot, Bunch
 from utility import temp_data
 from utility.benchmark import Bench
 from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping
+from wqflask.show_trait.SampleList import SampleList
 
 from utility.tools import locate, locate_ignore_error, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR
 from utility.external import shell
@@ -74,15 +75,41 @@ class RunMapping(object):
         self.vals = []
         if 'samples' in start_vars:
             self.samples = start_vars['samples'].split(",")
-            for sample in self.samples:
-                if (len(genofile_samplelist) == 0) or (sample in genofile_samplelist):
+            if (len(genofile_samplelist) != 0):
+                for sample in genofile_samplelist:
+                    if sample in self.samples:
+                        value = start_vars.get('value:' + sample)
+                        if value:
+                            self.vals.append(value)
+                    else:
+                        self.vals.append("x")
+            else:
+                for sample in self.samples:
                     value = start_vars.get('value:' + sample)
                     if value:
                         self.vals.append(value)
         else:
             self.samples = []
-            for sample in self.dataset.group.samplelist: # sample is actually the name of an individual
-                if (len(genofile_samplelist) == 0) or (sample in genofile_samplelist):
+            if (len(genofile_samplelist) != 0):
+                for sample in genofile_samplelist:
+                    if sample in self.dataset.group.samplelist:
+                        in_trait_data = False
+                        for item in self.this_trait.data:
+                            if self.this_trait.data[item].name == sample:
+                                value = start_vars['value:' + self.this_trait.data[item].name]
+                                self.samples.append(self.this_trait.data[item].name)
+                                self.vals.append(value)
+                                in_trait_data = True
+                                break
+                        if not in_trait_data:
+                            value = start_vars.get('value:' + sample)
+                            if value:
+                                self.samples.append(sample)
+                                self.vals.append(value)
+                    else:
+                        self.vals.append("x")
+            else:
+                for sample in self.dataset.group.samplelist: # sample is actually the name of an individual
                     in_trait_data = False
                     for item in self.this_trait.data:
                         if self.this_trait.data[item].name == sample:
@@ -204,8 +231,17 @@ class RunMapping(object):
         elif self.mapping_method == "rqtl_plink":
             results = self.run_rqtl_plink()
         elif self.mapping_method == "rqtl_geno":
+            perm_strata = []
+            if "perm_strata" in start_vars and "categorical_vars" in start_vars:
+                self.categorical_vars = start_vars["categorical_vars"].split(",")
+                if len(self.categorical_vars) and start_vars["perm_strata"] == "True":
+                    primary_samples = SampleList(dataset = self.dataset,
+                                                 sample_names = self.samples,
+                                                 this_trait = self.this_trait)
+
+                    perm_strata = get_perm_strata(self.this_trait, primary_samples, self.categorical_vars, self.samples)
             self.score_type = "LOD"
-            self.mapping_scale = "morgan"
+            #self.mapping_scale = "morgan"
             self.control_marker = start_vars['control_marker']
             self.do_control = start_vars['do_control']
             if 'mapmethod_rqtl_geno' in start_vars:
@@ -216,9 +252,9 @@ class RunMapping(object):
             #if start_vars['pair_scan'] == "true":
             #    self.pair_scan = True
             if self.permCheck and self.num_perm > 0:
-                self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.samples, self.covariates)
+                self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl_geno(self.vals, self.samples, self.dataset, self.method, self.model, self.permCheck, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates)
             else:
-                results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.samples, self.covariates)
+                results = rqtl_mapping.run_rqtl_geno(self.vals, self.samples, self.dataset, self.method, self.model, self.permCheck, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates)
         elif self.mapping_method == "reaper":
             if "startMb" in start_vars: #ZS: Check if first time page loaded, so it can default to ON
                 if "additiveCheck" in start_vars:
@@ -429,8 +465,8 @@ class RunMapping(object):
                       chr_lengths = chr_lengths,
                       num_perm = self.num_perm,
                       perm_results = self.perm_output,
-                      browser_files = browser_files,
                       significant = significant_for_browser,
+                      browser_files = browser_files,
                       selected_chr = this_chr
                   )
               else:
@@ -617,9 +653,14 @@ def get_chr_lengths(mapping_scale, dataset, qtl_results):
         this_chr = 1
         highest_pos = 0
         for i, result in enumerate(qtl_results):
-            if int(result['chr']) > this_chr or i == (len(qtl_results) - 1):
+            chr_as_num = 0
+            try:
+                chr_as_num = int(result['chr'])
+            except:
+                chr_as_num = 20
+            if chr_as_num > this_chr or i == (len(qtl_results) - 1):
                 chr_lengths.append({ "chr": str(this_chr), "size": str(highest_pos)})
-                this_chr = int(result['chr'])
+                this_chr = chr_as_num
                 highest_pos = 0
             else:
                 if float(result['Mb']) > highest_pos:
@@ -635,4 +676,25 @@ def get_genofile_samplelist(dataset):
         if genofile['location'] == dataset.group.genofile and 'sample_list' in genofile:
             genofile_samplelist = genofile['sample_list']
 
-    return genofile_samplelist
\ No newline at end of file
+    return genofile_samplelist
+
+def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples):
+    perm_strata_strings = []
+    for sample in used_samples:
+        if sample in sample_list.sample_attribute_values.keys():
+            combined_string = ""
+            for var in categorical_vars:
+                if var in sample_list.sample_attribute_values[sample].keys():
+                    combined_string += str(sample_list.sample_attribute_values[sample][var])
+                else:
+                    combined_string += "NA"
+        else:
+            combined_string = "NA"
+
+        perm_strata_strings.append(combined_string)
+
+    d = dict([(y,x+1) for x,y in enumerate(sorted(set(perm_strata_strings)))])
+    list_to_numbers = [d[x] for x in perm_strata_strings]
+    perm_strata = list_to_numbers
+
+    return perm_strata
\ No newline at end of file
diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py
index 451be50b..7e126a36 100644
--- a/wqflask/wqflask/show_trait/SampleList.py
+++ b/wqflask/wqflask/show_trait/SampleList.py
@@ -24,8 +24,8 @@ class SampleList(object):
                  dataset,
                  sample_names,
                  this_trait,
-                 sample_group_type,
-                 header):
+                 sample_group_type = "primary",
+                 header = "Samples"):
 
         self.dataset = dataset
         self.this_trait = this_trait
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index 10ce38a7..64deb942 100644
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -159,6 +159,8 @@ class ShowTrait(object):
             self.sample_group_types['samples_primary'] = self.dataset.group.name
         sample_lists = [group.sample_list for group in self.sample_groups]
 
+        categorical_var_list = get_categorical_variables(self.this_trait, self.sample_groups[0]) #ZS: Only using first samplelist, since I think mapping only uses those samples
+
         #ZS: Get list of chromosomes to select for mapping
         self.chr_list = [["All", -1]]
         for i, this_chr in enumerate(self.dataset.species.chromosomes.chromosomes):
@@ -226,6 +228,7 @@ class ShowTrait(object):
         hddn['mapping_display_all'] = True
         hddn['suggestive'] = 0
         hddn['num_perm'] = 0
+        hddn['categorical_vars'] = ""
         hddn['manhattan_plot'] = ""
         hddn['control_marker'] = ""
         if not self.temp_trait:
@@ -250,6 +253,7 @@ class ShowTrait(object):
                        sample_group_types = self.sample_group_types,
                        sample_lists = sample_lists,
                        attribute_names = self.sample_groups[0].attributes,
+                       categorical_vars = ",".join(categorical_var_list),
                        num_values = self.num_values,
                        qnorm_values = self.qnorm_vals,
                        zscore_values = self.z_scores,
@@ -570,10 +574,25 @@ def get_ncbi_summary(this_trait):
         #ZS: Need to switch this try/except to something that checks the output later
         try:
             response = requests.get("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=gene&id=%s&retmode=json" % this_trait.geneid)
-            logger.debug("NCBI:", json.loads(response.content)['result'][this_trait.geneid])
             summary = json.loads(response.content)['result'][this_trait.geneid]['summary']
             return summary
         except:
             return None
     else:
-        return None
\ No newline at end of file
+        return None
+
+def get_categorical_variables(this_trait, sample_list):
+    categorical_var_list = []
+
+    if len(sample_list.attributes) > 0:
+        for attribute in sample_list.attributes:
+            attribute_vals = []
+            for sample_name in this_trait.data.keys():
+                attribute_vals.append(this_trait.data[sample_name].extra_attributes[sample_list.attributes[attribute].name])
+
+            num_distinct = len(set(attribute_vals))
+
+            if num_distinct < 10:
+                categorical_var_list.append(sample_list.attributes[attribute].name)
+
+    return categorical_var_list
\ No newline at end of file
diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
index b26110d8..478ed87e 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
@@ -153,7 +153,7 @@
 
   //ZS: This is a list of inputs to be passed to the loading page, since not all inputs on the trait page are relevant to mapping
   var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale',
-                            'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'num_bootstrap', 'bootCheck', 'bootstrap_results',
+                            'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results',
                             'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'control_marker_db', 'do_control', 'genofile', 
                             'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 
                             'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples']
@@ -167,6 +167,7 @@
         $('input[name=selected_chr]').val($('#chr_rqtl_geno').val());
         $('input[name=genofile]').val($('#genofile_rqtl_geno').val());
         $('input[name=num_perm]').val($('input[name=num_perm_rqtl_geno]').val());
+        $('input[name=categorical_vars]').val(js_data.categorical_vars)
         $('input[name=manhattan_plot]').val($('input[name=manhattan_plot_rqtl]:checked').val());
         $('input[name=control_marker]').val($('input[name=control_rqtl_geno]').val());
         $('input[name=do_control]').val($('input[name=do_control_rqtl]:checked').val());
diff --git a/wqflask/wqflask/templates/collections/add.html b/wqflask/wqflask/templates/collections/add.html
index 058e269c..825dfb84 100644
--- a/wqflask/wqflask/templates/collections/add.html
+++ b/wqflask/wqflask/templates/collections/add.html
@@ -11,22 +11,9 @@
             {% else %}
             <input type="hidden" name="hash" value="{{ hash }}" />
             {% endif %}
-            <fieldset>
-              <legend>1. Create a new collection</legend>
-              <div style="margin-left: 20px;">
-                <!--<label>Collection name:</label>-->
-                <input type="text" name="new_collection" placeholder=" Name of new collection..."
-                    data-trigger="change" data-minlength="5" data-maxlength="50" style="width: 100%">
-                <button type="submit" name="create_new" class="btn btn-primary" style="margin-top: 20px;">Create collection</button>
-                {% if uc is not defined %}
-                <span class="help-block" style="color:red;">This collection will be saved to your computer for a year (or until you clear your cache).</span>
-                {% endif %}
-              </div>
-            </fieldset>
             {% if collections|length > 0 %}
-            <hr />
             <fieldset>
-              <legend>2. Add to an existing collection</legend>
+              <legend>1. Add to an existing collection</legend>
               <div style="margin-left: 20px;">  
                 <!--<label>Existing collection name:</label>-->
                 <select name="existing_collection" class="form-control">
@@ -44,6 +31,19 @@
               </div>
             </fieldset>
             {% endif %}
+            <hr />
+            <fieldset>
+              <legend>{% if collections|length > 0 %}2. {% else %}{% endif %}Create a new collection</legend>
+              <div style="margin-left: 20px;">
+                <!--<label>Collection name:</label>-->
+                <input type="text" name="new_collection" placeholder=" Name of new collection..."
+                    data-trigger="change" data-minlength="5" data-maxlength="50" style="width: 100%">
+                <button type="submit" name="create_new" class="btn btn-primary" style="margin-top: 20px;">Create collection</button>
+                {% if uc is not defined %}
+                <span class="help-block">This collection will be saved to your computer for a year (or until you clear your cache).</span>
+                {% endif %}
+              </div>
+            </fieldset>
         </form>
     </div>
 </div>
diff --git a/wqflask/wqflask/templates/collections/list.html b/wqflask/wqflask/templates/collections/list.html
index a2f1a1f7..3829b950 100644
--- a/wqflask/wqflask/templates/collections/list.html
+++ b/wqflask/wqflask/templates/collections/list.html
@@ -28,7 +28,7 @@
         <br>
         <div id="collections_list" style="width:50%; margin-top: 20px; margin-bottom: 20px;">
         {% if collections|length > 0 %}
-        <table class="table-hover table-striped cell-border" id='trait_table'>
+        <table class="table-hover table-striped cell-border" id='trait_table' style="float: left;">
             <thead>
                 <tr>
                     <th></th>
diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html
index 3b7f9671..59936a8e 100644
--- a/wqflask/wqflask/templates/collections/view.html
+++ b/wqflask/wqflask/templates/collections/view.html
@@ -88,8 +88,9 @@
             <button class="btn btn-danger" id="remove" disabled="disabled" type="button"><i class="icon-minus-sign"></i> Delete Rows</button>
             <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" title="Delete this collection" > Delete Collection</button>
             </form>
-            <br />
-            <br />
+            <div style="margin-top: 10px; margin-bottom: 5px;">
+                <b>Show/Hide Columns:</b>
+            </div>
             <div>
                 <table class="table-hover table-striped cell-border" id='trait_table' style="float: left;">
                     <thead>
@@ -127,6 +128,8 @@
                             </TD>
                             {% if this_trait.symbol %}
                             <TD data-export="{{ this_trait.symbol }}">{{ this_trait.symbol }}</TD>
+                            {% elif this_trait.abbreviation %}
+                            <TD data-export="{{ this_trait.abbreviation }}">{{ this_trait.abbreviation }}</TD>
                             {% else %}
                             <TD data-export="N/A">N/A</TD>
                             {% endif %}
@@ -139,9 +142,17 @@
                             {% endif %}
                             <TD data-export="{{ this_trait.location_repr }}">{{ this_trait.location_repr }}</TD>
                             <TD data-export="{{ this_trait.mean }}" align="right">{{ '%0.3f' % this_trait.mean|float }}</TD>
+                            {% if this_trait.LRS_score_repr|float > 0 %}
                             <TD data-export="{{ this_trait.LRS_score_repr }}" align="right">{{ '%0.3f' % this_trait.LRS_score_repr|float }}</TD>
+                            {% else %}
+                            <TD data-export="{{ this_trait.LRS_score_repr }}" align="right">N/A</TD>
+                            {% endif %}
                             <TD data-export="{{ this_trait.LRS_location_repr }}">{{ this_trait.LRS_location_repr }}</TD>
+                            {% if this_trait.additive|float > 0 %}
                             <TD data-export="{{ this_trait.additive }}" align="right">{{ '%0.3f' % this_trait.additive|float }}</TD>
+                            {% else %}
+                            <TD data-export="{{ this_trait.additive }}" align="right">N/A</TD>
+                            {% endif %}
                         </TR>
                     {% endfor %}
                     </tbody>
diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html
index 3e8baab6..03b03aa7 100644
--- a/wqflask/wqflask/templates/correlation_page.html
+++ b/wqflask/wqflask/templates/correlation_page.html
@@ -29,46 +29,42 @@
                 {{ this_trait.name }}:{{ this_trait.dataset }},
             {% endfor %}" >
 
+            <button id="corr_matrix" class="btn btn-primary submit_special" data-url="/corr_matrix" title="Correlation Matrix" >
+                Correlations
+            </button>
 
-                <button id="corr_matrix" class="btn btn-primary submit_special" data-url="/corr_matrix" title="Correlation Matrix" >
-                    Correlation Matrix
-                </button>
+            <button id="network_graph" class="btn btn-primary submit_special" data-url="/network_graph" title="Network Graph" >
+                Networks
+            </button>
 
+            <button id="send_to_webgestalt" class="btn btn-primary submit_special" data-url="/webgestalt_page" title="WebGestalt" >
+              WebGestalt
+            </button>
 
-                <button id="network_graph" class="btn btn-primary submit_special" data-url="/network_graph" title="Network Graph" >
-                    Network Graph
-                </button>
+            <button id="send_to_geneweaver" class="btn btn-primary submit_special" data-url="/geneweaver_page" title="GeneWeaver" >
+              GeneWeaver
+            </button>
 
+            <button id="send_to_bnw" class="btn btn-primary submit_special" data-url="/bnw_page" title="Bayesian Network" >
+              BNW
+           </button>
 
-                <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
-                    WGCNA Analysis
-                </button>
+            <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
+                WGCNA
+            </button>
 
+            <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
+                CTL Maps
+            </button>
 
-                <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
-                    CTL Analysis
-                </button>
+            <button id="heatmap" class="btn btn-primary submit_special" data-url="/heatmap" title="Heatmap" >
+                MultiMap
+            </button>
 
+            <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" >
+                Comparison Bar Chart
+            </button>
 
-                <button id="heatmap" class="btn btn-primary submit_special" data-url="/heatmap" title="Heatmap" >
-                    Heatmap
-                </button>
-
-                <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" >
-                    Comparison Bar Chart
-                </button>
-
-                <button id="send_to_webgestalt" class="btn btn-primary submit_special" data-url="/webgestalt_page" title="WebGestalt" >
-                    WebGestalt
-                </button>
-
-                <button id="send_to_geneweaver" class="btn btn-primary submit_special" data-url="/geneweaver_page" title="GeneWeaver" >
-                    GeneWeaver
-                </button>
-
-                <button id="send_to_bnw" class="btn btn-primary submit_special" data-url="/bnw_page" title="Bayesian Network" >
-                    BNW
-                </button>
           </form>
         </div>
         <br />
diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html
index bc614e01..49bcbff7 100644
--- a/wqflask/wqflask/templates/loading.html
+++ b/wqflask/wqflask/templates/loading.html
@@ -9,7 +9,7 @@
       <div style="min-height: 80vh; display: flex; align-items: center; text-align: center;">
         <div style="margin-bottom: 5px; left: 50%; margin-right: -50%; margin-top: 10%; transform: translate(-50%, -50%); position: absolute;">
         {% if start_vars.tool_used == "Mapping" %}
-        <h1>Computing the Map</h1>
+        <h1>Computing the Maps</h1>
         <br>
         <i>n</i> = {{ start_vars.num_vals }}
         <br>
diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html
index 162bde08..33221e0f 100644
--- a/wqflask/wqflask/templates/search_result_page.html
+++ b/wqflask/wqflask/templates/search_result_page.html
@@ -8,7 +8,7 @@
 {% endblock %}
 {% block content %}
 <!-- Start of body -->
-    <div style="padding-left: 10px; min-width: 1200px;">
+    <div style="padding-left: 10px; {% if dataset.type == 'Publish' %}min-width: 1000px;{% else %}min-width: 1200px;{% endif %}">
         <input type="hidden" name="uc_id" id="uc_id" value="{{ uc_id }}">
 
         <div style="padding-top: 10px; padding-bottom: 10px; font-size: 16px;">
@@ -68,45 +68,41 @@
 
 
                 <button id="corr_matrix" class="btn btn-primary submit_special" data-url="/corr_matrix" title="Correlation Matrix" >
-                    Correlation Matrix
+                    Correlations
                 </button>
 
-
                 <button id="network_graph" class="btn btn-primary submit_special" data-url="/network_graph" title="Network Graph" >
-                    Network Graph
+                    Networks
                 </button>
 
+                <button id="send_to_webgestalt" class="btn btn-primary submit_special" data-url="/webgestalt_page" title="WebGestalt" >
+                  WebGestalt
+                </button>
 
-                <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
-                    WGCNA Analysis
+                <button id="send_to_geneweaver" class="btn btn-primary submit_special" data-url="/geneweaver_page" title="GeneWeaver" >
+                  GeneWeaver
                 </button>
 
+                <button id="send_to_bnw" class="btn btn-primary submit_special" data-url="/bnw_page" title="Bayesian Network" >
+                  BNW
+               </button>
 
-                <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
-                    CTL Analysis
+                <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
+                    WGCNA
                 </button>
 
+                <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
+                    CTL Maps
+                </button>
 
                 <button id="heatmap" class="btn btn-primary submit_special" data-url="/heatmap" title="Heatmap" >
-                    Heatmap
+                    MultiMap
                 </button>
 
                 <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" >
                     Comparison Bar Chart
                 </button>
 
-                <button id="send_to_webgestalt" class="btn btn-primary submit_special" data-url="/webgestalt_page" title="WebGestalt" >
-                    WebGestalt
-                </button>
-
-                <button id="send_to_geneweaver" class="btn btn-primary submit_special" data-url="/geneweaver_page" title="GeneWeaver" >
-                    GeneWeaver
-                </button>
-
-                <button id="send_to_bnw" class="btn btn-primary submit_special" data-url="/bnw_page" title="Bayesian Network" >
-                    BNW
-                </button>
-
           </form>
         </div>
 
@@ -252,11 +248,13 @@
                     $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text());
                     $('td', row).eq(4).attr('title', $('td', row).eq(4).text());
                     $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text());
-                    $('td', row).slice(6,9).attr("align", "right");
+                    $('td', row).eq(4).attr('align', 'right');
+                    $('td', row).slice(6,10).attr("align", "right");
                     $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text());
                     $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text());
                     $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text());
                     $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text());
+                    $('td', row).eq(9).attr('data-export', $('td', row).eq(8).text());
                     {% elif dataset.type == 'Geno' %}
                     $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text());
                     {% endif %}
@@ -339,7 +337,7 @@
                     {
                       'title': "Description",
                       'type': "natural",
-                      'width': "25%",
+                      'width': "800px",
                       'data': null,
                       'render': function(data, type, row, meta) {
 			                  try {
@@ -359,7 +357,7 @@
                     {
                       'title': "Authors",
                       'type': "natural",
-                      'width': "25%",
+                      'width': "500px",
                       'data': null,
                       'render': function(data, type, row, meta) {
                         author_list = data.authors.split(",")
@@ -398,7 +396,7 @@
                     {
                       'title': "Max LRS Location",
                       'type': "natural",
-                      'width': "160px",
+                      'width': "200px",
                       'data': "lrs_location"
                     },
                     {
@@ -430,7 +428,6 @@
                     }
                 ],
                 'sDom': "Bitir",
-                'autoWidth': false,
                 'deferRender': true,
                 'paging': false,
                 'orderClasses': true,
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index 81661f86..27c3e398 100644
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -364,7 +364,7 @@
                   }
                 } );
 
-                primary_table.on( 'order.dt search.dt', function () {
+                primary_table.on( 'order.dt search.dt draw.dt', function () {
                   primary_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) {
                     cell.innerHTML = i+1;
                   } );
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index 01d90d21..777d4a2d 100644
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -243,6 +243,21 @@
                                 <input name="num_perm_rqtl_geno" value="200" type="text" class="form-control">
                             </div>
                         </div>
+                        {% if sample_groups[0].attributes|length > 0 %}
+                        <div class="mapping_method_fields form-group">
+                            <label style="text-align:right;" class="col-xs-3 control-label">Stratified</label>
+                            <div style="margin-left:20px;" class="col-xs-6 controls">
+                                <label class="radio-inline">
+                                    <input type="radio" name="perm_strata" value="True" checked=""> 
+                                    Yes
+                                </label>
+                                <label class="radio-inline">
+                                    <input type="radio" name="perm_strata" value="False" >
+                                    No
+                               </label>
+                            </div>
+                        </div>
+                        {% endif %}
                         <div class="mapping_method_fields form-group">
                             <label style="text-align:right;" for="control_for" class="col-xs-3 control-label">Control&nbsp;for</label>
                             <div style="margin-left:20px;" class="col-xs-6 controls">
@@ -365,7 +380,7 @@
              <dd>Maps traits with correction for kinship among samples using a linear mixed model method, and also allows users to fit multiple covariates such as sex, age, treatment, and genetic markers (<a href="https://www.ncbi.nlm.nih.gov/pubmed/24531419">PMID: 2453419</a>, and <a href="https://github.com/genetics-statistics/GEMMA"> GitHub code</a>). GEMMA incorporates the Leave One Chromosome Out (LOCO) method to ensure that the correction for kinship does not remove useful genetic variance near each marker. Markers can be filtered to include only those with minor allele frequencies (MAF) above a threshold. The default MAF is 0.05.</dd>
              {% elif mapping_method == "R/qtl" %}
              <dt style="margin-top: 20px;">R/qtl</dt>
-             <dd>The original R/qtl mapping code that supports most classic experimental crosses provided that they  do not have complex kinship or admixture (<a href="https://www.ncbi.nlm.nih.gov/pubmed/30591514">PMID: 30591514</a>). Both R/qtl, implemented here, and R/qtl2 are available as stand-alone R packages (<a href="https://kbroman.org/pages/software.html">R suite</a>).</dd>
+             <dd>The original R/qtl mapping package that supports classic experimental crosses including 4-parent F2 intercrosses (e.g., NIA ITP UM-HET3). R/qtl is ideal for populations that do not have complex kinship or admixture (<a href="https://www.ncbi.nlm.nih.gov/pubmed/12724300">PMID: 12724300</a>). Both R/qtl as implemented here, and R/qtl2 (<a href="https://www.ncbi.nlm.nih.gov/pubmed/30591514">PMID: 30591514</a>) are available as <a href="https://kbroman.org/pages/software.html">R suites</a>.</dd>
              {% elif mapping_method == "QTLReaper" %}
              <dt style="margin-top: 20px;">Haley-Knott Regression</dt>
              <dd>Fast linear mapping method (<a href="https://www.ncbi.nlm.nih.gov/pubmed/16718932">PMID 16718932</a>) works well with F2 intercrosses and backcrosses, but that is not recommended for complex or admixed populations (e.g., GWAS or heterogeneous stock studies) or for advanced intercrosses, recombinant inbred families, or diallel crosses. Interactive plots in GeneNetwork have relied on the fast HK mapping for two decades and we still use this method for mapping omics data sets and computing genome-wide permutation threshold (<a href="https://github.com/pjotrp/QTLReaper">QTL Reaper code</a>).</dd>
diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py
index 53a43c40..d75a03df 100644
--- a/wqflask/wqflask/user_session.py
+++ b/wqflask/wqflask/user_session.py
@@ -144,6 +144,7 @@ class UserSession(object):
 
         #ZS: Get user's collections if they exist
         collections = get_user_collections(self.redis_user_id)
+        collections = [item for item in collections if item['name'] != "Your Default Collection"] + [item for item in collections if item['name'] == "Your Default Collection"] #ZS: Ensure Default Collection is last in list
         return collections
 
     @property
@@ -199,7 +200,8 @@ class UserSession(object):
         this_collection = self.get_collection_by_id(collection_id)
 
         updated_collection = this_collection
-        updated_traits = this_collection['members'] + traits_to_add
+        current_members_minus_new = [member for member in this_collection['members'] if member not in traits_to_add]
+        updated_traits = traits_to_add + current_members_minus_new
 
         updated_collection['members'] = updated_traits
         updated_collection['num_members'] = len(updated_traits)
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 0fbe370d..923c89bd 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -15,6 +15,7 @@ import xlsxwriter
 import StringIO  # Todo: Use cStringIO?
 
 import gc
+import numpy as np
 
 import cPickle as pickle
 import uuid
@@ -392,9 +393,9 @@ def export_perm_data():
 
     buff = StringIO.StringIO()
     writer = csv.writer(buff)
-    writer.writerow(["Suggestive LRS (p=0.63) = " + str(perm_data[int(num_perm*0.37-1)])])
-    writer.writerow(["Significant LRS (p=0.05) = " + str(perm_data[int(num_perm*0.95-1)])])
-    writer.writerow(["Highly Significant LRS (p=0.01) = " + str(perm_data[int(num_perm*0.99-1)])])
+    writer.writerow(["Suggestive LRS (p=0.63) = " + str(np.percentile(np.array(perm_data), 67))])
+    writer.writerow(["Significant LRS (p=0.05) = " + str(np.percentile(np.array(perm_data), 95))])
+    writer.writerow(["Highly Significant LRS (p=0.01) = " + str(np.percentile(np.array(perm_data), 99))])
     writer.writerow("")
     writer.writerow([str(num_perm) + " Permutations"])
     writer.writerow("")
@@ -595,7 +596,7 @@ def loading_page():
 @app.route("/run_mapping", methods=('POST',))
 def mapping_results_page():
     initial_start_vars = request.form
-    logger.debug("Mapping called with initial_start_vars:", initial_start_vars.items())
+    #logger.debug("Mapping called with initial_start_vars:", initial_start_vars.items())
     logger.info(request.url)
     temp_uuid = initial_start_vars['temp_uuid']
     wanted = (
@@ -620,6 +621,9 @@ def mapping_results_page():
         'significant',
         'num_perm',
         'permCheck',
+        'perm_strata',
+        'strat_var',
+        'categorical_vars',
         'perm_output',
         'num_bootstrap',
         'bootCheck',
@@ -654,11 +658,11 @@ def mapping_results_page():
     for key, value in initial_start_vars.iteritems():
         if key in wanted or key.startswith(('value:')):
             start_vars[key] = value
-    logger.debug("Mapping called with start_vars:", start_vars)
+    #logger.debug("Mapping called with start_vars:", start_vars)
 
     version = "v3"
     key = "mapping_results:{}:".format(version) + json.dumps(start_vars, sort_keys=True)
-    logger.info("key is:", pf(key))
+    #logger.info("key is:", pf(key))
     with Bench("Loading cache"):
         result = None # Just for testing
         #result = Redis.get(key)