From af7d0bca229f3ebaa80a16d1ce3a2bf1a8abd5df Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 20 Apr 2016 08:37:55 +0000
Subject: [PATCH 023/100] WIP fixing all paths

---
 wqflask/base/data_set.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index e37a838f..0e5a3ac1 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -405,10 +405,11 @@ class DatasetGroup(object):
             #print("Cache not hit")
 
             from utility.tools import plink_command
-            PLINK_PATH,PLINK_COMMAND = plink_command()
+            PLINK_RUN = plink_command()
 
             geno_file_path = webqtlConfig.GENODIR+self.name+".geno"
             plink_file_path = PLINK_PATH+"/"+self.name+".fam"
+            # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES
 
             if os.path.isfile(plink_file_path):
                 self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path)
-- 
cgit v1.2.3


From 4b80bbfe261e8d0587a637b35416834e027f0999 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 20 Apr 2016 08:46:23 +0000
Subject: [PATCH 024/100] Sanitizing file handling

---
 wqflask/base/data_set.py                 | 29 ++++++++++-------------------
 wqflask/utility/tools.py                 | 20 +++++++++++++++-----
 wqflask/wqflask/show_trait/show_trait.py | 11 ++++-------
 3 files changed, 29 insertions(+), 31 deletions(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 0e5a3ac1..c6f34143 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -44,12 +44,15 @@ from dbFunction import webqtlDatabaseFunction
 from utility import webqtlUtil
 from utility.benchmark import Bench
 from utility import chunks
+from utility.tools import flat_files
 
 from maintenance import get_group_samplelists
 
 from MySQLdb import escape_string as escape
 from pprint import pformat as pf
 
+MAPPING_PATH = flat_files("mapping")
+
 # Used by create_database to instantiate objects
 # Each subclass will add to this
 DS_NAME_MAP = {}
@@ -404,15 +407,11 @@ class DatasetGroup(object):
         else:
             #print("Cache not hit")
 
-            from utility.tools import plink_command
-            PLINK_RUN = plink_command()
-
             geno_file_path = webqtlConfig.GENODIR+self.name+".geno"
-            plink_file_path = PLINK_PATH+"/"+self.name+".fam"
-            # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES
 
-            if os.path.isfile(plink_file_path):
-                self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path)
+            mapping_file_path = MAPPING_PATH+"/"+self.name+".fam"
+            if os.path.isfile(mapping_file_path):
+                self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path)
             elif os.path.isfile(geno_file_path):
                 self.samplelist = get_group_samplelists.get_samplelist("geno", geno_file_path)
             else:
@@ -441,18 +440,10 @@ class DatasetGroup(object):
 
         # reaper barfs on unicode filenames, so here we ensure it's a string
         full_filename = str(os.path.join(webqtlConfig.GENODIR, self.name + '.geno'))
-        if os.path.isfile(full_filename):
-            #print("Reading file: ", full_filename)
-            genotype_1.read(full_filename)
-            #print("File read")
-        else:
-            try:
-                full_filename = str(os.path.join(webqtlConfig.TMPDIR, self.name + '.geno'))
-                #print("Reading file")
-                genotype_1.read(full_filename)
-                #print("File read")
-            except IOError:
-                print("File doesn't exist!")
+        if not os.path.isfile(full_filename):
+            raise SystemError("File "+full_filename+" does not exist")
+        print("Reading file: ", full_filename)
+        genotype_1.read(full_filename)
 
         if genotype_1.type == "group" and self.parlist:
             genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1])       #, F1=_f1)
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index c3c9b292..44bbb1b0 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -52,17 +52,27 @@ def get_setting(command_id,guess=None):
                 raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path')
     return command
 
+def valid_bin(bin):
+    if os.path.islink(bin) or os.path.isfile(bin):
+        return bin
+    return None
+
+def valid_path(dir):
+    if os.path.isdir(dir):
+        return dir
+    return None
+
 def pylmm_command(guess=None):
-    return get_setting("PYLMM_RUN",guess)
+    return valid_bin(get_setting("PYLMM_RUN",guess))
 
 def gemma_command(guess=None):
-    return get_setting("GEMMA_RUN",guess)
+    return valid_bin(get_setting("GEMMA_RUN",guess))
 
 def plink_command(guess=None):
-    return get_setting("PLINK_RUN",guess)
+    return valid_bin(get_setting("PLINK_RUN",guess))
 
 def flat_files(subdir=None):
     base = get_setting("GENENETWORK_FILES")
     if subdir:
-        return base+"/"+subdir
-    return base
+        return valid_path(base+"/"+subdir)
+    return valid_path(base)
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index 458e48da..35f7fe5f 100755
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -16,8 +16,6 @@ from base import webqtlConfig
 from base import webqtlCaseData
 from wqflask.show_trait.SampleList import SampleList
 from utility import webqtlUtil, Plot, Bunch, helper_functions
-# from utility.tools import plink_command
-from utility.tools import flat_files
 from base.trait import GeneralTrait
 from base import data_set
 from dbFunction import webqtlDatabaseFunction
@@ -25,7 +23,8 @@ from basicStatistics import BasicStatisticsFunctions
 
 from pprint import pformat as pf
 
-MAPPING_FILES = flat_files("mapping")
+from utility.tools import flat_files
+MAPPING_PATH = flat_files("mapping")
 
 ###############################################
 #
@@ -34,8 +33,6 @@ MAPPING_FILES = flat_files("mapping")
 #
 ##############################################
 
-
-
 class ShowTrait(object):
 
     def __init__(self, kw):
@@ -162,8 +159,8 @@ class ShowTrait(object):
     def get_mapping_methods(self):
         '''Only display mapping methods when the dataset group's genotype file exists'''
         def check_plink_gemma():
-            if (os.path.isfile(MAPPYING_FILES+"/"+self.dataset.group.name+".bed") and
-                os.path.isfile(MAPPING_FILES+"/"+self.dataset.group.name+".map")):
+            if (os.path.isfile(MAPPYING_PATH+"/"+self.dataset.group.name+".bed") and
+                os.path.isfile(MAPPING_PATH+"/"+self.dataset.group.name+".map")):
                 return True
             else:
                 return False
-- 
cgit v1.2.3


From a5ff6520ad29266e1647ce54ad6b2c0f200dfe9a Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 20 Apr 2016 08:55:56 +0000
Subject: [PATCH 032/100] file locating: error checking is built-in

---
 wqflask/base/data_set.py | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index c6f34143..9ffc09ac 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -44,15 +44,13 @@ from dbFunction import webqtlDatabaseFunction
 from utility import webqtlUtil
 from utility.benchmark import Bench
 from utility import chunks
-from utility.tools import flat_files
+from utility.tools import locate
 
 from maintenance import get_group_samplelists
 
 from MySQLdb import escape_string as escape
 from pprint import pformat as pf
 
-MAPPING_PATH = flat_files("mapping")
-
 # Used by create_database to instantiate objects
 # Each subclass will add to this
 DS_NAME_MAP = {}
@@ -405,11 +403,10 @@ class DatasetGroup(object):
             #print("  type: ", type(self.samplelist))
             #print("  self.samplelist: ", self.samplelist)
         else:
-            #print("Cache not hit")
-
-            geno_file_path = webqtlConfig.GENODIR+self.name+".geno"
+            print("Cache not hit")
 
-            mapping_file_path = MAPPING_PATH+"/"+self.name+".fam"
+            geno_file_path = locate(self.name+".geno",'genotype')
+            mapping_file_path = locate(self.name+".fam",'mapping')
             if os.path.isfile(mapping_file_path):
                 self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path)
             elif os.path.isfile(geno_file_path):
@@ -439,10 +436,7 @@ class DatasetGroup(object):
         genotype_1 = reaper.Dataset()
 
         # reaper barfs on unicode filenames, so here we ensure it's a string
-        full_filename = str(os.path.join(webqtlConfig.GENODIR, self.name + '.geno'))
-        if not os.path.isfile(full_filename):
-            raise SystemError("File "+full_filename+" does not exist")
-        print("Reading file: ", full_filename)
+        full_filename = str(locate(self.name+'.geno','genotype'))
         genotype_1.read(full_filename)
 
         if genotype_1.type == "group" and self.parlist:
-- 
cgit v1.2.3


From 218fd0c7666583748098f0b7b4286b6d1cbf6838 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Tue, 23 Feb 2016 11:16:33 +0000
Subject: [PATCH 033/100] Refactored file searching

---
 wqflask/base/data_set.py | 16 ++++++++--------
 wqflask/utility/tools.py |  4 +++-
 2 files changed, 11 insertions(+), 9 deletions(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 9ffc09ac..ce13dd77 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -44,7 +44,7 @@ from dbFunction import webqtlDatabaseFunction
 from utility import webqtlUtil
 from utility.benchmark import Bench
 from utility import chunks
-from utility.tools import locate
+from utility.tools import locate, locate_without_error
 
 from maintenance import get_group_samplelists
 
@@ -405,15 +405,15 @@ class DatasetGroup(object):
         else:
             print("Cache not hit")
 
-            geno_file_path = locate(self.name+".geno",'genotype')
-            mapping_file_path = locate(self.name+".fam",'mapping')
-            if os.path.isfile(mapping_file_path):
-                self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path)
-            elif os.path.isfile(geno_file_path):
-                self.samplelist = get_group_samplelists.get_samplelist("geno", geno_file_path)
+            genotype_fn = locate_without_error(self.name+".geno",'genotype')
+            mapping_fn = locate_without_error(self.name+".fam",'mapping')
+            if mapping_fn:
+                self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn)
+            elif genotype_fn:
+                self.samplelist = get_group_samplelists.get_samplelist("geno", genotype_fn)
             else:
                 self.samplelist = None
-            #print("after get_samplelist")
+            print("Sample list: ",self.samplelist)
             Redis.set(key, json.dumps(self.samplelist))
             Redis.expire(key, 60*5)
 
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index dd40be99..d4c10c68 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -36,7 +36,7 @@ def get_setting(command_id,guess=None):
     """
     def value(command):
         if command:
-            sys.stderr.write("Found value "+command+"\n")
+            sys.stderr.write("Found path "+command+"\n")
             return command
         else:
             return None
@@ -95,6 +95,7 @@ def locate(name, subdir=None):
     if valid_path(base):
         lookfor = base + "/" + name
         if valid_file(lookfor):
+            print("Found: file "+lookfor+"\n")
             return lookfor
         else:
             raise IOError("Can not locate "+lookfor)
@@ -114,6 +115,7 @@ def locate_without_error(name, subdir=None):
     if valid_path(base):
         lookfor = base + "/" + name
         if valid_file(lookfor):
+            print("Found: file "+name+"\n")
             return lookfor
     sys.stderr.write("WARNING: file "+name+" not found\n")
     return None
-- 
cgit v1.2.3


From 5fffa1aa4260af6663c1f9b4cf7494d3ebc6995d Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 24 Feb 2016 11:01:29 +0000
Subject: [PATCH 037/100] WebQtlConfig: sanitizing naming and used vars

---
 bin/genenetwork2                                   |   2 +-
 wqflask/base/data_set.py                           | 194 ++++++++++-----------
 wqflask/base/webqtlFormData.py                     |   2 +-
 .../basicStatistics/BasicStatisticsFunctions.py    |   6 +-
 wqflask/maintenance/get_group_samplelists.py       |   3 +-
 wqflask/utility/tools.py                           |  11 +-
 wqflask/wqflask/correlation/show_corr_results.py   |   2 +-
 .../wqflask/correlation_matrix/show_corr_matrix.py |   1 -
 wqflask/wqflask/heatmap/heatmap.py                 |   2 -
 .../interval_analyst/IntervalAnalystPage.py        | 138 +++++++--------
 .../marker_regression/MarkerRegressionPage.py      |   6 +-
 .../wqflask/marker_regression/marker_regression.py |   2 +-
 .../marker_regression/marker_regression_gn1.py     | 116 ++++++------
 wqflask/wqflask/search_results.py                  |   1 -
 wqflask/wqflask/show_trait/show_trait.py           |   2 +-
 15 files changed, 246 insertions(+), 242 deletions(-)

(limited to 'wqflask/base')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 54eee02e..00ee09bf 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -1,6 +1,6 @@
 #! /bin/bash
 #
-# This will run the server with default settings
+# This will run the server with default settings.
 
 # Absolute path to this script, e.g. /home/user/bin/foo.sh
 SCRIPT=$(readlink -f "$0")
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index ce13dd77..6527657a 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -76,22 +76,22 @@ def create_dataset(dataset_name, dataset_type = None, get_samplelist = True):
 
 
 #def get_dataset_type_from_json(dataset_name):
-    
+
 class Dataset_Types(object):
-    
+
     def __init__(self):
         self.datasets = {}
         file_name = "wqflask/static/new/javascript/dataset_menu_structure.json"
         with open(file_name, 'r') as fh:
             data = json.load(fh)
-        
+
         print("*" * 70)
         for species in data['datasets']:
             for group in data['datasets'][species]:
                 for dataset_type in data['datasets'][species][group]:
                     for dataset in data['datasets'][species][group][dataset_type]:
                         #print("dataset is:", dataset)
-                        
+
                         short_dataset_name = dataset[1]
                         if dataset_type == "Phenotypes":
                             new_type = "Publish"
@@ -100,32 +100,32 @@ class Dataset_Types(object):
                         else:
                             new_type = "ProbeSet"
                         self.datasets[short_dataset_name] = new_type
-                            
+
     def __call__(self, name):
         return self.datasets[name]
-    
+
 # Do the intensive work at startup one time only
 Dataset_Getter = Dataset_Types()
 
 #
 #print("Running at startup:", get_dataset_type_from_json("HBTRC-MLPFC_0611"))
-                    
+
 
 def create_datasets_list():
     key = "all_datasets"
     result = Redis.get(key)
-    
+
     if result:
         print("Cache hit!!!")
         datasets = pickle.loads(result)
-        
+
     else:
         datasets = list()
         with Bench("Creating DataSets object"):
             type_dict = {'Publish': 'PublishFreeze',
                          'ProbeSet': 'ProbeSetFreeze',
                          'Geno': 'GenoFreeze'}
-        
+
             for dataset_type in type_dict:
                 query = "SELECT Name FROM {}".format(type_dict[dataset_type])
                 for result in g.db.execute(query).fetchall():
@@ -134,10 +134,10 @@ def create_datasets_list():
                     #print("type: {}\tname: {}".format(dataset_type, result.Name))
                     dataset = create_dataset(result.Name, dataset_type)
                     datasets.append(dataset)
-            
+
         Redis.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL))
         Redis.expire(key, 60*60)
-    
+
     return datasets
 
 
@@ -158,30 +158,30 @@ def mescape(*items):
 class Markers(object):
     """Todo: Build in cacheing so it saves us reading the same file more than once"""
     def __init__(self, name):
-        json_data_fh = open(os.path.join(webqtlConfig.NEWGENODIR + name + '.json'))
+        json_data_fh = open(locate(name + '.json','genotype/json'))
         try:
             markers = json.load(json_data_fh)
         except:
             markers = []
-    
+
         for marker in markers:
             if (marker['chr'] != "X") and (marker['chr'] != "Y"):
                 marker['chr'] = int(marker['chr'])
             marker['Mb'] = float(marker['Mb'])
-            
+
         self.markers = markers
         #print("self.markers:", self.markers)
-    
-    
+
+
     def add_pvalues(self, p_values):
         print("length of self.markers:", len(self.markers))
         print("length of p_values:", len(p_values))
-        
+
         if type(p_values) is list:
             # THIS IS only needed for the case when we are limiting the number of p-values calculated
             #if len(self.markers) > len(p_values):
             #    self.markers = self.markers[:len(p_values)]
-            
+
             for marker, p_value in itertools.izip(self.markers, p_values):
                 if not p_value:
                     continue
@@ -214,7 +214,7 @@ class Markers(object):
                     #self.markers.remove(marker)
                     #del self.markers[i]
             self.markers = filtered_markers
-            
+
 
         #for i, marker in enumerate(self.markers):
         #    if not 'p_value' in marker:
@@ -223,9 +223,9 @@ class Markers(object):
         #        #self.markers.remove(self.markers[i])
 
 class HumanMarkers(Markers):
-    
+
     def __init__(self, name, specified_markers = []):
-        marker_data_fh = open(os.path.join(webqtlConfig.PYLMM_PATH + name + '.bim'))
+        marker_data_fh = open(locate('genotype') + '/' + name + '.bim')
         self.markers = []
         for line in marker_data_fh:
             splat = line.strip().split()
@@ -244,7 +244,7 @@ class HumanMarkers(Markers):
                 marker['name'] = splat[1]
                 marker['Mb'] = float(splat[3]) / 1000000
             self.markers.append(marker)
-            
+
         #print("markers is: ", pf(self.markers))
 
 
@@ -257,26 +257,26 @@ class HumanMarkers(Markers):
         #    marker['lod_score'] = -math.log10(marker['p_value'])
         #    #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values
         #    marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61
-        
+
         #print("p_values2:", pf(p_values))
         super(HumanMarkers, self).add_pvalues(p_values)
-        
+
         #with Bench("deleting markers"):
         #    markers = []
         #    for marker in self.markers:
         #        if not marker['Mb'] <= 0 and not marker['chr'] == 0:
         #            markers.append(marker)
         #    self.markers = markers
-        
-    
+
+
 
 class DatasetGroup(object):
     """
     Each group has multiple datasets; each species has multiple groups.
-    
+
     For example, Mouse has multiple groups (BXD, BXA, etc), and each group
     has multiple datasets associated with it.
-    
+
     """
     def __init__(self, dataset):
         """This sets self.group and self.group_id"""
@@ -284,14 +284,14 @@ class DatasetGroup(object):
         self.name, self.id = g.db.execute(dataset.query_for_group).fetchone()
         if self.name == 'BXD300':
             self.name = "BXD"
-        
+
         self.f1list = None
         self.parlist = None
         self.get_f1_parent_strains()
         #print("parents/f1s: {}:{}".format(self.parlist, self.f1list))
-        
+
         self.species = webqtlDatabaseFunction.retrieve_species(self.name)
-        
+
         self.incparentsf1 = False
         self.allsamples = None
         self._datasets = None
@@ -302,7 +302,7 @@ class DatasetGroup(object):
     def get_markers(self):
         #print("self.species is:", self.species)
         if self.species == "human":
-            marker_class = HumanMarkers 
+            marker_class = HumanMarkers
         else:
             marker_class = Markers
 
@@ -356,7 +356,7 @@ class DatasetGroup(object):
                 dataset_menu.append(dict(tissue=None, datasets=[(dataset, dataset_short)]))
             else:
                 dataset_sub_menu = [item[1:] for item in dataset]
-                
+
                 tissue_already_exists = False
                 tissue_position = None
                 for i, tissue_dict in enumerate(dataset_menu):
@@ -384,7 +384,7 @@ class DatasetGroup(object):
             f1, f12, maternal, paternal = webqtlUtil.ParInfo[self.name]
         except KeyError:
             f1 = f12 = maternal = paternal = None
-            
+
         if f1 and f12:
             self.f1list = [f1, f12]
         if maternal and paternal:
@@ -455,18 +455,18 @@ class DatasetGroup(object):
 
         #self.samplelist = list(self.genotype.prgy)
         self.samplelist = list(genotype.prgy)
-        
+
         return genotype
 
 
 #class DataSets(object):
 #    """Builds a list of DataSets"""
-#    
+#
 #    def __init__(self):
 #        self.datasets = list()
-#        
+#
+
 
-        
         #query = """SELECT Name FROM ProbeSetFreeze
         #           UNION
         #           SELECT Name From PublishFreeze
@@ -501,7 +501,7 @@ class DataSet(object):
         self.check_confidentiality()
 
         self.retrieve_other_names()
-        
+
         self.group = DatasetGroup(self)   # sets self.group and self.group_id and gets genotype
         if get_samplelist == True:
              self.group.get_samplelist()
@@ -511,30 +511,30 @@ class DataSet(object):
     def get_desc(self):
         """Gets overridden later, at least for Temp...used by trait's get_given_name"""
         return None
-    
+
     #@staticmethod
     #def get_by_trait_id(trait_id):
     #    """Gets the dataset object given the trait id"""
-    #    
-    #    
     #
-    #    name = g.db.execute(""" SELECT 
-    #                        
+    #
+    #
+    #    name = g.db.execute(""" SELECT
+    #
     #                        """)
-    #    
+    #
     #    return DataSet(name)
 
     # Delete this eventually
     @property
     def riset():
         Weve_Renamed_This_As_Group
-        
-        
+
+
     #@property
     #def group(self):
     #    if not self._group:
     #        self.get_group()
-    #        
+    #
     #    return self._group
 
 
@@ -546,7 +546,7 @@ class DataSet(object):
         This is not meant to retrieve the data set info if no name at all is passed.
 
         """
-        
+
         try:
             if self.type == "ProbeSet":
                 query_args = tuple(escape(x) for x in (
@@ -582,17 +582,17 @@ class DataSet(object):
         except TypeError:
             print("Dataset {} is not yet available in GeneNetwork.".format(self.name))
             pass
-        
+
     def get_trait_data(self, sample_list=None):
         if sample_list:
             self.samplelist = sample_list
         else:
             self.samplelist = self.group.samplelist
-            
+
         if self.group.parlist != None and self.group.f1list != None:
             if (self.group.parlist + self.group.f1list) in self.samplelist:
                 self.samplelist += self.group.parlist + self.group.f1list
-        
+
         query = """
             SELECT Strain.Name, Strain.Id FROM Strain, Species
             WHERE Strain.Name IN {}
@@ -610,9 +610,9 @@ class DataSet(object):
         trait_sample_data = []
         for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks):
 
-        #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId 
+        #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId
         #tempTable = None
-        #if GeneId and db.type == "ProbeSet": 
+        #if GeneId and db.type == "ProbeSet":
         #    if method == "3":
         #        tempTable = self.getTempLiteratureTable(species=species,
         #                                                input_species_geneid=GeneId,
@@ -623,7 +623,7 @@ class DataSet(object):
         #                                        TissueProbeSetFreezeId=tissueProbeSetFreezeId,
         #                                        method=method,
         #                                        returnNumber=returnNumber)
-        
+
             if self.type == "Publish":
                 dataset_type = "Phenotype"
             else:
@@ -644,7 +644,7 @@ class DataSet(object):
                         left join {}Data as T{} on T{}.Id = {}XRef.DataId
                         and T{}.StrainId={}\n
                         """.format(*mescape(self.type, item, item, self.type, item, item))
-                        
+
             if self.type == "Publish":
                 query += """
                         WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId
@@ -661,16 +661,16 @@ class DataSet(object):
                         order by {}.Id
                         """.format(*mescape(self.type, self.type, self.type, self.type,
                                    self.name, dataset_type, self.type, self.type, dataset_type))
-                        
+
             #print("trait data query: ", query)
-            
+
             results = g.db.execute(query).fetchall()
             #print("query results:", results)
             trait_sample_data.append(results)
 
         trait_count = len(trait_sample_data[0])
         self.trait_data = collections.defaultdict(list)
-        
+
         # put all of the separate data together into a dictionary where the keys are
         # trait names and values are lists of sample values
         for trait_counter in range(trait_count):
@@ -683,9 +683,9 @@ class PhenotypeDataSet(DataSet):
     DS_NAME_MAP['Publish'] = 'PhenotypeDataSet'
 
     def setup(self):
-        
+
         #print("IS A PHENOTYPEDATASET")
-        
+
         # Fields in the database table
         self.search_fields = ['Phenotype.Post_publication_description',
                             'Phenotype.Pre_publication_description',
@@ -756,26 +756,26 @@ class PhenotypeDataSet(DataSet):
 
     def get_trait_info(self, trait_list, species = ''):
         for this_trait in trait_list:
-            
+
             if not this_trait.haveinfo:
                 this_trait.retrieve_info(get_qtl_info=True)
 
             description = this_trait.post_publication_description
-            
+
             #If the dataset is confidential and the user has access to confidential
             #phenotype traits, then display the pre-publication description instead
             #of the post-publication description
             if this_trait.confidential:
                 this_trait.description_display = ""
                 continue   # for now
-            
+
                 if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(
                         privilege=self.privilege,
                         userName=self.userName,
                         authorized_users=this_trait.authorized_users):
-                        
+
                     description = this_trait.pre_publication_description
-            
+
             if len(description) > 0:
                 this_trait.description_display = description.strip()
             else:
@@ -820,7 +820,7 @@ class PhenotypeDataSet(DataSet):
                         this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs
                         this_trait.LRS_score_value = LRS_score_value = this_trait.lrs
                         this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb))
-                        
+
     def retrieve_sample_data(self, trait):
         query = """
                     SELECT
@@ -878,7 +878,7 @@ class GenotypeDataSet(DataSet):
 
     def check_confidentiality(self):
         return geno_mrna_confidentiality(self)
-    
+
     def get_trait_list(self):
         query = """
             select Geno.Name
@@ -912,7 +912,7 @@ class GenotypeDataSet(DataSet):
 
                 this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb) )
                 this_trait.location_value = trait_location_value
-                
+
     def retrieve_sample_data(self, trait):
         query = """
                     SELECT
@@ -1004,7 +1004,7 @@ class MrnaAssayDataSet(DataSet):
 
     def check_confidentiality(self):
         return geno_mrna_confidentiality(self)
-        
+
     def get_trait_list_1(self):
         query = """
             select ProbeSet.Name
@@ -1020,7 +1020,7 @@ class MrnaAssayDataSet(DataSet):
             trait_data[trait[0]] = self.retrieve_sample_data(trait[0])
         #print("After retrieve_sample_data")
         return trait_data
-    
+
     #def get_trait_data(self):
     #    self.samplelist = self.group.samplelist + self.group.parlist + self.group.f1list
     #    query = """
@@ -1040,9 +1040,9 @@ class MrnaAssayDataSet(DataSet):
     #    trait_sample_data = []
     #    for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks):
     #
-    #    #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId 
+    #    #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId
     #    #tempTable = None
-    #    #if GeneId and db.type == "ProbeSet": 
+    #    #if GeneId and db.type == "ProbeSet":
     #    #    if method == "3":
     #    #        tempTable = self.getTempLiteratureTable(species=species,
     #    #                                                input_species_geneid=GeneId,
@@ -1053,7 +1053,7 @@ class MrnaAssayDataSet(DataSet):
     #    #                                        TissueProbeSetFreezeId=tissueProbeSetFreezeId,
     #    #                                        method=method,
     #    #                                        returnNumber=returnNumber)
-    #    
+    #
     #        temp = ['T%s.value' % item for item in sample_ids_step]
     #        query = "SELECT {}.Name,".format(escape(self.type))
     #        data_start_pos = 1
@@ -1067,7 +1067,7 @@ class MrnaAssayDataSet(DataSet):
     #                    left join {}Data as T{} on T{}.Id = {}XRef.DataId
     #                    and T{}.StrainId={}\n
     #                    """.format(*mescape(self.type, item, item, self.type, item, item))
-    #                    
+    #
     #        query += """
     #                WHERE {}XRef.{}FreezeId = {}Freeze.Id
     #                and {}Freeze.Name = '{}'
@@ -1080,7 +1080,7 @@ class MrnaAssayDataSet(DataSet):
     #
     #    trait_count = len(trait_sample_data[0])
     #    self.trait_data = collections.defaultdict(list)
-    #    
+    #
     #    # put all of the separate data together into a dictionary where the keys are
     #    # trait names and values are lists of sample values
     #    for trait_counter in range(trait_count):
@@ -1088,11 +1088,11 @@ class MrnaAssayDataSet(DataSet):
     #        for chunk_counter in range(int(number_chunks)):
     #            self.trait_data[trait_name] += (
     #                trait_sample_data[chunk_counter][trait_counter][data_start_pos:])
-    
+
 
     def get_trait_info(self, trait_list=None, species=''):
 
-        #  Note: setting trait_list to [] is probably not a great idea. 
+        #  Note: setting trait_list to [] is probably not a great idea.
         if not trait_list:
             trait_list = []
 
@@ -1155,7 +1155,7 @@ class MrnaAssayDataSet(DataSet):
             #print("query is:", pf(query))
 
             result = g.db.execute(query).fetchone()
-            
+
             mean = result[0] if result else 0
 
             if mean:
@@ -1176,7 +1176,7 @@ class MrnaAssayDataSet(DataSet):
                         Geno.SpeciesId = Species.Id
                 """.format(species, this_trait.locus)
                 result = g.db.execute(query).fetchone()
-                
+
                 if result:
                     #if result[0] and result[1]:
                     #    lrs_chr = result[0]
@@ -1184,7 +1184,7 @@ class MrnaAssayDataSet(DataSet):
                     lrs_chr, lrs_mb = result
                     #XZ: LRS_location_value is used for sorting
                     lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb)
-                    
+
                     #try:
                     #    lrs_location_value = int(lrs_chr)*1000 + float(lrs_mb)
                     #except:
@@ -1197,7 +1197,7 @@ class MrnaAssayDataSet(DataSet):
                     this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs
                     this_trait.LRS_score_value = this_trait.lrs
                     this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb))
-      
+
 
     def convert_location_to_value(self, chromosome, mb):
         try:
@@ -1208,7 +1208,7 @@ class MrnaAssayDataSet(DataSet):
             else:
                 location_value = (ord(str(chromosome).upper()[0])*1000 +
                                   float(mb))
-        
+
         return location_value
 
     def get_sequence(self):
@@ -1225,7 +1225,7 @@ class MrnaAssayDataSet(DataSet):
                 """ % (escape(self.name), escape(self.dataset.name))
         results = g.db.execute(query).fetchone()
         return results[0]
-   
+
     def retrieve_sample_data(self, trait):
         query = """
                     SELECT
@@ -1246,8 +1246,8 @@ class MrnaAssayDataSet(DataSet):
         results = g.db.execute(query).fetchall()
         #print("RETRIEVED RESULTS HERE:", results)
         return results
-    
-    
+
+
     def retrieve_genes(self, column_name):
         query = """
                     select ProbeSet.Name, ProbeSet.%s
@@ -1256,7 +1256,7 @@ class MrnaAssayDataSet(DataSet):
                     ProbeSetXRef.ProbeSetId=ProbeSet.Id;
                 """ % (column_name, escape(str(self.id)))
         results = g.db.execute(query).fetchall()
-        
+
         return dict(results)
 
     #def retrieve_gene_symbols(self):
@@ -1285,8 +1285,8 @@ class MrnaAssayDataSet(DataSet):
     #    for item in results:
     #        symbol_dict[item[0]] = item[1]
     #    return symbol_dict
-    
-    
+
+
 
 
 class TempDataSet(DataSet):
@@ -1308,8 +1308,8 @@ class TempDataSet(DataSet):
         self.id = 1
         self.fullname = 'Temporary Storage'
         self.shortname = 'Temp'
-        
-       
+
+
     @staticmethod
     def handle_pca(desc):
         if 'PCA' in desc:
@@ -1318,13 +1318,13 @@ class TempDataSet(DataSet):
         else:
             desc = desc[:desc.index('entered')].strip()
         return desc
-        
+
     def get_desc(self):
         g.db.execute('SELECT description FROM Temp WHERE Name=%s', self.name)
         desc = g.db.fetchone()[0]
         desc = self.handle_pca(desc)
-        return desc    
-        
+        return desc
+
     def get_group(self):
         self.cursor.execute("""
                     SELECT
@@ -1337,7 +1337,7 @@ class TempDataSet(DataSet):
             """, self.name)
         self.group, self.group_id = self.cursor.fetchone()
         #return self.group
-        
+
     def retrieve_sample_data(self, trait):
         query = """
                 SELECT
@@ -1351,7 +1351,7 @@ class TempDataSet(DataSet):
                 Order BY
                         Strain.Name
                 """ % escape(trait.name)
-                
+
         results = g.db.execute(query).fetchall()
 
 
diff --git a/wqflask/base/webqtlFormData.py b/wqflask/base/webqtlFormData.py
index 44fdcc3f..10251756 100755
--- a/wqflask/base/webqtlFormData.py
+++ b/wqflask/base/webqtlFormData.py
@@ -157,7 +157,7 @@ class webqtlFormData(object):
 
         self.genotype_1 = reaper.Dataset()
 
-        full_filename = os.path.join(webqtlConfig.GENODIR, self.group + '.geno')
+        full_filename = locate(self.group + '.geno','genotype')
 
         # reaper barfs on unicode filenames, so here we ensure it's a string
         full_filename = str(full_filename)
diff --git a/wqflask/basicStatistics/BasicStatisticsFunctions.py b/wqflask/basicStatistics/BasicStatisticsFunctions.py
index 74784853..e748a822 100755
--- a/wqflask/basicStatistics/BasicStatisticsFunctions.py
+++ b/wqflask/basicStatistics/BasicStatisticsFunctions.py
@@ -118,7 +118,7 @@ def plotNormalProbability(vals=None, RISet='', title=None, showstrains=0, specia
     Plot.plotXY(c, dataZ, dataX, dataLabel = dataLabel, XLabel='Expected Z score', connectdot=0, YLabel='Trait value', title=title, specialCases=specialStrains, showLabel = showLabel)
 
     filename= webqtlUtil.genRandStr("nP_")
-    c.save(webqtlConfig.IMGDIR+filename, format='gif')
+    c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif')
 
     img=HT.Image('/image/'+filename+'.gif',border=0)
 
@@ -145,7 +145,7 @@ def plotBoxPlot(vals):
 
     Plot.plotBoxPlot(canvas, XXX, offset=(xLeftOffset, xRightOffset, yTopOffset, yBottomOffset), XLabel= "Trait")
     filename= webqtlUtil.genRandStr("Box_")
-    canvas.save(webqtlConfig.IMGDIR+filename, format='gif')
+    canvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif')
     img=HT.Image('/image/'+filename+'.gif',border=0)
 
     plotLink = HT.Span("More about ", HT.Href(text="Box Plots", url="http://davidmlane.com/hyperstat/A37797.html", target="_blank", Class="fs13"))
@@ -201,7 +201,7 @@ def plotBarGraph(identification='', RISet='', vals=None, type="name"):
     Plot.plotBarText(c, tvals, tnames, variance=tvars, YLabel='Value', title=title, sLabel = sLabel, barSpace = sw)
 
     filename= webqtlUtil.genRandStr("Bar_")
-    c.save(webqtlConfig.IMGDIR+filename, format='gif')
+    c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif')
     img=HT.Image('/image/'+filename+'.gif',border=0)
 
     return img
diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py
index b8397b47..a9059fad 100755
--- a/wqflask/maintenance/get_group_samplelists.py
+++ b/wqflask/maintenance/get_group_samplelists.py
@@ -6,7 +6,6 @@ import gzip
 
 from base import webqtlConfig
 
-
 def process_genofiles(geno_dir=webqtlConfig.GENODIR):
     print("Yabba")
     #sys.exit("Dabba")
@@ -54,4 +53,4 @@ def get_samplelist_from_plink(genofilename):
         line = line.split(" ")
         samplelist.append(line[0])
 
-    return samplelist
\ No newline at end of file
+    return samplelist
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 0f2e4d88..c0f6a49a 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -79,6 +79,15 @@ def flat_files(subdir=None):
         return valid_path(base+"/"+subdir)
     return valid_path(base)
 
+def assert_dir(dir):
+    if not valid_path(dir):
+        raise Exception("ERROR: can not find directory "+dir)
+    return dir
+    
+def mk_dir(dir):
+    os.makedirs(dir)
+    return assert_dir(dir)
+
 def locate(name, subdir=None):
     """
     Locate a static flat file in the GENENETWORK_FILES environment.
@@ -98,7 +107,7 @@ def locate(name, subdir=None):
     if subdir: sys.stderr.write(subdir)
     raise IOError("Can not locate "+name+" in "+base)
 
-def locate_without_error(name, subdir=None):
+def locate_ignore_error(name, subdir=None):
     """
     Locate a static flat file in the GENENETWORK_FILES environment.
 
diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 21a2c26c..06b4860e 100755
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -943,7 +943,7 @@ class CorrelationResults(object):
                 use_tissue_corr = True
 
             DatabaseFileName = self.getFileName( target_db_name=self.target_db_name )
-            datasetFile = open(webqtlConfig.TEXTDIR+DatabaseFileName,'r')
+            datasetFile = open(webqtlConfig.CACHEDIR+DatabaseFileName,'r')
 
             #XZ, 01/08/2009: read the first line
             line = datasetFile.readline()
diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
index 6bc0ef77..f74e655d 100755
--- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
+++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py
@@ -43,7 +43,6 @@ from pprint import pformat as pf
 from htmlgen import HTMLgen2 as HT
 import reaper
 
-from base import webqtlConfig
 from utility.THCell import THCell
 from utility.TDCell import TDCell
 from base.trait import GeneralTrait
diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py
index 40f518f0..61847bc3 100644
--- a/wqflask/wqflask/heatmap/heatmap.py
+++ b/wqflask/wqflask/heatmap/heatmap.py
@@ -26,8 +26,6 @@ import reaper
 from base.trait import GeneralTrait
 from base import data_set
 from base import species
-from base import webqtlConfig
-from utility import webqtlUtil
 # from wqflask.my_pylmm.pyLMM import lmm
 # from wqflask.my_pylmm.pyLMM import input
 from utility import helper_functions
diff --git a/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py b/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py
index ec9aa29c..f45ec0c4 100755
--- a/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py
+++ b/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py
@@ -45,40 +45,40 @@ class IntervalAnalystPage(templatePage):
 	#A dictionary that lets us map the html form names "txStart_mm6" -> "Mb Start (mm8)"
 	#the first item is the short name (column headers) and the second item is the long name (dropdown list)
 	#   [short name, long name, category]
-	columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'], 
+	columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'],
 			"GeneDescription" : ["Description", "Gene Description", 'species'],
-			'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'], 
-			'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'], 
-			'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'],  
+			'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'],
+			'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'],
+			'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'],
 			"ProteinID" : ["Prot ID", "Protein ID", 'protein'],
-			"Chromosome" : ["Chr", "Chromosome", 'species'], 
-			"TxStart" : ["Start", "Mb Start", 'species'], 
-			"TxEnd" : ["End", "Mb End", 'species'], 
-			"GeneLength" : ["Length", "Kb Length", 'species'], 
-			"cdsStart" : ["CDS Start", "Mb CDS Start", 'species'], 
+			"Chromosome" : ["Chr", "Chromosome", 'species'],
+			"TxStart" : ["Start", "Mb Start", 'species'],
+			"TxEnd" : ["End", "Mb End", 'species'],
+			"GeneLength" : ["Length", "Kb Length", 'species'],
+			"cdsStart" : ["CDS Start", "Mb CDS Start", 'species'],
 			"cdsEnd" : ["CDS End", "Mb CDS End", 'species'],
-			"exonCount" : ["Num Exons", "Exon Count", 'species'], 
-			"exonStarts" : ["Exon Starts", "Exon Starts", 'species'], 
-			"exonEnds" : ["Exon Ends", "Exon Ends", 'species'], 
-			"Strand" : ["Strand", "Strand", 'species'], 
+			"exonCount" : ["Num Exons", "Exon Count", 'species'],
+			"exonStarts" : ["Exon Starts", "Exon Starts", 'species'],
+			"exonEnds" : ["Exon Ends", "Exon Ends", 'species'],
+			"Strand" : ["Strand", "Strand", 'species'],
 			"GeneID" : ["Gene ID", "Gene ID", 'species'],
-			"GenBankID" : ["GenBank", "GenBank ID", 'species'], 
+			"GenBankID" : ["GenBank", "GenBank ID", 'species'],
 			"UnigenID" : ["Unigen", "Unigen ID", 'species'],
-			"NM_ID" : ["NM ID", "NM ID", 'species'], 
+			"NM_ID" : ["NM ID", "NM ID", 'species'],
 			"kgID" : ["kg ID", "kg ID", 'species'],
-			"snpCount" : ["SNPs", "SNP Count", 'species'], 
-			"snpDensity" : ["SNP Density", "SNP Density", 'species'], 
-			"lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'], 
-			"lod" : ["LOD", "Likelihood Odds Ratio", 'misc'], 
-			"pearson" : ["Pearson", "Pearson Product Moment", 'misc'], 
-			"literature" : ["Lit Corr", "Literature Correlation", 'misc'], 
+			"snpCount" : ["SNPs", "SNP Count", 'species'],
+			"snpDensity" : ["SNP Density", "SNP Density", 'species'],
+			"lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'],
+			"lod" : ["LOD", "Likelihood Odds Ratio", 'misc'],
+			"pearson" : ["Pearson", "Pearson Product Moment", 'misc'],
+			"literature" : ["Lit Corr", "Literature Correlation", 'misc'],
 			}
 
 	###Species Freeze
 	speciesFreeze = {'mouse':'mm9', 'rat':'rn3', 'human':'hg19'}
 	for key in speciesFreeze.keys():
 		speciesFreeze[speciesFreeze[key]] = key
-	
+
 	def __init__(self, fd):
 
 		templatePage.__init__(self, fd)
@@ -86,7 +86,7 @@ class IntervalAnalystPage(templatePage):
 		fd.formdata['remote_ip'] = fd.remote_ip
 		if not self.openMysql():
 			return
-		
+
 		self.species = fd.formdata.getvalue("species", "mouse")
 		try:
 			self.startMb = float(fd.formdata.getvalue("startMb"))
@@ -96,7 +96,7 @@ class IntervalAnalystPage(templatePage):
 			self.endMb = float(fd.formdata.getvalue("endMb"))
 		except:
 			self.endMb = self.startMb + 10
-			
+
 		self.Chr = fd.formdata.getvalue("chromosome", "1")
 		self.xls = fd.formdata.getvalue("xls", "1")
 		try:
@@ -107,38 +107,38 @@ class IntervalAnalystPage(templatePage):
 			self.diffColDefault = self.diffCol = []
 			if self.species !=  'mouse':
 				self.diffColDefault = [2, 3]#default is B6 and D2 for other species
-			
+
 		controlFrm, dispFields = self.genControlForm(fd)
 		geneTable, filename = self.genGeneTable(fd, dispFields)
-		
+
 		infoTD = HT.TD(width=400, valign= "top")
-		infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"), 
-			HT.Strong("Species : "), self.species.title(), HT.BR(),  
-			HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(),  
-			HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(),  
+		infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"),
+			HT.Strong("Species : "), self.species.title(), HT.BR(),
+			HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(),
+			HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(),
 			)
 		if filename:
 			infoTD.append(HT.BR(), HT.BR(), HT.Href(text="Download", url = "/tmp/" + filename, Class="normalsize")
 					, " output in MS excel format.")
-		
+
 		mainTable = HT.TableLite(HT.TR(infoTD, HT.TD(controlFrm, Class="doubleBorder", width=400), HT.TD("&nbsp;", width="")), cellpadding=10)
 		mainTable.append(HT.TR(HT.TD(geneTable, colspan=3)))
 		self.dict['body'] = HT.TD(mainTable)
 		self.dict['title'] = "Interval Analyst"
-		
+
 	def genGeneTable(self, fd, dispFields):
 		filename = ""
 		if self.xls:
 			#import pyXLWriter as xl
 			filename = "IntAn_Chr%s_%2.6f-%2.6f" % (self.Chr, self.startMb, self.endMb)
 			filename += ".xls"
-			
+
 			# Create a new Excel workbook
 			workbook = xl.Writer(os.path.join(webqtlConfig.TMPDIR, filename))
 			worksheet = workbook.add_worksheet()
 			titleStyle = workbook.add_format(align = 'left', bold = 0, size=18, border = 1, border_color="gray")
 			headingStyle = workbook.add_format(align = 'center', bold = 1, size=13, fg_color = 0x1E, color="white", border = 1, border_color="gray")
-			
+
 			##Write title Info
 			worksheet.write([0, 0], "GeneNetwork Interval Analyst Table", titleStyle)
 			worksheet.write([1, 0], "%s%s" % (webqtlConfig.PORTADDR, os.path.join(webqtlConfig.CGIDIR, self._scriptfile)))
@@ -148,12 +148,12 @@ class IntervalAnalystPage(templatePage):
 			worksheet.write([4, 0], "Search by : %s" % fd.formdata['remote_ip'])
 			worksheet.write([5, 0], "view region : Chr %s %2.6f - %2.6f Mb" % (self.Chr, self.startMb, self.endMb))
 			nTitleRow = 7
-			
+
 		geneTable = HT.TableLite(Class="collap", cellpadding=5)
 		headerRow = HT.TR(HT.TD(" ", Class="fs13 fwb ffl b1 cw cbrb", width="1"))
 		if self.xls:
 			worksheet.write([nTitleRow, 0], "Index", headingStyle)
-			
+
 		for ncol, column in enumerate(dispFields):
 			if len(column) == 1:
 				headerRow.append(HT.TD(self.columnNames[column[0]][0], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1,align="Center"))
@@ -162,24 +162,24 @@ class IntervalAnalystPage(templatePage):
 					worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle)
 					worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle))
 			else:
-				headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]], 
+				headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]],
 					Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1, align="Center"))
 				if self.xls:
 					colTitle = self.columnNames[column[0]][0] + " (%s)" % self.speciesFreeze[column[1]]
 					worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle)
 					worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle))
-				#headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), 
-				#	"(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]), 
+				#headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(),
+				#	"(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]),
 				#	Class="colorBlue", NOWRAP=1, align="Center"))
 		geneTable.append(headerRow)
-		
+
 		geneCol = GeneUtil.loadGenes(self.cursor, self.Chr, self.diffColDefault, self.startMb, self.endMb, species=self.species)
 		for gIndex, theGO in enumerate(geneCol):
 			geneRow = HT.TR(HT.TD(gIndex+1, Class="fs12 fwn b1", align="right"))
 			if self.xls:
 				nTitleRow += 1
 				worksheet.write([nTitleRow, 0], gIndex + 1)
-				
+
 			for ncol, column in enumerate(dispFields):
 				if len(column) == 1 or column[1]== self.species:
 					keyValue = ""
@@ -196,17 +196,17 @@ class IntervalAnalystPage(templatePage):
 					curGO = theGO[subGO]
 					if theGO[subGO].has_key(fieldName):
 						keyValue = theGO[subGO][fieldName]
-				
+
 				if self.xls:
 					worksheet.write([nTitleRow, ncol+1], keyValue)
 				geneRow.append(self.formatTD(keyValue, fieldName, curSpecies, curGO))
-					
+
 			geneTable.append(geneRow)
-			
+
 		if self.xls:
 			workbook.close()
 		return geneTable, filename
-	
+
 	def formatTD(self, keyValue, fieldName, Species, theGO):
 		if keyValue is None:
 			keyValue = ""
@@ -219,7 +219,7 @@ class IntervalAnalystPage(templatePage):
 					keyValue = ""
 				return HT.TD(keyValue, Class="fs12 fwn b1", width=300)
 			elif fieldName in ("GeneSymbol"):
-				webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), 		
+				webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), 
 					HT.Image("/images/webqtl_search.gif", border=0, valign="top"), target="_blank")
 				if theGO['GeneID']:
 					geneSymbolLink = HT.Href(webqtlConfig.NCBI_LOCUSID % theGO['GeneID'], keyValue, Class="normalsize", target="_blank")
@@ -236,8 +236,8 @@ class IntervalAnalystPage(templatePage):
 				return HT.TD(keyValue, Class="fs12 fwn b1",align="right")
 			elif fieldName in ("snpCount"):
 				if keyValue:
-					snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'), 
-							theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]), 
+					snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'),
+							theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]),
 							text=theGO["snpCount"], target="_blank", Class="normalsize")
 				else:
 					snpString = keyValue
@@ -252,13 +252,13 @@ class IntervalAnalystPage(templatePage):
 				return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1)
 		else:
 			return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1,align="right")
-				
+
 	def genControlForm(self, fd):
 		##desc GeneList
 		self.cursor.execute("Desc GeneList")
 		GeneListFields = self.cursor.fetchall()
 		GeneListFields = map(lambda X: X[0], GeneListFields)
-		
+
 		#group columns by category--used for creating the dropdown list of possible columns
 		categories = {}
 		for item in self.columnNames.keys():
@@ -267,7 +267,7 @@ class IntervalAnalystPage(templatePage):
 				categories[category[-1]] = [item ]
 			else:
 				categories[category[-1]] = categories[category[-1]]+[item]
-	
+
 		##List All Species in the Gene Table
 		speciesDict = {}
 		self.cursor.execute("select Species.Name, GeneList.SpeciesId from Species, GeneList where \
@@ -292,34 +292,34 @@ class IntervalAnalystPage(templatePage):
 					pass
 				AppliedField.append(item2)
 			categories[specName] = AppliedField
-			
+
 		categoriesOrder += ['misc']
-		
+
 		############################################################
 		## Create the list of possible columns for the dropdown list
 		############################################################
 		allColumnsList = HT.Select(name="allColumns", Class="snpBrowserDropBox")
-		
+
 		for category in categoriesOrder:
 			allFields = categories[category]
 			if allFields:
 				geneOpt = HT.Optgroup(label=category.title())
 				for item in allFields:
 					if category in self.speciesFreeze.keys():
-						geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]), 
+						geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]),
 							"%s__%s" % (item, self.speciesFreeze[category])))
 					else:
 						geneOpt.append((self.columnNames[item][1], item))
 				geneOpt.sort()
 				allColumnsList.append(geneOpt)
-		
+
 		######################################
 		## Create the list of selected columns
 		######################################
-		
+
 		#cols contains the value of all the selected columns
 		submitCols = cols = fd.formdata.getvalue("columns", "default")
-		
+
 		if cols == "default":
 			if self.species=="mouse":  #these are the same columns that are shown on intervalPage.py
 				cols = ['GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'Strand', 'GeneLength', 'GeneID', 'NM_ID', 'snpCount', 'snpDensity']
@@ -331,12 +331,12 @@ class IntervalAnalystPage(templatePage):
 		else:
 			if type(cols)==type(""):
 				cols = [cols]
-			
+
 		colsLst = []
 		dispFields = []
 		for column in cols:
 			if submitCols == "default" and column not in ('GeneSymbol') and (column in GeneListFields or column in speciesField):
-				colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]), 
+				colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]),
 							"%s__%s" % (column, self.speciesFreeze[self.species])))
 				dispFields.append([column, self.species])
 			else:
@@ -346,17 +346,17 @@ class IntervalAnalystPage(templatePage):
 					dispFields.append([column])
 				else:
 					thisSpecies = self.speciesFreeze[column2[1]]
-					colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]), 
+					colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]),
 							column))
 					dispFields.append((column2[0], thisSpecies))
 		selectedColumnsList = HT.Select(name="columns", Class="snpBrowserSelectBox", multiple="true", data=colsLst, size=6)
-		
+
 		##########################
         ## Create the columns form
-		##########################		
+		##########################
 		columnsForm = HT.Form(name="columnsForm", submit=HT.Input(type='hidden'), cgi=os.path.join(webqtlConfig.CGIDIR, self._scriptfile), enctype="multipart/form-data")
 		columnsForm.append(HT.Input(type="hidden", name="fromdatabase", value= fd.formdata.getvalue("fromdatabase", "unknown")))
-		columnsForm.append(HT.Input(type="hidden", name="species", value=self.species))	
+		columnsForm.append(HT.Input(type="hidden", name="species", value=self.species))
 		if self.diffCol:
 			columnsForm.append(HT.Input(type="hidden", name="s1", value=self.diffCol[0]))
 			columnsForm.append(HT.Input(type="hidden", name="s2", value=self.diffCol[1]))
@@ -366,8 +366,8 @@ class IntervalAnalystPage(templatePage):
 		removeButton = HT.Input(type="button", name="remove", value="Remove", Class="button", onClick="removeFromList(this.form.columns.selectedIndex, this.form.columns)")
 		upButton = HT.Input(type="button", name="up", value="Up", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex-1, this.form.columns)")
 		downButton = HT.Input(type="button", name="down", value="Down", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex+1, this.form.columns)")
-		clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)")		
-		submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)")		
+		clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)")
+		submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)")
 
 		selectChrBox = HT.Select(name="chromosome")
 		self.cursor.execute("""
@@ -375,11 +375,11 @@ class IntervalAnalystPage(templatePage):
 				Chr_Length.Name, Length from Chr_Length, Species
 			where
 				Chr_Length.SpeciesId = Species.Id AND
-				Species.Name = '%s' 
+				Species.Name = '%s'
 			Order by
 				Chr_Length.OrderId
 			""" % self.species)
-		
+
 		results = self.cursor.fetchall()
 		for chrInfo in results:
 			selectChrBox.append((chrInfo[0], chrInfo[0]))
@@ -401,5 +401,5 @@ class IntervalAnalystPage(templatePage):
 		#columnsForm.append(HT.Input(type="hidden", name="sort", value=diffCol),
 		#		   HT.Input(type="hidden", name="identification", value=identification),
 		#		   HT.Input(type="hidden", name="traitInfo", value=traitInfo))
-		
+
 		return columnsForm, dispFields
diff --git a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py
index d02d80b3..455fcf95 100755
--- a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py
+++ b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py
@@ -140,7 +140,7 @@ class MarkerRegressionPage(templatePage):
                 intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight))
                 gifmap = self.plotIntMappingForPLINK(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, plinkResultDict=plinkResultDict)
 
-                intCanvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png')
+                intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png')
                 intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap')
 
                 TD_LR = HT.TR(HT.TD(HT.Blockquote(gifmap,intImg, HT.P()), bgColor='#eeeeee', height = 200))
@@ -249,7 +249,7 @@ class MarkerRegressionPage(templatePage):
                 intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight))
                 gifmap = self.plotIntMapping(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= "")
                 filename= webqtlUtil.genRandStr("Itvl_")
-                intCanvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png')
+                intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png')
                 intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap')
 
                 ################################################################
@@ -458,7 +458,7 @@ class MarkerRegressionPage(templatePage):
         #plotBar(myCanvas,10,10,390,290,LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test',identification=fd.identification)
         Plot.plotBar(myCanvas, LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test')
         filename= webqtlUtil.genRandStr("Reg_")
-        myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif')
+        myCanvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif')
         img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test')
             
         if fd.suggestive == None:
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index 36334317..265f9473 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -30,7 +30,7 @@ from flask import Flask, g
 from base.trait import GeneralTrait
 from base import data_set
 from base import species
-from base import webqtlConfig
+# from base import webqtlConfig
 from utility import webqtlUtil
 from utility import helper_functions
 from utility import Plot, Bunch
diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
index 01303b0f..decde579 100644
--- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py
+++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
@@ -178,8 +178,8 @@ class MarkerRegression(object):
         self.species = start_vars['species']
 
         #Needing for form submission when doing single chr mapping or remapping after changing options
-        self.vals = start_vars['vals'] 
-        self.mapping_method = start_vars['mapping_method'] 
+        self.vals = start_vars['vals']
+        self.mapping_method = start_vars['mapping_method']
         if self.mapping_method == "rqtl_geno":
             self.mapmethod_rqtl_geno = start_vars['method']
             self.mapmodel_rqtl_geno = start_vars['model']
@@ -232,7 +232,7 @@ class MarkerRegression(object):
                 self.significant = start_vars['significant']
         else:
             self.nperm = 0
-           
+
         if 'bootCheck' in start_vars.keys():
             self.bootChecked = start_vars['bootCheck']
         else:
@@ -308,7 +308,7 @@ class MarkerRegression(object):
         if 'showSNP' in start_vars.keys():
             self.SNPChecked = start_vars['showSNP']
         else:
-            self.SNPChecked = False  
+            self.SNPChecked = False
         if 'showGenes' in start_vars.keys():
             self.geneChecked = start_vars['showGenes']
         else:
@@ -321,7 +321,7 @@ class MarkerRegression(object):
             self.endMb = float(start_vars['endMb'])
         except:
             self.endMb = -1
-        try: 
+        try:
             self.lrsMax = float(start_vars['lrsMax'])
         except:
             self.lrsMax = 0
@@ -363,7 +363,7 @@ class MarkerRegression(object):
             self.ChrList.append((indChr.name, i))
 
 
-        
+
         self.ChrLengthMbList = g.db.execute("""
                 Select
                         Length from Chr_Length, InbredSet
@@ -517,7 +517,7 @@ class MarkerRegression(object):
            chrName = self.selectedChr
            # Draw the genes for this chromosome / region of this chromosome
            webqtldatabase = self.dataset.name
-        
+
            if self.dataset.group.species == "mouse":
                self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, webqtldatabase, "mouse")
            elif self.dataset.group.species == "rat":
@@ -533,11 +533,11 @@ class MarkerRegression(object):
         #
         #        #GENEID = fd.formdata.getvalue('GeneId') or None
         #        GENEID = None
-        # 
+        #
         #        geneTableContainer = HT.Div(Id="sortable") #Div to hold table
         #        geneTable = self.geneTable(self.geneCol,GENEID)
         #        geneTableContainer.append(geneTable)
-        #        
+        #
         #        mainfmName = webqtlUtil.genRandStr("fm_")
         #        tableForm = HT.Form(cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name=mainfmName, submit=HT.Input(type='hidden'))
         #        tableForm.append(HT.Input(name='FormID', value='', type='hidden'))
@@ -552,22 +552,22 @@ class MarkerRegression(object):
         #else:
         showLocusForm = ""
         intCanvas = pid.PILCanvas(size=(self.graphWidth, self.graphHeight))
-        gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm)    
+        gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm)
 
         self.gifmap = gifmap.__str__()
         #print("GIFMAP:", gifmap.__str__())
 
         self.filename= webqtlUtil.genRandStr("Itvl_")
-        intCanvas.save(os.path.join(webqtlConfig.IMGDIR, self.filename), format='jpeg')
+        intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename), format='jpeg')
         intImg=HT.Image('/image/'+self.filename+'.png', border=0, usemap='#WebQTLImageMap')
 
         #Scales plot differently for high resolution
         if self.draw2X:
             intCanvasX2 = pid.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2))
             gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2)
-            intCanvasX2.save(os.path.join(webqtlConfig.IMGDIR, self.filename+"X2"), format='png')
+            intCanvasX2.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename+"X2"), format='png')
             #DLintImgX2=HT.Href(text='Download',url = '/image/'+self.filename+'X2.png', Class='smallsize', target='_blank')
- 
+
         #textUrl = self.writeQTL2Text(fd, self.filename)
 
         ################################################################
@@ -597,7 +597,7 @@ class MarkerRegression(object):
             showLocusForm.append(intImg)
         else:
             showLocusForm = intImg
-        
+
         if self.permChecked and self.nperm > 0 and not self.multipleInterval and 0 < self.nperm:
             self.perm_filename = self.drawPermutationHistogram()
             #perm_text_file = self.permutationTextFile()
@@ -667,7 +667,7 @@ class MarkerRegression(object):
             TD_LR.append(HT.Blockquote(tableForm))
 
         self.body = TD_LR
-       
+
         #self.dict['body'] = TD_LR
         #self.dict['title'] = "Mapping"
 
@@ -858,7 +858,7 @@ class MarkerRegression(object):
         BootCoord = []
         i = 0
         startX = xLeftOffset
-        
+
         if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes
             for j, _chr in enumerate(self.genotype):
                 BootCoord.append( [])
@@ -881,8 +881,8 @@ class MarkerRegression(object):
                         else:
                             Xc = startX + (_locus.cM-_chr[0].cM)*plotXScale
                         BootCoord[-1].append([Xc, self.bootResult[i]])
-                    i += 1      
-                
+                    i += 1
+
         #reduce bootResult
         if self.selectedChr > -1:
             maxBootBar = 80.0
@@ -1411,7 +1411,7 @@ class MarkerRegression(object):
             if _strains[ii] in self.dataset.group.samplelist:
                 temp = GeneralObject(name=_strains[ii], value=_val)
                 smd.append(temp)
-            
+
 
         smd.sort(lambda A, B: cmp(A.value, B.value))
         smd.reverse()
@@ -1566,14 +1566,14 @@ class MarkerRegression(object):
                     firstGene = 0
                 else:
                     lastGene = 0
-                     
+
         for j, _geno in enumerate (self.genotype[0][1].genotype):
 
             plotbxd=0
             for item in smd:
                 if item.name == samplelist[j]:
-                    plotbxd=1    
-                    
+                    plotbxd=1
+
             if (plotbxd == 1):
 
                 ind = 0
@@ -1620,28 +1620,28 @@ class MarkerRegression(object):
 
         currentChromosome = self.genotype[0].name
         i = 0
-        
+
         paddingTop = yTopOffset
         ucscPaddingTop = paddingTop + self.WEBQTL_BAND_HEIGHT + self.BAND_SPACING
         ensemblPaddingTop = ucscPaddingTop + self.UCSC_BAND_HEIGHT + self.BAND_SPACING
-        
+
         if zoom == 1:
             for pixel in range(xLeftOffset, xLeftOffset + plotWidth, pixelStep):
-    
+
                 calBase = self.kONE_MILLION*(startMb + (endMb-startMb)*(pixel-xLeftOffset-0.0)/plotWidth)
-    
+
                 xBrowse1 = pixel
                 xBrowse2 = min(xLeftOffset + plotWidth, (pixel + pixelStep - 1))
-    
+
                 WEBQTL_COORDS = "%d, %d, %d, %d" % (xBrowse1, paddingTop, xBrowse2, (paddingTop+self.WEBQTL_BAND_HEIGHT))
                 bandWidth = xBrowse2 - xBrowse1
                 WEBQTL_HREF = "javascript:rangeView('%s', %f, %f)" % (self.selectedChr - 1, max(0, (calBase-webqtlZoomWidth))/1000000.0, (calBase+webqtlZoomWidth)/1000000.0)
-    
+
                 WEBQTL_TITLE = "Click to view this section of the genome in WebQTL"
                 gifmap.areas.append(HT.Area(shape='rect',coords=WEBQTL_COORDS,href=WEBQTL_HREF, title=WEBQTL_TITLE))
                 canvas.drawRect(xBrowse1, paddingTop, xBrowse2, (paddingTop + self.WEBQTL_BAND_HEIGHT), edgeColor=self.CLICKABLE_WEBQTL_REGION_COLOR, fillColor=self.CLICKABLE_WEBQTL_REGION_COLOR)
                 canvas.drawLine(xBrowse1, paddingTop, xBrowse1, (paddingTop + self.WEBQTL_BAND_HEIGHT), color=self.CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR)
-    
+
                 UCSC_COORDS = "%d, %d, %d, %d" %(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT))
                 if self.species == "mouse":
                     UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=%s/snp/chr%s" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases, webqtlConfig.PORTADDR, self.selectedChr)
@@ -1651,7 +1651,7 @@ class MarkerRegression(object):
                 gifmap.areas.append(HT.Area(shape='rect',coords=UCSC_COORDS,href=UCSC_HREF, title=UCSC_TITLE))
                 canvas.drawRect(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), edgeColor=self.CLICKABLE_UCSC_REGION_COLOR, fillColor=self.CLICKABLE_UCSC_REGION_COLOR)
                 canvas.drawLine(xBrowse1, ucscPaddingTop, xBrowse1, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), color=self.CLICKABLE_UCSC_REGION_OUTLINE_COLOR)
-    
+
                 ENSEMBL_COORDS = "%d, %d, %d, %d" %(xBrowse1, ensemblPaddingTop, xBrowse2, (ensemblPaddingTop+self.ENSEMBL_BAND_HEIGHT))
                 if self.species == "mouse":
                     ENSEMBL_HREF = "http://www.ensembl.org/Mus_musculus/contigview?highlight=&chr=%s&vc_start=%d&vc_end=%d&x=35&y=12" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases)
@@ -1766,7 +1766,7 @@ class MarkerRegression(object):
             ChrAInfo = []
             preLpos = -1
             distinctCount = 0.0
-            
+
             #if len(self.genotype) > 1:
             if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes
                 for i, _chr in enumerate(self.genotype):
@@ -1809,7 +1809,7 @@ class MarkerRegression(object):
             offsetA = -stepA
             lineColor = pid.lightblue
             startPosX = xLeftOffset
-            
+
             for j, ChrInfo in enumerate(ChrAInfo):
                 preLpos = -1
                 for i, item in enumerate(ChrInfo):
@@ -1877,7 +1877,7 @@ class MarkerRegression(object):
             # lodm = self.LODFACTOR
         # else:
             # lodm = 1.0
- 
+
         #ZS: This is a mess, but I don't know a better way to account for different mapping methods returning results in different formats + the option to change between LRS and LOD
         if self.lrsMax <= 0:  #sliding scale
             if "lrs_value" in self.qtlresults[0]:
@@ -1890,7 +1890,7 @@ class MarkerRegression(object):
                 else:
                     if self.permChecked and self.nperm > 0 and not self.multipleInterval:
                         self.significant = min(self.significant, webqtlConfig.MAXLRS)
-                        self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS)                    
+                        self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS)
                     else:
                         pass
             else:
@@ -1903,10 +1903,10 @@ class MarkerRegression(object):
                 else:
                     if self.permChecked and self.nperm > 0 and not self.multipleInterval:
                         self.significant = min(self.significant, webqtlConfig.MAXLRS)
-                        self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS)  
+                        self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS)
                     else:
                         pass
-                        
+
             if self.permChecked and self.nperm > 0 and not self.multipleInterval:
                 LRS_LOD_Max = max(self.significant, LRS_LOD_Max)
 
@@ -1923,7 +1923,7 @@ class MarkerRegression(object):
             LRSScale = 2.5
         else:
             LRSScale = 1.0
-           
+
         LRSAxisList = Plot.frange(LRSScale, LRS_LOD_Max, LRSScale)
         #make sure the user's value appears on the y-axis
         #update by NL 6-21-2011: round the LOD value to 100 when LRS_LOD_Max is equal to 460
@@ -1953,7 +1953,7 @@ class MarkerRegression(object):
 
             #"Significant" and "Suggestive" Drawing Routine
             # ======= Draw the thick lines for "Significant" and "Suggestive" =====  (crowell: I tried to make the SNPs draw over these lines, but piddle wouldn't have it...)
-            
+
             #ZS: I don't know if what I did here with this inner function is clever or overly complicated, but it's the only way I could think of to avoid duplicating the code inside this function
             def add_suggestive_significant_lines_and_legend(start_pos_x, chr_length_dist):
                 rightEdge = int(start_pos_x + chr_length_dist*plotXScale - self.SUGGESTIVE_WIDTH/1.5)
@@ -1976,13 +1976,13 @@ class MarkerRegression(object):
 
                 start_pos_x +=  (chr_length_dist+self.GraphInterval)*plotXScale
                 return start_pos_x
-            
+
             for i, _chr in enumerate(self.genotype):
                 if self.selectedChr != -1:
                     if _chr.name == self.ChrList[self.selectedChr][0]:
                         startPosX = add_suggestive_significant_lines_and_legend(startPosX, self.ChrLengthDistList[0])
                         break
-                    else: 
+                    else:
                         continue
                 else:
                     startPosX = add_suggestive_significant_lines_and_legend(startPosX, self.ChrLengthDistList[i])
@@ -1997,7 +1997,7 @@ class MarkerRegression(object):
             #else:
             #    dominanceMax = -1
             lrsEdgeWidth = 2
-            
+
         if zoom == 2:
             lrsEdgeWidth = 2 * lrsEdgeWidth
 
@@ -2018,7 +2018,7 @@ class MarkerRegression(object):
             if qtlresult['chr'] != previous_chr and self.selectedChr == -1:
                 if self.manhattan_plot != True:
                     canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
-                    
+
                 if not self.multipleInterval and self.additiveChecked:
                     plusColor = self.ADDITIVE_COLOR_POSITIVE
                     minusColor = self.ADDITIVE_COLOR_NEGATIVE
@@ -2048,7 +2048,7 @@ class MarkerRegression(object):
                                     canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
                                 else:
                                     canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
-                    
+
                 LRSCoordXY = []
                 AdditiveCoordXY = []
                 previous_chr = qtlresult['chr']
@@ -2061,7 +2061,7 @@ class MarkerRegression(object):
 
             #startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale
 
-            #for j, _chr in enumerate(self.genotype):   
+            #for j, _chr in enumerate(self.genotype):
             #ZS: This is beause the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used
             if self.plotScale == "physic":
                 this_chr = str(self.ChrList[self.selectedChr][0])
@@ -2126,7 +2126,7 @@ class MarkerRegression(object):
 
         if self.manhattan_plot != True:
             canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
-            
+
         if not self.multipleInterval and self.additiveChecked:
             plusColor = self.ADDITIVE_COLOR_POSITIVE
             minusColor = self.ADDITIVE_COLOR_NEGATIVE
@@ -2156,7 +2156,7 @@ class MarkerRegression(object):
                             canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
                         else:
                             canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
-                    
+
         if not self.multipleInterval and INTERCROSS and self.dominanceChecked:
             plusColor = self.DOMINANCE_COLOR_POSITIVE
             minusColor = self.DOMINANCE_COLOR_NEGATIVE
@@ -2186,7 +2186,7 @@ class MarkerRegression(object):
                             canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
                         else:
                             canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth))
-                
+
 
         ###draw additive scale
         if not self.multipleInterval and self.additiveChecked:
@@ -2222,7 +2222,7 @@ class MarkerRegression(object):
         if zoom == 2:
             fontZoom = 1.5
             yTopOffset += 30
-        
+
         #calculate plot scale
         if self.plotScale != 'physic':
             self.ChrLengthDistList = self.ChrLengthCMList
@@ -2589,13 +2589,13 @@ class MarkerRegression(object):
             perm_output = [value/4.16 for value in self.perm_output]
         else:
             perm_output = self.perm_output
-            
+
         Plot.plotBar(myCanvas, perm_output, XLabel=self.LRS_LOD, YLabel='Frequency', title=' Histogram of Permutation Test')
         filename= webqtlUtil.genRandStr("Reg_")
         myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif')
-        
+
         return filename
-        
+
         # img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test')
 
         # self.suggestive = self.perm_output[int(self.nperm*0.37-1)]
@@ -2609,9 +2609,9 @@ class MarkerRegression(object):
         # permutation.append(HT.TR(HT.TD(img)),
                            # HT.TR(HT.TD('')),
                            # HT.TR(HT.TD('Total of %d permutations'%self.nperm)))
-        
+
         # return permutation
-    
+
     def permutationTextFile(self):
         filename= webqtlUtil.genRandStr("Reg_")
         fpText = open('%s.txt' % (webqtlConfig.TMPDIR+filename), 'wb')
@@ -2624,12 +2624,12 @@ class MarkerRegression(object):
                               '&nbsp;&nbsp;&nbsp;&nbsp;Significant LRS =%3.2f\n'%self.significant,
                               HT.BR(),
                               '&nbsp;&nbsp;&nbsp;&nbsp;Highly Significant LRS =%3.2f\n' % self.highlysignificant)
-        
+
         for lrs_value in self.perm_output:
             fpText.write(str(lrs_value) + "\n")
-        
+
         textUrl = HT.Href(text = 'Download Permutation Results', url= '/tmp/'+filename+'.txt', target = "_blank", Class='fs12 fwn')
-        
+
         return textUrl
 
     def geneTable(self, geneCol, refGene=None):
@@ -2654,7 +2654,7 @@ class MarkerRegression(object):
         else:
             gene_table = ""
 
-        return gene_table    
+        return gene_table
 
     def getLiteratureCorrelation(cursor,geneId1=None,geneId2=None):
         if not geneId1 or not geneId2:
@@ -2954,4 +2954,4 @@ And by voluntary, according to HFG when I talked to him, they have a choice betw
 
         sortby = ("", "")
 
-        return sortby
\ No newline at end of file
+        return sortby
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index 52fe2e34..9941a4d3 100755
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -26,7 +26,6 @@ from MySQLdb import escape_string as escape
 # Instead of importing HT we're going to build a class below until we can eliminate it
 # from htmlgen import HTMLgen2 as HT
 
-from base import webqtlConfig
 from utility.benchmark import Bench
 from base.data_set import create_dataset
 from base.trait import GeneralTrait
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index 35f7fe5f..e80cf191 100755
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -166,7 +166,7 @@ class ShowTrait(object):
                 return False
 
         def check_pylmm_rqtl():
-            if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.NEWGENODIR+self.dataset.group.name+".json") > 0):
+            if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.JSON_GENODIR+self.dataset.group.name+".json") > 0):
                 return True
             else:
                 return False
-- 
cgit v1.2.3


From 31a1c126185adb747028d035717ad72728a52b0e Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 24 Feb 2016 11:25:10 +0000
Subject: -a

[PATCH 038/100] Fix compile errors
---
 wqflask/base/data_set.py | 6 +++---
 wqflask/utility/tools.py | 6 ++++--
 2 files changed, 7 insertions(+), 5 deletions(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 6527657a..8b2a9f01 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -44,7 +44,7 @@ from dbFunction import webqtlDatabaseFunction
 from utility import webqtlUtil
 from utility.benchmark import Bench
 from utility import chunks
-from utility.tools import locate, locate_without_error
+from utility.tools import locate, locate_ignore_error
 
 from maintenance import get_group_samplelists
 
@@ -405,8 +405,8 @@ class DatasetGroup(object):
         else:
             print("Cache not hit")
 
-            genotype_fn = locate_without_error(self.name+".geno",'genotype')
-            mapping_fn = locate_without_error(self.name+".fam",'mapping')
+            genotype_fn = locate_ignore_error(self.name+".geno",'genotype')
+            mapping_fn = locate_ignore_error(self.name+".fam",'mapping')
             if mapping_fn:
                 self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn)
             elif genotype_fn:
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index c0f6a49a..67c5128a 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -1,3 +1,4 @@
+
 # Tools/paths finder resolves external paths from settings and/or environment
 # variables
 
@@ -83,9 +84,10 @@ def assert_dir(dir):
     if not valid_path(dir):
         raise Exception("ERROR: can not find directory "+dir)
     return dir
-    
+
 def mk_dir(dir):
-    os.makedirs(dir)
+    if not valid_path(dir):
+        os.makedirs(dir)
     return assert_dir(dir)
 
 def locate(name, subdir=None):
-- 
cgit v1.2.3


From c5dbe982272ef95c6598c36172444b1e8263c01a Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 24 Feb 2016 11:39:05 +0000
Subject: -a

[PATCH 039/100] Removing commented code
---
 wqflask/base/data_set.py | 228 -----------------------------------------------
 1 file changed, 228 deletions(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 8b2a9f01..52ac95f0 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -58,25 +58,13 @@ DS_NAME_MAP = {}
 def create_dataset(dataset_name, dataset_type = None, get_samplelist = True):
     if not dataset_type:
         dataset_type = Dataset_Getter(dataset_name)
-        #dataset_type = get_dataset_type_from_json(dataset_name)
 
         print("dataset_type is:", dataset_type)
-        #query = """
-        #    SELECT DBType.Name
-        #    FROM DBList, DBType
-        #    WHERE DBList.Name = '{}' and
-        #          DBType.Id = DBList.DBTypeId
-        #    """.format(escape(dataset_name))
-        #dataset_type = g.db.execute(query).fetchone().Name
-
 
     dataset_ob = DS_NAME_MAP[dataset_type]
     dataset_class = globals()[dataset_ob]
     return dataset_class(dataset_name, get_samplelist)
 
-
-#def get_dataset_type_from_json(dataset_name):
-
 class Dataset_Types(object):
 
     def __init__(self):
@@ -90,8 +78,6 @@ class Dataset_Types(object):
             for group in data['datasets'][species]:
                 for dataset_type in data['datasets'][species][group]:
                     for dataset in data['datasets'][species][group][dataset_type]:
-                        #print("dataset is:", dataset)
-
                         short_dataset_name = dataset[1]
                         if dataset_type == "Phenotypes":
                             new_type = "Publish"
@@ -107,10 +93,6 @@ class Dataset_Types(object):
 # Do the intensive work at startup one time only
 Dataset_Getter = Dataset_Types()
 
-#
-#print("Running at startup:", get_dataset_type_from_json("HBTRC-MLPFC_0611"))
-
-
 def create_datasets_list():
     key = "all_datasets"
     result = Redis.get(key)
@@ -215,13 +197,6 @@ class Markers(object):
                     #del self.markers[i]
             self.markers = filtered_markers
 
-
-        #for i, marker in enumerate(self.markers):
-        #    if not 'p_value' in marker:
-        #        #print("self.markers[i]", self.markers[i])
-        #        del self.markers[i]
-        #        #self.markers.remove(self.markers[i])
-
 class HumanMarkers(Markers):
 
     def __init__(self, name, specified_markers = []):
@@ -249,26 +224,8 @@ class HumanMarkers(Markers):
 
 
     def add_pvalues(self, p_values):
-        #for marker, p_value in itertools.izip(self.markers, p_values):
-        #    if marker['Mb'] <= 0 and marker['chr'] == 0:
-        #        continue
-        #    marker['p_value'] = p_value
-        #    print("p_value is:", marker['p_value'])
-        #    marker['lod_score'] = -math.log10(marker['p_value'])
-        #    #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values
-        #    marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61
-
-        #print("p_values2:", pf(p_values))
         super(HumanMarkers, self).add_pvalues(p_values)
 
-        #with Bench("deleting markers"):
-        #    markers = []
-        #    for marker in self.markers:
-        #        if not marker['Mb'] <= 0 and not marker['chr'] == 0:
-        #            markers.append(marker)
-        #    self.markers = markers
-
-
 
 class DatasetGroup(object):
     """
@@ -311,12 +268,6 @@ class DatasetGroup(object):
     def datasets(self):
         key = "group_dataset_menu:v2:" + self.name
         print("key is2:", key)
-        #with Bench("Loading cache"):
-        #    result = Redis.get(key)
-        #if result:
-        #    self._datasets = pickle.loads(result)
-        #    return self._datasets
-
         dataset_menu = []
         print("[tape4] webqtlConfig.PUBLICTHRESH:", webqtlConfig.PUBLICTHRESH)
         print("[tape4] type webqtlConfig.PUBLICTHRESH:", type(webqtlConfig.PUBLICTHRESH))
@@ -425,11 +376,6 @@ class DatasetGroup(object):
 
     def read_genotype_file(self):
         '''Read genotype from .geno file instead of database'''
-        #if self.group == 'BXD300':
-        #    self.group = 'BXD'
-        #
-        #assert self.group, "self.group needs to be set"
-
         #genotype_1 is Dataset Object without parents and f1
         #genotype_2 is Dataset Object with parents and f1 (not for intercross)
 
@@ -446,40 +392,16 @@ class DatasetGroup(object):
 
         #determine default genotype object
         if self.incparentsf1 and genotype_1.type != "intercross":
-            #self.genotype = genotype_2
             genotype = genotype_2
         else:
             self.incparentsf1 = 0
-            #self.genotype = genotype_1
             genotype = genotype_1
 
-        #self.samplelist = list(self.genotype.prgy)
         self.samplelist = list(genotype.prgy)
 
         return genotype
 
 
-#class DataSets(object):
-#    """Builds a list of DataSets"""
-#
-#    def __init__(self):
-#        self.datasets = list()
-#
-
-
-        #query = """SELECT Name FROM ProbeSetFreeze
-        #           UNION
-        #           SELECT Name From PublishFreeze
-        #           UNION
-        #           SELECT Name From GenoFreeze"""
-        #
-        #for result in g.db.execute(query).fetchall():
-        #    dataset = DataSet(result.Name)
-        #    self.datasets.append(dataset)
-
-#ds = DataSets()
-#print("[orange] ds:", ds.datasets)
-
 class DataSet(object):
     """
     DataSet class defines a dataset in webqtl, can be either Microarray,
@@ -512,32 +434,11 @@ class DataSet(object):
         """Gets overridden later, at least for Temp...used by trait's get_given_name"""
         return None
 
-    #@staticmethod
-    #def get_by_trait_id(trait_id):
-    #    """Gets the dataset object given the trait id"""
-    #
-    #
-    #
-    #    name = g.db.execute(""" SELECT
-    #
-    #                        """)
-    #
-    #    return DataSet(name)
-
     # Delete this eventually
     @property
     def riset():
         Weve_Renamed_This_As_Group
 
-
-    #@property
-    #def group(self):
-    #    if not self._group:
-    #        self.get_group()
-    #
-    #    return self._group
-
-
     def retrieve_other_names(self):
         """
         If the data set name parameter is not found in the 'Name' field of the data set table,
@@ -609,21 +510,6 @@ class DataSet(object):
         number_chunks = int(math.ceil(len(sample_ids) / chunk_size))
         trait_sample_data = []
         for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks):
-
-        #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId
-        #tempTable = None
-        #if GeneId and db.type == "ProbeSet":
-        #    if method == "3":
-        #        tempTable = self.getTempLiteratureTable(species=species,
-        #                                                input_species_geneid=GeneId,
-        #                                                returnNumber=returnNumber)
-        #
-        #    if method == "4" or method == "5":
-        #        tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol,
-        #                                        TissueProbeSetFreezeId=tissueProbeSetFreezeId,
-        #                                        method=method,
-        #                                        returnNumber=returnNumber)
-
             if self.type == "Publish":
                 dataset_type = "Phenotype"
             else:
@@ -1013,83 +899,11 @@ class MrnaAssayDataSet(DataSet):
             and ProbeSetFreezeId = {}
             """.format(escape(str(self.id)))
         results = g.db.execute(query).fetchall()
-        #print("After get_trait_list query")
         trait_data = {}
         for trait in results:
-            #print("Retrieving sample_data for ", trait[0])
             trait_data[trait[0]] = self.retrieve_sample_data(trait[0])
-        #print("After retrieve_sample_data")
         return trait_data
 
-    #def get_trait_data(self):
-    #    self.samplelist = self.group.samplelist + self.group.parlist + self.group.f1list
-    #    query = """
-    #        SELECT Strain.Name, Strain.Id FROM Strain, Species
-    #        WHERE Strain.Name IN {}
-    #        and Strain.SpeciesId=Species.Id
-    #        and Species.name = '{}'
-    #        """.format(create_in_clause(self.samplelist), *mescape(self.group.species))
-    #    results = dict(g.db.execute(query).fetchall())
-    #    sample_ids = [results[item] for item in self.samplelist]
-    #
-    #    # MySQL limits the number of tables that can be used in a join to 61,
-    #    # so we break the sample ids into smaller chunks
-    #    # Postgres doesn't have that limit, so we can get rid of this after we transition
-    #    chunk_size = 50
-    #    number_chunks = int(math.ceil(len(sample_ids) / chunk_size))
-    #    trait_sample_data = []
-    #    for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks):
-    #
-    #    #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId
-    #    #tempTable = None
-    #    #if GeneId and db.type == "ProbeSet":
-    #    #    if method == "3":
-    #    #        tempTable = self.getTempLiteratureTable(species=species,
-    #    #                                                input_species_geneid=GeneId,
-    #    #                                                returnNumber=returnNumber)
-    #    #
-    #    #    if method == "4" or method == "5":
-    #    #        tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol,
-    #    #                                        TissueProbeSetFreezeId=tissueProbeSetFreezeId,
-    #    #                                        method=method,
-    #    #                                        returnNumber=returnNumber)
-    #
-    #        temp = ['T%s.value' % item for item in sample_ids_step]
-    #        query = "SELECT {}.Name,".format(escape(self.type))
-    #        data_start_pos = 1
-    #        query += string.join(temp, ', ')
-    #        query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(self.type,
-    #                                                                 self.type,
-    #                                                                 self.type))
-    #
-    #        for item in sample_ids_step:
-    #            query += """
-    #                    left join {}Data as T{} on T{}.Id = {}XRef.DataId
-    #                    and T{}.StrainId={}\n
-    #                    """.format(*mescape(self.type, item, item, self.type, item, item))
-    #
-    #        query += """
-    #                WHERE {}XRef.{}FreezeId = {}Freeze.Id
-    #                and {}Freeze.Name = '{}'
-    #                and {}.Id = {}XRef.{}Id
-    #                order by {}.Id
-    #                """.format(*mescape(self.type, self.type, self.type, self.type,
-    #                           self.name, self.type, self.type, self.type, self.type))
-    #        results = g.db.execute(query).fetchall()
-    #        trait_sample_data.append(results)
-    #
-    #    trait_count = len(trait_sample_data[0])
-    #    self.trait_data = collections.defaultdict(list)
-    #
-    #    # put all of the separate data together into a dictionary where the keys are
-    #    # trait names and values are lists of sample values
-    #    for trait_counter in range(trait_count):
-    #        trait_name = trait_sample_data[0][trait_counter][0]
-    #        for chunk_counter in range(int(number_chunks)):
-    #            self.trait_data[trait_name] += (
-    #                trait_sample_data[chunk_counter][trait_counter][data_start_pos:])
-
-
     def get_trait_info(self, trait_list=None, species=''):
 
         #  Note: setting trait_list to [] is probably not a great idea.
@@ -1178,22 +992,9 @@ class MrnaAssayDataSet(DataSet):
                 result = g.db.execute(query).fetchone()
 
                 if result:
-                    #if result[0] and result[1]:
-                    #    lrs_chr = result[0]
-                    #    lrs_mb = result[1]
                     lrs_chr, lrs_mb = result
                     #XZ: LRS_location_value is used for sorting
                     lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb)
-
-                    #try:
-                    #    lrs_location_value = int(lrs_chr)*1000 + float(lrs_mb)
-                    #except:
-                    #    if lrs_chr.upper() == 'X':
-                    #        lrs_location_value = 20*1000 + float(lrs_mb)
-                    #    else:
-                    #        lrs_location_value = (ord(str(LRS_chr).upper()[0])*1000 +
-                    #                              float(lrs_mb))
-
                     this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs
                     this_trait.LRS_score_value = this_trait.lrs
                     this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb))
@@ -1259,35 +1060,6 @@ class MrnaAssayDataSet(DataSet):
 
         return dict(results)
 
-    #def retrieve_gene_symbols(self):
-    #    query = """
-    #                select ProbeSet.Name, ProbeSet.Symbol, ProbeSet.GeneId
-    #                from ProbeSet,ProbeSetXRef
-    #                where ProbeSetXRef.ProbeSetFreezeId = %s and
-    #                ProbeSetXRef.ProbeSetId=ProbeSet.Id;
-    #            """ % (self.id)
-    #    results = g.db.execute(query).fetchall()
-    #    symbol_dict = {}
-    #    for item in results:
-    #        symbol_dict[item[0]] = item[1]
-    #    return symbol_dict
-    #
-    #def retrieve_gene_ids(self):
-    #    query = """
-    #                select ProbeSet.Name, ProbeSet.GeneId
-    #                from ProbeSet,ProbeSetXRef
-    #                where ProbeSetXRef.ProbeSetFreezeId = %s and
-    #                ProbeSetXRef.ProbeSetId=ProbeSet.Id;
-    #            """ % (self.id)
-    #    return process_and_run_query(query)
-    #    results = g.db.execute(query).fetchall()
-    #    symbol_dict = {}
-    #    for item in results:
-    #        symbol_dict[item[0]] = item[1]
-    #    return symbol_dict
-
-
-
 
 class TempDataSet(DataSet):
     '''Temporary user-generated data set'''
-- 
cgit v1.2.3


From 9164ffc23879a8f9338939c06c36d3d49c3c899a Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 20 Apr 2016 10:26:54 +0000
Subject: Add file

---
 wqflask/base/webqtlConfig.py | 84 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100755 wqflask/base/webqtlConfig.py

(limited to 'wqflask/base')

diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
new file mode 100755
index 00000000..cdce119a
--- /dev/null
+++ b/wqflask/base/webqtlConfig.py
@@ -0,0 +1,84 @@
+#########################################'
+#      Environment Variables - public
+#
+# Note: much of this needs to handled by the settings/environment
+# scripts. But rather than migrating everything in one go, we'll
+# take it a step at a time. First the hard coded paths get replaced
+# with those in utility/tools.py
+# 
+#########################################
+
+from utility.tools import mk_dir, assert_dir, flat_files, TEMPDIR
+
+#Debug Level
+#1 for debug, mod python will reload import each time
+DEBUG = 1
+
+#USER privilege
+USERDICT = {'guest':1,'user':2, 'admin':3, 'root':4}
+
+#minimum number of informative strains
+KMININFORMATIVE = 5
+
+#maximum number of traits for interval mapping
+MULTIPLEMAPPINGLIMIT = 11
+
+#maximum number of traits for correlation
+MAXCORR = 100
+
+#Daily download limit from one IP
+DAILYMAXIMUM = 1000
+
+#maximum LRS value
+MAXLRS = 460.0
+
+#temporary data life span
+MAXLIFE = 86400
+
+#MINIMUM Database public value
+PUBLICTHRESH = 0
+
+#NBCI address
+NCBI_LOCUSID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s"
+UCSC_REFSEQ = "http://genome.cse.ucsc.edu/cgi-bin/hgGene?db=%s&hgg_gene=%s&hgg_chrom=chr%s&hgg_start=%s&hgg_end=%s"
+GENBANK_ID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=Nucleotide&cmd=search&doptcmdl=DocSum&term=%s"
+OMIM_ID = "http://www.ncbi.nlm.nih.gov/omim/%s"
+UNIGEN_ID = "http://www.ncbi.nlm.nih.gov/UniGene/clust.cgi?ORG=%s&CID=%s";
+HOMOLOGENE_ID = "http://www.ncbi.nlm.nih.gov/sites/entrez?Db=homologene&Cmd=DetailsSearch&Term=%s"
+PUBMEDLINK_URL = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids=%s&dopt=Abstract"
+UCSC_POS = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=mammal&org=%s&db=%s&position=chr%s:%s-%s&pix=800&Submit=submit"
+UCSC_BLAT = 'http://genome.ucsc.edu/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s'
+UTHSC_BLAT = 'http://ucscbrowser.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s'
+UCSC_GENOME = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=http://web2qtl.utmem.edu:88/snp/chr%s"
+ENSEMBLE_BLAT = 'http://www.ensembl.org/Mus_musculus/featureview?type=AffyProbe&id=%s'
+DBSNP = 'http://www.ncbi.nlm.nih.gov/SNP/snp_ref.cgi?type=rs&rs=%s'
+UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s&hgt.customText=http://gbic.biol.rug.nl/~ralberts/tracks/%s/%s"
+GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit"
+ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s"
+
+# HTMLPATH = GNROOT + 'genotype_files/'
+# PYLMM_PATH
+# IMGDIR = GNROOT + '/wqflask/wqflask/static/output/'
+
+# Temporary storage:
+TMPDIR               = mk_dir(TEMPDIR+'/gn2/')
+CACHEDIR             = mk_dir(TEMPDIR+'/cache/')
+# We can no longer write into the git tree:
+GENERATED_IMAGE_DIR  = mk_dir(TMPDIR+'/generate/')
+GENERATED_TEXT_DIR   = mk_dir(TMPDIR+'/generate_text/')
+
+# Flat file directories
+GENODIR              = flat_files('genotype')+'/'
+JSON_GENODIR         = assert_dir(GENODIR+'json/')
+
+# SITENAME = 'GN'
+# PORTADDR = "http://50.16.251.170"
+# BASEHREF = '<base href="http://50.16.251.170/">'
+
+INFOPAGEHREF = '/dbdoc/%s.html'
+CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR'
+SCRIPTFILE = 'main.py'
+
+# GLOSSARYFILE = "/glossary.html"
+# REFRESHSTR = '<meta http-equiv="refresh" content="5;url=%s' + SCRIPTFILE +'?sid=%s">'
+# REFRESHDIR = '%s' + SCRIPTFILE +'?sid=%s'
-- 
cgit v1.2.3


From b63e4b5111269dcaf4a9b9eec7d387d8aecb3ba5 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Fri, 13 May 2016 03:07:41 -0500
Subject: Fix paths for images

---
 wqflask/base/webqtlConfig.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index cdce119a..f62207e0 100755
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -64,8 +64,8 @@ ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?sp
 TMPDIR               = mk_dir(TEMPDIR+'/gn2/')
 CACHEDIR             = mk_dir(TEMPDIR+'/cache/')
 # We can no longer write into the git tree:
-GENERATED_IMAGE_DIR  = mk_dir(TMPDIR+'/generate/')
-GENERATED_TEXT_DIR   = mk_dir(TMPDIR+'/generate_text/')
+GENERATED_IMAGE_DIR  = mk_dir(TMPDIR+'/generated/')
+GENERATED_TEXT_DIR   = mk_dir(TMPDIR+'/generated_text/')
 
 # Flat file directories
 GENODIR              = flat_files('genotype')+'/'
-- 
cgit v1.2.3


From 409848a4b3873fb0512ae17135b9794030eff024 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Fri, 13 May 2016 03:21:20 -0500
Subject: WebqtlConfig.py: Constant handling

---
 wqflask/base/webqtlConfig.py                               | 7 +------
 wqflask/wqflask/marker_regression/marker_regression_gn1.py | 2 +-
 2 files changed, 2 insertions(+), 7 deletions(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index f62207e0..6890dacf 100755
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -71,14 +71,9 @@ GENERATED_TEXT_DIR   = mk_dir(TMPDIR+'/generated_text/')
 GENODIR              = flat_files('genotype')+'/'
 JSON_GENODIR         = assert_dir(GENODIR+'json/')
 
-# SITENAME = 'GN'
-# PORTADDR = "http://50.16.251.170"
-# BASEHREF = '<base href="http://50.16.251.170/">'
+PORTADDR = "http://50.16.251.170"
 
 INFOPAGEHREF = '/dbdoc/%s.html'
 CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR'
 SCRIPTFILE = 'main.py'
 
-# GLOSSARYFILE = "/glossary.html"
-# REFRESHSTR = '<meta http-equiv="refresh" content="5;url=%s' + SCRIPTFILE +'?sid=%s">'
-# REFRESHDIR = '%s' + SCRIPTFILE +'?sid=%s'
diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
index 97e4b934..4460c06d 100644
--- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py
+++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
@@ -674,7 +674,7 @@ class MarkerRegression(object):
 
         fpText.write("Source: WebQTL, The GeneNetwork (%s)\n" % webqtlConfig.PORTADDR)
         #
-        fpText.write("Site: %s\n" % webqtlConfig.SITENAME)
+        fpText.write("Site: GN\n")
         fpText.write("Page: Map Viewer\n")
         fpText.write(time.strftime("Date and Time (US Center): %b %d, %Y at %I.%M %p\n", time.localtime()))
         fpText.write("Trait ID: %s\n" % self.this_trait.name)
-- 
cgit v1.2.3


From 3706cf389a8ead64f02c2ed997fd3defbede9832 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Fri, 13 May 2016 08:48:18 +0000
Subject: Removing ref. to HTMLPATH

---
 wqflask/base/webqtlConfig.py                              | 6 +++---
 wqflask/wqflask/marker_regression/MarkerRegressionPage.py | 4 ++--
 wqflask/wqflask/marker_regression/gemma_mapping.py        | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'wqflask/base')

diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index 6890dacf..0358bcbf 100755
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -56,9 +56,9 @@ UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s
 GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit"
 ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s"
 
-# HTMLPATH = GNROOT + 'genotype_files/'
-# PYLMM_PATH
-# IMGDIR = GNROOT + '/wqflask/wqflask/static/output/'
+# The following paths are no longer in use!
+#   HTMLPATH is replaced by GENODIR
+#   IMGDIR is replaced by GENERATED_IMAGE_DIR
 
 # Temporary storage:
 TMPDIR               = mk_dir(TEMPDIR+'/gn2/')
diff --git a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py
index 455fcf95..4c3391e5 100755
--- a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py
+++ b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py
@@ -72,7 +72,7 @@ class MarkerRegressionPage(templatePage):
             #automatically generate pheno txt file for PLINK
             self.genPhenoTxtFileForPlink(phenoFileName=plinkOutputFileName,RISetName=fd.RISet,probesetName=probesetName, valueDict=allTraitValueDict)
             # os.system full path is required for input and output files; specify missing value is -9999
-            plink_command = '%splink/plink --noweb --ped %splink/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %splink/%s.map --pheno %s/%s.txt --pheno-name %s --missing-phenotype -9999 --out %s%s --assoc ' % (webqtlConfig.HTMLPATH, webqtlConfig.HTMLPATH,  fd.RISet, webqtlConfig.HTMLPATH, fd.RISet, webqtlConfig.TMPDIR, plinkOutputFileName, probesetName, webqtlConfig.TMPDIR, plinkOutputFileName)
+            plink_command = '%splink/plink --noweb --ped %splink/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %splink/%s.map --pheno %s/%s.txt --pheno-name %s --missing-phenotype -9999 --out %s%s --assoc ' % (webqtlConfig.GENODIR, webqtlConfig.GENODIR,  fd.RISet, webqtlConfig.GENODIR, fd.RISet, webqtlConfig.TMPDIR, plinkOutputFileName, probesetName, webqtlConfig.TMPDIR, plinkOutputFileName)
 
             os.system(plink_command)
 
@@ -1597,7 +1597,7 @@ class MarkerRegressionPage(templatePage):
 
     # get strain name from ped file in order
     def getStrainNameFromPedFile(self, RISetName=''):
-        pedFileopen= open("%splink/%s.ped"%(webqtlConfig.HTMLPATH, RISetName),"r")
+        pedFileopen= open("%splink/%s.ped"%(webqtlConfig.GENODIR, RISetName),"r")
         line =pedFileopen.readline()
         strainNameList=[]
 
diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py
index 8fb086c1..caea5802 100644
--- a/wqflask/wqflask/marker_regression/gemma_mapping.py
+++ b/wqflask/wqflask/marker_regression/gemma_mapping.py
@@ -11,7 +11,7 @@ def run_gemma(this_dataset, samples, vals):
     gen_pheno_txt_file(this_dataset, samples, vals)
 
     # Don't do this!
-    # os.chdir("{}gemma".format(webqtlConfig.HTMLPATH))
+    # os.chdir("{}gemma".format(webqtlConfig.GENODIR))
 
     # use GEMMA_RUN in the next one, create a unique temp file
     
-- 
cgit v1.2.3