Merge pull request #255 from lyan6/master

Correlation Scatterplot updates

17 files changed, 399 insertions, 182 deletions

diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index 1e47e183..e5f10edf 100644
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -65,7 +65,7 @@ ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?sp
 # want to reach this base dir
 assert_writable_dir(TEMPDIR)
 
-TMPDIR               = mk_dir(TEMPDIR+'gn2')
+TMPDIR               = mk_dir(TEMPDIR+'/gn2/')
 assert_writable_dir(TMPDIR)
 
 CACHEDIR             = mk_dir(TMPDIR+'/cache/')
diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py
index e5726656..a5fd86e2 100644
--- a/wqflask/maintenance/gen_select_dataset.py
+++ b/wqflask/maintenance/gen_select_dataset.py
@@ -57,7 +57,7 @@ import urlparse
 
 from pprint import pformat as pf
 
-#Engine = sa.create_engine(zach_settings.SQLALCHEMY_DATABASE_URI)
+#Engine = sa.create_engine(zach_settings.SQL_URI)
 
 # build MySql database connection
 
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index c28c617a..393ff2df 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -13,6 +13,11 @@ logger = logging.getLogger(__name__ )
 
 OVERRIDES = {}
 
+def app_set(command_id, value):
+    """Set application wide value"""
+    app.config.setdefault(command_id,value)
+    value
+
 def get_setting(command_id,guess=None):
     """Resolve a setting from the environment or the global settings in
     app.config, with valid_path is a function checking whether the
@@ -40,6 +45,7 @@ def get_setting(command_id,guess=None):
     def value(command):
         if command:
             # sys.stderr.write("Found "+command+"\n")
+            app_set(command_id,command)
             return command
         else:
             return None
@@ -89,6 +95,18 @@ def valid_path(dir):
         return dir
     return None
 
+def js_path(module=None):
+    """
+    Find the JS module in the two paths
+    """
+    try_gn   = get_setting("JS_GN_PATH")+"/"+module
+    if valid_path(try_gn):
+        return try_gn
+    try_guix = get_setting("JS_GUIX_PATH")+"/"+module
+    if valid_path(try_guix):
+        return try_guix
+    raise "No JS path found for "+module+" (check JS_GN_PATH)"
+
 def pylmm_command(guess=None):
     return valid_bin(get_setting("PYLMM_COMMAND",guess))
 
@@ -205,21 +223,27 @@ SQL_URI            = get_setting('SQL_URI')
 LOG_LEVEL          = get_setting('LOG_LEVEL')
 LOG_LEVEL_DEBUG    = get_setting_int('LOG_LEVEL_DEBUG')
 LOG_SQL            = get_setting_bool('LOG_SQL')
-LOG_SQLALCHEMY     = get_setting_bool('LOG_SQLALCHEMY')
+LOG_SQL_ALCHEMY    = get_setting_bool('LOG_SQL_ALCHEMY')
 LOG_BENCH          = get_setting_bool('LOG_BENCH')
 LOG_FORMAT         = "%(message)s"    # not yet in use
 USE_REDIS          = get_setting_bool('USE_REDIS')
 USE_GN_SERVER      = get_setting_bool('USE_GN_SERVER')
 
 GENENETWORK_FILES  = get_setting('GENENETWORK_FILES')
-TEMP_TRAITS        = get_setting('TEMP_TRAITS')
-
-PYLMM_COMMAND      = pylmm_command()
-GEMMA_COMMAND      = gemma_command()
-GEMMA_RESULTS_PATH = get_setting('GEMMA_RESULTS_PATH')
-PLINK_COMMAND      = plink_command()
+JS_GUIX_PATH       = get_setting('JS_GUIX_PATH')
+# assert_dir(JS_GUIX_PATH) - don't enforce right now
+JS_GN_PATH         = get_setting('JS_GN_PATH')
+# assert_dir(JS_GN_PATH)
+
+PYLMM_COMMAND      = app_set("PYLMM_COMMAND",pylmm_command())
+GEMMA_COMMAND      = app_set("GEMMA_COMMAND",gemma_command())
+PLINK_COMMAND      = app_set("PLINK_COMMAND",plink_command())
 TEMPDIR            = tempdir() # defaults to UNIX TMPDIR
 
+# ---- Handle specific JS modules
+JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher"))
+assert_dir(JS_TWITTER_POST_FETCHER_PATH)
+
 from six import string_types
 
 if os.environ.get('WQFLASK_OVERRIDES'):
@@ -234,5 +258,3 @@ if os.environ.get('WQFLASK_OVERRIDES'):
             else:
                 OVERRIDES[k] = cmd
             logger.debug(OVERRIDES)
-
-assert_dir(get_setting("TWITTER_POST_FETCHER_JS_PATH"))
diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py
index 4df872ad..96c2c301 100644
--- a/wqflask/wqflask/database.py
+++ b/wqflask/wqflask/database.py
@@ -22,8 +22,8 @@ def init_db():
     # they will be registered properly on the metadata.  Otherwise
     # you will have to import them first before calling init_db()
     #import yourapplication.models
+    logger.info("Initializing database connection")
     import wqflask.model
-    logger.debug("Creating all model metadata")
     Base.metadata.create_all(bind=engine)
     logger.info("Done creating all model metadata")
 
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index 87532445..8882c515 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -37,7 +37,7 @@ from utility import temp_data
 from utility.benchmark import Bench
 from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping
 
-from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, GEMMA_RESULTS_PATH, PLINK_COMMAND, TEMPDIR
+from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR
 from utility.external import shell
 from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR
 
diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
index 49959a74..82a44796 100644
--- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py
+++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
@@ -1018,7 +1018,7 @@ class MarkerRegression(object):
         else:
             if self.selectedChr > -1:
                 for i, qtlresult in enumerate(self.qtlresults):
-                    if qtlresult['Chr'] != self.selectedChr:
+                    if qtlresult['chr'] != self.selectedChr:
                         continue
 
                     if i==0 and qtlresult['Mb'] >= Mb:
@@ -1027,7 +1027,7 @@ class MarkerRegression(object):
 
                     #the trait's position is between two traits
                     if i > 0 and self.qtlresults[i-1]['Mb'] < Mb and qtlresult['Mb'] >= Mb:
-                        locPixel = xLeftOffset + plotXScale*(self.qtlresults[i-1]['cM']+(qtlresult['cM']-self.qtlresults[i-1]['cM'])*(Mb - self.qtlresults[i-1]['Mb'])/(qtlresult['Mb']-self.qtlresults[i-1]['Mb']))
+                        locPixel = xLeftOffset + plotXScale*(self.qtlresults[i-1]['Mb']+(qtlresult['Mb']-self.qtlresults[i-1]['Mb'])*(Mb - self.qtlresults[i-1]['Mb'])/(qtlresult['Mb']-self.qtlresults[i-1]['Mb']))
                         break
 
                     #the trait's position is on the right of the last genotype
@@ -1036,11 +1036,12 @@ class MarkerRegression(object):
             else:
                 locPixel = xLeftOffset
                 for i, _chr in enumerate(self.ChrList):
-                    if _chr != Chr:
-                        locPixel += (self.ChrLengthDistList[i] + self.GraphInterval)*plotXScale
-                    else:
-                        locPixel += (Mb*(_chr[-1].cM-_chr[0].cM)/self.ChrLengthCMList[i])*plotXScale
-                        break
+                    if i < (len(self.ChrList)-1):
+                        if _chr != Chr:
+                            locPixel += (self.ChrLengthDistList[i] + self.GraphInterval)*plotXScale
+                        else:
+                            locPixel += (Mb*(_chr[-1].cM-_chr[0].cM)/self.ChrLengthCMList[i])*plotXScale
+                            break
         if locPixel >= 0 and self.plotScale == 'physic':
             traitPixel = ((locPixel, yZero), (locPixel-6, yZero+12), (locPixel+6, yZero+12))
             canvas.drawPolygon(traitPixel, edgeColor=pid.black, fillColor=self.TRANSCRIPT_LOCATION_COLOR, closed=1)
diff --git a/wqflask/wqflask/static/new/css/corr_scatter_plot2.css b/wqflask/wqflask/static/new/css/corr_scatter_plot2.css
index 92e777c2..5aa9b9bf 100644
--- a/wqflask/wqflask/static/new/css/corr_scatter_plot2.css
+++ b/wqflask/wqflask/static/new/css/corr_scatter_plot2.css
@@ -1,19 +1,40 @@
 .nvd3 .nv-axis.nv-x text {
-	font-size: 16px;
-	font-weight: normal;
-	fill: black;
+  font-size: 16px;
+  font-weight: normal;
+  fill: black;
 }
 
 .nvd3 .nv-axis.nv-y text {
-	font-size: 16px;
-	font-weight: normal;
-	fill: black;
+  font-size: 16px;
+  font-weight: normal;
+  fill:black;
 }
 
-.nv-y .tick.zero line {
-	stroke: black;
+.nv-x .nv-axis g path.domain {
+    stroke: black;
+    stroke-width: 2;
 }
 
 .nv-y .nv-axis g path.domain {
-	stroke: black;
+    stroke: black;
+    stroke-width: 2;
+}
+
+.nvd3 .nv-axis.nv-x path.domain {
+    stroke-opacity: 1;
+}
+
+.nvd3 .nv-axis.nv-y path.domain {
+    stroke-opacity: 1;
+}
+
+line.nv-regLine {
+    stroke: red !important;
+}
+
+.nv-axisMin-x,
+.nv-axisMax-x,
+.nv-axisMin-y,
+.nv-axisMax-y {
+    display: none;
 }
diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
index 861af3d6..f3712b5d 100644
--- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
+++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
@@ -1340,14 +1340,14 @@
                   "GSE9588 Human Liver Normal (Mar11) Both Sexes"
                ],
                [
-                  "384",
-                  "HLCF_0311",
-                  "GSE9588 Human Liver Normal (Mar11) Females"
-               ],
-               [
                   "383",
                   "HLCM_0311",
                   "GSE9588 Human Liver Normal (Mar11) Males"
+               ],
+               [
+                  "384",
+                  "HLCF_0311",
+                  "GSE9588 Human Liver Normal (Mar11) Females"
                ]
             ],
             "Phenotypes": [
@@ -1493,6 +1493,15 @@
                ]
             ]
          },
+         "Islets-Gerling": {
+            "Phenotypes": [
+               [
+                  "None",
+                  "Islets-GerlingPublish",
+                  "Islets-Gerling Published Phenotypes"
+               ]
+            ]
+         },
          "TIGEM-Retina-RNA-Seq": {
             "Phenotypes": [
                [
@@ -1561,6 +1570,11 @@
             ],
             "Heart mRNA": [
                [
+                  "820",
+                  "UCLA_AXB_BXA_Aor_Jan16",
+                  "UCLA AXB/BXA Aorta Affy M430 2.0 (Jan16) RMA"
+               ],
+               [
                   "421",
                   "IRCM_AXBXA_HRI0213",
                   "IRCM AXB/BXA Mouse Heart ILM MouseRef-8 v2.0 (Feb13) RankInv"
@@ -1568,6 +1582,11 @@
             ],
             "Liver mRNA": [
                [
+                  "822",
+                  "UCLA_AXB_BXA_Liv_Jan16",
+                  "UCLA AXB/BXA Liver Affy M430 2.0 (Jan16) RMA"
+               ],
+               [
                   "352",
                   "GSE16780AB_UCLA_ML0911",
                   "GSE16780 UCLA Mouse AXB/BXA Liver Affy HT M430A (Sep11) RMA"
@@ -1727,11 +1746,6 @@
             ],
             "Striatum mRNA": [
                [
-                  "85",
-                  "SA_M2_0905_P",
-                  "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) PDNN"
-               ],
-               [
                   "84",
                   "SA_M2_0905_R",
                   "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) RMA"
@@ -1740,22 +1754,27 @@
                   "83",
                   "SA_M2_0905_M",
                   "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) MAS5"
+               ],
+               [
+                  "85",
+                  "SA_M2_0905_P",
+                  "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) PDNN"
                ]
             ]
          },
          "BHF2": {
             "Adipose mRNA": [
                [
-                  "197",
-                  "UCLA_BHF2_ADIPOSE_FEMALE",
-                  "UCLA BHF2 Adipose Female mlratio"
-               ],
-               [
                   "196",
                   "UCLA_BHF2_ADIPOSE_MALE",
                   "UCLA BHF2 Adipose Male mlratio"
                ],
                [
+                  "197",
+                  "UCLA_BHF2_ADIPOSE_FEMALE",
+                  "UCLA BHF2 Adipose Female mlratio"
+               ],
+               [
                   "165",
                   "UCLA_BHF2_ADIPOSE_0605",
                   "UCLA BHF2 Adipose (June05) mlratio"
@@ -1763,16 +1782,16 @@
             ],
             "Brain mRNA": [
                [
-                  "199",
-                  "UCLA_BHF2_BRAIN_FEMALE",
-                  "UCLA BHF2 Brain Female mlratio"
-               ],
-               [
                   "198",
                   "UCLA_BHF2_BRAIN_MALE",
                   "UCLA BHF2 Brain Male mlratio"
                ],
                [
+                  "199",
+                  "UCLA_BHF2_BRAIN_FEMALE",
+                  "UCLA BHF2 Brain Female mlratio"
+               ],
+               [
                   "166",
                   "UCLA_BHF2_BRAIN_0605",
                   "UCLA BHF2 Brain (June05) mlratio"
@@ -1787,16 +1806,16 @@
             ],
             "Liver mRNA": [
                [
-                  "201",
-                  "UCLA_BHF2_LIVER_FEMALE",
-                  "UCLA BHF2 Liver Female mlratio"
-               ],
-               [
                   "200",
                   "UCLA_BHF2_LIVER_MALE",
                   "UCLA BHF2 Liver Male mlratio"
                ],
                [
+                  "201",
+                  "UCLA_BHF2_LIVER_FEMALE",
+                  "UCLA BHF2 Liver Female mlratio"
+               ],
+               [
                   "167",
                   "UCLA_BHF2_LIVER_0605",
                   "UCLA BHF2 Liver (June05) mlratio"
@@ -2000,11 +2019,6 @@
                   "UTHSC Brain mRNA U74Av2 (Nov05) PDNN"
                ],
                [
-                  "81",
-                  "BR_U_0805_P",
-                  "UTHSC Brain mRNA U74Av2 (Aug05) PDNN"
-               ],
-               [
                   "80",
                   "BR_U_0805_M",
                   "UTHSC Brain mRNA U74Av2 (Aug05) MAS5"
@@ -2015,6 +2029,11 @@
                   "UTHSC Brain mRNA U74Av2 (Aug05) RMA"
                ],
                [
+                  "81",
+                  "BR_U_0805_P",
+                  "UTHSC Brain mRNA U74Av2 (Aug05) PDNN"
+               ],
+               [
                   "42",
                   "CB_M_0204_P",
                   "INIA Brain mRNA M430 (Feb04) PDNN"
@@ -2142,6 +2161,13 @@
                   "BXD Genotypes"
                ]
             ],
+            "Heart mRNA": [
+               [
+                  "819",
+                  "UCLA_BXD_Aor_Jan16",
+                  "UCLA BXD Aorta Affy M430 2.0 (Jan16) RMA"
+               ]
+            ],
             "Hematopoietic Cells mRNA": [
                [
                   "149",
@@ -2218,6 +2244,26 @@
                   "UMUTAffy Hippocampus Exon (Feb09) RMA"
                ],
                [
+                  "814",
+                  "UTHSC_ILM_BXD_hipp_NOSb_0217",
+                  "UTHSC BXD Hippocampus ILM v6.1 NOS Balanced (Feb17) RankInv"
+               ],
+               [
+                  "815",
+                  "UTHSC_ILM_BXD_hipp_NOEb_0217",
+                  "UTHSC BXD Hippocampus ILM v6.1 NOE Balanced (Feb17) RankInv"
+               ],
+               [
+                  "816",
+                  "UTHSC_ILM_BXD_hipp_RSSb_0217",
+                  "UTHSC BXD Hippocampus ILM v6.1 RSS Balanced (Feb17) RankInv"
+               ],
+               [
+                  "817",
+                  "UTHSC_ILM_BXD_hipp_RSEb_0217",
+                  "UTHSC BXD Hippocampus ILM v6.1 RSE Balanced (Feb17) RankInv"
+               ],
+               [
                   "780",
                   "UTHSC_ILM_BXD_hipp_NOEb_0216",
                   "UTHSC BXD Hippocampus ILM v6.1 NOE Balanced (Feb16) RankInv"
@@ -2391,14 +2437,24 @@
             ],
             "Liver Proteome": [
                [
+                  "540",
+                  "EPFLETHZBXDprotCD0514",
+                  "EPFL/ETHZ BXD Liver, Chow Diet (Jun16) Top100 SWATH"
+               ],
+               [
+                  "541",
+                  "EPFLETHZBXDprotHFD0514",
+                  "EPFL/ETHZ BXD Liver, High Fat Diet (Jun16) Top100 SWATH"
+               ],
+               [
                   "704",
                   "EPFLETHZBXDprotCD_LS1114",
-                  "EPFL/ETHZ BXD Liver, Chow Diet (Oct14) Light SWATH"
+                  "EPFL/ETHZ BXD Liver, Chow Diet (Oct14) Top10 SWATH"
                ],
                [
                   "705",
                   "EPFLETHZBXDprotHF_LS1114",
-                  "EPFL/ETHZ BXD Liver, High Fat Diet (Oct14) Light SWATH"
+                  "EPFL/ETHZ BXD Liver, High Fat Diet Diet (Oct14) Top10 SWATH"
                ],
                [
                   "703",
@@ -2414,20 +2470,15 @@
                   "489",
                   "EPFLBXDprotHFDRPN0214",
                   "EPFL/LISP BXD Liver, Soluble Proteins HFD (Feb14) SRM"
-               ],
-               [
-                  "540",
-                  "EPFLETHZBXDprotCD0514",
-                  "EPFL/ETHZ BXD Liver, Soluble Proteins CD (Jun14) SWATH"
-               ],
-               [
-                  "541",
-                  "EPFLETHZBXDprotHFD0514",
-                  "EPFL/ETHZ BXD Liver, Soluble Proteins HFD (Jun14) SWATH"
                ]
             ],
             "Liver mRNA": [
                [
+                  "818",
+                  "UCLA_BXD_Liv_Jan16",
+                  "UCLA BXD Liver Affy M430 2.0 (Jan16) RMA"
+               ],
+               [
                   "430",
                   "EPFLMouseLiverRMA0413",
                   "EPFL/LISP BXD CD+HFD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA"
@@ -2443,6 +2494,11 @@
                   "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA"
                ],
                [
+                  "433",
+                  "EPFLMouseLiverBothExRMA0413",
+                  "EPFL/LISP BXD CD+HFD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA Exon Level"
+               ],
+               [
                   "700",
                   "UTHSC-VGX_MmBXDHepatocytesRMA1014",
                   "UT-VGX Hepatocytes Affy Mouse Gene 1.0 ST Gene Level (Oct14) RMA"
@@ -2466,6 +2522,51 @@
                   "702",
                   "SUH_Liv_RMAEx_0611",
                   "SUH BXD Liver CCl4-treated Affy Mouse Gene 1.0 ST Exon Level (Jun11) RMA"
+               ],
+               [
+                  "256",
+                  "GenEx_BXD_liverEt_M5_0912",
+                  "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Both Sexes"
+               ],
+               [
+                  "257",
+                  "GenEx_BXD_liverEt_M5M_0912",
+                  "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Males"
+               ],
+               [
+                  "258",
+                  "GenEx_BXD_liverEt_M5F_0912",
+                  "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Females"
+               ],
+               [
+                  "307",
+                  "GenEx_BXD_liverEt_RMA_0211",
+                  "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Both Sexes"
+               ],
+               [
+                  "308",
+                  "GenEx_BXD_liverEt_RMA_M_0211",
+                  "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Males"
+               ],
+               [
+                  "309",
+                  "GenEx_BXD_liverEt_RMA_F_0211",
+                  "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Females"
+               ],
+               [
+                  "310",
+                  "GenEx_BXD_liverSal_RMA_0211",
+                  "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Both Sexes"
+               ],
+               [
+                  "311",
+                  "GenEx_BXD_liverSal_RMA_M_0211",
+                  "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Males"
+               ],
+               [
+                  "312",
+                  "GenEx_BXD_liverSal_RMA_F_0211",
+                  "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Females"
                ]
             ],
             "Lung mRNA": [
@@ -2546,14 +2647,14 @@
                   "HQF BXD Neocortex ILM6v1.1 (Feb08) RankInv"
                ],
                [
-                  "275",
-                  "DevNeocortex_ILM6.2P14RInv_1110",
-                  "BIDMC/UTHSC Dev Neocortex P14 ILMv6.2 (Nov10) RankInv"
-               ],
-               [
                   "274",
                   "DevNeocortex_ILM6.2P3RInv_1110",
                   "BIDMC/UTHSC Dev Neocortex P3 ILMv6.2 (Nov10) RankInv"
+               ],
+               [
+                  "275",
+                  "DevNeocortex_ILM6.2P14RInv_1110",
+                  "BIDMC/UTHSC Dev Neocortex P14 ILMv6.2 (Nov10) RankInv"
                ]
             ],
             "Nucleus Accumbens mRNA": [
@@ -2761,11 +2862,6 @@
             ],
             "Ventral Tegmental Area mRNA": [
                [
-                  "230",
-                  "VCUEtvsSal_0609_R",
-                  "VCU BXD VTA Et vs Sal M430 2.0 (Jun09) RMA"
-               ],
-               [
                   "229",
                   "VCUEtOH_0609_R",
                   "VCU BXD VTA EtOH M430 2.0 (Jun09) RMA"
@@ -2774,6 +2870,11 @@
                   "228",
                   "VCUSal_0609_R",
                   "VCU BXD VTA Sal M430 2.0 (Jun09) RMA"
+               ],
+               [
+                  "230",
+                  "VCUEtvsSal_0609_R",
+                  "VCU BXD VTA Et vs Sal M430 2.0 (Jun09) RMA"
                ]
             ]
          },
@@ -2920,6 +3021,27 @@
                ]
             ]
          },
+         "CIE-RMA": {
+            "Midbrain mRNA": [
+               [
+                  "830",
+                  "INIA_UTHSC_Mid_AffyMTA1_Apr17",
+                  "INIA-UTHSC Midbrain CIE Affy MTA 1.0 GeneLevel (Apr17) RMA"
+               ],
+               [
+                  "834",
+                  "INIA_UTHSC_Mid_AffyMTA1_Ex_May17",
+                  "INIA-UTHSC Midbrain CIE Affy MTA 1.0 Exon Level (Apr17) RMA"
+               ]
+            ],
+            "Phenotypes": [
+               [
+                  "None",
+                  "CIE-RMAPublish",
+                  "CIE-RMA Published Phenotypes"
+               ]
+            ]
+         },
          "CMS": {
             "Phenotypes": [
                [
@@ -3021,6 +3143,13 @@
                   "CXB Genotypes"
                ]
             ],
+            "Heart mRNA": [
+               [
+                  "821",
+                  "UCLA_CXB_Aor_Jan16",
+                  "UCLA CXB Aorta Affy M430 2.0 (Jan16) RMA"
+               ]
+            ],
             "Hippocampus mRNA": [
                [
                   "100",
@@ -3033,6 +3162,13 @@
                   "Hippocampus Consortium M430v2 CXB (Dec05) PDNN"
                ]
             ],
+            "Liver mRNA": [
+               [
+                  "823",
+                  "UCLA_CXB_Liv_Jan16",
+                  "UCLA CXB Liver Affy M430 2.0 (Jan16) RMA"
+               ]
+            ],
             "Phenotypes": [
                [
                   "628",
@@ -3048,6 +3184,7 @@
                ]
             ]
          },
+         "EMSR": {},
          "HS": {
             "Hippocampus mRNA": [
                [
@@ -3545,6 +3682,10 @@
             "Brain, Development: Normal Gene Expression (Yale/Sestan)"
          ],
          [
+            "Islets-Gerling",
+            "Pancreatic: Islets (UTHSC/Gerling)"
+         ],
+         [
             "TIGEM-Retina-RNA-Seq",
             "Retina: Normal Adult Gene Expression, RNA-Seq (TIGEM)"
          ]
@@ -3616,7 +3757,11 @@
          ],
          [
             "CIE-INIA",
-            "Chronic Intermittent Ethanol"
+            "Chronic Intermittent Ethanol Phase 1"
+         ],
+         [
+            "CIE-RMA",
+            "Chronic Intermittent Ethanol Phase 2"
          ],
          [
             "CMS",
@@ -3631,6 +3776,10 @@
             "CXB"
          ],
          [
+            "EMSR",
+            "Ethanol-Medicated Stress Reduction"
+         ],
+         [
             "HS",
             "Heterogeneous Stock"
          ],
@@ -4414,6 +4563,12 @@
                "Ventrolateral Prefrontal Cortex mRNA"
             ]
          ],
+         "Islets-Gerling": [
+            [
+               "Phenotypes",
+               "Phenotypes"
+            ]
+         ],
          "TIGEM-Retina-RNA-Seq": [
             [
                "Phenotypes",
@@ -4630,6 +4785,10 @@
                "Gastrointestinal mRNA"
             ],
             [
+               "Heart mRNA",
+               "Heart mRNA"
+            ],
+            [
                "Hematopoietic Cells mRNA",
                "Hematopoietic Cells mRNA"
             ],
@@ -4780,6 +4939,16 @@
                "LCM Brain Regions mRNA"
             ]
          ],
+         "CIE-RMA": [
+            [
+               "Phenotypes",
+               "Phenotypes"
+            ],
+            [
+               "Midbrain mRNA",
+               "Midbrain mRNA"
+            ]
+         ],
          "CMS": [
             [
                "Phenotypes",
@@ -4822,14 +4991,23 @@
                "Genotypes"
             ],
             [
+               "Heart mRNA",
+               "Heart mRNA"
+            ],
+            [
                "Hippocampus mRNA",
                "Hippocampus mRNA"
             ],
             [
+               "Liver mRNA",
+               "Liver mRNA"
+            ],
+            [
                "Spleen mRNA",
                "Spleen mRNA"
             ]
          ],
+         "EMSR": [],
          "HS": [
             [
                "Phenotypes",
diff --git a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js
index 7a5a86bd..96763aca 100644
--- a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js
+++ b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js
@@ -1,74 +1,64 @@
 var chart;
 
 nv.addGraph(function() {
-	//
-	chart = nv.models.scatterChart();
-	//
-	chart.showLegend(false);
-	chart.duration(300);
-	chart.color(d3.scale.category10().range());
-	chart.pointRange([200,0]);
-	//
-	// chart.showDistX(true);
-	// chart.showDistY(true);
-	chart.xAxis.tickFormat(d3.format('.02f'));
-	chart.yAxis.tickFormat(d3.format('.02f'));
-	// chart.showXAxis(false);
-	// chart.showYAxis(false);
-	chart.xAxis.axisLabel(js_data.trait_1);
-	chart.yAxis.axisLabel(js_data.trait_2);
-	//
-	xmin = d3.min(js_data.data[0]);
-	xmax = d3.max(js_data.data[0]);
-	xrange = xmax - xmin;
-	ymin = d3.min(js_data.data[1]);
-	ymax = d3.max(js_data.data[1]);
-	yrange = ymax - ymin;
-	chart.xDomain([xmin - xrange/10, xmax + xrange/10]);
-	chart.yDomain([ymin - yrange/10, ymax + yrange/10]);
-	//
-	d3.select('#scatterplot2 svg').datum(nv.log(getdata())).call(chart);
-	nv.utils.windowResize(chart.update);
-	return chart;
+    //
+    chart = nv.models.scatterChart();
+    //
+    chart.showLegend(false);
+    chart.duration(300);
+    chart.color(d3.scale.category10().range());
+    chart.pointRange([200,0]);
+    //
+    chart.xAxis.axisLabel(js_data.trait_1);
+    chart.xAxis.axisLabelDistance(11);
+    chart.yAxis.axisLabel(js_data.trait_2);
+    chart.yAxis.axisLabelDistance(11);
+    //
+    xmin = d3.min(js_data.data[0]);
+    xmax = d3.max(js_data.data[0]);
+    xrange = xmax - xmin;
+    ymin = d3.min(js_data.data[1]);
+    ymax = d3.max(js_data.data[1]);
+    yrange = ymax - ymin;
+    chart.xDomain([xmin - xrange/10, xmax + xrange/10]);
+    chart.yDomain([ymin - yrange/10, ymax + yrange/10]);
+    chart.xAxis.tickFormat(d3.format(checkformat(xrange)));
+    chart.yAxis.tickFormat(d3.format(checkformat(yrange)));
+    //
+    chart.tooltip.contentGenerator(function (obj) {
+        return '<b style="font-size: 18px">(' + obj.point.x + ', ' + obj.point.y + ')</b>';
+    });
+    //
+    d3.select('#scatterplot2 svg').datum(nv.log(getdata())).call(chart);
+    nv.utils.windowResize(chart.update);
+    return chart;
 });
 
 function getdata () {
-	var data = [];
-	data.push({
-			values: [],
-			slope: js_data.slope,
-			intercept: js_data.intercept
-		});
-	for (j = 0; j < js_data.data[0].length; j++) {
-		data[0].values.push({
-			x: js_data.data[0][j],
-			y: js_data.data[1][j],
-			size: 10,
-			shape: 'circle'
-		});
-	}
-	return data;
+    var data = [];
+    data.push({
+            values: [],
+            slope: js_data.slope,
+            intercept: js_data.intercept
+        });
+    for (j = 0; j < js_data.data[0].length; j++) {
+        data[0].values.push({
+            x: js_data.data[0][j],
+            y: js_data.data[1][j],
+            size: 10,
+            shape: 'circle'
+        });
+    }
+    return data;
 }
-	
-function randomData(groups, points) {
-	var data = [],
-		shapes = ['circle'],
-		random = d3.random.normal();
-	for (i = 0; i < groups; i++) {
-		data.push({
-			key: 'Group ' + i,
-			values: [],
-			slope: Math.random() - .01,
-			intercept: Math.random() - .5
-		});
-		for (j = 0; j < points; j++) {
-			data[i].values.push({
-				x: random(),
-				y: random(),
-				size: Math.random(),
-				shape: shapes[j % shapes.length]
-			});
-		}
-	}
-	return data;
+    
+function checkformat (range) {
+    cell = range / 10.0;
+    if (cell >= 1) {
+        return ",r";
+    } else {
+        cell = -Math.log(cell);
+        n = cell.toString().split(".")[0].length;
+        return ",.0" + n + "f";
+    }
 }
diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html
index ec55ebeb..9ca3d63a 100644
--- a/wqflask/wqflask/templates/base.html
+++ b/wqflask/wqflask/templates/base.html
@@ -144,7 +144,7 @@
         <p>
           Development and source code on <a href="https://github.com/genenetwork/">github</a> with <a href="https://github.com/genenetwork/genenetwork2/issues">issue tracker</a> and <a href="https://github.com/genenetwork/genenetwork2/blob/master/README.md">documentation</a>. Join the <a href="http://listserv.uthsc.edu/mailman/listinfo/genenetwork-dev">mailing list</a> and find us on <a href="https://webchat.freenode.net/">IRC</a> (#genenetwork channel).
         {% if version: %}
-        <p><small>GeneNetwork v{{ version }}</small></p>
+        <p><small>GeneNetwork {{ version }}</small></p>
         {% endif %}
 
       </div>
diff --git a/wqflask/wqflask/templates/corr_scatterplot.html b/wqflask/wqflask/templates/corr_scatterplot.html
index 4464025e..05891c95 100644
--- a/wqflask/wqflask/templates/corr_scatterplot.html
+++ b/wqflask/wqflask/templates/corr_scatterplot.html
@@ -53,16 +53,6 @@
 

 <h2>Correlation Scatterplot</h2>

 

-<table class="table">

-	<tbody>

-		<tr><th class="text-right">num overlap</th>	<td>{{jsdata.num_overlap}}</td></tr>

-		<tr><th class="text-right">slope</th>		<td>{{jsdata.slope}}</td></tr>

-		<tr><th class="text-right">intercept</th>	<td>{{jsdata.intercept}}</td></tr>

-		<tr><th class="text-right">r value</th>		<td>{{jsdata.r_value}}</td></tr>

-		<tr><th class="text-right">p value</th>		<td>{{jsdata.p_value}}</td></tr>

-	</tbody>

-</table>

-

 <!--

 <table class="table">

 	<tr>

@@ -85,12 +75,12 @@
 -->

 

 <div id="scatterplot2">

-	<svg style="width: 1000px; height: 800px;"></svg>

+	<svg style="width: 1000px; height: 800px; margin-left: 10px;"></svg>

 </div>

 

 <br>

 

-<div style="margin-left: 75px;">

+<div style="margin-left: 80px;">

 

     {% if trait_1.dataset.type == "ProbeSet" %}

 		<div>

@@ -144,6 +134,18 @@
 	

 </div>

 

+<br>

+

+<table class="table table-hover table-striped table-bordered" style="width: 250px; margin-left: 80px;">

+	<tbody>

+		<tr><th>Number</th>		<td>{{jsdata.num_overlap}}</td></tr>

+		<tr><th>Slope</th>		<td>{{'%0.3f' % jsdata.slope}}</td></tr>

+		<tr><th>Intercept</th>	<td>{{'%0.3f' % jsdata.intercept}}</td></tr>

+		<tr><th>R value</th>	<td>{{'%0.3f' % jsdata.r_value}}</td></tr>

+		<tr><th>P value</th>	<td>{{'%0.3e' % jsdata.p_value}}</td></tr>

+	</tbody>

+</table>

+

 {% endblock %}

 

 {% block js %}

diff --git a/wqflask/wqflask/templates/error.html b/wqflask/wqflask/templates/error.html
index c707a4fc..2f1e06fa 100644
--- a/wqflask/wqflask/templates/error.html
+++ b/wqflask/wqflask/templates/error.html
@@ -35,7 +35,7 @@
   </p>
 
   <pre>
-    GeneNetwork v{{ version }} {{ stack[0] }}
+    GeneNetwork {{ version }} {{ stack[0] }}
     {{ message }} (error)
     {{ stack[-3] }}
     {{ stack[-2] }}
@@ -50,7 +50,7 @@
   <a href="#Stack" class="btn btn-default" data-toggle="collapse">Toggle full stack trace</a>
   <div id="Stack" class="collapse">
     <pre>
-      GeneNetwork v{{ version }} {% for line in stack %} {{ line }}
+      GeneNetwork {{ version }} {% for line in stack %} {{ line }}
       {% endfor %}
     </pre>
   </div>
diff --git a/wqflask/wqflask/templates/marker_regression_gn1.html b/wqflask/wqflask/templates/marker_regression_gn1.html
index 4cea51b2..65debd10 100644
--- a/wqflask/wqflask/templates/marker_regression_gn1.html
+++ b/wqflask/wqflask/templates/marker_regression_gn1.html
@@ -14,7 +14,7 @@
         <input type="hidden" name="temp_uuid" value="{{ temp_uuid }}">
         <input type="hidden" name="trait_id" value="{{ this_trait.name }}">
         <input type="hidden" name="dataset" value="{{ dataset.name }}">
-        {% if mapping_method == "reaper" %}
+        {% if mapping_method == "reaper" or mapping_method == "rqtl_geno" %}
         <input type="hidden" name="genofile" value="{{ dataset.group.genofile }}">
         {% endif %}
         <input type="hidden" name="method" value="{{ mapping_method }}">
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index 56b6cc62..5a880647 100644
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -129,6 +129,7 @@
     <script type="text/javascript" src="/static/new/js_external/d3-tip.min.js"></script>
     <script type="text/javascript" src="/static/new/js_external/jstat.min.js"></script>
     <script type="text/javascript" src="/static/new/js_external/shapiro-wilk.js"></script>
+    <script type="text/javascript" src="/static/new/js_external/saveSvgAsPng.js"></script>
     <script type="text/javascript" src="/static/new/javascript/colorbrewer.js"></script>
 
     <script type="text/javascript" src="/static/new/javascript/stats.js"></script>
diff --git a/wqflask/wqflask/templates/show_trait_statistics.html b/wqflask/wqflask/templates/show_trait_statistics.html
index 0a0e9990..ac44b656 100644
--- a/wqflask/wqflask/templates/show_trait_statistics.html
+++ b/wqflask/wqflask/templates/show_trait_statistics.html
@@ -94,12 +94,15 @@
                 <div class="tab-pane" id="probability_plot">
                     <div style="padding: 20px" class="form-horizontal">
                         {% if sample_groups|length != 1 %}
+                        Select Group:
                         <select class="prob_plot_samples_group">
                             {% for group, pretty_group in sample_group_types.items() %}
                                 <option value="{{ group }}">{{ pretty_group }}</option>
                             {% endfor %}
                         </select>
-                        <br><br>
+                        <button type="button" class="btn btn-default" id="down_prob_plot">Export as PNG</button>
+                        <br>
+                        <br>
                         {% endif %}
 
                         <div id="prob_plot_container">
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index 1e831896..f7fcd2d0 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -46,7 +46,7 @@ from wqflask.database import db_session
 from wqflask import model
 
 from utility import Bunch, Struct, after
-from utility.tools import LOG_SQL, LOG_SQLALCHEMY
+from utility.tools import LOG_SQL, LOG_SQL_ALCHEMY
 
 import logging
 from utility.logger import getLogger
@@ -69,7 +69,7 @@ class AnonUser(object):
         if self.cookie:
             logger.debug("already is cookie")
             self.anon_id = verify_cookie(self.cookie)
-            
+
         else:
             logger.debug("creating new cookie")
             self.anon_id, self.cookie = create_signed_cookie()
@@ -78,17 +78,17 @@ class AnonUser(object):
         @after.after_this_request
         def set_cookie(response):
             response.set_cookie(self.cookie_name, self.cookie)
-            
+
     def add_collection(self, new_collection):
         collection_dict = dict(name = new_collection.name,
                                created_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
                                changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
                                num_members = new_collection.num_members,
                                members = new_collection.get_members())
-                               
+
         Redis.set(self.key, json.dumps(collection_dict))
         Redis.expire(self.key, 60 * 60 * 24 * 5)
-            
+
     def delete_collection(self, collection_name):
         existing_collections = self.get_collections()
         updated_collections = []
@@ -106,7 +106,7 @@ class AnonUser(object):
                 updated_collections.append(this_collection)
 
         Redis.set(self.key, json.dumps(updated_collections))
-            
+
     def get_collections(self):
         json_collections = Redis.get(self.key)
         if json_collections == None or json_collections == "None":
@@ -117,7 +117,7 @@ class AnonUser(object):
                 collection['created_timestamp'] = datetime.datetime.strptime(collection['created_timestamp'], '%b %d %Y %I:%M%p')
                 collection['changed_timestamp'] = datetime.datetime.strptime(collection['changed_timestamp'], '%b %d %Y %I:%M%p')
             return collections
-            
+
     def import_traits_to_user(self):
         collections_list = json.loads(Redis.get(self.key))
         for collection in collections_list:
@@ -131,7 +131,7 @@ class AnonUser(object):
                 uc.members = json.dumps(collection['members'])
                 db_session.add(uc)
                 db_session.commit()
-            
+
     def display_num_collections(self):
         """
         Returns the number of collections or a blank string if there are zero.
@@ -217,7 +217,7 @@ class UserSession(object):
         """Actual sqlalchemy record"""
         # Only look it up once if needed, then store it
         try:
-            if LOG_SQLALCHEMY:
+            if LOG_SQL_ALCHEMY:
                 logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG)
 
             # Already did this before
@@ -237,7 +237,7 @@ class UserSession(object):
 def before_request():
     g.user_session = UserSession()
     g.cookie_session = AnonUser()
-    
+
 class UsersManager(object):
     def __init__(self):
         self.users = model.User.query.all()
@@ -541,9 +541,9 @@ class LoginUser(object):
                 import_col = "true"
             else:
                 import_col = "false"
-            
-            #g.cookie_session.import_traits_to_user()           
-                
+
+            #g.cookie_session.import_traits_to_user()
+
             return self.actual_login(user, import_collections=import_col)
 
         else:
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 39f4a686..07151425 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -53,7 +53,7 @@ from wqflask.ctl import ctl_analysis
 #from wqflask.trait_submission import submit_trait
 
 from utility import temp_data
-from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION
+from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,JS_TWITTER_POST_FETCHER_PATH
 from utility.helper_functions import get_species_groups
 
 from base import webqtlFormData
@@ -147,9 +147,8 @@ def tmp_page(img_path):
                             img_base64 = bytesarray )
 
 @app.route("/twitter/<path:filename>")
-def bd_files(filename):
-    bd_path = app.config['TWITTER_POST_FETCHER_JS_PATH']
-    return send_from_directory(bd_path, filename)
+def twitter(filename):
+    return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename)
 
 #@app.route("/data_sharing")
 #def data_sharing_page():