Merge branch 'master' of github.com:genenetwork/genenetwork2

53 files changed, 343 insertions, 76 deletions

diff --git a/bin/test-website b/bin/test-website
index 18994cc5..b693bd60 100755
--- a/bin/test-website
+++ b/bin/test-website
@@ -4,7 +4,7 @@
 USAGE = <<EOT
 This is Mechanical-Rob - an automated web server tester for
                          Genenetwork.org that uses the brilliant
-                         mechanize gem.
+                         mechanize gem with minitest.
 
 To use this software you need to install mechanize. Run it from
 the root of the genenetwork2 source tree as, for example,
@@ -15,6 +15,10 @@ If you are using the small deployment database you can use
 
   ./bin/test-website --skip -n
 
+To run individual tests on localhost you can do
+
+  ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/"
+
 For more information see http://genenetwork.org/
 
 EOT
@@ -36,6 +40,10 @@ opts = OptionParser.new do |o|
       end
   end
 
+  o.on('--all', 'Run all tests') do
+    options[:all] = true
+  end
+
   o.on('-l','--link-checker', 'Check for dead links') do
     options[:link_checker] = true
   end
@@ -84,16 +92,16 @@ require 'minitest/autorun'
 libpath = File.dirname(File.dirname(__FILE__))
 $: << File.join(libpath,'test/lib')
 
-if options[:link_checker]
-  require 'link_checker'
-else
-  require 'main_web_functionality'
+require 'main_web_functionality'
+
+if options[:all] or options[:mapping_test]
+  require 'mapping'
 end
 
-if options[:navigation_test]
-  require 'navigation'
+if options[:all] or options[:link_checker]
+  require 'link_checker'
 end
 
-if options[:mapping_test]
-  require 'mapping'
+if options[:all] or options[:navigation_test]
+  require 'navigation'
 end
diff --git a/doc/Architecture.org b/doc/Architecture.org
index ec56f9a9..c5876196 100644
--- a/doc/Architecture.org
+++ b/doc/Architecture.org
@@ -26,29 +26,27 @@ Findable, Accessible, Interoperable, and Re-usable (FAIR) principles
 (see the Wilkinson Nature paper on [[http://www.nature.com/articles/sdata201618][FAIR Guiding Principles for
 scientific data management and stewardship]]).
 
-With GN2 we are solving these requirements by assigning unique
-identifiers (cryptographic HASH values calculated over immutable data
-content and including that value in the file names or directories) and
-making these identifiers available through web interfaces (e.g.,
-through a REST API). This means that at any point in the future the
-exact same data can be retrieved using a known non-changeable
-identifier (see also
+GeneNetwork (GN2) solves this by assigning unique identifiers
+(cryptographic HASH values calculated over immutable data content),
+including these values in file or directory names, and making them
+available through web interfaces (e.g., through a through a REST
+API). This means that at any point in the future the exact same data
+can be retrieved using a known non-changeable identifier (see also
 https://github.com/pjotrp/genenetwork2/blob/staging/doc/submit-data.org).
 
 Synchronisation, integrity checking and backups become trivial using
 these HASH values, even for very large datasets. Since everything is
 managed at the file system level we can also use Unix authorisation
-systems. HIPAA compliancy is achieved by using HASH values and
+systems. HIPAA compliancy is achieved by using HASH references and
 bringing the software into the controlled HIPAA environment.
 
-In the context of GeneNetwork we are using git and github for version
-control of software source code
-(https://github.com/genenetwork/). Software can be treated just like
-data, i.e., git uses HASH identifiers to retrieve specific versions of
-source. I.e., versions of source code are identifiable and retrievable
-and can be matched with data into an analysis pipeline. The
-combination of software and data, again, makes a unique HASH value
-which identifies the analysis pipe-line.
+In the context of GeneNetwork we are using git for version control of
+software source code (https://github.com/genenetwork/). Software can
+be treated just like data, i.e., git uses HASH identifiers to retrieve
+specific versions of source. I.e., versions of source code are
+identifiable and retrievable and can be matched with data into an
+analysis pipeline. The combination of software and data, again, makes
+a unique HASH value which identifies the analysis pipeline.
 
 For combining runnable software and data into an analysis pipeline we
 use GNU Guix which, yet again, turns everything into a unique HASH
@@ -68,12 +66,12 @@ deployment (described in JOSS paper). We also use automated test tools
 (Ruby mechanize) for integration testing of the web services and we
 use unit testing of all backend services. All our software source code
 is published as `free and open source software' (FOSS) which means
-that anyone can view code on github, comment on it, or even
-contribute. GeneNetwork is becoming increasingly modular and has a
-growing number of contributers who, in principle, abide by the THE
-SMALL TOOLS MANIFESTO FOR BIOINFORMATICS which we wrote up
-(https://github.com/pjotrp/bioinformatics) and was signed by 51
-bioinformaticians.
+that anyone can view code on github, comment on, or even contribute
+to. GeneNetwork is becoming increasingly modular and has a growing
+number of contributers who subscribe to the principles of THE SMALL
+TOOLS MANIFESTO FOR BIOINFORMATICS
+(https://github.com/pjotrp/bioinformatics) which we drew up and was
+signed by over fifty bioinformaticians.
 
 * Webserver
 
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 9dddd0ad..547478df 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -7,6 +7,7 @@
 # e.g.
 #
 #   env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2
+#   env LOG_LEVEL=DEBUG ./bin/genenetwork2 ~/gn2_settings.py
 #
 # Note also that in the near future we will additionally fetch
 # settings from a JSON file
@@ -54,6 +55,7 @@ USE_GN_SERVER   = 'False'   # Use GN_SERVER SQL calls
 HOME=os.environ['HOME']
 LOGFILE = HOME+"/genenetwork2.log"
 GENENETWORK_FILES = HOME+"/gn2_data"  # base dir for all static data files
+LOCAL_PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files
 
 # ---- GN2 Executables
 PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read())
diff --git a/test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json b/test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json
new file mode 100644
index 00000000..15428553
--- /dev/null
+++ b/test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json
@@ -0,0 +1,142 @@
+{
+    "temp_uuid" : "unknown",
+"maf" : "0.01",
+"control_marker" : "mCV24506226",
+"dataset" : "HC_M2_0606_P",
+ "do_control" : "false",
+ "maf" : "0.01",
+ "manhattan_plot" : "False",
+ "mapmethod_rqtl_geno" : "em",
+ "mapmodel_rqtl_geno" : "normal",
+ "method" : "rqtl_geno",
+ "num_bootstrap" : "2000",
+ "num_perm" : "",
+ "pair_scan" : "false",
+ "suggestive" : "0",
+ "trait_id" : "1433387_at",
+ "value:129S1/SvImJ" : "6.920",
+ "value:A/J" : "7.437",
+ "value:AKR/J" : "7.550",
+ "value:B6D2F1" : "7.742",
+ "value:BALB/cByJ" : "6.613",
+ "value:BALB/cJ" : "6.664",
+ "value:BXD1" : "6.749",
+ "value:BXD100" : "x",
+ "value:BXD101" : "x",
+ "value:BXD102" : "x",
+ "value:BXD103" : "x",
+ "value:BXD11" : "7.398",
+ "value:BXD12" : "7.073",
+ "value:BXD13" : "8.191",
+ "value:BXD14" : "x",
+ "value:BXD15" : "7.406",
+ "value:BXD16" : "6.890",
+ "value:BXD18" : "x",
+ "value:BXD19" : "6.980",
+ "value:BXD2" : "7.248",
+ "value:BXD20" : "7.554",
+ "value:BXD21" : "7.316",
+ "value:BXD22" : "7.327",
+ "value:BXD23" : "7.605",
+ "value:BXD24" : "7.306",
+ "value:BXD24a" : "x",
+ "value:BXD25" : "x",
+ "value:BXD27" : "7.800",
+ "value:BXD28" : "7.023",
+ "value:BXD29" : "7.518",
+ "value:BXD30" : "x",
+ "value:BXD31" : "7.301",
+ "value:BXD32" : "7.161",
+ "value:BXD33" : "7.140",
+ "value:BXD34" : "8.180",
+ "value:BXD35" : "x",
+ "value:BXD36" : "x",
+ "value:BXD37" : "x",
+ "value:BXD38" : "7.350",
+ "value:BXD39" : "6.976",
+ "value:BXD40" : "7.411",
+ "value:BXD41" : "x",
+ "value:BXD42" : "7.050",
+ "value:BXD43" : "6.866",
+ "value:BXD44" : "7.246",
+ "value:BXD45" : "7.576",
+ "value:BXD48" : "7.108",
+ "value:BXD48a" : "7.192",
+ "value:BXD49" : "x",
+ "value:BXD5" : "6.771",
+ "value:BXD50" : "7.528",
+ "value:BXD51" : "7.408",
+ "value:BXD52" : "x",
+ "value:BXD53" : "x",
+ "value:BXD54" : "x",
+ "value:BXD55" : "6.720",
+ "value:BXD56" : "x",
+ "value:BXD59" : "x",
+ "value:BXD6" : "7.372",
+ "value:BXD60" : "7.178",
+ "value:BXD61" : "7.198",
+ "value:BXD62" : "7.618",
+ "value:BXD63" : "7.076",
+ "value:BXD64" : "6.576",
+ "value:BXD65" : "7.046",
+ "value:BXD65a" : "7.046",
+ "value:BXD65b" : "7.385",
+ "value:BXD66" : "7.108",
+ "value:BXD67" : "7.342",
+ "value:BXD68" : "7.668",
+ "value:BXD69" : "7.310",
+ "value:BXD70" : "7.278",
+ "value:BXD71" : "x",
+ "value:BXD72" : "x",
+ "value:BXD73" : "7.342",
+ "value:BXD73a" : "7.841",
+ "value:BXD74" : "7.298",
+ "value:BXD75" : "7.133",
+ "value:BXD76" : "7.448",
+ "value:BXD77" : "7.014",
+ "value:BXD78" : "x",
+ "value:BXD79" : "7.916",
+ "value:BXD8" : "7.148",
+ "value:BXD81" : "x",
+ "value:BXD83" : "7.322",
+ "value:BXD84" : "7.298",
+ "value:BXD85" : "7.243",
+ "value:BXD86" : "7.719",
+ "value:BXD87" : "7.145",
+ "value:BXD88" : "x",
+ "value:BXD89" : "7.162",
+ "value:BXD9" : "6.708",
+ "value:BXD90" : "7.497",
+ "value:BXD91" : "x",
+ "value:BXD93" : "7.558",
+ "value:BXD94" : "6.817",
+ "value:BXD95" : "x",
+ "value:BXD98" : "7.753",
+ "value:BXD99" : "8.076",
+ "value:C3H/HeJ" : "7.536",
+ "value:C57BL/6ByJ" : "6.940",
+ "value:C57BL/6J" : "7.309",
+ "value:CAST/EiJ" : "8.796",
+ "value:CXB1" : "6.430",
+ "value:CXB10" : "6.958",
+ "value:CXB11" : "6.182",
+ "value:CXB12" : "6.432",
+ "value:CXB13" : "6.388",
+ "value:CXB2" : "7.687",
+ "value:CXB3" : "6.312",
+ "value:CXB4" : "6.453",
+ "value:CXB5" : "6.959",
+ "value:CXB6" : "7.022",
+ "value:CXB7" : "6.334",
+ "value:CXB8" : "6.790",
+ "value:CXB9" : "6.545",
+ "value:D2B6F1" : "7.615",
+ "value:DBA/2J" : "7.816",
+ "value:KK/HlJ" : "8.035",
+ "value:LG/J" : "7.594",
+ "value:NOD/ShiLtJ" : "8.220",
+ "value:NZO/HlLtJ" : "7.629",
+ "value:PWD/PhJ" : "6.320",
+ "value:PWK/PhJ" : "7.235",
+ "value:WSB/EiJ" : "7.650"
+}
diff --git a/test/lib/gntest.rb b/test/lib/gntest.rb
new file mode 100644
index 00000000..865ef51d
--- /dev/null
+++ b/test/lib/gntest.rb
@@ -0,0 +1,5 @@
+require 'minitest/autorun'
+require 'mechanize'
+
+# ---- Use some default parameters if not set
+$host = "http://localhost:5003" if !$host
diff --git a/test/lib/mapping.rb b/test/lib/mapping.rb
index 08b6b5da..4c8ff632 100644
--- a/test/lib/mapping.rb
+++ b/test/lib/mapping.rb
@@ -1,5 +1,6 @@
 # In these tests we navigate from the main page to a specific trait then hit the different mapping tool buttons (In this case pylMM and r/qtl) followed by computing the results (marker regressions).
 
+require 'gntest'
 
 class MappingTest
 end
@@ -11,41 +12,46 @@ describe MappingTest do
   end
 
   describe MappingTest do
-    it "pyLMM mapping tool selection" do
-      url = $host+'/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P'
-      page = @agent.get(url)
-      # get the form
-      # form = @agent.page.form_with(:name => "my-form")
-      # get the button you want from the form
-      # page.forms.each do |form|
-      #   p form
-      # end
-      # button = page.button_with(:value => "pylmm_compute")
-      # submit the form using that button
-      # agent.submit(form, button)
-      # link = page.link_with(text: 'pyLMM')
-      # page = link.click
-      # puts page.uri
-      # link = page.link_with(text: 'Compute')
-      # page = link.click
-      # puts page.uri
-      # probe_link.uri.to_s.must_equal "/marker_regression"
+    it "pylmm mapping tool selection" do
+      url = $host+'/marker_regression'
+
+      json = JSON::load(File.read('test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json'))
+      json["method"] = "pylmm"
+      # p json
+      page = @agent.post(URI.encode(url), json)
+      # Unpacking the page is slow - but the run is enough as a test
+      # form = page.forms_with("marker_regression")[0]
+      # form.fields.select { |fld| fld.name == 'dataset' }.first.value.must_equal 'HC_M2_0606_P'
+      # form.fields.select { |fld| fld.name == 'value:BXD1' }.first.value.must_equal '6.749'
     end
   end
 
-end
-
-describe MappingTest do
+  describe MappingTest do
     it "R/qtl mapping tool selection" do
-      break if $options[:skip_broken]
-      page = @agent.get($host+'/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P')
-      link = page.link_with(text: 'R/qtl')
-      page = link.click
-      puts page.uri
-      form.field_with(:name => 'Methods').options[2].select
-      link = page.link_with(text: 'Compute')
-      page = link.click
-      puts page.uri
-      probe_link.uri.to_s.must_equal "/marker_regression"
+      url = $host+'/marker_regression' # ?trait_id=1435395_s_at&dataset=HC_M2_0606_P'
+
+      json = JSON::load(File.read('test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json'))
+      # p json
+      page = @agent.post(URI.encode(url),
+                         json,
+                         ({'Content-Type' => 'application/x-www-form-urlencoded'}))
+      form = page.forms_with("marker_regression")[0]
+      form.fields.select { |fld| fld.name == 'dataset' }.first.value.must_equal 'HC_M2_0606_P'
+      form.fields.select { |fld| fld.name == 'value:BXD1' }.first.value.must_equal '6.749'
     end
+  end
+
+  describe MappingTest do
+    it "CIM mapping tool selection (using reaper)" do
+      url = $host+'/marker_regression'
+
+      json = JSON::load(File.read('test/data/input/mapping/1435395_s_at_HC_M2_0606_P.json'))
+      json["method"] = "reaper"
+      page = @agent.post(URI.encode(url), json)
+      form = page.forms_with("marker_regression")[0]
+      form.fields.select { |fld| fld.name == 'dataset' }.first.value.must_equal 'HC_M2_0606_P'
+      form.fields.select { |fld| fld.name == 'value:BXD1' }.first.value.must_equal '6.749'
+    end
+  end
+
 end
diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py
index 900e050c..32032ba7 100644
--- a/wqflask/base/trait.py
+++ b/wqflask/base/trait.py
@@ -363,7 +363,7 @@ class GeneralTrait(object):
                             FROM
                                     Homologene, Species, InbredSet
                             WHERE
-                                    Homologene.GeneId =%s AND
+                                    Homologene.GeneId ='%s' AND
                                     InbredSet.Name = '%s' AND
                                     InbredSet.SpeciesId = Species.Id AND
                                     Species.TaxonomyId = Homologene.TaxonomyId
diff --git a/wqflask/maintenance/__init__.py b/wqflask/maintenance/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/wqflask/maintenance/__init__.py
diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py
index ddc0ea82..b873e16f 100644
--- a/wqflask/utility/logger.py
+++ b/wqflask/utility/logger.py
@@ -31,6 +31,7 @@ import string
 from inspect import isfunction
 from pprint import pformat as pf
 from inspect import stack
+import datetime
 
 from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL, LOG_FORMAT
 
@@ -66,7 +67,10 @@ LOG_LEVEL_DEBUG (NYI).
 
     def error(self,*args):
         """Call logging.error for multiple args"""
-        self.collect(self.logger.error,*args)
+        now = datetime.datetime.utcnow()
+        time_str = now.strftime('%H:%M:%S UTC %Y%m%d')
+        l = [time_str]+list(args)
+        self.collect(self.logger.error,*l)
 
     def infof(self,*args):
         """Call logging.info for multiple args lazily"""
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index f2a2eb8c..f28407b3 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -41,6 +41,9 @@ from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMM
 from utility.external import shell
 from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR
 
+import utility.logger
+logger = utility.logger.getLogger(__name__ )
+
 class MarkerRegression(object):
 
     def __init__(self, start_vars, temp_uuid):
@@ -197,6 +200,7 @@ class MarkerRegression(object):
 
             self.control_marker = start_vars['control_marker']
             self.do_control = start_vars['do_control']
+            logger.info("Running qtlreaper")
             results = self.gen_reaper_results()
         elif self.mapping_method == "plink":
             results = self.run_plink()
diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
index 9cef3cec..bc147f75 100644
--- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py
+++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
@@ -44,6 +44,9 @@ from utility import Plot
 from wqflask.interval_analyst import GeneUtil
 from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR, GENERATED_IMAGE_DIR
 
+import utility.logger
+logger = utility.logger.getLogger(__name__ )
+
 #########################################
 #      Inteval Mapping Plot Page
 #########################################
@@ -159,6 +162,7 @@ class MarkerRegression(object):
 
         #if not self.openMysql():
         #    return
+        logger.info("Running qtlreaper")
 
         #helper_functions.get_species_dataset_trait(self, start_vars)
 
@@ -211,7 +215,7 @@ class MarkerRegression(object):
             self.plotScale = start_vars['mapping_scale']
         else:
             self.plotScale = "physic"
-            
+
         self.manhattan_plot = start_vars['manhattan_plot']
 
         if 'permCheck' in start_vars.keys():
@@ -514,13 +518,13 @@ class MarkerRegression(object):
 
             if self.dataset.group.species == "mouse":
                 if self.selectedChr == 20:
-                    chrName = "X" 
+                    chrName = "X"
                 else:
                     chrName = self.selectedChr
                 self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, webqtldatabase, "mouse")
             elif self.dataset.group.species == "rat":
                 if self.selectedChr == 21:
-                    chrName = "X" 
+                    chrName = "X"
                 else:
                     chrName = self.selectedChr
                 self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, webqtldatabase, "rat")
@@ -860,7 +864,7 @@ class MarkerRegression(object):
         BootCoord = []
         i = 0
         startX = xLeftOffset
-     
+
         if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes
             for j, _chr in enumerate(self.genotype):
                 BootCoord.append( [])
@@ -883,8 +887,8 @@ class MarkerRegression(object):
                         else:
                             Xc = startX + (_locus.cM-_chr[0].cM)*plotXScale
                         BootCoord[-1].append([Xc, self.bootResult[i]])
-                    i += 1   
-                    
+                    i += 1
+
         #reduce bootResult
         if self.selectedChr > -1:
             maxBootBar = 80.0
@@ -1851,7 +1855,7 @@ class MarkerRegression(object):
                 if j == 0:
                     canvas.drawLine(startPosX,yZero,startPosX,yZero+40, color=lineColor)
                 startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale
-                
+
             centimorganLabelFont = pid.Font(ttf="verdana", size=18*zoom*1.5, bold=0)
             canvas.drawString("Centimorgans", xLeftOffset + (plotWidth - canvas.stringWidth("Megabases", font=centimorganLabelFont))/2,
                     strYLoc + canvas.fontHeight(MBLabelFont)+ 10*(zoom%2) + 10, font=centimorganLabelFont, color=pid.black)
@@ -2963,7 +2967,7 @@ class MarkerRegression(object):
                     chr_as_int = 20
                 else:
                     chr_as_int = int(theGO["Chromosome"]) - 1
-                    
+
                 geneLength = (float(theGO["TxEnd"]) - float(theGO["TxStart"]))
                 #geneLengthURL = "javascript:centerIntervalMapOnRange2('%s', %f, %f, document.changeViewForm)" % (theGO["Chromosome"], float(theGO["TxStart"])-(geneLength*0.1), float(theGO["TxEnd"])+(geneLength*0.1))
                 geneLengthURL = "javascript:rangeView('%s', %f, %f)" % (theGO["Chromosome"], float(theGO["TxStart"])-(geneLength*0.1), float(theGO["TxEnd"])+(geneLength*0.1))
diff --git a/wqflask/wqflask/static/gif/error/Wild-Type-Mouse.gif b/wqflask/wqflask/static/gif/error/Wild-Type-Mouse.gif
new file mode 100644
index 00000000..2c68b5ee
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/Wild-Type-Mouse.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-aliens-29.gif b/wqflask/wqflask/static/gif/error/animated-gifs-aliens-29.gif
new file mode 100644
index 00000000..e9d38277
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-aliens-29.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-angels-04.gif b/wqflask/wqflask/static/gif/error/animated-gifs-angels-04.gif
new file mode 100644
index 00000000..94e11847
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-angels-04.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-cats-016.gif b/wqflask/wqflask/static/gif/error/animated-gifs-cats-016.gif
new file mode 100644
index 00000000..7e6ec9a3
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-cats-016.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-cats-031.gif b/wqflask/wqflask/static/gif/error/animated-gifs-cats-031.gif
new file mode 100644
index 00000000..af7ef655
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-cats-031.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-cell-phones-03.gif b/wqflask/wqflask/static/gif/error/animated-gifs-cell-phones-03.gif
new file mode 100644
index 00000000..89c79ddf
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-cell-phones-03.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-cell-phones-16.gif b/wqflask/wqflask/static/gif/error/animated-gifs-cell-phones-16.gif
new file mode 100644
index 00000000..7530d180
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-cell-phones-16.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-13.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-13.gif
new file mode 100644
index 00000000..afb05c62
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-13.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-28.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-28.gif
new file mode 100644
index 00000000..f5b4a563
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-28.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-32.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-32.gif
new file mode 100644
index 00000000..7258e594
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-32.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-42.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-42.gif
new file mode 100644
index 00000000..ed1f8722
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-42.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-60.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-60.gif
new file mode 100644
index 00000000..f58d69f1
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-60.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-64.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-64.gif
new file mode 100644
index 00000000..5d5b4fdf
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-64.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-65.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-65.gif
new file mode 100644
index 00000000..b4b10845
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-65.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-72.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-72.gif
new file mode 100644
index 00000000..e60cb4fe
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-72.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-74.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-74.gif
new file mode 100644
index 00000000..bd7b72f3
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-74.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-computers-75.gif b/wqflask/wqflask/static/gif/error/animated-gifs-computers-75.gif
new file mode 100644
index 00000000..916d6b33
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-computers-75.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-construction-sites-038.gif b/wqflask/wqflask/static/gif/error/animated-gifs-construction-sites-038.gif
new file mode 100644
index 00000000..0ec782c4
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-construction-sites-038.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-dogs-04.gif b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-04.gif
new file mode 100644
index 00000000..9515c18a
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-04.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-dogs-14.gif b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-14.gif
new file mode 100644
index 00000000..f1e2e1f5
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-14.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-dogs-18.gif b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-18.gif
new file mode 100644
index 00000000..572849d5
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-18.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-dogs-47.gif b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-47.gif
new file mode 100644
index 00000000..d808c9ee
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-47.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-dogs-50.gif b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-50.gif
new file mode 100644
index 00000000..9865ee45
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-dogs-50.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-lava-lamps-01.gif b/wqflask/wqflask/static/gif/error/animated-gifs-lava-lamps-01.gif
new file mode 100644
index 00000000..ee9c113d
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-lava-lamps-01.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-mice-02.gif b/wqflask/wqflask/static/gif/error/animated-gifs-mice-02.gif
new file mode 100644
index 00000000..5ca2ee5c
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-mice-02.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-mice-09.gif b/wqflask/wqflask/static/gif/error/animated-gifs-mice-09.gif
new file mode 100644
index 00000000..7cb361e4
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-mice-09.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-mice-24.gif b/wqflask/wqflask/static/gif/error/animated-gifs-mice-24.gif
new file mode 100644
index 00000000..96a26450
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-mice-24.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-smileys-063.gif b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-063.gif
new file mode 100644
index 00000000..62de166c
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-063.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-smileys-068.gif b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-068.gif
new file mode 100644
index 00000000..3550e978
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-068.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-smileys-134.gif b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-134.gif
new file mode 100644
index 00000000..954ab614
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-134.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-smileys-211.gif b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-211.gif
new file mode 100644
index 00000000..596174d7
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-211.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-smileys-234.gif b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-234.gif
new file mode 100644
index 00000000..5aba636b
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-smileys-234.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-001.gif b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-001.gif
new file mode 100644
index 00000000..7896ff1f
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-001.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-002.gif b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-002.gif
new file mode 100644
index 00000000..89da6441
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-002.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-005.gif b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-005.gif
new file mode 100644
index 00000000..b7887630
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-005.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-012.gif b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-012.gif
new file mode 100644
index 00000000..f6697d02
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-012.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-056.gif b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-056.gif
new file mode 100644
index 00000000..2b2496a4
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-056.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-059.gif b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-059.gif
new file mode 100644
index 00000000..f2188656
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-059.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-060.gif b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-060.gif
new file mode 100644
index 00000000..aa8f7bd3
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-060.gif
diff --git a/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-069.gif b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-069.gif
new file mode 100644
index 00000000..473212e4
--- /dev/null
+++ b/wqflask/wqflask/static/gif/error/animated-gifs-stickmen-069.gif
diff --git a/wqflask/wqflask/templates/error.html b/wqflask/wqflask/templates/error.html
new file mode 100644
index 00000000..7ab2bf2f
--- /dev/null
+++ b/wqflask/wqflask/templates/error.html
@@ -0,0 +1,61 @@
+{% extends "base.html" %}
+{% block title %}Error: {{message}}{% endblock %}
+{% block content %}
+<!-- Start of body -->
+
+<div class="col-md-8">
+<div class="form-group has-error">
+  <div class="control-label" for="inputError1">
+
+    <img src="/static/gif/error/{{ error_image }}">
+
+    <h1>ERROR</h1>
+
+    <p>
+      This error is not what we wanted to see. Unfortunately errors
+      are part of all software systems and we need to resolve this
+      together.
+    </p>
+    <p>
+      <b>It is important to report this ERROR so we can fix it for everyone</b>.
+    </p>
+
+    <p>
+      Report to the GeneNetwork team by recording the steps you take
+      to reproduce this ERROR. Next to those steps, copy-paste below
+      stack trace, either as
+      a <a href="https://github.com/genenetwork/genenetwork2/issues/new">new
+      issue</a> or E-mail this full page to one of the developers
+      directly.
+    </p>
+  </div>
+
+  <p>
+  (GeneNetwork error: {{message[:128]}})
+  </p>
+
+  <pre>
+    {{ stack[0] }}
+    {{ message }} (error)
+    {{ stack[-3] }}
+    {{ stack[-2] }}
+  </pre>
+
+  <p>
+    To check if this already a known issue, search the
+    <a href="https://github.com/genenetwork/genenetwork2/issues">issue
+      tracker</a>.
+  </p>
+
+  <a href="#Stack" class="btn btn-default" data-toggle="collapse">Toggle full stack trace</a>
+  <div id="Stack" class="collapse">
+    <pre>
+      {% for line in stack %} {{ line }}
+      {% endfor %}
+    </pre>
+  </div>
+</div>
+</div>
+
+
+{% endblock %}
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 41e2c7be..cf316bee 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -4,6 +4,11 @@
 
 from __future__ import absolute_import, division, print_function
 
+import traceback # for error page
+import os        # for error gifs
+import random    # for random error gif
+import datetime  # for errors
+import time      # for errors
 import sys
 import csv
 import xlsxwriter
@@ -26,7 +31,7 @@ import base64
 import array
 import sqlalchemy
 from wqflask import app
-from flask import g, Response, request, render_template, send_from_directory, jsonify, redirect
+from flask import g, Response, request, make_response, render_template, send_from_directory, jsonify, redirect
 from wqflask import search_results
 from wqflask import gsearch
 from wqflask import update_search_results
@@ -58,6 +63,8 @@ from wqflask import user_manager
 from wqflask import collect
 from wqflask.database import db_session
 
+import werkzeug
+
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
 
@@ -82,6 +89,31 @@ def shutdown_session(exception=None):
 #    from wqflask import tracer
 #    tracer.turn_on()
 
+@app.errorhandler(Exception)
+def handle_bad_request(e):
+    err_msg = str(e)
+    logger.error(err_msg)
+    logger.error(request.url)
+    # get the stack trace and send it to the logger
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    logger.error(traceback.format_exc())
+    now = datetime.datetime.utcnow()
+    time_str = now.strftime('%l:%M%p UTC %b %d, %Y')
+    formatted_lines = [request.url + " ("+time_str+")"]+traceback.format_exc().splitlines()
+
+    # Handle random animations
+    # Use a cookie to have one animation on refresh
+    animation = request.cookies.get(err_msg[:64])
+    if not animation:
+        list = [fn for fn in os.listdir("./wqflask/static/gif/error") if fn.endswith(".gif") ]
+        animation = random.choice(list)
+
+    resp = make_response(render_template("error.html",message=err_msg,stack=formatted_lines,error_image=animation))
+
+    # logger.error("Set cookie %s with %s" % (err_msg, animation))
+    resp.set_cookie(err_msg[:64],animation)
+    return resp
+
 @app.route("/")
 def index_page():
     logger.info("Sending index_page")
@@ -395,6 +427,7 @@ def mapping_results_container_page():
 @app.route("/marker_regression", methods=('POST',))
 def marker_regression_page():
     initial_start_vars = request.form
+    logger.debug("Marker regression called with initial_start_vars:", initial_start_vars.items())
     temp_uuid = initial_start_vars['temp_uuid']
     wanted = (
         'trait_id',
@@ -432,11 +465,11 @@ def marker_regression_page():
         'mapmethod_rqtl_geno',
         'mapmodel_rqtl_geno'
     )
-    logger.info("Marker regression called with initial_start_vars:", initial_start_vars)
     start_vars = {}
     for key, value in initial_start_vars.iteritems():
         if key in wanted or key.startswith(('value:')):
             start_vars[key] = value
+    logger.debug("Marker regression called with start_vars:", start_vars)
 
     version = "v3"
     key = "marker_regression:{}:".format(version) + json.dumps(start_vars, sort_keys=True)
@@ -546,7 +579,7 @@ def network_graph_page():
         return render_template("network_graph.html", **template_vars.__dict__)
     else:
         return render_template("empty_collection.html", **{'tool':'Network Graph'})
-    
+
 @app.route("/corr_compute", methods=('POST',))
 def corr_compute_page():
     logger.info("In corr_compute, request.form is:", pf(request.form))