Merge pull request #180 from pjotrp/zsloan

Support maintenance scripts

14 files changed, 430 insertions, 61 deletions

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index d3bf3299..3a8c3ff4 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -1,6 +1,14 @@
 #! /bin/bash
 #
-# This will run the GN2 server (with default settings if none supplied).
+# This will run the GN2 server (with default settings if none supplied). Pass in
+# your own settings file, e.g.
+#
+#   ./bin/genenetwork2 ~/my_settings.py
+#
+# To run a maintenance script with settings (instead of the webserver) add that with
+# a -c switch, e.g.
+#
+#   ./bin/genenetwork2 ~/my_settings.py -c ./wqflask/maintenance/gen_select_dataset.py
 #
 # Environment settings can be used to preconfigure as well as a
 # settings.py file.
@@ -18,7 +26,12 @@ echo $GN2_BASE_PATH
 
 # Handle settings parameter
 settings=$1
-if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi
+if [ -z $settings ]; then
+    # get default
+    settings=$GN2_BASE_PATH/etc/default_settings.py
+else
+    shift
+fi
 if [ ! -e $settings ]; then
     echo "ERROR: can not locate settings file - pass it in the command line"
     exit 1
@@ -32,7 +45,15 @@ export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH
 if [ -z $TEMPDIR ]; then
     TEMPDIR="/tmp"
 fi
-                         
+
+# Now handle command parameter -c
+if [ $1 = '-c' ] ; then
+    echo PYTHONPATH=$PYTHONPATH
+    echo RUNNING COMMAND $2
+    /usr/bin/env python $2
+    exit 0
+fi
+
 echo "Starting the redis server:"
 echo -n "dir $TEMPDIR
 dbfilename gn2.rdb
diff --git a/doc/Architecture.org b/doc/Architecture.org
index ed19889c..04e05e40 100644
--- a/doc/Architecture.org
+++ b/doc/Architecture.org
@@ -1,19 +1,20 @@
-* GeneNetwork Architecture
-
 #+TITLE: Installing GeneNetwork services
 
 * Table of Contents                                                     :TOC:
- - [[#genenetwork-architecture][GeneNetwork Architecture]]
-   - [[#introduction][Introduction]]
-   - [[#webserver][Webserver]]
-   - [[#gnserver-rest][GnServer (REST)]]
+ - [[#introduction][Introduction]]
+ - [[#webserver][Webserver]]
+ - [[#gnserver-rest][GnServer (REST)]]
+ - [[#gnexec][GnExec]]
+ - [[#database][Database]]
+   - [[#phenotypes][Phenotypes]]
+   - [[#genotypes][Genotypes]]
 
-** Introduction
+* Introduction
 
 This document describes the architecture of GN2. Because GN2 is
 evolving, only a high-level overview is given here.
 
-** Webserver
+* Webserver
 
 The main [[https://github.com/genenetwork/genenetwork2][GN2 webserver]] is built on [[http://flask.pocoo.org/][Python flask]] and this GN2 source
 code can be found on [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask][github]] in the wqflask directory. The routing
@@ -44,7 +45,7 @@ Consortium M430v2 (Jun06) PDNN to find all records with MEAN between
 15 and 16 and with LRS between 23 and 46.'. Then the results are added
 to a table which is displayed using a JS [[https://datatables.net/][DataTable container]].
 
-** GnServer (REST)
+* GnServer (REST)
 
 The [[https://github.com/genenetwork/gn_server][GnServer REST API]] is built on high performance [[http://elixir-lang.org/][Elixir]] with [[https://github.com/falood/maru][Maru]].
 Mainly the GnServer serves JSON requests, for example to fetch data
@@ -52,4 +53,118 @@ from the database. To get the menu data in YAML you can do something like
 
 : curl localhost:8880/int/menu/main.json|ruby extra/json2yaml.rb
 
-(json2yaml.rb is in the gn_server repo).
+(json2yaml.rb is in the gn_server repo). For the current API definition
+see [[https://github.com/genenetwork/gn_server/doc/API.md][GnServer REST API]] documentation.
+
+* GnExec
+
+GnExec, also written in Elixir, executes commands using a separate
+daemon.
+
+* Database
+** Phenotypes
+
+Phenotypes are stored in the SQL database.  For what happens at the
+database level see [[database.org]]. A test database can be downloaded -
+see the installation [[./README.org][instructions]].
+
+** Genotypes
+
+Genotypes are stored in genotype files. These are part of the GNU Guix
+distribution, see the installation [[./README.org][instructions]]. Genotype files are
+currently in GN1 format, and will be aligned with the [[http://kbroman.org/qtl2/pages/sampledata.html][R/qtl2 formats]].
+
+GN1-style (still default GN2) for the stored file BXD.geno:
+
+#+begin_src js
+@name:BXD
+@type:riset
+@mat:B
+@pat:D
+@het:H
+@unk:U
+Chr Locus cM  Mb  BXD1  BXD2  BXD5  BXD6  BXD8  BXD9  BXD11 BXD12 BXD13 BXD14 BX
+D15 BXD16 BXD18 BXD19 BXD20 BXD21 BXD22 BXD23 BXD24a  BXD24 BXD25 BXD27 BXD28 BX
+D29 BXD30 BXD31 BXD32 BXD33 BXD34 BXD35 BXD36 BXD37 BXD38 BXD39 BXD40 BXD41 BXD4
+2 BXD43 BXD44 BXD45 BXD48 BXD49 BXD50 BXD51 BXD52 BXD53 BXD54 BXD55 BXD56 BXD59
+BXD60 BXD61 BXD62 BXD63 BXD64 BXD65 BXD66 BXD67 BXD68 BXD69 BXD70 BXD71 BXD72 BX
+D73 BXD74 BXD75 BXD76 BXD77 BXD78 BXD79 BXD80 BXD81 BXD83 BXD84 BXD85 BXD86 BXD8
+7 BXD88 BXD89 BXD90 BXD91 BXD92 BXD93 BXD94 BXD95 BXD96 BXD97 BXD98 BXD99 BXD100
+  BXD101  BXD102  BXD103
+1 rs6269442 0.0 3.482275  B B D D D B B D B B D D B D D D D B B B D B D D B B B
+B B B B B B D B D B B D B B H H B D B B H H B B D D D D D B B H B B B B D B D B
+D D D D D H B D D B D B B D D B D D B B B B B B B D
+1 rs6365999 0.0 4.811062  B B D D D B B D B B D D B D D D D B B B D B D D B B B
+B B B B B B D B D B B D B B H H B D B B H H B B D D D D D B B H B B B B D B D B
+D D D D D H B D D B D B B D D B D D B B B B B B U D
+...
+#+end_src
+
+and, for example, in the method run_rqtl_geno this file gets
+loaded. For GnServer, however, we only want to deal with standardized
+R/qtl formatted data, so with gn_extra we convert the original format
+into R/qtl format with geno2rqtl with one adaptation: the geno table
+is transposed so now becomes
+
+#+begin_src js
+marker,BXD1,BXD2,BXD5,BXD6,BXD8,BXD9,BXD11,BXD12,BXD13,BXD14,BXD15,BXD16,BXD18,BXD19,BXD20,BXD21,BXD22,BXD23,BXD24a,BXD24,BXD25,BXD27,BXD28,BXD29,BXD30,BXD31,BXD32,BXD33,BXD34,BXD35,BXD36,BXD37,BXD38,BXD39,BXD40,BXD41,BXD42,BXD43,BXD44,BXD45,BXD48,BXD49,BXD50,BXD51,BXD52,BXD53,BXD54,BXD55,BXD56,BXD59,BXD60,BXD61,BXD62,BXD63,BXD64,BXD65,BXD66,BXD67,BXD68,BXD69,BXD70,BXD71,BXD72,BXD73,BXD74,BXD75,BXD76,BXD77,BXD78,BXD79,BXD80,BXD81,BXD83,BXD84,BXD85,BXD86,BXD87,BXD88,BXD89,BXD90,BXD91,BXD92,BXD93,BXD94,BXD95,BXD96,BXD97,BXD98,BXD99,BXD100,BXD101,BXD102,BXD103
+1,B,B,D,D,D,B,B,D,B,B,D,D,B,D,D,D,D,B,B,B,D,B,D,D,B,B,B,B,B,B,B,B,B,D,B,D,B,B,D,B,B,H,H,B,D,B,B,H,H,B,B,D,D,D,D,D,B,B,H,B,B,B,B,D,B,D,B,D,D,D,D,D,H,B,D,D,B,D,B,B,D,D,B,D,D,B,B,B,B,B,B,B,D
+2,B,B,D,D,D,B,B,D,B,B,D,D,B,D,D,D,D,B,B,B,D,B,D,D,B,B,B,B,B,B,B,B,B,D,B,D,B,B,D,B,B,H,H,B,D,B,B,H,H,B,B,D,D,D,D,D,B,B,H,B,B,B,B,D,B,D,B,D,D,D,D,D,H,B,D,D,B,D,B,B,D,D,B,D,D,B,B,B,B,B,B,U,D
+3,B,B,D,D,D,B,B,D,B,B,D,D,B,D,D,D,D,B,B,B,D,B,D,D,B,B,B,B,B,B,B,B,B,D,B,D,B,D,D,B,B,H,H,B,B,B,B,H,H,B,B,D,D,D,D,B,B,B,H,B,B,B,B,D,B,D,B,D,D,D,D,D,H,B,D,D,B,D,B,B,D,D,B,D,D,B,B,B,B,B,B,U,D
+...
+#+end_src js
+
+i.e. individuals are columns and markers are rows. Alternatively it could look like
+
+#+begin_src js
+marker,BXD1,BXD2,BXD5,BXD6,BXD8,BXD9,BXD11,BXD12,BXD13,BXD14,BXD15,BXD16,BXD18,BXD19,BXD20,BXD21,BXD22,BXD23,BXD24a,BXD24,BXD25,BXD27,BXD28,BXD29,BXD30,BXD31,BXD32,BXD33,BXD34,BXD35,BXD36,BXD37,BXD38,BXD39,BXD40,BXD41,BXD42,BXD43,BXD44,BXD45,BXD48,BXD49,BXD50,BXD51,BXD52,BXD53,BXD54,BXD55,BXD56,BXD59,BXD60,BXD61,BXD62,BXD63,BXD64,BXD65,BXD66,BXD67,BXD68,BXD69,BXD70,BXD71,BXD72,BXD73,BXD74,BXD75,BXD76,BXD77,BXD78,BXD79,BXD80,BXD81,BXD83,BXD84,BXD85,BXD86,BXD87,BXD88,BXD89,BXD90,BXD91,BXD92,BXD93,BXD94,BXD95,BXD96,BXD97,BXD98,BXD99,BXD100,BXD101,BXD102,BXD103
+rs6269442,B,B,D,D,D,B,B,D,B,B,D,D,B,D,D,D,D,B,B,B,D,B,D,D,B,B,B,B,B,B,B,B,B,D,B,D,B,B,D,B,B,H,H,B,D,B,B,H,H,B,B,D,D,D,D,D,B,B,H,B,B,B,B,D,B,D,B,D,D,D,D,D,H,B,D,D,B,D,B,B,D,D,B,D,D,B,B,B,B,B,B,B,D
+rs6365999,B,B,D,D,D,B,B,D,B,B,D,D,B,D,D,D,D,B,B,B,D,B,D,D,B,B,B,B,B,B,B,B,B,D,B,D,B,B,D,B,B,H,H,B,D,B,B,H,H,B,B,D,D,D,D,D,B,B,H,B,B,B,B,D,B,D,B,D,D,D,D,D,H,B,D,D,B,D,B,B,D,D,B,D,D,B,B,B,B,B,B,U,D
+rs6376963,B,B,D,D,D,B,B,D,B,B,D,D,B,D,D,D,D,B,B,B,D,B,D,D,B,B,B,B,B,B,B,B,B,D,B,D,B,D,D,B,B,H,H,B,B,B,B,H,H,B,B,D,D,D,D,B,B,B,H,B,B,B,B,D,B,D,B,D,D,D,D,D,H,B,D,D,B,D,B,B,D,D,B,D,D,B,B,B,B,B,B,U,D
+#+end_src js
+
+This is also the format provided by R/qtl in
+https://github.com/rqtl/qtl2data/tree/master/DO_Recla which we will
+use as the base line for the REST server. In the meta json file the
+genotype data is tagged as transposed:
+
+#+begin_src js
+{
+"description": "DO data from Recla et al. (2014) Mamm Genome 25:211-222",
+"crosstype": "do",
+"geno": "recla_geno.csv",
+"geno_transposed": true,
+"founder_geno": "recla_foundergeno.csv",
+"founder_geno_transposed": true,
+"genotypes": {
+  "1": "1",
+  "2": "2",
+  "3": "3"
+},
+"pheno": "recla_pheno.csv",
+"pheno_transposed": false,
+"covar": "recla_covar.csv",
+"sex": {
+  "covar": "Sex",
+  "female": "female",
+  "male": "male"
+},
+"x_chr": "X",
+"cross_info": {
+  "covar": "ngen"
+},
+"gmap": "recla_gmap.csv",
+"pmap": "recla_pmap.csv",
+"alleles": ["A", "B", "C", "D", "E", "F", "G", "H"]
+}
+#+end_src
+
+Meanwhile the gmap file looks like
+
+#+begin_src js
+marker,chr,pos,Mb
+rs6269442,1,0.0,3.482275
+rs6365999,1,0.0,4.811062
+rs6376963,1,0.895,5.008089
+rs3677817,1,1.185,5.176058
+#+end_src
diff --git a/doc/README.org b/doc/README.org
index b3c78f29..2b27d562 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -6,7 +6,7 @@
    - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]]
    - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]]
    - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]]
-   - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]]
+   - [[#step-4-install-and-run-gn2][Step 4: Install and run GN2]]
  - [[#run-mysql-server][Run MySQL server]]
  - [[#gn2-dependency-graph][GN2 Dependency Graph]]
  - [[#source-deployment][Source deployment]]
@@ -20,6 +20,7 @@
    - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]]
    - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]]
    - [[#cant-run-a-module][Can't run a module]]
+   - [[#rpy2-error-show-now-found][Rpy2 error 'show' now found]]
  - [[#irc-session][IRC session]]
 
 * Introduction
@@ -117,7 +118,7 @@ cd guix-gn-latest
 ** Step 3: Authorize the GN Guix server
 
 GN2 has its own GNU Guix binary distribution server. To trust it you have
-to add the following key 
+to add the following key
 
 #+begin_src scheme
 (public-key
@@ -136,9 +137,9 @@ guix archive --authorize
 
 and hit Ctrl-D.
 
-Now you can use the substitute server to install GN2 binaries. 
+Now you can use the substitute server to install GN2 binaries.
 
-** Step 4: Install and run GN2 
+** Step 4: Install and run GN2
 
 Since this is a quick and dirty install we are going to override the
 GNU Guix package path by pointing the package path to our repository:
@@ -208,7 +209,7 @@ https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip
 Check the md5sum.
 
 After installation inflate the database binary in the MySQL directory
-(this installation path is subject to change soon) 
+(this installation path is subject to change soon)
 
 : chown -R mysql:mysql db_webqtl_s/
 : chmod 700 db_webqtl_s/
@@ -271,10 +272,10 @@ R_LIBS_SITE are set) from the information given by guix:
 Inside the repository:
 
 : cd genenetwork2
-: ./bin/genenetwork2 
+: ./bin/genenetwork2
 
-Will fire up your local repo http://localhost:5003/ using the  
-settings in ./etc/default_settings.py. These settings may 
+Will fire up your local repo http://localhost:5003/ using the
+settings in ./etc/default_settings.py. These settings may
 not reflect your system. To override settings create your own from a copy of
 default_settings.py and pass it into GN2 with
 
@@ -348,7 +349,7 @@ Make dirs
 
 Add users
 
-: adduser nobody ; addgroup nobody 
+: adduser nobody ; addgroup nobody
 
 Run nginx
 
@@ -392,6 +393,12 @@ Make a note of the paths with
 ./pre-inst-env guix package --search-paths
 #+end_src bash
 
+or this should also work if guix is installed
+
+#+begin_src bash
+guix package --search-paths
+#+end_src bash
+
 After setting the paths for the server
 
 #+begin_src bash
@@ -413,7 +420,7 @@ genenetwork2
 will start the default server which listens on port 5003, i.e.,
 http://localhost:5003/.
 
-OK, we are where we were before with step 4. Only difference is that we 
+OK, we are where we were before with step 4. Only difference is that we
 used our own compiled guix server.
 
 * Trouble shooting
@@ -433,7 +440,7 @@ On one system:
 : export R_LIBS_SITE="$HOME/.guix-profile/site-library/"
 : export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0"
 
-and perhaps a few more. 
+and perhaps a few more.
 ** ERROR: can not find directory $HOME/gn2_data
 
 The default settings file looks in your $HOME/gn2_data. Since these
@@ -447,6 +454,21 @@ In rare cases, development modules are not brought in with Guix
 because no source code is available. This can lead to missing modules
 on a running server. Please check with the authors when a module
 is missing.
+** Rpy2 error 'show' now found
+
+This error
+
+: __show = rpy2.rinterface.baseenv.get("show")
+: LookupError: 'show' not found
+
+means that R was updated in your path, and that Rpy2 needs to be
+recompiled against this R - don't you love informative messages?
+
+In our case it means that GN's PYTHONPATH is not in sync with
+R_LIBS_SITE. Please check your GNU Guix GN2 installation paths,
+you man need to reinstall. Note that this may be the point you
+may want to start using profiles (see profile section).
+
 * IRC session
 
 Here an IRC session where we installed GN2 from scratch using GNU Guix
@@ -466,7 +488,7 @@ and a download of the test database.
 <user01> set to the ones in ~/.guix-profile/
 <pjotrp> good, and you are in gn-latest-guix repo  [07:06]
 <user01> yep  [07:07]
-<pjotrp> git log shows 
+<pjotrp> git log shows
 
 Author: David Thompson <dthompson2@worcester.edu>
 Date:   Sun Mar 27 21:20:19 2016 -0400
@@ -488,7 +510,7 @@ genenetwork2-files-small        1.0     out ../guix-bioinformatics/gn/packages/g
 <user01> hah, I don't have screen installed yet  [07:11]
 <pjotrp> comes with guix ;)  [07:12]
 <pjotrp> no worries, you can run it any way you want
-<pjotrp> $HOME/.guix-profile/bin/guix-daemon --build-users-group=guixbuild 
+<pjotrp> $HOME/.guix-profile/bin/guix-daemon --build-users-group=guixbuild
 <user01> then something's weird, because it says I don't have it
 <pjotrp> oh, you need to install it first  [07:13]
 <pjotrp> guix package -A screen
@@ -546,11 +568,11 @@ The following derivations would be built:
 <pjotrp> https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org
 <pjotrp> this is exactly what we are doing now
 <user01> alrighty  [07:35]
-<pjotrp> To see if a remote server has a guix server running it should respond 
+<pjotrp> To see if a remote server has a guix server running it should respond
                                                                         [07:36]
 <pjotrp> lynx http://guix.genenetwork.org:8080 --dump
 <pjotrp> Resource not found: /
-<pjotrp> 
+<pjotrp>
 <pjotrp> you see that?
 <user01> yes  [07:37]
 <pjotrp> good. The main hydra server is too slow. So on my laptop I forced
@@ -558,7 +580,7 @@ The following derivations would be built:
 <pjotrp> env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix
          package -i genenetwork2 --dry-run
          --substitute-urls="http://mirror.hydra.gnu.org"
-<pjotrp> 
+<pjotrp>
 <pjotrp> the list looks the same to me  [07:40]
 <user01> me too
 <pjotrp> note that some packages will be built and some downloaded, right?
@@ -688,7 +710,7 @@ The following derivations would be built:
 <pjotrp> everything should be pre-built from guix.genenetwork.org
 <pjotrp> you are downloading?
 <user02> yes  [09:15]
-<pjotrp> cool. Maybe an idea to set up a server 
+<pjotrp> cool. Maybe an idea to set up a server
 <pjotrp> for your own use
 <user02> Stuck at downloading preprocesscore
 <pjotrp> should not  [09:24]
@@ -735,7 +757,7 @@ The following derivations would be built:
 <pjotrp> should be at
          /gnu/store/y1f3r2xs3fhyadd46nd2aqbr2p9qv2ra-r-biocpreprocesscore-1.32.0
                                                                         [09:33]
-<pjotrp> 
+<pjotrp>
 <user03> pjotrp: Possibly we should use the archive utility of Guix to do
         deployment to avoid such out-of-sync differences :)  [09:34]
 <pjotrp> maybe. I did not get archive to update profiles properly  [09:37]
@@ -802,7 +824,7 @@ The following derivations would be built:
 <pjotrp> but do not checkout that genetwork2_diet
 <pjotrp> we reverted to the main tree
 <pjotrp> clone git@github.com:genenetwork/genenetwork2.git  [09:53]
-<pjotrp> instead and checkout the staging branch 
+<pjotrp> instead and checkout the staging branch
 <pjotrp> that is effectively my branch  [09:54]
 <pjotrp> when that is done you should be able to fire up the webserver from
          there  [09:55]
@@ -825,7 +847,7 @@ The following derivations would be built:
 <user01> yep
 <pjotrp> that can also run on remote files over ssh
 <pjotrp> that's an alternative
-<pjotrp> kudos for using emacs :), wdyt user03 
+<pjotrp> kudos for using emacs :), wdyt user03
 <user02> 79 minutes to go downloading the db
 <pjotrp> user02: sorry about that  [09:59]
 <pjotrp> it is 2GB
@@ -850,7 +872,7 @@ The following derivations would be built:
          --substitute-urls="http://guix.genenetwork.org:8080"   [10:08]
 <pjotrp> elixir  1.2.3   out
          ../guix-bioinformatics/gn/packages/elixir.scm:31:2
-<pjotrp> 
+<pjotrp>
 <pjotrp> I am building it on guix.genenetwork.org right now  [10:09]
 <user01> nice  [10:10]
 #+end_src
diff --git a/doc/database.org b/doc/database.org
index d4c04848..624174a4 100644
--- a/doc/database.org
+++ b/doc/database.org
@@ -207,10 +207,18 @@ Metadata
 
 ?
 
-** Chr_Length
+** Chr_Length (/cross/BXD.json)
 
 Default mm9, column for mm8
 
+select * from Chr_Length;
+
+| Name | SpeciesId | OrderId | Length    | Length_mm8 |
+| 1    |         1 |       1 | 197195432 |  197069962 |
+| 2    |         1 |       2 | 181748087 |  181976762 |
+
+Table should be merged with
+
 ** Dataset_mbat
 
 Menu for BXD (linkouts)
@@ -275,10 +283,19 @@ Wiki info (nightly updated from NCBI)
 
 XRef should be foreign keys
 
-** Geno
+** Geno (genotype/marker/'marker'.json)
 
 SNP or marker info
 
+INFO:base.trait:.sql: retrieve_info:
+                                select Geno.Chr, Geno.Mb from Geno, Species
+                                where Species.Name = 'mouse' and
+                                Geno.Name = 'rs3693478' and
+                                Geno.SpeciesId = Species.Id
+
+| Id | SpeciesId | Name       | Marker_Name | Chr  | Mb        | Sequence                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | Source  | chr_num | Source2 | Comments | used_by_geno_file | Mb_mm8    | Chr_mm8 |
+|  1 |         1 | 01.001.695 | 01.001.695  | 1    |  4.678288 | GCCCTGCCCACCTCAGAGCAAGCTGCCACCCAGGAGTCCGTGTTTCAGGAGATGTGTGAGGAGGGCCTGCTGGAGGAGTGTGATGGTGAGGATGAGGCAGGCCGTGCCGCG[T/C]AGCCAGAGGCTGGTGATGGGACCACCGAGATCTCACCCACTGGTGCTGCTGATCCTGAGAAGAGGATGGAGAAGAAGACGGAGCAGCAGCACACCGGCGGCGGGAGAAAGCTGCTCGTAAGCTGCTCGTAAGCTACGGGTGCAGCAGGCTGCACTTAGGGCAGCCCGGCTTCAGCACCAAGAACTCTTCAGGCTGCATGGGATCAAGGCCCAGGTGGCCCGAAGGCTGGCAGAACTCGCACACGGGAGGGAGCAGCAGCGCATACAGCGACTGGCAGAGGCTGACAAGCCCCGAAGGCTGGGACGACTCAAGTACCAGGCTCCTGACATTGATGTGCAGCTCAGCTCTGAGCTGTCTGGCCCACTCAGGACACTGAAACCAGAAGGTCACATTCTCCAAGACAGGTTCAAGAGCTTCCAGAAGAGAAATATGATTGAGCCCCGAGAACGAGCCAAGTTCAAGCGCAAATAAAAAATGAAGTTGGTGGAGAAGCGGGCCTACCATGAGATTCAGTTGTAGCTGTGCAGATGTCGGAGCCCCGCCCCTCAATAAAGTTCTGTGACAAAAAAAAAAAAAAAAAAAGAAGAAGAAGAAGAAAAGGAAAAAAAAGAAGAAAAAGAAAAAAAAAGAAAAAAGAAAAAGAAAACACATCACTTGGCAAAACTCCATAGACTCTATGTGATTCATGTTTCAAACATGCACCTA | GNF_SNP |       1 | GNF     | NULL     | NULL              |  4.678288 | 1       |
+
 ** GenoCode
 
 Belongs to someone else
@@ -311,7 +328,7 @@ Heritability for probeset(?)
 
 Homology, not used much
 
-** InbredSet
+** InbredSet (/cross/BXD.info)
 
 Group in menu
 
@@ -489,6 +506,35 @@ select count(*) from ProbeSet limit 5;
 +------+--------+----------+----------+--------+----------------------------------------------+------+-----------+----------+--------+-----------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+--------------+-------------+--------+----------+-------------------------+-----------------------+----------------------+-------------------------+-----------------------+------------------+----------------------+------+----------+---------------+--------------+------+---------+----------+---------+----------+--------------------------+---------------------+---------+-----------+-----------------------------+---------------------------+--------------+-------------+-----------+-------------+------------+---------+------------+----------+-----------+------------+------------+---------------+---------+-----------+---------+------------------+-------------+------+--------+-------------+----------------+-----------------+
 2 rows in set (0.00 sec)
 
+** ProbeSetXRef (phenotypes/dataset_name.json)
+
+For every probe set (read dataset measuring point):
+
+select * from ProbeSetXRef;
+
+| ProbeSetFreezeId | ProbeSetId | DataId   | Locus_old | LRS_old | pValue_old | mean             | se   | Locus      | LRS               | pValue | additive              | h2   |
+|              112 |     123528 | 23439389 | NULL      |    NULL |       NULL |  6.7460707070707 | NULL | rs6239372  |  10.9675593568894 |  0.567 |    0.0448545966228878 | NULL |
+|              112 |     123527 | 23439388 | NULL      |    NULL |       NULL | 6.19416161616162 | NULL | rs13476936 |  10.9075670392762 |  0.567 |   -0.0358456732993988 | NULL |
+
+where ProbeSetFreezeId is the dataset (experiment). ProbesetId refers
+to the probe set information (measuring point). DataId points to the
+data point. The othe values are used for search.
+
+It is used in search thus:
+
+SELECT distinct ProbeSet.Name as TNAME,
+  ProbeSetXRef.Mean as TMEAN, ProbeSetXRef.LRS as TLRS,
+  ProbeSetXRef.PVALUE as TPVALUE, ProbeSet.Chr_num as TCHR_NUM,
+  ProbeSet.Mb as TMB, ProbeSet.Symbol as TSYMBOL,
+  ProbeSet.name_num as TNAME_NUM
+FROM ProbeSetXRef, ProbeSet
+WHERE ProbeSet.Id = ProbeSetXRef.ProbeSetId
+  and ProbeSetXRef.ProbeSetFreezeId = 112
+  ORDER BY ProbeSet.symbol ASC limit 5;
+
+| TNAME      | TMEAN            | TLRS               | TPVALUE               | TCHR_NUM | TMB        | TSYMBOL       | TNAME_NUM |
+| 1445618_at | 7.05679797979798 |   13.5417452764616 |                  0.17 |        8 |  75.077895 | NULL          |   1445618 |
+| 1452452_at |            7.232 |   30.4944361132252 | 0.0000609756097560421 |       12 |    12.6694 | NULL          |   1452452 |
 
 ** ProbeSetData
 
@@ -691,6 +737,19 @@ show indexes from ProbeSetFreeze;
 | ProbeSetFreeze |          1 | NameIndex |            1 | Name2       | A         |           2 |     NULL | NULL   |      | BTREE      |         |               |
 +----------------+------------+-----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+
 
+** ProbeSetSE
+
+ select * from ProbeSetSE limit 5;
++--------+----------+----------+
+| DataId | StrainId | error    |
++--------+----------+----------+
+|      1 |        1 | 0.681091 |
+|      1 |        2 | 0.361151 |
+|      1 |        3 | 0.364342 |
+|      1 |        4 | 0.827588 |
+|      1 |        5 | 0.303492 |
++--------+----------+----------+
+
 ** Publication and publishdata (all pheno)
 
 Phenotype pubs
@@ -794,6 +853,18 @@ INFO:db.call:.sql: __init__:
 
 INFO:db.call:.sql: ('BXD', 1)
 
+The actual search is
+
+SELECT distinct ProbeSet.Name as TNAME, 0 as thistable,
+  ProbeSetXRef.Mean as TMEAN, ProbeSetXRef.LRS as TLRS,
+  ProbeSetXRef.PVALUE as TPVALUE, ProbeSet.Chr_num as TCHR_NUM,
+  ProbeSet.Mb as TMB, ProbeSet.Symbol as TSYMBOL,
+  ProbeSet.name_num as TNAME_NUM
+FROM ProbeSetXRef, ProbeSet
+WHERE ProbeSet.Id = ProbeSetXRef.ProbeSetId
+  and ProbeSetXRef.ProbeSetFreezeId = 112
+  ORDER BY ProbeSet.symbol ASC limit 5;
+
 INFO:base.species:.sql: __init__:
                 Select
                         Chr_Length.Name, Chr_Length.OrderId, Length from Chr_Length, InbredSet
@@ -874,7 +945,87 @@ INFO:base.data_set:.sql: get_trait_info:
 
 (that is a bug!).
 
-** Fetch phenotypes
+** Fetch phenotype information
+*** Through the trait page
+
+When hitting the trait page, e.g.
+
+curl "http://localhost:5003/show_trait?trait_id=1443823_s_aet=HC_M2_0606_P"
+
+First the BXD's are queried with
+
+DEBUG:base.data_set:.get_samplelist: Sample list: : ['BXD1',
+ 'BXD2',
+ 'BXD5',
+ ...
+
+main probeset info (trait) is retrieved with
+
+SELECT ProbeSet.name, ProbeSet.symbol, ProbeSet.description, ProbeSet.probe_target_description, ProbeSet.chr, ProbeSet.mb, ProbeSet.alias, ProbeSet.geneid, ProbeSet.genbankid, ProbeSet.unigeneid, ProbeSet.omim, ProbeSet.refseq_transcriptid, ProbeSet.blatseq, ProbeSet.targetseq, ProbeSet.chipid, ProbeSet.comments, ProbeSet.strand_probe, ProbeSet.strand_gene, ProbeSet.probe_set_target_region, ProbeSet.probe_set_specificity, ProbeSet.probe_set_blat_score, ProbeSet.probe_set_blat_mb_start, ProbeSet.probe_set_blat_mb_end, ProbeSet.probe_set_strand, ProbeSet.probe_set_note_by_rw, ProbeSet.flag
+                    FROM ProbeSet, ProbeSetFreeze, ProbeSetXRef
+                    WHERE
+                            ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND
+                            ProbeSetXRef.ProbeSetId = ProbeSet.Id AND
+                            ProbeSetFreeze.Name = 'HC_M2_0606_P' AND
+                            ProbeSet.Name = '1443823_s_at'
+
+Followed by
+
+INFO:base.trait:.sql: retrieve_info:
+   SELECT
+                                    ProbeSetXRef.Locus, ProbeSetXRef.LRS, ProbeSetXRef.pValue, ProbeSetXRef.mean, ProbeSetXRef.additive
+                            FROM
+                                    ProbeSetXRef, ProbeSet
+                            WHERE
+                                    ProbeSetXRef.ProbeSetId = ProbeSet.Id AND
+                                    ProbeSet.Name = "1443823_s_at" AND
+                                    ProbeSetXRef.ProbeSetFreezeId =112
+
+| Locus       | LRS             | pValue                 | mean             | additive             |
+| NES13033186 | 35.466324074542 | 0.00000900000000003676 | 15.0551313131313 | -0.16750405405405402 |
+
+Then the interesting bit, the sample data is fetched with
+
+INFO:base.data_set:.sql: retrieve_sample_data:
+                   SELECT
+                            Strain.Name, ProbeSetData.value, ProbeSetSE.error, ProbeSetData.Id, Strain.Name2
+                    FROM
+                            (ProbeSetData, ProbeSetFreeze, Strain, ProbeSet, ProbeSetXRef)
+                    left join ProbeSetSE on
+                            (ProbeSetSE.DataId = ProbeSetData.Id AND ProbeSetSE.StrainId = ProbeSetData.StrainId)
+                    WHERE
+                            ProbeSet.Name = '1443823_s_at' AND ProbeSetXRef.ProbeSetId = ProbeSet.Id AND
+                            ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND
+                            ProbeSetFreeze.Name = 'HC_M2_0606_P' AND
+                            ProbeSetXRef.DataId = ProbeSetData.Id AND
+                            ProbeSetData.StrainId = Strain.Id
+                    Order BY
+                            Strain.Name
+
+| Name        | value  | error | Id       | Name2       |
+| 129S1/SvImJ | 14.552 |  NULL | 23422417 | 129S1/SvImJ |
+| A/J         |  14.34 |  NULL | 23422417 | A/J         |
+| AKR/J       | 14.338 |  NULL | 23422417 | AKR/J       |
+| B6D2F1      | 15.251 |  NULL | 23422417 | B6D2F1      |
+| BALB/cByJ   | 14.164 |  NULL | 23422417 | BALB/cByJ   |
+| BALB/cJ     | 14.563 |  NULL | 23422417 | BALB/cJ     |
+| BXD1        | 15.198 |  NULL | 23422417 | BXD1        |
+| BXD11       | 15.084 |  NULL | 23422417 | BXD11       |
+| BXD12       | 15.192 |  NULL | 23422417 | BXD12       |
+ etc.
+
+Then some repeated queries and this fetchest the nearest SNP
+
+INFO:wqflask.show_trait.show_trait:.sql: get_nearest_marker:
+ SELECT Geno.Name FROM Geno, GenoXRef, GenoFreeze WHERE Geno.Chr = '1'
+ AND GenoXRef.GenoId = Geno.Id AND GenoFreeze.Id =
+ GenoXRef.GenoFreezeId AND GenoFreeze.Name = 'BXDGeno' ORDER BY ABS(
+ Geno.Mb - 173.149434) limit 1;
+
+| Name        |
+| NES13033186 |
+
+*** Digging deeper
 
 To get at phenotype data ProbeSetData is the main table (almost all
 important molecular assay data is in this table including probe set
@@ -1014,3 +1165,51 @@ select * from ProbeSetData limit 5;
 5 rows in set (0.00 sec)
 
 linked by ProbeSetXRef.dataid.
+** Fetch genotype information
+
+*** SNPs
+
+The SNP count info for the BXD is calculated like this
+
+#+begin_src python
+        while startMb<endMb:
+            snp_count = g.db.execute("""
+                    select
+                            count(*) from BXDSnpPosition
+                    where
+                            Chr = '%s' AND Mb >= %2.6f AND Mb < %2.6f AND
+                            StrainId1 = %d AND StrainId2 = %d
+                    """ % (chrName, startMb, startMb+stepMb, strainId1, strainId2)).fetchone()[0]
+            SNPCounts.append(snp_count)
+            startMb += stepMb
+#+end_src
+
+select * from BXDSnpPosition limit 5;
++------+-----------+-----------+----------+
+| Chr  | StrainId1 | StrainId2 | Mb       |
++------+-----------+-----------+----------+
+| 1    |         2 |         3 | 0.002477 |
+| 1    |         2 |         3 | 0.002592 |
+| 1    |         2 |         3 |  0.00283 |
+| 1    |         2 |         3 | 0.002994 |
+| 1    |         2 |         3 | 0.003299 |
++------+-----------+-----------+----------+
+
+Other SNP tables containing
+
+select * from SnpSource limit 5;
+Empty set (0.00 sec)
+
+select * from SnpAll limit 5;
+Empty set (0.00 sec)
+
+mysql> select * from SnpAll limit 5;
+Empty set (0.00 sec)
+
+mysql> select * from SnpPattern limit 5;
+Empty set (0.00 sec)
+
+mysql> select * from SnpSource limit 5;
+Empty set (0.00 sec)
+
+Hmmm. This is the test database. Then there are the plink files and VCF files.
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 1b4e1195..6cd3c8e6 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -91,7 +91,7 @@ Publish or ProbeSet. E.g.
         if USE_GN_SERVER:
             data = menu_main()
         else:
-            file_name = "wqflask/static/new/javascript/dataset_menu_structure.json"
+            file_name = "wqflask/wqflask/static/new/javascript/dataset_menu_structure.json"
             with open(file_name, 'r') as fh:
                 data = json.load(fh)
 
diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index f76d8140..8c67a6fd 100644
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -80,4 +80,3 @@ PORTADDR = "http://50.16.251.170"
 INFOPAGEHREF = '/dbdoc/%s.html'
 CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR'
 SCRIPTFILE = 'main.py'
-
diff --git a/wqflask/maintenance/README.md b/wqflask/maintenance/README.md
new file mode 100644
index 00000000..873eaa32
--- /dev/null
+++ b/wqflask/maintenance/README.md
@@ -0,0 +1,4 @@
+Maintenance files have been moved into a separate repository named
+*gn_extra*. See https://github.com/genenetwork/gn_extra
+
+
diff --git a/wqflask/maintenance/__init__.py b/wqflask/maintenance/__init__.py
deleted file mode 100644
index e69de29b..00000000
--- a/wqflask/maintenance/__init__.py
+++ /dev/null
diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py
index d39bf4a5..f2f0830f 100644
--- a/wqflask/maintenance/gen_select_dataset.py
+++ b/wqflask/maintenance/gen_select_dataset.py
@@ -1,7 +1,9 @@
 """Script that generates the data for the main dropdown menus on the home page
 
 Writes out data as /static/new/javascript/dataset_menu_structure.json
-It needs to be run manually when database has been changed.
+It needs to be run manually when database has been changed. Run it as
+
+  ./bin/genenetwork2 ~/my_settings.py -c ./wqflask/maintenance/gen_select_dataset.py
 
 """
 
@@ -37,9 +39,15 @@ from __future__ import print_function, division
 #print("cdict is:", cdict)
 
 import sys
-# import zach_settings # no hard code paths!
 
-# import MySQLdb
+# NEW: Note we prepend the current path - otherwise a guix instance of GN2 is used instead
+sys.path.insert(0,'./wqflask')
+# NEW: import app to avoid a circular dependency on utility.tools
+from wqflask import app
+
+from utility.tools import locate, locate_ignore_error, TEMPDIR, SQL_URI
+
+import MySQLdb
 
 # import simplejson as json
 import urlparse
@@ -55,14 +63,13 @@ from pprint import pformat as pf
 
 #conn = Engine.connect()
 
-print('ERROR: This conversion is now OBSOLETE as the menu gets built from the database in Javascript using GN_SERVER instead!')
-sys.exit()
+print('WARNING: This conversion is now OBSOLETE as the menu gets built from the database in Javascript using GN_SERVER instead!')
 
 
 def parse_db_uri(db_uri):
     """Converts a database URI to the db name, host name, user name, and password"""
 
-    parsed_uri = urlparse.urlparse(zach_settings.DB_URI)
+    parsed_uri = urlparse.urlparse(SQL_URI)
 
     db_conn_info = dict(
                         db = parsed_uri.path[1:],
@@ -70,6 +77,7 @@ def parse_db_uri(db_uri):
                         user = parsed_uri.username,
                         passwd = parsed_uri.password)
 
+    print(db_conn_info)
     return db_conn_info
 
 
@@ -258,7 +266,7 @@ def build_datasets(species, group, type_name):
 def main():
     """Generates and outputs (as json file) the data for the main dropdown menus on the home page"""
 
-    parse_db_uri(zach_settings.SQL_URI)
+    parse_db_uri(SQL_URI)
 
     species = get_species()
     groups = get_groups(species)
@@ -281,7 +289,7 @@ def main():
 
     #print("data:", data)
 
-    output_file = """../wqflask/static/new/javascript/dataset_menu_structure.json"""
+    output_file = """./wqflask/wqflask/static/new/javascript/dataset_menu_structure.json"""
 
     with open(output_file, 'w') as fh:
         json.dump(data, fh, indent="   ", sort_keys=True)
@@ -297,6 +305,6 @@ def _test_it():
     #print("build_datasets:", pf(datasets))
 
 if __name__ == '__main__':
-    Conn = MySQLdb.Connect(**parse_db_uri(zach_settings.SQL_URI))
+    Conn = MySQLdb.Connect(**parse_db_uri(SQL_URI))
     Cursor = Conn.cursor()
     main()
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index bb8241f5..2c8cc5c5 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -3,6 +3,7 @@
 
 import os
 import sys
+
 from wqflask import app
 
 # Use the standard logger here to avoid a circular dependency
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index f3b018b8..a1065adc 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -57,7 +57,7 @@ class MarkerRegression(object):
 
         all_samples_ordered = self.dataset.group.all_samples_ordered()
         primary_sample_names = list(all_samples_ordered)
-        
+
         for sample in self.dataset.group.samplelist:
             in_trait_data = False
             for item in self.this_trait.data:
@@ -241,9 +241,9 @@ class MarkerRegression(object):
                         highest_chr = marker['chr']
                     if ('lod_score' in marker.keys()) or ('lrs_value' in marker.keys()):
                         self.qtl_results.append(marker)
-       
+
             self.trimmed_markers = trim_markers_for_table(results)
-			
+
             self.json_data['chr'] = []
             self.json_data['pos'] = []
             self.json_data['lod.hk'] = []
@@ -560,7 +560,7 @@ class MarkerRegression(object):
 
 
     def gen_pheno_txt_file_plink(self, pheno_filename = ''):
-        ped_sample_list = self.get_samples_from_ped_file()	
+        ped_sample_list = self.get_samples_from_ped_file()
         output_file = open("%s%s.txt" % (TMPDIR, pheno_filename), "wb")
         header = 'FID\tIID\t%s\n' % self.this_trait.name
         output_file.write(header)
@@ -595,7 +595,7 @@ class MarkerRegression(object):
         output_file.close()
 
     def gen_pheno_txt_file_rqtl(self, pheno_filename = ''):
-        ped_sample_list = self.get_samples_from_ped_file()	
+        ped_sample_list = self.get_samples_from_ped_file()
         output_file = open("%s%s.txt" % (TMPDIR, pheno_filename), "wb")
         header = 'FID\tIID\t%s\n' % self.this_trait.name
         output_file.write(header)
@@ -653,7 +653,7 @@ class MarkerRegression(object):
             genotype = genotype.addinterval()
 
         samples, values, variances, sample_aliases = self.this_trait.export_informative()
-        
+
         trimmed_samples = []
         trimmed_values = []
         for i in range(0, len(samples)):
@@ -818,7 +818,7 @@ class MarkerRegression(object):
         """Runs permutations and gets significant and suggestive LOD scores"""
 
         top_lod_scores = []
-	
+
         #print("self.num_perm:", self.num_perm)
 
         for permutation in range(self.num_perm):
@@ -1084,7 +1084,7 @@ def create_snp_iterator_file(group):
 
 def trim_markers_for_table(markers):
     num_markers = len(markers)
-	
+
     if 'lod_score' in markers[0].keys():
         sorted_markers = sorted(markers, key=lambda k: k['lod_score'], reverse=True)
     else:
diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
index 3db5b0f6..9cef3cec 100644
--- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py
+++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
@@ -386,7 +386,7 @@ class MarkerRegression(object):
             self.GraphInterval = self.MbGraphInterval #Mb
         else:
             self.GraphInterval = self.cMGraphInterval #cM
-			
+
         ################################################################
         # Get Trait Values and Infomation
         ################################################################
diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py
index 5e3b092e..7e7503d4 100644
--- a/wqflask/wqflask/show_trait/SampleList.py
+++ b/wqflask/wqflask/show_trait/SampleList.py
@@ -32,7 +32,7 @@ class SampleList(object):
         self.sample_attribute_values = {}
 
         self.get_attributes()
-        logger.debug("camera: attributes are:", pf(self.attributes))
+        # logger.debug("camera: attributes are:", pf(self.attributes))
 
         if self.this_trait and self.dataset and self.dataset.type == 'ProbeSet':
             self.get_extra_attribute_values()
@@ -55,7 +55,7 @@ class SampleList(object):
                 sample.extra_info['url'] = "/mouseCross.html#AXB/BXA"
                 sample.extra_info['css_class'] = "fs12"
 
-            logger.debug("  type of sample:", type(sample))
+            # logger.debug("  type of sample:", type(sample))
 
             if sample_group_type == 'primary':
                 sample.this_id = "Primary_" + str(counter)
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index 3eea3f4a..912beabe 100644
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -146,7 +146,7 @@ class ShowTrait(object):
         else:
             self.sample_group_types['samples_primary'] = self.dataset.group.name
         sample_lists = [group.sample_list for group in self.sample_groups]
-        logger.debug("sample_lists is:", pf(sample_lists))
+        # logger.debug("sample_lists is:", pf(sample_lists))
 
         self.get_mapping_methods()