aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorzsloan2017-07-03 11:33:56 -0500
committerGitHub2017-07-03 11:33:56 -0500
commit5c7b50d579dcf08a16a45ccde4a299b6cce6aaab (patch)
tree228e98034186383c02556693453695dc864d30ea
parentb41898f11cd20b063db1df41a34f6515f331c21b (diff)
parente15cef7c47d6da9085c44d0835e86620162d1607 (diff)
downloadgenenetwork2-5c7b50d579dcf08a16a45ccde4a299b6cce6aaab.tar.gz
Merge pull request #255 from lyan6/master
Correlation Scatterplot updates
-rw-r--r--VERSION2
-rwxr-xr-xbin/genenetwork2104
-rw-r--r--doc/GUIX-Reproducible-from-source.org356
-rw-r--r--doc/README.org440
-rw-r--r--doc/development.org43
-rw-r--r--etc/VERSION1
-rw-r--r--etc/default_settings.py33
-rw-r--r--test/lib/main_web_functionality.rb4
-rw-r--r--wqflask/base/webqtlConfig.py2
-rw-r--r--wqflask/maintenance/gen_select_dataset.py2
-rw-r--r--wqflask/utility/tools.py40
-rw-r--r--wqflask/wqflask/database.py2
-rw-r--r--wqflask/wqflask/marker_regression/marker_regression.py2
-rw-r--r--wqflask/wqflask/marker_regression/marker_regression_gn1.py15
-rw-r--r--wqflask/wqflask/static/new/css/corr_scatter_plot2.css39
-rw-r--r--wqflask/wqflask/static/new/javascript/dataset_menu_structure.json284
-rw-r--r--wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js122
-rw-r--r--wqflask/wqflask/templates/base.html2
-rw-r--r--wqflask/wqflask/templates/corr_scatterplot.html26
-rw-r--r--wqflask/wqflask/templates/error.html4
-rw-r--r--wqflask/wqflask/templates/marker_regression_gn1.html2
-rw-r--r--wqflask/wqflask/templates/show_trait.html1
-rw-r--r--wqflask/wqflask/templates/show_trait_statistics.html5
-rw-r--r--wqflask/wqflask/user_manager.py26
-rw-r--r--wqflask/wqflask/views.py7
25 files changed, 979 insertions, 585 deletions
diff --git a/VERSION b/VERSION
index 144b089a..3e0b7cab 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.10-pre1
+2.10-pre4
diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 52d3155c..236d8f63 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -1,34 +1,39 @@
#! /bin/bash
#
# This will run the GN2 server (with default settings if none
-# supplied). Pass in your own settings file, e.g.
+# supplied). Typically you need a GNU Guix profile which is set with
+# an environment variable (this profile is dictated by the
+# installation path of genenetwork). Say your profile is in
+# ~/opt/gn-latest-guix
#
-# ./bin/genenetwork2 ~/my_settings.py
+# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2
#
-# But better is to retain the defaults and override them with
-# JSON file (see the deploy docs).
+# You can pass in your own settings file, e.g.
#
-# ./bin/genenetwork2 ~/my_overrides.json
+# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_settings.py
#
# To run a maintenance script with settings (instead of the webserver) add that with
# a -c switch, e.g.
#
-# ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py
-#
-# Environment settings can be used to preconfigure as well as a
-# settings.py file.
-#
-# GN2_BASE_PATH is base directory of wqflask source code
-#
+# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py
SCRIPT=$(readlink -f "$0")
GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
+GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR)
echo GN2_BASE_DIR=$GN2_BASE_DIR
-GIT_HASH=$(git rev-parse HEAD)
-GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-export GN_VERSION=$(cat $GN2_BASE_DIR/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9}
+GUIX_SITE=$GN2_BASE_DIR/lib/python2.7/site-packages
+if [ -d $GUIX_SITE ]; then
+ echo INFO: GN2 is running from GNU Guix
+ GN2_BASE_DIR=$GUIX_SITE
+ export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION)
+else
+ echo INFO: GN2 is running from a source tree
+ GIT_HASH=$(git rev-parse HEAD)
+ GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+ export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9}
+fi
echo GN_VERSION=$GN_VERSION
# Handle settings parameter (can be .py or .json)
@@ -41,13 +46,8 @@ else
shift
fi
if [ ! -e $settings ]; then
- GUIX_ETC=$GN2_BASE_DIR/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg
- if [ -d $GUIX_ETC ]; then
- echo INFO: GN2 is actually running from GNU Guix
- else
- echo "ERROR: can not locate settings file - pass it in the command line"
- exit 1
- fi
+ echo "ERROR: can not locate settings file - pass it in the command line"
+ exit 1
fi
export WQFLASK_SETTINGS=$settings # Python
export WQFLASK_OVERRIDES=$overrides # JSON
@@ -55,30 +55,40 @@ export WQFLASK_OVERRIDES=$overrides # JSON
echo WQFLASK_SETTINGS=$settings
echo WQFLASK_OVERRIDES=$overrides
-# ---- Checks and balances
-if [ -z $GUIX_ETC ]; then
- if [ -z $GN2_PROFILE ] ; then
- echo "WARNING: GN2_PROFILE has not been set - I hope you know what you are doing!"
- else
- export PATH=$GN2_PROFILE/bin:$PATH
- export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages
- export R_LIBS_SITE=$GN2_PROFILE/site-library
- export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0"
- export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0"
- export XDG_DATA_DIRS="$GN2_PROFILE/share"
- export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules"
- export LC_ALL=C
- fi
- if [ -z $PYTHONPATH ] ; then
- echo "WARNING PYTHONPATH has not been set - use GN2_PROFILE!"
- fi
- if [ ! -d $R_LIBS_SITE ] ; then
- echo "ERROR: R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!"
- echo "Paste into your shell the output of (for example)"
- echo "guix package -p \$GN2_PROFILE --search-paths"
- exit 1
+if [ -z $GN2_PROFILE ] ; then
+ echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!"
+ export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2)))
+ if [ -d $GN2_PROFILE ]; then
+ echo "Best guess is $GN2_PROFILE"
fi
fi
+if [ -z $GN2_PROFILE ]; then
+ read -p "PRESS [ENTER] TO CONTINUE..."
+else
+ export PATH=$GN2_PROFILE/bin:$PATH
+ export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages
+ export R_LIBS_SITE=$GN2_PROFILE/site-library
+ export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript
+ export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0"
+ export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0"
+ export XDG_DATA_DIRS="$GN2_PROFILE/share"
+ export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules"
+ export LC_ALL=C # FIXME
+ export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2"
+ export PLINK_COMMAND="$GN2_PROFILE/bin/plink2"
+ export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis"
+ export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma"
+fi
+if [ -z $PYTHONPATH ] ; then
+ echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!"
+ exit 1
+fi
+if [ ! -d $R_LIBS_SITE ] ; then
+ echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!"
+ echo "Paste into your shell the output of (for example)"
+ echo "guix package -p \$GN2_PROFILE --search-paths"
+ exit 1
+fi
# We may change this one:
export PYTHONPATH=$GN2_BASE_DIR/wqflask:$PYTHONPATH
@@ -88,6 +98,8 @@ if [ -z $TMPDIR ]; then
TMPDIR="/tmp"
fi
+# Show environment settings
+set|grep guix
set|grep $GN2_PROFILE
set|grep TMPDIR
@@ -97,7 +109,7 @@ if [ "$1" = '-c' ] ; then
cmd=${2#wqflask/}
echo PYTHONPATH=$PYTHONPATH
echo RUNNING COMMAND $cmd
- /usr/bin/env python $cmd
+ python $cmd
exit 0
fi
@@ -109,4 +121,4 @@ dbfilename gn2.rdb
# Start the flask server running GN2
cd $GN2_BASE_DIR/wqflask
echo "Starting with $settings"
-/usr/bin/env python runserver.py
+python runserver.py
diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org
index 4399ea26..83adce99 100644
--- a/doc/GUIX-Reproducible-from-source.org
+++ b/doc/GUIX-Reproducible-from-source.org
@@ -2,19 +2,188 @@
* Table of Contents :TOC:
- [[#introduction][Introduction]]
- - [[#binary-deployment][Binary deployment]]
+ - [[#binary-deployment-through-gnu-guix][Binary deployment through GNU Guix]]
+ - [[#quick-installation-recipe][Quick installation recipe]]
+ - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]]
+ - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]]
+ - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]]
+ - [[#step-4-install-and-run-gn2][Step 4: Install and run GN2]]
- [[#from-source-deployment][From source deployment]]
- [[#create-archive][Create archive]]
+ - [[#source-deployment][Source deployment]]
+ - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]]
+ - [[#set-up-nginx-port-forwarding][Set up nginx port forwarding]]
+ - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]]
+ - [[#update-to-recent-guix][Update to recent guix]]
+ - [[#install-gn2][Install GN2]]
+ - [[#run-gn2][Run GN2]]
* Introduction
Large system deployments tend to get very complex. In this document we
explain the GeneNetwork deployment system which is based on GNU Guix
-(see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]).
+(see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]] and the main [[README.org]] doc).
-* Binary deployment
+* Binary deployment through GNU Guix
+** Quick installation recipe
-NYA (will go to README)
+This is a recipe for quick and dirty installation of GN2. For
+convenience everything is installed as root, though in reality only
+GNU Guix has to be installed as root. I tested this recipe on a fresh
+install of Debian 8.3.0 (in KVM) though it should work on any modern
+Linux distribution (including CentOS).
+
+
+Note that GN2 consists of an approx. 5 GB installation including
+database. If you use a virtual machine we recommend to use at least
+double.
+
+** Step 1: Install GNU Guix
+
+Fetch the GNU Guix binary from [[https://www.gnu.org/software/guix/download/][here]] (middle panel) and follow
+[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html][instructions]]. Essentially, download and unpack the tar ball (which
+creates directories in /gnu and /var/guix), add build users and group
+(Guix builds software as unpriviliged users) and run the Guix daemon
+after fixing the paths (also known as the 'profile').
+
+Once you have succeeded, you have to [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-the-key][set the key]] (getting permission
+to download binaries from the GNU server) and you should be able to
+install the hello package using binary packages (no building)
+
+#+begin_src bash
+export PATH=~/.guix-profile/bin:$PATH
+guix pull
+guix package -i hello --dry-run
+#+end_src
+
+Which should show something like
+
+: The following files would be downloaded:
+: /gnu/store/zby49aqfbd9w9br4l52mvb3y6f9vfv22-hello-2.10
+: ...
+#+end_src
+
+means binary installs. The actual installation command of 'hello' is
+
+#+begin_src bash
+guix package -i hello
+hello
+ Hello, world!
+#+end_src
+
+If you actually see things building it means that Guix is not yet
+properly installed and up-to-date, i.e., the key is missing or you
+need to do a 'guix pull'. Press Ctrl-C to interrupt.
+
+If you need more help we have another writeup in [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#binary-installation][guix-notes]]. To get
+rid of the locale warning see [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-locale][set-locale]].
+
+** Step 2: Checkout the GN2 git repositories
+
+To fixate the software dependency graph GN2 uses git repositories of
+Guix packages. First install git if it is missing
+
+#+begin_src bash
+guix package -i git
+export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
+#+end_src
+
+check out the git repositories (gn-deploy branch)
+
+#+begin_src bash
+cd ~
+mkdir genenetwork
+cd genenetwork
+git clone --branch gn-deploy https://github.com/genenetwork/guix-bioinformatics
+git clone --branch gn-deploy --recursive https://github.com/genenetwork/guix guix-gn-deploy
+cd guix-gn-deploy
+#+end_src bash
+
+To test whether this is working try:
+
+#+begin_src bash
+#+end_src bash
+
+** Step 3: Authorize the GN Guix server
+
+GN2 has its own GNU Guix binary distribution server. To trust it you have
+to add the following key
+
+#+begin_src scheme
+(public-key
+ (ecc
+ (curve Ed25519)
+ (q #11217788B41ADC8D5B8E71BD87EF699C65312EC387752899FE9C888856F5C769#)
+ )
+)
+#+end_src
+
+by pasting it into the command
+
+#+begin_src bash
+guix archive --authorize
+#+end_src
+
+and hit Ctrl-D.
+
+Now you can use the substitute server to install GN2 binaries.
+
+** Step 4: Install and run GN2
+
+Since this is a quick and dirty install we are going to override the
+GNU Guix package path by pointing the package path to our repository:
+
+#+begin_src bash
+rm /root/.config/guix/latest
+ln -s ~/genenetwork/guix-gn-deploy/ /root/.config/guix/latest
+#+end_src
+
+Now check whether you can find the GN2 package with
+
+#+begin_src bash
+env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -A genenetwork2
+ genenetwork2 2.0-a8fcff4 out gn/packages/genenetwork.scm:144:2
+#+end_src
+
+(ignore the source file newer then ... messages, this is caused by the
+/root/.config/guix/latest override).
+
+And install with
+
+#+begin_src bash
+env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ \
+ guix package -i genenetwork2 \
+ --substitute-urls="http://guix.genenetwork.org"
+#+end_src
+
+Note: the order of the substitute url's may make a difference in speed
+(put the one first that is fastest for your location and time of day).
+
+Note: if your system starts building or gives an error it may well be
+Step 3 did not succeed. The installation should actually be smooth at
+this point and only do binary installs (no compiling).
+
+After installation you should be able to run genenetwork2 after updating
+the Guix suggested environment vars. Check the output of
+
+#+begin_src bash
+guix package --search-paths
+export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages"
+export R_LIBS_SITE="/root/.guix-profile/site-library/"
+#+end_src
+
+and copy-paste the listed exports into the terminal before running:
+
+#+begin_src bash
+genenetwork2
+#+end_src
+
+It will complain that the database is missing. See the next section on
+running MySQL server for downloading and installing a MySQL GN2
+database. After installing the database restart genenetwork2 and point
+your browser at [[http://localhost:5003/]].
+
+End of the GN2 installation recipe!
* From source deployment
@@ -52,3 +221,182 @@ gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix
* Create archive
: env GUIX_PACKAGE_PATH=../../genenetwork/guix-bioinformatics/ ./pre-inst-env guix archive --export -r genenetwork2 > guix_gn2-2.0-9e9475053.nar
+
+
+* Source deployment
+
+This section gives a more elaborate instruction for installing GN2
+from source.
+
+First execute above 4 steps:
+
+ - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]]
+ - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]]
+ - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]]
+ - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]]
+
+
+** Run your own copy of GN2
+
+At some point you may want to fix the source code. Assuming you have
+Guix and Genenetwork2 installed (as described above) clone the GN2
+repository from https://github.com/genenetwork/genenetwork2.
+
+Copy-paste the paths into your terminal (mainly so PYTHON_PATH and
+R_LIBS_SITE are set) from the information given by guix:
+
+: guix package --search-paths
+
+Inside the repository:
+
+: cd genenetwork2
+: ./bin/genenetwork2
+
+Will fire up your local repo http://localhost:5003/ using the
+settings in ./etc/default_settings.py. These settings may
+not reflect your system. To override settings create your own from a copy of
+default_settings.py and pass it into GN2 with
+
+: ./bin/genenetwork2 $HOME/my_settings.py
+
+and everything *should* work (note the full path to the settings
+file). This way we develop against the exact same dependency graph of
+software.
+
+If something is not working, take a hint from the settings file
+that comes in the Guix installation. It sits in something like
+
+: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py
+
+** Set up nginx port forwarding
+
+nginx can be used as a reverse proxy for GN2. For example, we want to
+expose GN2 on port 80 while it is running on port 5003. Essentially
+the configuration looks like
+
+#+begin_src js
+ server {
+ listen 80;
+ server_name test-gn2.genenetwork.org;
+ access_log logs/test-gn2.access.log;
+
+ proxy_connect_timeout 3000;
+ proxy_send_timeout 3000;
+ proxy_read_timeout 3000;
+ send_timeout 3000;
+
+ location / {
+ proxy_set_header Host $http_host;
+ proxy_set_header Connection keep-alive;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Host $server_name;
+ proxy_pass http://127.0.0.1:5003;
+ }
+}
+#+end_src js
+
+Install the nginx webserver (as root)
+
+: guix package -i nginx
+
+The nginx example configuration examples can be found in the Guix
+store through
+
+: ls -l /root/.guix-profile/sbin/nginx
+: lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx
+
+Use that path
+
+: ls /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/share/nginx/conf/
+: fastcgi.conf koi-win scgi_params
+: fastcgi.conf.default mime.types scgi_params.default
+: fastcgi_params mime.types.default uwsgi_params
+: fastcgi_params.default nginx.conf uwsgi_params.default
+: koi-utf nginx.conf.default win-utf
+
+And copy any relevant files to /etc/nginx. A configuration file for
+GeneNetwork (reverse proxy) port forwarding can be found in the source
+repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc
+(still as root)
+: cp ./etc/nginx-genenetwork.conf /etc/nginx/
+
+Make dirs
+
+: mkdir -p /var/spool/nginx/logs
+
+Add users
+
+: adduser nobody ; addgroup nobody
+
+Run nginx
+
+: /root/.guix-profile/sbin/nginx -c /etc/nginx/nginx-genenetwork.conf -p /var/spool/nginx
+
+* Source deployment and other information on reproducibility
+
+See the document [[GUIX-Reproducible-from-source.org]].
+
+** Update to recent guix
+
+We now compile Guix from scratch.
+
+Create, install and run a recent version of the guix-daemon by
+compiling the guix repository you have installed with git in
+step 2. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully after
+
+: cd ~/genenetwork/guix-gn-deploy
+
+Make sure to restart the guix daemon and run guix client from this
+directory.
+
+** Install GN2
+
+Reinstall genenetwork2 using the new tree
+
+#+begin_src bash
+env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --substitute-urls="http://guix.genenetwork.org https://mirror.guixsd.org"
+#+end_src bash
+
+Note the use of ./pre-inst-env here!
+
+Actually, it should be the same installation as in step 4, so nothing
+gets downloaded.
+
+** Run GN2
+
+Make a note of the paths with
+
+#+begin_src bash
+./pre-inst-env guix package --search-paths
+#+end_src bash
+
+or this should also work if guix is installed
+
+#+begin_src bash
+guix package --search-paths
+#+end_src bash
+
+After setting the paths for the server
+
+#+begin_src bash
+export PATH=~/.guix-profile/bin:$PATH
+export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages"
+export R_LIBS_SITE="$HOME/.guix-profile/site-library/"
+export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0"
+export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0"
+export XDG_DATA_DIRS="$HOME/.guix-profile/share"
+export GIO_EXTRA_MODULES="$HOME/.guix-profile/lib/gio/modules"
+#+end_src bash
+
+run the main script (in ~/.guix-profile/bin)
+
+#+begin_src bash
+genenetwork2
+#+end_src bash
+
+will start the default server which listens on port 5003, i.e.,
+http://localhost:5003/.
+
+OK, we are where we were before with step 4. Only difference is that we
+used our own compiled guix server.
diff --git a/doc/README.org b/doc/README.org
index b38ea664..a39ef603 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -2,33 +2,27 @@
* Table of Contents :TOC:
- [[#introduction][Introduction]]
- - [[#quick-installation-recipe][Quick installation recipe]]
- - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]]
- - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]]
- - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]]
- - [[#step-4-install-and-run-gn2][Step 4: Install and run GN2]]
+ - [[#install][Install]]
+ - [[#tarball][Tarball]]
+ - [[#docker][Docker]]
+ - [[#with-source][With source]]
+ - [[#running-gn2][Running GN2]]
- [[#run-mysql-server][Run MySQL server]]
+ - [[#install-mysql-with-gnu-guix][Install MySQL with GNU GUIx]]
+ - [[#load-the-small-database-in-mysql][Load the small database in MySQL]]
- [[#gn2-dependency-graph][GN2 Dependency Graph]]
- - [[#source-deployment][Source deployment]]
- - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]]
- - [[#set-up-nginx-port-forwarding][Set up nginx port forwarding]]
- - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]]
- - [[#update-to-recent-guix][Update to recent guix]]
- - [[#install-gn2][Install GN2]]
- - [[#run-gn2][Run GN2]]
+ - [[#working-with-the-gn2-source-code][Working with the GN2 source code]]
- [[#trouble-shooting][Trouble shooting]]
- [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]]
- [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]]
- [[#cant-run-a-module][Can't run a module]]
- [[#rpy2-error-show-now-found][Rpy2 error 'show' now found]]
+ - [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]]
+ - [[#read-more][Read more]]
- [[#irc-session][IRC session]]
* Introduction
-If you want to understand the architecture of GN2 read
-[[Architecture.org]]. The rest of this document is mostly on deployment
-of GN2.
-
Large system deployments can get very [[http://biogems.info/contrib/genenetwork/gn2.svg ][complex]]. In this document we
explain the GeneNetwork version 2 (GN2) reproducible deployment system
which is based on GNU Guix (see also [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix
@@ -37,195 +31,113 @@ system can be used to install GN with all its files and dependencies.
The official installation path is from a checked out version of the
main Guix package tree and that of the Genenetwork package
tree. Current supported versions can be found as the SHA values of
-'gn-latest' branches of [[https://github.com/genenetwork/guix-bioinformatics/tree/gn-latest][Guix bioinformatics]] and [[https://github.com/genenetwork/guix/tree/gn-latest][GNU Guix main]].
+'gn-latest' branches of [[https://gitlab.com/genenetwork/guix-bioinformatics][Guix bioinformatics]] and [[https://gitlab.com/genenetwork/guix][GNU Guix]].
For a full view of runtime dependencies as defined by GNU Guix, see
-the [[#gn2-dependency-graph][GN2 Dependency Graph]].
-
-* Quick installation recipe
-
-This is a recipe for quick and dirty installation of GN2. For
-convenience everything is installed as root, though in reality only
-GNU Guix has to be installed as root. I tested this recipe on a fresh
-install of Debian 8.3.0 (in KVM) though it should work on any modern
-Linux distribution (including CentOS). For more elaborate installation
-instructions see [[#source-deployment][Source deployment]].
-
-Note that GN2 consists of an approx. 5 GB installation including
-database. If you use a virtual machine we recommend to use at least
-double.
-
-** Step 1: Install GNU Guix
-
-Fetch the GNU Guix binary from [[https://www.gnu.org/software/guix/download/][here]] (middle panel) and follow
-[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html][instructions]]. Essentially, download and unpack the tar ball (which
-creates directories in /gnu and /var/guix), add build users and group
-(Guix builds software as unpriviliged users) and run the Guix daemon
-after fixing the paths (also known as the 'profile').
-
-Once you have succeeded, you have to [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-the-key][set the key]] (getting permission
-to download binaries from the GNU server) and you should be able to
-install the hello package using binary packages (no building)
-
-#+begin_src bash
-export PATH=~/.guix-profile/bin:$PATH
-guix pull
-guix package -i hello --dry-run
-#+end_src
-
-Which should show something like
-
-: The following files would be downloaded:
-: /gnu/store/zby49aqfbd9w9br4l52mvb3y6f9vfv22-hello-2.10
-: ...
-#+end_src
-
-means binary installs. The actual installation command of 'hello' is
-
-#+begin_src bash
-guix package -i hello
-hello
- Hello, world!
-#+end_src
-
-If you actually see things building it means that Guix is not yet
-properly installed and up-to-date, i.e., the key is missing or you
-need to do a 'guix pull'. Press Ctrl-C to interrupt.
-
-If you need more help we have another writeup in [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#binary-installation][guix-notes]]. To get
-rid of the locale warning see [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-locale][set-locale]].
-
-** Step 2: Checkout the GN2 git repositories
+an example of the [[#gn2-dependency-graph][GN2 Dependency Graph]].
-To fixate the software dependency graph GN2 uses git repositories of
-Guix packages. First install git if it is missing
+* Install
-#+begin_src bash
-guix package -i git
-export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
-#+end_src
+The quickest way to install GN2 is by using a binary installation
+(tarball or Docker image). These installations are bundled by GNU
+Guix and include all dependencies. You can install GeneNetwork on most
+Linux distributions, including Debian, Ubuntu, Fedora and CentOS,
+provided you have administrator privileges (root). The alternative is
+a Docker installation.
-check out the git repositories (gn-deploy branch)
+** Tarball
-#+begin_src bash
-cd ~
-mkdir genenetwork
-cd genenetwork
-git clone --branch gn-deploy https://github.com/genenetwork/guix-bioinformatics
-git clone --branch gn-deploy --recursive https://github.com/genenetwork/guix guix-gn-deploy
-cd guix-gn-deploy
-#+end_src bash
+Download the ~800Mb tarball from
+[[http://files.genenetwork.org/software/binary_tarball/]]. Validate the checksum and
+unpack to root, for example
-To test whether this is working try:
+: tar xvzf genenetwork2-2.10rc3-1538ffd-tarball-pack.tar.gz
+: mv /gnu /
+: mv /opt/genenetwork2 /opt/
-#+begin_src bash
-#+end_src bash
+Now you shoud be able to start the server with
+: /opt/genenetwork2/bin/genenetwork2
-** Step 3: Authorize the GN Guix server
+When the server stops with a MySQL error [[#run-mysql-server][Run MySQL server]]
+and set SQL_URI to point at it. For example:
-GN2 has its own GNU Guix binary distribution server. To trust it you have
-to add the following key
+: export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s
-#+begin_src scheme
-(public-key
- (ecc
- (curve Ed25519)
- (q #11217788B41ADC8D5B8E71BD87EF699C65312EC387752899FE9C888856F5C769#)
- )
-)
-#+end_src
+See also [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]].
-by pasting it into the command
+** Docker
-#+begin_src bash
-guix archive --authorize
-#+end_src
+Docker images are also available through
+[[http://files.genenetwork.org/software/]]. Validate the checksum and run
+with [[https://docs.docker.com/engine/reference/commandline/load/][Docker load]].
-and hit Ctrl-D.
+** With source
-Now you can use the substitute server to install GN2 binaries.
+For more elaborate installation instructions on deploying GeneNetwork from
+source see [[#source-deployment][Source deployment]].
-** Step 4: Install and run GN2
+* Running GN2
-Since this is a quick and dirty install we are going to override the
-GNU Guix package path by pointing the package path to our repository:
+Default settings for GN2 are listed in a file called
+[[../etc/default_settings.py][default_settings.py]]. You can copy this file and pass it as a new
+parameter to the genenetwork2 command, e.g.
-#+begin_src bash
-rm /root/.config/guix/latest
-ln -s ~/genenetwork/guix-gn-deploy/ /root/.config/guix/latest
-#+end_src
+: genenetwork2 mysettings.py
-Now check whether you can find the GN2 package with
+or you can set environment variables to override individual parameters, e.g.
-#+begin_src bash
-env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -A genenetwork2
- genenetwork2 2.0-a8fcff4 out gn/packages/genenetwork.scm:144:2
-#+end_src
-
-(ignore the source file newer then ... messages, this is caused by the
-/root/.config/guix/latest override).
+: env SERVER_PORT=5004 SQL_URI=mysql://user:pwd@dbhostname/db_webqtl genenetwork2
-And install with
+the debug and logging switches can be particularly useful when
+developing GN2.
-#+begin_src bash
-env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ \
- guix package -i genenetwork2 \
- --substitute-urls="http://guix.genenetwork.org"
-#+end_src
-
-Note: the order of the substitute url's may make a difference in speed
-(put the one first that is fastest for your location and time of day).
-
-Note: if your system starts building or gives an error it may well be
-Step 3 did not succeed. The installation should actually be smooth at
-this point and only do binary installs (no compiling).
-
-After installation you should be able to run genenetwork2 after updating
-the Guix suggested environment vars. Check the output of
-
-#+begin_src bash
-guix package --search-paths
-export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages"
-export R_LIBS_SITE="/root/.guix-profile/site-library/"
-#+end_src
+* Run MySQL server
+** Install MySQL with GNU GUIx
-and copy-paste the listed exports into the terminal before running:
+These are the steps you can take to install a fresh installation of
+mysql (which comes as part of the GNU Guix genenetwork2 install).
-#+begin_src bash
-genenetwork2
-#+end_src
+As root configure and run
-It will complain that the database is missing. See the next section on
-running MySQL server for downloading and installing a MySQL GN2
-database. After installing the database restart genenetwork2 and point
-your browser at [[http://localhost:5003/]].
+: adduser mysql && addgroup mysql
+: mysqld --datadir=/var/mysql --initialize-insecure
+: mkdir -p /var/run/mysqld
+: chown mysql.mysql ~/mysql /var/run/mysqld
+: su mysql -c mysqld --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048
-End of the GN2 installation recipe!
+/etc/my.cnf
+[mysqld]
+user=root
-* Run MySQL server
+** Load the small database in MySQL
At this point we require the underlying distribution to install and
-run mysqld. Currently we have two databases for deployment,
+run mysqld (see next section for GNU Guix). Currently we have two databases for deployment,
'db_webqtl_s' is the small testing database containing experiments
from BXD mice and 'db_webqtl_plant' which contains all plant related
material.
Download one database from
-http://files.genenetwork.org/raw_database/
-https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip
+[[http://files.genenetwork.org/raw_database/]]
+
+[[https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip]]
Check the md5sum.
After installation inflate the database binary in the MySQL directory
-(this installation path is subject to change soon)
+: cd ~/mysql
: chown -R mysql:mysql db_webqtl_s/
: chmod 700 db_webqtl_s/
: chmod 660 db_webqtl_s/*
-restart MySQL service (mysqld). Login as root and
+restart MySQL service (mysqld). Login as root
+
+: myslq -u root
+
+and
: mysql> show databases;
: +--------------------+
@@ -241,9 +153,12 @@ Set permissions and match password in your settings file below:
: mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password';
+You may need to change "localhost" to whatever domain you are
+connecting from (mysql will give an error).
+
Note that if the mysql connection is not working, try connecting to
the IP address and check server firewall, hosts.allow and mysql IP
-configuration.
+configuration (see below).
Note for the plant database you can rename it to db_webqtl_s, or
change the settings in etc/default_settings.py to match your path.
@@ -255,183 +170,9 @@ Graph of all runtime dependencies as installed by GNU Guix.
#+ATTR_HTML: :title GN2_graph
http://biogems.info/contrib/genenetwork/gn2.svg
-* Source deployment
-
-This section gives a more elaborate instruction for installing GN2
-from source.
-
-First execute above 4 steps:
-
- - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]]
- - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]]
- - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]]
- - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]]
-
-
-** Run your own copy of GN2
-
-At some point you may want to fix the source code. Assuming you have
-Guix and Genenetwork2 installed (as described above) clone the GN2
-repository from https://github.com/genenetwork/genenetwork2.
-
-Copy-paste the paths into your terminal (mainly so PYTHON_PATH and
-R_LIBS_SITE are set) from the information given by guix:
-
-: guix package --search-paths
-
-Inside the repository:
-
-: cd genenetwork2
-: ./bin/genenetwork2
-
-Will fire up your local repo http://localhost:5003/ using the
-settings in ./etc/default_settings.py. These settings may
-not reflect your system. To override settings create your own from a copy of
-default_settings.py and pass it into GN2 with
-
-: ./bin/genenetwork2 $HOME/my_settings.py
-
-and everything *should* work (note the full path to the settings
-file). This way we develop against the exact same dependency graph of
-software.
-
-If something is not working, take a hint from the settings file
-that comes in the Guix installation. It sits in something like
+* Working with the GN2 source code
-: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py
-
-** Set up nginx port forwarding
-
-nginx can be used as a reverse proxy for GN2. For example, we want to
-expose GN2 on port 80 while it is running on port 5003. Essentially
-the configuration looks like
-
-#+begin_src js
- server {
- listen 80;
- server_name test-gn2.genenetwork.org;
- access_log logs/test-gn2.access.log;
-
- proxy_connect_timeout 3000;
- proxy_send_timeout 3000;
- proxy_read_timeout 3000;
- send_timeout 3000;
-
- location / {
- proxy_set_header Host $http_host;
- proxy_set_header Connection keep-alive;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Host $server_name;
- proxy_pass http://127.0.0.1:5003;
- }
-}
-#+end_src js
-
-Install the nginx webserver (as root)
-
-: guix package -i nginx
-
-The nginx example configuration examples can be found in the Guix
-store through
-
-: ls -l /root/.guix-profile/sbin/nginx
-: lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx
-
-Use that path
-
-: ls /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/share/nginx/conf/
-: fastcgi.conf koi-win scgi_params
-: fastcgi.conf.default mime.types scgi_params.default
-: fastcgi_params mime.types.default uwsgi_params
-: fastcgi_params.default nginx.conf uwsgi_params.default
-: koi-utf nginx.conf.default win-utf
-
-And copy any relevant files to /etc/nginx. A configuration file for
-GeneNetwork (reverse proxy) port forwarding can be found in the source
-repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc
-(still as root)
-: cp ./etc/nginx-genenetwork.conf /etc/nginx/
-
-Make dirs
-
-: mkdir -p /var/spool/nginx/logs
-
-Add users
-
-: adduser nobody ; addgroup nobody
-
-Run nginx
-
-: /root/.guix-profile/sbin/nginx -c /etc/nginx/nginx-genenetwork.conf -p /var/spool/nginx
-
-* Source deployment and other information on reproducibility
-
-See the document [[GUIX-Reproducible-from-source.org]].
-
-** Update to recent guix
-
-We now compile Guix from scratch.
-
-Create, install and run a recent version of the guix-daemon by
-compiling the guix repository you have installed with git in
-step 2. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully after
-
-: cd ~/genenetwork/guix-gn-deploy
-
-Make sure to restart the guix daemon and run guix client from this
-directory.
-
-** Install GN2
-
-Reinstall genenetwork2 using the new tree
-
-#+begin_src bash
-env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --substitute-urls="http://guix.genenetwork.org https://mirror.guixsd.org"
-#+end_src bash
-
-Note the use of ./pre-inst-env here!
-
-Actually, it should be the same installation as in step 4, so nothing
-gets downloaded.
-
-** Run GN2
-
-Make a note of the paths with
-
-#+begin_src bash
-./pre-inst-env guix package --search-paths
-#+end_src bash
-
-or this should also work if guix is installed
-
-#+begin_src bash
-guix package --search-paths
-#+end_src bash
-
-After setting the paths for the server
-
-#+begin_src bash
-export PATH=~/.guix-profile/bin:$PATH
-export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages"
-export R_LIBS_SITE="$HOME/.guix-profile/site-library/"
-export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0"
-export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0"
-export XDG_DATA_DIRS="$HOME/.guix-profile/share"
-export GIO_EXTRA_MODULES="$HOME/.guix-profile/lib/gio/modules"
-#+end_src bash
-
-run the main script (in ~/.guix-profile/bin)
-
-#+begin_src bash
-genenetwork2
-#+end_src bash
-
-will start the default server which listens on port 5003, i.e.,
-http://localhost:5003/.
-
-OK, we are where we were before with step 4. Only difference is that we
-used our own compiled guix server.
+See [[development.org]].
* Trouble shooting
@@ -479,6 +220,31 @@ R_LIBS_SITE. Please check your GNU Guix GN2 installation paths,
you man need to reinstall. Note that this may be the point you
may want to start using profiles (see profile section).
+** Mysql can't connect server through socket ERROR
+
+The following error
+
+: sqlalchemy.exc.OperationalError: (_mysql_exceptions.OperationalError) (2002, 'Can\'t connect to local MySQL server through socket \'/run/mysqld/mysqld.sock\' (2 "No such file or directory")')
+
+means that MySQL is trying to connect locally to a non-existent MySQL
+server, something you may see in a container. Typically replicated with something like
+
+: mysql -h localhost
+
+try to connect over the network interface instead, e.g.
+
+: mysql -h 127.0.0.1
+
+if that works run genenetwork after setting SQL_URI to something like
+
+: export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s
+
+* Read more
+
+If you want to understand the architecture of GN2 read
+[[Architecture.org]]. The rest of this document is mostly on deployment
+of GN2.
+
* IRC session
Here an IRC session where we installed GN2 from scratch using GNU Guix
diff --git a/doc/development.org b/doc/development.org
new file mode 100644
index 00000000..5e6e318b
--- /dev/null
+++ b/doc/development.org
@@ -0,0 +1,43 @@
+* Development
+
+** Using GN2_PROFILE
+
+After cloning the git source tree you can run the contained GN2 using
+an existing GN2_PROFILE, i.e., use a profile that was create to run a
+binary installation of GN2. This profile may be found by typing
+
+: which genenetwork2
+: /home/wrk/opt/gn-latest-guix/bin/genenetwork2
+
+An example of running the development version would be
+
+: env GN2_PROFILE=/home/wrk/opt/gn-latest-guix ./bin/genenetwork2
+
+Profiles are stored in /gnu/store, so you may pick one up there
+
+: readlink -f $(dirname $(dirname `which genenetwork2`))
+: /gnu/store/dvckpaw770b00l6rv4ijql8wrk11iypv-profile
+
+and use that instead.
+
+Note that the genenetwork2 script sets up the environment for running
+the webserver. This includes path to R modules and python modules. These
+are output on startup. To make sure there is no environment pollution you can
+
+** Javascript modules
+
+As of release 2.10-pre4 we Javascript modules are installed in three places:
+
+1. JS_GUIX_PATH: the Guix store - these are Guix pre-packaged modules
+2. The git source tree (./wqflask/wqflask/static/packages/)
+3. JS_GN_PATH: a local directory containing (temporary) development modules
+
+Packages currently in git (2) will move to JS_GUIX_PATH (1) over
+time. This is to keep better track of origin updates. Putting packages
+in git (2) is actively discouraged(!), unless there are GN2 specific
+adaptations to the original Javascript modules.
+
+JS_GN_PATH (3) is for development purposes. By default is is set to
+$HOME/genenetwork/javascript. Say you are working on an updated
+version of a JS module not yet in (1) you can simply check out that
+module in that path and it should show up.
diff --git a/etc/VERSION b/etc/VERSION
new file mode 100644
index 00000000..1785aa28
--- /dev/null
+++ b/etc/VERSION
@@ -0,0 +1 @@
+2.10rc3
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 6acea637..1b609414 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -9,6 +9,8 @@
# env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2
# env LOG_LEVEL=DEBUG ./bin/genenetwork2 ~/gn2_settings.py
#
+# Typically you need to set GN2_PROFILE too.
+#
# Note also that in the near future we will additionally fetch
# settings from a JSON file
#
@@ -18,10 +20,9 @@
import os
import sys
-GN_VERSION = open("../VERSION","r").read()
+GN_VERSION = open("../etc/VERSION","r").read()
SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s"
-SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password@localhost/db_webqtl_s'
-SQLALCHEMY_POOL_RECYCLE = 3600
+SQL_ALCHEMY_POOL_RECYCLE = 3600
GN_SERVER_URL = "http://localhost:8880/"
# ---- Flask configuration (see website)
@@ -45,24 +46,26 @@ WEBSERVER_DEPLOY = None # Deployment specifics (nyi)
LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL)
LOG_LEVEL_DEBUG = '0' # logger.debugf log level (0-5, 5 = show all)
LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls
-LOG_SQLALCHEMY = 'False'
+LOG_SQL_ALCHEMY = 'False'
LOG_BENCH = True # Log bench marks
USE_REDIS = True # REDIS caching (note that redis will be phased out)
USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls
+HOME = os.environ['HOME']
-# Paths to JS libraries
+# ---- Default locations
+GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files
-TWITTER_POST_FETCHER_JS_PATH = os.environ['HOME']+"/genenetwork/Twitter-Post-Fetcher"
+# ---- Path overrides for Genenetwork - the defaults are normally
+# picked up from Guix or in the HOME directory
-# ---- Path overrides for Genenetwork
# TMPDIR is normally picked up from the environment
-HOME=os.environ['HOME']
-LOGFILE = HOME+"/genenetwork2.log"
-GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files
-LOCAL_PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files
+# PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused)
+
+# ---- Local path to JS libraries - for development modules (only)
+JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript"
-# ---- GN2 Executables
-PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read())
-PLINK_COMMAND = str.strip(os.popen("which plink2").read())
-GEMMA_COMMAND = str.strip(os.popen("which gemma").read())
+# ---- GN2 Executables (overwrite for testing only)
+# PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read())
+# PLINK_COMMAND = str.strip(os.popen("which plink2").read())
+# GEMMA_COMMAND = str.strip(os.popen("which gemma").read())
diff --git a/test/lib/main_web_functionality.rb b/test/lib/main_web_functionality.rb
index ec61de2d..9ae49995 100644
--- a/test/lib/main_web_functionality.rb
+++ b/test/lib/main_web_functionality.rb
@@ -13,16 +13,20 @@ describe MainWebFunctionality do
it "Get to trait page" do
page = @agent.get($host)
+ p page
form = page.forms[1]
form.buttons[0].value.must_equal "Search" # main menu is loaded
# http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult
+ form.fields[0].value.must_equal "searchResult"
form.fields[2].value = "MEAN=(15 16) LRS=(23 46)"
form.fields[3].value = "mouse"
form.fields[4].value = "BXD"
form.fields[5].value = "Hippocampus mRNA"
form.fields[6].value = "HC_M2_0606_P"
search_page = @agent.submit(form, form.buttons.first)
+ p "=================="
+ p search_page
probe_link = search_page.links.find { |l| l.text =~ /1435395_s_at/ }
probe_link.uri.to_s.must_equal "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P"
show_trait_page = probe_link.click
diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index 1e47e183..e5f10edf 100644
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -65,7 +65,7 @@ ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?sp
# want to reach this base dir
assert_writable_dir(TEMPDIR)
-TMPDIR = mk_dir(TEMPDIR+'gn2')
+TMPDIR = mk_dir(TEMPDIR+'/gn2/')
assert_writable_dir(TMPDIR)
CACHEDIR = mk_dir(TMPDIR+'/cache/')
diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py
index e5726656..a5fd86e2 100644
--- a/wqflask/maintenance/gen_select_dataset.py
+++ b/wqflask/maintenance/gen_select_dataset.py
@@ -57,7 +57,7 @@ import urlparse
from pprint import pformat as pf
-#Engine = sa.create_engine(zach_settings.SQLALCHEMY_DATABASE_URI)
+#Engine = sa.create_engine(zach_settings.SQL_URI)
# build MySql database connection
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index c28c617a..393ff2df 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -13,6 +13,11 @@ logger = logging.getLogger(__name__ )
OVERRIDES = {}
+def app_set(command_id, value):
+ """Set application wide value"""
+ app.config.setdefault(command_id,value)
+ value
+
def get_setting(command_id,guess=None):
"""Resolve a setting from the environment or the global settings in
app.config, with valid_path is a function checking whether the
@@ -40,6 +45,7 @@ def get_setting(command_id,guess=None):
def value(command):
if command:
# sys.stderr.write("Found "+command+"\n")
+ app_set(command_id,command)
return command
else:
return None
@@ -89,6 +95,18 @@ def valid_path(dir):
return dir
return None
+def js_path(module=None):
+ """
+ Find the JS module in the two paths
+ """
+ try_gn = get_setting("JS_GN_PATH")+"/"+module
+ if valid_path(try_gn):
+ return try_gn
+ try_guix = get_setting("JS_GUIX_PATH")+"/"+module
+ if valid_path(try_guix):
+ return try_guix
+ raise "No JS path found for "+module+" (check JS_GN_PATH)"
+
def pylmm_command(guess=None):
return valid_bin(get_setting("PYLMM_COMMAND",guess))
@@ -205,21 +223,27 @@ SQL_URI = get_setting('SQL_URI')
LOG_LEVEL = get_setting('LOG_LEVEL')
LOG_LEVEL_DEBUG = get_setting_int('LOG_LEVEL_DEBUG')
LOG_SQL = get_setting_bool('LOG_SQL')
-LOG_SQLALCHEMY = get_setting_bool('LOG_SQLALCHEMY')
+LOG_SQL_ALCHEMY = get_setting_bool('LOG_SQL_ALCHEMY')
LOG_BENCH = get_setting_bool('LOG_BENCH')
LOG_FORMAT = "%(message)s" # not yet in use
USE_REDIS = get_setting_bool('USE_REDIS')
USE_GN_SERVER = get_setting_bool('USE_GN_SERVER')
GENENETWORK_FILES = get_setting('GENENETWORK_FILES')
-TEMP_TRAITS = get_setting('TEMP_TRAITS')
-
-PYLMM_COMMAND = pylmm_command()
-GEMMA_COMMAND = gemma_command()
-GEMMA_RESULTS_PATH = get_setting('GEMMA_RESULTS_PATH')
-PLINK_COMMAND = plink_command()
+JS_GUIX_PATH = get_setting('JS_GUIX_PATH')
+# assert_dir(JS_GUIX_PATH) - don't enforce right now
+JS_GN_PATH = get_setting('JS_GN_PATH')
+# assert_dir(JS_GN_PATH)
+
+PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command())
+GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command())
+PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command())
TEMPDIR = tempdir() # defaults to UNIX TMPDIR
+# ---- Handle specific JS modules
+JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher"))
+assert_dir(JS_TWITTER_POST_FETCHER_PATH)
+
from six import string_types
if os.environ.get('WQFLASK_OVERRIDES'):
@@ -234,5 +258,3 @@ if os.environ.get('WQFLASK_OVERRIDES'):
else:
OVERRIDES[k] = cmd
logger.debug(OVERRIDES)
-
-assert_dir(get_setting("TWITTER_POST_FETCHER_JS_PATH"))
diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py
index 4df872ad..96c2c301 100644
--- a/wqflask/wqflask/database.py
+++ b/wqflask/wqflask/database.py
@@ -22,8 +22,8 @@ def init_db():
# they will be registered properly on the metadata. Otherwise
# you will have to import them first before calling init_db()
#import yourapplication.models
+ logger.info("Initializing database connection")
import wqflask.model
- logger.debug("Creating all model metadata")
Base.metadata.create_all(bind=engine)
logger.info("Done creating all model metadata")
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index 87532445..8882c515 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -37,7 +37,7 @@ from utility import temp_data
from utility.benchmark import Bench
from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping
-from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, GEMMA_RESULTS_PATH, PLINK_COMMAND, TEMPDIR
+from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR
from utility.external import shell
from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR
diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
index 49959a74..82a44796 100644
--- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py
+++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py
@@ -1018,7 +1018,7 @@ class MarkerRegression(object):
else:
if self.selectedChr > -1:
for i, qtlresult in enumerate(self.qtlresults):
- if qtlresult['Chr'] != self.selectedChr:
+ if qtlresult['chr'] != self.selectedChr:
continue
if i==0 and qtlresult['Mb'] >= Mb:
@@ -1027,7 +1027,7 @@ class MarkerRegression(object):
#the trait's position is between two traits
if i > 0 and self.qtlresults[i-1]['Mb'] < Mb and qtlresult['Mb'] >= Mb:
- locPixel = xLeftOffset + plotXScale*(self.qtlresults[i-1]['cM']+(qtlresult['cM']-self.qtlresults[i-1]['cM'])*(Mb - self.qtlresults[i-1]['Mb'])/(qtlresult['Mb']-self.qtlresults[i-1]['Mb']))
+ locPixel = xLeftOffset + plotXScale*(self.qtlresults[i-1]['Mb']+(qtlresult['Mb']-self.qtlresults[i-1]['Mb'])*(Mb - self.qtlresults[i-1]['Mb'])/(qtlresult['Mb']-self.qtlresults[i-1]['Mb']))
break
#the trait's position is on the right of the last genotype
@@ -1036,11 +1036,12 @@ class MarkerRegression(object):
else:
locPixel = xLeftOffset
for i, _chr in enumerate(self.ChrList):
- if _chr != Chr:
- locPixel += (self.ChrLengthDistList[i] + self.GraphInterval)*plotXScale
- else:
- locPixel += (Mb*(_chr[-1].cM-_chr[0].cM)/self.ChrLengthCMList[i])*plotXScale
- break
+ if i < (len(self.ChrList)-1):
+ if _chr != Chr:
+ locPixel += (self.ChrLengthDistList[i] + self.GraphInterval)*plotXScale
+ else:
+ locPixel += (Mb*(_chr[-1].cM-_chr[0].cM)/self.ChrLengthCMList[i])*plotXScale
+ break
if locPixel >= 0 and self.plotScale == 'physic':
traitPixel = ((locPixel, yZero), (locPixel-6, yZero+12), (locPixel+6, yZero+12))
canvas.drawPolygon(traitPixel, edgeColor=pid.black, fillColor=self.TRANSCRIPT_LOCATION_COLOR, closed=1)
diff --git a/wqflask/wqflask/static/new/css/corr_scatter_plot2.css b/wqflask/wqflask/static/new/css/corr_scatter_plot2.css
index 92e777c2..5aa9b9bf 100644
--- a/wqflask/wqflask/static/new/css/corr_scatter_plot2.css
+++ b/wqflask/wqflask/static/new/css/corr_scatter_plot2.css
@@ -1,19 +1,40 @@
.nvd3 .nv-axis.nv-x text {
- font-size: 16px;
- font-weight: normal;
- fill: black;
+ font-size: 16px;
+ font-weight: normal;
+ fill: black;
}
.nvd3 .nv-axis.nv-y text {
- font-size: 16px;
- font-weight: normal;
- fill: black;
+ font-size: 16px;
+ font-weight: normal;
+ fill:black;
}
-.nv-y .tick.zero line {
- stroke: black;
+.nv-x .nv-axis g path.domain {
+ stroke: black;
+ stroke-width: 2;
}
.nv-y .nv-axis g path.domain {
- stroke: black;
+ stroke: black;
+ stroke-width: 2;
+}
+
+.nvd3 .nv-axis.nv-x path.domain {
+ stroke-opacity: 1;
+}
+
+.nvd3 .nv-axis.nv-y path.domain {
+ stroke-opacity: 1;
+}
+
+line.nv-regLine {
+ stroke: red !important;
+}
+
+.nv-axisMin-x,
+.nv-axisMax-x,
+.nv-axisMin-y,
+.nv-axisMax-y {
+ display: none;
}
diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
index 861af3d6..f3712b5d 100644
--- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
+++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
@@ -1340,14 +1340,14 @@
"GSE9588 Human Liver Normal (Mar11) Both Sexes"
],
[
- "384",
- "HLCF_0311",
- "GSE9588 Human Liver Normal (Mar11) Females"
- ],
- [
"383",
"HLCM_0311",
"GSE9588 Human Liver Normal (Mar11) Males"
+ ],
+ [
+ "384",
+ "HLCF_0311",
+ "GSE9588 Human Liver Normal (Mar11) Females"
]
],
"Phenotypes": [
@@ -1493,6 +1493,15 @@
]
]
},
+ "Islets-Gerling": {
+ "Phenotypes": [
+ [
+ "None",
+ "Islets-GerlingPublish",
+ "Islets-Gerling Published Phenotypes"
+ ]
+ ]
+ },
"TIGEM-Retina-RNA-Seq": {
"Phenotypes": [
[
@@ -1561,6 +1570,11 @@
],
"Heart mRNA": [
[
+ "820",
+ "UCLA_AXB_BXA_Aor_Jan16",
+ "UCLA AXB/BXA Aorta Affy M430 2.0 (Jan16) RMA"
+ ],
+ [
"421",
"IRCM_AXBXA_HRI0213",
"IRCM AXB/BXA Mouse Heart ILM MouseRef-8 v2.0 (Feb13) RankInv"
@@ -1568,6 +1582,11 @@
],
"Liver mRNA": [
[
+ "822",
+ "UCLA_AXB_BXA_Liv_Jan16",
+ "UCLA AXB/BXA Liver Affy M430 2.0 (Jan16) RMA"
+ ],
+ [
"352",
"GSE16780AB_UCLA_ML0911",
"GSE16780 UCLA Mouse AXB/BXA Liver Affy HT M430A (Sep11) RMA"
@@ -1727,11 +1746,6 @@
],
"Striatum mRNA": [
[
- "85",
- "SA_M2_0905_P",
- "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) PDNN"
- ],
- [
"84",
"SA_M2_0905_R",
"OHSU/VA B6D2F2 Striatum M430v2 (Sep05) RMA"
@@ -1740,22 +1754,27 @@
"83",
"SA_M2_0905_M",
"OHSU/VA B6D2F2 Striatum M430v2 (Sep05) MAS5"
+ ],
+ [
+ "85",
+ "SA_M2_0905_P",
+ "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) PDNN"
]
]
},
"BHF2": {
"Adipose mRNA": [
[
- "197",
- "UCLA_BHF2_ADIPOSE_FEMALE",
- "UCLA BHF2 Adipose Female mlratio"
- ],
- [
"196",
"UCLA_BHF2_ADIPOSE_MALE",
"UCLA BHF2 Adipose Male mlratio"
],
[
+ "197",
+ "UCLA_BHF2_ADIPOSE_FEMALE",
+ "UCLA BHF2 Adipose Female mlratio"
+ ],
+ [
"165",
"UCLA_BHF2_ADIPOSE_0605",
"UCLA BHF2 Adipose (June05) mlratio"
@@ -1763,16 +1782,16 @@
],
"Brain mRNA": [
[
- "199",
- "UCLA_BHF2_BRAIN_FEMALE",
- "UCLA BHF2 Brain Female mlratio"
- ],
- [
"198",
"UCLA_BHF2_BRAIN_MALE",
"UCLA BHF2 Brain Male mlratio"
],
[
+ "199",
+ "UCLA_BHF2_BRAIN_FEMALE",
+ "UCLA BHF2 Brain Female mlratio"
+ ],
+ [
"166",
"UCLA_BHF2_BRAIN_0605",
"UCLA BHF2 Brain (June05) mlratio"
@@ -1787,16 +1806,16 @@
],
"Liver mRNA": [
[
- "201",
- "UCLA_BHF2_LIVER_FEMALE",
- "UCLA BHF2 Liver Female mlratio"
- ],
- [
"200",
"UCLA_BHF2_LIVER_MALE",
"UCLA BHF2 Liver Male mlratio"
],
[
+ "201",
+ "UCLA_BHF2_LIVER_FEMALE",
+ "UCLA BHF2 Liver Female mlratio"
+ ],
+ [
"167",
"UCLA_BHF2_LIVER_0605",
"UCLA BHF2 Liver (June05) mlratio"
@@ -2000,11 +2019,6 @@
"UTHSC Brain mRNA U74Av2 (Nov05) PDNN"
],
[
- "81",
- "BR_U_0805_P",
- "UTHSC Brain mRNA U74Av2 (Aug05) PDNN"
- ],
- [
"80",
"BR_U_0805_M",
"UTHSC Brain mRNA U74Av2 (Aug05) MAS5"
@@ -2015,6 +2029,11 @@
"UTHSC Brain mRNA U74Av2 (Aug05) RMA"
],
[
+ "81",
+ "BR_U_0805_P",
+ "UTHSC Brain mRNA U74Av2 (Aug05) PDNN"
+ ],
+ [
"42",
"CB_M_0204_P",
"INIA Brain mRNA M430 (Feb04) PDNN"
@@ -2142,6 +2161,13 @@
"BXD Genotypes"
]
],
+ "Heart mRNA": [
+ [
+ "819",
+ "UCLA_BXD_Aor_Jan16",
+ "UCLA BXD Aorta Affy M430 2.0 (Jan16) RMA"
+ ]
+ ],
"Hematopoietic Cells mRNA": [
[
"149",
@@ -2218,6 +2244,26 @@
"UMUTAffy Hippocampus Exon (Feb09) RMA"
],
[
+ "814",
+ "UTHSC_ILM_BXD_hipp_NOSb_0217",
+ "UTHSC BXD Hippocampus ILM v6.1 NOS Balanced (Feb17) RankInv"
+ ],
+ [
+ "815",
+ "UTHSC_ILM_BXD_hipp_NOEb_0217",
+ "UTHSC BXD Hippocampus ILM v6.1 NOE Balanced (Feb17) RankInv"
+ ],
+ [
+ "816",
+ "UTHSC_ILM_BXD_hipp_RSSb_0217",
+ "UTHSC BXD Hippocampus ILM v6.1 RSS Balanced (Feb17) RankInv"
+ ],
+ [
+ "817",
+ "UTHSC_ILM_BXD_hipp_RSEb_0217",
+ "UTHSC BXD Hippocampus ILM v6.1 RSE Balanced (Feb17) RankInv"
+ ],
+ [
"780",
"UTHSC_ILM_BXD_hipp_NOEb_0216",
"UTHSC BXD Hippocampus ILM v6.1 NOE Balanced (Feb16) RankInv"
@@ -2391,14 +2437,24 @@
],
"Liver Proteome": [
[
+ "540",
+ "EPFLETHZBXDprotCD0514",
+ "EPFL/ETHZ BXD Liver, Chow Diet (Jun16) Top100 SWATH"
+ ],
+ [
+ "541",
+ "EPFLETHZBXDprotHFD0514",
+ "EPFL/ETHZ BXD Liver, High Fat Diet (Jun16) Top100 SWATH"
+ ],
+ [
"704",
"EPFLETHZBXDprotCD_LS1114",
- "EPFL/ETHZ BXD Liver, Chow Diet (Oct14) Light SWATH"
+ "EPFL/ETHZ BXD Liver, Chow Diet (Oct14) Top10 SWATH"
],
[
"705",
"EPFLETHZBXDprotHF_LS1114",
- "EPFL/ETHZ BXD Liver, High Fat Diet (Oct14) Light SWATH"
+ "EPFL/ETHZ BXD Liver, High Fat Diet Diet (Oct14) Top10 SWATH"
],
[
"703",
@@ -2414,20 +2470,15 @@
"489",
"EPFLBXDprotHFDRPN0214",
"EPFL/LISP BXD Liver, Soluble Proteins HFD (Feb14) SRM"
- ],
- [
- "540",
- "EPFLETHZBXDprotCD0514",
- "EPFL/ETHZ BXD Liver, Soluble Proteins CD (Jun14) SWATH"
- ],
- [
- "541",
- "EPFLETHZBXDprotHFD0514",
- "EPFL/ETHZ BXD Liver, Soluble Proteins HFD (Jun14) SWATH"
]
],
"Liver mRNA": [
[
+ "818",
+ "UCLA_BXD_Liv_Jan16",
+ "UCLA BXD Liver Affy M430 2.0 (Jan16) RMA"
+ ],
+ [
"430",
"EPFLMouseLiverRMA0413",
"EPFL/LISP BXD CD+HFD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA"
@@ -2443,6 +2494,11 @@
"EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA"
],
[
+ "433",
+ "EPFLMouseLiverBothExRMA0413",
+ "EPFL/LISP BXD CD+HFD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA Exon Level"
+ ],
+ [
"700",
"UTHSC-VGX_MmBXDHepatocytesRMA1014",
"UT-VGX Hepatocytes Affy Mouse Gene 1.0 ST Gene Level (Oct14) RMA"
@@ -2466,6 +2522,51 @@
"702",
"SUH_Liv_RMAEx_0611",
"SUH BXD Liver CCl4-treated Affy Mouse Gene 1.0 ST Exon Level (Jun11) RMA"
+ ],
+ [
+ "256",
+ "GenEx_BXD_liverEt_M5_0912",
+ "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Both Sexes"
+ ],
+ [
+ "257",
+ "GenEx_BXD_liverEt_M5M_0912",
+ "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Males"
+ ],
+ [
+ "258",
+ "GenEx_BXD_liverEt_M5F_0912",
+ "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Females"
+ ],
+ [
+ "307",
+ "GenEx_BXD_liverEt_RMA_0211",
+ "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Both Sexes"
+ ],
+ [
+ "308",
+ "GenEx_BXD_liverEt_RMA_M_0211",
+ "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Males"
+ ],
+ [
+ "309",
+ "GenEx_BXD_liverEt_RMA_F_0211",
+ "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Females"
+ ],
+ [
+ "310",
+ "GenEx_BXD_liverSal_RMA_0211",
+ "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Both Sexes"
+ ],
+ [
+ "311",
+ "GenEx_BXD_liverSal_RMA_M_0211",
+ "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Males"
+ ],
+ [
+ "312",
+ "GenEx_BXD_liverSal_RMA_F_0211",
+ "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Females"
]
],
"Lung mRNA": [
@@ -2546,14 +2647,14 @@
"HQF BXD Neocortex ILM6v1.1 (Feb08) RankInv"
],
[
- "275",
- "DevNeocortex_ILM6.2P14RInv_1110",
- "BIDMC/UTHSC Dev Neocortex P14 ILMv6.2 (Nov10) RankInv"
- ],
- [
"274",
"DevNeocortex_ILM6.2P3RInv_1110",
"BIDMC/UTHSC Dev Neocortex P3 ILMv6.2 (Nov10) RankInv"
+ ],
+ [
+ "275",
+ "DevNeocortex_ILM6.2P14RInv_1110",
+ "BIDMC/UTHSC Dev Neocortex P14 ILMv6.2 (Nov10) RankInv"
]
],
"Nucleus Accumbens mRNA": [
@@ -2761,11 +2862,6 @@
],
"Ventral Tegmental Area mRNA": [
[
- "230",
- "VCUEtvsSal_0609_R",
- "VCU BXD VTA Et vs Sal M430 2.0 (Jun09) RMA"
- ],
- [
"229",
"VCUEtOH_0609_R",
"VCU BXD VTA EtOH M430 2.0 (Jun09) RMA"
@@ -2774,6 +2870,11 @@
"228",
"VCUSal_0609_R",
"VCU BXD VTA Sal M430 2.0 (Jun09) RMA"
+ ],
+ [
+ "230",
+ "VCUEtvsSal_0609_R",
+ "VCU BXD VTA Et vs Sal M430 2.0 (Jun09) RMA"
]
]
},
@@ -2920,6 +3021,27 @@
]
]
},
+ "CIE-RMA": {
+ "Midbrain mRNA": [
+ [
+ "830",
+ "INIA_UTHSC_Mid_AffyMTA1_Apr17",
+ "INIA-UTHSC Midbrain CIE Affy MTA 1.0 GeneLevel (Apr17) RMA"
+ ],
+ [
+ "834",
+ "INIA_UTHSC_Mid_AffyMTA1_Ex_May17",
+ "INIA-UTHSC Midbrain CIE Affy MTA 1.0 Exon Level (Apr17) RMA"
+ ]
+ ],
+ "Phenotypes": [
+ [
+ "None",
+ "CIE-RMAPublish",
+ "CIE-RMA Published Phenotypes"
+ ]
+ ]
+ },
"CMS": {
"Phenotypes": [
[
@@ -3021,6 +3143,13 @@
"CXB Genotypes"
]
],
+ "Heart mRNA": [
+ [
+ "821",
+ "UCLA_CXB_Aor_Jan16",
+ "UCLA CXB Aorta Affy M430 2.0 (Jan16) RMA"
+ ]
+ ],
"Hippocampus mRNA": [
[
"100",
@@ -3033,6 +3162,13 @@
"Hippocampus Consortium M430v2 CXB (Dec05) PDNN"
]
],
+ "Liver mRNA": [
+ [
+ "823",
+ "UCLA_CXB_Liv_Jan16",
+ "UCLA CXB Liver Affy M430 2.0 (Jan16) RMA"
+ ]
+ ],
"Phenotypes": [
[
"628",
@@ -3048,6 +3184,7 @@
]
]
},
+ "EMSR": {},
"HS": {
"Hippocampus mRNA": [
[
@@ -3545,6 +3682,10 @@
"Brain, Development: Normal Gene Expression (Yale/Sestan)"
],
[
+ "Islets-Gerling",
+ "Pancreatic: Islets (UTHSC/Gerling)"
+ ],
+ [
"TIGEM-Retina-RNA-Seq",
"Retina: Normal Adult Gene Expression, RNA-Seq (TIGEM)"
]
@@ -3616,7 +3757,11 @@
],
[
"CIE-INIA",
- "Chronic Intermittent Ethanol"
+ "Chronic Intermittent Ethanol Phase 1"
+ ],
+ [
+ "CIE-RMA",
+ "Chronic Intermittent Ethanol Phase 2"
],
[
"CMS",
@@ -3631,6 +3776,10 @@
"CXB"
],
[
+ "EMSR",
+ "Ethanol-Medicated Stress Reduction"
+ ],
+ [
"HS",
"Heterogeneous Stock"
],
@@ -4414,6 +4563,12 @@
"Ventrolateral Prefrontal Cortex mRNA"
]
],
+ "Islets-Gerling": [
+ [
+ "Phenotypes",
+ "Phenotypes"
+ ]
+ ],
"TIGEM-Retina-RNA-Seq": [
[
"Phenotypes",
@@ -4630,6 +4785,10 @@
"Gastrointestinal mRNA"
],
[
+ "Heart mRNA",
+ "Heart mRNA"
+ ],
+ [
"Hematopoietic Cells mRNA",
"Hematopoietic Cells mRNA"
],
@@ -4780,6 +4939,16 @@
"LCM Brain Regions mRNA"
]
],
+ "CIE-RMA": [
+ [
+ "Phenotypes",
+ "Phenotypes"
+ ],
+ [
+ "Midbrain mRNA",
+ "Midbrain mRNA"
+ ]
+ ],
"CMS": [
[
"Phenotypes",
@@ -4822,14 +4991,23 @@
"Genotypes"
],
[
+ "Heart mRNA",
+ "Heart mRNA"
+ ],
+ [
"Hippocampus mRNA",
"Hippocampus mRNA"
],
[
+ "Liver mRNA",
+ "Liver mRNA"
+ ],
+ [
"Spleen mRNA",
"Spleen mRNA"
]
],
+ "EMSR": [],
"HS": [
[
"Phenotypes",
diff --git a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js
index 7a5a86bd..96763aca 100644
--- a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js
+++ b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js
@@ -1,74 +1,64 @@
var chart;
nv.addGraph(function() {
- //
- chart = nv.models.scatterChart();
- //
- chart.showLegend(false);
- chart.duration(300);
- chart.color(d3.scale.category10().range());
- chart.pointRange([200,0]);
- //
- // chart.showDistX(true);
- // chart.showDistY(true);
- chart.xAxis.tickFormat(d3.format('.02f'));
- chart.yAxis.tickFormat(d3.format('.02f'));
- // chart.showXAxis(false);
- // chart.showYAxis(false);
- chart.xAxis.axisLabel(js_data.trait_1);
- chart.yAxis.axisLabel(js_data.trait_2);
- //
- xmin = d3.min(js_data.data[0]);
- xmax = d3.max(js_data.data[0]);
- xrange = xmax - xmin;
- ymin = d3.min(js_data.data[1]);
- ymax = d3.max(js_data.data[1]);
- yrange = ymax - ymin;
- chart.xDomain([xmin - xrange/10, xmax + xrange/10]);
- chart.yDomain([ymin - yrange/10, ymax + yrange/10]);
- //
- d3.select('#scatterplot2 svg').datum(nv.log(getdata())).call(chart);
- nv.utils.windowResize(chart.update);
- return chart;
+ //
+ chart = nv.models.scatterChart();
+ //
+ chart.showLegend(false);
+ chart.duration(300);
+ chart.color(d3.scale.category10().range());
+ chart.pointRange([200,0]);
+ //
+ chart.xAxis.axisLabel(js_data.trait_1);
+ chart.xAxis.axisLabelDistance(11);
+ chart.yAxis.axisLabel(js_data.trait_2);
+ chart.yAxis.axisLabelDistance(11);
+ //
+ xmin = d3.min(js_data.data[0]);
+ xmax = d3.max(js_data.data[0]);
+ xrange = xmax - xmin;
+ ymin = d3.min(js_data.data[1]);
+ ymax = d3.max(js_data.data[1]);
+ yrange = ymax - ymin;
+ chart.xDomain([xmin - xrange/10, xmax + xrange/10]);
+ chart.yDomain([ymin - yrange/10, ymax + yrange/10]);
+ chart.xAxis.tickFormat(d3.format(checkformat(xrange)));
+ chart.yAxis.tickFormat(d3.format(checkformat(yrange)));
+ //
+ chart.tooltip.contentGenerator(function (obj) {
+ return '<b style="font-size: 18px">(' + obj.point.x + ', ' + obj.point.y + ')</b>';
+ });
+ //
+ d3.select('#scatterplot2 svg').datum(nv.log(getdata())).call(chart);
+ nv.utils.windowResize(chart.update);
+ return chart;
});
function getdata () {
- var data = [];
- data.push({
- values: [],
- slope: js_data.slope,
- intercept: js_data.intercept
- });
- for (j = 0; j < js_data.data[0].length; j++) {
- data[0].values.push({
- x: js_data.data[0][j],
- y: js_data.data[1][j],
- size: 10,
- shape: 'circle'
- });
- }
- return data;
+ var data = [];
+ data.push({
+ values: [],
+ slope: js_data.slope,
+ intercept: js_data.intercept
+ });
+ for (j = 0; j < js_data.data[0].length; j++) {
+ data[0].values.push({
+ x: js_data.data[0][j],
+ y: js_data.data[1][j],
+ size: 10,
+ shape: 'circle'
+ });
+ }
+ return data;
}
-
-function randomData(groups, points) {
- var data = [],
- shapes = ['circle'],
- random = d3.random.normal();
- for (i = 0; i < groups; i++) {
- data.push({
- key: 'Group ' + i,
- values: [],
- slope: Math.random() - .01,
- intercept: Math.random() - .5
- });
- for (j = 0; j < points; j++) {
- data[i].values.push({
- x: random(),
- y: random(),
- size: Math.random(),
- shape: shapes[j % shapes.length]
- });
- }
- }
- return data;
+
+function checkformat (range) {
+ cell = range / 10.0;
+ if (cell >= 1) {
+ return ",r";
+ } else {
+ cell = -Math.log(cell);
+ n = cell.toString().split(".")[0].length;
+ return ",.0" + n + "f";
+ }
}
diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html
index ec55ebeb..9ca3d63a 100644
--- a/wqflask/wqflask/templates/base.html
+++ b/wqflask/wqflask/templates/base.html
@@ -144,7 +144,7 @@
<p>
Development and source code on <a href="https://github.com/genenetwork/">github</a> with <a href="https://github.com/genenetwork/genenetwork2/issues">issue tracker</a> and <a href="https://github.com/genenetwork/genenetwork2/blob/master/README.md">documentation</a>. Join the <a href="http://listserv.uthsc.edu/mailman/listinfo/genenetwork-dev">mailing list</a> and find us on <a href="https://webchat.freenode.net/">IRC</a> (#genenetwork channel).
{% if version: %}
- <p><small>GeneNetwork v{{ version }}</small></p>
+ <p><small>GeneNetwork {{ version }}</small></p>
{% endif %}
</div>
diff --git a/wqflask/wqflask/templates/corr_scatterplot.html b/wqflask/wqflask/templates/corr_scatterplot.html
index 4464025e..05891c95 100644
--- a/wqflask/wqflask/templates/corr_scatterplot.html
+++ b/wqflask/wqflask/templates/corr_scatterplot.html
@@ -53,16 +53,6 @@
<h2>Correlation Scatterplot</h2>
-<table class="table">
- <tbody>
- <tr><th class="text-right">num overlap</th> <td>{{jsdata.num_overlap}}</td></tr>
- <tr><th class="text-right">slope</th> <td>{{jsdata.slope}}</td></tr>
- <tr><th class="text-right">intercept</th> <td>{{jsdata.intercept}}</td></tr>
- <tr><th class="text-right">r value</th> <td>{{jsdata.r_value}}</td></tr>
- <tr><th class="text-right">p value</th> <td>{{jsdata.p_value}}</td></tr>
- </tbody>
-</table>
-
<!--
<table class="table">
<tr>
@@ -85,12 +75,12 @@
-->
<div id="scatterplot2">
- <svg style="width: 1000px; height: 800px;"></svg>
+ <svg style="width: 1000px; height: 800px; margin-left: 10px;"></svg>
</div>
<br>
-<div style="margin-left: 75px;">
+<div style="margin-left: 80px;">
{% if trait_1.dataset.type == "ProbeSet" %}
<div>
@@ -144,6 +134,18 @@
</div>
+<br>
+
+<table class="table table-hover table-striped table-bordered" style="width: 250px; margin-left: 80px;">
+ <tbody>
+ <tr><th>Number</th> <td>{{jsdata.num_overlap}}</td></tr>
+ <tr><th>Slope</th> <td>{{'%0.3f' % jsdata.slope}}</td></tr>
+ <tr><th>Intercept</th> <td>{{'%0.3f' % jsdata.intercept}}</td></tr>
+ <tr><th>R value</th> <td>{{'%0.3f' % jsdata.r_value}}</td></tr>
+ <tr><th>P value</th> <td>{{'%0.3e' % jsdata.p_value}}</td></tr>
+ </tbody>
+</table>
+
{% endblock %}
{% block js %}
diff --git a/wqflask/wqflask/templates/error.html b/wqflask/wqflask/templates/error.html
index c707a4fc..2f1e06fa 100644
--- a/wqflask/wqflask/templates/error.html
+++ b/wqflask/wqflask/templates/error.html
@@ -35,7 +35,7 @@
</p>
<pre>
- GeneNetwork v{{ version }} {{ stack[0] }}
+ GeneNetwork {{ version }} {{ stack[0] }}
{{ message }} (error)
{{ stack[-3] }}
{{ stack[-2] }}
@@ -50,7 +50,7 @@
<a href="#Stack" class="btn btn-default" data-toggle="collapse">Toggle full stack trace</a>
<div id="Stack" class="collapse">
<pre>
- GeneNetwork v{{ version }} {% for line in stack %} {{ line }}
+ GeneNetwork {{ version }} {% for line in stack %} {{ line }}
{% endfor %}
</pre>
</div>
diff --git a/wqflask/wqflask/templates/marker_regression_gn1.html b/wqflask/wqflask/templates/marker_regression_gn1.html
index 4cea51b2..65debd10 100644
--- a/wqflask/wqflask/templates/marker_regression_gn1.html
+++ b/wqflask/wqflask/templates/marker_regression_gn1.html
@@ -14,7 +14,7 @@
<input type="hidden" name="temp_uuid" value="{{ temp_uuid }}">
<input type="hidden" name="trait_id" value="{{ this_trait.name }}">
<input type="hidden" name="dataset" value="{{ dataset.name }}">
- {% if mapping_method == "reaper" %}
+ {% if mapping_method == "reaper" or mapping_method == "rqtl_geno" %}
<input type="hidden" name="genofile" value="{{ dataset.group.genofile }}">
{% endif %}
<input type="hidden" name="method" value="{{ mapping_method }}">
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index 56b6cc62..5a880647 100644
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -129,6 +129,7 @@
<script type="text/javascript" src="/static/new/js_external/d3-tip.min.js"></script>
<script type="text/javascript" src="/static/new/js_external/jstat.min.js"></script>
<script type="text/javascript" src="/static/new/js_external/shapiro-wilk.js"></script>
+ <script type="text/javascript" src="/static/new/js_external/saveSvgAsPng.js"></script>
<script type="text/javascript" src="/static/new/javascript/colorbrewer.js"></script>
<script type="text/javascript" src="/static/new/javascript/stats.js"></script>
diff --git a/wqflask/wqflask/templates/show_trait_statistics.html b/wqflask/wqflask/templates/show_trait_statistics.html
index 0a0e9990..ac44b656 100644
--- a/wqflask/wqflask/templates/show_trait_statistics.html
+++ b/wqflask/wqflask/templates/show_trait_statistics.html
@@ -94,12 +94,15 @@
<div class="tab-pane" id="probability_plot">
<div style="padding: 20px" class="form-horizontal">
{% if sample_groups|length != 1 %}
+ Select Group:
<select class="prob_plot_samples_group">
{% for group, pretty_group in sample_group_types.items() %}
<option value="{{ group }}">{{ pretty_group }}</option>
{% endfor %}
</select>
- <br><br>
+ <button type="button" class="btn btn-default" id="down_prob_plot">Export as PNG</button>
+ <br>
+ <br>
{% endif %}
<div id="prob_plot_container">
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index 1e831896..f7fcd2d0 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -46,7 +46,7 @@ from wqflask.database import db_session
from wqflask import model
from utility import Bunch, Struct, after
-from utility.tools import LOG_SQL, LOG_SQLALCHEMY
+from utility.tools import LOG_SQL, LOG_SQL_ALCHEMY
import logging
from utility.logger import getLogger
@@ -69,7 +69,7 @@ class AnonUser(object):
if self.cookie:
logger.debug("already is cookie")
self.anon_id = verify_cookie(self.cookie)
-
+
else:
logger.debug("creating new cookie")
self.anon_id, self.cookie = create_signed_cookie()
@@ -78,17 +78,17 @@ class AnonUser(object):
@after.after_this_request
def set_cookie(response):
response.set_cookie(self.cookie_name, self.cookie)
-
+
def add_collection(self, new_collection):
collection_dict = dict(name = new_collection.name,
created_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'),
num_members = new_collection.num_members,
members = new_collection.get_members())
-
+
Redis.set(self.key, json.dumps(collection_dict))
Redis.expire(self.key, 60 * 60 * 24 * 5)
-
+
def delete_collection(self, collection_name):
existing_collections = self.get_collections()
updated_collections = []
@@ -106,7 +106,7 @@ class AnonUser(object):
updated_collections.append(this_collection)
Redis.set(self.key, json.dumps(updated_collections))
-
+
def get_collections(self):
json_collections = Redis.get(self.key)
if json_collections == None or json_collections == "None":
@@ -117,7 +117,7 @@ class AnonUser(object):
collection['created_timestamp'] = datetime.datetime.strptime(collection['created_timestamp'], '%b %d %Y %I:%M%p')
collection['changed_timestamp'] = datetime.datetime.strptime(collection['changed_timestamp'], '%b %d %Y %I:%M%p')
return collections
-
+
def import_traits_to_user(self):
collections_list = json.loads(Redis.get(self.key))
for collection in collections_list:
@@ -131,7 +131,7 @@ class AnonUser(object):
uc.members = json.dumps(collection['members'])
db_session.add(uc)
db_session.commit()
-
+
def display_num_collections(self):
"""
Returns the number of collections or a blank string if there are zero.
@@ -217,7 +217,7 @@ class UserSession(object):
"""Actual sqlalchemy record"""
# Only look it up once if needed, then store it
try:
- if LOG_SQLALCHEMY:
+ if LOG_SQL_ALCHEMY:
logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG)
# Already did this before
@@ -237,7 +237,7 @@ class UserSession(object):
def before_request():
g.user_session = UserSession()
g.cookie_session = AnonUser()
-
+
class UsersManager(object):
def __init__(self):
self.users = model.User.query.all()
@@ -541,9 +541,9 @@ class LoginUser(object):
import_col = "true"
else:
import_col = "false"
-
- #g.cookie_session.import_traits_to_user()
-
+
+ #g.cookie_session.import_traits_to_user()
+
return self.actual_login(user, import_collections=import_col)
else:
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 39f4a686..07151425 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -53,7 +53,7 @@ from wqflask.ctl import ctl_analysis
#from wqflask.trait_submission import submit_trait
from utility import temp_data
-from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION
+from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,JS_TWITTER_POST_FETCHER_PATH
from utility.helper_functions import get_species_groups
from base import webqtlFormData
@@ -147,9 +147,8 @@ def tmp_page(img_path):
img_base64 = bytesarray )
@app.route("/twitter/<path:filename>")
-def bd_files(filename):
- bd_path = app.config['TWITTER_POST_FETCHER_JS_PATH']
- return send_from_directory(bd_path, filename)
+def twitter(filename):
+ return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename)
#@app.route("/data_sharing")
#def data_sharing_page():