diff options
25 files changed, 979 insertions, 585 deletions
diff --git a/VERSION b/VERSION index 144b089a..3e0b7cab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10-pre1 +2.10-pre4 diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 52d3155c..236d8f63 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,34 +1,39 @@ #! /bin/bash # # This will run the GN2 server (with default settings if none -# supplied). Pass in your own settings file, e.g. +# supplied). Typically you need a GNU Guix profile which is set with +# an environment variable (this profile is dictated by the +# installation path of genenetwork). Say your profile is in +# ~/opt/gn-latest-guix # -# ./bin/genenetwork2 ~/my_settings.py +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 # -# But better is to retain the defaults and override them with -# JSON file (see the deploy docs). +# You can pass in your own settings file, e.g. # -# ./bin/genenetwork2 ~/my_overrides.json +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_settings.py # # To run a maintenance script with settings (instead of the webserver) add that with # a -c switch, e.g. # -# ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py -# -# Environment settings can be used to preconfigure as well as a -# settings.py file. -# -# GN2_BASE_PATH is base directory of wqflask source code -# +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py SCRIPT=$(readlink -f "$0") GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) +GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) echo GN2_BASE_DIR=$GN2_BASE_DIR -GIT_HASH=$(git rev-parse HEAD) -GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -export GN_VERSION=$(cat $GN2_BASE_DIR/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9} +GUIX_SITE=$GN2_BASE_DIR/lib/python2.7/site-packages +if [ -d $GUIX_SITE ]; then + echo INFO: GN2 is running from GNU Guix + GN2_BASE_DIR=$GUIX_SITE + export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION) +else + echo INFO: GN2 is running from a source tree + GIT_HASH=$(git rev-parse HEAD) + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9} +fi echo GN_VERSION=$GN_VERSION # Handle settings parameter (can be .py or .json) @@ -41,13 +46,8 @@ else shift fi if [ ! -e $settings ]; then - GUIX_ETC=$GN2_BASE_DIR/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg - if [ -d $GUIX_ETC ]; then - echo INFO: GN2 is actually running from GNU Guix - else - echo "ERROR: can not locate settings file - pass it in the command line" - exit 1 - fi + echo "ERROR: can not locate settings file - pass it in the command line" + exit 1 fi export WQFLASK_SETTINGS=$settings # Python export WQFLASK_OVERRIDES=$overrides # JSON @@ -55,30 +55,40 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides -# ---- Checks and balances -if [ -z $GUIX_ETC ]; then - if [ -z $GN2_PROFILE ] ; then - echo "WARNING: GN2_PROFILE has not been set - I hope you know what you are doing!" - else - export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages - export R_LIBS_SITE=$GN2_PROFILE/site-library - export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" - export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" - export XDG_DATA_DIRS="$GN2_PROFILE/share" - export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" - export LC_ALL=C - fi - if [ -z $PYTHONPATH ] ; then - echo "WARNING PYTHONPATH has not been set - use GN2_PROFILE!" - fi - if [ ! -d $R_LIBS_SITE ] ; then - echo "ERROR: R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" - echo "Paste into your shell the output of (for example)" - echo "guix package -p \$GN2_PROFILE --search-paths" - exit 1 +if [ -z $GN2_PROFILE ] ; then + echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" + export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) + if [ -d $GN2_PROFILE ]; then + echo "Best guess is $GN2_PROFILE" fi fi +if [ -z $GN2_PROFILE ]; then + read -p "PRESS [ENTER] TO CONTINUE..." +else + export PATH=$GN2_PROFILE/bin:$PATH + export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages + export R_LIBS_SITE=$GN2_PROFILE/site-library + export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript + export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" + export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" + export XDG_DATA_DIRS="$GN2_PROFILE/share" + export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" + export LC_ALL=C # FIXME + export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" + export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" + export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" + export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" +fi +if [ -z $PYTHONPATH ] ; then + echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" + exit 1 +fi +if [ ! -d $R_LIBS_SITE ] ; then + echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" + echo "Paste into your shell the output of (for example)" + echo "guix package -p \$GN2_PROFILE --search-paths" + exit 1 +fi # We may change this one: export PYTHONPATH=$GN2_BASE_DIR/wqflask:$PYTHONPATH @@ -88,6 +98,8 @@ if [ -z $TMPDIR ]; then TMPDIR="/tmp" fi +# Show environment settings +set|grep guix set|grep $GN2_PROFILE set|grep TMPDIR @@ -97,7 +109,7 @@ if [ "$1" = '-c' ] ; then cmd=${2#wqflask/} echo PYTHONPATH=$PYTHONPATH echo RUNNING COMMAND $cmd - /usr/bin/env python $cmd + python $cmd exit 0 fi @@ -109,4 +121,4 @@ dbfilename gn2.rdb # Start the flask server running GN2 cd $GN2_BASE_DIR/wqflask echo "Starting with $settings" -/usr/bin/env python runserver.py +python runserver.py diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index 4399ea26..83adce99 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -2,19 +2,188 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#binary-deployment][Binary deployment]] + - [[#binary-deployment-through-gnu-guix][Binary deployment through GNU Guix]] + - [[#quick-installation-recipe][Quick installation recipe]] + - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]] + - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]] + - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] + - [[#step-4-install-and-run-gn2][Step 4: Install and run GN2]] - [[#from-source-deployment][From source deployment]] - [[#create-archive][Create archive]] + - [[#source-deployment][Source deployment]] + - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] + - [[#set-up-nginx-port-forwarding][Set up nginx port forwarding]] + - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] + - [[#update-to-recent-guix][Update to recent guix]] + - [[#install-gn2][Install GN2]] + - [[#run-gn2][Run GN2]] * Introduction Large system deployments tend to get very complex. In this document we explain the GeneNetwork deployment system which is based on GNU Guix -(see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). +(see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]] and the main [[README.org]] doc). -* Binary deployment +* Binary deployment through GNU Guix +** Quick installation recipe -NYA (will go to README) +This is a recipe for quick and dirty installation of GN2. For +convenience everything is installed as root, though in reality only +GNU Guix has to be installed as root. I tested this recipe on a fresh +install of Debian 8.3.0 (in KVM) though it should work on any modern +Linux distribution (including CentOS). + + +Note that GN2 consists of an approx. 5 GB installation including +database. If you use a virtual machine we recommend to use at least +double. + +** Step 1: Install GNU Guix + +Fetch the GNU Guix binary from [[https://www.gnu.org/software/guix/download/][here]] (middle panel) and follow +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html][instructions]]. Essentially, download and unpack the tar ball (which +creates directories in /gnu and /var/guix), add build users and group +(Guix builds software as unpriviliged users) and run the Guix daemon +after fixing the paths (also known as the 'profile'). + +Once you have succeeded, you have to [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-the-key][set the key]] (getting permission +to download binaries from the GNU server) and you should be able to +install the hello package using binary packages (no building) + +#+begin_src bash +export PATH=~/.guix-profile/bin:$PATH +guix pull +guix package -i hello --dry-run +#+end_src + +Which should show something like + +: The following files would be downloaded: +: /gnu/store/zby49aqfbd9w9br4l52mvb3y6f9vfv22-hello-2.10 +: ... +#+end_src + +means binary installs. The actual installation command of 'hello' is + +#+begin_src bash +guix package -i hello +hello + Hello, world! +#+end_src + +If you actually see things building it means that Guix is not yet +properly installed and up-to-date, i.e., the key is missing or you +need to do a 'guix pull'. Press Ctrl-C to interrupt. + +If you need more help we have another writeup in [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#binary-installation][guix-notes]]. To get +rid of the locale warning see [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-locale][set-locale]]. + +** Step 2: Checkout the GN2 git repositories + +To fixate the software dependency graph GN2 uses git repositories of +Guix packages. First install git if it is missing + +#+begin_src bash +guix package -i git +export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt +#+end_src + +check out the git repositories (gn-deploy branch) + +#+begin_src bash +cd ~ +mkdir genenetwork +cd genenetwork +git clone --branch gn-deploy https://github.com/genenetwork/guix-bioinformatics +git clone --branch gn-deploy --recursive https://github.com/genenetwork/guix guix-gn-deploy +cd guix-gn-deploy +#+end_src bash + +To test whether this is working try: + +#+begin_src bash +#+end_src bash + +** Step 3: Authorize the GN Guix server + +GN2 has its own GNU Guix binary distribution server. To trust it you have +to add the following key + +#+begin_src scheme +(public-key + (ecc + (curve Ed25519) + (q #11217788B41ADC8D5B8E71BD87EF699C65312EC387752899FE9C888856F5C769#) + ) +) +#+end_src + +by pasting it into the command + +#+begin_src bash +guix archive --authorize +#+end_src + +and hit Ctrl-D. + +Now you can use the substitute server to install GN2 binaries. + +** Step 4: Install and run GN2 + +Since this is a quick and dirty install we are going to override the +GNU Guix package path by pointing the package path to our repository: + +#+begin_src bash +rm /root/.config/guix/latest +ln -s ~/genenetwork/guix-gn-deploy/ /root/.config/guix/latest +#+end_src + +Now check whether you can find the GN2 package with + +#+begin_src bash +env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -A genenetwork2 + genenetwork2 2.0-a8fcff4 out gn/packages/genenetwork.scm:144:2 +#+end_src + +(ignore the source file newer then ... messages, this is caused by the +/root/.config/guix/latest override). + +And install with + +#+begin_src bash +env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ \ + guix package -i genenetwork2 \ + --substitute-urls="http://guix.genenetwork.org" +#+end_src + +Note: the order of the substitute url's may make a difference in speed +(put the one first that is fastest for your location and time of day). + +Note: if your system starts building or gives an error it may well be +Step 3 did not succeed. The installation should actually be smooth at +this point and only do binary installs (no compiling). + +After installation you should be able to run genenetwork2 after updating +the Guix suggested environment vars. Check the output of + +#+begin_src bash +guix package --search-paths +export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages" +export R_LIBS_SITE="/root/.guix-profile/site-library/" +#+end_src + +and copy-paste the listed exports into the terminal before running: + +#+begin_src bash +genenetwork2 +#+end_src + +It will complain that the database is missing. See the next section on +running MySQL server for downloading and installing a MySQL GN2 +database. After installing the database restart genenetwork2 and point +your browser at [[http://localhost:5003/]]. + +End of the GN2 installation recipe! * From source deployment @@ -52,3 +221,182 @@ gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix * Create archive : env GUIX_PACKAGE_PATH=../../genenetwork/guix-bioinformatics/ ./pre-inst-env guix archive --export -r genenetwork2 > guix_gn2-2.0-9e9475053.nar + + +* Source deployment + +This section gives a more elaborate instruction for installing GN2 +from source. + +First execute above 4 steps: + + - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]] + - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]] + - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] + - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]] + + +** Run your own copy of GN2 + +At some point you may want to fix the source code. Assuming you have +Guix and Genenetwork2 installed (as described above) clone the GN2 +repository from https://github.com/genenetwork/genenetwork2. + +Copy-paste the paths into your terminal (mainly so PYTHON_PATH and +R_LIBS_SITE are set) from the information given by guix: + +: guix package --search-paths + +Inside the repository: + +: cd genenetwork2 +: ./bin/genenetwork2 + +Will fire up your local repo http://localhost:5003/ using the +settings in ./etc/default_settings.py. These settings may +not reflect your system. To override settings create your own from a copy of +default_settings.py and pass it into GN2 with + +: ./bin/genenetwork2 $HOME/my_settings.py + +and everything *should* work (note the full path to the settings +file). This way we develop against the exact same dependency graph of +software. + +If something is not working, take a hint from the settings file +that comes in the Guix installation. It sits in something like + +: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py + +** Set up nginx port forwarding + +nginx can be used as a reverse proxy for GN2. For example, we want to +expose GN2 on port 80 while it is running on port 5003. Essentially +the configuration looks like + +#+begin_src js + server { + listen 80; + server_name test-gn2.genenetwork.org; + access_log logs/test-gn2.access.log; + + proxy_connect_timeout 3000; + proxy_send_timeout 3000; + proxy_read_timeout 3000; + send_timeout 3000; + + location / { + proxy_set_header Host $http_host; + proxy_set_header Connection keep-alive; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + proxy_pass http://127.0.0.1:5003; + } +} +#+end_src js + +Install the nginx webserver (as root) + +: guix package -i nginx + +The nginx example configuration examples can be found in the Guix +store through + +: ls -l /root/.guix-profile/sbin/nginx +: lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx + +Use that path + +: ls /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/share/nginx/conf/ +: fastcgi.conf koi-win scgi_params +: fastcgi.conf.default mime.types scgi_params.default +: fastcgi_params mime.types.default uwsgi_params +: fastcgi_params.default nginx.conf uwsgi_params.default +: koi-utf nginx.conf.default win-utf + +And copy any relevant files to /etc/nginx. A configuration file for +GeneNetwork (reverse proxy) port forwarding can be found in the source +repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc +(still as root) +: cp ./etc/nginx-genenetwork.conf /etc/nginx/ + +Make dirs + +: mkdir -p /var/spool/nginx/logs + +Add users + +: adduser nobody ; addgroup nobody + +Run nginx + +: /root/.guix-profile/sbin/nginx -c /etc/nginx/nginx-genenetwork.conf -p /var/spool/nginx + +* Source deployment and other information on reproducibility + +See the document [[GUIX-Reproducible-from-source.org]]. + +** Update to recent guix + +We now compile Guix from scratch. + +Create, install and run a recent version of the guix-daemon by +compiling the guix repository you have installed with git in +step 2. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully after + +: cd ~/genenetwork/guix-gn-deploy + +Make sure to restart the guix daemon and run guix client from this +directory. + +** Install GN2 + +Reinstall genenetwork2 using the new tree + +#+begin_src bash +env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --substitute-urls="http://guix.genenetwork.org https://mirror.guixsd.org" +#+end_src bash + +Note the use of ./pre-inst-env here! + +Actually, it should be the same installation as in step 4, so nothing +gets downloaded. + +** Run GN2 + +Make a note of the paths with + +#+begin_src bash +./pre-inst-env guix package --search-paths +#+end_src bash + +or this should also work if guix is installed + +#+begin_src bash +guix package --search-paths +#+end_src bash + +After setting the paths for the server + +#+begin_src bash +export PATH=~/.guix-profile/bin:$PATH +export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +export R_LIBS_SITE="$HOME/.guix-profile/site-library/" +export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0" +export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0" +export XDG_DATA_DIRS="$HOME/.guix-profile/share" +export GIO_EXTRA_MODULES="$HOME/.guix-profile/lib/gio/modules" +#+end_src bash + +run the main script (in ~/.guix-profile/bin) + +#+begin_src bash +genenetwork2 +#+end_src bash + +will start the default server which listens on port 5003, i.e., +http://localhost:5003/. + +OK, we are where we were before with step 4. Only difference is that we +used our own compiled guix server. diff --git a/doc/README.org b/doc/README.org index b38ea664..a39ef603 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,33 +2,27 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#quick-installation-recipe][Quick installation recipe]] - - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]] - - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]] - - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] - - [[#step-4-install-and-run-gn2][Step 4: Install and run GN2]] + - [[#install][Install]] + - [[#tarball][Tarball]] + - [[#docker][Docker]] + - [[#with-source][With source]] + - [[#running-gn2][Running GN2]] - [[#run-mysql-server][Run MySQL server]] + - [[#install-mysql-with-gnu-guix][Install MySQL with GNU GUIx]] + - [[#load-the-small-database-in-mysql][Load the small database in MySQL]] - [[#gn2-dependency-graph][GN2 Dependency Graph]] - - [[#source-deployment][Source deployment]] - - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] - - [[#set-up-nginx-port-forwarding][Set up nginx port forwarding]] - - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] - - [[#update-to-recent-guix][Update to recent guix]] - - [[#install-gn2][Install GN2]] - - [[#run-gn2][Run GN2]] + - [[#working-with-the-gn2-source-code][Working with the GN2 source code]] - [[#trouble-shooting][Trouble shooting]] - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]] - [[#cant-run-a-module][Can't run a module]] - [[#rpy2-error-show-now-found][Rpy2 error 'show' now found]] + - [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]] + - [[#read-more][Read more]] - [[#irc-session][IRC session]] * Introduction -If you want to understand the architecture of GN2 read -[[Architecture.org]]. The rest of this document is mostly on deployment -of GN2. - Large system deployments can get very [[http://biogems.info/contrib/genenetwork/gn2.svg ][complex]]. In this document we explain the GeneNetwork version 2 (GN2) reproducible deployment system which is based on GNU Guix (see also [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix @@ -37,195 +31,113 @@ system can be used to install GN with all its files and dependencies. The official installation path is from a checked out version of the main Guix package tree and that of the Genenetwork package tree. Current supported versions can be found as the SHA values of -'gn-latest' branches of [[https://github.com/genenetwork/guix-bioinformatics/tree/gn-latest][Guix bioinformatics]] and [[https://github.com/genenetwork/guix/tree/gn-latest][GNU Guix main]]. +'gn-latest' branches of [[https://gitlab.com/genenetwork/guix-bioinformatics][Guix bioinformatics]] and [[https://gitlab.com/genenetwork/guix][GNU Guix]]. For a full view of runtime dependencies as defined by GNU Guix, see -the [[#gn2-dependency-graph][GN2 Dependency Graph]]. - -* Quick installation recipe - -This is a recipe for quick and dirty installation of GN2. For -convenience everything is installed as root, though in reality only -GNU Guix has to be installed as root. I tested this recipe on a fresh -install of Debian 8.3.0 (in KVM) though it should work on any modern -Linux distribution (including CentOS). For more elaborate installation -instructions see [[#source-deployment][Source deployment]]. - -Note that GN2 consists of an approx. 5 GB installation including -database. If you use a virtual machine we recommend to use at least -double. - -** Step 1: Install GNU Guix - -Fetch the GNU Guix binary from [[https://www.gnu.org/software/guix/download/][here]] (middle panel) and follow -[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html][instructions]]. Essentially, download and unpack the tar ball (which -creates directories in /gnu and /var/guix), add build users and group -(Guix builds software as unpriviliged users) and run the Guix daemon -after fixing the paths (also known as the 'profile'). - -Once you have succeeded, you have to [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-the-key][set the key]] (getting permission -to download binaries from the GNU server) and you should be able to -install the hello package using binary packages (no building) - -#+begin_src bash -export PATH=~/.guix-profile/bin:$PATH -guix pull -guix package -i hello --dry-run -#+end_src - -Which should show something like - -: The following files would be downloaded: -: /gnu/store/zby49aqfbd9w9br4l52mvb3y6f9vfv22-hello-2.10 -: ... -#+end_src - -means binary installs. The actual installation command of 'hello' is - -#+begin_src bash -guix package -i hello -hello - Hello, world! -#+end_src - -If you actually see things building it means that Guix is not yet -properly installed and up-to-date, i.e., the key is missing or you -need to do a 'guix pull'. Press Ctrl-C to interrupt. - -If you need more help we have another writeup in [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#binary-installation][guix-notes]]. To get -rid of the locale warning see [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-locale][set-locale]]. - -** Step 2: Checkout the GN2 git repositories +an example of the [[#gn2-dependency-graph][GN2 Dependency Graph]]. -To fixate the software dependency graph GN2 uses git repositories of -Guix packages. First install git if it is missing +* Install -#+begin_src bash -guix package -i git -export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt -#+end_src +The quickest way to install GN2 is by using a binary installation +(tarball or Docker image). These installations are bundled by GNU +Guix and include all dependencies. You can install GeneNetwork on most +Linux distributions, including Debian, Ubuntu, Fedora and CentOS, +provided you have administrator privileges (root). The alternative is +a Docker installation. -check out the git repositories (gn-deploy branch) +** Tarball -#+begin_src bash -cd ~ -mkdir genenetwork -cd genenetwork -git clone --branch gn-deploy https://github.com/genenetwork/guix-bioinformatics -git clone --branch gn-deploy --recursive https://github.com/genenetwork/guix guix-gn-deploy -cd guix-gn-deploy -#+end_src bash +Download the ~800Mb tarball from +[[http://files.genenetwork.org/software/binary_tarball/]]. Validate the checksum and +unpack to root, for example -To test whether this is working try: +: tar xvzf genenetwork2-2.10rc3-1538ffd-tarball-pack.tar.gz +: mv /gnu / +: mv /opt/genenetwork2 /opt/ -#+begin_src bash -#+end_src bash +Now you shoud be able to start the server with +: /opt/genenetwork2/bin/genenetwork2 -** Step 3: Authorize the GN Guix server +When the server stops with a MySQL error [[#run-mysql-server][Run MySQL server]] +and set SQL_URI to point at it. For example: -GN2 has its own GNU Guix binary distribution server. To trust it you have -to add the following key +: export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s -#+begin_src scheme -(public-key - (ecc - (curve Ed25519) - (q #11217788B41ADC8D5B8E71BD87EF699C65312EC387752899FE9C888856F5C769#) - ) -) -#+end_src +See also [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]]. -by pasting it into the command +** Docker -#+begin_src bash -guix archive --authorize -#+end_src +Docker images are also available through +[[http://files.genenetwork.org/software/]]. Validate the checksum and run +with [[https://docs.docker.com/engine/reference/commandline/load/][Docker load]]. -and hit Ctrl-D. +** With source -Now you can use the substitute server to install GN2 binaries. +For more elaborate installation instructions on deploying GeneNetwork from +source see [[#source-deployment][Source deployment]]. -** Step 4: Install and run GN2 +* Running GN2 -Since this is a quick and dirty install we are going to override the -GNU Guix package path by pointing the package path to our repository: +Default settings for GN2 are listed in a file called +[[../etc/default_settings.py][default_settings.py]]. You can copy this file and pass it as a new +parameter to the genenetwork2 command, e.g. -#+begin_src bash -rm /root/.config/guix/latest -ln -s ~/genenetwork/guix-gn-deploy/ /root/.config/guix/latest -#+end_src +: genenetwork2 mysettings.py -Now check whether you can find the GN2 package with +or you can set environment variables to override individual parameters, e.g. -#+begin_src bash -env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -A genenetwork2 - genenetwork2 2.0-a8fcff4 out gn/packages/genenetwork.scm:144:2 -#+end_src - -(ignore the source file newer then ... messages, this is caused by the -/root/.config/guix/latest override). +: env SERVER_PORT=5004 SQL_URI=mysql://user:pwd@dbhostname/db_webqtl genenetwork2 -And install with +the debug and logging switches can be particularly useful when +developing GN2. -#+begin_src bash -env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ \ - guix package -i genenetwork2 \ - --substitute-urls="http://guix.genenetwork.org" -#+end_src - -Note: the order of the substitute url's may make a difference in speed -(put the one first that is fastest for your location and time of day). - -Note: if your system starts building or gives an error it may well be -Step 3 did not succeed. The installation should actually be smooth at -this point and only do binary installs (no compiling). - -After installation you should be able to run genenetwork2 after updating -the Guix suggested environment vars. Check the output of - -#+begin_src bash -guix package --search-paths -export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages" -export R_LIBS_SITE="/root/.guix-profile/site-library/" -#+end_src +* Run MySQL server +** Install MySQL with GNU GUIx -and copy-paste the listed exports into the terminal before running: +These are the steps you can take to install a fresh installation of +mysql (which comes as part of the GNU Guix genenetwork2 install). -#+begin_src bash -genenetwork2 -#+end_src +As root configure and run -It will complain that the database is missing. See the next section on -running MySQL server for downloading and installing a MySQL GN2 -database. After installing the database restart genenetwork2 and point -your browser at [[http://localhost:5003/]]. +: adduser mysql && addgroup mysql +: mysqld --datadir=/var/mysql --initialize-insecure +: mkdir -p /var/run/mysqld +: chown mysql.mysql ~/mysql /var/run/mysqld +: su mysql -c mysqld --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048 -End of the GN2 installation recipe! +/etc/my.cnf +[mysqld] +user=root -* Run MySQL server +** Load the small database in MySQL At this point we require the underlying distribution to install and -run mysqld. Currently we have two databases for deployment, +run mysqld (see next section for GNU Guix). Currently we have two databases for deployment, 'db_webqtl_s' is the small testing database containing experiments from BXD mice and 'db_webqtl_plant' which contains all plant related material. Download one database from -http://files.genenetwork.org/raw_database/ -https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip +[[http://files.genenetwork.org/raw_database/]] + +[[https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip]] Check the md5sum. After installation inflate the database binary in the MySQL directory -(this installation path is subject to change soon) +: cd ~/mysql : chown -R mysql:mysql db_webqtl_s/ : chmod 700 db_webqtl_s/ : chmod 660 db_webqtl_s/* -restart MySQL service (mysqld). Login as root and +restart MySQL service (mysqld). Login as root + +: myslq -u root + +and : mysql> show databases; : +--------------------+ @@ -241,9 +153,12 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; +You may need to change "localhost" to whatever domain you are +connecting from (mysql will give an error). + Note that if the mysql connection is not working, try connecting to the IP address and check server firewall, hosts.allow and mysql IP -configuration. +configuration (see below). Note for the plant database you can rename it to db_webqtl_s, or change the settings in etc/default_settings.py to match your path. @@ -255,183 +170,9 @@ Graph of all runtime dependencies as installed by GNU Guix. #+ATTR_HTML: :title GN2_graph http://biogems.info/contrib/genenetwork/gn2.svg -* Source deployment - -This section gives a more elaborate instruction for installing GN2 -from source. - -First execute above 4 steps: - - - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]] - - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]] - - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] - - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]] - - -** Run your own copy of GN2 - -At some point you may want to fix the source code. Assuming you have -Guix and Genenetwork2 installed (as described above) clone the GN2 -repository from https://github.com/genenetwork/genenetwork2. - -Copy-paste the paths into your terminal (mainly so PYTHON_PATH and -R_LIBS_SITE are set) from the information given by guix: - -: guix package --search-paths - -Inside the repository: - -: cd genenetwork2 -: ./bin/genenetwork2 - -Will fire up your local repo http://localhost:5003/ using the -settings in ./etc/default_settings.py. These settings may -not reflect your system. To override settings create your own from a copy of -default_settings.py and pass it into GN2 with - -: ./bin/genenetwork2 $HOME/my_settings.py - -and everything *should* work (note the full path to the settings -file). This way we develop against the exact same dependency graph of -software. - -If something is not working, take a hint from the settings file -that comes in the Guix installation. It sits in something like +* Working with the GN2 source code -: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py - -** Set up nginx port forwarding - -nginx can be used as a reverse proxy for GN2. For example, we want to -expose GN2 on port 80 while it is running on port 5003. Essentially -the configuration looks like - -#+begin_src js - server { - listen 80; - server_name test-gn2.genenetwork.org; - access_log logs/test-gn2.access.log; - - proxy_connect_timeout 3000; - proxy_send_timeout 3000; - proxy_read_timeout 3000; - send_timeout 3000; - - location / { - proxy_set_header Host $http_host; - proxy_set_header Connection keep-alive; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; - proxy_pass http://127.0.0.1:5003; - } -} -#+end_src js - -Install the nginx webserver (as root) - -: guix package -i nginx - -The nginx example configuration examples can be found in the Guix -store through - -: ls -l /root/.guix-profile/sbin/nginx -: lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx - -Use that path - -: ls /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/share/nginx/conf/ -: fastcgi.conf koi-win scgi_params -: fastcgi.conf.default mime.types scgi_params.default -: fastcgi_params mime.types.default uwsgi_params -: fastcgi_params.default nginx.conf uwsgi_params.default -: koi-utf nginx.conf.default win-utf - -And copy any relevant files to /etc/nginx. A configuration file for -GeneNetwork (reverse proxy) port forwarding can be found in the source -repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc -(still as root) -: cp ./etc/nginx-genenetwork.conf /etc/nginx/ - -Make dirs - -: mkdir -p /var/spool/nginx/logs - -Add users - -: adduser nobody ; addgroup nobody - -Run nginx - -: /root/.guix-profile/sbin/nginx -c /etc/nginx/nginx-genenetwork.conf -p /var/spool/nginx - -* Source deployment and other information on reproducibility - -See the document [[GUIX-Reproducible-from-source.org]]. - -** Update to recent guix - -We now compile Guix from scratch. - -Create, install and run a recent version of the guix-daemon by -compiling the guix repository you have installed with git in -step 2. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully after - -: cd ~/genenetwork/guix-gn-deploy - -Make sure to restart the guix daemon and run guix client from this -directory. - -** Install GN2 - -Reinstall genenetwork2 using the new tree - -#+begin_src bash -env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --substitute-urls="http://guix.genenetwork.org https://mirror.guixsd.org" -#+end_src bash - -Note the use of ./pre-inst-env here! - -Actually, it should be the same installation as in step 4, so nothing -gets downloaded. - -** Run GN2 - -Make a note of the paths with - -#+begin_src bash -./pre-inst-env guix package --search-paths -#+end_src bash - -or this should also work if guix is installed - -#+begin_src bash -guix package --search-paths -#+end_src bash - -After setting the paths for the server - -#+begin_src bash -export PATH=~/.guix-profile/bin:$PATH -export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" -export R_LIBS_SITE="$HOME/.guix-profile/site-library/" -export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0" -export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0" -export XDG_DATA_DIRS="$HOME/.guix-profile/share" -export GIO_EXTRA_MODULES="$HOME/.guix-profile/lib/gio/modules" -#+end_src bash - -run the main script (in ~/.guix-profile/bin) - -#+begin_src bash -genenetwork2 -#+end_src bash - -will start the default server which listens on port 5003, i.e., -http://localhost:5003/. - -OK, we are where we were before with step 4. Only difference is that we -used our own compiled guix server. +See [[development.org]]. * Trouble shooting @@ -479,6 +220,31 @@ R_LIBS_SITE. Please check your GNU Guix GN2 installation paths, you man need to reinstall. Note that this may be the point you may want to start using profiles (see profile section). +** Mysql can't connect server through socket ERROR + +The following error + +: sqlalchemy.exc.OperationalError: (_mysql_exceptions.OperationalError) (2002, 'Can\'t connect to local MySQL server through socket \'/run/mysqld/mysqld.sock\' (2 "No such file or directory")') + +means that MySQL is trying to connect locally to a non-existent MySQL +server, something you may see in a container. Typically replicated with something like + +: mysql -h localhost + +try to connect over the network interface instead, e.g. + +: mysql -h 127.0.0.1 + +if that works run genenetwork after setting SQL_URI to something like + +: export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s + +* Read more + +If you want to understand the architecture of GN2 read +[[Architecture.org]]. The rest of this document is mostly on deployment +of GN2. + * IRC session Here an IRC session where we installed GN2 from scratch using GNU Guix diff --git a/doc/development.org b/doc/development.org new file mode 100644 index 00000000..5e6e318b --- /dev/null +++ b/doc/development.org @@ -0,0 +1,43 @@ +* Development + +** Using GN2_PROFILE + +After cloning the git source tree you can run the contained GN2 using +an existing GN2_PROFILE, i.e., use a profile that was create to run a +binary installation of GN2. This profile may be found by typing + +: which genenetwork2 +: /home/wrk/opt/gn-latest-guix/bin/genenetwork2 + +An example of running the development version would be + +: env GN2_PROFILE=/home/wrk/opt/gn-latest-guix ./bin/genenetwork2 + +Profiles are stored in /gnu/store, so you may pick one up there + +: readlink -f $(dirname $(dirname `which genenetwork2`)) +: /gnu/store/dvckpaw770b00l6rv4ijql8wrk11iypv-profile + +and use that instead. + +Note that the genenetwork2 script sets up the environment for running +the webserver. This includes path to R modules and python modules. These +are output on startup. To make sure there is no environment pollution you can + +** Javascript modules + +As of release 2.10-pre4 we Javascript modules are installed in three places: + +1. JS_GUIX_PATH: the Guix store - these are Guix pre-packaged modules +2. The git source tree (./wqflask/wqflask/static/packages/) +3. JS_GN_PATH: a local directory containing (temporary) development modules + +Packages currently in git (2) will move to JS_GUIX_PATH (1) over +time. This is to keep better track of origin updates. Putting packages +in git (2) is actively discouraged(!), unless there are GN2 specific +adaptations to the original Javascript modules. + +JS_GN_PATH (3) is for development purposes. By default is is set to +$HOME/genenetwork/javascript. Say you are working on an updated +version of a JS module not yet in (1) you can simply check out that +module in that path and it should show up. diff --git a/etc/VERSION b/etc/VERSION new file mode 100644 index 00000000..1785aa28 --- /dev/null +++ b/etc/VERSION @@ -0,0 +1 @@ +2.10rc3 diff --git a/etc/default_settings.py b/etc/default_settings.py index 6acea637..1b609414 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -9,6 +9,8 @@ # env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2 # env LOG_LEVEL=DEBUG ./bin/genenetwork2 ~/gn2_settings.py # +# Typically you need to set GN2_PROFILE too. +# # Note also that in the near future we will additionally fetch # settings from a JSON file # @@ -18,10 +20,9 @@ import os import sys -GN_VERSION = open("../VERSION","r").read() +GN_VERSION = open("../etc/VERSION","r").read() SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" -SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password@localhost/db_webqtl_s' -SQLALCHEMY_POOL_RECYCLE = 3600 +SQL_ALCHEMY_POOL_RECYCLE = 3600 GN_SERVER_URL = "http://localhost:8880/" # ---- Flask configuration (see website) @@ -45,24 +46,26 @@ WEBSERVER_DEPLOY = None # Deployment specifics (nyi) LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) LOG_LEVEL_DEBUG = '0' # logger.debugf log level (0-5, 5 = show all) LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls -LOG_SQLALCHEMY = 'False' +LOG_SQL_ALCHEMY = 'False' LOG_BENCH = True # Log bench marks USE_REDIS = True # REDIS caching (note that redis will be phased out) USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls +HOME = os.environ['HOME'] -# Paths to JS libraries +# ---- Default locations +GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files -TWITTER_POST_FETCHER_JS_PATH = os.environ['HOME']+"/genenetwork/Twitter-Post-Fetcher" +# ---- Path overrides for Genenetwork - the defaults are normally +# picked up from Guix or in the HOME directory -# ---- Path overrides for Genenetwork # TMPDIR is normally picked up from the environment -HOME=os.environ['HOME'] -LOGFILE = HOME+"/genenetwork2.log" -GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files -LOCAL_PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files +# PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) + +# ---- Local path to JS libraries - for development modules (only) +JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript" -# ---- GN2 Executables -PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) -PLINK_COMMAND = str.strip(os.popen("which plink2").read()) -GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) +# ---- GN2 Executables (overwrite for testing only) +# PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) +# PLINK_COMMAND = str.strip(os.popen("which plink2").read()) +# GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) diff --git a/test/lib/main_web_functionality.rb b/test/lib/main_web_functionality.rb index ec61de2d..9ae49995 100644 --- a/test/lib/main_web_functionality.rb +++ b/test/lib/main_web_functionality.rb @@ -13,16 +13,20 @@ describe MainWebFunctionality do it "Get to trait page" do page = @agent.get($host) + p page form = page.forms[1] form.buttons[0].value.must_equal "Search" # main menu is loaded # http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult + form.fields[0].value.must_equal "searchResult" form.fields[2].value = "MEAN=(15 16) LRS=(23 46)" form.fields[3].value = "mouse" form.fields[4].value = "BXD" form.fields[5].value = "Hippocampus mRNA" form.fields[6].value = "HC_M2_0606_P" search_page = @agent.submit(form, form.buttons.first) + p "==================" + p search_page probe_link = search_page.links.find { |l| l.text =~ /1435395_s_at/ } probe_link.uri.to_s.must_equal "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" show_trait_page = probe_link.click diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 1e47e183..e5f10edf 100644 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -65,7 +65,7 @@ ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?sp # want to reach this base dir assert_writable_dir(TEMPDIR) -TMPDIR = mk_dir(TEMPDIR+'gn2') +TMPDIR = mk_dir(TEMPDIR+'/gn2/') assert_writable_dir(TMPDIR) CACHEDIR = mk_dir(TMPDIR+'/cache/') diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index e5726656..a5fd86e2 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -57,7 +57,7 @@ import urlparse from pprint import pformat as pf -#Engine = sa.create_engine(zach_settings.SQLALCHEMY_DATABASE_URI) +#Engine = sa.create_engine(zach_settings.SQL_URI) # build MySql database connection diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index c28c617a..393ff2df 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -13,6 +13,11 @@ logger = logging.getLogger(__name__ ) OVERRIDES = {} +def app_set(command_id, value): + """Set application wide value""" + app.config.setdefault(command_id,value) + value + def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with valid_path is a function checking whether the @@ -40,6 +45,7 @@ def get_setting(command_id,guess=None): def value(command): if command: # sys.stderr.write("Found "+command+"\n") + app_set(command_id,command) return command else: return None @@ -89,6 +95,18 @@ def valid_path(dir): return dir return None +def js_path(module=None): + """ + Find the JS module in the two paths + """ + try_gn = get_setting("JS_GN_PATH")+"/"+module + if valid_path(try_gn): + return try_gn + try_guix = get_setting("JS_GUIX_PATH")+"/"+module + if valid_path(try_guix): + return try_guix + raise "No JS path found for "+module+" (check JS_GN_PATH)" + def pylmm_command(guess=None): return valid_bin(get_setting("PYLMM_COMMAND",guess)) @@ -205,21 +223,27 @@ SQL_URI = get_setting('SQL_URI') LOG_LEVEL = get_setting('LOG_LEVEL') LOG_LEVEL_DEBUG = get_setting_int('LOG_LEVEL_DEBUG') LOG_SQL = get_setting_bool('LOG_SQL') -LOG_SQLALCHEMY = get_setting_bool('LOG_SQLALCHEMY') +LOG_SQL_ALCHEMY = get_setting_bool('LOG_SQL_ALCHEMY') LOG_BENCH = get_setting_bool('LOG_BENCH') LOG_FORMAT = "%(message)s" # not yet in use USE_REDIS = get_setting_bool('USE_REDIS') USE_GN_SERVER = get_setting_bool('USE_GN_SERVER') GENENETWORK_FILES = get_setting('GENENETWORK_FILES') -TEMP_TRAITS = get_setting('TEMP_TRAITS') - -PYLMM_COMMAND = pylmm_command() -GEMMA_COMMAND = gemma_command() -GEMMA_RESULTS_PATH = get_setting('GEMMA_RESULTS_PATH') -PLINK_COMMAND = plink_command() +JS_GUIX_PATH = get_setting('JS_GUIX_PATH') +# assert_dir(JS_GUIX_PATH) - don't enforce right now +JS_GN_PATH = get_setting('JS_GN_PATH') +# assert_dir(JS_GN_PATH) + +PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) +GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) +PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) TEMPDIR = tempdir() # defaults to UNIX TMPDIR +# ---- Handle specific JS modules +JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher")) +assert_dir(JS_TWITTER_POST_FETCHER_PATH) + from six import string_types if os.environ.get('WQFLASK_OVERRIDES'): @@ -234,5 +258,3 @@ if os.environ.get('WQFLASK_OVERRIDES'): else: OVERRIDES[k] = cmd logger.debug(OVERRIDES) - -assert_dir(get_setting("TWITTER_POST_FETCHER_JS_PATH")) diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 4df872ad..96c2c301 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -22,8 +22,8 @@ def init_db(): # they will be registered properly on the metadata. Otherwise # you will have to import them first before calling init_db() #import yourapplication.models + logger.info("Initializing database connection") import wqflask.model - logger.debug("Creating all model metadata") Base.metadata.create_all(bind=engine) logger.info("Done creating all model metadata") diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 87532445..8882c515 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -37,7 +37,7 @@ from utility import temp_data from utility.benchmark import Bench from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_mapping, plink_mapping -from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, GEMMA_RESULTS_PATH, PLINK_COMMAND, TEMPDIR +from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR from utility.external import shell from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 49959a74..82a44796 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -1018,7 +1018,7 @@ class MarkerRegression(object): else: if self.selectedChr > -1: for i, qtlresult in enumerate(self.qtlresults): - if qtlresult['Chr'] != self.selectedChr: + if qtlresult['chr'] != self.selectedChr: continue if i==0 and qtlresult['Mb'] >= Mb: @@ -1027,7 +1027,7 @@ class MarkerRegression(object): #the trait's position is between two traits if i > 0 and self.qtlresults[i-1]['Mb'] < Mb and qtlresult['Mb'] >= Mb: - locPixel = xLeftOffset + plotXScale*(self.qtlresults[i-1]['cM']+(qtlresult['cM']-self.qtlresults[i-1]['cM'])*(Mb - self.qtlresults[i-1]['Mb'])/(qtlresult['Mb']-self.qtlresults[i-1]['Mb'])) + locPixel = xLeftOffset + plotXScale*(self.qtlresults[i-1]['Mb']+(qtlresult['Mb']-self.qtlresults[i-1]['Mb'])*(Mb - self.qtlresults[i-1]['Mb'])/(qtlresult['Mb']-self.qtlresults[i-1]['Mb'])) break #the trait's position is on the right of the last genotype @@ -1036,11 +1036,12 @@ class MarkerRegression(object): else: locPixel = xLeftOffset for i, _chr in enumerate(self.ChrList): - if _chr != Chr: - locPixel += (self.ChrLengthDistList[i] + self.GraphInterval)*plotXScale - else: - locPixel += (Mb*(_chr[-1].cM-_chr[0].cM)/self.ChrLengthCMList[i])*plotXScale - break + if i < (len(self.ChrList)-1): + if _chr != Chr: + locPixel += (self.ChrLengthDistList[i] + self.GraphInterval)*plotXScale + else: + locPixel += (Mb*(_chr[-1].cM-_chr[0].cM)/self.ChrLengthCMList[i])*plotXScale + break if locPixel >= 0 and self.plotScale == 'physic': traitPixel = ((locPixel, yZero), (locPixel-6, yZero+12), (locPixel+6, yZero+12)) canvas.drawPolygon(traitPixel, edgeColor=pid.black, fillColor=self.TRANSCRIPT_LOCATION_COLOR, closed=1) diff --git a/wqflask/wqflask/static/new/css/corr_scatter_plot2.css b/wqflask/wqflask/static/new/css/corr_scatter_plot2.css index 92e777c2..5aa9b9bf 100644 --- a/wqflask/wqflask/static/new/css/corr_scatter_plot2.css +++ b/wqflask/wqflask/static/new/css/corr_scatter_plot2.css @@ -1,19 +1,40 @@ .nvd3 .nv-axis.nv-x text { - font-size: 16px; - font-weight: normal; - fill: black; + font-size: 16px; + font-weight: normal; + fill: black; } .nvd3 .nv-axis.nv-y text { - font-size: 16px; - font-weight: normal; - fill: black; + font-size: 16px; + font-weight: normal; + fill:black; } -.nv-y .tick.zero line { - stroke: black; +.nv-x .nv-axis g path.domain { + stroke: black; + stroke-width: 2; } .nv-y .nv-axis g path.domain { - stroke: black; + stroke: black; + stroke-width: 2; +} + +.nvd3 .nv-axis.nv-x path.domain { + stroke-opacity: 1; +} + +.nvd3 .nv-axis.nv-y path.domain { + stroke-opacity: 1; +} + +line.nv-regLine { + stroke: red !important; +} + +.nv-axisMin-x, +.nv-axisMax-x, +.nv-axisMin-y, +.nv-axisMax-y { + display: none; } diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json index 861af3d6..f3712b5d 100644 --- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json +++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json @@ -1340,14 +1340,14 @@ "GSE9588 Human Liver Normal (Mar11) Both Sexes" ], [ - "384", - "HLCF_0311", - "GSE9588 Human Liver Normal (Mar11) Females" - ], - [ "383", "HLCM_0311", "GSE9588 Human Liver Normal (Mar11) Males" + ], + [ + "384", + "HLCF_0311", + "GSE9588 Human Liver Normal (Mar11) Females" ] ], "Phenotypes": [ @@ -1493,6 +1493,15 @@ ] ] }, + "Islets-Gerling": { + "Phenotypes": [ + [ + "None", + "Islets-GerlingPublish", + "Islets-Gerling Published Phenotypes" + ] + ] + }, "TIGEM-Retina-RNA-Seq": { "Phenotypes": [ [ @@ -1561,6 +1570,11 @@ ], "Heart mRNA": [ [ + "820", + "UCLA_AXB_BXA_Aor_Jan16", + "UCLA AXB/BXA Aorta Affy M430 2.0 (Jan16) RMA" + ], + [ "421", "IRCM_AXBXA_HRI0213", "IRCM AXB/BXA Mouse Heart ILM MouseRef-8 v2.0 (Feb13) RankInv" @@ -1568,6 +1582,11 @@ ], "Liver mRNA": [ [ + "822", + "UCLA_AXB_BXA_Liv_Jan16", + "UCLA AXB/BXA Liver Affy M430 2.0 (Jan16) RMA" + ], + [ "352", "GSE16780AB_UCLA_ML0911", "GSE16780 UCLA Mouse AXB/BXA Liver Affy HT M430A (Sep11) RMA" @@ -1727,11 +1746,6 @@ ], "Striatum mRNA": [ [ - "85", - "SA_M2_0905_P", - "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) PDNN" - ], - [ "84", "SA_M2_0905_R", "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) RMA" @@ -1740,22 +1754,27 @@ "83", "SA_M2_0905_M", "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) MAS5" + ], + [ + "85", + "SA_M2_0905_P", + "OHSU/VA B6D2F2 Striatum M430v2 (Sep05) PDNN" ] ] }, "BHF2": { "Adipose mRNA": [ [ - "197", - "UCLA_BHF2_ADIPOSE_FEMALE", - "UCLA BHF2 Adipose Female mlratio" - ], - [ "196", "UCLA_BHF2_ADIPOSE_MALE", "UCLA BHF2 Adipose Male mlratio" ], [ + "197", + "UCLA_BHF2_ADIPOSE_FEMALE", + "UCLA BHF2 Adipose Female mlratio" + ], + [ "165", "UCLA_BHF2_ADIPOSE_0605", "UCLA BHF2 Adipose (June05) mlratio" @@ -1763,16 +1782,16 @@ ], "Brain mRNA": [ [ - "199", - "UCLA_BHF2_BRAIN_FEMALE", - "UCLA BHF2 Brain Female mlratio" - ], - [ "198", "UCLA_BHF2_BRAIN_MALE", "UCLA BHF2 Brain Male mlratio" ], [ + "199", + "UCLA_BHF2_BRAIN_FEMALE", + "UCLA BHF2 Brain Female mlratio" + ], + [ "166", "UCLA_BHF2_BRAIN_0605", "UCLA BHF2 Brain (June05) mlratio" @@ -1787,16 +1806,16 @@ ], "Liver mRNA": [ [ - "201", - "UCLA_BHF2_LIVER_FEMALE", - "UCLA BHF2 Liver Female mlratio" - ], - [ "200", "UCLA_BHF2_LIVER_MALE", "UCLA BHF2 Liver Male mlratio" ], [ + "201", + "UCLA_BHF2_LIVER_FEMALE", + "UCLA BHF2 Liver Female mlratio" + ], + [ "167", "UCLA_BHF2_LIVER_0605", "UCLA BHF2 Liver (June05) mlratio" @@ -2000,11 +2019,6 @@ "UTHSC Brain mRNA U74Av2 (Nov05) PDNN" ], [ - "81", - "BR_U_0805_P", - "UTHSC Brain mRNA U74Av2 (Aug05) PDNN" - ], - [ "80", "BR_U_0805_M", "UTHSC Brain mRNA U74Av2 (Aug05) MAS5" @@ -2015,6 +2029,11 @@ "UTHSC Brain mRNA U74Av2 (Aug05) RMA" ], [ + "81", + "BR_U_0805_P", + "UTHSC Brain mRNA U74Av2 (Aug05) PDNN" + ], + [ "42", "CB_M_0204_P", "INIA Brain mRNA M430 (Feb04) PDNN" @@ -2142,6 +2161,13 @@ "BXD Genotypes" ] ], + "Heart mRNA": [ + [ + "819", + "UCLA_BXD_Aor_Jan16", + "UCLA BXD Aorta Affy M430 2.0 (Jan16) RMA" + ] + ], "Hematopoietic Cells mRNA": [ [ "149", @@ -2218,6 +2244,26 @@ "UMUTAffy Hippocampus Exon (Feb09) RMA" ], [ + "814", + "UTHSC_ILM_BXD_hipp_NOSb_0217", + "UTHSC BXD Hippocampus ILM v6.1 NOS Balanced (Feb17) RankInv" + ], + [ + "815", + "UTHSC_ILM_BXD_hipp_NOEb_0217", + "UTHSC BXD Hippocampus ILM v6.1 NOE Balanced (Feb17) RankInv" + ], + [ + "816", + "UTHSC_ILM_BXD_hipp_RSSb_0217", + "UTHSC BXD Hippocampus ILM v6.1 RSS Balanced (Feb17) RankInv" + ], + [ + "817", + "UTHSC_ILM_BXD_hipp_RSEb_0217", + "UTHSC BXD Hippocampus ILM v6.1 RSE Balanced (Feb17) RankInv" + ], + [ "780", "UTHSC_ILM_BXD_hipp_NOEb_0216", "UTHSC BXD Hippocampus ILM v6.1 NOE Balanced (Feb16) RankInv" @@ -2391,14 +2437,24 @@ ], "Liver Proteome": [ [ + "540", + "EPFLETHZBXDprotCD0514", + "EPFL/ETHZ BXD Liver, Chow Diet (Jun16) Top100 SWATH" + ], + [ + "541", + "EPFLETHZBXDprotHFD0514", + "EPFL/ETHZ BXD Liver, High Fat Diet (Jun16) Top100 SWATH" + ], + [ "704", "EPFLETHZBXDprotCD_LS1114", - "EPFL/ETHZ BXD Liver, Chow Diet (Oct14) Light SWATH" + "EPFL/ETHZ BXD Liver, Chow Diet (Oct14) Top10 SWATH" ], [ "705", "EPFLETHZBXDprotHF_LS1114", - "EPFL/ETHZ BXD Liver, High Fat Diet (Oct14) Light SWATH" + "EPFL/ETHZ BXD Liver, High Fat Diet Diet (Oct14) Top10 SWATH" ], [ "703", @@ -2414,20 +2470,15 @@ "489", "EPFLBXDprotHFDRPN0214", "EPFL/LISP BXD Liver, Soluble Proteins HFD (Feb14) SRM" - ], - [ - "540", - "EPFLETHZBXDprotCD0514", - "EPFL/ETHZ BXD Liver, Soluble Proteins CD (Jun14) SWATH" - ], - [ - "541", - "EPFLETHZBXDprotHFD0514", - "EPFL/ETHZ BXD Liver, Soluble Proteins HFD (Jun14) SWATH" ] ], "Liver mRNA": [ [ + "818", + "UCLA_BXD_Liv_Jan16", + "UCLA BXD Liver Affy M430 2.0 (Jan16) RMA" + ], + [ "430", "EPFLMouseLiverRMA0413", "EPFL/LISP BXD CD+HFD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA" @@ -2443,6 +2494,11 @@ "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA" ], [ + "433", + "EPFLMouseLiverBothExRMA0413", + "EPFL/LISP BXD CD+HFD Liver Affy Mouse Gene 1.0 ST (Apr13) RMA Exon Level" + ], + [ "700", "UTHSC-VGX_MmBXDHepatocytesRMA1014", "UT-VGX Hepatocytes Affy Mouse Gene 1.0 ST Gene Level (Oct14) RMA" @@ -2466,6 +2522,51 @@ "702", "SUH_Liv_RMAEx_0611", "SUH BXD Liver CCl4-treated Affy Mouse Gene 1.0 ST Exon Level (Jun11) RMA" + ], + [ + "256", + "GenEx_BXD_liverEt_M5_0912", + "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Both Sexes" + ], + [ + "257", + "GenEx_BXD_liverEt_M5M_0912", + "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Males" + ], + [ + "258", + "GenEx_BXD_liverEt_M5F_0912", + "GenEx BXD EtOH Liver Affy M430 2.0 (Sep12) MAS5 Females" + ], + [ + "307", + "GenEx_BXD_liverEt_RMA_0211", + "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Both Sexes" + ], + [ + "308", + "GenEx_BXD_liverEt_RMA_M_0211", + "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Males" + ], + [ + "309", + "GenEx_BXD_liverEt_RMA_F_0211", + "GenEx BXD EtOH Liver Affy M430 2.0 (Feb11) RMA Females" + ], + [ + "310", + "GenEx_BXD_liverSal_RMA_0211", + "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Both Sexes" + ], + [ + "311", + "GenEx_BXD_liverSal_RMA_M_0211", + "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Males" + ], + [ + "312", + "GenEx_BXD_liverSal_RMA_F_0211", + "GenEx BXD Sal Liver Affy M430 2.0 (Feb11) RMA Females" ] ], "Lung mRNA": [ @@ -2546,14 +2647,14 @@ "HQF BXD Neocortex ILM6v1.1 (Feb08) RankInv" ], [ - "275", - "DevNeocortex_ILM6.2P14RInv_1110", - "BIDMC/UTHSC Dev Neocortex P14 ILMv6.2 (Nov10) RankInv" - ], - [ "274", "DevNeocortex_ILM6.2P3RInv_1110", "BIDMC/UTHSC Dev Neocortex P3 ILMv6.2 (Nov10) RankInv" + ], + [ + "275", + "DevNeocortex_ILM6.2P14RInv_1110", + "BIDMC/UTHSC Dev Neocortex P14 ILMv6.2 (Nov10) RankInv" ] ], "Nucleus Accumbens mRNA": [ @@ -2761,11 +2862,6 @@ ], "Ventral Tegmental Area mRNA": [ [ - "230", - "VCUEtvsSal_0609_R", - "VCU BXD VTA Et vs Sal M430 2.0 (Jun09) RMA" - ], - [ "229", "VCUEtOH_0609_R", "VCU BXD VTA EtOH M430 2.0 (Jun09) RMA" @@ -2774,6 +2870,11 @@ "228", "VCUSal_0609_R", "VCU BXD VTA Sal M430 2.0 (Jun09) RMA" + ], + [ + "230", + "VCUEtvsSal_0609_R", + "VCU BXD VTA Et vs Sal M430 2.0 (Jun09) RMA" ] ] }, @@ -2920,6 +3021,27 @@ ] ] }, + "CIE-RMA": { + "Midbrain mRNA": [ + [ + "830", + "INIA_UTHSC_Mid_AffyMTA1_Apr17", + "INIA-UTHSC Midbrain CIE Affy MTA 1.0 GeneLevel (Apr17) RMA" + ], + [ + "834", + "INIA_UTHSC_Mid_AffyMTA1_Ex_May17", + "INIA-UTHSC Midbrain CIE Affy MTA 1.0 Exon Level (Apr17) RMA" + ] + ], + "Phenotypes": [ + [ + "None", + "CIE-RMAPublish", + "CIE-RMA Published Phenotypes" + ] + ] + }, "CMS": { "Phenotypes": [ [ @@ -3021,6 +3143,13 @@ "CXB Genotypes" ] ], + "Heart mRNA": [ + [ + "821", + "UCLA_CXB_Aor_Jan16", + "UCLA CXB Aorta Affy M430 2.0 (Jan16) RMA" + ] + ], "Hippocampus mRNA": [ [ "100", @@ -3033,6 +3162,13 @@ "Hippocampus Consortium M430v2 CXB (Dec05) PDNN" ] ], + "Liver mRNA": [ + [ + "823", + "UCLA_CXB_Liv_Jan16", + "UCLA CXB Liver Affy M430 2.0 (Jan16) RMA" + ] + ], "Phenotypes": [ [ "628", @@ -3048,6 +3184,7 @@ ] ] }, + "EMSR": {}, "HS": { "Hippocampus mRNA": [ [ @@ -3545,6 +3682,10 @@ "Brain, Development: Normal Gene Expression (Yale/Sestan)" ], [ + "Islets-Gerling", + "Pancreatic: Islets (UTHSC/Gerling)" + ], + [ "TIGEM-Retina-RNA-Seq", "Retina: Normal Adult Gene Expression, RNA-Seq (TIGEM)" ] @@ -3616,7 +3757,11 @@ ], [ "CIE-INIA", - "Chronic Intermittent Ethanol" + "Chronic Intermittent Ethanol Phase 1" + ], + [ + "CIE-RMA", + "Chronic Intermittent Ethanol Phase 2" ], [ "CMS", @@ -3631,6 +3776,10 @@ "CXB" ], [ + "EMSR", + "Ethanol-Medicated Stress Reduction" + ], + [ "HS", "Heterogeneous Stock" ], @@ -4414,6 +4563,12 @@ "Ventrolateral Prefrontal Cortex mRNA" ] ], + "Islets-Gerling": [ + [ + "Phenotypes", + "Phenotypes" + ] + ], "TIGEM-Retina-RNA-Seq": [ [ "Phenotypes", @@ -4630,6 +4785,10 @@ "Gastrointestinal mRNA" ], [ + "Heart mRNA", + "Heart mRNA" + ], + [ "Hematopoietic Cells mRNA", "Hematopoietic Cells mRNA" ], @@ -4780,6 +4939,16 @@ "LCM Brain Regions mRNA" ] ], + "CIE-RMA": [ + [ + "Phenotypes", + "Phenotypes" + ], + [ + "Midbrain mRNA", + "Midbrain mRNA" + ] + ], "CMS": [ [ "Phenotypes", @@ -4822,14 +4991,23 @@ "Genotypes" ], [ + "Heart mRNA", + "Heart mRNA" + ], + [ "Hippocampus mRNA", "Hippocampus mRNA" ], [ + "Liver mRNA", + "Liver mRNA" + ], + [ "Spleen mRNA", "Spleen mRNA" ] ], + "EMSR": [], "HS": [ [ "Phenotypes", diff --git a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js index 7a5a86bd..96763aca 100644 --- a/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js +++ b/wqflask/wqflask/static/new/javascript/draw_corr_scatterplot-2.js @@ -1,74 +1,64 @@ var chart; nv.addGraph(function() { - // - chart = nv.models.scatterChart(); - // - chart.showLegend(false); - chart.duration(300); - chart.color(d3.scale.category10().range()); - chart.pointRange([200,0]); - // - // chart.showDistX(true); - // chart.showDistY(true); - chart.xAxis.tickFormat(d3.format('.02f')); - chart.yAxis.tickFormat(d3.format('.02f')); - // chart.showXAxis(false); - // chart.showYAxis(false); - chart.xAxis.axisLabel(js_data.trait_1); - chart.yAxis.axisLabel(js_data.trait_2); - // - xmin = d3.min(js_data.data[0]); - xmax = d3.max(js_data.data[0]); - xrange = xmax - xmin; - ymin = d3.min(js_data.data[1]); - ymax = d3.max(js_data.data[1]); - yrange = ymax - ymin; - chart.xDomain([xmin - xrange/10, xmax + xrange/10]); - chart.yDomain([ymin - yrange/10, ymax + yrange/10]); - // - d3.select('#scatterplot2 svg').datum(nv.log(getdata())).call(chart); - nv.utils.windowResize(chart.update); - return chart; + // + chart = nv.models.scatterChart(); + // + chart.showLegend(false); + chart.duration(300); + chart.color(d3.scale.category10().range()); + chart.pointRange([200,0]); + // + chart.xAxis.axisLabel(js_data.trait_1); + chart.xAxis.axisLabelDistance(11); + chart.yAxis.axisLabel(js_data.trait_2); + chart.yAxis.axisLabelDistance(11); + // + xmin = d3.min(js_data.data[0]); + xmax = d3.max(js_data.data[0]); + xrange = xmax - xmin; + ymin = d3.min(js_data.data[1]); + ymax = d3.max(js_data.data[1]); + yrange = ymax - ymin; + chart.xDomain([xmin - xrange/10, xmax + xrange/10]); + chart.yDomain([ymin - yrange/10, ymax + yrange/10]); + chart.xAxis.tickFormat(d3.format(checkformat(xrange))); + chart.yAxis.tickFormat(d3.format(checkformat(yrange))); + // + chart.tooltip.contentGenerator(function (obj) { + return '<b style="font-size: 18px">(' + obj.point.x + ', ' + obj.point.y + ')</b>'; + }); + // + d3.select('#scatterplot2 svg').datum(nv.log(getdata())).call(chart); + nv.utils.windowResize(chart.update); + return chart; }); function getdata () { - var data = []; - data.push({ - values: [], - slope: js_data.slope, - intercept: js_data.intercept - }); - for (j = 0; j < js_data.data[0].length; j++) { - data[0].values.push({ - x: js_data.data[0][j], - y: js_data.data[1][j], - size: 10, - shape: 'circle' - }); - } - return data; + var data = []; + data.push({ + values: [], + slope: js_data.slope, + intercept: js_data.intercept + }); + for (j = 0; j < js_data.data[0].length; j++) { + data[0].values.push({ + x: js_data.data[0][j], + y: js_data.data[1][j], + size: 10, + shape: 'circle' + }); + } + return data; } - -function randomData(groups, points) { - var data = [], - shapes = ['circle'], - random = d3.random.normal(); - for (i = 0; i < groups; i++) { - data.push({ - key: 'Group ' + i, - values: [], - slope: Math.random() - .01, - intercept: Math.random() - .5 - }); - for (j = 0; j < points; j++) { - data[i].values.push({ - x: random(), - y: random(), - size: Math.random(), - shape: shapes[j % shapes.length] - }); - } - } - return data; + +function checkformat (range) { + cell = range / 10.0; + if (cell >= 1) { + return ",r"; + } else { + cell = -Math.log(cell); + n = cell.toString().split(".")[0].length; + return ",.0" + n + "f"; + } } diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index ec55ebeb..9ca3d63a 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -144,7 +144,7 @@ <p> Development and source code on <a href="https://github.com/genenetwork/">github</a> with <a href="https://github.com/genenetwork/genenetwork2/issues">issue tracker</a> and <a href="https://github.com/genenetwork/genenetwork2/blob/master/README.md">documentation</a>. Join the <a href="http://listserv.uthsc.edu/mailman/listinfo/genenetwork-dev">mailing list</a> and find us on <a href="https://webchat.freenode.net/">IRC</a> (#genenetwork channel). {% if version: %} - <p><small>GeneNetwork v{{ version }}</small></p> + <p><small>GeneNetwork {{ version }}</small></p> {% endif %} </div> diff --git a/wqflask/wqflask/templates/corr_scatterplot.html b/wqflask/wqflask/templates/corr_scatterplot.html index 4464025e..05891c95 100644 --- a/wqflask/wqflask/templates/corr_scatterplot.html +++ b/wqflask/wqflask/templates/corr_scatterplot.html @@ -53,16 +53,6 @@ <h2>Correlation Scatterplot</h2> -<table class="table"> - <tbody> - <tr><th class="text-right">num overlap</th> <td>{{jsdata.num_overlap}}</td></tr> - <tr><th class="text-right">slope</th> <td>{{jsdata.slope}}</td></tr> - <tr><th class="text-right">intercept</th> <td>{{jsdata.intercept}}</td></tr> - <tr><th class="text-right">r value</th> <td>{{jsdata.r_value}}</td></tr> - <tr><th class="text-right">p value</th> <td>{{jsdata.p_value}}</td></tr> - </tbody> -</table> - <!-- <table class="table"> <tr> @@ -85,12 +75,12 @@ --> <div id="scatterplot2"> - <svg style="width: 1000px; height: 800px;"></svg> + <svg style="width: 1000px; height: 800px; margin-left: 10px;"></svg> </div> <br> -<div style="margin-left: 75px;"> +<div style="margin-left: 80px;"> {% if trait_1.dataset.type == "ProbeSet" %} <div> @@ -144,6 +134,18 @@ </div> +<br> + +<table class="table table-hover table-striped table-bordered" style="width: 250px; margin-left: 80px;"> + <tbody> + <tr><th>Number</th> <td>{{jsdata.num_overlap}}</td></tr> + <tr><th>Slope</th> <td>{{'%0.3f' % jsdata.slope}}</td></tr> + <tr><th>Intercept</th> <td>{{'%0.3f' % jsdata.intercept}}</td></tr> + <tr><th>R value</th> <td>{{'%0.3f' % jsdata.r_value}}</td></tr> + <tr><th>P value</th> <td>{{'%0.3e' % jsdata.p_value}}</td></tr> + </tbody> +</table> + {% endblock %} {% block js %} diff --git a/wqflask/wqflask/templates/error.html b/wqflask/wqflask/templates/error.html index c707a4fc..2f1e06fa 100644 --- a/wqflask/wqflask/templates/error.html +++ b/wqflask/wqflask/templates/error.html @@ -35,7 +35,7 @@ </p> <pre> - GeneNetwork v{{ version }} {{ stack[0] }} + GeneNetwork {{ version }} {{ stack[0] }} {{ message }} (error) {{ stack[-3] }} {{ stack[-2] }} @@ -50,7 +50,7 @@ <a href="#Stack" class="btn btn-default" data-toggle="collapse">Toggle full stack trace</a> <div id="Stack" class="collapse"> <pre> - GeneNetwork v{{ version }} {% for line in stack %} {{ line }} + GeneNetwork {{ version }} {% for line in stack %} {{ line }} {% endfor %} </pre> </div> diff --git a/wqflask/wqflask/templates/marker_regression_gn1.html b/wqflask/wqflask/templates/marker_regression_gn1.html index 4cea51b2..65debd10 100644 --- a/wqflask/wqflask/templates/marker_regression_gn1.html +++ b/wqflask/wqflask/templates/marker_regression_gn1.html @@ -14,7 +14,7 @@ <input type="hidden" name="temp_uuid" value="{{ temp_uuid }}"> <input type="hidden" name="trait_id" value="{{ this_trait.name }}"> <input type="hidden" name="dataset" value="{{ dataset.name }}"> - {% if mapping_method == "reaper" %} + {% if mapping_method == "reaper" or mapping_method == "rqtl_geno" %} <input type="hidden" name="genofile" value="{{ dataset.group.genofile }}"> {% endif %} <input type="hidden" name="method" value="{{ mapping_method }}"> diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index 56b6cc62..5a880647 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -129,6 +129,7 @@ <script type="text/javascript" src="/static/new/js_external/d3-tip.min.js"></script> <script type="text/javascript" src="/static/new/js_external/jstat.min.js"></script> <script type="text/javascript" src="/static/new/js_external/shapiro-wilk.js"></script> + <script type="text/javascript" src="/static/new/js_external/saveSvgAsPng.js"></script> <script type="text/javascript" src="/static/new/javascript/colorbrewer.js"></script> <script type="text/javascript" src="/static/new/javascript/stats.js"></script> diff --git a/wqflask/wqflask/templates/show_trait_statistics.html b/wqflask/wqflask/templates/show_trait_statistics.html index 0a0e9990..ac44b656 100644 --- a/wqflask/wqflask/templates/show_trait_statistics.html +++ b/wqflask/wqflask/templates/show_trait_statistics.html @@ -94,12 +94,15 @@ <div class="tab-pane" id="probability_plot"> <div style="padding: 20px" class="form-horizontal"> {% if sample_groups|length != 1 %} + Select Group: <select class="prob_plot_samples_group"> {% for group, pretty_group in sample_group_types.items() %} <option value="{{ group }}">{{ pretty_group }}</option> {% endfor %} </select> - <br><br> + <button type="button" class="btn btn-default" id="down_prob_plot">Export as PNG</button> + <br> + <br> {% endif %} <div id="prob_plot_container"> diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 1e831896..f7fcd2d0 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -46,7 +46,7 @@ from wqflask.database import db_session from wqflask import model from utility import Bunch, Struct, after -from utility.tools import LOG_SQL, LOG_SQLALCHEMY +from utility.tools import LOG_SQL, LOG_SQL_ALCHEMY import logging from utility.logger import getLogger @@ -69,7 +69,7 @@ class AnonUser(object): if self.cookie: logger.debug("already is cookie") self.anon_id = verify_cookie(self.cookie) - + else: logger.debug("creating new cookie") self.anon_id, self.cookie = create_signed_cookie() @@ -78,17 +78,17 @@ class AnonUser(object): @after.after_this_request def set_cookie(response): response.set_cookie(self.cookie_name, self.cookie) - + def add_collection(self, new_collection): collection_dict = dict(name = new_collection.name, created_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), num_members = new_collection.num_members, members = new_collection.get_members()) - + Redis.set(self.key, json.dumps(collection_dict)) Redis.expire(self.key, 60 * 60 * 24 * 5) - + def delete_collection(self, collection_name): existing_collections = self.get_collections() updated_collections = [] @@ -106,7 +106,7 @@ class AnonUser(object): updated_collections.append(this_collection) Redis.set(self.key, json.dumps(updated_collections)) - + def get_collections(self): json_collections = Redis.get(self.key) if json_collections == None or json_collections == "None": @@ -117,7 +117,7 @@ class AnonUser(object): collection['created_timestamp'] = datetime.datetime.strptime(collection['created_timestamp'], '%b %d %Y %I:%M%p') collection['changed_timestamp'] = datetime.datetime.strptime(collection['changed_timestamp'], '%b %d %Y %I:%M%p') return collections - + def import_traits_to_user(self): collections_list = json.loads(Redis.get(self.key)) for collection in collections_list: @@ -131,7 +131,7 @@ class AnonUser(object): uc.members = json.dumps(collection['members']) db_session.add(uc) db_session.commit() - + def display_num_collections(self): """ Returns the number of collections or a blank string if there are zero. @@ -217,7 +217,7 @@ class UserSession(object): """Actual sqlalchemy record""" # Only look it up once if needed, then store it try: - if LOG_SQLALCHEMY: + if LOG_SQL_ALCHEMY: logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG) # Already did this before @@ -237,7 +237,7 @@ class UserSession(object): def before_request(): g.user_session = UserSession() g.cookie_session = AnonUser() - + class UsersManager(object): def __init__(self): self.users = model.User.query.all() @@ -541,9 +541,9 @@ class LoginUser(object): import_col = "true" else: import_col = "false" - - #g.cookie_session.import_traits_to_user() - + + #g.cookie_session.import_traits_to_user() + return self.actual_login(user, import_collections=import_col) else: diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 39f4a686..07151425 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -53,7 +53,7 @@ from wqflask.ctl import ctl_analysis #from wqflask.trait_submission import submit_trait from utility import temp_data -from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION +from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,JS_TWITTER_POST_FETCHER_PATH from utility.helper_functions import get_species_groups from base import webqtlFormData @@ -147,9 +147,8 @@ def tmp_page(img_path): img_base64 = bytesarray ) @app.route("/twitter/<path:filename>") -def bd_files(filename): - bd_path = app.config['TWITTER_POST_FETCHER_JS_PATH'] - return send_from_directory(bd_path, filename) +def twitter(filename): + return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename) #@app.route("/data_sharing") #def data_sharing_page(): |