diff options
26 files changed, 616 insertions, 191 deletions
diff --git a/VERSION b/VERSION index 3e0b7cab..a9a7884c 100644..120000 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10-pre4 +etc/VERSION \ No newline at end of file diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5e791885..8886e4bc 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,5 +1,14 @@ #! /bin/bash # +# This is the startup script for GN2. It sets the environment variables to pick +# up a Guix profile and allows for overriding parameters. +# +# Typical usage +# +# env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py +# +# Where GN2_PROFILE points to the GNU Guix profile used for deployment. +# # This will run the GN2 server (with default settings if none # supplied). Typically you need a GNU Guix profile which is set with # an environment variable (this profile is dictated by the @@ -16,8 +25,27 @@ # a -c switch, e.g. # # env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py +# +# For development you may want to run +# +# env GN2_PROFILE=~/opt/gn-latest-guix WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 +# +# For staging and production we use gunicorn. Run with something like +# (note you have to provide the server port). Provide a settings file! +# +# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod +# +# For development use +# +# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +# +# For extra flexibility you can also provide gunicorn parameters yourself with something like +# +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" -SCRIPT=$(readlink -f "$0") +SCRIPT=$(realpath "$0") +echo SCRIPT=$SCRIPT +echo GN2_PROFILE=$GN2_PROFILE GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) @@ -36,15 +64,27 @@ else fi echo GN_VERSION=$GN_VERSION +if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then + echo "Can not use $1 switch without default settings file" + exit 1 +fi # Handle settings parameter (can be .py or .json) -settings=$1 +if [ ! -z $1 ]; then + settings=$(realpath "$1") + if [ ! -e $settings ]; then + settings=$GN2_BASE_DIR/etc/default_settings.py + else + shift + fi +fi + ext="${settings##*.}" -if [ -z "$settings" -o "$ext" = "json" -o "$ext" = "JSON" ]; then +if [ "$ext" = "json" -o "$ext" = "JSON" ]; then overrides=$settings - settings=$GN2_BASE_DIR/etc/default_settings.py else - shift + echo $settings fi + if [ ! -e $settings ]; then echo "ERROR: can not locate settings file - pass it in the command line" exit 1 @@ -61,6 +101,8 @@ if [ -z $GN2_PROFILE ] ; then if [ -d $GN2_PROFILE ]; then echo "Best guess is $GN2_PROFILE" fi + echo "ERROR: always set GN2_PROFILE" + exit 1 fi if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." @@ -79,7 +121,12 @@ else export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" - export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" + if [ -z $GEMMA_WRAPPER_COMMAND ]; then + export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" + fi + if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi + if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi + if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi fi if [ -z $PYTHONPATH ] ; then echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" @@ -105,14 +152,51 @@ set|grep guix set|grep $GN2_PROFILE set|grep TMPDIR -# Now handle command parameter -c +# Now handle command parameter -c which runs python if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} echo PYTHONPATH=$PYTHONPATH echo RUNNING COMMAND $cmd python $cmd - exit 0 + exit $? +fi +# Now handle command parameter -cli which runs in bash +if [ "$1" = "-cli" ] ; then + echo "HERE" + cd $GN2_BASE_DIR/wqflask + cmd=$2 + echo PYTHONPATH=$PYTHONPATH + echo RUNNING COMMAND $cmd + $cmd + exit $? +fi +if [ "$1" = '-gunicorn' ] ; then + cd $GN2_BASE_DIR/wqflask + cmd=$2 + echo PYTHONPATH=$PYTHONPATH + echo RUNNING gunicorn $cmd + gunicorn $cmd + exit $? +fi +if [ "$1" = '-gunicorn-dev' ] ; then + cd $GN2_BASE_DIR/wqflask + echo PYTHONPATH=$PYTHONPATH + if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi + cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" + echo RUNNING gunicorn $cmd + gunicorn $cmd + exit $? +fi +if [ "$1" = '-gunicorn-prod' ] ; then + cd $GN2_BASE_DIR/wqflask + echo PYTHONPATH=$PYTHONPATH + if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi + PID=$TMPDIR/gunicorn.$USER.pid + cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID -k eventlet --workers 20 --keep-alive 1200 --max-requests 1000 --timeout 1200 wsgi" + echo RUNNING gunicorn $cmd + gunicorn $cmd + exit $? fi echo "Starting the redis server:" @@ -120,6 +204,9 @@ echo -n "dir $TMPDIR dbfilename gn2.rdb " | redis-server - & +# Overrides for packages that are not yet public (currently r-auwerx) +# export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib + # Start the flask server running GN2 cd $GN2_BASE_DIR/wqflask echo "Starting with $settings" diff --git a/doc/README.org b/doc/README.org index a39ef603..937a9549 100644 --- a/doc/README.org +++ b/doc/README.org @@ -104,11 +104,29 @@ As root configure and run : mysqld --datadir=/var/mysql --initialize-insecure : mkdir -p /var/run/mysqld : chown mysql.mysql ~/mysql /var/run/mysqld -: su mysql -c mysqld --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048 +: mysqld -u mysql --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048" -/etc/my.cnf -[mysqld] -user=root +If you want to run as root you may have to set + +: /etc/my.cnf +: [mysqld] +: user=root + +To check error output in a file on start-up run with something like + +: mysqld -u mysql --console --explicit_defaults_for_timestamp --datadir=/gnu/mysql --log-error=~/test.log + +Other tips are that Guix installs mysqld in your profile, so this may work + +: /home/user/.guix-profile/bin/mysqld -u mysql --explicit_defaults_for_timestamp --datadir=/gnu/mysql + +When you get errors like: + +: qlalchemy.exc.IntegrityError: (_mysql_exceptions.IntegrityError) (1215, 'Cannot add foreign key constraint') + +you may need to set + +: set foreign_key_checks=0 ** Load the small database in MySQL diff --git a/doc/database.org b/doc/database.org index 624174a4..5107b660 100644 --- a/doc/database.org +++ b/doc/database.org @@ -1,9 +1,19 @@ -- github Document reduction issue +* Database Information + +WARNING: This document contains information on the GN databases which +will change over time. The GN database is currently MySQL based and, +while efficient, contains a number of design choices we want to grow +'out' of. Especially with an eye on reproducibility we want to +introduce versioning. + +So do not treat the information in this document as a final way of +accessing data. It is better to use the +[[https://github.com/genenetwork/gn_server/blob/master/doc/API.md][REST API]]. * The small test database (2GB) The default install comes with a smaller database which includes a -number of the BSD's and the Human liver dataset (GSE9588). +number of the BXD's and the Human liver dataset (GSE9588). * GeneNetwork database @@ -750,9 +760,30 @@ show indexes from ProbeSetFreeze; | 1 | 5 | 0.303492 | +--------+----------+----------+ -** Publication and publishdata (all pheno) +** Publication + +Publication: + +| Id | PubMed_ID | Abstract | Title | Pages | Month | Year | + -Phenotype pubs +** Publishdata (all pheno) + +One of three phenotype tables. + +mysql> select * from PublishData limit 5; ++---------+----------+-------+ +| Id | StrainId | value | ++---------+----------+-------+ +| 8966353 | 349 | 29.6 | +| 8966353 | 350 | 27.8 | +| 8966353 | 351 | 26.6 | +| 8966353 | 352 | 28.5 | +| 8966353 | 353 | 24.6 | ++---------+----------+-------+ +5 rows in set (0.25 sec) + +See below for phenotype access. ** QuickSearch @@ -1073,7 +1104,37 @@ select * from ProbeSetXRef limit 5; i.e., for Strain Id 1 (DataId) 1, the locus '10.095.400' has a phenotype value of 5.742. -GeneNetwork1 already has a limited REST interface, if you do +Interestingly ProbeData and PublishData have the same layout as +ProbeSetData. ProbeData is only in use for Affy assays - and not used +for computations. PublishData contains trait values. ProbeSetData.id +matches ProbeSetXRef.DataId while PublishData.id matches +PublishXRef.DataId. + +select * from PublishXRef limit 3; ++-------+-------------+-------------+---------------+---------+----------------+------------------+-----------+----------+-------------------------------------------------------+ +| Id | InbredSetId | PhenotypeId | PublicationId | DataId | Locus | LRS | additive | Sequence | comments | ++-------+-------------+-------------+---------------+---------+----------------+------------------+-----------+----------+-------------------------------------------------------+ +| 10001 | 8 | 1 | 1 | 8966353 | D2Mit5 | 10.18351644706 | -1.20875 | 1 | | +| 10001 | 7 | 2 | 53 | 8966813 | D7Mit25UT | 9.85534330983917 | -2.86875 | 1 | | +| 10001 | 4 | 3 | 81 | 8966947 | CEL-6_57082524 | 11.7119505898121 | -23.28875 | 1 | elissa modified Abstract at Tue Jun 7 11:38:00 2005 | ++-------+-------------+-------------+---------------+---------+----------------+------------------+-----------+----------+-------------------------------------------------------+ +3 rows in set (0.00 sec) + +ties the trait data (PublishData) with the inbredsetid (matching +PublishFreeze.InbredSetId), locus and publication. + +select * from PublishFreeze -> ; ++----+------------+--------------------------+-------------+------------+--------+-------------+-----------------+-----------------+ +| Id | Name | FullName | ShortName | CreateTime | public | InbredSetId | confidentiality | AuthorisedUsers | ++----+------------+--------------------------+-------------+------------+--------+-------------+-----------------+-----------------+ +| 1 | BXDPublish | BXD Published Phenotypes | BXDPublish | 2004-07-17 | 2 | 1 | 0 | NULL | +| 18 | HLCPublish | HLC Published Phenotypes | HLC Publish | 2012-02-20 | 2 | 34 | 0 | NULL | ++----+------------+--------------------------+-------------+------------+--------+-------------+-----------------+-----------------+ +2 rows in set (0.02 sec) + +which gives us the datasets. + +GeneNetwork1 has a limited REST interface, if you do : curl "http://robot.genenetwork.org/webqtl/main.py?cmd=get&probeset=1443823_s_at&db=HC_M2_0606_P" @@ -1082,6 +1143,9 @@ we get : ProbeSetID B6D2F1 C57BL/6J DBA/2J BXD1 BXD2 BXD5 BXD6 BXD8 BXD9 BXD11 BXD12 BXD13 BXD15 BXD16 BXD19 BXD20 BXD21 BXD22 BXD23 BXD24 BXD27 BXD28 BXD29 BXD31 BXD32 BXD33 BXD34 BXD38 BXD39 BXD40 BXD42 BXD67 BXD68 BXD43 BXD44 BXD45 BXD48 BXD50 BXD51 BXD55 BXD60 BXD61 BXD62 BXD63 BXD64 BXD65 BXD66 BXD69 BXD70 BXD73 BXD74 BXD75 BXD76 BXD77 BXD79 BXD73a BXD83 BXD84 BXD85 BXD86 BXD87 BXD89 BXD90 BXD65b BXD93 BXD94 A/J AKR/J C3H/HeJ C57BL/6ByJ CXB1 CXB2 CXB3 CXB4 CXB5 CXB6 CXB7 CXB8 CXB9 CXB10 CXB11 CXB12 CXB13 BXD48a 129S1/SvImJ BALB/cJ BALB/cByJ LG/J NOD/ShiLtJ PWD/PhJ BXD65a BXD98 BXD99 CAST/EiJ KK/HlJ WSB/EiJ NZO/HlLtJ PWK/PhJ D2B6F1 : 1443823_s_at 15.251 15.626 14.716 15.198 14.918 15.057 15.232 14.968 14.87 15.084 15.192 14.924 15.343 15.226 15.364 15.36 14.792 14.908 15.344 14.948 15.08 15.021 15.176 15.14 14.796 15.443 14.636 14.921 15.22 15.62 14.816 15.39 15.428 14.982 15.05 15.13 14.722 14.636 15.242 15.527 14.825 14.416 15.125 15.362 15.226 15.176 15.328 14.895 15.141 15.634 14.922 14.764 15.122 15.448 15.398 15.089 14.765 15.234 15.302 14.774 14.979 15.212 15.29 15.012 15.041 15.448 14.34 14.338 14.809 15.046 14.816 15.232 14.933 15.255 15.21 14.766 14.8 15.506 15.749 15.274 15.599 15.673 14.651 14.692 14.552 14.563 14.164 14.546 15.044 14.695 15.162 14.772 14.645 15.493 14.75 14.786 15.003 15.148 15.221 +(see https://github.com/genenetwork/gn_server/blob/master/doc/API.md +for the latest REST API). + getTraitData is defined in the file [[https://github.com/genenetwork/genenetwork/blob/master/web/webqtl/textUI/cmdClass.py#L134][web/webqtl/textUI/cmdClass.py]]. probe is None, so the code at line 199 is run @@ -1165,6 +1229,97 @@ select * from ProbeSetData limit 5; 5 rows in set (0.00 sec) linked by ProbeSetXRef.dataid. + +*** For PublishData: + +List datasets for BXD (InbredSetId=1): + +select * from PublishXRef where InbredSetId=1 limit 3; ++-------+-------------+-------------+---------------+---------+-----------+------------------+------------------+----------+--------------------------------------------------------------------------------+ +| Id | InbredSetId | PhenotypeId | PublicationId | DataId | Locus | LRS | additive | Sequence | comments | ++-------+-------------+-------------+---------------+---------+-----------+------------------+------------------+----------+--------------------------------------------------------------------------------+ +| 10001 | 1 | 4 | 116 | 8967043 | rs8253516 | 13.4974914158039 | 2.39444444444444 | 1 | robwilliams modified post_publication_description at Mon Jul 30 14:58:10 2012 + | +| 10002 | 1 | 10 | 116 | 8967044 | rs3666069 | 22.0042692151629 | 2.08178571428572 | 1 | robwilliams modified phenotype at Thu Oct 28 21:43:28 2010 + | +| 10003 | 1 | 15 | 116 | 8967045 | D18Mit4 | 15.5929163293343 | 19.0882352941176 | 1 | robwilliams modified phenotype at Mon May 23 20:52:19 2011 + | ++-------+-------------+-------------+---------------+---------+-----------+------------------+------------------+----------+--------------------------------------------------------------------------------+ + +where ID is the 'record' or, effectively, dataset. + +select distinct(publicationid) from PublishXRef where InbredSetId=1 limit 3; ++---------------+ +| publicationid | ++---------------+ +| 116 | +| 117 | +| 118 | ++---------------+ + +select distinct +PublishXRef.id,publicationid,phenotypeid,Phenotype.post_publication_description +from PublishXRef,Phenotype where InbredSetId=1 and +phenotypeid=Phenotype.id limit 3; ++-------+---------------+-------------+----------------------------------------------------------------------------------------------------------------------------+ +| id | publicationid | phenotypeid | post_publication_description | ++-------+---------------+-------------+----------------------------------------------------------------------------------------------------------------------------+ +| 10001 | 116 | 4 | Central nervous system, morphology: Cerebellum weight [mg] | +| 10002 | 116 | 10 | Central nervous system, morphology: Cerebellum weight after adjustment for covariance with brain size [mg] | +| 10003 | 116 | 15 | Central nervous system, morphology: Brain weight, male and female adult average, unadjusted for body weight, age, sex [mg] | ++-------+---------------+-------------+----------------------------------------------------------------------------------------------------------------------------+ + +The id field is the same that is used in the GN2 web interface and the +PublicationID ties the datasets together. + +To list trait values: + +SELECT Strain.Name, PublishData.id, PublishData.value from +(Strain,PublishData, PublishXRef) Where PublishData.StrainId = +Strain.id limit 3; + ++------+---------+-------+ +| Name | id | value | ++------+---------+-------+ +| CXB1 | 8966353 | 29.6 | +| CXB1 | 8966353 | 29.6 | +| CXB1 | 8966353 | 29.6 | ++------+---------+-------+ + +here id should match dataid again: + +SELECT Strain.Name, PublishData.id, PublishData.value from +(Strain,PublishData, PublishXRef) Where PublishData.StrainId = +Strain.id and PublishXRef.dataid=8967043 and +PublishXRef.dataid=PublishData.id limit 3; ++------+---------+-------+ +| Name | id | value | ++------+---------+-------+ +| BXD1 | 8967043 | 61.4 | +| BXD2 | 8967043 | 49 | +| BXD5 | 8967043 | 62.5 | ++------+---------+-------+ + +*** Datasets + +The REST API aims to present a unified interface for genotype and +phenotype data. Phenotype datasets appear in two major forms in the +database and we want to present them as one resource. + +Dataset names are defined in ProbeSetFreeze.name and Published.id -> +publication (we'll ignore the probe dataset that uses +ProbeFreeze.name). These tables should be meshed. It looks like the +ids are non-overlapping with the publish record IDs starting at 10,001 +(someone has been smart, though it sets the limit of probesets now to +10,000). + +The datasets are organized differently in these tables. All published +BXD data is grouped on BXDpublished with the publications as +'datasets'. So, that is how we list them in the REST API. + +To fetch all the datasets we first list ProbeSetFreeze entries. Then +we list the Published entries. + ** Fetch genotype information *** SNPs diff --git a/etc/VERSION b/etc/VERSION index 1785aa28..ca9e199c 100644 --- a/etc/VERSION +++ b/etc/VERSION @@ -1 +1 @@ -2.10rc3 +2.11-rc1 diff --git a/etc/default_settings.py b/etc/default_settings.py index c00f6c8f..699d21f1 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -2,7 +2,7 @@ # webserver running in developer mode with limited console # output. Copy this file and run it from ./bin/genenetwork2 configfile # -# Note that these settings are fetched in ./wqflask/utilities/tools.py +# Note: these settings are fetched in ./wqflask/utilities/tools.py # which has support for overriding them through environment variables, # e.g. # @@ -14,8 +14,12 @@ # Note also that in the near future we will additionally fetch # settings from a JSON file # -# Note that values for False and 0 have to be strings here - otherwise +# Note: values for False and 0 have to be strings here - otherwise # Flask won't pick them up +# +# For GNU Guix deployment also check the paths in +# +# ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py import os import sys @@ -23,7 +27,7 @@ import sys GN_VERSION = open("../etc/VERSION","r").read() SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" SQL_ALCHEMY_POOL_RECYCLE = 3600 -GN_SERVER_URL = "http://localhost:8880/" +GN_SERVER_URL = "http://localhost:8880/" # REST API server # ---- Flask configuration (see website) TRAP_BAD_REQUEST_ERRORS = True @@ -34,7 +38,7 @@ SECURITY_RECOVERABLE = True SECURITY_EMAIL_SENDER = "no-reply@genenetwork.org" SECURITY_POST_LOGIN_VIEW = "/thank_you" -SERVER_PORT = 5003 +SERVER_PORT = 5003 # running on localhost SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' # ---- Behavioural settings (defaults) note that logger and log levels can @@ -42,6 +46,7 @@ SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) WEBSERVER_BRANDING = None # Set the branding (nyi) WEBSERVER_DEPLOY = None # Deployment specifics (nyi) +WEBSERVER_URL = "http://localhost:"+str(SERVER_PORT)+"/" # external URL LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) LOG_LEVEL_DEBUG = '0' # logger.debugf log level (0-5, 5 = show all) diff --git a/wqflask/__init__.py b/wqflask/__init__.py index 315b709e..e69de29b 100644 --- a/wqflask/__init__.py +++ b/wqflask/__init__.py @@ -1 +0,0 @@ -from wqflask import app diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 6649f8af..a4eaaa2e 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -332,7 +332,7 @@ class DatasetGroup(object): if check_plink_gemma(): marker_class = HumanMarkers else: - marker_class = Markers + marker_class = Markers if self.genofile: self.markers = marker_class(self.genofile[:-5]) diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 1ef2bc26..1e66e957 100644 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -82,6 +82,7 @@ assert_writable_dir(GENERATED_TEXT_DIR) # Flat file directories GENODIR = flat_files('genotype')+'/' assert_dir(GENODIR) +assert_dir(GENODIR+'bimbam') # for gemma # JSON genotypes are OBSOLETE JSON_GENODIR = flat_files('genotype/json')+'/' diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py new file mode 100644 index 00000000..14a2d689 --- /dev/null +++ b/wqflask/run_gunicorn.py @@ -0,0 +1,19 @@ +# Run with gunicorn, see ./bin/genenetwork2 for an example +# +# Run standalone with +# +# ./bin/genenetwork2 ./etc/default_settings.py -c run_gunicorn.py + +# from flask import Flask +# application = Flask(__name__) + +print "Starting up Gunicorn process" + +from wqflask import app + +@app.route("/gunicorn") +def hello(): + return "<h1 style='color:blue'>Hello There!</h1>" + +if __name__ == "__main__": + app.run(host='0.0.0.0') diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 50805643..a0c76e51 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -1,5 +1,7 @@ # Starts the webserver with the ./bin/genenetwork2 command # +# This uses Werkzeug WSGI, see ./run_gunicorn.py for the alternative +# # Please note, running with host set externally below combined with # debug mode is a security risk unless you have a firewall setup, e.g. # @@ -22,11 +24,19 @@ ENDC = '\033[0m' import os app.config['SECRET_KEY'] = os.urandom(24) -from utility.tools import WEBSERVER_MODE,get_setting_int +from utility.tools import WEBSERVER_MODE,get_setting_int,get_setting,get_setting_bool port = get_setting_int("SERVER_PORT") -logger.info("GN2 is running. Visit %shttp://localhost:%s/%s" % (BLUE,port,ENDC)) +print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]") + +if get_setting_bool("USE_GN_SERVER"): + import requests + page = requests.get(get_setting("GN_SERVER_URL")) + if page.status_code != 200: + raise Exception("API server not found!") + +print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) werkzeug_logger = logging.getLogger('werkzeug') diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py index bacb0aa4..128706df 100644 --- a/wqflask/utility/logger.py +++ b/wqflask/utility/logger.py @@ -72,7 +72,7 @@ LOG_LEVEL_DEBUG (NYI). def warning(self,*args): """Call logging.warning for multiple args""" self.collect(self.logger.warning,*args) - self.logger.warning(self.collect(*args)) + # self.logger.warning(self.collect(*args)) def error(self,*args): """Call logging.error for multiple args""" diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index c5685cdd..d3113302 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -16,7 +16,7 @@ OVERRIDES = {} def app_set(command_id, value): """Set application wide value""" app.config.setdefault(command_id,value) - value + return value def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in @@ -51,7 +51,7 @@ def get_setting(command_id,guess=None): return None # ---- Check whether environment exists - logger.debug("Looking for "+command_id+"\n") + # print("Looking for "+command_id+"\n") command = value(os.environ.get(command_id)) if command is None or command == "": command = OVERRIDES.get(command_id) @@ -63,7 +63,7 @@ def get_setting(command_id,guess=None): if command is None or command == "": # print command raise Exception(command_id+' setting unknown or faulty (update default_settings.py?).') - logger.debug("Set "+command_id+"="+str(command)) + # print("Set "+command_id+"="+str(command)) return command def get_setting_bool(id): @@ -105,7 +105,7 @@ def js_path(module=None): try_guix = get_setting("JS_GUIX_PATH")+"/"+module if valid_path(try_guix): return try_guix - raise "No JS path found for "+module+" (check JS_GN_PATH)" + raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)" def pylmm_command(guess=None): return assert_bin(get_setting("PYLMM_COMMAND",guess)) @@ -147,9 +147,14 @@ def assert_writable_dir(dir): fh.close() os.remove(fn) except IOError: - raise Exception('Unable to write test.txt to directory ' + dir ) + raise Exception('Unable to write test.txt to directory ' + dir) return dir +def assert_file(fn): + if not valid_file(fn): + raise Exception('Unable to find file '+fn) + return fn + def mk_dir(dir): if not valid_path(dir): os.makedirs(dir) @@ -174,6 +179,9 @@ def locate(name, subdir=None): if subdir: sys.stderr.write(subdir) raise Exception("Can not locate "+name+" in "+base) +def locate_phewas(name, subdir=None): + return locate(name,'/phewas/'+subdir) + def locate_ignore_error(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -239,15 +247,17 @@ USE_GN_SERVER = get_setting_bool('USE_GN_SERVER') GENENETWORK_FILES = get_setting('GENENETWORK_FILES') JS_GUIX_PATH = get_setting('JS_GUIX_PATH') -# assert_dir(JS_GUIX_PATH) - don't enforce right now +assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) -PYLMM_COMMAND = pylmm_command() -GEMMA_COMMAND = gemma_command() +PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) +GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) +assert(GEMMA_COMMAND is not None) +PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) GEMMA_WRAPPER_COMMAND = gemma_wrapper_command() -PLINK_COMMAND = plink_command() -TEMPDIR = tempdir() # defaults to UNIX TMPDIR +TEMPDIR = tempdir() # defaults to UNIX TMPDIR +assert_dir(TEMPDIR) # ---- Handle specific JS modules JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher")) @@ -257,7 +267,7 @@ from six import string_types if os.environ.get('WQFLASK_OVERRIDES'): jsonfn = get_setting('WQFLASK_OVERRIDES') - logger.error("WQFLASK_OVERRIDES: %s" % jsonfn) + logger.info("WQFLASK_OVERRIDES: %s" % jsonfn) with open(jsonfn) as data_file: overrides = json.load(data_file) for k in overrides: @@ -267,3 +277,6 @@ if os.environ.get('WQFLASK_OVERRIDES'): else: OVERRIDES[k] = cmd logger.debug(OVERRIDES) + +# assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData") +assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 24432ad0..3d1c0d17 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -75,6 +75,46 @@ def print_mem(stage=""): mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss #print("{}: {}".format(stage, mem/1024)) +def is_float(value): + try: + float(value) + return True + except: + return False + +def is_int(value): + try: + int(value) + return True + except: + return False + +def is_str(value): + if value is None: + return False + try: + str(value) + return True + except: + return False + +def get_float(vars,name,default=None): + if name in vars: + if is_float(vars[name]): + return float(vars[name]) + return None + +def get_int(vars,name,default=None): + if name in vars: + if is_int(vars[name]): + return float(vars[name]) + return default + +def get_string(vars,name,default=None): + if name in vars: + if not vars[name] is None: + return str(vars[name]) + return default class AuthException(Exception): pass @@ -96,7 +136,19 @@ class CorrelationResults(object): # get trait list from db (database name) # calculate correlation with Base vector and targets - print("TESTING...") + # Check parameters + assert('corr_type' in start_vars) + assert(is_str(start_vars['corr_type'])) + assert('dataset' in start_vars) + # assert('group' in start_vars) permitted to be empty? + assert('corr_sample_method' in start_vars) + assert('corr_samples_group' in start_vars) + assert('corr_dataset' in start_vars) + assert('min_expr' in start_vars) + assert('corr_return_results' in start_vars) + if 'loc_chr' in start_vars: + assert('min_loc_mb' in start_vars) + assert('max_loc_mb' in start_vars) with Bench("Doing correlations"): if start_vars['dataset'] == "Temp": @@ -115,27 +167,17 @@ class CorrelationResults(object): self.sample_data = {} self.corr_type = start_vars['corr_type'] self.corr_method = start_vars['corr_sample_method'] - if 'min_expr' in start_vars: - if start_vars['min_expr'] != "": - self.min_expr = float(start_vars['min_expr']) - else: - self.min_expr = None - self.p_range_lower = float(start_vars['p_range_lower']) - self.p_range_upper = float(start_vars['p_range_upper']) + self.min_expr = get_float(start_vars,'min_expr') + self.p_range_lower = get_float(start_vars,'p_range_lower',-1.0) + self.p_range_upper = get_float(start_vars,'p_range_upper',1.0) if ('loc_chr' in start_vars and 'min_loc_mb' in start_vars and 'max_loc_mb' in start_vars): - self.location_chr = start_vars['loc_chr'] - if start_vars['min_loc_mb'].isdigit(): - self.min_location_mb = start_vars['min_loc_mb'] - else: - self.min_location_mb = None - if start_vars['max_loc_mb'].isdigit(): - self.max_location_mb = start_vars['max_loc_mb'] - else: - self.max_location_mb = None + self.location_chr = get_string(start_vars,'loc_chr') + self.min_location_mb = get_int(start_vars,'min_loc_mb') + self.max_location_mb = get_int(start_vars,'max_loc_mb') self.get_formatted_corr_type() self.return_number = int(start_vars['corr_return_results']) @@ -183,7 +225,7 @@ class CorrelationResults(object): else: for trait, values in self.target_dataset.trait_data.iteritems(): self.get_sample_r_and_p_values(trait, values) - + elif self.corr_type == "lit": self.trait_geneid_dict = self.dataset.retrieve_genes("GeneId") lit_corr_data = self.do_lit_correlation_for_all_traits() @@ -564,7 +606,7 @@ class CorrelationResults(object): self.this_trait_vals.append(sample_value) target_vals.append(target_sample_value) - self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(self.this_trait_vals, target_vals) + self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(self.this_trait_vals, target_vals) #ZS: 2015 could add biweight correlation, see http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3465711/ if self.corr_method == 'pearson': @@ -574,8 +616,8 @@ class CorrelationResults(object): if num_overlap > 5: self.correlation_data[trait] = [sample_r, sample_p, num_overlap] - - + + """ correlations = [] @@ -673,8 +715,8 @@ class CorrelationResults(object): method=self.method) return trait_list - """ - + """ + def do_tissue_corr_for_all_traits_2(self): """Comments Possibly Out of Date!!!!! @@ -1089,7 +1131,7 @@ class CorrelationResults(object): totalTraits = len(traits) #XZ, 09/18/2008: total trait number return traits - + def calculate_corr_for_all_tissues(self, tissue_dataset_id=None): symbol_corr_dict = {} @@ -1129,7 +1171,7 @@ class CorrelationResults(object): values_2.append(target_value) correlation = calCorrelation(values_1, values_2) self.correlation_data[trait] = correlation - + def getFileName(self, target_db_name): ### dcrowell August 2008 """Returns the name of the reference database file with which correlations are calculated. Takes argument cursor which is a cursor object of any instance of a subclass of templatePage @@ -1144,7 +1186,7 @@ class CorrelationResults(object): return FileName def do_parallel_correlation(self, db_filename, num_overlap): - + #XZ, 01/14/2009: This method is for parallel computing only. #XZ: It is supposed to be called when "Genetic Correlation, Pearson's r" (method 1) #XZ: or "Genetic Correlation, Spearman's rho" (method 2) is selected @@ -1313,7 +1355,7 @@ class CorrelationResults(object): z_value = z_value*math.sqrt(nOverlap-3) sample_p = 2.0*(1.0 - reaper.normp(abs(z_value))) - correlation_data[traitdataName] = [sample_r, sample_p, nOverlap] + correlation_data[traitdataName] = [sample_r, sample_p, nOverlap] # traitinfo = [traitdataName, sample_r, nOverlap] # allcorrelations.append(traitinfo) @@ -1321,7 +1363,7 @@ class CorrelationResults(object): return correlation_data # return allcorrelations - + datasetFile = open(webqtlConfig.GENERATED_TEXT_DIR+db_filename,'r') print("Invoking parallel computing") @@ -1378,5 +1420,3 @@ class CorrelationResults(object): # for one_result in results: # for one_traitinfo in one_result: # allcorrelations.append( one_traitinfo ) - - diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index a24e43d4..68920130 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -3,7 +3,7 @@ import os, math, string, random, json from base import webqtlConfig from base.trait import GeneralTrait from base.data_set import create_dataset -from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR +from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR, assert_bin, assert_file import utility.logger logger = utility.logger.getLogger(__name__ ) @@ -11,6 +11,7 @@ logger = utility.logger.getLogger(__name__ ) def run_gemma(this_dataset, samples, vals, covariates, method, use_loco): """Generates p-values for each marker using GEMMA""" + assert_bin(GEMMA_COMMAND); if this_dataset.group.genofile != None: genofile_name = this_dataset.group.genofile[:-5] else: @@ -27,7 +28,7 @@ def run_gemma(this_dataset, samples, vals, covariates, method, use_loco): if i < (len(this_chromosomes) - 1): chr_list_string += this_chromosomes[i+1].name + "," else: - chr_list_string += this_chromosomes[i+1].name + chr_list_string += this_chromosomes[i+1].name if covariates != "": gen_covariates_file(this_dataset, covariates) @@ -209,8 +210,13 @@ def parse_gemma_output(genofile_name): def parse_loco_output(this_dataset, gwa_output_filename): output_filelist = [] - with open("{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json") as data_file: - data = json.load(data_file) + jsonfn = "{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json" + assert_file(jsonfn) + try: + with open(jsonfn) as data_file: + data = json.load(data_file) + except: + logger.error("Can not parse "+jsonfn) files = data['files'] for file in files: @@ -247,4 +253,4 @@ def parse_loco_output(this_dataset, gwa_output_filename): included_markers.append(line.split("\t")[1]) p_values.append(float(line.split("\t")[10])) - return marker_obs \ No newline at end of file + return marker_obs diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index bcb14451..087b95b4 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -79,6 +79,16 @@ class MarkerRegression(object): self.samples.append(sample) self.vals.append(value) + #ZS: Check if genotypes exist in the DB in order to create links for markers + if "geno_db_exists" in start_vars: + self.geno_db_exists = start_vars['geno_db_exists'] + else: + try: + geno_dataset = data_set.create_dataset(self.dataset.group.name + "Geno") + self.geno_db_exists = "True" + except: + self.geno_db_exists = "False" + self.mapping_method = start_vars['method'] if "results_path" in start_vars: self.mapping_results_path = start_vars['results_path'] diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 93bd9d42..211cf187 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -174,6 +174,8 @@ class MarkerRegression(object): if 'genofile_string' in start_vars: self.genofile_string = start_vars['genofile_string'] + self.geno_db_exists = start_vars['geno_db_exists'] + #Needing for form submission when doing single chr mapping or remapping after changing options self.samples = start_vars['samples'] self.vals = start_vars['vals'] @@ -576,7 +578,7 @@ class MarkerRegression(object): self.gifmap = gifmap.__str__() self.filename= webqtlUtil.genRandStr("Itvl_") - intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename), format='jpeg') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename), format='png') intImg=HT.Image('/image/'+self.filename+'.png', border=0, usemap='#WebQTLImageMap') #Scales plot differently for high resolution @@ -616,7 +618,7 @@ class MarkerRegression(object): else: showLocusForm = intImg - if self.permChecked and self.nperm > 0 and not self.multipleInterval and 0 < self.nperm: + if (self.permChecked and self.nperm > 0) and not (self.multipleInterval and 0 < self.nperm): self.perm_filename = self.drawPermutationHistogram() #perm_text_file = self.permutationTextFile() @@ -1200,8 +1202,10 @@ class MarkerRegression(object): if self.controlLocus and self.doControl != "false": string2 = 'Using %s as control' % self.controlLocus else: - if self.mapping_method == "gemma": + if self.mapping_method == "gemma" or self.mapping_method == "gemma_bimbam": string2 = 'Using GEMMA mapping method with no control for other QTLs.' + elif self.mapping_method == "rqtl_plink" or self.mapping_method == "rqtl_geno": + string2 = 'Using R/qtl mapping method with no control for other QTLs.' elif self.mapping_method == "plink": string2 = 'Using PLINK mapping method with no control for other QTLs.' else: @@ -1963,6 +1967,8 @@ class MarkerRegression(object): if self.permChecked and self.nperm > 0 and not self.multipleInterval: LRS_LOD_Max = max(self.significant, LRS_LOD_Max) + else: + LRS_LOD_Max = 1.15*LRS_LOD_Max #genotype trait will give infinite LRS LRS_LOD_Max = min(LRS_LOD_Max, webqtlConfig.MAXLRS) @@ -2164,10 +2170,7 @@ class MarkerRegression(object): # Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/LRS_LOD_Max if self.manhattan_plot == True: - if previous_chr_as_int % 2 == 1: - point_color = pid.grey - else: - point_color = pid.black + point_color = pid.black canvas.drawString("5", Xc-canvas.stringWidth("5",font=symbolFont)/2+1,Yc+2,color=point_color, font=symbolFont) else: LRSCoordXY.append((Xc, Yc)) diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json index 8de85a86..b7ebb9ed 100644 --- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json +++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json @@ -2974,6 +2974,20 @@ ] ] }, + "BXD-Harvested": { + "Liver mRNA": [ + [ + "843", + "UTHSC-BXD-Harv_Liv-0118", + "UTHSC BXD Harvested Liver RNA-Seq (Jan18) Log2 **" + ], + [ + "842", + "UTHSC-BXD-Liv-0917", + "UTHSC BXD Liver Affy Clariom S GeneLevel Main (Sep17) RMA **" + ] + ] + }, "BXD300": { "Genotypes": [ [ @@ -3864,6 +3878,10 @@ "BXD Bone" ], [ + "BXD-Harvested", + "BXD NIA Longevity Study" + ], + [ "BXD300", "BXD300" ], @@ -5007,6 +5025,12 @@ "Phenotypes" ] ], + "BXD-Harvested": [ + [ + "Liver mRNA", + "Liver mRNA" + ] + ], "BXD300": [ [ "Phenotypes", diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index 2f1d836a..4d98f5d8 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -245,6 +245,7 @@ //$("#static_progress_bar_container").modal(); url = "/loading"; $('input[name=method]').val("gemma_bimbam"); + $('input[name=num_perm]').val(0); $('input[name=genofile]').val($('#genofile_gemma').val()); $('input[name=maf]').val($('input[name=maf_gemma]').val()); form_data = $('#trait_data_form').serialize(); diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index cede0e86..46136ddb 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -1,20 +1,20 @@ <title>Loading Mapping Results</title> <link REL="stylesheet" TYPE="text/css" href="/static/packages/bootstrap/css/bootstrap.css" /> -<link REL="stylesheet" TYPE="text/css" href="/static/packages/bootstrap/css/non-responsive.css" /> -<link REL="stylesheet" TYPE="text/css" href="/static/packages/bootstrap/css/docs.css" /> -<link rel="stylesheet" type="text/css" href="/static/packages/colorbox/example4/colorbox.css" /> -<link rel="stylesheet" type="text/css" href="/static/new/css/main.css" /> <form method="post" action="/marker_regression" name="loading_form" id="loading_form" class="form-horizontal"> {% for key, value in start_vars.iteritems() %} <input type="hidden" name="{{ key }}" value="{{ value }}"> {% endfor %} <div class="container"> - <div class="row"> - <div style="margin-left: 46%; margin-right: 50%; min-height: 100vh; display: flex; align-items: center;"> - <h1>Loading Mapping Results...</h1> + <div> + <div style="min-height: 80vh; display: flex; align-items: center;"> + <div class="center-block" style="margin-left: 38%; margin-right: 38%; position: absolute; height:50px; width:24%; top:40%;"> + <h1>Loading Mapping Results...</h1> + </div> </div> - <div class="progress center-block" style="margin-left: 25%; margin-right: 25%; position: absolute; height:50px; width:50%; top:50%;"> - <div class="progress-bar progress-bar-striped active" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width:100%;"></div> + <div style="min-height: 80vh; display: flex; align-items: center;"> + <div class="progress center-block" style="margin-left: 25%; margin-right: 25%; position: absolute; height:50px; width:50%; top:50%;"> + <div class="progress-bar progress-bar-striped active" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width:100%;"></div> + </div> </div> </div> </div> diff --git a/wqflask/wqflask/templates/marker_regression_gn1.html b/wqflask/wqflask/templates/marker_regression_gn1.html index d13c68cd..08a934a6 100644 --- a/wqflask/wqflask/templates/marker_regression_gn1.html +++ b/wqflask/wqflask/templates/marker_regression_gn1.html @@ -15,6 +15,7 @@ <input type="hidden" name="trait_id" value="{{ this_trait.name }}"> <input type="hidden" name="dataset" value="{{ dataset.name }}"> <input type="hidden" name="genofile" value="{{ genofile_string }}"> + <input type="hidden" name="geno_db_exists" value="{{ geno_db_exists }}"> <input type="hidden" name="results_path" value="{{ mapping_results_path }}"> <input type="hidden" name="method" value="{{ mapping_method }}"> {% for sample in samples %} @@ -154,7 +155,7 @@ <div class="tab-pane active" id="gn1_map"> <div class="qtlcharts"> {{ gifmap|safe }} - <img src="/generated/{{ filename }}.jpeg" usemap="#WebQTLImageMap"> + <img src="/generated/{{ filename }}.png" usemap="#WebQTLImageMap"> {% if additiveChecked|upper == "ON" %} <br> <span style="white-space: nowrap;">A positive additive coefficient (green line) indicates that {{ dataset.group.parlist[1] }} alleles increase trait values. In contrast, a negative additive coefficient (orange line) indicates that {{ dataset.group.parlist[0] }} alleles increase trait values.</span> @@ -180,10 +181,10 @@ </form> {% if selectedChr == -1 %} - <div style="width:{% if 'additive' in trimmed_markers[0] %}45%{% else %}35%{% endif %};"> - <h2>Results</h2> - <div id="table_container"> - <table id="qtl_results" class="table table-hover table-striped nowrap"> + <div class="container" style="padding-left: 30px; width:{% if 'additive' in trimmed_markers[0] %}45%{% else %}35%{% endif %};"> + <h2>Mapping Statistics</h2> + <div id="table_container" style="border-style: solid; border-width: 1px; border-color: black;"> + <table id="trait_table" class="table table-hover table-striped nowrap"> <thead> <tr> <th></th> @@ -194,7 +195,7 @@ {% if plotScale != "physic" %} <th>cM</th> {% else %} - <th>Mb</th> + <th align="right">Mb</th> {% endif %} {% if 'additive' in trimmed_markers[0] %} <th>Add Eff</th> @@ -209,11 +210,11 @@ <tr> <td align="center" style="padding-right: 0px;"> <input type="checkbox" name="selectCheck" - class="checkbox edit_sample_checkbox" - value="{{ marker.name }}" checked="checked"> + class="checkbox trait_checkbox" + value="{{ marker.name }}"> </td> <td align="right">{{ loop.index }}</td> - <td>{{ marker.name }}</td> + <td>{% if geno_db_exists == "True" %}<a href="/show_trait?trait_id={{ marker.name }}&dataset={{ dataset.group.name }}Geno">{{ marker.name }}</a>{% else %}{{ marker.name }}{% endif %}</td> {% if LRS_LOD == "LOD" or LRS_LOD == "-log(p)" %} {% if 'lod_score' in marker %} <td align="right">{{ '%0.2f' | format(marker.lod_score|float) }}</td> @@ -294,6 +295,7 @@ </script> {% endif %} + <script language="javascript" type="text/javascript" src="/static/new/javascript/search_results.js"></script> {% if mapping_method != "gemma" and mapping_method != "plink" %} <script language="javascript" type="text/javascript" src="/static/new/javascript/panelutil.js"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/chr_lod_chart.js"></script> @@ -304,7 +306,7 @@ <script type="text/javascript" charset="utf-8"> $(document).ready( function () { console.time("Creating table"); - $('#qtl_results').DataTable( { + $('#trait_table').DataTable( { {% if mapping_method != "reaper" %} "columns": [ { "type": "natural", "width": "5%" }, diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index a291baf3..dd4325e7 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -28,7 +28,9 @@ --> <div class="container"> <h2>Trait Data and Analysis for <b>{{ this_trait.name }}</b></h2> + {% if this_trait.dataset.type != 'Publish' %} <h3>{{ this_trait.description_fmt }}</h3> + {% endif %} </div> <form method="post" action="/corr_compute" target="_blank" name="trait_page" id="trait_data_form" diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 2a62733e..1e6d41c2 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -8,6 +8,22 @@ <td>Tissue</td> <td>{{ this_trait.dataset.tissue }}</td> </tr> + <tr> + <td>Phenotype</td> + <td><div style="width:40%;">{{ this_trait.description_fmt }}</div></td> + </tr> + <tr> + <td>Authors</td> + <td><div style="width:40%;">{{ this_trait.authors }}</div></td> + </tr> + <tr> + <td>Title</td> + <td><div style="width:40%;">{{ this_trait.title }}</div></td> + </tr> + <tr> + <td>Journal</td> + <td>{{ this_trait.journal }} (<a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids={{ this_trait.pubmed_id }}&dop=Abstract" title="PubMed">{{ this_trait.year }}</a>)</td> + </tr> {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} {% if this_trait.symbol != None %} @@ -50,22 +66,29 @@ </td> </tr> {% endif %} + {% if this_trait.pubmed_id or this_trait.geneid or this_trait.omim or this_trait.symbol %} + {% if this_trait.dataset.type != 'Publish' %} <tr> <td>Resource Links</td> <td> - {% if this_trait.geneid != None %} + {% if this_trait.pubmed_id %} + <a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids={{ this_trait.pubmed_id }}&dop=Abstract" title="PubMed"> + PubMed + </a> + {% endif %} + {% if this_trait.geneid %} <a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids={{ this_trait.geneid }}" title="Info from NCBI Entrez Gene"> NCBI </a> {% endif %} - {% if this_trait.omim != None %} + {% if this_trait.omim %} <a href="http://www.ncbi.nlm.nih.gov/omim/{{ this_trait.omim }}" title="Summary from On Mendelion Inheritance in Man"> OMIM </a> {% endif %} - {% if this_trait.symbol != None %} + {% if this_trait.symbol %} <a href="http://www.genotation.org/Getd2g.pl?gene_list={{ this_trait.symbol }}" title="Related descriptive, genomic, clinical, functional and drug-therapy information"> Genotation </a> @@ -77,6 +100,8 @@ {% endif %} </td> </tr> + {% endif %} + {% endif %} </table> <div style="margin-bottom:15px;" class="btn-toolbar"> diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 77f78415..dcec2b9e 100644 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -6,16 +6,16 @@ <ul class="nav nav-pills"> {% if dataset.group.mapping_id == "1" %} <li class="active"> - <a href="#interval_mapping" data-toggle="tab">Interval Mapping</a> + <a href="#gemma" data-toggle="tab">GEMMA</a> </li> <li> - <a href="#pylmm" data-toggle="tab">pyLMM</a> + <a href="#interval_mapping" data-toggle="tab">Interval Mapping</a> </li> <li> <a href="#rqtl_geno" data-toggle="tab">R/qtl</a> </li> <li> - <a href="#gemma" data-toggle="tab">GEMMA</a> + <a href="#pylmm" data-toggle="tab">pyLMM</a> </li> {% endif %} {% for mapping_method in dataset.group.mapping_names %} @@ -37,7 +37,60 @@ <div class="tab-content"> {# if use_pylmm_rqtl and not use_plink_gemma and dataset.group.species != "human" #} {% if dataset.group.mapping_id == "1" %} - <div class="tab-pane active" id="interval_mapping"> + <div class="tab-pane active" id="gemma"> + <div style="padding-top: 10px;" class="form-horizontal"> + {% if genofiles and genofiles|length>0 %} + <div class="mapping_method_fields form-group"> + <label for="genofiles" class="col-xs-3 control-label">Genotypes</label> + <div style="margin-left: 20px;" class="col-xs-8 controls"> + <select id="genofile_gemma" class="form-control"> + {% for item in genofiles %} + <option value="{{item['location']}}:{{item['title']}}">{{item['title']}}</option> + {% endfor %} + </select> + </div> + </div> + {% endif %} + <div class="mapping_method_fields form-group"> + <label for="maf_gemma" class="col-xs-5 control-label">Minor allele threshold</label> + <div style="margin-left: 20px;" class="col-xs-3 controls"> + <input name="maf_gemma" value="0.01" type="text" class="form-control"> + </div> + </div> + <div class="mapping_method_fields form-group"> + <label class="col-xs-4 control-label">Use LOCO</label> + <div style="margin-left: 20px;" class="col-xs-4 controls"> + <label class="radio-inline"> + <input type="radio" name="use_loco" value="True"> + Yes + </label> + <label class="radio-inline"> + <input type="radio" name="use_loco" value="False" checked=""> + No + </label> + </div> + </div> + </div> + <div style="padding-top: 5px; padding-bottom: 5px; padding-left: 20px;" class="form-horizontal"> + <div class="mapping_method_fields form-group"> + <button type="button" id="select_covariates" class="btn btn-default"> + Select Covariates + </button> + <button type="button" id="remove_covariates" class="btn btn-default"> + Remove Covariates + </button> + </div> + </div> + + <div class="form-group"> + <div class="col-xs-4 controls"> + <button id="gemma_bimbam_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression"> + Compute + </button> + </div> + </div> + </div> + <div class="tab-pane" id="interval_mapping"> <div style="margin-top: 20px" class="form-horizontal"> {% if genofiles and genofiles|length>0 %} <div class="mapping_method_fields form-group"> @@ -131,62 +184,6 @@ <!--<div id="alert_placeholder"></div>--> </div> </div> - <div class="tab-pane" id="pylmm"> - <div style="margin-top: 20px" class="form-horizontal"> - {% if genofiles and genofiles|length>0 %} - <div class="mapping_method_fields form-group"> - <label for="genofiles" class="col-xs-3 control-label">Genotypes</label> - <div style="margin-left: 20px;" class="col-xs-8 controls"> - <select id="genofile_pylmm" class="form-control"> - {% for item in genofiles %} - <option value="{{item['location']}}:{{item['title']}}">{{item['title']}}</option> - {% endfor %} - </select> - </div> - </div> - {% endif %} -<!-- - <div class="mapping_method_fields form-group"> - <label for="control_for" class="col-xs-3 control-label">Control for</label> - <div style="margin-left: 20px;" class="col-xs-4 controls"> - {% if dataset.type == 'ProbeSet' and this_trait.locus_chr != "" %} - <input name="control_pylmm" value="{{ nearest_marker }}" type="text" /> - {% else %} - <input name="control_pylmm" value="" type="text" /> - {% endif %} - <label class="radio-inline"> - <input type="radio" name="do_control_pylmm" value="true"> - Yes - </label> - <label class="radio-inline"> - <input type="radio" name="do_control_pylmm" value="false" checked=""> - No - </label> - </div> - </div> - <div class="mapping_method_fields form-group"> - <label style="text-align:left;" class="col-xs-12 control-label">Manhattan Plot</label> - <div class="col-xs-12 controls"> - <label class="radio-inline"> - <input type="radio" name="manhattan_plot_pylmm" value="True"> - Yes - </label> - <label class="radio-inline"> - <input type="radio" name="manhattan_plot_pylmm" value="False" checked=""> - No - </label> - </div> - </div> ---> - <div class="form-group"> - <div style="padding-left:15px;" class="controls"> - <button id="pylmm_compute" class="btn submit_special btn-success" title="Compute Marker Regression"> - <i class="icon-ok-circle icon-white"></i> Compute - </button> - </div> - </div> - </div> - </div> <div class="tab-pane" id="rqtl_geno"> <div style="margin-top: 20px" class="form-horizontal"> @@ -294,13 +291,13 @@ </div> </div> </div> - <div class="tab-pane" id="gemma"> - <div style="padding-top: 10px;" class="form-horizontal"> + <div class="tab-pane" id="pylmm"> + <div style="margin-top: 20px" class="form-horizontal"> {% if genofiles and genofiles|length>0 %} <div class="mapping_method_fields form-group"> <label for="genofiles" class="col-xs-3 control-label">Genotypes</label> <div style="margin-left: 20px;" class="col-xs-8 controls"> - <select id="genofile_gemma" class="form-control"> + <select id="genofile_pylmm" class="form-control"> {% for item in genofiles %} <option value="{{item['location']}}:{{item['title']}}">{{item['title']}}</option> {% endfor %} @@ -308,42 +305,45 @@ </div> </div> {% endif %} +<!-- <div class="mapping_method_fields form-group"> - <label for="maf_gemma" class="col-xs-5 control-label">Minor allele threshold</label> - <div style="margin-left: 20px;" class="col-xs-3 controls"> - <input name="maf_gemma" value="0.01" type="text" class="form-control"> + <label for="control_for" class="col-xs-3 control-label">Control for</label> + <div style="margin-left: 20px;" class="col-xs-4 controls"> + {% if dataset.type == 'ProbeSet' and this_trait.locus_chr != "" %} + <input name="control_pylmm" value="{{ nearest_marker }}" type="text" /> + {% else %} + <input name="control_pylmm" value="" type="text" /> + {% endif %} + <label class="radio-inline"> + <input type="radio" name="do_control_pylmm" value="true"> + Yes + </label> + <label class="radio-inline"> + <input type="radio" name="do_control_pylmm" value="false" checked=""> + No + </label> </div> </div> <div class="mapping_method_fields form-group"> - <label class="col-xs-4 control-label">Use LOCO</label> - <div style="margin-left: 20px;" class="col-xs-4 controls"> + <label style="text-align:left;" class="col-xs-12 control-label">Manhattan Plot</label> + <div class="col-xs-12 controls"> <label class="radio-inline"> - <input type="radio" name="use_loco" value="True"> + <input type="radio" name="manhattan_plot_pylmm" value="True"> Yes </label> <label class="radio-inline"> - <input type="radio" name="use_loco" value="False" checked=""> + <input type="radio" name="manhattan_plot_pylmm" value="False" checked=""> No </label> </div> </div> - </div> - <div style="padding-top: 5px; padding-bottom: 5px; padding-left: 20px;" class="form-horizontal"> - <div class="mapping_method_fields form-group"> - <button type="button" id="select_covariates" class="btn btn-default"> - Select Covariates - </button> - <button type="button" id="remove_covariates" class="btn btn-default"> - Remove Covariates - </button> - </div> - </div> - - <div class="form-group"> - <div class="col-xs-4 controls"> - <button id="gemma_bimbam_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression"> - Compute - </button> +--> + <div class="form-group"> + <div style="padding-left:15px;" class="controls"> + <button id="pylmm_compute" class="btn submit_special btn-success" title="Compute Marker Regression"> + <i class="icon-ok-circle icon-white"></i> Compute + </button> + </div> </div> </div> </div> diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 68e779a1..4e81c29c 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -557,6 +557,7 @@ def marker_regression_page(): wanted = ( 'trait_id', 'dataset', + 'geno_db_exists', 'method', 'mapping_results_path', 'trimmed_markers', @@ -765,7 +766,7 @@ def corr_scatter_plot_page(): def submit_bnw(): logger.error(request.url) template_vars = get_bnw_input(request.form) - return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) + return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) # Todo: Can we simplify this? -Sam def sharing_info_page(): diff --git a/wqflask/wsgi.py b/wqflask/wsgi.py new file mode 100644 index 00000000..be9c7b37 --- /dev/null +++ b/wqflask/wsgi.py @@ -0,0 +1,4 @@ +from run_gunicorn import app as application # expect application as a name + +if __name__ == "__main__": + application.run() |