From 847f911257e66be454f16443ac53c107064eb6aa Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:10:30 +0000 Subject: Docs --- doc/GUIX-Reproducible-from-source.org | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index b88eb9e8..871156ed 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -31,5 +31,12 @@ Next build guix (and run) following the instructions in [[https://github.com/pjo Once that is done we can add the guix-bioinformatics path with -: env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A slurm +: env GUIX_PACKAGE_PATH=../guix-bioinformatics command +such as + +#+begin_src sh :lang bash +gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A genenetwork +genenetwork1 1.0-d622c803b out ../guix-bioinformatics/gn/packages/bioinformatics.scm:163:2 +genenetwork2 2.0-9e9475053 out ../guix-bioinformatics/gn/packages/bioinformatics.scm:215:2 +#+end_src sh :lang bash -- cgit v1.2.3 From 94459e2cb31cbbd2f3f58c74e26dd762e981f7da Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:13:04 +0000 Subject: [PATCH 010/100] README --- doc/README.org | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/doc/README.org b/doc/README.org index f6ab6a52..9206cd35 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,7 +3,6 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - [[#binary-deployment][Binary deployment]] - - [[#from-source-deployment][From source deployment]] * Introduction @@ -15,14 +14,3 @@ explain the GeneNetwork deployment system which is based on GNU Guix NYA -* From source deployment - -GNU Guix allows for [[https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org][reproducible deployment]] based on a checked out -Guix repository - use gn-stable for that: - -#+begin_src sh :lang bash -mkdir genenetwork -cd genenetwork -git checkout https://github.com/genenetwork/guix.git gn-stable-guix -git checkout https://github.com/genenetwork/guix-bioinformatics.git -#+end_src -- cgit v1.2.3 From c9a48705184c02f0b74b01bae1d0ed8d9328550a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:14:04 +0000 Subject: [PATCH 012/100] Adding configuration for running a default server --- etc/default_settings.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 etc/default_settings.py diff --git a/etc/default_settings.py b/etc/default_settings.py new file mode 100644 index 00000000..7aa24fce --- /dev/null +++ b/etc/default_settings.py @@ -0,0 +1,25 @@ +LOGFILE = "/var/log/genenetwork/wqflask.log" + +# This is needed because Flask turns key errors into a +# 400 bad request response with no exception/log +TRAP_BAD_REQUEST_ERRORS = True + +DB_URI = "mysql://gn2:default@localhost/db_webqtl" +SQLALCHEMY_DATABASE_URI = 'mysql://gn2:default@localhost/db_webqtl' + +# http://pythonhosted.org/Flask-Security/configuration.html +SECURITY_CONFIRMABLE = True +SECURITY_TRACKABLE = True +SECURITY_REGISTERABLE = True +SECURITY_RECOVERABLE = True + +SECURITY_EMAIL_SENDER = "no-reply@genenetwork.org" +SECURITY_POST_LOGIN_VIEW = "/thank_you" +SQLALCHEMY_POOL_RECYCLE = 3600 + +SERVER_PORT = 5003 +SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' + +# Path overrides for Genenetwork +# PYLMM_PATH = 'UNUSED' + -- cgit v1.2.3 From e1cdb11c211394472e2f8cdd033fd0f93dbf5cea Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sat, 13 Feb 2016 11:22:02 +0100 Subject: Docs: Use ./doc dir instead of ./misc --- INSTALL.md | 2 +- doc/gn_installation_notes.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index b8c9c977..3ef2d5b3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -86,4 +86,4 @@ GN2 can locate PYLMM through PYLMM_PATH in setting.py (or in ENV) ## Other information -Check also the ./misc/ directory for settings +Check also the ./doc directory for settings diff --git a/doc/gn_installation_notes.txt b/doc/gn_installation_notes.txt index 584080f7..efea0309 100644 --- a/doc/gn_installation_notes.txt +++ b/doc/gn_installation_notes.txt @@ -96,7 +96,7 @@ Before installing from requirements.txt, install numpy separately: pip install numpy==1.7.0 (or whatever version we're using) Install from requirements.txt (after activating virtualenv): -pip install -r gene/misc/requirements.txt +pip install -r gene/doc/requirements.txt =========================================== @@ -343,4 +343,4 @@ python runserver.py To do full upgrade (as opposed to apt-get upgrade) sudo aptitude full-upgrade -=========================================== \ No newline at end of file +=========================================== -- cgit v1.2.3 From 82f731d11a870262b5d4ed693b86d7dcc40493a1 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sat, 13 Feb 2016 12:21:07 +0100 Subject: * ./bin/genenetwork2: New file --- bin/genenetwork2 | 15 +++++++++++++++ etc/default_settings.py | 1 - 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100755 bin/genenetwork2 diff --git a/bin/genenetwork2 b/bin/genenetwork2 new file mode 100755 index 00000000..9cac31d5 --- /dev/null +++ b/bin/genenetwork2 @@ -0,0 +1,15 @@ +#! /bin/bash +# +# This will run the server with default settings + +# Absolute path to this script, e.g. /home/user/bin/foo.sh +SCRIPT=$(readlink -f "$0") +# Absolute path this script is in, thus /home/user/bin +SCRIPTPATH=$(dirname $(dirname "$SCRIPT")) +echo $SCRIPTPATH + +export PROFILE=$SCRIPTPATH +export PYTHONPATH=$PROFILE/wqflask:$PYTHONPATH +export WQFLASK_SETTINGS=$PROFILE/etc/default_settings.py + +/usr/bin/env python ./wqflask/runserver.py diff --git a/etc/default_settings.py b/etc/default_settings.py index 7aa24fce..0c7d7bad 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -12,7 +12,6 @@ SECURITY_CONFIRMABLE = True SECURITY_TRACKABLE = True SECURITY_REGISTERABLE = True SECURITY_RECOVERABLE = True - SECURITY_EMAIL_SENDER = "no-reply@genenetwork.org" SECURITY_POST_LOGIN_VIEW = "/thank_you" SQLALCHEMY_POOL_RECYCLE = 3600 -- cgit v1.2.3 From b0ae93454b6c7ef0ed40a9df36d4e30d19eb1fc9 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sat, 13 Feb 2016 12:23:09 +0100 Subject: Removed import htmlgen as is not used (apparently) --- wqflask/wqflask/search_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index a57bfffe..52fe2e34 100755 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -24,7 +24,7 @@ from flask import Flask, g from MySQLdb import escape_string as escape # Instead of importing HT we're going to build a class below until we can eliminate it -from htmlgen import HTMLgen2 as HT +# from htmlgen import HTMLgen2 as HT from base import webqtlConfig from utility.benchmark import Bench -- cgit v1.2.3 From 0ddefeab559746b6ab89a20e07edc1c04ee76e28 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sat, 13 Feb 2016 16:15:38 +0100 Subject: * bin/genenetwork2: path handling --- bin/genenetwork2 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 9cac31d5..d0177045 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -5,11 +5,11 @@ # Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") # Absolute path this script is in, thus /home/user/bin -SCRIPTPATH=$(dirname $(dirname "$SCRIPT")) -echo $SCRIPTPATH +GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) +echo $GN2_BASE_PATH -export PROFILE=$SCRIPTPATH -export PYTHONPATH=$PROFILE/wqflask:$PYTHONPATH -export WQFLASK_SETTINGS=$PROFILE/etc/default_settings.py +export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH +export WQFLASK_SETTINGS=$GN2_BASE_PATH/etc/default_settings.py -/usr/bin/env python ./wqflask/runserver.py +cd $GN2_BASE_PATH/wqflask +/usr/bin/env python runserver.py -- cgit v1.2.3 From 73ef712f3fd02df7ffa84a2b35ee5648f689d31b Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sun, 14 Feb 2016 10:33:05 +0100 Subject: README: add URL --- doc/README.org | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/README.org b/doc/README.org index 9206cd35..8e0823de 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,7 +2,8 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#binary-deployment][Binary deployment]] + - [[#binary-deployment-wip][Binary deployment (WIP)]] + - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -10,7 +11,12 @@ Large system deployments tend to get very complex. In this document we explain the GeneNetwork deployment system which is based on GNU Guix (see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). -* Binary deployment +* Binary deployment (WIP) -NYA +GN can be deployed either as a binary tarball or as a GNU Guix +package. + +* Source deployment and other information on reproducibility + +See the document [[GUIX-Reproducible-from-source.org]]. -- cgit v1.2.3 From beb6a9360ecc9edca19692e4081efc15292fb7d1 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:28:44 +0000 Subject: [PATCH 018/100] Find external tools: refactored code to work with GNU Guix --- etc/default_settings.py | 8 +- wqflask/utility/tools.py | 92 ++++++++++------------ .../wqflask/marker_regression/marker_regression.py | 8 +- 3 files changed, 56 insertions(+), 52 deletions(-) diff --git a/etc/default_settings.py b/etc/default_settings.py index 0c7d7bad..929bd687 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -1,3 +1,5 @@ +import os + LOGFILE = "/var/log/genenetwork/wqflask.log" # This is needed because Flask turns key errors into a @@ -20,5 +22,7 @@ SERVER_PORT = 5003 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' # Path overrides for Genenetwork -# PYLMM_PATH = 'UNUSED' - +HOME=os.environ.get('HOME') +PYLMM_PATH = HOME+"/izip/git/opensource/python/pylmm_gn2/" +PLINK_PATH = HOME+"/.guix-profile/bin" +GEMMA_PATH = HOME+"/.guix-profile/bin" diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index b8a41f60..0db195df 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -9,76 +9,70 @@ import os import sys from wqflask import app -def get_setting(id,default,guess,get_valid_path): - """ - Resolve a setting from the environment or the global settings in app.config +def get_setting(id,default,guess,find_path): + """Resolve a setting from the environment or the global settings in + app.config, with get_valid_path is a function checking whether the + path points to an expected directory an returns the full path e.g. + + guess = os.environ.get('HOME')+'/pylmm' + get_setting('PYLMM_PATH',default,guess,get_valid_path) + + first tries the environment variable in +id+, next gets the Flask + app setting for the same +id+, next tries the path passed in with + +default+ and finally does an educated +guess+. + + In all, the environment overrides the others, next is the flask + setting, then the default and finally the guess (which is + $HOME/repo). A valid path is returned. If none is resolved an + exception is thrown. + + Note that we do not use the system path. This is on purpose + because it will mess up controlled (reproducible) deployment. The + proper way is to either use the GNU Guix defaults as listed in + etc/default_settings.py or override them yourself by creating a + different settings.py file (or setting the environment). + """ # ---- Check whether environment exists - path = get_valid_path(os.environ.get(id)) + path = find_path(os.environ.get(id)) # ---- Check whether setting exists setting = app.config.get(id) if not path: - path = get_valid_path(setting) + path = find_path(setting) # ---- Check whether default exists if not path: - path = get_valid_path(default) + path = find_path(default) # ---- Guess directory if not path: + guess = os.environ.get('HOME')+guess if not setting: setting = guess - path = get_valid_path(guess) + path = find_path(guess) if not path: - raise Exception(id+' '+setting+' path unknown or faulty (update settings.py?). '+id+' should point to the root of the git repository') - + raise Exception(id+' '+setting+' path unknown or faulty (update settings.py?). '+id+' should point to the path') return path -def pylmm_command(default=None): - """ - Return the path to the repository and the python command to call - """ - def get_valid_path(path): +def find_command(command,id1,default,guess): + def find_path(path): """Test for a valid repository""" if path: - sys.stderr.write("Trying PYLMM_PATH in "+path+"\n") - if path and os.path.isfile(path+'/pylmm_gn2/lmm.py'): + sys.stderr.write("Trying "+id1+" in "+path+"\n") + binary = str.split(command)[0] + if path and os.path.isfile(path+'/'+binary): return path else: None - guess = os.environ.get('HOME')+'/pylmm_gn2' - path = get_setting('PYLMM_PATH',default,guess,get_valid_path) - pylmm_command = 'python '+path+'/pylmm_gn2/lmm.py' - return path,pylmm_command - -def plink_command(default=None): - """ - Return the path to the repository and the python command to call - """ - def get_valid_path(path): - """Test for a valid repository""" - if path: - sys.stderr.write("Trying PLINK_PATH in "+path+"\n") - if path and os.path.isfile(path+'/plink'): - return path - else: - None + path = get_setting(id1,default,guess,find_path) + binary = path+'/'+command + sys.stderr.write("Found "+binary+"\n") + return path,binary - guess = os.environ.get('HOME')+'/plink_gemma' - path = get_setting('PLINK_PATH',default,guess,get_valid_path) - plink_command = path+'/plink' - return path,plink_command +def pylmm_command(default=None): + return find_command('pylmm_gn2/lmm.py',"PYLMM_PATH",default,'/pylmm2') def gemma_command(default=None): - def get_valid_path(path): - """Test for a valid repository""" - if path: - sys.stderr.write("Trying PLINK_PATH in "+path+"\n") - if path and os.path.isfile(path+'/plink'): - return path - else: - None + return find_command('gemma',"GEMMA_PATH",default,'/gemma') - guess = os.environ.get('HOME')+'/plink' - path = get_setting('PLINK_PATH',default,guess,get_valid_path) - gemma_command = path+'/gemma' - return path, gemma_command \ No newline at end of file +def plink_command(default=None): + return find_command('plink2',"PLINK_PATH",default,'/plink') diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index af320f65..a657510d 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -42,9 +42,15 @@ from wqflask.marker_regression import gemma_mapping #from wqflask.marker_regression import plink_mapping #from wqflask.marker_regression import rqtl_mapping +# Check for valid binary paths of pylmm, plink, rqtl etc. This code +# runs at startup, so a missing binary will balk before running the +# service + +GEMMA_PATH,GEMMA_COMMAND = gemma_command() PYLMM_PATH,PYLMM_COMMAND = pylmm_command() PLINK_PATH,PLINK_COMMAND = plink_command() -GEMMA_PATH,GEMMA_COMMAND = gemma_command() +# RQTL_PATH,RQTL_COMMAND = rqtl_command() + class MarkerRegression(object): -- cgit v1.2.3 From f1a9ce85521180b9204d14203677a92f944e2859 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 16 Feb 2016 10:29:47 +0000 Subject: Remover dependency to logger_tree (may reintroduce later) --- wqflask/runserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 59ebf0d4..e4392b3f 100755 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -25,8 +25,8 @@ file_handler = logging.FileHandler(app.config['LOGFILE']) file_handler.setLevel(logging.DEBUG) app.logger.addHandler(file_handler) -import logging_tree -logging_tree.printout() +# import logging_tree +# logging_tree.printout() app.run(host='0.0.0.0', port=app.config['SERVER_PORT'], -- cgit v1.2.3 From c76299fa981add7b8cf56126c8d7ffeb9a6c4034 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 16 Feb 2016 10:30:27 +0000 Subject: Successful startup of GN2 with MySQL --- bin/genenetwork2 | 8 +++++++- doc/README.org | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index d0177045..a48e8b93 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -8,8 +8,14 @@ SCRIPT=$(readlink -f "$0") GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) echo $GN2_BASE_PATH +settings=$1 +if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH -export WQFLASK_SETTINGS=$GN2_BASE_PATH/etc/default_settings.py +export WQFLASK_SETTINGS=$settings + +# start the redis server +redis-server & cd $GN2_BASE_PATH/wqflask +echo "Starting with $settings" /usr/bin/env python runserver.py diff --git a/doc/README.org b/doc/README.org index 8e0823de..375a5d20 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,6 +3,8 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - [[#binary-deployment-wip][Binary deployment (WIP)]] + - [[#run-mysql-server][Run MySQL server]] + - [[#start-the-gn2-server][Start the GN2 server]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -16,6 +18,53 @@ explain the GeneNetwork deployment system which is based on GNU Guix GN can be deployed either as a binary tarball or as a GNU Guix package. +** Run MySQL server + +At this point we require the underlying distribution to install +and run mysqld. + +Download one of + +http://files.genenetwork.org/raw_database/ +https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip + +Check the md5sum. + +After installation inflate the database binary in the MySQL directory +(this is subject to change soon) + +: chown -R mysql:mysql db_webqtl_s/ +: chmod 700 db_webqtl_s/ +: chmod 660 db_webqtl_s/* + +restart MySQL service (mysqld). Login as root and + +: mysql> show databases; +: +--------------------+ +: | Database | +: +--------------------+ +: | information_schema | +: | db_webqtl_s | +: | mysql | +: | performance_schema | +: +--------------------+ + +Set permissions and match password in your settings file below: + +: mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; + +** Start the GN2 server + +Copy the default_settings.py file and modify + +: DB_URI = """mysql://gn2:mysql_password@localhost/db_webqtl""" +: SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password/db_webqtl' +: SECRET_KEY = 'secret_key' + +Start the server with your settings file + +: ./bin/genenetwork2 ~/my_gn2_settings.py + * Source deployment and other information on reproducibility See the document [[GUIX-Reproducible-from-source.org]]. -- cgit v1.2.3 From af7d0bca229f3ebaa80a16d1ce3a2bf1a8abd5df Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:37:55 +0000 Subject: [PATCH 023/100] WIP fixing all paths --- README.md | 2 + doc/GUIX-Reproducible-from-source.org | 14 +++- doc/README.org | 23 +++++- etc/default_settings.py | 15 ++-- wqflask/base/data_set.py | 3 +- wqflask/utility/tools.py | 82 ++++++++++------------ wqflask/wqflask/database.py | 5 +- .../wqflask/marker_regression/marker_regression.py | 6 +- wqflask/wqflask/show_trait/show_trait.py | 10 +-- 9 files changed, 94 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 3d95d05f..db495910 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ Elissa Chesler, Jintao Wang, Kenneth Manly, Robert W. Williams, and colleagues. Code and primary web service managed by Dr. Robert W. Williams and the University of Tennessee Health Science Center, Memphis TN, USA. +Join #genenetwork on irc.freenode.net. + Email labwilliams@gmail.com or rwilliams@uthsc.edu Older version available on SourceForge http://sourceforge.net/projects/genenetwork/ diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index 871156ed..4399ea26 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -4,6 +4,7 @@ - [[#introduction][Introduction]] - [[#binary-deployment][Binary deployment]] - [[#from-source-deployment][From source deployment]] + - [[#create-archive][Create archive]] * Introduction @@ -33,10 +34,21 @@ Once that is done we can add the guix-bioinformatics path with : env GUIX_PACKAGE_PATH=../guix-bioinformatics command -such as +So +#+begin_src sh :lang bash #+begin_src sh :lang bash gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A genenetwork genenetwork1 1.0-d622c803b out ../guix-bioinformatics/gn/packages/bioinformatics.scm:163:2 genenetwork2 2.0-9e9475053 out ../guix-bioinformatics/gn/packages/bioinformatics.scm:215:2 #+end_src sh :lang bash + +Install with + +#+begin_src sh :lang bash +gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -i genenetwork2 +#+end_src sh :lang bash + +* Create archive + +: env GUIX_PACKAGE_PATH=../../genenetwork/guix-bioinformatics/ ./pre-inst-env guix archive --export -r genenetwork2 > guix_gn2-2.0-9e9475053.nar diff --git a/doc/README.org b/doc/README.org index 375a5d20..d9730948 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,6 +3,7 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - [[#binary-deployment-wip][Binary deployment (WIP)]] + - [[#install-genenetwork-server][Install genenetwork server]] - [[#run-mysql-server][Run MySQL server]] - [[#start-the-gn2-server][Start the GN2 server]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] @@ -16,7 +17,27 @@ explain the GeneNetwork deployment system which is based on GNU Guix * Binary deployment (WIP) GN can be deployed either as a binary tarball or as a GNU Guix -package. +package. First install GNU Guix following the instructions of the +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]] and, update guix with a +'guix pull' and make guix visible in the path. More information +exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. + +With guix running you should be able to install python, for example. + +: guix package -i python2 + +This will make python appear in $HOME/.guix-profile/bin/python. Suggested +environment settings can be seen with + +: guix package --search-paths + +** Install genenetwork server + +Fetch the nar file and install it after adding the key with, for +example + +: cat signing-key.pub |.guix-profile/bin/guix archive --authorize +: guix archive --import < guix_gn2-2.0-9e9475053.nar ** Run MySQL server diff --git a/etc/default_settings.py b/etc/default_settings.py index 929bd687..48d3f66b 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -1,13 +1,14 @@ import os +import sys -LOGFILE = "/var/log/genenetwork/wqflask.log" +LOGFILE = "/tmp/genenetwork2.log" # This is needed because Flask turns key errors into a # 400 bad request response with no exception/log TRAP_BAD_REQUEST_ERRORS = True -DB_URI = "mysql://gn2:default@localhost/db_webqtl" -SQLALCHEMY_DATABASE_URI = 'mysql://gn2:default@localhost/db_webqtl' +DB_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" +SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password@localhost/db_webqtl_s' # http://pythonhosted.org/Flask-Security/configuration.html SECURITY_CONFIRMABLE = True @@ -22,7 +23,7 @@ SERVER_PORT = 5003 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' # Path overrides for Genenetwork -HOME=os.environ.get('HOME') -PYLMM_PATH = HOME+"/izip/git/opensource/python/pylmm_gn2/" -PLINK_PATH = HOME+"/.guix-profile/bin" -GEMMA_PATH = HOME+"/.guix-profile/bin" +GENENETWORK_FILES = "../../gn2_data" +PYLMM_RUN = os.popen("which pylmm_redis").read() +PLINK_RUN = os.popen("which plink2").read() +GEMMA_RUN = os.popen("which gemma").read() diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index e37a838f..0e5a3ac1 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -405,10 +405,11 @@ class DatasetGroup(object): #print("Cache not hit") from utility.tools import plink_command - PLINK_PATH,PLINK_COMMAND = plink_command() + PLINK_RUN = plink_command() geno_file_path = webqtlConfig.GENODIR+self.name+".geno" plink_file_path = PLINK_PATH+"/"+self.name+".fam" + # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES if os.path.isfile(plink_file_path): self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 0db195df..c3c9b292 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -9,22 +9,22 @@ import os import sys from wqflask import app -def get_setting(id,default,guess,find_path): +def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with get_valid_path is a function checking whether the - path points to an expected directory an returns the full path e.g. + path points to an expected directory and returns the full path to + the binary command guess = os.environ.get('HOME')+'/pylmm' - get_setting('PYLMM_PATH',default,guess,get_valid_path) + get_setting('PYLMM_PATH',guess) first tries the environment variable in +id+, next gets the Flask - app setting for the same +id+, next tries the path passed in with - +default+ and finally does an educated +guess+. + app setting for the same +id+ and finally does an educated + +guess+. In all, the environment overrides the others, next is the flask - setting, then the default and finally the guess (which is - $HOME/repo). A valid path is returned. If none is resolved an - exception is thrown. + setting, then the guess. A valid path to the binary command is + returned. If none is resolved an exception is thrown. Note that we do not use the system path. This is on purpose because it will mess up controlled (reproducible) deployment. The @@ -33,46 +33,36 @@ def get_setting(id,default,guess,find_path): different settings.py file (or setting the environment). """ - # ---- Check whether environment exists - path = find_path(os.environ.get(id)) - # ---- Check whether setting exists - setting = app.config.get(id) - if not path: - path = find_path(setting) - # ---- Check whether default exists - if not path: - path = find_path(default) - # ---- Guess directory - if not path: - guess = os.environ.get('HOME')+guess - if not setting: - setting = guess - path = find_path(guess) - if not path: - raise Exception(id+' '+setting+' path unknown or faulty (update settings.py?). '+id+' should point to the path') - return path - -def find_command(command,id1,default,guess): - def find_path(path): - """Test for a valid repository""" - if path: - sys.stderr.write("Trying "+id1+" in "+path+"\n") - binary = str.split(command)[0] - if path and os.path.isfile(path+'/'+binary): - return path + def valid(command): + if command: + sys.stderr.write("Found value "+command+"\n") + return command else: - None + return None + + # ---- Check whether environment exists + sys.stderr.write("Looking for "+command_id+"\n") + command = valid(os.environ.get(command_id)) + if not command: + # ---- Check whether setting exists in app + command = valid(app.config.get(command_id)) + if not command: + command = valid(guess) + if not command: + raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path') + return command - path = get_setting(id1,default,guess,find_path) - binary = path+'/'+command - sys.stderr.write("Found "+binary+"\n") - return path,binary +def pylmm_command(guess=None): + return get_setting("PYLMM_RUN",guess) -def pylmm_command(default=None): - return find_command('pylmm_gn2/lmm.py',"PYLMM_PATH",default,'/pylmm2') +def gemma_command(guess=None): + return get_setting("GEMMA_RUN",guess) -def gemma_command(default=None): - return find_command('gemma',"GEMMA_PATH",default,'/gemma') +def plink_command(guess=None): + return get_setting("PLINK_RUN",guess) -def plink_command(default=None): - return find_command('plink2',"PLINK_PATH",default,'/plink') +def flat_files(subdir=None): + base = get_setting("GENENETWORK_FILES") + if subdir: + return base+"/"+subdir + return base diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 159c5d6c..2f544d44 100755 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -24,8 +24,9 @@ def init_db(): # you will have to import them first before calling init_db() #import yourapplication.models import wqflask.model - print("Creating all..") + print("database.py: Creating all model metadata..") Base.metadata.create_all(bind=engine) - print("Done creating all...") + print("database.py: Done creating all model metadata...") + print("Point your browser at http://localhost:5003/") init_db() diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index a657510d..b0f5ed69 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -46,9 +46,9 @@ from wqflask.marker_regression import gemma_mapping # runs at startup, so a missing binary will balk before running the # service -GEMMA_PATH,GEMMA_COMMAND = gemma_command() -PYLMM_PATH,PYLMM_COMMAND = pylmm_command() -PLINK_PATH,PLINK_COMMAND = plink_command() +GEMMA_RUN = gemma_command() +PYLMM_RUN = pylmm_command() +PLINK_RUN = plink_command() # RQTL_PATH,RQTL_COMMAND = rqtl_command() diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 2d4c952a..458e48da 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -16,7 +16,8 @@ from base import webqtlConfig from base import webqtlCaseData from wqflask.show_trait.SampleList import SampleList from utility import webqtlUtil, Plot, Bunch, helper_functions -from utility.tools import pylmm_command, plink_command +# from utility.tools import plink_command +from utility.tools import flat_files from base.trait import GeneralTrait from base import data_set from dbFunction import webqtlDatabaseFunction @@ -24,8 +25,7 @@ from basicStatistics import BasicStatisticsFunctions from pprint import pformat as pf -PYLMM_PATH,PYLMM_COMMAND = pylmm_command() -PLINK_PATH,PLINK_COMMAND = plink_command() +MAPPING_FILES = flat_files("mapping") ############################################### # @@ -162,8 +162,8 @@ class ShowTrait(object): def get_mapping_methods(self): '''Only display mapping methods when the dataset group's genotype file exists''' def check_plink_gemma(): - if (os.path.isfile(PLINK_PATH+"/"+self.dataset.group.name+".bed") and - os.path.isfile(PLINK_PATH+"/"+self.dataset.group.name+".map")): + if (os.path.isfile(MAPPYING_FILES+"/"+self.dataset.group.name+".bed") and + os.path.isfile(MAPPING_FILES+"/"+self.dataset.group.name+".map")): return True else: return False -- cgit v1.2.3 From 4b80bbfe261e8d0587a637b35416834e027f0999 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:46:23 +0000 Subject: [PATCH 024/100] Sanitizing file handling --- wqflask/base/data_set.py | 29 ++++++++++------------------- wqflask/utility/tools.py | 20 +++++++++++++++----- wqflask/wqflask/show_trait/show_trait.py | 11 ++++------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 0e5a3ac1..c6f34143 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,12 +44,15 @@ from dbFunction import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks +from utility.tools import flat_files from maintenance import get_group_samplelists from MySQLdb import escape_string as escape from pprint import pformat as pf +MAPPING_PATH = flat_files("mapping") + # Used by create_database to instantiate objects # Each subclass will add to this DS_NAME_MAP = {} @@ -404,15 +407,11 @@ class DatasetGroup(object): else: #print("Cache not hit") - from utility.tools import plink_command - PLINK_RUN = plink_command() - geno_file_path = webqtlConfig.GENODIR+self.name+".geno" - plink_file_path = PLINK_PATH+"/"+self.name+".fam" - # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES - if os.path.isfile(plink_file_path): - self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path) + mapping_file_path = MAPPING_PATH+"/"+self.name+".fam" + if os.path.isfile(mapping_file_path): + self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path) elif os.path.isfile(geno_file_path): self.samplelist = get_group_samplelists.get_samplelist("geno", geno_file_path) else: @@ -441,18 +440,10 @@ class DatasetGroup(object): # reaper barfs on unicode filenames, so here we ensure it's a string full_filename = str(os.path.join(webqtlConfig.GENODIR, self.name + '.geno')) - if os.path.isfile(full_filename): - #print("Reading file: ", full_filename) - genotype_1.read(full_filename) - #print("File read") - else: - try: - full_filename = str(os.path.join(webqtlConfig.TMPDIR, self.name + '.geno')) - #print("Reading file") - genotype_1.read(full_filename) - #print("File read") - except IOError: - print("File doesn't exist!") + if not os.path.isfile(full_filename): + raise SystemError("File "+full_filename+" does not exist") + print("Reading file: ", full_filename) + genotype_1.read(full_filename) if genotype_1.type == "group" and self.parlist: genotype_2 = genotype_1.add(Mat=self.parlist[0], Pat=self.parlist[1]) #, F1=_f1) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index c3c9b292..44bbb1b0 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -52,17 +52,27 @@ def get_setting(command_id,guess=None): raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path') return command +def valid_bin(bin): + if os.path.islink(bin) or os.path.isfile(bin): + return bin + return None + +def valid_path(dir): + if os.path.isdir(dir): + return dir + return None + def pylmm_command(guess=None): - return get_setting("PYLMM_RUN",guess) + return valid_bin(get_setting("PYLMM_RUN",guess)) def gemma_command(guess=None): - return get_setting("GEMMA_RUN",guess) + return valid_bin(get_setting("GEMMA_RUN",guess)) def plink_command(guess=None): - return get_setting("PLINK_RUN",guess) + return valid_bin(get_setting("PLINK_RUN",guess)) def flat_files(subdir=None): base = get_setting("GENENETWORK_FILES") if subdir: - return base+"/"+subdir - return base + return valid_path(base+"/"+subdir) + return valid_path(base) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 458e48da..35f7fe5f 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -16,8 +16,6 @@ from base import webqtlConfig from base import webqtlCaseData from wqflask.show_trait.SampleList import SampleList from utility import webqtlUtil, Plot, Bunch, helper_functions -# from utility.tools import plink_command -from utility.tools import flat_files from base.trait import GeneralTrait from base import data_set from dbFunction import webqtlDatabaseFunction @@ -25,7 +23,8 @@ from basicStatistics import BasicStatisticsFunctions from pprint import pformat as pf -MAPPING_FILES = flat_files("mapping") +from utility.tools import flat_files +MAPPING_PATH = flat_files("mapping") ############################################### # @@ -34,8 +33,6 @@ MAPPING_FILES = flat_files("mapping") # ############################################## - - class ShowTrait(object): def __init__(self, kw): @@ -162,8 +159,8 @@ class ShowTrait(object): def get_mapping_methods(self): '''Only display mapping methods when the dataset group's genotype file exists''' def check_plink_gemma(): - if (os.path.isfile(MAPPYING_FILES+"/"+self.dataset.group.name+".bed") and - os.path.isfile(MAPPING_FILES+"/"+self.dataset.group.name+".map")): + if (os.path.isfile(MAPPYING_PATH+"/"+self.dataset.group.name+".bed") and + os.path.isfile(MAPPING_PATH+"/"+self.dataset.group.name+".map")): return True else: return False -- cgit v1.2.3 From 94e3ac8e2fff5da72fd1c0358fdb6fdad9a7830c Mon Sep 17 00:00:00 2001 From: pjotrp Date: Mon, 22 Feb 2016 16:26:13 +0000 Subject: Introducing TEMPDIR --- wqflask/utility/tools.py | 14 ++++++++++---- wqflask/wqflask/views.py | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 44bbb1b0..dc7bc33c 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -9,6 +9,7 @@ import os import sys from wqflask import app + def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with get_valid_path is a function checking whether the @@ -33,7 +34,7 @@ def get_setting(command_id,guess=None): different settings.py file (or setting the environment). """ - def valid(command): + def value(command): if command: sys.stderr.write("Found value "+command+"\n") return command @@ -42,12 +43,12 @@ def get_setting(command_id,guess=None): # ---- Check whether environment exists sys.stderr.write("Looking for "+command_id+"\n") - command = valid(os.environ.get(command_id)) + command = value(os.environ.get(command_id)) if not command: # ---- Check whether setting exists in app - command = valid(app.config.get(command_id)) + command = value(app.config.get(command_id)) if not command: - command = valid(guess) + command = value(guess) if not command: raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path') return command @@ -76,3 +77,8 @@ def flat_files(subdir=None): if subdir: return valid_path(base+"/"+subdir) return valid_path(base) + +def tempdir(): + return valid_path(get_setting("TEMPDIR","/tmp")) + +TEMPDIR = tempdir() diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3bf64a18..929da649 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -50,6 +50,7 @@ from wqflask.correlation import corr_scatter_plot from wqflask.wgcna import wgcna_analysis from utility import temp_data +from utility.tools import TEMPDIR from base import webqtlFormData from utility.benchmark import Bench @@ -423,7 +424,7 @@ def marker_regression_page(): print("img_path:", img_path) initial_start_vars = request.form print("initial_start_vars:", initial_start_vars) - imgfile = open('/home/zas1024/tmp/' + img_path, 'rb') + imgfile = open(TEMPDIR + '/' + img_path, 'rb') imgdata = imgfile.read() imgB64 = imgdata.encode("base64") bytesarray = array.array('B', imgB64) -- cgit v1.2.3 From 3be9e4a0950b1ea2d560463032c6f4104baf738e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:49:59 +0000 Subject: [PATCH 026/100] Introducing cached values PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND and TEMPDIR --- wqflask/utility/tools.py | 8 +++++++- wqflask/wqflask/marker_regression/marker_regression.py | 11 +---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index dc7bc33c..5d69e699 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -81,4 +81,10 @@ def flat_files(subdir=None): def tempdir(): return valid_path(get_setting("TEMPDIR","/tmp")) -TEMPDIR = tempdir() +# Cached values + +PYLMM_COMMAND = pylmm_command() +GEMMA_COMMAND = pylmm_command() +PLINK_COMMAND = pylmm_command() +FLAT_FILES = flat_files() +TEMPDIR = tempdir() diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index b0f5ed69..a76a5967 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -36,21 +36,12 @@ from utility import helper_functions from utility import Plot, Bunch from utility import temp_data from utility.benchmark import Bench -from utility.tools import pylmm_command, plink_command, gemma_command from wqflask.marker_regression import gemma_mapping #from wqflask.marker_regression import qtl_reaper_mapping #from wqflask.marker_regression import plink_mapping #from wqflask.marker_regression import rqtl_mapping -# Check for valid binary paths of pylmm, plink, rqtl etc. This code -# runs at startup, so a missing binary will balk before running the -# service - -GEMMA_RUN = gemma_command() -PYLMM_RUN = pylmm_command() -PLINK_RUN = plink_command() -# RQTL_PATH,RQTL_COMMAND = rqtl_command() - +from utility.tools import PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND class MarkerRegression(object): -- cgit v1.2.3 From 1675f34eeff84d7ec0f43c1676f9107e202dda88 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:51:57 +0000 Subject: [PATCH 027/100] Introduce tools.locate --- wqflask/utility/tools.py | 14 +++++++++++++- wqflask/wqflask/marker_regression/marker_regression.py | 5 ++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 5d69e699..624df179 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -78,11 +78,23 @@ def flat_files(subdir=None): return valid_path(base+"/"+subdir) return valid_path(base) +def locate(name, subdir=None): + base = get_setting("GENENETWORK_FILES") + if subdir: + base = base+"/"+subdir + if valid_path(base): + lookfor = base + "/" + name + if valid_path(lookfor): + return lookfor + else: + raise IOError("Can not locate "+lookfor) + raise IOError("Can not locate "+name) + def tempdir(): return valid_path(get_setting("TEMPDIR","/tmp")) + # Cached values - PYLMM_COMMAND = pylmm_command() GEMMA_COMMAND = pylmm_command() PLINK_COMMAND = pylmm_command() diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index a76a5967..36334317 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -41,7 +41,7 @@ from wqflask.marker_regression import gemma_mapping #from wqflask.marker_regression import plink_mapping #from wqflask.marker_regression import rqtl_mapping -from utility.tools import PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND +from utility.tools import locate, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND class MarkerRegression(object): @@ -979,8 +979,7 @@ class MarkerRegression(object): #def gen_human_results(self, pheno_vector, tempdata): def gen_human_results(self, pheno_vector, key, temp_uuid): - file_base = os.path.join(webqtlConfig.PYLMM_PATH, self.dataset.group.name) - print("file_base:", file_base) + file_base = locate(self.dataset.group.name,"mapping") plink_input = input.plink(file_base, type='b') input_file_name = os.path.join(webqtlConfig.SNP_PATH, self.dataset.group.name + ".snps.gz") -- cgit v1.2.3 From fe69b32993ba48623be49404ae07c817a568a01a Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 23 Feb 2016 07:41:22 +0000 Subject: Tell redis where to store its persistent db --- bin/genenetwork2 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index a48e8b93..54eee02e 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -13,8 +13,14 @@ if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH export WQFLASK_SETTINGS=$settings +if [ -z $TEMPDIR ]; then + TEMPDIR="/tmp" +fi + # start the redis server -redis-server & +echo -n "dir $TEMPDIR +dbfilename gn2.rdb +" | redis-server - & cd $GN2_BASE_PATH/wqflask echo "Starting with $settings" -- cgit v1.2.3 From ae8834ac9c0f15f2c397bb6727b8a8079bf175eb Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 23 Feb 2016 07:46:43 +0000 Subject: tools.py: add function locate_without_error --- wqflask/utility/tools.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 624df179..af1ab353 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -79,6 +79,11 @@ def flat_files(subdir=None): return valid_path(base) def locate(name, subdir=None): + """ + Locate a static flat file in the GENENETWORK_FILES environment. + + This function throws an error when the file is not found. + """ base = get_setting("GENENETWORK_FILES") if subdir: base = base+"/"+subdir @@ -89,7 +94,24 @@ def locate(name, subdir=None): else: raise IOError("Can not locate "+lookfor) raise IOError("Can not locate "+name) - + +def locate_without_error(name, subdir=None): + """ + Locate a static flat file in the GENENETWORK_FILES environment. + + This function does not throw an error when the file is not found + but returns None. + """ + base = get_setting("GENENETWORK_FILES") + if subdir: + base = base+"/"+subdir + if valid_path(base): + lookfor = base + "/" + name + if valid_path(lookfor): + return lookfor + sys.stderr.write("WARNING: file "+name+" not found\n") + return None + def tempdir(): return valid_path(get_setting("TEMPDIR","/tmp")) -- cgit v1.2.3 From 6c0f35e56e8d26a47d210b12c171bb050d1ab150 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 23 Feb 2016 09:27:57 +0000 Subject: Fine tuning file locating --- wqflask/utility/tools.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index af1ab353..dd40be99 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -54,10 +54,15 @@ def get_setting(command_id,guess=None): return command def valid_bin(bin): - if os.path.islink(bin) or os.path.isfile(bin): + if os.path.islink(bin) or valid_file(bin): return bin return None +def valid_file(fn): + if os.path.isfile(fn): + return fn + return None + def valid_path(dir): if os.path.isdir(dir): return dir @@ -89,11 +94,12 @@ def locate(name, subdir=None): base = base+"/"+subdir if valid_path(base): lookfor = base + "/" + name - if valid_path(lookfor): + if valid_file(lookfor): return lookfor else: raise IOError("Can not locate "+lookfor) - raise IOError("Can not locate "+name) + if subdir: sys.stderr.write(subdir) + raise IOError("Can not locate "+name+" in "+base) def locate_without_error(name, subdir=None): """ @@ -107,7 +113,7 @@ def locate_without_error(name, subdir=None): base = base+"/"+subdir if valid_path(base): lookfor = base + "/" + name - if valid_path(lookfor): + if valid_file(lookfor): return lookfor sys.stderr.write("WARNING: file "+name+" not found\n") return None -- cgit v1.2.3 From a5ff6520ad29266e1647ce54ad6b2c0f200dfe9a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:55:56 +0000 Subject: [PATCH 032/100] file locating: error checking is built-in --- wqflask/base/data_set.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index c6f34143..9ffc09ac 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,15 +44,13 @@ from dbFunction import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks -from utility.tools import flat_files +from utility.tools import locate from maintenance import get_group_samplelists from MySQLdb import escape_string as escape from pprint import pformat as pf -MAPPING_PATH = flat_files("mapping") - # Used by create_database to instantiate objects # Each subclass will add to this DS_NAME_MAP = {} @@ -405,11 +403,10 @@ class DatasetGroup(object): #print(" type: ", type(self.samplelist)) #print(" self.samplelist: ", self.samplelist) else: - #print("Cache not hit") - - geno_file_path = webqtlConfig.GENODIR+self.name+".geno" + print("Cache not hit") - mapping_file_path = MAPPING_PATH+"/"+self.name+".fam" + geno_file_path = locate(self.name+".geno",'genotype') + mapping_file_path = locate(self.name+".fam",'mapping') if os.path.isfile(mapping_file_path): self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path) elif os.path.isfile(geno_file_path): @@ -439,10 +436,7 @@ class DatasetGroup(object): genotype_1 = reaper.Dataset() # reaper barfs on unicode filenames, so here we ensure it's a string - full_filename = str(os.path.join(webqtlConfig.GENODIR, self.name + '.geno')) - if not os.path.isfile(full_filename): - raise SystemError("File "+full_filename+" does not exist") - print("Reading file: ", full_filename) + full_filename = str(locate(self.name+'.geno','genotype')) genotype_1.read(full_filename) if genotype_1.type == "group" and self.parlist: -- cgit v1.2.3 From 218fd0c7666583748098f0b7b4286b6d1cbf6838 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 23 Feb 2016 11:16:33 +0000 Subject: [PATCH 033/100] Refactored file searching --- wqflask/base/data_set.py | 16 ++++++++-------- wqflask/utility/tools.py | 4 +++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 9ffc09ac..ce13dd77 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,7 +44,7 @@ from dbFunction import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks -from utility.tools import locate +from utility.tools import locate, locate_without_error from maintenance import get_group_samplelists @@ -405,15 +405,15 @@ class DatasetGroup(object): else: print("Cache not hit") - geno_file_path = locate(self.name+".geno",'genotype') - mapping_file_path = locate(self.name+".fam",'mapping') - if os.path.isfile(mapping_file_path): - self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_file_path) - elif os.path.isfile(geno_file_path): - self.samplelist = get_group_samplelists.get_samplelist("geno", geno_file_path) + genotype_fn = locate_without_error(self.name+".geno",'genotype') + mapping_fn = locate_without_error(self.name+".fam",'mapping') + if mapping_fn: + self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn) + elif genotype_fn: + self.samplelist = get_group_samplelists.get_samplelist("geno", genotype_fn) else: self.samplelist = None - #print("after get_samplelist") + print("Sample list: ",self.samplelist) Redis.set(key, json.dumps(self.samplelist)) Redis.expire(key, 60*5) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index dd40be99..d4c10c68 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -36,7 +36,7 @@ def get_setting(command_id,guess=None): """ def value(command): if command: - sys.stderr.write("Found value "+command+"\n") + sys.stderr.write("Found path "+command+"\n") return command else: return None @@ -95,6 +95,7 @@ def locate(name, subdir=None): if valid_path(base): lookfor = base + "/" + name if valid_file(lookfor): + print("Found: file "+lookfor+"\n") return lookfor else: raise IOError("Can not locate "+lookfor) @@ -114,6 +115,7 @@ def locate_without_error(name, subdir=None): if valid_path(base): lookfor = base + "/" + name if valid_file(lookfor): + print("Found: file "+name+"\n") return lookfor sys.stderr.write("WARNING: file "+name+" not found\n") return None -- cgit v1.2.3 From e1e6adeb07536b6c4afc849e7f3a994dca3fda1d Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 23 Feb 2016 11:28:26 +0000 Subject: This threw an error when sample is missing in the DB. Let sample pass without data if the sample/genotype does not exist in the database. --- wqflask/wqflask/correlation/show_corr_results.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 98596ca4..09d5ecdb 100755 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -708,12 +708,12 @@ class CorrelationResults(object): for sample in sample_names: if sample not in excluded_samples: - value = start_vars['value:' + sample] - if value.strip().lower() == 'x': - self.sample_data[str(sample)] = None - else: + print("Looking for",sample,"in",start_vars) + value = start_vars.get('value:' + sample) + if value and not value.strip().lower() == 'x': self.sample_data[str(sample)] = float(value) - + else: + self.sample_data[str(sample)] = None ##XZ, 12/16/2008: the input geneid is of mouse type -- cgit v1.2.3 From e5951ad5f3bb244e4dc53cd04c291dcbdef544a0 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 23 Feb 2016 11:41:44 +0000 Subject: With missing samples to not add them to the result set --- wqflask/wqflask/correlation/show_corr_results.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 09d5ecdb..21a2c26c 100755 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -710,10 +710,11 @@ class CorrelationResults(object): if sample not in excluded_samples: print("Looking for",sample,"in",start_vars) value = start_vars.get('value:' + sample) - if value and not value.strip().lower() == 'x': - self.sample_data[str(sample)] = float(value) - else: - self.sample_data[str(sample)] = None + if value: + if not value.strip().lower() == 'x': + self.sample_data[str(sample)] = float(value) + else: + self.sample_data[str(sample)] = None ##XZ, 12/16/2008: the input geneid is of mouse type -- cgit v1.2.3 From 92fda7557645199276bf4f8a65c40252c4d83c21 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Wed, 24 Feb 2016 09:25:43 +0000 Subject: Settled on the _COMMAND syntax over _RUN --- etc/default_settings.py | 6 +++--- wqflask/utility/tools.py | 10 +++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/etc/default_settings.py b/etc/default_settings.py index 48d3f66b..6a218f26 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -24,6 +24,6 @@ SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a # Path overrides for Genenetwork GENENETWORK_FILES = "../../gn2_data" -PYLMM_RUN = os.popen("which pylmm_redis").read() -PLINK_RUN = os.popen("which plink2").read() -GEMMA_RUN = os.popen("which gemma").read() +PYLMM_COMMAND = os.popen("which pylmm_redis").read() +PLINK_COMMAND = os.popen("which plink2").read() +GEMMA_COMMAND = os.popen("which gemma").read() diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index d4c10c68..0f2e4d88 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -1,9 +1,5 @@ # Tools/paths finder resolves external paths from settings and/or environment # variables -# -# Currently supported: -# -# PYLMM_PATH finds the root of the git repository of the pylmm_gn2 tool import os import sys @@ -69,13 +65,13 @@ def valid_path(dir): return None def pylmm_command(guess=None): - return valid_bin(get_setting("PYLMM_RUN",guess)) + return valid_bin(get_setting("PYLMM_COMMAND",guess)) def gemma_command(guess=None): - return valid_bin(get_setting("GEMMA_RUN",guess)) + return valid_bin(get_setting("GEMMA_COMMAND",guess)) def plink_command(guess=None): - return valid_bin(get_setting("PLINK_RUN",guess)) + return valid_bin(get_setting("PLINK_COMMAND",guess)) def flat_files(subdir=None): base = get_setting("GENENETWORK_FILES") -- cgit v1.2.3 From 5fffa1aa4260af6663c1f9b4cf7494d3ebc6995d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 11:01:29 +0000 Subject: [PATCH 037/100] WebQtlConfig: sanitizing naming and used vars --- bin/genenetwork2 | 2 +- wqflask/base/data_set.py | 194 ++++++++++----------- wqflask/base/webqtlFormData.py | 2 +- .../basicStatistics/BasicStatisticsFunctions.py | 6 +- wqflask/maintenance/get_group_samplelists.py | 3 +- wqflask/utility/tools.py | 11 +- wqflask/wqflask/correlation/show_corr_results.py | 2 +- .../wqflask/correlation_matrix/show_corr_matrix.py | 1 - wqflask/wqflask/heatmap/heatmap.py | 2 - .../interval_analyst/IntervalAnalystPage.py | 138 +++++++-------- .../marker_regression/MarkerRegressionPage.py | 6 +- .../wqflask/marker_regression/marker_regression.py | 2 +- .../marker_regression/marker_regression_gn1.py | 116 ++++++------ wqflask/wqflask/search_results.py | 1 - wqflask/wqflask/show_trait/show_trait.py | 2 +- 15 files changed, 246 insertions(+), 242 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 54eee02e..00ee09bf 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,6 +1,6 @@ #! /bin/bash # -# This will run the server with default settings +# This will run the server with default settings. # Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index ce13dd77..6527657a 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -76,22 +76,22 @@ def create_dataset(dataset_name, dataset_type = None, get_samplelist = True): #def get_dataset_type_from_json(dataset_name): - + class Dataset_Types(object): - + def __init__(self): self.datasets = {} file_name = "wqflask/static/new/javascript/dataset_menu_structure.json" with open(file_name, 'r') as fh: data = json.load(fh) - + print("*" * 70) for species in data['datasets']: for group in data['datasets'][species]: for dataset_type in data['datasets'][species][group]: for dataset in data['datasets'][species][group][dataset_type]: #print("dataset is:", dataset) - + short_dataset_name = dataset[1] if dataset_type == "Phenotypes": new_type = "Publish" @@ -100,32 +100,32 @@ class Dataset_Types(object): else: new_type = "ProbeSet" self.datasets[short_dataset_name] = new_type - + def __call__(self, name): return self.datasets[name] - + # Do the intensive work at startup one time only Dataset_Getter = Dataset_Types() # #print("Running at startup:", get_dataset_type_from_json("HBTRC-MLPFC_0611")) - + def create_datasets_list(): key = "all_datasets" result = Redis.get(key) - + if result: print("Cache hit!!!") datasets = pickle.loads(result) - + else: datasets = list() with Bench("Creating DataSets object"): type_dict = {'Publish': 'PublishFreeze', 'ProbeSet': 'ProbeSetFreeze', 'Geno': 'GenoFreeze'} - + for dataset_type in type_dict: query = "SELECT Name FROM {}".format(type_dict[dataset_type]) for result in g.db.execute(query).fetchall(): @@ -134,10 +134,10 @@ def create_datasets_list(): #print("type: {}\tname: {}".format(dataset_type, result.Name)) dataset = create_dataset(result.Name, dataset_type) datasets.append(dataset) - + Redis.set(key, pickle.dumps(datasets, pickle.HIGHEST_PROTOCOL)) Redis.expire(key, 60*60) - + return datasets @@ -158,30 +158,30 @@ def mescape(*items): class Markers(object): """Todo: Build in cacheing so it saves us reading the same file more than once""" def __init__(self, name): - json_data_fh = open(os.path.join(webqtlConfig.NEWGENODIR + name + '.json')) + json_data_fh = open(locate(name + '.json','genotype/json')) try: markers = json.load(json_data_fh) except: markers = [] - + for marker in markers: if (marker['chr'] != "X") and (marker['chr'] != "Y"): marker['chr'] = int(marker['chr']) marker['Mb'] = float(marker['Mb']) - + self.markers = markers #print("self.markers:", self.markers) - - + + def add_pvalues(self, p_values): print("length of self.markers:", len(self.markers)) print("length of p_values:", len(p_values)) - + if type(p_values) is list: # THIS IS only needed for the case when we are limiting the number of p-values calculated #if len(self.markers) > len(p_values): # self.markers = self.markers[:len(p_values)] - + for marker, p_value in itertools.izip(self.markers, p_values): if not p_value: continue @@ -214,7 +214,7 @@ class Markers(object): #self.markers.remove(marker) #del self.markers[i] self.markers = filtered_markers - + #for i, marker in enumerate(self.markers): # if not 'p_value' in marker: @@ -223,9 +223,9 @@ class Markers(object): # #self.markers.remove(self.markers[i]) class HumanMarkers(Markers): - + def __init__(self, name, specified_markers = []): - marker_data_fh = open(os.path.join(webqtlConfig.PYLMM_PATH + name + '.bim')) + marker_data_fh = open(locate('genotype') + '/' + name + '.bim') self.markers = [] for line in marker_data_fh: splat = line.strip().split() @@ -244,7 +244,7 @@ class HumanMarkers(Markers): marker['name'] = splat[1] marker['Mb'] = float(splat[3]) / 1000000 self.markers.append(marker) - + #print("markers is: ", pf(self.markers)) @@ -257,26 +257,26 @@ class HumanMarkers(Markers): # marker['lod_score'] = -math.log10(marker['p_value']) # #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values # marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 - + #print("p_values2:", pf(p_values)) super(HumanMarkers, self).add_pvalues(p_values) - + #with Bench("deleting markers"): # markers = [] # for marker in self.markers: # if not marker['Mb'] <= 0 and not marker['chr'] == 0: # markers.append(marker) # self.markers = markers - - + + class DatasetGroup(object): """ Each group has multiple datasets; each species has multiple groups. - + For example, Mouse has multiple groups (BXD, BXA, etc), and each group has multiple datasets associated with it. - + """ def __init__(self, dataset): """This sets self.group and self.group_id""" @@ -284,14 +284,14 @@ class DatasetGroup(object): self.name, self.id = g.db.execute(dataset.query_for_group).fetchone() if self.name == 'BXD300': self.name = "BXD" - + self.f1list = None self.parlist = None self.get_f1_parent_strains() #print("parents/f1s: {}:{}".format(self.parlist, self.f1list)) - + self.species = webqtlDatabaseFunction.retrieve_species(self.name) - + self.incparentsf1 = False self.allsamples = None self._datasets = None @@ -302,7 +302,7 @@ class DatasetGroup(object): def get_markers(self): #print("self.species is:", self.species) if self.species == "human": - marker_class = HumanMarkers + marker_class = HumanMarkers else: marker_class = Markers @@ -356,7 +356,7 @@ class DatasetGroup(object): dataset_menu.append(dict(tissue=None, datasets=[(dataset, dataset_short)])) else: dataset_sub_menu = [item[1:] for item in dataset] - + tissue_already_exists = False tissue_position = None for i, tissue_dict in enumerate(dataset_menu): @@ -384,7 +384,7 @@ class DatasetGroup(object): f1, f12, maternal, paternal = webqtlUtil.ParInfo[self.name] except KeyError: f1 = f12 = maternal = paternal = None - + if f1 and f12: self.f1list = [f1, f12] if maternal and paternal: @@ -455,18 +455,18 @@ class DatasetGroup(object): #self.samplelist = list(self.genotype.prgy) self.samplelist = list(genotype.prgy) - + return genotype #class DataSets(object): # """Builds a list of DataSets""" -# +# # def __init__(self): # self.datasets = list() -# +# + - #query = """SELECT Name FROM ProbeSetFreeze # UNION # SELECT Name From PublishFreeze @@ -501,7 +501,7 @@ class DataSet(object): self.check_confidentiality() self.retrieve_other_names() - + self.group = DatasetGroup(self) # sets self.group and self.group_id and gets genotype if get_samplelist == True: self.group.get_samplelist() @@ -511,30 +511,30 @@ class DataSet(object): def get_desc(self): """Gets overridden later, at least for Temp...used by trait's get_given_name""" return None - + #@staticmethod #def get_by_trait_id(trait_id): # """Gets the dataset object given the trait id""" - # - # # - # name = g.db.execute(""" SELECT - # + # + # + # name = g.db.execute(""" SELECT + # # """) - # + # # return DataSet(name) # Delete this eventually @property def riset(): Weve_Renamed_This_As_Group - - + + #@property #def group(self): # if not self._group: # self.get_group() - # + # # return self._group @@ -546,7 +546,7 @@ class DataSet(object): This is not meant to retrieve the data set info if no name at all is passed. """ - + try: if self.type == "ProbeSet": query_args = tuple(escape(x) for x in ( @@ -582,17 +582,17 @@ class DataSet(object): except TypeError: print("Dataset {} is not yet available in GeneNetwork.".format(self.name)) pass - + def get_trait_data(self, sample_list=None): if sample_list: self.samplelist = sample_list else: self.samplelist = self.group.samplelist - + if self.group.parlist != None and self.group.f1list != None: if (self.group.parlist + self.group.f1list) in self.samplelist: self.samplelist += self.group.parlist + self.group.f1list - + query = """ SELECT Strain.Name, Strain.Id FROM Strain, Species WHERE Strain.Name IN {} @@ -610,9 +610,9 @@ class DataSet(object): trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId + #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId #tempTable = None - #if GeneId and db.type == "ProbeSet": + #if GeneId and db.type == "ProbeSet": # if method == "3": # tempTable = self.getTempLiteratureTable(species=species, # input_species_geneid=GeneId, @@ -623,7 +623,7 @@ class DataSet(object): # TissueProbeSetFreezeId=tissueProbeSetFreezeId, # method=method, # returnNumber=returnNumber) - + if self.type == "Publish": dataset_type = "Phenotype" else: @@ -644,7 +644,7 @@ class DataSet(object): left join {}Data as T{} on T{}.Id = {}XRef.DataId and T{}.StrainId={}\n """.format(*mescape(self.type, item, item, self.type, item, item)) - + if self.type == "Publish": query += """ WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId @@ -661,16 +661,16 @@ class DataSet(object): order by {}.Id """.format(*mescape(self.type, self.type, self.type, self.type, self.name, dataset_type, self.type, self.type, dataset_type)) - + #print("trait data query: ", query) - + results = g.db.execute(query).fetchall() #print("query results:", results) trait_sample_data.append(results) trait_count = len(trait_sample_data[0]) self.trait_data = collections.defaultdict(list) - + # put all of the separate data together into a dictionary where the keys are # trait names and values are lists of sample values for trait_counter in range(trait_count): @@ -683,9 +683,9 @@ class PhenotypeDataSet(DataSet): DS_NAME_MAP['Publish'] = 'PhenotypeDataSet' def setup(self): - + #print("IS A PHENOTYPEDATASET") - + # Fields in the database table self.search_fields = ['Phenotype.Post_publication_description', 'Phenotype.Pre_publication_description', @@ -756,26 +756,26 @@ class PhenotypeDataSet(DataSet): def get_trait_info(self, trait_list, species = ''): for this_trait in trait_list: - + if not this_trait.haveinfo: this_trait.retrieve_info(get_qtl_info=True) description = this_trait.post_publication_description - + #If the dataset is confidential and the user has access to confidential #phenotype traits, then display the pre-publication description instead #of the post-publication description if this_trait.confidential: this_trait.description_display = "" continue # for now - + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait( privilege=self.privilege, userName=self.userName, authorized_users=this_trait.authorized_users): - + description = this_trait.pre_publication_description - + if len(description) > 0: this_trait.description_display = description.strip() else: @@ -820,7 +820,7 @@ class PhenotypeDataSet(DataSet): this_trait.LRS_score_repr = LRS_score_repr = '%3.1f' % this_trait.lrs this_trait.LRS_score_value = LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb)) - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -878,7 +878,7 @@ class GenotypeDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - + def get_trait_list(self): query = """ select Geno.Name @@ -912,7 +912,7 @@ class GenotypeDataSet(DataSet): this_trait.location_repr = 'Chr%s: %.6f' % (this_trait.chr, float(this_trait.mb) ) this_trait.location_value = trait_location_value - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1004,7 +1004,7 @@ class MrnaAssayDataSet(DataSet): def check_confidentiality(self): return geno_mrna_confidentiality(self) - + def get_trait_list_1(self): query = """ select ProbeSet.Name @@ -1020,7 +1020,7 @@ class MrnaAssayDataSet(DataSet): trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) #print("After retrieve_sample_data") return trait_data - + #def get_trait_data(self): # self.samplelist = self.group.samplelist + self.group.parlist + self.group.f1list # query = """ @@ -1040,9 +1040,9 @@ class MrnaAssayDataSet(DataSet): # trait_sample_data = [] # for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): # - # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId + # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId # #tempTable = None - # #if GeneId and db.type == "ProbeSet": + # #if GeneId and db.type == "ProbeSet": # # if method == "3": # # tempTable = self.getTempLiteratureTable(species=species, # # input_species_geneid=GeneId, @@ -1053,7 +1053,7 @@ class MrnaAssayDataSet(DataSet): # # TissueProbeSetFreezeId=tissueProbeSetFreezeId, # # method=method, # # returnNumber=returnNumber) - # + # # temp = ['T%s.value' % item for item in sample_ids_step] # query = "SELECT {}.Name,".format(escape(self.type)) # data_start_pos = 1 @@ -1067,7 +1067,7 @@ class MrnaAssayDataSet(DataSet): # left join {}Data as T{} on T{}.Id = {}XRef.DataId # and T{}.StrainId={}\n # """.format(*mescape(self.type, item, item, self.type, item, item)) - # + # # query += """ # WHERE {}XRef.{}FreezeId = {}Freeze.Id # and {}Freeze.Name = '{}' @@ -1080,7 +1080,7 @@ class MrnaAssayDataSet(DataSet): # # trait_count = len(trait_sample_data[0]) # self.trait_data = collections.defaultdict(list) - # + # # # put all of the separate data together into a dictionary where the keys are # # trait names and values are lists of sample values # for trait_counter in range(trait_count): @@ -1088,11 +1088,11 @@ class MrnaAssayDataSet(DataSet): # for chunk_counter in range(int(number_chunks)): # self.trait_data[trait_name] += ( # trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) - + def get_trait_info(self, trait_list=None, species=''): - # Note: setting trait_list to [] is probably not a great idea. + # Note: setting trait_list to [] is probably not a great idea. if not trait_list: trait_list = [] @@ -1155,7 +1155,7 @@ class MrnaAssayDataSet(DataSet): #print("query is:", pf(query)) result = g.db.execute(query).fetchone() - + mean = result[0] if result else 0 if mean: @@ -1176,7 +1176,7 @@ class MrnaAssayDataSet(DataSet): Geno.SpeciesId = Species.Id """.format(species, this_trait.locus) result = g.db.execute(query).fetchone() - + if result: #if result[0] and result[1]: # lrs_chr = result[0] @@ -1184,7 +1184,7 @@ class MrnaAssayDataSet(DataSet): lrs_chr, lrs_mb = result #XZ: LRS_location_value is used for sorting lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb) - + #try: # lrs_location_value = int(lrs_chr)*1000 + float(lrs_mb) #except: @@ -1197,7 +1197,7 @@ class MrnaAssayDataSet(DataSet): this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs this_trait.LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb)) - + def convert_location_to_value(self, chromosome, mb): try: @@ -1208,7 +1208,7 @@ class MrnaAssayDataSet(DataSet): else: location_value = (ord(str(chromosome).upper()[0])*1000 + float(mb)) - + return location_value def get_sequence(self): @@ -1225,7 +1225,7 @@ class MrnaAssayDataSet(DataSet): """ % (escape(self.name), escape(self.dataset.name)) results = g.db.execute(query).fetchone() return results[0] - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1246,8 +1246,8 @@ class MrnaAssayDataSet(DataSet): results = g.db.execute(query).fetchall() #print("RETRIEVED RESULTS HERE:", results) return results - - + + def retrieve_genes(self, column_name): query = """ select ProbeSet.Name, ProbeSet.%s @@ -1256,7 +1256,7 @@ class MrnaAssayDataSet(DataSet): ProbeSetXRef.ProbeSetId=ProbeSet.Id; """ % (column_name, escape(str(self.id))) results = g.db.execute(query).fetchall() - + return dict(results) #def retrieve_gene_symbols(self): @@ -1285,8 +1285,8 @@ class MrnaAssayDataSet(DataSet): # for item in results: # symbol_dict[item[0]] = item[1] # return symbol_dict - - + + class TempDataSet(DataSet): @@ -1308,8 +1308,8 @@ class TempDataSet(DataSet): self.id = 1 self.fullname = 'Temporary Storage' self.shortname = 'Temp' - - + + @staticmethod def handle_pca(desc): if 'PCA' in desc: @@ -1318,13 +1318,13 @@ class TempDataSet(DataSet): else: desc = desc[:desc.index('entered')].strip() return desc - + def get_desc(self): g.db.execute('SELECT description FROM Temp WHERE Name=%s', self.name) desc = g.db.fetchone()[0] desc = self.handle_pca(desc) - return desc - + return desc + def get_group(self): self.cursor.execute(""" SELECT @@ -1337,7 +1337,7 @@ class TempDataSet(DataSet): """, self.name) self.group, self.group_id = self.cursor.fetchone() #return self.group - + def retrieve_sample_data(self, trait): query = """ SELECT @@ -1351,7 +1351,7 @@ class TempDataSet(DataSet): Order BY Strain.Name """ % escape(trait.name) - + results = g.db.execute(query).fetchall() diff --git a/wqflask/base/webqtlFormData.py b/wqflask/base/webqtlFormData.py index 44fdcc3f..10251756 100755 --- a/wqflask/base/webqtlFormData.py +++ b/wqflask/base/webqtlFormData.py @@ -157,7 +157,7 @@ class webqtlFormData(object): self.genotype_1 = reaper.Dataset() - full_filename = os.path.join(webqtlConfig.GENODIR, self.group + '.geno') + full_filename = locate(self.group + '.geno','genotype') # reaper barfs on unicode filenames, so here we ensure it's a string full_filename = str(full_filename) diff --git a/wqflask/basicStatistics/BasicStatisticsFunctions.py b/wqflask/basicStatistics/BasicStatisticsFunctions.py index 74784853..e748a822 100755 --- a/wqflask/basicStatistics/BasicStatisticsFunctions.py +++ b/wqflask/basicStatistics/BasicStatisticsFunctions.py @@ -118,7 +118,7 @@ def plotNormalProbability(vals=None, RISet='', title=None, showstrains=0, specia Plot.plotXY(c, dataZ, dataX, dataLabel = dataLabel, XLabel='Expected Z score', connectdot=0, YLabel='Trait value', title=title, specialCases=specialStrains, showLabel = showLabel) filename= webqtlUtil.genRandStr("nP_") - c.save(webqtlConfig.IMGDIR+filename, format='gif') + c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) @@ -145,7 +145,7 @@ def plotBoxPlot(vals): Plot.plotBoxPlot(canvas, XXX, offset=(xLeftOffset, xRightOffset, yTopOffset, yBottomOffset), XLabel= "Trait") filename= webqtlUtil.genRandStr("Box_") - canvas.save(webqtlConfig.IMGDIR+filename, format='gif') + canvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) plotLink = HT.Span("More about ", HT.Href(text="Box Plots", url="http://davidmlane.com/hyperstat/A37797.html", target="_blank", Class="fs13")) @@ -201,7 +201,7 @@ def plotBarGraph(identification='', RISet='', vals=None, type="name"): Plot.plotBarText(c, tvals, tnames, variance=tvars, YLabel='Value', title=title, sLabel = sLabel, barSpace = sw) filename= webqtlUtil.genRandStr("Bar_") - c.save(webqtlConfig.IMGDIR+filename, format='gif') + c.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) return img diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py index b8397b47..a9059fad 100755 --- a/wqflask/maintenance/get_group_samplelists.py +++ b/wqflask/maintenance/get_group_samplelists.py @@ -6,7 +6,6 @@ import gzip from base import webqtlConfig - def process_genofiles(geno_dir=webqtlConfig.GENODIR): print("Yabba") #sys.exit("Dabba") @@ -54,4 +53,4 @@ def get_samplelist_from_plink(genofilename): line = line.split(" ") samplelist.append(line[0]) - return samplelist \ No newline at end of file + return samplelist diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 0f2e4d88..c0f6a49a 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -79,6 +79,15 @@ def flat_files(subdir=None): return valid_path(base+"/"+subdir) return valid_path(base) +def assert_dir(dir): + if not valid_path(dir): + raise Exception("ERROR: can not find directory "+dir) + return dir + +def mk_dir(dir): + os.makedirs(dir) + return assert_dir(dir) + def locate(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -98,7 +107,7 @@ def locate(name, subdir=None): if subdir: sys.stderr.write(subdir) raise IOError("Can not locate "+name+" in "+base) -def locate_without_error(name, subdir=None): +def locate_ignore_error(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 21a2c26c..06b4860e 100755 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -943,7 +943,7 @@ class CorrelationResults(object): use_tissue_corr = True DatabaseFileName = self.getFileName( target_db_name=self.target_db_name ) - datasetFile = open(webqtlConfig.TEXTDIR+DatabaseFileName,'r') + datasetFile = open(webqtlConfig.CACHEDIR+DatabaseFileName,'r') #XZ, 01/08/2009: read the first line line = datasetFile.readline() diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 6bc0ef77..f74e655d 100755 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -43,7 +43,6 @@ from pprint import pformat as pf from htmlgen import HTMLgen2 as HT import reaper -from base import webqtlConfig from utility.THCell import THCell from utility.TDCell import TDCell from base.trait import GeneralTrait diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 40f518f0..61847bc3 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -26,8 +26,6 @@ import reaper from base.trait import GeneralTrait from base import data_set from base import species -from base import webqtlConfig -from utility import webqtlUtil # from wqflask.my_pylmm.pyLMM import lmm # from wqflask.my_pylmm.pyLMM import input from utility import helper_functions diff --git a/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py b/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py index ec9aa29c..f45ec0c4 100755 --- a/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py +++ b/wqflask/wqflask/interval_analyst/IntervalAnalystPage.py @@ -45,40 +45,40 @@ class IntervalAnalystPage(templatePage): #A dictionary that lets us map the html form names "txStart_mm6" -> "Mb Start (mm8)" #the first item is the short name (column headers) and the second item is the long name (dropdown list) # [short name, long name, category] - columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'], + columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'], "GeneDescription" : ["Description", "Gene Description", 'species'], - 'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'], - 'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'], - 'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'], + 'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'], + 'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'], + 'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'], "ProteinID" : ["Prot ID", "Protein ID", 'protein'], - "Chromosome" : ["Chr", "Chromosome", 'species'], - "TxStart" : ["Start", "Mb Start", 'species'], - "TxEnd" : ["End", "Mb End", 'species'], - "GeneLength" : ["Length", "Kb Length", 'species'], - "cdsStart" : ["CDS Start", "Mb CDS Start", 'species'], + "Chromosome" : ["Chr", "Chromosome", 'species'], + "TxStart" : ["Start", "Mb Start", 'species'], + "TxEnd" : ["End", "Mb End", 'species'], + "GeneLength" : ["Length", "Kb Length", 'species'], + "cdsStart" : ["CDS Start", "Mb CDS Start", 'species'], "cdsEnd" : ["CDS End", "Mb CDS End", 'species'], - "exonCount" : ["Num Exons", "Exon Count", 'species'], - "exonStarts" : ["Exon Starts", "Exon Starts", 'species'], - "exonEnds" : ["Exon Ends", "Exon Ends", 'species'], - "Strand" : ["Strand", "Strand", 'species'], + "exonCount" : ["Num Exons", "Exon Count", 'species'], + "exonStarts" : ["Exon Starts", "Exon Starts", 'species'], + "exonEnds" : ["Exon Ends", "Exon Ends", 'species'], + "Strand" : ["Strand", "Strand", 'species'], "GeneID" : ["Gene ID", "Gene ID", 'species'], - "GenBankID" : ["GenBank", "GenBank ID", 'species'], + "GenBankID" : ["GenBank", "GenBank ID", 'species'], "UnigenID" : ["Unigen", "Unigen ID", 'species'], - "NM_ID" : ["NM ID", "NM ID", 'species'], + "NM_ID" : ["NM ID", "NM ID", 'species'], "kgID" : ["kg ID", "kg ID", 'species'], - "snpCount" : ["SNPs", "SNP Count", 'species'], - "snpDensity" : ["SNP Density", "SNP Density", 'species'], - "lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'], - "lod" : ["LOD", "Likelihood Odds Ratio", 'misc'], - "pearson" : ["Pearson", "Pearson Product Moment", 'misc'], - "literature" : ["Lit Corr", "Literature Correlation", 'misc'], + "snpCount" : ["SNPs", "SNP Count", 'species'], + "snpDensity" : ["SNP Density", "SNP Density", 'species'], + "lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'], + "lod" : ["LOD", "Likelihood Odds Ratio", 'misc'], + "pearson" : ["Pearson", "Pearson Product Moment", 'misc'], + "literature" : ["Lit Corr", "Literature Correlation", 'misc'], } ###Species Freeze speciesFreeze = {'mouse':'mm9', 'rat':'rn3', 'human':'hg19'} for key in speciesFreeze.keys(): speciesFreeze[speciesFreeze[key]] = key - + def __init__(self, fd): templatePage.__init__(self, fd) @@ -86,7 +86,7 @@ class IntervalAnalystPage(templatePage): fd.formdata['remote_ip'] = fd.remote_ip if not self.openMysql(): return - + self.species = fd.formdata.getvalue("species", "mouse") try: self.startMb = float(fd.formdata.getvalue("startMb")) @@ -96,7 +96,7 @@ class IntervalAnalystPage(templatePage): self.endMb = float(fd.formdata.getvalue("endMb")) except: self.endMb = self.startMb + 10 - + self.Chr = fd.formdata.getvalue("chromosome", "1") self.xls = fd.formdata.getvalue("xls", "1") try: @@ -107,38 +107,38 @@ class IntervalAnalystPage(templatePage): self.diffColDefault = self.diffCol = [] if self.species != 'mouse': self.diffColDefault = [2, 3]#default is B6 and D2 for other species - + controlFrm, dispFields = self.genControlForm(fd) geneTable, filename = self.genGeneTable(fd, dispFields) - + infoTD = HT.TD(width=400, valign= "top") - infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"), - HT.Strong("Species : "), self.species.title(), HT.BR(), - HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(), - HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(), + infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"), + HT.Strong("Species : "), self.species.title(), HT.BR(), + HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(), + HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(), ) if filename: infoTD.append(HT.BR(), HT.BR(), HT.Href(text="Download", url = "/tmp/" + filename, Class="normalsize") , " output in MS excel format.") - + mainTable = HT.TableLite(HT.TR(infoTD, HT.TD(controlFrm, Class="doubleBorder", width=400), HT.TD(" ", width="")), cellpadding=10) mainTable.append(HT.TR(HT.TD(geneTable, colspan=3))) self.dict['body'] = HT.TD(mainTable) self.dict['title'] = "Interval Analyst" - + def genGeneTable(self, fd, dispFields): filename = "" if self.xls: #import pyXLWriter as xl filename = "IntAn_Chr%s_%2.6f-%2.6f" % (self.Chr, self.startMb, self.endMb) filename += ".xls" - + # Create a new Excel workbook workbook = xl.Writer(os.path.join(webqtlConfig.TMPDIR, filename)) worksheet = workbook.add_worksheet() titleStyle = workbook.add_format(align = 'left', bold = 0, size=18, border = 1, border_color="gray") headingStyle = workbook.add_format(align = 'center', bold = 1, size=13, fg_color = 0x1E, color="white", border = 1, border_color="gray") - + ##Write title Info worksheet.write([0, 0], "GeneNetwork Interval Analyst Table", titleStyle) worksheet.write([1, 0], "%s%s" % (webqtlConfig.PORTADDR, os.path.join(webqtlConfig.CGIDIR, self._scriptfile))) @@ -148,12 +148,12 @@ class IntervalAnalystPage(templatePage): worksheet.write([4, 0], "Search by : %s" % fd.formdata['remote_ip']) worksheet.write([5, 0], "view region : Chr %s %2.6f - %2.6f Mb" % (self.Chr, self.startMb, self.endMb)) nTitleRow = 7 - + geneTable = HT.TableLite(Class="collap", cellpadding=5) headerRow = HT.TR(HT.TD(" ", Class="fs13 fwb ffl b1 cw cbrb", width="1")) if self.xls: worksheet.write([nTitleRow, 0], "Index", headingStyle) - + for ncol, column in enumerate(dispFields): if len(column) == 1: headerRow.append(HT.TD(self.columnNames[column[0]][0], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1,align="Center")) @@ -162,24 +162,24 @@ class IntervalAnalystPage(templatePage): worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle) worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle)) else: - headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]], + headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1, align="Center")) if self.xls: colTitle = self.columnNames[column[0]][0] + " (%s)" % self.speciesFreeze[column[1]] worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle) worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle)) - #headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), - # "(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]), + #headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), + # "(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]), # Class="colorBlue", NOWRAP=1, align="Center")) geneTable.append(headerRow) - + geneCol = GeneUtil.loadGenes(self.cursor, self.Chr, self.diffColDefault, self.startMb, self.endMb, species=self.species) for gIndex, theGO in enumerate(geneCol): geneRow = HT.TR(HT.TD(gIndex+1, Class="fs12 fwn b1", align="right")) if self.xls: nTitleRow += 1 worksheet.write([nTitleRow, 0], gIndex + 1) - + for ncol, column in enumerate(dispFields): if len(column) == 1 or column[1]== self.species: keyValue = "" @@ -196,17 +196,17 @@ class IntervalAnalystPage(templatePage): curGO = theGO[subGO] if theGO[subGO].has_key(fieldName): keyValue = theGO[subGO][fieldName] - + if self.xls: worksheet.write([nTitleRow, ncol+1], keyValue) geneRow.append(self.formatTD(keyValue, fieldName, curSpecies, curGO)) - + geneTable.append(geneRow) - + if self.xls: workbook.close() return geneTable, filename - + def formatTD(self, keyValue, fieldName, Species, theGO): if keyValue is None: keyValue = "" @@ -219,7 +219,7 @@ class IntervalAnalystPage(templatePage): keyValue = "" return HT.TD(keyValue, Class="fs12 fwn b1", width=300) elif fieldName in ("GeneSymbol"): - webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), + webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), HT.Image("/images/webqtl_search.gif", border=0, valign="top"), target="_blank") if theGO['GeneID']: geneSymbolLink = HT.Href(webqtlConfig.NCBI_LOCUSID % theGO['GeneID'], keyValue, Class="normalsize", target="_blank") @@ -236,8 +236,8 @@ class IntervalAnalystPage(templatePage): return HT.TD(keyValue, Class="fs12 fwn b1",align="right") elif fieldName in ("snpCount"): if keyValue: - snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'), - theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]), + snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'), + theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]), text=theGO["snpCount"], target="_blank", Class="normalsize") else: snpString = keyValue @@ -252,13 +252,13 @@ class IntervalAnalystPage(templatePage): return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1) else: return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1,align="right") - + def genControlForm(self, fd): ##desc GeneList self.cursor.execute("Desc GeneList") GeneListFields = self.cursor.fetchall() GeneListFields = map(lambda X: X[0], GeneListFields) - + #group columns by category--used for creating the dropdown list of possible columns categories = {} for item in self.columnNames.keys(): @@ -267,7 +267,7 @@ class IntervalAnalystPage(templatePage): categories[category[-1]] = [item ] else: categories[category[-1]] = categories[category[-1]]+[item] - + ##List All Species in the Gene Table speciesDict = {} self.cursor.execute("select Species.Name, GeneList.SpeciesId from Species, GeneList where \ @@ -292,34 +292,34 @@ class IntervalAnalystPage(templatePage): pass AppliedField.append(item2) categories[specName] = AppliedField - + categoriesOrder += ['misc'] - + ############################################################ ## Create the list of possible columns for the dropdown list ############################################################ allColumnsList = HT.Select(name="allColumns", Class="snpBrowserDropBox") - + for category in categoriesOrder: allFields = categories[category] if allFields: geneOpt = HT.Optgroup(label=category.title()) for item in allFields: if category in self.speciesFreeze.keys(): - geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]), + geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]), "%s__%s" % (item, self.speciesFreeze[category]))) else: geneOpt.append((self.columnNames[item][1], item)) geneOpt.sort() allColumnsList.append(geneOpt) - + ###################################### ## Create the list of selected columns ###################################### - + #cols contains the value of all the selected columns submitCols = cols = fd.formdata.getvalue("columns", "default") - + if cols == "default": if self.species=="mouse": #these are the same columns that are shown on intervalPage.py cols = ['GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'Strand', 'GeneLength', 'GeneID', 'NM_ID', 'snpCount', 'snpDensity'] @@ -331,12 +331,12 @@ class IntervalAnalystPage(templatePage): else: if type(cols)==type(""): cols = [cols] - + colsLst = [] dispFields = [] for column in cols: if submitCols == "default" and column not in ('GeneSymbol') and (column in GeneListFields or column in speciesField): - colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]), + colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]), "%s__%s" % (column, self.speciesFreeze[self.species]))) dispFields.append([column, self.species]) else: @@ -346,17 +346,17 @@ class IntervalAnalystPage(templatePage): dispFields.append([column]) else: thisSpecies = self.speciesFreeze[column2[1]] - colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]), + colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]), column)) dispFields.append((column2[0], thisSpecies)) selectedColumnsList = HT.Select(name="columns", Class="snpBrowserSelectBox", multiple="true", data=colsLst, size=6) - + ########################## ## Create the columns form - ########################## + ########################## columnsForm = HT.Form(name="columnsForm", submit=HT.Input(type='hidden'), cgi=os.path.join(webqtlConfig.CGIDIR, self._scriptfile), enctype="multipart/form-data") columnsForm.append(HT.Input(type="hidden", name="fromdatabase", value= fd.formdata.getvalue("fromdatabase", "unknown"))) - columnsForm.append(HT.Input(type="hidden", name="species", value=self.species)) + columnsForm.append(HT.Input(type="hidden", name="species", value=self.species)) if self.diffCol: columnsForm.append(HT.Input(type="hidden", name="s1", value=self.diffCol[0])) columnsForm.append(HT.Input(type="hidden", name="s2", value=self.diffCol[1])) @@ -366,8 +366,8 @@ class IntervalAnalystPage(templatePage): removeButton = HT.Input(type="button", name="remove", value="Remove", Class="button", onClick="removeFromList(this.form.columns.selectedIndex, this.form.columns)") upButton = HT.Input(type="button", name="up", value="Up", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex-1, this.form.columns)") downButton = HT.Input(type="button", name="down", value="Down", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex+1, this.form.columns)") - clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)") - submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)") + clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)") + submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)") selectChrBox = HT.Select(name="chromosome") self.cursor.execute(""" @@ -375,11 +375,11 @@ class IntervalAnalystPage(templatePage): Chr_Length.Name, Length from Chr_Length, Species where Chr_Length.SpeciesId = Species.Id AND - Species.Name = '%s' + Species.Name = '%s' Order by Chr_Length.OrderId """ % self.species) - + results = self.cursor.fetchall() for chrInfo in results: selectChrBox.append((chrInfo[0], chrInfo[0])) @@ -401,5 +401,5 @@ class IntervalAnalystPage(templatePage): #columnsForm.append(HT.Input(type="hidden", name="sort", value=diffCol), # HT.Input(type="hidden", name="identification", value=identification), # HT.Input(type="hidden", name="traitInfo", value=traitInfo)) - + return columnsForm, dispFields diff --git a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py index d02d80b3..455fcf95 100755 --- a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py +++ b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py @@ -140,7 +140,7 @@ class MarkerRegressionPage(templatePage): intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight)) gifmap = self.plotIntMappingForPLINK(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, plinkResultDict=plinkResultDict) - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png') intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap') TD_LR = HT.TR(HT.TD(HT.Blockquote(gifmap,intImg, HT.P()), bgColor='#eeeeee', height = 200)) @@ -249,7 +249,7 @@ class MarkerRegressionPage(templatePage): intCanvas = pid.PILCanvas(size=(self.graphWidth,self.graphHeight)) gifmap = self.plotIntMapping(fd, intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= "") filename= webqtlUtil.genRandStr("Itvl_") - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, filename), format='png') intImg=HT.Image('/image/'+filename+'.png', border=0, usemap='#WebQTLImageMap') ################################################################ @@ -458,7 +458,7 @@ class MarkerRegressionPage(templatePage): #plotBar(myCanvas,10,10,390,290,LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test',identification=fd.identification) Plot.plotBar(myCanvas, LRSArray,XLabel='LRS',YLabel='Frequency',title=' Histogram of Permutation Test') filename= webqtlUtil.genRandStr("Reg_") - myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif') + myCanvas.save(webqtlConfig.GENERATED_IMAGE_DIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test') if fd.suggestive == None: diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 36334317..265f9473 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -30,7 +30,7 @@ from flask import Flask, g from base.trait import GeneralTrait from base import data_set from base import species -from base import webqtlConfig +# from base import webqtlConfig from utility import webqtlUtil from utility import helper_functions from utility import Plot, Bunch diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 01303b0f..decde579 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -178,8 +178,8 @@ class MarkerRegression(object): self.species = start_vars['species'] #Needing for form submission when doing single chr mapping or remapping after changing options - self.vals = start_vars['vals'] - self.mapping_method = start_vars['mapping_method'] + self.vals = start_vars['vals'] + self.mapping_method = start_vars['mapping_method'] if self.mapping_method == "rqtl_geno": self.mapmethod_rqtl_geno = start_vars['method'] self.mapmodel_rqtl_geno = start_vars['model'] @@ -232,7 +232,7 @@ class MarkerRegression(object): self.significant = start_vars['significant'] else: self.nperm = 0 - + if 'bootCheck' in start_vars.keys(): self.bootChecked = start_vars['bootCheck'] else: @@ -308,7 +308,7 @@ class MarkerRegression(object): if 'showSNP' in start_vars.keys(): self.SNPChecked = start_vars['showSNP'] else: - self.SNPChecked = False + self.SNPChecked = False if 'showGenes' in start_vars.keys(): self.geneChecked = start_vars['showGenes'] else: @@ -321,7 +321,7 @@ class MarkerRegression(object): self.endMb = float(start_vars['endMb']) except: self.endMb = -1 - try: + try: self.lrsMax = float(start_vars['lrsMax']) except: self.lrsMax = 0 @@ -363,7 +363,7 @@ class MarkerRegression(object): self.ChrList.append((indChr.name, i)) - + self.ChrLengthMbList = g.db.execute(""" Select Length from Chr_Length, InbredSet @@ -517,7 +517,7 @@ class MarkerRegression(object): chrName = self.selectedChr # Draw the genes for this chromosome / region of this chromosome webqtldatabase = self.dataset.name - + if self.dataset.group.species == "mouse": self.geneCol = GeneUtil.loadGenes(chrName, self.diffCol, self.startMb, self.endMb, webqtldatabase, "mouse") elif self.dataset.group.species == "rat": @@ -533,11 +533,11 @@ class MarkerRegression(object): # # #GENEID = fd.formdata.getvalue('GeneId') or None # GENEID = None - # + # # geneTableContainer = HT.Div(Id="sortable") #Div to hold table # geneTable = self.geneTable(self.geneCol,GENEID) # geneTableContainer.append(geneTable) - # + # # mainfmName = webqtlUtil.genRandStr("fm_") # tableForm = HT.Form(cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name=mainfmName, submit=HT.Input(type='hidden')) # tableForm.append(HT.Input(name='FormID', value='', type='hidden')) @@ -552,22 +552,22 @@ class MarkerRegression(object): #else: showLocusForm = "" intCanvas = pid.PILCanvas(size=(self.graphWidth, self.graphHeight)) - gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) + gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) self.gifmap = gifmap.__str__() #print("GIFMAP:", gifmap.__str__()) self.filename= webqtlUtil.genRandStr("Itvl_") - intCanvas.save(os.path.join(webqtlConfig.IMGDIR, self.filename), format='jpeg') + intCanvas.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename), format='jpeg') intImg=HT.Image('/image/'+self.filename+'.png', border=0, usemap='#WebQTLImageMap') #Scales plot differently for high resolution if self.draw2X: intCanvasX2 = pid.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2)) gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2) - intCanvasX2.save(os.path.join(webqtlConfig.IMGDIR, self.filename+"X2"), format='png') + intCanvasX2.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename+"X2"), format='png') #DLintImgX2=HT.Href(text='Download',url = '/image/'+self.filename+'X2.png', Class='smallsize', target='_blank') - + #textUrl = self.writeQTL2Text(fd, self.filename) ################################################################ @@ -597,7 +597,7 @@ class MarkerRegression(object): showLocusForm.append(intImg) else: showLocusForm = intImg - + if self.permChecked and self.nperm > 0 and not self.multipleInterval and 0 < self.nperm: self.perm_filename = self.drawPermutationHistogram() #perm_text_file = self.permutationTextFile() @@ -667,7 +667,7 @@ class MarkerRegression(object): TD_LR.append(HT.Blockquote(tableForm)) self.body = TD_LR - + #self.dict['body'] = TD_LR #self.dict['title'] = "Mapping" @@ -858,7 +858,7 @@ class MarkerRegression(object): BootCoord = [] i = 0 startX = xLeftOffset - + if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes for j, _chr in enumerate(self.genotype): BootCoord.append( []) @@ -881,8 +881,8 @@ class MarkerRegression(object): else: Xc = startX + (_locus.cM-_chr[0].cM)*plotXScale BootCoord[-1].append([Xc, self.bootResult[i]]) - i += 1 - + i += 1 + #reduce bootResult if self.selectedChr > -1: maxBootBar = 80.0 @@ -1411,7 +1411,7 @@ class MarkerRegression(object): if _strains[ii] in self.dataset.group.samplelist: temp = GeneralObject(name=_strains[ii], value=_val) smd.append(temp) - + smd.sort(lambda A, B: cmp(A.value, B.value)) smd.reverse() @@ -1566,14 +1566,14 @@ class MarkerRegression(object): firstGene = 0 else: lastGene = 0 - + for j, _geno in enumerate (self.genotype[0][1].genotype): plotbxd=0 for item in smd: if item.name == samplelist[j]: - plotbxd=1 - + plotbxd=1 + if (plotbxd == 1): ind = 0 @@ -1620,28 +1620,28 @@ class MarkerRegression(object): currentChromosome = self.genotype[0].name i = 0 - + paddingTop = yTopOffset ucscPaddingTop = paddingTop + self.WEBQTL_BAND_HEIGHT + self.BAND_SPACING ensemblPaddingTop = ucscPaddingTop + self.UCSC_BAND_HEIGHT + self.BAND_SPACING - + if zoom == 1: for pixel in range(xLeftOffset, xLeftOffset + plotWidth, pixelStep): - + calBase = self.kONE_MILLION*(startMb + (endMb-startMb)*(pixel-xLeftOffset-0.0)/plotWidth) - + xBrowse1 = pixel xBrowse2 = min(xLeftOffset + plotWidth, (pixel + pixelStep - 1)) - + WEBQTL_COORDS = "%d, %d, %d, %d" % (xBrowse1, paddingTop, xBrowse2, (paddingTop+self.WEBQTL_BAND_HEIGHT)) bandWidth = xBrowse2 - xBrowse1 WEBQTL_HREF = "javascript:rangeView('%s', %f, %f)" % (self.selectedChr - 1, max(0, (calBase-webqtlZoomWidth))/1000000.0, (calBase+webqtlZoomWidth)/1000000.0) - + WEBQTL_TITLE = "Click to view this section of the genome in WebQTL" gifmap.areas.append(HT.Area(shape='rect',coords=WEBQTL_COORDS,href=WEBQTL_HREF, title=WEBQTL_TITLE)) canvas.drawRect(xBrowse1, paddingTop, xBrowse2, (paddingTop + self.WEBQTL_BAND_HEIGHT), edgeColor=self.CLICKABLE_WEBQTL_REGION_COLOR, fillColor=self.CLICKABLE_WEBQTL_REGION_COLOR) canvas.drawLine(xBrowse1, paddingTop, xBrowse1, (paddingTop + self.WEBQTL_BAND_HEIGHT), color=self.CLICKABLE_WEBQTL_REGION_OUTLINE_COLOR) - + UCSC_COORDS = "%d, %d, %d, %d" %(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT)) if self.species == "mouse": UCSC_HREF = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=%s/snp/chr%s" % (self._ucscDb, self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases, webqtlConfig.PORTADDR, self.selectedChr) @@ -1651,7 +1651,7 @@ class MarkerRegression(object): gifmap.areas.append(HT.Area(shape='rect',coords=UCSC_COORDS,href=UCSC_HREF, title=UCSC_TITLE)) canvas.drawRect(xBrowse1, ucscPaddingTop, xBrowse2, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), edgeColor=self.CLICKABLE_UCSC_REGION_COLOR, fillColor=self.CLICKABLE_UCSC_REGION_COLOR) canvas.drawLine(xBrowse1, ucscPaddingTop, xBrowse1, (ucscPaddingTop+self.UCSC_BAND_HEIGHT), color=self.CLICKABLE_UCSC_REGION_OUTLINE_COLOR) - + ENSEMBL_COORDS = "%d, %d, %d, %d" %(xBrowse1, ensemblPaddingTop, xBrowse2, (ensemblPaddingTop+self.ENSEMBL_BAND_HEIGHT)) if self.species == "mouse": ENSEMBL_HREF = "http://www.ensembl.org/Mus_musculus/contigview?highlight=&chr=%s&vc_start=%d&vc_end=%d&x=35&y=12" % (self.selectedChr, max(0, calBase-flankingWidthInBases), calBase+flankingWidthInBases) @@ -1766,7 +1766,7 @@ class MarkerRegression(object): ChrAInfo = [] preLpos = -1 distinctCount = 0.0 - + #if len(self.genotype) > 1: if self.selectedChr == -1: #ZS: If viewing full genome/all chromosomes for i, _chr in enumerate(self.genotype): @@ -1809,7 +1809,7 @@ class MarkerRegression(object): offsetA = -stepA lineColor = pid.lightblue startPosX = xLeftOffset - + for j, ChrInfo in enumerate(ChrAInfo): preLpos = -1 for i, item in enumerate(ChrInfo): @@ -1877,7 +1877,7 @@ class MarkerRegression(object): # lodm = self.LODFACTOR # else: # lodm = 1.0 - + #ZS: This is a mess, but I don't know a better way to account for different mapping methods returning results in different formats + the option to change between LRS and LOD if self.lrsMax <= 0: #sliding scale if "lrs_value" in self.qtlresults[0]: @@ -1890,7 +1890,7 @@ class MarkerRegression(object): else: if self.permChecked and self.nperm > 0 and not self.multipleInterval: self.significant = min(self.significant, webqtlConfig.MAXLRS) - self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) + self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) else: pass else: @@ -1903,10 +1903,10 @@ class MarkerRegression(object): else: if self.permChecked and self.nperm > 0 and not self.multipleInterval: self.significant = min(self.significant, webqtlConfig.MAXLRS) - self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) + self.suggestive = min(self.suggestive, webqtlConfig.MAXLRS) else: pass - + if self.permChecked and self.nperm > 0 and not self.multipleInterval: LRS_LOD_Max = max(self.significant, LRS_LOD_Max) @@ -1923,7 +1923,7 @@ class MarkerRegression(object): LRSScale = 2.5 else: LRSScale = 1.0 - + LRSAxisList = Plot.frange(LRSScale, LRS_LOD_Max, LRSScale) #make sure the user's value appears on the y-axis #update by NL 6-21-2011: round the LOD value to 100 when LRS_LOD_Max is equal to 460 @@ -1953,7 +1953,7 @@ class MarkerRegression(object): #"Significant" and "Suggestive" Drawing Routine # ======= Draw the thick lines for "Significant" and "Suggestive" ===== (crowell: I tried to make the SNPs draw over these lines, but piddle wouldn't have it...) - + #ZS: I don't know if what I did here with this inner function is clever or overly complicated, but it's the only way I could think of to avoid duplicating the code inside this function def add_suggestive_significant_lines_and_legend(start_pos_x, chr_length_dist): rightEdge = int(start_pos_x + chr_length_dist*plotXScale - self.SUGGESTIVE_WIDTH/1.5) @@ -1976,13 +1976,13 @@ class MarkerRegression(object): start_pos_x += (chr_length_dist+self.GraphInterval)*plotXScale return start_pos_x - + for i, _chr in enumerate(self.genotype): if self.selectedChr != -1: if _chr.name == self.ChrList[self.selectedChr][0]: startPosX = add_suggestive_significant_lines_and_legend(startPosX, self.ChrLengthDistList[0]) break - else: + else: continue else: startPosX = add_suggestive_significant_lines_and_legend(startPosX, self.ChrLengthDistList[i]) @@ -1997,7 +1997,7 @@ class MarkerRegression(object): #else: # dominanceMax = -1 lrsEdgeWidth = 2 - + if zoom == 2: lrsEdgeWidth = 2 * lrsEdgeWidth @@ -2018,7 +2018,7 @@ class MarkerRegression(object): if qtlresult['chr'] != previous_chr and self.selectedChr == -1: if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and self.additiveChecked: plusColor = self.ADDITIVE_COLOR_POSITIVE minusColor = self.ADDITIVE_COLOR_NEGATIVE @@ -2048,7 +2048,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + LRSCoordXY = [] AdditiveCoordXY = [] previous_chr = qtlresult['chr'] @@ -2061,7 +2061,7 @@ class MarkerRegression(object): #startPosX += (self.ChrLengthDistList[j]+self.GraphInterval)*plotXScale - #for j, _chr in enumerate(self.genotype): + #for j, _chr in enumerate(self.genotype): #ZS: This is beause the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used if self.plotScale == "physic": this_chr = str(self.ChrList[self.selectedChr][0]) @@ -2126,7 +2126,7 @@ class MarkerRegression(object): if self.manhattan_plot != True: canvas.drawPolygon(LRSCoordXY,edgeColor=thisLRSColor,closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and self.additiveChecked: plusColor = self.ADDITIVE_COLOR_POSITIVE minusColor = self.ADDITIVE_COLOR_NEGATIVE @@ -2156,7 +2156,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + if not self.multipleInterval and INTERCROSS and self.dominanceChecked: plusColor = self.DOMINANCE_COLOR_POSITIVE minusColor = self.DOMINANCE_COLOR_NEGATIVE @@ -2186,7 +2186,7 @@ class MarkerRegression(object): canvas.drawLine(Xc0, Yc0, Xc, Yc, color=plusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) else: canvas.drawLine(Xc0, yZero - (Yc0-yZero), Xc, yZero - (Yc-yZero), color=minusColor, width=lineWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth)) - + ###draw additive scale if not self.multipleInterval and self.additiveChecked: @@ -2222,7 +2222,7 @@ class MarkerRegression(object): if zoom == 2: fontZoom = 1.5 yTopOffset += 30 - + #calculate plot scale if self.plotScale != 'physic': self.ChrLengthDistList = self.ChrLengthCMList @@ -2589,13 +2589,13 @@ class MarkerRegression(object): perm_output = [value/4.16 for value in self.perm_output] else: perm_output = self.perm_output - + Plot.plotBar(myCanvas, perm_output, XLabel=self.LRS_LOD, YLabel='Frequency', title=' Histogram of Permutation Test') filename= webqtlUtil.genRandStr("Reg_") myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif') - + return filename - + # img=HT.Image('/image/'+filename+'.gif',border=0,alt='Histogram of Permutation Test') # self.suggestive = self.perm_output[int(self.nperm*0.37-1)] @@ -2609,9 +2609,9 @@ class MarkerRegression(object): # permutation.append(HT.TR(HT.TD(img)), # HT.TR(HT.TD('')), # HT.TR(HT.TD('Total of %d permutations'%self.nperm))) - + # return permutation - + def permutationTextFile(self): filename= webqtlUtil.genRandStr("Reg_") fpText = open('%s.txt' % (webqtlConfig.TMPDIR+filename), 'wb') @@ -2624,12 +2624,12 @@ class MarkerRegression(object): '    Significant LRS =%3.2f\n'%self.significant, HT.BR(), '    Highly Significant LRS =%3.2f\n' % self.highlysignificant) - + for lrs_value in self.perm_output: fpText.write(str(lrs_value) + "\n") - + textUrl = HT.Href(text = 'Download Permutation Results', url= '/tmp/'+filename+'.txt', target = "_blank", Class='fs12 fwn') - + return textUrl def geneTable(self, geneCol, refGene=None): @@ -2654,7 +2654,7 @@ class MarkerRegression(object): else: gene_table = "" - return gene_table + return gene_table def getLiteratureCorrelation(cursor,geneId1=None,geneId2=None): if not geneId1 or not geneId2: @@ -2954,4 +2954,4 @@ And by voluntary, according to HFG when I talked to him, they have a choice betw sortby = ("", "") - return sortby \ No newline at end of file + return sortby diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 52fe2e34..9941a4d3 100755 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -26,7 +26,6 @@ from MySQLdb import escape_string as escape # Instead of importing HT we're going to build a class below until we can eliminate it # from htmlgen import HTMLgen2 as HT -from base import webqtlConfig from utility.benchmark import Bench from base.data_set import create_dataset from base.trait import GeneralTrait diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 35f7fe5f..e80cf191 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -166,7 +166,7 @@ class ShowTrait(object): return False def check_pylmm_rqtl(): - if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.NEWGENODIR+self.dataset.group.name+".json") > 0): + if os.path.isfile(webqtlConfig.GENODIR+self.dataset.group.name+".geno") and (os.path.getsize(webqtlConfig.JSON_GENODIR+self.dataset.group.name+".json") > 0): return True else: return False -- cgit v1.2.3 From 31a1c126185adb747028d035717ad72728a52b0e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 11:25:10 +0000 Subject: -a [PATCH 038/100] Fix compile errors --- wqflask/base/data_set.py | 6 +++--- wqflask/utility/tools.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 6527657a..8b2a9f01 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -44,7 +44,7 @@ from dbFunction import webqtlDatabaseFunction from utility import webqtlUtil from utility.benchmark import Bench from utility import chunks -from utility.tools import locate, locate_without_error +from utility.tools import locate, locate_ignore_error from maintenance import get_group_samplelists @@ -405,8 +405,8 @@ class DatasetGroup(object): else: print("Cache not hit") - genotype_fn = locate_without_error(self.name+".geno",'genotype') - mapping_fn = locate_without_error(self.name+".fam",'mapping') + genotype_fn = locate_ignore_error(self.name+".geno",'genotype') + mapping_fn = locate_ignore_error(self.name+".fam",'mapping') if mapping_fn: self.samplelist = get_group_samplelists.get_samplelist("plink", mapping_fn) elif genotype_fn: diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index c0f6a49a..67c5128a 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -1,3 +1,4 @@ + # Tools/paths finder resolves external paths from settings and/or environment # variables @@ -83,9 +84,10 @@ def assert_dir(dir): if not valid_path(dir): raise Exception("ERROR: can not find directory "+dir) return dir - + def mk_dir(dir): - os.makedirs(dir) + if not valid_path(dir): + os.makedirs(dir) return assert_dir(dir) def locate(name, subdir=None): -- cgit v1.2.3 From c5dbe982272ef95c6598c36172444b1e8263c01a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 11:39:05 +0000 Subject: -a [PATCH 039/100] Removing commented code --- wqflask/base/data_set.py | 228 ----------------------------------------------- 1 file changed, 228 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8b2a9f01..52ac95f0 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -58,25 +58,13 @@ DS_NAME_MAP = {} def create_dataset(dataset_name, dataset_type = None, get_samplelist = True): if not dataset_type: dataset_type = Dataset_Getter(dataset_name) - #dataset_type = get_dataset_type_from_json(dataset_name) print("dataset_type is:", dataset_type) - #query = """ - # SELECT DBType.Name - # FROM DBList, DBType - # WHERE DBList.Name = '{}' and - # DBType.Id = DBList.DBTypeId - # """.format(escape(dataset_name)) - #dataset_type = g.db.execute(query).fetchone().Name - dataset_ob = DS_NAME_MAP[dataset_type] dataset_class = globals()[dataset_ob] return dataset_class(dataset_name, get_samplelist) - -#def get_dataset_type_from_json(dataset_name): - class Dataset_Types(object): def __init__(self): @@ -90,8 +78,6 @@ class Dataset_Types(object): for group in data['datasets'][species]: for dataset_type in data['datasets'][species][group]: for dataset in data['datasets'][species][group][dataset_type]: - #print("dataset is:", dataset) - short_dataset_name = dataset[1] if dataset_type == "Phenotypes": new_type = "Publish" @@ -107,10 +93,6 @@ class Dataset_Types(object): # Do the intensive work at startup one time only Dataset_Getter = Dataset_Types() -# -#print("Running at startup:", get_dataset_type_from_json("HBTRC-MLPFC_0611")) - - def create_datasets_list(): key = "all_datasets" result = Redis.get(key) @@ -215,13 +197,6 @@ class Markers(object): #del self.markers[i] self.markers = filtered_markers - - #for i, marker in enumerate(self.markers): - # if not 'p_value' in marker: - # #print("self.markers[i]", self.markers[i]) - # del self.markers[i] - # #self.markers.remove(self.markers[i]) - class HumanMarkers(Markers): def __init__(self, name, specified_markers = []): @@ -249,26 +224,8 @@ class HumanMarkers(Markers): def add_pvalues(self, p_values): - #for marker, p_value in itertools.izip(self.markers, p_values): - # if marker['Mb'] <= 0 and marker['chr'] == 0: - # continue - # marker['p_value'] = p_value - # print("p_value is:", marker['p_value']) - # marker['lod_score'] = -math.log10(marker['p_value']) - # #Using -log(p) for the LRS; need to ask Rob how he wants to get LRS from p-values - # marker['lrs_value'] = -math.log10(marker['p_value']) * 4.61 - - #print("p_values2:", pf(p_values)) super(HumanMarkers, self).add_pvalues(p_values) - #with Bench("deleting markers"): - # markers = [] - # for marker in self.markers: - # if not marker['Mb'] <= 0 and not marker['chr'] == 0: - # markers.append(marker) - # self.markers = markers - - class DatasetGroup(object): """ @@ -311,12 +268,6 @@ class DatasetGroup(object): def datasets(self): key = "group_dataset_menu:v2:" + self.name print("key is2:", key) - #with Bench("Loading cache"): - # result = Redis.get(key) - #if result: - # self._datasets = pickle.loads(result) - # return self._datasets - dataset_menu = [] print("[tape4] webqtlConfig.PUBLICTHRESH:", webqtlConfig.PUBLICTHRESH) print("[tape4] type webqtlConfig.PUBLICTHRESH:", type(webqtlConfig.PUBLICTHRESH)) @@ -425,11 +376,6 @@ class DatasetGroup(object): def read_genotype_file(self): '''Read genotype from .geno file instead of database''' - #if self.group == 'BXD300': - # self.group = 'BXD' - # - #assert self.group, "self.group needs to be set" - #genotype_1 is Dataset Object without parents and f1 #genotype_2 is Dataset Object with parents and f1 (not for intercross) @@ -446,40 +392,16 @@ class DatasetGroup(object): #determine default genotype object if self.incparentsf1 and genotype_1.type != "intercross": - #self.genotype = genotype_2 genotype = genotype_2 else: self.incparentsf1 = 0 - #self.genotype = genotype_1 genotype = genotype_1 - #self.samplelist = list(self.genotype.prgy) self.samplelist = list(genotype.prgy) return genotype -#class DataSets(object): -# """Builds a list of DataSets""" -# -# def __init__(self): -# self.datasets = list() -# - - - #query = """SELECT Name FROM ProbeSetFreeze - # UNION - # SELECT Name From PublishFreeze - # UNION - # SELECT Name From GenoFreeze""" - # - #for result in g.db.execute(query).fetchall(): - # dataset = DataSet(result.Name) - # self.datasets.append(dataset) - -#ds = DataSets() -#print("[orange] ds:", ds.datasets) - class DataSet(object): """ DataSet class defines a dataset in webqtl, can be either Microarray, @@ -512,32 +434,11 @@ class DataSet(object): """Gets overridden later, at least for Temp...used by trait's get_given_name""" return None - #@staticmethod - #def get_by_trait_id(trait_id): - # """Gets the dataset object given the trait id""" - # - # - # - # name = g.db.execute(""" SELECT - # - # """) - # - # return DataSet(name) - # Delete this eventually @property def riset(): Weve_Renamed_This_As_Group - - #@property - #def group(self): - # if not self._group: - # self.get_group() - # - # return self._group - - def retrieve_other_names(self): """ If the data set name parameter is not found in the 'Name' field of the data set table, @@ -609,21 +510,6 @@ class DataSet(object): number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - - #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId - #tempTable = None - #if GeneId and db.type == "ProbeSet": - # if method == "3": - # tempTable = self.getTempLiteratureTable(species=species, - # input_species_geneid=GeneId, - # returnNumber=returnNumber) - # - # if method == "4" or method == "5": - # tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol, - # TissueProbeSetFreezeId=tissueProbeSetFreezeId, - # method=method, - # returnNumber=returnNumber) - if self.type == "Publish": dataset_type = "Phenotype" else: @@ -1013,83 +899,11 @@ class MrnaAssayDataSet(DataSet): and ProbeSetFreezeId = {} """.format(escape(str(self.id))) results = g.db.execute(query).fetchall() - #print("After get_trait_list query") trait_data = {} for trait in results: - #print("Retrieving sample_data for ", trait[0]) trait_data[trait[0]] = self.retrieve_sample_data(trait[0]) - #print("After retrieve_sample_data") return trait_data - #def get_trait_data(self): - # self.samplelist = self.group.samplelist + self.group.parlist + self.group.f1list - # query = """ - # SELECT Strain.Name, Strain.Id FROM Strain, Species - # WHERE Strain.Name IN {} - # and Strain.SpeciesId=Species.Id - # and Species.name = '{}' - # """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) - # results = dict(g.db.execute(query).fetchall()) - # sample_ids = [results[item] for item in self.samplelist] - # - # # MySQL limits the number of tables that can be used in a join to 61, - # # so we break the sample ids into smaller chunks - # # Postgres doesn't have that limit, so we can get rid of this after we transition - # chunk_size = 50 - # number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - # trait_sample_data = [] - # for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - # - # #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId - # #tempTable = None - # #if GeneId and db.type == "ProbeSet": - # # if method == "3": - # # tempTable = self.getTempLiteratureTable(species=species, - # # input_species_geneid=GeneId, - # # returnNumber=returnNumber) - # # - # # if method == "4" or method == "5": - # # tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol, - # # TissueProbeSetFreezeId=tissueProbeSetFreezeId, - # # method=method, - # # returnNumber=returnNumber) - # - # temp = ['T%s.value' % item for item in sample_ids_step] - # query = "SELECT {}.Name,".format(escape(self.type)) - # data_start_pos = 1 - # query += string.join(temp, ', ') - # query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(self.type, - # self.type, - # self.type)) - # - # for item in sample_ids_step: - # query += """ - # left join {}Data as T{} on T{}.Id = {}XRef.DataId - # and T{}.StrainId={}\n - # """.format(*mescape(self.type, item, item, self.type, item, item)) - # - # query += """ - # WHERE {}XRef.{}FreezeId = {}Freeze.Id - # and {}Freeze.Name = '{}' - # and {}.Id = {}XRef.{}Id - # order by {}.Id - # """.format(*mescape(self.type, self.type, self.type, self.type, - # self.name, self.type, self.type, self.type, self.type)) - # results = g.db.execute(query).fetchall() - # trait_sample_data.append(results) - # - # trait_count = len(trait_sample_data[0]) - # self.trait_data = collections.defaultdict(list) - # - # # put all of the separate data together into a dictionary where the keys are - # # trait names and values are lists of sample values - # for trait_counter in range(trait_count): - # trait_name = trait_sample_data[0][trait_counter][0] - # for chunk_counter in range(int(number_chunks)): - # self.trait_data[trait_name] += ( - # trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) - - def get_trait_info(self, trait_list=None, species=''): # Note: setting trait_list to [] is probably not a great idea. @@ -1178,22 +992,9 @@ class MrnaAssayDataSet(DataSet): result = g.db.execute(query).fetchone() if result: - #if result[0] and result[1]: - # lrs_chr = result[0] - # lrs_mb = result[1] lrs_chr, lrs_mb = result #XZ: LRS_location_value is used for sorting lrs_location_value = self.convert_location_to_value(lrs_chr, lrs_mb) - - #try: - # lrs_location_value = int(lrs_chr)*1000 + float(lrs_mb) - #except: - # if lrs_chr.upper() == 'X': - # lrs_location_value = 20*1000 + float(lrs_mb) - # else: - # lrs_location_value = (ord(str(LRS_chr).upper()[0])*1000 + - # float(lrs_mb)) - this_trait.LRS_score_repr = '%3.1f' % this_trait.lrs this_trait.LRS_score_value = this_trait.lrs this_trait.LRS_location_repr = 'Chr%s: %.6f' % (lrs_chr, float(lrs_mb)) @@ -1259,35 +1060,6 @@ class MrnaAssayDataSet(DataSet): return dict(results) - #def retrieve_gene_symbols(self): - # query = """ - # select ProbeSet.Name, ProbeSet.Symbol, ProbeSet.GeneId - # from ProbeSet,ProbeSetXRef - # where ProbeSetXRef.ProbeSetFreezeId = %s and - # ProbeSetXRef.ProbeSetId=ProbeSet.Id; - # """ % (self.id) - # results = g.db.execute(query).fetchall() - # symbol_dict = {} - # for item in results: - # symbol_dict[item[0]] = item[1] - # return symbol_dict - # - #def retrieve_gene_ids(self): - # query = """ - # select ProbeSet.Name, ProbeSet.GeneId - # from ProbeSet,ProbeSetXRef - # where ProbeSetXRef.ProbeSetFreezeId = %s and - # ProbeSetXRef.ProbeSetId=ProbeSet.Id; - # """ % (self.id) - # return process_and_run_query(query) - # results = g.db.execute(query).fetchall() - # symbol_dict = {} - # for item in results: - # symbol_dict[item[0]] = item[1] - # return symbol_dict - - - class TempDataSet(DataSet): '''Temporary user-generated data set''' -- cgit v1.2.3 From 1e9d7476697bf8b60ab1004f25b295f81d94de49 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Wed, 24 Feb 2016 11:49:14 +0000 Subject: When a sample does not exist we should not set sample_data to None. Better to leave it out so it never gets processed again. --- wqflask/wqflask/correlation/show_corr_results.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 06b4860e..dd661092 100755 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -708,14 +708,11 @@ class CorrelationResults(object): for sample in sample_names: if sample not in excluded_samples: - print("Looking for",sample,"in",start_vars) + # print("Looking for",sample,"in",start_vars) value = start_vars.get('value:' + sample) if value: if not value.strip().lower() == 'x': self.sample_data[str(sample)] = float(value) - else: - self.sample_data[str(sample)] = None - ##XZ, 12/16/2008: the input geneid is of mouse type #def checkForLitInfo(self,geneId): -- cgit v1.2.3 From 575e00b1061c58952cba38af5ab078ca0081b4d7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 16:34:52 +0000 Subject: [PATCH 041/100] Fixes for running tools --- etc/default_settings.py | 6 +++--- wqflask/utility/tools.py | 4 ++-- wqflask/wqflask/marker_regression/marker_regression.py | 3 +-- wqflask/wqflask/marker_regression/marker_regression_gn1.py | 7 ++++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/etc/default_settings.py b/etc/default_settings.py index 6a218f26..60a3a7b4 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -24,6 +24,6 @@ SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a # Path overrides for Genenetwork GENENETWORK_FILES = "../../gn2_data" -PYLMM_COMMAND = os.popen("which pylmm_redis").read() -PLINK_COMMAND = os.popen("which plink2").read() -GEMMA_COMMAND = os.popen("which gemma").read() +PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) +PLINK_COMMAND = str.strip(os.popen("which plink2").read()) +GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 67c5128a..51189fa3 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -133,7 +133,7 @@ def tempdir(): # Cached values PYLMM_COMMAND = pylmm_command() -GEMMA_COMMAND = pylmm_command() -PLINK_COMMAND = pylmm_command() +GEMMA_COMMAND = gemma_command() +PLINK_COMMAND = plink_command() FLAT_FILES = flat_files() TEMPDIR = tempdir() diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 265f9473..910d0c3c 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -937,8 +937,7 @@ class MarkerRegression(object): Redis.expire(key, 60*60) print("before printing command") - command = PYLMM_COMMAND + ' --key {} --species {}'.format(key, - "other") + command = PYLMM_COMMAND + ' --key {} --species {}'.format(key, "other") print("command is:", command) print("after printing command") diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index decde579..4edc1891 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -28,6 +28,7 @@ import time import string from math import * import piddle as pid +import piddlePIL as pil import sys,os import cPickle import httplib, urllib @@ -551,7 +552,7 @@ class MarkerRegression(object): # showLocusForm = webqtlUtil.genRandStr("fm_") #else: showLocusForm = "" - intCanvas = pid.PILCanvas(size=(self.graphWidth, self.graphHeight)) + intCanvas = pil.PILCanvas(size=(self.graphWidth, self.graphHeight)) gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) self.gifmap = gifmap.__str__() @@ -563,7 +564,7 @@ class MarkerRegression(object): #Scales plot differently for high resolution if self.draw2X: - intCanvasX2 = pid.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2)) + intCanvasX2 = pil.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2)) gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2) intCanvasX2.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename+"X2"), format='png') #DLintImgX2=HT.Href(text='Download',url = '/image/'+self.filename+'X2.png', Class='smallsize', target='_blank') @@ -2582,7 +2583,7 @@ class MarkerRegression(object): ######################################### # Permutation Graph ######################################### - myCanvas = pid.PILCanvas(size=(400,300)) + myCanvas = pil.PILCanvas(size=(400,300)) if 'lod_score' in self.qtlresults[0] and self.LRS_LOD == "LRS": perm_output = [value*4.16 for value in self.perm_output] elif 'lod_score' not in self.qtlresults[0] and self.LRS_LOD == "LOD": -- cgit v1.2.3 From 9395e9f3cc2d06e069c5de797b2d6e8e59cfba7c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 24 Feb 2016 17:00:04 +0000 Subject: [PATCH 042/100] Sanitizing locations --- wqflask/utility/tools.py | 5 ++--- wqflask/wqflask/marker_regression/marker_regression.py | 13 ++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 51189fa3..b8eff12a 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -1,4 +1,3 @@ - # Tools/paths finder resolves external paths from settings and/or environment # variables @@ -105,9 +104,9 @@ def locate(name, subdir=None): print("Found: file "+lookfor+"\n") return lookfor else: - raise IOError("Can not locate "+lookfor) + raise Exception("Can not locate "+lookfor) if subdir: sys.stderr.write(subdir) - raise IOError("Can not locate "+name+" in "+base) + raise Exception("Can not locate "+name+" in "+base) def locate_ignore_error(name, subdir=None): """ diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 910d0c3c..e9cfb06d 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -41,7 +41,8 @@ from wqflask.marker_regression import gemma_mapping #from wqflask.marker_regression import plink_mapping #from wqflask.marker_regression import rqtl_mapping -from utility.tools import locate, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND +from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND +from utility.external import shell class MarkerRegression(object): @@ -413,8 +414,8 @@ class MarkerRegression(object): write_cross = ro.r["write.cross"] # Map the write.cross function GENOtoCSVR = ro.r["GENOtoCSVR"] # Map the GENOtoCSVR function - genofilelocation = webqtlConfig.HTMLPATH + "genotypes/" + self.dataset.group.name + ".geno" - crossfilelocation = webqtlConfig.HTMLPATH + "genotypes/" + self.dataset.group.name + ".cross" + genofilelocation = locate(self.dataset.group.name + ".geno", "genotype") + crossfilelocation = locate(self.dataset.group.name + ".cross", "genotype") #print("Conversion of geno to cross at location:", genofilelocation, " to ", crossfilelocation) @@ -860,9 +861,7 @@ class MarkerRegression(object): Redis.expire(key, 60*60) command = PYLMM_COMMAND+' --key {} --species {}'.format(key,"other") - - os.system(command) - + shell(command) json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60) results = json.loads(json_results[1]) @@ -941,7 +940,7 @@ class MarkerRegression(object): print("command is:", command) print("after printing command") - os.system(command) + shell(command) #t_stats, p_values = lmm.run(key) #lmm.run(key) -- cgit v1.2.3 From e42b21c66b09128ac6a9c18ca018287516d8c309 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Wed, 24 Feb 2016 17:41:55 +0000 Subject: Minor fixes --- wqflask/utility/external.py | 9 +++++++++ wqflask/wqflask/marker_regression/marker_regression_gn1.py | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 wqflask/utility/external.py diff --git a/wqflask/utility/external.py b/wqflask/utility/external.py new file mode 100644 index 00000000..bd8cf584 --- /dev/null +++ b/wqflask/utility/external.py @@ -0,0 +1,9 @@ +# Call external program + +import os +import sys +import subprocess + +def shell(command): + if not subprocess.call(command, shell=True): + raise Exception("ERROR: failed on "+command) diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 4edc1891..a68c9586 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -29,6 +29,7 @@ import string from math import * import piddle as pid import piddlePIL as pil +from piddle import Font import sys,os import cPickle import httplib, urllib @@ -941,7 +942,7 @@ class MarkerRegression(object): bootScale = bootScale[:-1] + [highestPercent] bootOffset = 50*fontZoom - bootScaleFont=pid.Font(ttf="verdana",size=13*fontZoom,bold=0) + bootScaleFont=Font(ttf="verdana",size=13*fontZoom,bold=0) canvas.drawRect(canvas.size[0]-bootOffset,yZero-bootHeightThresh,canvas.size[0]-bootOffset-15*zoom,yZero,fillColor = pid.yellow) canvas.drawLine(canvas.size[0]-bootOffset+4, yZero, canvas.size[0]-bootOffset, yZero, color=pid.black) canvas.drawString('0%' ,canvas.size[0]-bootOffset+10,yZero+5,font=bootScaleFont,color=pid.black) @@ -2259,7 +2260,7 @@ class MarkerRegression(object): chrFontZoom = 2 else: chrFontZoom = 1 - chrLabelFont=pid.Font(ttf="verdana",size=24*chrFontZoom,bold=0) + chrLabelFont=Font(ttf="verdana",size=24*chrFontZoom,bold=0) for i, _chr in enumerate(self.genotype): if (i % 2 == 0): -- cgit v1.2.3 From a4686a6e2fc96e3fc59d99823a25662fec793d2b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 25 Feb 2016 08:59:57 +0000 Subject: [PATCH 045/100] Reverted on previous changes. With a corrected python2-piddle module the thing works. Turns out that the version we are using is not the same as the public one. Despite the same version number. --- wqflask/wqflask/marker_regression/marker_regression_gn1.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index a68c9586..8a593ec4 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -28,8 +28,8 @@ import time import string from math import * import piddle as pid -import piddlePIL as pil -from piddle import Font +# import piddlePIL as pil +# from piddle import Font import sys,os import cPickle import httplib, urllib @@ -553,7 +553,7 @@ class MarkerRegression(object): # showLocusForm = webqtlUtil.genRandStr("fm_") #else: showLocusForm = "" - intCanvas = pil.PILCanvas(size=(self.graphWidth, self.graphHeight)) + intCanvas = pid.PILCanvas(size=(self.graphWidth, self.graphHeight)) gifmap = self.plotIntMapping(intCanvas, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm) self.gifmap = gifmap.__str__() @@ -565,7 +565,7 @@ class MarkerRegression(object): #Scales plot differently for high resolution if self.draw2X: - intCanvasX2 = pil.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2)) + intCanvasX2 = pid.PILCanvas(size=(self.graphWidth*2,self.graphHeight*2)) gifmapX2 = self.plotIntMapping(intCanvasX2, startMb = self.startMb, endMb = self.endMb, showLocusForm= showLocusForm, zoom=2) intCanvasX2.save(os.path.join(webqtlConfig.GENERATED_IMAGE_DIR, self.filename+"X2"), format='png') #DLintImgX2=HT.Href(text='Download',url = '/image/'+self.filename+'X2.png', Class='smallsize', target='_blank') @@ -2260,7 +2260,7 @@ class MarkerRegression(object): chrFontZoom = 2 else: chrFontZoom = 1 - chrLabelFont=Font(ttf="verdana",size=24*chrFontZoom,bold=0) + chrLabelFont=pid.Font(ttf="verdana",size=24*chrFontZoom,bold=0) for i, _chr in enumerate(self.genotype): if (i % 2 == 0): @@ -2584,7 +2584,7 @@ class MarkerRegression(object): ######################################### # Permutation Graph ######################################### - myCanvas = pil.PILCanvas(size=(400,300)) + myCanvas = pid.PILCanvas(size=(400,300)) if 'lod_score' in self.qtlresults[0] and self.LRS_LOD == "LRS": perm_output = [value*4.16 for value in self.perm_output] elif 'lod_score' not in self.qtlresults[0] and self.LRS_LOD == "LOD": -- cgit v1.2.3 From 2e039da988b585fde5a09e94d7fd7b56262752f2 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Thu, 25 Feb 2016 11:22:02 +0000 Subject: external program should fail on return value not zero --- wqflask/utility/external.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/external.py b/wqflask/utility/external.py index bd8cf584..50afea08 100644 --- a/wqflask/utility/external.py +++ b/wqflask/utility/external.py @@ -5,5 +5,5 @@ import sys import subprocess def shell(command): - if not subprocess.call(command, shell=True): + if subprocess.call(command, shell=True) != 0: raise Exception("ERROR: failed on "+command) -- cgit v1.2.3 From ec6c3158a48a7d81596c848cb8b7f3a487bcc28a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 25 Feb 2016 12:43:49 +0000 Subject: [PATCH 047/100] This commit moves the image generation dir *outside* the git repository. --- wqflask/wqflask/marker_regression/marker_regression_gn1.py | 12 ------------ wqflask/wqflask/templates/marker_regression_gn1.html | 2 +- wqflask/wqflask/views.py | 7 ++++++- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 8a593ec4..245e3f04 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -28,8 +28,6 @@ import time import string from math import * import piddle as pid -# import piddlePIL as pil -# from piddle import Font import sys,os import cPickle import httplib, urllib @@ -40,21 +38,11 @@ from htmlgen import HTMLgen2 as HT from base import webqtlConfig from base.GeneralObject import GeneralObject -#from base.webqtlTrait import webqtlTrait -#from base.templatePage import templatePage from utility import webqtlUtil from utility import helper_functions from utility import Plot -#from utility.THCell import THCell -#from utility.TDCell import TDCell from wqflask.interval_analyst import GeneUtil -#from dbFunction import webqtlDatabaseFunction - -#import logging -#logging.basicConfig(filename="/tmp/gn_leiyan.log", level=logging.INFO) -#_log = logging.getLogger("gn\web\webqtl\intervalMapping\IntervalMappingPage.py") - ######################################### # Inteval Mapping Plot Page ######################################### diff --git a/wqflask/wqflask/templates/marker_regression_gn1.html b/wqflask/wqflask/templates/marker_regression_gn1.html index f3f33fad..311429f8 100644 --- a/wqflask/wqflask/templates/marker_regression_gn1.html +++ b/wqflask/wqflask/templates/marker_regression_gn1.html @@ -143,7 +143,7 @@
{{ gifmap|safe }} - + {% if additiveChecked|upper == "ON" %}
A positive additive coefficient (green line) indicates that {{ dataset.group.parlist[1] }} alleles increase trait values. In contrast, a negative additive coefficient (orange line) indicates that {{ dataset.group.parlist[0] }} alleles increase trait values. diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 929da649..b35bbbf9 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -30,7 +30,7 @@ import sqlalchemy from wqflask import app from flask import (render_template, request, make_response, Response, - Flask, g, config, jsonify, redirect, url_for) + Flask, g, config, jsonify, redirect, url_for, send_from_directory) from wqflask import search_results from wqflask import gsearch @@ -53,6 +53,7 @@ from utility import temp_data from utility.tools import TEMPDIR from base import webqtlFormData +from base.webqtlConfig import GENERATED_IMAGE_DIR from utility.benchmark import Bench from pprint import pformat as pf @@ -173,6 +174,10 @@ def docedit(): doc = docs.Docs(request.args['entry']) return render_template("docedit.html", **doc.__dict__) +@app.route('/generated/') +def generated_file(filename): + return send_from_directory(GENERATED_IMAGE_DIR,filename) + @app.route("/help") def help(): doc = docs.Docs("help") -- cgit v1.2.3 From f755e7d4b478e8acc9613d663e40c2a0aca7c18f Mon Sep 17 00:00:00 2001 From: pjotrp Date: Thu, 25 Feb 2016 12:53:30 +0000 Subject: Update INSTALL.md --- INSTALL.md | 49 ++++++++++++++++++------------------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 3ef2d5b3..3e6d2f02 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,17 +1,25 @@ # INSTALL Genenetwork2 (GN2) +## Guix installs + +The recommended way of installing GN2 is through GNU Guix. + ## Use a Docker image A Docker image can be generated from [here](https://github.com/lomereiter/gn2-docker). -## Fetch GN2 from github +(no longer up-to-date). + +## Install GN2 from source + +Currently a difficult exercise. More information coming soon. Clone the repository (currently ~800Mb) to local git clone git@github.com:genenetwork2/genenetwork2.git -## Dependencies +### Dependencies GN2 requires @@ -19,7 +27,7 @@ GN2 requires * redis-server * mysql-server -## Required python modules +### Required python modules Install the following python modules (it is probably wise to use a local Python with environment for this) @@ -32,19 +40,21 @@ Python with environment for this) * pp * Flask-SQLAlchemy -## Set up local file settings.py +(not complete) + +### Set up local file settings.py ```python LOGFILE = """/tmp/flask_gn_log""" -#This is needed because Flask turns key errors into a -#400 bad request response with no exception/log + #This is needed because Flask turns key errors into a + #400 bad request response with no exception/log TRAP_BAD_REQUEST_ERRORS = True DB_URI = """mysql://gn2:password@localhost/db_webqtl""" SQLALCHEMY_DATABASE_URI = 'mysql://gn2:password@localhost/db_webqtl' -# http://pythonhosted.org/Flask-Security/configuration.html + # http://pythonhosted.org/Flask-Security/configuration.html SECURITY_CONFIRMABLE = True SECURITY_TRACKABLE = True SECURITY_REGISTERABLE = True @@ -59,30 +69,7 @@ SERVER_PORT = 5051 SECRET_HMAC_CODE = '*' ``` -```sh -# Use a working copy of python -export python=$HOME/ve27/bin/python -export WQFLASK_SETTINGS=$HOME/settings.py -source /home/pjotr/ve27/bin/activate -cd genenetwork2/wqflask -$python ./runserver.py - -or - -$python ./secure_server.py -``` - -## Running tools - -### pylmm - -To run pylmm check out the repository at https://github.com/genenetwork/pylmm_gn2. - -Next update the setting.py file to point at the tree - -GN2 can locate PYLMM through PYLMM_PATH in setting.py (or in ENV) - - PYLMM_PATH = '/home/test/opensource/python/pylmm_gn2' +(not complete) ## Other information -- cgit v1.2.3 From 84fef98c7929d3d42e1a7efa33bff74cbbb468c7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 25 Feb 2016 12:57:53 +0000 Subject: [PATCH 049/100] SNP_PATH: removal --- wqflask/wqflask/marker_regression/marker_regression.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index e9cfb06d..1b32b05b 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -1069,7 +1069,11 @@ class MarkerRegression(object): return trimmed_genotype_data def create_snp_iterator_file(group): - plink_file_base = os.path.join(webqtlConfig.PYLMM_PATH, group) + """ + This function is only called by main below + """ + raise Exception("Paths are undefined here") + plink_file_base = os.path.join(webqtlConfig.TMPDIR, group) plink_input = input.plink(plink_file_base, type='b') data = dict(plink_input = list(plink_input), @@ -1128,7 +1132,5 @@ def get_markers_from_csv(included_markers, p_values, group_name): return markers -if __name__ == '__main__': - import cPickle as pickle - import gzip - create_snp_iterator_file("HLC") + if __name__ == '__main__': + import cPickle as pickle -- cgit v1.2.3 From 0ce72520d5b8f6ef38cfb43637b39ce209d5c7f7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 25 Feb 2016 13:19:40 +0000 Subject: [PATCH 050/100] Using out-of-tree storage. Added comments for plink, gemma. --- wqflask/wqflask/marker_regression/gemma_mapping.py | 7 ++-- .../wqflask/marker_regression/marker_regression.py | 39 +++------------------- wqflask/wqflask/views.py | 2 +- 3 files changed, 10 insertions(+), 38 deletions(-) diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index cfcd4783..ae426621 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -12,8 +12,11 @@ def run_gemma(this_dataset, samples, vals): gen_pheno_txt_file(this_dataset, samples, vals) - os.chdir(GEMMA_PATH) + # Don't do this! + # os.chdir("{}gemma".format(webqtlConfig.HTMLPATH)) + # use GEMMA_RUN in the next one, create a unique temp file + gemma_command = GEMMA_COMMAND + ' -bfile %s/%s -k %s/output/%s.cXX.txt -lmm 1 -o %s_output' % (GEMMA_PATH, this_dataset.group.name, GEMMA_PATH, @@ -46,4 +49,4 @@ def parse_gemma_output(this_dataset): p_values.append(float(line.split("\t")[10])) #print("p_values: ", p_values) - return included_markers, p_values \ No newline at end of file + return included_markers, p_values diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 1b32b05b..d1c0d9bc 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -30,16 +30,12 @@ from flask import Flask, g from base.trait import GeneralTrait from base import data_set from base import species -# from base import webqtlConfig from utility import webqtlUtil from utility import helper_functions from utility import Plot, Bunch from utility import temp_data from utility.benchmark import Bench from wqflask.marker_regression import gemma_mapping -#from wqflask.marker_regression import qtl_reaper_mapping -#from wqflask.marker_regression import plink_mapping -#from wqflask.marker_regression import rqtl_mapping from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND from utility.external import shell @@ -297,22 +293,14 @@ class MarkerRegression(object): included_markers, p_values = self.parse_gemma_output() self.dataset.group.get_specified_markers(markers = included_markers) - - #for marker in self.dataset.group.markers.markers: - # if marker['name'] not in included_markers: - # print("marker:", marker) - # self.dataset.group.markers.markers.remove(marker) - # #del self.dataset.group.markers.markers[marker] - self.dataset.group.markers.add_pvalues(p_values) - return self.dataset.group.markers.markers - def parse_gemma_output(self): included_markers = [] p_values = [] - with open("/home/zas1024/gene/web/gemma/output/{}_output.assoc.txt".format(self.dataset.group.name)) as output_file: + # Use a temporary file name here! + with open(webqtlConfig.GENERATED_TEXT_DIR+"/{}_output.assoc.txt".format(self.dataset.group.name)) as output_file: for line in output_file: if line.startswith("chr"): continue @@ -325,31 +313,12 @@ class MarkerRegression(object): def gen_pheno_txt_file(self): """Generates phenotype file for GEMMA""" - - #with open("/home/zas1024/gene/web/gemma/tmp_pheno/{}.txt".format(filename), "w") as outfile: - # for sample, i in enumerate(self.samples): - # print("sample:" + str(i)) - # print("self.vals[i]:" + str(self.vals[sample])) - # outfile.write(str(i) + "\t" + str(self.vals[sample]) + "\n") - - with open("/home/zas1024/gene/web/gemma/{}.fam".format(self.dataset.group.name), "w") as outfile: + with open(webqtlConfig.GENERATED_TEXT_DIR+"{}.fam".format(self.dataset.group.name), "w") as outfile: for i, sample in enumerate(self.samples): outfile.write(str(sample) + " " + str(sample) + " 0 0 0 " + str(self.vals[i]) + "\n") - - #def gen_plink_for_gemma(self, filename): - # - # make_bed = "/home/zas1024/plink/plink --file /home/zas1024/plink/%s --noweb --no-fid --no-parents --no-sex --no-pheno --pheno %s%s.txt --out %s%s --make-bed" % (webqtlConfig.HTMLPATH, - # webqtlConfig.HTMLPATH, - # self.dataset.group.name, - # webqtlConfig.TMPDIR, - # filename, - # webqtlConfig.TMPDIR, - # filename) - # - # def run_rqtl_plink(self): - os.chdir("/home/zas1024/plink") + # os.chdir("/home/zas1024/plink") never do this!! output_filename = webqtlUtil.genRandStr("%s_%s_"%(self.dataset.group.name, self.this_trait.name)) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b35bbbf9..4979d45d 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -458,7 +458,7 @@ def export_pdf(): svg_xml = request.form.get("data", "Invalid data") print("svg_xml:", svg_xml) filename = request.form.get("filename", "interval_map_pdf") - filepath = "/home/zas1024/gene/wqflask/output/"+filename + filepath = GENERATED_IMAGE_DIR+filename pdf_file = cairosvg.svg2pdf(bytestring=svg_xml) response = Response(pdf_file, mimetype="application/pdf") response.headers["Content-Disposition"] = "attachment; filename=%s"%filename -- cgit v1.2.3 From 33bc6a57394f9fd3242a33e7cb29f01baa0692b1 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Thu, 25 Feb 2016 13:32:58 +0000 Subject: More hard-coded --- wqflask/maintenance/gen_select_dataset.py | 3 +-- wqflask/wqflask/heatmap/heatmap.py | 5 +++-- wqflask/wqflask/marker_regression/marker_regression.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index e080050e..489d291f 100755 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -36,8 +36,7 @@ from __future__ import print_function, division #print("cdict is:", cdict) import sys -sys.path.append("/home/zas1024/") -import zach_settings +import zach_settings # no hard code paths! import MySQLdb diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index 61847bc3..2445b37f 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -31,6 +31,7 @@ from base import species from utility import helper_functions from utility import Plot, Bunch from utility import temp_data +from utility.tools import PYLMM_COMMAND from MySQLdb import escape_string as escape @@ -212,7 +213,7 @@ class Heatmap(object): #Redis.expire(key, 60*60) #print("before printing command") # - #command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key, + #command = 'python lmm.py --key {} --species {}'.format(key, # "other") #print("command is:", command) #print("after printing command") @@ -271,7 +272,7 @@ class Heatmap(object): Redis.expire(key, 60*60) print("before printing command") - command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key, + command = PYLMM_COMMAND+' --key {} --species {}'.format(key, "other") print("command is:", command) print("after printing command") diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index d1c0d9bc..abd1ce85 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -318,7 +318,7 @@ class MarkerRegression(object): outfile.write(str(sample) + " " + str(sample) + " 0 0 0 " + str(self.vals[i]) + "\n") def run_rqtl_plink(self): - # os.chdir("/home/zas1024/plink") never do this!! + # os.chdir("") never do this inside a webserver!! output_filename = webqtlUtil.genRandStr("%s_%s_"%(self.dataset.group.name, self.this_trait.name)) -- cgit v1.2.3 From d00ed869d6df48180c56bbf9bc9f39afa7f5d3fa Mon Sep 17 00:00:00 2001 From: pjotrp Date: Mon, 29 Feb 2016 10:53:02 +0000 Subject: r-wgcna: Fix paths --- wqflask/wqflask/views.py | 2 +- wqflask/wqflask/wgcna/wgcna_analysis.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 4979d45d..7f331492 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -98,7 +98,7 @@ def tmp_page(img_path): print("img_path:", img_path) initial_start_vars = request.form print("initial_start_vars:", initial_start_vars) - imgfile = open(webqtlConfig.TMPDIR + img_path, 'rb') + imgfile = open(GENERATED_IMAGE_DIR + img_path, 'rb') imgdata = imgfile.read() imgB64 = imgdata.encode("base64") bytesarray = array.array('B', imgB64) diff --git a/wqflask/wqflask/wgcna/wgcna_analysis.py b/wqflask/wqflask/wgcna/wgcna_analysis.py index 9e9f41bc..6901712b 100644 --- a/wqflask/wqflask/wgcna/wgcna_analysis.py +++ b/wqflask/wqflask/wgcna/wgcna_analysis.py @@ -6,7 +6,7 @@ import scipy as sp # SciPy import rpy2.robjects as ro # R Objects import rpy2.rinterface as ri -from base import webqtlConfig # For paths and stuff +from base.webqtlConfig import GENERATED_IMAGE_DIR from utility import webqtlUtil # Random number for the image import base64 @@ -127,7 +127,7 @@ class WGCNA(object): # The iconic WCGNA plot of the modules in the hanging tree self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" - self.results['imgloc'] = webqtlConfig.TMPDIR + self.results['imgurl'] + self.results['imgloc'] = GENERATED_IMAGE_DIR + self.results['imgurl'] r_png(self.results['imgloc'], width=1000, height=600) mergedColors = self.r_labels2colors(network[1]) self.r_plotDendroAndColors(network[5][0], mergedColors, "Module colors", dendroLabels = False, hang = 0.03, addGuide = True, guideHang = 0.05) -- cgit v1.2.3 From 9f3ac3290ab7c00705c88fb648ee981ebe50c45a Mon Sep 17 00:00:00 2001 From: pjotrp Date: Mon, 29 Feb 2016 11:00:44 +0000 Subject: Removed symlinks --- wqflask/wqflask/static/css | 1 - wqflask/wqflask/static/images | 1 - wqflask/wqflask/static/javascript | 1 - 3 files changed, 3 deletions(-) delete mode 120000 wqflask/wqflask/static/css delete mode 120000 wqflask/wqflask/static/images delete mode 120000 wqflask/wqflask/static/javascript diff --git a/wqflask/wqflask/static/css b/wqflask/wqflask/static/css deleted file mode 120000 index 9d8c2f68..00000000 --- a/wqflask/wqflask/static/css +++ /dev/null @@ -1 +0,0 @@ -../../../web/css \ No newline at end of file diff --git a/wqflask/wqflask/static/images b/wqflask/wqflask/static/images deleted file mode 120000 index 12f0f8b5..00000000 --- a/wqflask/wqflask/static/images +++ /dev/null @@ -1 +0,0 @@ -../../../web/images \ No newline at end of file diff --git a/wqflask/wqflask/static/javascript b/wqflask/wqflask/static/javascript deleted file mode 120000 index 5f58faf4..00000000 --- a/wqflask/wqflask/static/javascript +++ /dev/null @@ -1 +0,0 @@ -../../../web/javascript \ No newline at end of file -- cgit v1.2.3 From 6cf118eb55453d0ca041cf139bae4839ceb0286a Mon Sep 17 00:00:00 2001 From: pjotrp Date: Mon, 29 Feb 2016 11:01:11 +0000 Subject: Remove dbdoc --- wqflask/wqflask/static/dbdoc/TODO.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 wqflask/wqflask/static/dbdoc/TODO.md diff --git a/wqflask/wqflask/static/dbdoc/TODO.md b/wqflask/wqflask/static/dbdoc/TODO.md deleted file mode 100644 index c0a8bab7..00000000 --- a/wqflask/wqflask/static/dbdoc/TODO.md +++ /dev/null @@ -1 +0,0 @@ -TODO: Add all database documentation into this folder -- cgit v1.2.3 From 2fd8e09c3de9ee539e47bdb416a4c0df89ff15ab Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 1 Mar 2016 08:55:10 +0000 Subject: setup: prepare for Guix runtime --- MANIFEST.in | 5 +---- bin/genenetwork2 | 18 +++++++++++++++--- setup.py | 15 ++++++++++++--- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index f4ea2316..bf23f9aa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,10 +17,6 @@ include wqflask/wqflask/templates/new_security/* include wqflask/wqflask/correlation/* include wqflask/wqflask/interval_mapping/* include wqflask/wqflask/interval_analyst/* -include wqflask/wqflask/my_pylmm/* -include wqflask/wqflask/my_pylmm/scripts/* -include wqflask/wqflask/my_pylmm/pyLMM/* -include wqflask/wqflask/my_pylmm/data/* include wqflask/wqflask/static/* include wqflask/wqflask/static/dbdoc/* include wqflask/wqflask/static/new/* @@ -260,3 +256,4 @@ include wqflask/flask_security/templates/security/email/* include wqflask/dbFunction/* include wqflask/basicStatistics/* include wqflask/base/* +include etc/*.py diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 00ee09bf..e5f04ee1 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,6 +1,9 @@ #! /bin/bash # -# This will run the server with default settings. +# This will run the GN2 server (with default settings if none supplied). +# +# Environment settings can be used to preconfigure as well as a +# settings.py file. # Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") @@ -8,20 +11,29 @@ SCRIPT=$(readlink -f "$0") GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) echo $GN2_BASE_PATH +# Handle setting parameter settings=$1 if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi -export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH +if [ ! -e $settings ]; then + echo "ERROR: can not load settings file $settings" + exit 1 +fi export WQFLASK_SETTINGS=$settings +# We may change this one: +export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH + +# TEMPDIR defaults to /tmp if nothing else if [ -z $TEMPDIR ]; then TEMPDIR="/tmp" fi -# start the redis server +# Start the redis server echo -n "dir $TEMPDIR dbfilename gn2.rdb " | redis-server - & +# Start the flask server running GN2 cd $GN2_BASE_PATH/wqflask echo "Starting with $settings" /usr/bin/env python runserver.py diff --git a/setup.py b/setup.py index 1b11acc0..a9b71fab 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,17 @@ +# Run setup from python - this script is used by the GNU Guix builder. + from setuptools import setup, find_packages setup(name='genenetwork2', version='2.0', + author = "The GeneNetwork Team", + author_email='rwilliams@uthsc.edu', + url = "https://github.com/genenetwork/genenetwork2/blob/master/README.md", + description = 'Website and tools for genetics.', include_package_data=True, - # packages=find_packages() - packages=['wqflask'] - ) + packages=['wqflask','etc'], + scripts=['bin/genenetwork2'], + # package_data = { + # 'etc': ['*.py'] + # } +) -- cgit v1.2.3 From 8b2598efcf276ef6a1edd06921a9cc28dbfcad14 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 1 Mar 2016 09:13:30 +0000 Subject: Handle settings in guix store --- bin/genenetwork2 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index e5f04ee1..682d8b3d 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -11,12 +11,15 @@ SCRIPT=$(readlink -f "$0") GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) echo $GN2_BASE_PATH -# Handle setting parameter +# Handle settings parameter settings=$1 if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi if [ ! -e $settings ]; then - echo "ERROR: can not load settings file $settings" - exit 1 + settings=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py + if [ ! -e $settings ]; then + echo "ERROR: can not locate settings file - pass it in the command line" + exit 1 + fi fi export WQFLASK_SETTINGS=$settings -- cgit v1.2.3 From c52bf79ba92d2a632637536a9ea607c22ef2d9bf Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 1 Mar 2016 09:29:24 +0000 Subject: GN2_BASE_PATH --- bin/genenetwork2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 682d8b3d..a0a013fc 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -15,7 +15,8 @@ echo $GN2_BASE_PATH settings=$1 if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi if [ ! -e $settings ]; then - settings=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py + GN2_BASE_PATH=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg + settings=$GN2_BASE_PATH/etc/default_settings.py if [ ! -e $settings ]; then echo "ERROR: can not locate settings file - pass it in the command line" exit 1 -- cgit v1.2.3 From a66858e99c2195d90a187899db6f4dd8966a0a2c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 3 Mar 2016 11:21:01 +0000 Subject: Show error when GENODIR is missing --- bin/genenetwork2 | 13 +++++++------ etc/default_settings.py | 2 +- wqflask/utility/tools.py | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index a0a013fc..bbb2a19f 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -9,18 +9,19 @@ SCRIPT=$(readlink -f "$0") # Absolute path this script is in, thus /home/user/bin GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) + +GN2_GUIX_PATH=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg +if [ -d $GN2_GUIX_PATH ]; then + GN2_BASE_PATH=$GN2_GUIX_PATH +fi echo $GN2_BASE_PATH # Handle settings parameter settings=$1 if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi if [ ! -e $settings ]; then - GN2_BASE_PATH=$GN2_BASE_PATH/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg - settings=$GN2_BASE_PATH/etc/default_settings.py - if [ ! -e $settings ]; then - echo "ERROR: can not locate settings file - pass it in the command line" - exit 1 - fi + echo "ERROR: can not locate settings file - pass it in the command line" + exit 1 fi export WQFLASK_SETTINGS=$settings diff --git a/etc/default_settings.py b/etc/default_settings.py index 60a3a7b4..0cf40265 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -23,7 +23,7 @@ SERVER_PORT = 5003 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' # Path overrides for Genenetwork -GENENETWORK_FILES = "../../gn2_data" +GENENETWORK_FILES = os.environ['HOME']+"/gn2_data" PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) PLINK_COMMAND = str.strip(os.popen("which plink2").read()) GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index b8eff12a..9405a9c6 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -76,8 +76,8 @@ def plink_command(guess=None): def flat_files(subdir=None): base = get_setting("GENENETWORK_FILES") if subdir: - return valid_path(base+"/"+subdir) - return valid_path(base) + return assert_dir(base+"/"+subdir) + return assert_dir(base) def assert_dir(dir): if not valid_path(dir): -- cgit v1.2.3 From b5765fb91adf6845913dd60e993d47c3f691f90a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:06:24 +0000 Subject: [PATCH 061/100] Put R/qtl .cross file in TMPDIR --- .../wqflask/marker_regression/marker_regression.py | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index abd1ce85..39d7c78a 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -39,6 +39,7 @@ from wqflask.marker_regression import gemma_mapping from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND from utility.external import shell +from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR class MarkerRegression(object): @@ -324,7 +325,7 @@ class MarkerRegression(object): self.gen_pheno_txt_file_plink(pheno_filename = output_filename) - rqtl_command = './plink --noweb --ped %s.ped --no-fid --no-parents --no-sex --no-pheno --map %s.map --pheno %s/%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (self.dataset.group.name, self.dataset.group.name, webqtlConfig.TMPDIR, plink_output_filename, self.this_trait.name, self.maf, webqtlConfig.TMPDIR, plink_output_filename) + rqtl_command = './plink --noweb --ped %s.ped --no-fid --no-parents --no-sex --no-pheno --map %s.map --pheno %s/%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (self.dataset.group.name, self.dataset.group.name, TMPDIR, plink_output_filename, self.this_trait.name, self.maf, TMPDIR, plink_output_filename) os.system(rqtl_command) @@ -381,10 +382,11 @@ class MarkerRegression(object): calc_genoprob = ro.r["calc.genoprob"] # Map the calc.genoprob function read_cross = ro.r["read.cross"] # Map the read.cross function write_cross = ro.r["write.cross"] # Map the write.cross function - GENOtoCSVR = ro.r["GENOtoCSVR"] # Map the GENOtoCSVR function + GENOtoCSVR = ro.r["GENOtoCSVR"] # Map the local GENOtoCSVR function - genofilelocation = locate(self.dataset.group.name + ".geno", "genotype") - crossfilelocation = locate(self.dataset.group.name + ".cross", "genotype") + crossname = self.dataset.group.name + genofilelocation = locate(crossname + ".geno", "genotype") + crossfilelocation = TMPDIR + crossname + ".cross" #print("Conversion of geno to cross at location:", genofilelocation, " to ", crossfilelocation) @@ -411,7 +413,7 @@ class MarkerRegression(object): #print("Pair scan results:", result_data_frame) self.pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png" - png(file=webqtlConfig.TMPDIR+self.pair_scan_filename) + png(file=TMPDIR+self.pair_scan_filename) plot(result_data_frame) dev_off() @@ -521,8 +523,8 @@ class MarkerRegression(object): self.gen_pheno_txt_file_plink(pheno_filename = plink_output_filename) - plink_command = PLINK_COMMAND + ' --noweb --bed %s/%s.bed --bim %s/%s.bim --fam %s/%s.fam --no-fid --no-parents --no-sex --no-pheno --pheno %s%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (PLINK_PATH, self.dataset.group.name, PLINK_PATH, self.dataset.group.name, PLINK_PATH, self.dataset.group.name, webqtlConfig.TMPDIR, plink_output_filename, self.this_trait.name, self.maf, webqtlConfig.TMPDIR, plink_output_filename) - #print("plink_command:", plink_command) + plink_command = PLINK_COMMAND + ' --noweb --ped %s/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %s/%s.map --pheno %s%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (PLINK_PATH, self.dataset.group.name, PLINK_PATH, self.dataset.group.name, TMPDIR, plink_output_filename, self.this_trait.name, self.maf, TMPDIR, plink_output_filename) + print("plink_command:", plink_command) os.system(plink_command) @@ -543,7 +545,7 @@ class MarkerRegression(object): def gen_pheno_txt_file_plink(self, pheno_filename = ''): ped_sample_list = self.get_samples_from_ped_file() - output_file = open("%s%s.txt" % (webqtlConfig.TMPDIR, pheno_filename), "wb") + output_file = open("%s%s.txt" % (TMPDIR, pheno_filename), "wb") header = 'FID\tIID\t%s\n' % self.this_trait.name output_file.write(header) @@ -578,7 +580,7 @@ class MarkerRegression(object): def gen_pheno_txt_file_rqtl(self, pheno_filename = ''): ped_sample_list = self.get_samples_from_ped_file() - output_file = open("%s%s.txt" % (webqtlConfig.TMPDIR, pheno_filename), "wb") + output_file = open("%s%s.txt" % (TMPDIR, pheno_filename), "wb") header = 'FID\tIID\t%s\n' % self.this_trait.name output_file.write(header) @@ -720,7 +722,7 @@ class MarkerRegression(object): threshold_p_value = 0.01 - result_fp = open("%s%s.qassoc"% (webqtlConfig.TMPDIR, output_filename), "rb") + result_fp = open("%s%s.qassoc"% (TMPDIR, output_filename), "rb") header_line = result_fp.readline()# read header line line = result_fp.readline() @@ -1042,7 +1044,7 @@ def create_snp_iterator_file(group): This function is only called by main below """ raise Exception("Paths are undefined here") - plink_file_base = os.path.join(webqtlConfig.TMPDIR, group) + plink_file_base = os.path.join(TMPDIR, group) plink_input = input.plink(plink_file_base, type='b') data = dict(plink_input = list(plink_input), -- cgit v1.2.3 From 95d9380be80738d85ae85c3232ea1741fa07232a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:17:03 +0000 Subject: Simplify the README so people can actually READ it. --- README.md | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index db495910..98cb3e80 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,11 @@ -genenetwork on github (May 7, 2012 by Lei Yan and Rob Williams) - -www.genenetwork.org +Source code for the GeneNetwork2 server http://www.genenetwork.org/ Released under Affero General Public License 3 (AGPLv3). See also LICENSE.txt -For background see: http://en.wikipedia.org/wiki/Genenetwork - -WWW service initiated January, 1994 as The Portable Dictionary of the Mouse Genome and -June 15, 2001 as WebQTL. - -This code and the main web service is currently operated by Lei Yan, Zachary Sloan, -Arthur Centeno. Design and code by Xiaodong Zhou, Christian Fernandez, Sam Ockman, Ning Liu, Rudi Alberts, -Elissa Chesler, Jintao Wang, Kenneth Manly, Robert W. Williams, and colleagues. - -Code and primary web service managed by Dr. Robert W. Williams and the University of Tennessee Health Science Center, -Memphis TN, USA. +See also: http://en.wikipedia.org/wiki/Genenetwork Join #genenetwork on irc.freenode.net. -Email labwilliams@gmail.com or rwilliams@uthsc.edu - -Older version available on SourceForge http://sourceforge.net/projects/genenetwork/ - -Funded by the National Institutes of Health and -University of Tennessee Center for Integrative and Translational Genomics - - -=========== +Code and primary web service managed by Dr. Robert W. Williams and the +University of Tennessee Health Science Center, Memphis TN, USA. -- cgit v1.2.3 From e1c42eaed65de9cd9a99cc4731f3ede66cfe94bb Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:17:33 +0000 Subject: docs: muck out outdated info --- doc/gn_installation_notes.txt | 346 -------------------------------------- doc/new_variable_names.txt | 6 - doc/notes.txt | 311 ---------------------------------- doc/notes_DA.txt | 10 -- doc/old/gn_installation_notes.txt | 346 ++++++++++++++++++++++++++++++++++++++ doc/old/notes.txt | 311 ++++++++++++++++++++++++++++++++++ doc/requirements.txt | 36 ---- doc/todo.txt | 2 - 8 files changed, 657 insertions(+), 711 deletions(-) delete mode 100644 doc/gn_installation_notes.txt delete mode 100644 doc/new_variable_names.txt delete mode 100644 doc/notes.txt delete mode 100644 doc/notes_DA.txt create mode 100644 doc/old/gn_installation_notes.txt create mode 100644 doc/old/notes.txt delete mode 100644 doc/requirements.txt delete mode 100644 doc/todo.txt diff --git a/doc/gn_installation_notes.txt b/doc/gn_installation_notes.txt deleted file mode 100644 index efea0309..00000000 --- a/doc/gn_installation_notes.txt +++ /dev/null @@ -1,346 +0,0 @@ -UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration - -Enable UFW: -sudo ufw enable -Check status of UFW: -sudo ufw status verbose - -Limit ssh: -sudo ufw limit ssh - -Enable http, https: -sudo ufw allow http -sudo ufw allow https - -=========================================== - -http://evilrouters.net/2011/02/18/using-etckeeper-with-git-on-ubuntu/ - -Install etckeeper: -sudo apt-get install etckeeper - -Edit /etc/etckeeper/etckeeper.conf to comment out VCS="bzr" and comment in VCS="git" - -Initialize repository: -sudo etckeeper init - -Do initial commit: -sudo etckeeper commit "Initial commit." - -=========================================== - -Get git: - -sudo apt-get install git - -=========================================== - -Clone code from git repository: -git clone http://github.com/zsloan/genenetwork.git gene - -Pull from branch in git repository: -git pull origin flask(or whatever the branch is) - -**"apt-cache search" searches for text in the description for various packages, while -"apt-file search" searches for files within the packages** - -Search for package with a specified file that can be installed with apt-get -apt-file search _______ - -============================================ - -Create trash directory: -mkdir ~/.trash - -============================================ - -Using dpkg - -Check which system packages are already installed: -dpkg -l | less - -=========================================== - -Installing virtualenv: -sudo pip install virtualenv - -Tell virtualenv which directory to use: -virtualenv ve27 - -Start up virtual environment: -source ~/ve27/bin/activate - -=========================================== - -Install libmysqlclient-dev (Mysql-Python dependency) -sudo apt-get install libmysqlclient-dev - -Install python-dev (numpy dependency) -sudo apt-get install python-dev - -Install scipy dependencies: -sudo apt-get install libatlas-base-dev gfortran g++ - -=========================================== - -Install pip: -sudo apt-get install python-pip - -REMEMBER TO SOURCE VE BEFORE INSTALLING - -Comment out in requirements.txt: -Reaper -numarray - -Before installing from requirements.txt, install numpy separately: -pip install numpy==1.7.0 (or whatever version we're using) - -Install from requirements.txt (after activating virtualenv): -pip install -r gene/doc/requirements.txt - -=========================================== - -Installing QTL Reaper - -wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb -mv -v qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb qtlreaper-1.1.1.tar.gz -tar xvf qtlreaper-1.1.1.tar.gz (to unzip) -mkdir /home/zas1024/ve27/include/python2.7/Reaper -python setup.py install - -=========================================== - -Installing numarray 1.5.2 - -wget http://downloads.sourceforge.net/project/numpy/Old%20Numarray/1.5.2/numarray-1.5.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%2520Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload&ts=1358978306&use_mirror=superb-dca2 -mv -v numarray-1.5.2.tar.gz?r=http:%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%20Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload numarray-1.5.2.tar.gz -tar xvf numarray-1.5.2.tar.gz -sudo python setup.py install - -=========================================== - -Installing nginx - -sudo -s -nginx=stable # use nginx=development for latest development version -echo "deb http://ppa.launchpad.net/nginx/$nginx/ubuntu lucid main" > /etc/apt/sources.list.d/nginx-$nginx-lucid.list -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C300EE8C -apt-get update -apt-get install nginx - -Create configuration file in ~/gene/wqflask/other_config/nginx_conf/ (modeled off of the others) -Create symbolic link to config file in /etc/nginx/sites-enabled/: -ln -s dir_to_link_to linking_dir - -Run nginx: -sudo /usr/sbin/nginx - -============================================ - -Copy over zach_settings.py to /home/zas1024 directory (or whatever is home directory) - -export WQFLASK_SETTINGS=~/gene/wqflask/cfg/zach_settings.py (or wherever file is located) - -============================================ - -Install MySQL Server - -sudo apt-get install mysql-server - -mkdir /mnt/big -fdisk /dev/sdb -m: for help -n: new partion -... -w: write to table and exit - -Start MySQL server: -service mysql start - -Stop MySQL server: -service mysql stop - -Change root password: -mysql> UPDATE mysql.user SET Password=PASSWORD('your password') WHERE User='root'; - -Setup accounts in MySQL (first need to delete anonymous/non-root accounts): -#; use mysql; -#; select * from user; -#; delete from user where Host!="localhost"; -#; delete from user where User!="root"; -#; update user set Password = Password('yourpassword') where User='root'; -#; GRANT ALL ON *.* TO 'yourusername'@'%' IDENTIFIED BY 'yourpassword'; -#; select * from user; - -Use Percona to optimise my.cfg file: -https://tools.percona.com/wizard - -To check # CPUs: -grep -c processor /proc/cpuinfo - -To check memory, can use htop - -============================================ - -Check RSA key: -ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key - -03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) - -============================================ - -Installing redis server - -In the dls directory: -$ wget http://redis.googlecode.com/files/redis-2.6.14.tar.gz -$ tar xzf redis-2.6.14.tar.gz -$ cd redis-2.6.14 -$ make - -Run redis (from inside redis-2.6.14 directory): -src/redis-server - -For the rest of the installation instructions, refer to: -http://redis.io/topics/quickstart - -To continue installation (from the README): - -In order to install Redis binaries into /usr/local/bin just use: - - % make install - -Make install will just install binaries in your system, but will not configure -init scripts and configuration files in the appropriate place. This is not -needed if you want just to play a bit with Redis, but if you are installing -it the proper way for a production system, we have a script doing this -for Ubuntu and Debian systems: - - % cd utils - % ./install_server - -The script will ask you a few questions and will setup everything you need -to run Redis properly as a background daemon that will start again on -system reboots. - -In the redis config file (/etc/redis/6379.conf) uncomment the following line: -bind 127.0.0.1 - -This is so only the local computer has access - -============================================ - -Check RSA key: -ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key - -03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) - -============================================ - -Using Yolk - -Install Yolk: -pip install yolk - -Check packages installed in this virtual environment: -yolk -l - -Checks packages that have updates available: -yolk -U - -=========================================== - -Installing yaml - -Install libyaml-dev: -sudo apt-get install libyaml-dev - -Install yaml: -pip install pyyaml - -=========================================== - -Install MySQL Client - -To fix error "mysql_config not found" while installing packages with pip: -sudo apt-get install libmysqlclient-dev - -=========================================== - -Installing R - -sudo apt-get install r-base-dev - -=========================================== - -Installing rpy2 - -pip install rpy2 - -=========================================== - -Install Parallel Python (pp) - -wget http://www.parallelpython.com/downloads/pp/pp-1.6.3.tar.gz -tar xvf pp-1.6.3.tar.gz -python setup.py install - -========================================== - -Install requests - -pip install requests - -========================================== - -Install logging_tree - -pip install logging_tree - -========================================== - -Install colordiff (just to make it easier to view differences in files, not necessary for GN) - -sudo apt-get install colordiff - -========================================== - -Install NTP (network time protocol) - -sudo apt-get install ntp - -========================================== - -To get server running: - -!If having seemingly inexplicable problems with imports, make sure I've started the environment! - -Start up virtual environment: -source ~/ve27/bin/activate============== - -Install requests - -pip install requests - -======================= - -To set WQFLASK_SETTINGS environment variable: -export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) - -To change screen environment variable (if man not working or to get color, for example): -export TERM=screen - -To search for commands in history if necessary: -history | grep "(whatever is being searched for)" - -Run web server: -/usr/local/nginx/sbin/nginx - -Run server: -python runserver.py - -=========================================== - -To do full upgrade (as opposed to apt-get upgrade) -sudo aptitude full-upgrade - -=========================================== diff --git a/doc/new_variable_names.txt b/doc/new_variable_names.txt deleted file mode 100644 index c11c160e..00000000 --- a/doc/new_variable_names.txt +++ /dev/null @@ -1,6 +0,0 @@ -RISet/riset -> group -webqtlDataset.py -> data_set.py -webqtlDataset (class object) -> DataSet -database/db -> dataset/data_set -DataEditingPage -> show_trait.py/show_trait.html -webqtlTrait -> GeneralTrait \ No newline at end of file diff --git a/doc/notes.txt b/doc/notes.txt deleted file mode 100644 index f8ce2759..00000000 --- a/doc/notes.txt +++ /dev/null @@ -1,311 +0,0 @@ -Clone code from git repository: -git clone http://github.com/zsloan/genenetwork.git gene - -Pull from branch in git repository: -git pull origin flask(or whatever the branch is) - -Add all files in current directory branch to be staged to be committed -git add -A - -Git push to my github repository -git push https://github.com/zsloan/genenetwork.git master - -Get the default path being used when a command is typed: -which _______ - -Change default path python looks at when importing -export PYTHONPATH="/home/zas1024/gene/wqflask" - -Get the paths checked when a user types a command: -echo $PATH - -Get all environment variables: -set - - -**** apt-get is an Ubuntu system-wide package manager, while pip is a Python-only system -for python packages **** - -Upgrade system packages for security updates, etc: -apt-get update -apt-get dist-upgrade (can probably be used in place of apt-get upgrade) -apt-get upgrade - -**"apt-cache search" searches for text in the description for various packages, while -"apt-file search" searches for files within the packages** - -Show information about given result item returned by apt-cache search: -apt-cache show _______ - -Search for package with a specified file that can be installed with apt-get -apt-file search _______ - -============================================ - -Check which system packages are installed: -dpkg -l | less - -============================================ - -NEVER USE SUDO WITH PIP - -To fix error "mysql_config not found": -sudo apt-get install libmysqlclient-dev - -Create requirements.txt file: -pip freeze > requirements.txt (or whatever filename) - -Install from requirements.txt: -pip install -r gene/wqflask/requirements.txt -t ve27 - -============================================ - -To get server running: - -!If having seemingly inexplicable problems with imports, make sure I've started the environment! - -Note: Virtual environment is not activated when running as root - -Start up virtual environment: -source ~/ve27/bin/activate - -Deactivate virtual environment: -deactivate (just type deactivate) - -To set WQFLASK_SETTINGS environment variable: -export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) - -zach_settings.py is the local config file with the DB_URI -default_settings.py is the config file with the configuration information that would apply -to any running instance of GeneNetwork - -To change screen environment variable (if man not working or to get color, for example): -export TERM=screen - -To search for commands in history if necessary: -history | grep "(whatever is being searched for)" - -Reload web server: -/usr/sbin/nginx -s reload - -Run server: -python runserver.py - -Run sendmail.py -python send_mail.py - -=========================================== - -UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration - -Enable UFW: -sudo ufw enable -Check status of UFW: -sudo ufw status verbose - -=========================================== - -Update user password: -use mysql; -UPDATE user SET password=PASSWORD('whatever') WHERE user='webqtl'; -FLUSH PRIVILEGES; - -=========================================== - -Redis things - -Restart redis server: -sudo /etc/init.d/redis_6379 stop -sudo /etc/init.d/redis_6379 start - -=========================================== - -Run pylint: -pylint file_name - -For options: -pylint --help - -=========================================== - -Start screen session: - -If "no option -R" or running for first time: -byobu-select-backend -2. screen - -byobu -RD (to start) -control-a then :multiuser on -control-a then :acladd sam - -More info on sharing byobu screen with other users: -http://wiki.networksecuritytoolkit.org/nstwiki/index.php/HowTo_Share_A_Terminal_Session_Using_Screen - -control-a c to create channel - -type: screen -list for sessions -screen -r zas1024/25679.byobu - -or if only one: - -screen -r zas1024/ - -=========================================== - -Start up log: -Go to /tmp and tail -f flask_gn_log - -=========================================== - -Coffeescript Stuff: - -coffee -c (filename) -coffee -c -w . (to watch for changes and recompile in current directory; the "." is for current directory) -coffee --help (for information about setting options) - -=========================================== - -Unset ASKPASS when trying to git push - -unset SSH_ASKPASS - -=========================================== - -htop: Gives information on processes, cpu/memory load, etc -dstat: Also gives various system information, resource usage, etc -df: Reports file system disk space usage - -=========================================== - -ll h*: Finds items in the directory starting with h - -=========================================== - -du -hms * | sort -n : Gives size used by different directories --h: human readable --m: in megabytes (default) --s: summarize - -=========================================== - -rm -rfv - -=========================================== - -cp -a (archive; copies recursively and doesn't follow symbol links) - -i (interactive, prompts before overwrite) - -v (verbose) - -b (backup) - -mv (same as above, but with no -a) - -=========================================== - -Add user: -sudo adduser -Edit /etc/sudoers to give user root privileges - - -=========================================== - -tidyp - Improves/beautifies html code -tidyp -m -i -w 100 index_page.html - -=========================================== - -ps ax - View processes -ps ux (the "u" is for user) - -kill (process #) - -=========================================== - -Installing QTL Reaper - -wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb -tar xvf qtlreaper-1.1.1.tar.gz (to unzip) -python setup.py build -python setup.py install - -=========================================== - -Using Yolk - -Install Yolk: -pip install yolk - -Check packages installed in this virtual environment: -yolk -l - -Checks packages that have updates available: -yolk -U - -=========================================== - -Python stuff: - -Classes should always inherit "object" -To iterate through dictionary items: for X, Y in MyDictionary.items(): - -=========================================== - -Change import path from command line -export PYTHONPATH=~/gene/wqflask (or whatever) - -=========================================== - -Create symbolic link: -ln -s dir_to_link_to linking_dir - -=========================================== - -Reboot server -sudo reboot - -=========================================== - -Using grep: - -grep -ir (search string) (directory) - --i ignores case --r searches recursively through subdirectories of directory you specify - -=========================================== - -Command line arguments: - -Use argparse to deal with command line arguments (instead of argv or optparse) - -=========================================== - -Change owner/group: - -chown zas1024 somefile (change owner of somefile to zas1024) -chown zas1024: somefile (change owner and group of somefile to zas1024) -sudo chown root: somefile (change owner/group of somefile to root) - -Change permissions: - -chmod g-wr (remove write/read privileges for group) -chmod g+w (add write privileges for group) -chmod u+rwx (add read, write, and execute privileges for user) -etc. - -=========================================== - -MySQL Things - - -Create fulltext (MATCH AGAINST syntax) index: -CREATE FULLTEXT INDEX __________ ON Table (Column); - -If <4 character searches not working: -Add ft_min_word_len = 2 (to make te minimum characters 2) to config file -Rebuild relevant tables with fulltext indexes with "REPAIR TABLE _______ QUICK" - -=========================================== - -To delete all .pyc files (for example if there's some issue you're having trouble resolving): -find . -name '*.pyc' -delete -http://stackoverflow.com/questions/785519/how-do-i-remove-all-pyc-files-from-a-project - diff --git a/doc/notes_DA.txt b/doc/notes_DA.txt deleted file mode 100644 index 410e0182..00000000 --- a/doc/notes_DA.txt +++ /dev/null @@ -1,10 +0,0 @@ -Danny's notes about the genenetwork source - -Location of static files: - -Location of HTML templates: wqflask/wqflask/templates/ - -Entry point of the wqflask app: wqflask/wqflask/__init__.py - -Application routes: wqflask/wqflask/views.py - diff --git a/doc/old/gn_installation_notes.txt b/doc/old/gn_installation_notes.txt new file mode 100644 index 00000000..efea0309 --- /dev/null +++ b/doc/old/gn_installation_notes.txt @@ -0,0 +1,346 @@ +UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration + +Enable UFW: +sudo ufw enable +Check status of UFW: +sudo ufw status verbose + +Limit ssh: +sudo ufw limit ssh + +Enable http, https: +sudo ufw allow http +sudo ufw allow https + +=========================================== + +http://evilrouters.net/2011/02/18/using-etckeeper-with-git-on-ubuntu/ + +Install etckeeper: +sudo apt-get install etckeeper + +Edit /etc/etckeeper/etckeeper.conf to comment out VCS="bzr" and comment in VCS="git" + +Initialize repository: +sudo etckeeper init + +Do initial commit: +sudo etckeeper commit "Initial commit." + +=========================================== + +Get git: + +sudo apt-get install git + +=========================================== + +Clone code from git repository: +git clone http://github.com/zsloan/genenetwork.git gene + +Pull from branch in git repository: +git pull origin flask(or whatever the branch is) + +**"apt-cache search" searches for text in the description for various packages, while +"apt-file search" searches for files within the packages** + +Search for package with a specified file that can be installed with apt-get +apt-file search _______ + +============================================ + +Create trash directory: +mkdir ~/.trash + +============================================ + +Using dpkg + +Check which system packages are already installed: +dpkg -l | less + +=========================================== + +Installing virtualenv: +sudo pip install virtualenv + +Tell virtualenv which directory to use: +virtualenv ve27 + +Start up virtual environment: +source ~/ve27/bin/activate + +=========================================== + +Install libmysqlclient-dev (Mysql-Python dependency) +sudo apt-get install libmysqlclient-dev + +Install python-dev (numpy dependency) +sudo apt-get install python-dev + +Install scipy dependencies: +sudo apt-get install libatlas-base-dev gfortran g++ + +=========================================== + +Install pip: +sudo apt-get install python-pip + +REMEMBER TO SOURCE VE BEFORE INSTALLING + +Comment out in requirements.txt: +Reaper +numarray + +Before installing from requirements.txt, install numpy separately: +pip install numpy==1.7.0 (or whatever version we're using) + +Install from requirements.txt (after activating virtualenv): +pip install -r gene/doc/requirements.txt + +=========================================== + +Installing QTL Reaper + +wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb +mv -v qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb qtlreaper-1.1.1.tar.gz +tar xvf qtlreaper-1.1.1.tar.gz (to unzip) +mkdir /home/zas1024/ve27/include/python2.7/Reaper +python setup.py install + +=========================================== + +Installing numarray 1.5.2 + +wget http://downloads.sourceforge.net/project/numpy/Old%20Numarray/1.5.2/numarray-1.5.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%2520Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload&ts=1358978306&use_mirror=superb-dca2 +mv -v numarray-1.5.2.tar.gz?r=http:%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%20Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload numarray-1.5.2.tar.gz +tar xvf numarray-1.5.2.tar.gz +sudo python setup.py install + +=========================================== + +Installing nginx + +sudo -s +nginx=stable # use nginx=development for latest development version +echo "deb http://ppa.launchpad.net/nginx/$nginx/ubuntu lucid main" > /etc/apt/sources.list.d/nginx-$nginx-lucid.list +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C300EE8C +apt-get update +apt-get install nginx + +Create configuration file in ~/gene/wqflask/other_config/nginx_conf/ (modeled off of the others) +Create symbolic link to config file in /etc/nginx/sites-enabled/: +ln -s dir_to_link_to linking_dir + +Run nginx: +sudo /usr/sbin/nginx + +============================================ + +Copy over zach_settings.py to /home/zas1024 directory (or whatever is home directory) + +export WQFLASK_SETTINGS=~/gene/wqflask/cfg/zach_settings.py (or wherever file is located) + +============================================ + +Install MySQL Server + +sudo apt-get install mysql-server + +mkdir /mnt/big +fdisk /dev/sdb +m: for help +n: new partion +... +w: write to table and exit + +Start MySQL server: +service mysql start + +Stop MySQL server: +service mysql stop + +Change root password: +mysql> UPDATE mysql.user SET Password=PASSWORD('your password') WHERE User='root'; + +Setup accounts in MySQL (first need to delete anonymous/non-root accounts): +#; use mysql; +#; select * from user; +#; delete from user where Host!="localhost"; +#; delete from user where User!="root"; +#; update user set Password = Password('yourpassword') where User='root'; +#; GRANT ALL ON *.* TO 'yourusername'@'%' IDENTIFIED BY 'yourpassword'; +#; select * from user; + +Use Percona to optimise my.cfg file: +https://tools.percona.com/wizard + +To check # CPUs: +grep -c processor /proc/cpuinfo + +To check memory, can use htop + +============================================ + +Check RSA key: +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key + +03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) + +============================================ + +Installing redis server + +In the dls directory: +$ wget http://redis.googlecode.com/files/redis-2.6.14.tar.gz +$ tar xzf redis-2.6.14.tar.gz +$ cd redis-2.6.14 +$ make + +Run redis (from inside redis-2.6.14 directory): +src/redis-server + +For the rest of the installation instructions, refer to: +http://redis.io/topics/quickstart + +To continue installation (from the README): + +In order to install Redis binaries into /usr/local/bin just use: + + % make install + +Make install will just install binaries in your system, but will not configure +init scripts and configuration files in the appropriate place. This is not +needed if you want just to play a bit with Redis, but if you are installing +it the proper way for a production system, we have a script doing this +for Ubuntu and Debian systems: + + % cd utils + % ./install_server + +The script will ask you a few questions and will setup everything you need +to run Redis properly as a background daemon that will start again on +system reboots. + +In the redis config file (/etc/redis/6379.conf) uncomment the following line: +bind 127.0.0.1 + +This is so only the local computer has access + +============================================ + +Check RSA key: +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key + +03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) + +============================================ + +Using Yolk + +Install Yolk: +pip install yolk + +Check packages installed in this virtual environment: +yolk -l + +Checks packages that have updates available: +yolk -U + +=========================================== + +Installing yaml + +Install libyaml-dev: +sudo apt-get install libyaml-dev + +Install yaml: +pip install pyyaml + +=========================================== + +Install MySQL Client + +To fix error "mysql_config not found" while installing packages with pip: +sudo apt-get install libmysqlclient-dev + +=========================================== + +Installing R + +sudo apt-get install r-base-dev + +=========================================== + +Installing rpy2 + +pip install rpy2 + +=========================================== + +Install Parallel Python (pp) + +wget http://www.parallelpython.com/downloads/pp/pp-1.6.3.tar.gz +tar xvf pp-1.6.3.tar.gz +python setup.py install + +========================================== + +Install requests + +pip install requests + +========================================== + +Install logging_tree + +pip install logging_tree + +========================================== + +Install colordiff (just to make it easier to view differences in files, not necessary for GN) + +sudo apt-get install colordiff + +========================================== + +Install NTP (network time protocol) + +sudo apt-get install ntp + +========================================== + +To get server running: + +!If having seemingly inexplicable problems with imports, make sure I've started the environment! + +Start up virtual environment: +source ~/ve27/bin/activate============== + +Install requests + +pip install requests + +======================= + +To set WQFLASK_SETTINGS environment variable: +export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) + +To change screen environment variable (if man not working or to get color, for example): +export TERM=screen + +To search for commands in history if necessary: +history | grep "(whatever is being searched for)" + +Run web server: +/usr/local/nginx/sbin/nginx + +Run server: +python runserver.py + +=========================================== + +To do full upgrade (as opposed to apt-get upgrade) +sudo aptitude full-upgrade + +=========================================== diff --git a/doc/old/notes.txt b/doc/old/notes.txt new file mode 100644 index 00000000..f8ce2759 --- /dev/null +++ b/doc/old/notes.txt @@ -0,0 +1,311 @@ +Clone code from git repository: +git clone http://github.com/zsloan/genenetwork.git gene + +Pull from branch in git repository: +git pull origin flask(or whatever the branch is) + +Add all files in current directory branch to be staged to be committed +git add -A + +Git push to my github repository +git push https://github.com/zsloan/genenetwork.git master + +Get the default path being used when a command is typed: +which _______ + +Change default path python looks at when importing +export PYTHONPATH="/home/zas1024/gene/wqflask" + +Get the paths checked when a user types a command: +echo $PATH + +Get all environment variables: +set + + +**** apt-get is an Ubuntu system-wide package manager, while pip is a Python-only system +for python packages **** + +Upgrade system packages for security updates, etc: +apt-get update +apt-get dist-upgrade (can probably be used in place of apt-get upgrade) +apt-get upgrade + +**"apt-cache search" searches for text in the description for various packages, while +"apt-file search" searches for files within the packages** + +Show information about given result item returned by apt-cache search: +apt-cache show _______ + +Search for package with a specified file that can be installed with apt-get +apt-file search _______ + +============================================ + +Check which system packages are installed: +dpkg -l | less + +============================================ + +NEVER USE SUDO WITH PIP + +To fix error "mysql_config not found": +sudo apt-get install libmysqlclient-dev + +Create requirements.txt file: +pip freeze > requirements.txt (or whatever filename) + +Install from requirements.txt: +pip install -r gene/wqflask/requirements.txt -t ve27 + +============================================ + +To get server running: + +!If having seemingly inexplicable problems with imports, make sure I've started the environment! + +Note: Virtual environment is not activated when running as root + +Start up virtual environment: +source ~/ve27/bin/activate + +Deactivate virtual environment: +deactivate (just type deactivate) + +To set WQFLASK_SETTINGS environment variable: +export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) + +zach_settings.py is the local config file with the DB_URI +default_settings.py is the config file with the configuration information that would apply +to any running instance of GeneNetwork + +To change screen environment variable (if man not working or to get color, for example): +export TERM=screen + +To search for commands in history if necessary: +history | grep "(whatever is being searched for)" + +Reload web server: +/usr/sbin/nginx -s reload + +Run server: +python runserver.py + +Run sendmail.py +python send_mail.py + +=========================================== + +UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration + +Enable UFW: +sudo ufw enable +Check status of UFW: +sudo ufw status verbose + +=========================================== + +Update user password: +use mysql; +UPDATE user SET password=PASSWORD('whatever') WHERE user='webqtl'; +FLUSH PRIVILEGES; + +=========================================== + +Redis things + +Restart redis server: +sudo /etc/init.d/redis_6379 stop +sudo /etc/init.d/redis_6379 start + +=========================================== + +Run pylint: +pylint file_name + +For options: +pylint --help + +=========================================== + +Start screen session: + +If "no option -R" or running for first time: +byobu-select-backend +2. screen + +byobu -RD (to start) +control-a then :multiuser on +control-a then :acladd sam + +More info on sharing byobu screen with other users: +http://wiki.networksecuritytoolkit.org/nstwiki/index.php/HowTo_Share_A_Terminal_Session_Using_Screen + +control-a c to create channel + +type: screen -list for sessions +screen -r zas1024/25679.byobu + +or if only one: + +screen -r zas1024/ + +=========================================== + +Start up log: +Go to /tmp and tail -f flask_gn_log + +=========================================== + +Coffeescript Stuff: + +coffee -c (filename) +coffee -c -w . (to watch for changes and recompile in current directory; the "." is for current directory) +coffee --help (for information about setting options) + +=========================================== + +Unset ASKPASS when trying to git push + +unset SSH_ASKPASS + +=========================================== + +htop: Gives information on processes, cpu/memory load, etc +dstat: Also gives various system information, resource usage, etc +df: Reports file system disk space usage + +=========================================== + +ll h*: Finds items in the directory starting with h + +=========================================== + +du -hms * | sort -n : Gives size used by different directories +-h: human readable +-m: in megabytes (default) +-s: summarize + +=========================================== + +rm -rfv + +=========================================== + +cp -a (archive; copies recursively and doesn't follow symbol links) + -i (interactive, prompts before overwrite) + -v (verbose) + -b (backup) + +mv (same as above, but with no -a) + +=========================================== + +Add user: +sudo adduser +Edit /etc/sudoers to give user root privileges + + +=========================================== + +tidyp - Improves/beautifies html code +tidyp -m -i -w 100 index_page.html + +=========================================== + +ps ax - View processes +ps ux (the "u" is for user) + +kill (process #) + +=========================================== + +Installing QTL Reaper + +wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb +tar xvf qtlreaper-1.1.1.tar.gz (to unzip) +python setup.py build +python setup.py install + +=========================================== + +Using Yolk + +Install Yolk: +pip install yolk + +Check packages installed in this virtual environment: +yolk -l + +Checks packages that have updates available: +yolk -U + +=========================================== + +Python stuff: + +Classes should always inherit "object" +To iterate through dictionary items: for X, Y in MyDictionary.items(): + +=========================================== + +Change import path from command line +export PYTHONPATH=~/gene/wqflask (or whatever) + +=========================================== + +Create symbolic link: +ln -s dir_to_link_to linking_dir + +=========================================== + +Reboot server +sudo reboot + +=========================================== + +Using grep: + +grep -ir (search string) (directory) + +-i ignores case +-r searches recursively through subdirectories of directory you specify + +=========================================== + +Command line arguments: + +Use argparse to deal with command line arguments (instead of argv or optparse) + +=========================================== + +Change owner/group: + +chown zas1024 somefile (change owner of somefile to zas1024) +chown zas1024: somefile (change owner and group of somefile to zas1024) +sudo chown root: somefile (change owner/group of somefile to root) + +Change permissions: + +chmod g-wr (remove write/read privileges for group) +chmod g+w (add write privileges for group) +chmod u+rwx (add read, write, and execute privileges for user) +etc. + +=========================================== + +MySQL Things - + +Create fulltext (MATCH AGAINST syntax) index: +CREATE FULLTEXT INDEX __________ ON Table (Column); + +If <4 character searches not working: +Add ft_min_word_len = 2 (to make te minimum characters 2) to config file +Rebuild relevant tables with fulltext indexes with "REPAIR TABLE _______ QUICK" + +=========================================== + +To delete all .pyc files (for example if there's some issue you're having trouble resolving): +find . -name '*.pyc' -delete +http://stackoverflow.com/questions/785519/how-do-i-remove-all-pyc-files-from-a-project + diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index 39ee5652..00000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,36 +0,0 @@ -BeautifulSoup==3.2.1 -Flask==0.9 -Flask-Login==0.1.3 -Flask-Mail==0.7.6 -Flask-Principal==0.3.4 -Flask-SQLAlchemy==0.16 -Flask-Security==1.6.0 -Flask-WTF==0.8.3 -Jinja2==2.6 -MySQL-python==1.2.4 -PyYAML==3.10 -#Reaper==1.0 -Reindent==0.1.1 -SQLAlchemy==0.8.0 -WTForms==1.0.3 -Werkzeug==0.8.3 -apache-libcloud==0.12.3 -argparse==1.2.1 -blinker==1.2 -cairosvg==1.0.15 -itsdangerous==0.17 -logging-tree==1.2 -logilab-astng==0.24.3 -logilab-common==0.59.1 -#numarray==1.5.2 -numpy==1.7.0 -passlib==1.6.1 -pp==1.6.3 -pylint==0.27.0 -redis==2.7.2 -requests==1.1.0 -scipy==0.11.0 -simplejson==3.0.7 -wsgiref==0.1.2 -yolk==0.4.3 -XlsxWriter==0.7.2 diff --git a/doc/todo.txt b/doc/todo.txt deleted file mode 100644 index 1d781b13..00000000 --- a/doc/todo.txt +++ /dev/null @@ -1,2 +0,0 @@ -- Ask Rob about potentially recoding qtlreaper -- Ask Rob about Probe/cellid traits \ No newline at end of file -- cgit v1.2.3 From 8e01f8a28d3124c763bc3de8e95e59ec63a582b4 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:18:05 +0000 Subject: Removed outdated info --- README-wqflask.txt | 51 --------------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 README-wqflask.txt diff --git a/README-wqflask.txt b/README-wqflask.txt deleted file mode 100644 index 03e3a1cc..00000000 --- a/README-wqflask.txt +++ /dev/null @@ -1,51 +0,0 @@ -This readme concerns the directory wqflask - an officially sanctioned fork of the main GeneNetwork -code. It's still very early in the process - but we eventually want to port all of the code -in GeneNetwork to Flask and Jinja2. For more information about the project in general, see -the file README.md. - -For more information about the port to Flask, please keep reading. - -************************* - -Requirements: - -* Python 2.7 - -* virtualenv 1.7.1.2 or later - -* Other python dependencies are listed in the file wqflask/requirements.txt - -************************** - -Installation: - -We highly recommend you create a virtual enviornment called ve27 in your home directory. - - -Get into your home directory -> cd ~ - -Create a virtual environment -> virtualenv ve27 - -Activate the environment -> source ~/ve27/bin/activate - -Install dependencies -> pip install -r ~/gene/wqflask/requirements.txt -(Or replace gene with the name of the directory holding your repository) - -************************** - -Running the program: - -Assuming your enviornment is activated (source ~/ve27/bin/activate) just run: - -> python ~/gene/wqflask/runserver.py - -The program as configured runs on port 5000 and does not serve static files. - -You'll have to run a webserver to serve pages on port 80 and to server the static files (or -flask could also be configured to serve the static pages). - -A sample configuration file for nginx is in the directory: wqflask/other_config/nginx.conf -- cgit v1.2.3 From ce07647da3b8a5411a0b9e2d3f67302594ad4aa9 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:19:22 +0000 Subject: Remove outdated installation instructions --- INSTALL.md | 76 -------------------------------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 INSTALL.md diff --git a/INSTALL.md b/INSTALL.md deleted file mode 100644 index 3e6d2f02..00000000 --- a/INSTALL.md +++ /dev/null @@ -1,76 +0,0 @@ -# INSTALL Genenetwork2 (GN2) - -## Guix installs - -The recommended way of installing GN2 is through GNU Guix. - -## Use a Docker image - -A Docker image can be generated from -[here](https://github.com/lomereiter/gn2-docker). - -(no longer up-to-date). - -## Install GN2 from source - -Currently a difficult exercise. More information coming soon. - -Clone the repository (currently ~800Mb) to local - - git clone git@github.com:genenetwork2/genenetwork2.git - -### Dependencies - -GN2 requires - -* python -* redis-server -* mysql-server - -### Required python modules - -Install the following python modules (it is probably wise to use a local -Python with environment for this) - -* Flask -* pyyaml -* redis -* qtlreaper -* numarray -* pp -* Flask-SQLAlchemy - -(not complete) - -### Set up local file settings.py - -```python -LOGFILE = """/tmp/flask_gn_log""" - - #This is needed because Flask turns key errors into a - #400 bad request response with no exception/log -TRAP_BAD_REQUEST_ERRORS = True - -DB_URI = """mysql://gn2:password@localhost/db_webqtl""" -SQLALCHEMY_DATABASE_URI = 'mysql://gn2:password@localhost/db_webqtl' - - # http://pythonhosted.org/Flask-Security/configuration.html -SECURITY_CONFIRMABLE = True -SECURITY_TRACKABLE = True -SECURITY_REGISTERABLE = True -SECURITY_RECOVERABLE = True - -SECURITY_EMAIL_SENDER = "no-reply@genenetwork.org" -SECURITY_POST_LOGIN_VIEW = "/thank_you" -SQLALCHEMY_POOL_RECYCLE = 3600 - -SERVER_PORT = 5051 - -SECRET_HMAC_CODE = '*' -``` - -(not complete) - -## Other information - -Check also the ./doc directory for settings -- cgit v1.2.3 From a1e77b1dbd308a376e0e8c7c02d71f0b4b669241 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:21:13 +0000 Subject: README --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 98cb3e80..d07abc42 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ Source code for the GeneNetwork2 server http://www.genenetwork.org/ -Released under Affero General Public License 3 (AGPLv3). See also -LICENSE.txt +This GeneNetwork2 source code is released under the Affero General +Public License 3 (AGPLv3). See also LICENSE.txt -See also: http://en.wikipedia.org/wiki/Genenetwork +For more information visit http://www.genenetwork.org/ and +http://en.wikipedia.org/wiki/Genenetwork -Join #genenetwork on irc.freenode.net. +IRC on #genenetwork on irc.freenode.net. Code and primary web service managed by Dr. Robert W. Williams and the University of Tennessee Health Science Center, Memphis TN, USA. -- cgit v1.2.3 From f7a6c602c36bd3905aaa4d743a1ae1933d6bfcf9 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:23:01 +0000 Subject: README --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index d07abc42..6a69d6cd 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,18 @@ +# GENENETWORK + Source code for the GeneNetwork2 server http://www.genenetwork.org/ +## Install + +See the [installation docs](doc/README.org) + +## License + This GeneNetwork2 source code is released under the Affero General Public License 3 (AGPLv3). See also LICENSE.txt +## More information + For more information visit http://www.genenetwork.org/ and http://en.wikipedia.org/wiki/Genenetwork -- cgit v1.2.3 From 791a80c3409327e5578dc36cc575169b8ee5b13b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:41:14 +0000 Subject: README --- README.md | 5 ++++- doc/README.org | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6a69d6cd..d2d15781 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,10 @@ Source code for the GeneNetwork2 server http://www.genenetwork.org/ ## Install -See the [installation docs](doc/README.org) +The recommended installation is with GNU Guix which allows you to +deploy GN2 and dependencies as a self contained unit on any machine. +The database can be run separately as well as the source tree (for +developers). See the [installation docs](doc/README.org). ## License diff --git a/doc/README.org b/doc/README.org index d9730948..fd19141c 100644 --- a/doc/README.org +++ b/doc/README.org @@ -1,4 +1,4 @@ -#+TITLE: Installing GeneNetwork services with GNU Guix +#+TITLE: Installing GeneNetwork services * Table of Contents :TOC: - [[#introduction][Introduction]] @@ -12,7 +12,8 @@ Large system deployments tend to get very complex. In this document we explain the GeneNetwork deployment system which is based on GNU Guix -(see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). +(see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install +GN with all its files and dependencies. * Binary deployment (WIP) -- cgit v1.2.3 From b4376b2a4a1a03abec6020275253ad3c9f25a6e7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 5 Mar 2016 16:12:18 +0000 Subject: README --- doc/README.org | 168 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 144 insertions(+), 24 deletions(-) diff --git a/doc/README.org b/doc/README.org index fd19141c..ecc9b799 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,10 +2,19 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#binary-deployment-wip][Binary deployment (WIP)]] - - [[#install-genenetwork-server][Install genenetwork server]] + - [[#binary-deployment][Binary deployment]] + - [[#install-guix-using-a-tar-ball][Install Guix using a tar ball]] + - [[#fix-locale][Fix locale]] + - [[#authorize-our-archives][Authorize our archives]] + - [[#download-and-install-the-gn2-archive][Download and install the GN2 archive]] + - [[#source-deployment][Source deployment]] + - [[#install-guix][Install guix]] + - [[#checkout-the-git-repositories][Checkout the git repositories]] + - [[#run-guix-daemon][Run guix-daemon]] + - [[#install-gn2][Install GN2]] + - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] - - [[#start-the-gn2-server][Start the GN2 server]] + - [[#other][Other]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -15,30 +24,136 @@ explain the GeneNetwork deployment system which is based on GNU Guix (see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install GN with all its files and dependencies. -* Binary deployment (WIP) +* Binary deployment + +Note binary deployment is not working pending a few improvements +to GNU Guix. See source deployment instead. + +** Install Guix using a tar ball GN can be deployed either as a binary tarball or as a GNU Guix package. First install GNU Guix following the instructions of the -[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]] and, update guix with a -'guix pull' and make guix visible in the path. More information -exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]]. -With guix running you should be able to install python, for example. +With guix-daemon running you should be able to install the hello +package: -: guix package -i python2 +: guix package -i hello -This will make python appear in $HOME/.guix-profile/bin/python. Suggested -environment settings can be seen with +** Fix locale -: guix package --search-paths +You may want to + +#+begin_src sh :lang bash +export GUIX_LOCPATH=$HOME/.guix-profile/lib/locale +export LC_ALL=en_US.utf8 +#+end_src sh :lang bash + +** Authorize our archives + +Next add our archive key to guix (as root): + +#+begin_src scheme +echo "(public-key + (ecc + (curve Ed25519) + (q #E9A95686D8437186302E07C7AB9BF3913F435026C2D389AF27D9C66FD6EBB649#) + ) + ) +"|guix archive --authorize +#+end_src scheme + +if you have trouble finding a suitable guix try + +: ls /gnu/store/*guix-*/bin/guix + +and you should be able to use this directly, e.g. + +: alias guix=/gnu/store/632msbms2yaldfnlrb5lbnlnmn9yjisw-guix-0.9.0/bin/guix +: guix --version + +** Download and install the GN2 archive + +Find the archive on + + http://files.genenetwork.org/software/ + +download and install with + +#+begin_src bash +guix archive --import < genenetwork2-data-hash.nar +#+end_src bash + +and you should see a list of packages installing, e.g. + +#+begin_src bash +importing path `/gnu/store/l1zs2drn3zdzl5ysjcmhibcpa35p9zfc-python2-mysqlclient-1.3.7' +importing path `/gnu/store/n7kfg4knibvblggy8ci2liscl7vz5wkg-python2-parallel-1.6.4' +importing path `/gnu/store/qvv16qwlq59gp5d07lwbf5n8ndsi3il3-python2-sqlalchemy-1.0.11' +importing path `/gnu/store/qw872mbmr9ir0a9drv9xw9pvjk05ywwy-python2-xlsxwriter-0.8.4' +importing path `/gnu/store/wc112m1xfy3p08v14bdzay2ki2rirdsm-pylmm-gn2-1.0-3c6d1cac8' +importing path `/gnu/store/zfkcy17c2ks3cd9ks14irdabqvmlfpyn-python2-flask-sqlalchemy-2.1' +importing path `/gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff4' +(etc.) +#+end_src bash + +The following packages need to be added and the R path set + +: export R_LIBS_SITE="/home/wrk/.guix-profile/site-library/" +: guix package -i /gnu/store/w0dqg9dshq53j8xhcnqgvnvms2s6y5k5-r-wgcna-1.49-425bc170cc0873ddbd414675ac40f6d4d724c7cb +: guix package -i /gnu/store/k60bdlm0v7xic88j2z5c1jb1jvc371mn-r-qtl-1.38-4 -** Install genenetwork server +You can add the last one to your profile -Fetch the nar file and install it after adding the key with, for -example +: guix package -i /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff +: export PATH=~/.guix-profile/bin:$PATH +: genenetwork2 -: cat signing-key.pub |.guix-profile/bin/guix archive --authorize -: guix archive --import < guix_gn2-2.0-9e9475053.nar + or run it directly with + +: /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff/bin/genenetwork2 + +* Source deployment + +** Install guix + +Deploying from source is also straightforward. Install GNU Guix using +a binary tar ball as described [[https://github.com/pjotrp/guix-notes][here]]. + +** Checkout the git repositories + +Check out the guix and guix-bioinformatics repositories: + +#+begin_src bash +cd ~ +mkdir genenetwork +cd genenetwork +git clone https://github.com/genenetwork/guix-bioinformatics +git clone --recursive --branch gn-latest https://github.com/genenetwork/guix guix-gn-latest +cd guix-gn-latest +#+end_src bash + +** Run guix-daemon + +At this point you may decide to create, install and run a recent +version of the guix-daemon by compiling the guix repository. Follow +[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. + +** Install GN2 + +#+begin_src bash +env GUIX_PACKAGE_PATH=../guix-bioinformatics/ \ + ./pre-inst-env guix package -i genenetwork2 +#+end_src bash + +** Run GN2 + +#+begin_src bash +export PATH=~/.guix-profile/bin:$PATH +genenetwork2 +#+end_src bash + +will start the default server which listens on port 5003. ** Run MySQL server @@ -75,17 +190,22 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; -** Start the GN2 server -Copy the default_settings.py file and modify +** Other + +Update guix with a 'guix pull' and make guix visible in the path. +More information exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. -: DB_URI = """mysql://gn2:mysql_password@localhost/db_webqtl""" -: SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password/db_webqtl' -: SECRET_KEY = 'secret_key' +With guix running you should be able to install python, for example. + +: guix package -i python2 + +This will make python appear in $HOME/.guix-profile/bin/python. Suggested +environment settings can be seen with + +: guix package --search-paths -Start the server with your settings file -: ./bin/genenetwork2 ~/my_gn2_settings.py * Source deployment and other information on reproducibility -- cgit v1.2.3 From d8b3b06ada09b7aeead3a179673c3121d786ad46 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 5 Mar 2016 16:30:34 +0000 Subject: README --- doc/README.org | 140 ++++++++++----------------------------------------------- 1 file changed, 23 insertions(+), 117 deletions(-) diff --git a/doc/README.org b/doc/README.org index ecc9b799..adb545f6 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,19 +2,13 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#binary-deployment][Binary deployment]] - - [[#install-guix-using-a-tar-ball][Install Guix using a tar ball]] - - [[#fix-locale][Fix locale]] - - [[#authorize-our-archives][Authorize our archives]] - - [[#download-and-install-the-gn2-archive][Download and install the GN2 archive]] - [[#source-deployment][Source deployment]] - [[#install-guix][Install guix]] - [[#checkout-the-git-repositories][Checkout the git repositories]] - - [[#run-guix-daemon][Run guix-daemon]] - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] - - [[#other][Other]] + - [[#notes][Notes]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -24,94 +18,9 @@ explain the GeneNetwork deployment system which is based on GNU Guix (see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install GN with all its files and dependencies. -* Binary deployment - -Note binary deployment is not working pending a few improvements -to GNU Guix. See source deployment instead. - -** Install Guix using a tar ball - -GN can be deployed either as a binary tarball or as a GNU Guix -package. First install GNU Guix following the instructions of the -[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]]. - -With guix-daemon running you should be able to install the hello -package: - -: guix package -i hello - -** Fix locale - -You may want to - -#+begin_src sh :lang bash -export GUIX_LOCPATH=$HOME/.guix-profile/lib/locale -export LC_ALL=en_US.utf8 -#+end_src sh :lang bash - -** Authorize our archives - -Next add our archive key to guix (as root): - -#+begin_src scheme -echo "(public-key - (ecc - (curve Ed25519) - (q #E9A95686D8437186302E07C7AB9BF3913F435026C2D389AF27D9C66FD6EBB649#) - ) - ) -"|guix archive --authorize -#+end_src scheme - -if you have trouble finding a suitable guix try - -: ls /gnu/store/*guix-*/bin/guix - -and you should be able to use this directly, e.g. - -: alias guix=/gnu/store/632msbms2yaldfnlrb5lbnlnmn9yjisw-guix-0.9.0/bin/guix -: guix --version - -** Download and install the GN2 archive - -Find the archive on - - http://files.genenetwork.org/software/ - -download and install with - -#+begin_src bash -guix archive --import < genenetwork2-data-hash.nar -#+end_src bash - -and you should see a list of packages installing, e.g. - -#+begin_src bash -importing path `/gnu/store/l1zs2drn3zdzl5ysjcmhibcpa35p9zfc-python2-mysqlclient-1.3.7' -importing path `/gnu/store/n7kfg4knibvblggy8ci2liscl7vz5wkg-python2-parallel-1.6.4' -importing path `/gnu/store/qvv16qwlq59gp5d07lwbf5n8ndsi3il3-python2-sqlalchemy-1.0.11' -importing path `/gnu/store/qw872mbmr9ir0a9drv9xw9pvjk05ywwy-python2-xlsxwriter-0.8.4' -importing path `/gnu/store/wc112m1xfy3p08v14bdzay2ki2rirdsm-pylmm-gn2-1.0-3c6d1cac8' -importing path `/gnu/store/zfkcy17c2ks3cd9ks14irdabqvmlfpyn-python2-flask-sqlalchemy-2.1' -importing path `/gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff4' -(etc.) -#+end_src bash - -The following packages need to be added and the R path set - -: export R_LIBS_SITE="/home/wrk/.guix-profile/site-library/" -: guix package -i /gnu/store/w0dqg9dshq53j8xhcnqgvnvms2s6y5k5-r-wgcna-1.49-425bc170cc0873ddbd414675ac40f6d4d724c7cb -: guix package -i /gnu/store/k60bdlm0v7xic88j2z5c1jb1jvc371mn-r-qtl-1.38-4 - -You can add the last one to your profile - -: guix package -i /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff -: export PATH=~/.guix-profile/bin:$PATH -: genenetwork2 - - or run it directly with - -: /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff/bin/genenetwork2 +The official installation path is via a GNU Guix installation from a +checked out version of the Guix package tree (this is required for +reproducibility). * Source deployment @@ -120,9 +29,14 @@ You can add the last one to your profile Deploying from source is also straightforward. Install GNU Guix using a binary tar ball as described [[https://github.com/pjotrp/guix-notes][here]]. +If it works you should be able to install a package with + +: guix package -i hello + ** Checkout the git repositories -Check out the guix and guix-bioinformatics repositories: +Check out the two relevant guix and guix-bioinformatics git +repositories: #+begin_src bash cd ~ @@ -133,17 +47,20 @@ git clone --recursive --branch gn-latest https://github.com/genenetwork/guix gui cd guix-gn-latest #+end_src bash -** Run guix-daemon +*** Update the path -At this point you may decide to create, install and run a recent -version of the guix-daemon by compiling the guix repository. Follow -[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. +Now make sure the guix update path is inline with ours + +#+begin_src bash +mkdir -p ~/.config/guix +ln -s ~/genenetwork/guix-gn-latest ~/.config/guix/latest +#+end_src bash ** Install GN2 #+begin_src bash -env GUIX_PACKAGE_PATH=../guix-bioinformatics/ \ - ./pre-inst-env guix package -i genenetwork2 +export GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ +guix package -i genenetwork2 #+end_src bash ** Run GN2 @@ -190,22 +107,11 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; +** Notes -** Other - -Update guix with a 'guix pull' and make guix visible in the path. -More information exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. - -With guix running you should be able to install python, for example. - -: guix package -i python2 - -This will make python appear in $HOME/.guix-profile/bin/python. Suggested -environment settings can be seen with - -: guix package --search-paths - - +At some point you may decide to create, install and run a recent +version of the guix-daemon by compiling the guix repository. Follow +[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. * Source deployment and other information on reproducibility -- cgit v1.2.3 From 87442be548c8779bcc06693490b9e11ccba26dc5 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 5 Mar 2016 16:45:06 +0000 Subject: README --- doc/README.org | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/doc/README.org b/doc/README.org index adb545f6..ec0054e4 100644 --- a/doc/README.org +++ b/doc/README.org @@ -5,10 +5,11 @@ - [[#source-deployment][Source deployment]] - [[#install-guix][Install guix]] - [[#checkout-the-git-repositories][Checkout the git repositories]] + - [[#update-guix][Update guix]] + - [[#update-the-path][Update the path]] - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] - - [[#notes][Notes]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -47,9 +48,15 @@ git clone --recursive --branch gn-latest https://github.com/genenetwork/guix gui cd guix-gn-latest #+end_src bash -*** Update the path +** Update guix -Now make sure the guix update path is inline with ours +At some point you may decide to create, install and run a recent +version of the guix-daemon by compiling the guix repository. Follow +[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. + +** Update the path + +Now make sure the guix update path is in line with ours #+begin_src bash mkdir -p ~/.config/guix @@ -60,7 +67,8 @@ ln -s ~/genenetwork/guix-gn-latest ~/.config/guix/latest #+begin_src bash export GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ -guix package -i genenetwork2 +cd ~/genenetwork/guix-gn-latest +./pre-inst guix package -i genenetwork2 #+end_src bash ** Run GN2 @@ -107,11 +115,6 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; -** Notes - -At some point you may decide to create, install and run a recent -version of the guix-daemon by compiling the guix repository. Follow -[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. * Source deployment and other information on reproducibility -- cgit v1.2.3 From a203e362ec57351f29274b32b7c80dc9392d591d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 6 Mar 2016 11:15:50 +0000 Subject: README --- doc/README.org | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/doc/README.org b/doc/README.org index ec0054e4..efaec8a6 100644 --- a/doc/README.org +++ b/doc/README.org @@ -6,7 +6,6 @@ - [[#install-guix][Install guix]] - [[#checkout-the-git-repositories][Checkout the git repositories]] - [[#update-guix][Update guix]] - - [[#update-the-path][Update the path]] - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] @@ -54,21 +53,12 @@ At some point you may decide to create, install and run a recent version of the guix-daemon by compiling the guix repository. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. -** Update the path - -Now make sure the guix update path is in line with ours - -#+begin_src bash -mkdir -p ~/.config/guix -ln -s ~/genenetwork/guix-gn-latest ~/.config/guix/latest -#+end_src bash - ** Install GN2 #+begin_src bash export GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ cd ~/genenetwork/guix-gn-latest -./pre-inst guix package -i genenetwork2 +./pre-inst-env guix package -i genenetwork2 #+end_src bash ** Run GN2 -- cgit v1.2.3 From 3a34a3de80ce33afb7fb600620d693be26583462 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 6 Mar 2016 13:03:14 +0000 Subject: README --- doc/README.org | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/doc/README.org b/doc/README.org index efaec8a6..85c6da1c 100644 --- a/doc/README.org +++ b/doc/README.org @@ -9,6 +9,7 @@ - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] + - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -68,7 +69,8 @@ export PATH=~/.guix-profile/bin:$PATH genenetwork2 #+end_src bash -will start the default server which listens on port 5003. +will start the default server which listens on port 5003, i.e., +http://localhost:5003/. ** Run MySQL server @@ -105,6 +107,27 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; +** Run your own copy of GN2 + +At some point you may want to fix the source code. Clone the GN2 +repository from https://github.com/genenetwork/genenetwork2_diet + +Inside the repository: + +: cd genenetwork2_diet +: ./bin/genenetwork2 + +Will fire up your local repo http://localhost:5003/ using the +settings in ./etc/default_settings.py + +To override settings create your own from a copy of +default_settings.py and pass it into GN2 with + +: ./bin/genenetwork2 $HOME/my_settings.py + +and everything *should* work. This way we develop against +the exact same dependency graph of software. + * Source deployment and other information on reproducibility -- cgit v1.2.3 From 9b632ebbeb73de1369a49a712d614fd8a2b1ab30 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 6 Mar 2016 15:55:48 +0000 Subject: README --- doc/README.org | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/README.org b/doc/README.org index 85c6da1c..7d4d7e2d 100644 --- a/doc/README.org +++ b/doc/README.org @@ -62,10 +62,29 @@ cd ~/genenetwork/guix-gn-latest ./pre-inst-env guix package -i genenetwork2 #+end_src bash +Make a note of the paths with + +#+begin_src bash +./pre-inst-env guix package --search-paths +#+end_src bash + ** Run GN2 +After setting the paths for the server + #+begin_src bash export PATH=~/.guix-profile/bin:$PATH +export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +export R_LIBS_SITE="$HOME/.guix-profile/site-library/" +export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0" +export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0" +export XDG_DATA_DIRS="$HOME/.guix-profile/share" +export GIO_EXTRA_MODULES="$HOME/.guix-profile/lib/gio/modules" +#+end_src bash + +run the main script (in ~/.guix-profile/bin) + +#+begin_src bash genenetwork2 #+end_src bash @@ -125,8 +144,9 @@ default_settings.py and pass it into GN2 with : ./bin/genenetwork2 $HOME/my_settings.py -and everything *should* work. This way we develop against -the exact same dependency graph of software. +and everything *should* work (note the full path to the settings +file). This way we develop against the exact same dependency graph of +software. * Source deployment and other information on reproducibility -- cgit v1.2.3 From bbbca36baf3739c15e7a276c22c1c37700992e60 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 16 Mar 2016 15:08:56 -0500 Subject: README --- doc/README.org | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/README.org b/doc/README.org index 7d4d7e2d..86c8faaa 100644 --- a/doc/README.org +++ b/doc/README.org @@ -133,6 +133,7 @@ repository from https://github.com/genenetwork/genenetwork2_diet Inside the repository: +: git clone : cd genenetwork2_diet : ./bin/genenetwork2 -- cgit v1.2.3 From 3a3498c519eca58abec3dab08b8309cc15e00e7d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 21 Mar 2016 16:57:11 -0500 Subject: Docs: Database and Guix --- doc/GUIX-archive.org | 106 ++++++++++++++++++++++ doc/database.org | 245 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 351 insertions(+) create mode 100644 doc/GUIX-archive.org create mode 100644 doc/database.org diff --git a/doc/GUIX-archive.org b/doc/GUIX-archive.org new file mode 100644 index 00000000..67ab5cd0 --- /dev/null +++ b/doc/GUIX-archive.org @@ -0,0 +1,106 @@ +* Binary deployment + +Note binary deployment is not working pending a few improvements +to GNU Guix. See source deployment instead. + +** Install Guix using a tar ball + +GN can be deployed either as a binary tarball or as a GNU Guix +package. First install GNU Guix following the instructions of the +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]]. + +With guix-daemon running you should be able to install the hello +package: + +: guix package -i hello + +** Fix locale + +You may want to + +#+begin_src sh :lang bash +export GUIX_LOCPATH=$HOME/.guix-profile/lib/locale +export LC_ALL=en_US.utf8 +#+end_src sh :lang bash + +** Authorize our archives + +Next add our archive key to guix (as root): + +#+begin_src scheme +echo "(public-key + (ecc + (curve Ed25519) + (q #E9A95686D8437186302E07C7AB9BF3913F435026C2D389AF27D9C66FD6EBB649#) + ) + ) +"|guix archive --authorize +#+end_src scheme + +if you have trouble finding a suitable guix try + +: ls /gnu/store/*guix-*/bin/guix + +and you should be able to use this directly, e.g. + +: alias guix=/gnu/store/632msbms2yaldfnlrb5lbnlnmn9yjisw-guix-0.9.0/bin/guix +: guix --version + +** Download and install the GN2 archive + +Find the archive on + + http://files.genenetwork.org/software/ + +download and install with + +#+begin_src bash +guix archive --import < genenetwork2-data-hash.nar +#+end_src bash + +and you should see a list of packages installing, e.g. + +#+begin_src bash +importing path `/gnu/store/l1zs2drn3zdzl5ysjcmhibcpa35p9zfc-python2-mysqlclient-1.3.7' +importing path `/gnu/store/n7kfg4knibvblggy8ci2liscl7vz5wkg-python2-parallel-1.6.4' +importing path `/gnu/store/qvv16qwlq59gp5d07lwbf5n8ndsi3il3-python2-sqlalchemy-1.0.11' +importing path `/gnu/store/qw872mbmr9ir0a9drv9xw9pvjk05ywwy-python2-xlsxwriter-0.8.4' +importing path `/gnu/store/wc112m1xfy3p08v14bdzay2ki2rirdsm-pylmm-gn2-1.0-3c6d1cac8' +importing path `/gnu/store/zfkcy17c2ks3cd9ks14irdabqvmlfpyn-python2-flask-sqlalchemy-2.1' +importing path `/gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff4' +(etc.) +#+end_src bash + +The following packages need to be added and the R path set + +: export R_LIBS_SITE="/home/wrk/.guix-profile/site-library/" +: guix package -i /gnu/store/w0dqg9dshq53j8xhcnqgvnvms2s6y5k5-r-wgcna-1.49-425bc170cc0873ddbd414675ac40f6d4d724c7cb +: guix package -i /gnu/store/k60bdlm0v7xic88j2z5c1jb1jvc371mn-r-qtl-1.38-4 + +You can add the last one to your profile + +: guix package -i /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff +: export PATH=~/.guix-profile/bin:$PATH +: genenetwork2 + + or run it directly with + +: /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff/bin/genenetwork2 + + + +** Other + +Update guix with a 'guix pull' and make guix visible in the path. +More information exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. + +With guix running you should be able to install python, for example. + +: guix package -i python2 + +This will make python appear in $HOME/.guix-profile/bin/python. Suggested +environment settings can be seen with + +: guix package --search-paths + + diff --git a/doc/database.org b/doc/database.org new file mode 100644 index 00000000..6c079225 --- /dev/null +++ b/doc/database.org @@ -0,0 +1,245 @@ +- github Document reduction issue +- github bio-vcf issue + + +* GeneNetwork Database + +All *Data tables are large + +** User access + +GN1 uses access table and GN2 uses user table. + +** AvgMethod + +Probesetfreeze refers to AvgMethod + +** BXDSnPosition + +Snp table (all snps) + +Mapping in GN1 shows snps when you select a chromosome. + +** CaseAttribute(XRef) + +Metadata + +** CeleralINFO_mm6 + +? + +** Chr_Length + +Default mm9, column for mm8 + +** Dataset_mbat + +Menu for BXD (linkouts) + +** DatasetMapInvestigator + +Arthur? + +** DataSets + +Information/metadata + +** DatasetStatus + +Arthur private/public + +** DBList and DBType + +Hooked in API (URL encoding) + +** Docs + +GN2 only (see menu bar) + +** Ensembl* + +Probe information + +(will be deprecated) + +** Genbank + +Linkout and not important + +** GeneCategory + +Not important. GeneWiki notes function classification. + +Deprecate. + +** GeneChip + +** GeneIDXRef + +Interspecies gene comparison + +** GeneList + +Track info + +** Genlist_rn3(3) + +Rat list + +** GeneMap_cuiyan + +Link outs + +** GeneRIF + +Wiki info (nightly updated from NCBI) + +XRef should be foreign keys + +** Geno + +SNP or marker info + +** GenoCode + +Belongs to someone else + +** GenoData + +Allele info + +** GenoFreeze + +Big menu (Freeze refers to menu) + +** GenoSE + +SE standard err, not used + +** GenoXREF + +Very important. Key links between Geno, GenoData + +** GORef + +GO terms + +** H2 + +Heritability for probeset(?) + +** Homologene + +Homology, not used much + +** InbredSet + +Group in menu + +** Indelall, SnpAll, SnpPattern, SnpSource + +Indel Snp browser (variant browser Gn1) + +** Info* + +Infra system PhP + +Data Info button + +Infosystem users has separate entries + +Also Investigators, User, Organizations, + +** LCorrRamin3 + +Lit. Correlations Prof. Ramin + +** Login + +GN2 login info + +** MachineAccessLog + +Old + +** MappingMethod + +GN1 + +** News + +GN2 + +** NStrain + +pheno publishfreeze (menu) + xref (keys) + xref links to publish (pubmed), phenotype, pubishdata +geno genofreeze + xref (keys) + xref links to publish (pubmed), genotype, genodata +probeset/expr. probesetfreeze + xref (keys) + xref links to publish (pubmed), probeset, probesetdata +probe/expr. probefreeze + xref (keys) + xref links to publish (pubmed), probe, probedata + +Each dataset has 3 values (real value (1), number of samples (2), stderr (3)) + +NStrain = number of phenotype samples + +ProbesetFreeze contains all data, incl. metabolomic. + +** Probe + +Probedata - main molecular data. Probesets, metabolome, + +** Publication and publishdata (all pheno) + +Phenotype pubs + +** QuickSearch + +No longer used + +** role + +empty + +** Sample* + +No longer used + +** Species & Strain (should be sample) + +Menu + +** InbredSet + +Menu + +** TableComments + +Metadata on DB + +** Temp* + +User upload data + +** Tissue + +Menu - 3rd level + +** TissueP* + +Correlation tables + +** User collection + +User selection - retained + +** UserPrivilege + +** Vlookup + -- cgit v1.2.3 From 58b1955cd4236acfed35d0adc5d3a620fdb0fa4b Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 16:04:58 +0100 Subject: Cleaning up the WGCNA parameter input template --- wqflask/wqflask/templates/wgcna_setup.html | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html index 8ab8c4a0..0e46f59d 100644 --- a/wqflask/wqflask/templates/wgcna_setup.html +++ b/wqflask/wqflask/templates/wgcna_setup.html @@ -10,43 +10,43 @@ Error:

- Too few phenotypes as input + Too few phenotypes selected to run a WGCNA analysis, please select more.

Please make sure you select enough phenotypes / genes to perform WGCNA, your collection needs to contain at least 4 different phenotypes. You provided {{request.form['trait_list'].split(',')|length}} phenotypes as input
{% else %}
-
+
-
+
-
+
-
-
+
+
-
-
+
+
-
- +
+ {% endif %}
{% endblock %} -- cgit v1.2.3 From 980bc6c5f49814147b03c9385f89a677c19d3aef Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 16:32:18 +0100 Subject: [PATCH 078/100] Adding empty templates for CTL in GN2 --- wqflask/wqflask/templates/wgcna_setup.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html index 0e46f59d..b4a5730d 100644 --- a/wqflask/wqflask/templates/wgcna_setup.html +++ b/wqflask/wqflask/templates/wgcna_setup.html @@ -10,7 +10,7 @@ Error:

- Too few phenotypes selected to run a WGCNA analysis, please select more. + Too few phenotypes as input

Please make sure you select enough phenotypes / genes to perform WGCNA, your collection needs to contain at least 4 different phenotypes. You provided {{request.form['trait_list'].split(',')|length}} phenotypes as input
-- cgit v1.2.3 From 5847ac9fba55f6c75ca3f8ee12047e10efbf8341 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 16:34:32 +0100 Subject: Adding initial file to perform CTL analysis --- wqflask/wqflask/ctl/__init__.py | 0 wqflask/wqflask/ctl/ctl_analysis.py | 87 +++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100755 wqflask/wqflask/ctl/__init__.py create mode 100644 wqflask/wqflask/ctl/ctl_analysis.py diff --git a/wqflask/wqflask/ctl/__init__.py b/wqflask/wqflask/ctl/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py new file mode 100644 index 00000000..13f08a0f --- /dev/null +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -0,0 +1,87 @@ +# CTL analysis for GN2 +# Author / Maintainer: Danny Arends +import sys +from numpy import * +import scipy as sp # SciPy +import rpy2.robjects as ro # R Objects +import rpy2.rinterface as ri + +from base.webqtlConfig import GENERATED_IMAGE_DIR +from utility import webqtlUtil # Random number for the image + +import base64 +import array + +from utility import helper_functions + +from rpy2.robjects.packages import importr +utils = importr("utils") + +## Get pointers to some common R functions +r_library = ro.r["library"] # Map the library function +r_options = ro.r["options"] # Map the options function +r_read_csv = ro.r["read.csv"] # Map the read.csv function +r_dim = ro.r["dim"] # Map the dim function +r_c = ro.r["c"] # Map the c function +r_cat = ro.r["cat"] # Map the cat function +r_paste = ro.r["paste"] # Map the paste function +r_unlist = ro.r["unlist"] # Map the unlist function +r_unique = ro.r["unique"] # Map the unique function +r_length = ro.r["length"] # Map the length function +r_unlist = ro.r["unlist"] # Map the unlist function +r_list = ro.r.list # Map the list function +r_matrix = ro.r.matrix # Map the matrix function +r_seq = ro.r["seq"] # Map the seq function +r_table = ro.r["table"] # Map the table function +r_names = ro.r["names"] # Map the names function +r_sink = ro.r["sink"] # Map the sink function +r_is_NA = ro.r["is.na"] # Map the is.na function +r_file = ro.r["file"] # Map the file function +r_png = ro.r["png"] # Map the png function for plotting +r_dev_off = ro.r["dev.off"] # Map the dev.off function + +class CTL(object): + def __init__(self): + print("Initialization of CTL") + #log = r_file("/tmp/genenetwork_wcgna.log", open = "wt") + #r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file + #r_sink(log, type = "message") + r_library("ctl") # Load WGCNA - Should only be done once, since it is quite expensive + r_options(stringsAsFactors = False) + print("Initialization of CTL done, package loaded in R session") + self.r_CTLscan = ro.r["CTLscan"] # Map the CTLscan function + print("Obtained pointers to CTL functions") + + def run_analysis(self, requestform): + print("Starting CTL analysis on dataset") + + self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] + print("Retrieved phenotype data from database", requestform['trait_list']) + helper_functions.get_trait_db_obs(self, self.trait_db_list) + + self.input = {} # self.input contains the phenotype values we need to send to R + strains = [] # All the strains we have data for (contains duplicates) + traits = [] # All the traits we have data for (should not contain duplicates) + for trait in self.trait_list: + traits.append(trait[0].name) + self.input[trait[0].name] = {} + for strain in trait[0].data: + strains.append(strain) + self.input[trait[0].name][strain] = trait[0].data[strain].value + sys.stdout.flush() + + def render_image(self, results): + print("pre-loading imgage results:", self.results['imgloc']) + imgfile = open(self.results['imgloc'], 'rb') + imgdata = imgfile.read() + imgB64 = imgdata.encode("base64") + bytesarray = array.array('B', imgB64) + self.results['imgdata'] = bytesarray + + def process_results(self, results): + print("Processing WGCNA output") + template_vars = {} + template_vars["input"] = self.input + sys.stdout.flush() + return(dict(template_vars)) + -- cgit v1.2.3 From eb2bee6dce67f7e1ca04bd96a477e2eae8fcb543 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 16:36:32 +0100 Subject: Connecting the CTL analysis to the collection overview, by adding a button --- wqflask/wqflask/templates/collections/view.html | 12 ++++++++++++ wqflask/wqflask/views.py | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index f92d9984..c1563b9c 100755 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -55,6 +55,18 @@
+
+ {% if uc %} + + {% endif %} + +
+ +
+
{% if uc %} diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 7f331492..5e8fb1fe 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -196,6 +196,15 @@ def wcgna_results(): result = wgcna.process_results(wgcnaA) # After the analysis is finished store the result return render_template("wgcna_results.html", **result) # Display them using the template +@app.route("/ctl_setup", methods=('POST',)) +def ctl_setup(): + print("In ctl, request.form is:", request.form) # We are going to get additional user input for the analysis + return render_template("ctl_setup.html", **request.form) # Display them using the template + +@app.route("/ctl_results", methods=('POST',)) +def ctl_results(): + print("In ctl, request.form is:", request.form) + return render_template("ctl_results.html", **result) # Display them using the template @app.route("/news") def news_route(): -- cgit v1.2.3 From 47f0a34f040189898ad03bb2ddd34e25295c3eb8 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 17:57:54 +0100 Subject: Loading the library and mapping the scan function --- wqflask/wqflask/ctl/ctl_analysis.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 13f08a0f..3159ad7f 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -46,10 +46,10 @@ class CTL(object): #log = r_file("/tmp/genenetwork_wcgna.log", open = "wt") #r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file #r_sink(log, type = "message") - r_library("ctl") # Load WGCNA - Should only be done once, since it is quite expensive + r_library("ctl") # Load CTL - Should only be done once, since it is quite expensive r_options(stringsAsFactors = False) print("Initialization of CTL done, package loaded in R session") - self.r_CTLscan = ro.r["CTLscan"] # Map the CTLscan function + self.r_CTLscan = ro.r["CTLscan"] # Map the CTLscan function print("Obtained pointers to CTL functions") def run_analysis(self, requestform): @@ -68,6 +68,8 @@ class CTL(object): for strain in trait[0].data: strains.append(strain) self.input[trait[0].name][strain] = trait[0].data[strain].value + + self.results = {} sys.stdout.flush() def render_image(self, results): @@ -79,7 +81,7 @@ class CTL(object): self.results['imgdata'] = bytesarray def process_results(self, results): - print("Processing WGCNA output") + print("Processing CTL output") template_vars = {} template_vars["input"] = self.input sys.stdout.flush() -- cgit v1.2.3 From 3e1e63f3280a652e57cef5b4a526b321142296ce Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 17:58:23 +0100 Subject: [PATCH 082/100] Testing passing information between the templates --- wqflask/wqflask/templates/ctl_results.html | 9 +++++++++ wqflask/wqflask/templates/ctl_setup.html | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 wqflask/wqflask/templates/ctl_results.html create mode 100644 wqflask/wqflask/templates/ctl_setup.html diff --git a/wqflask/wqflask/templates/ctl_results.html b/wqflask/wqflask/templates/ctl_results.html new file mode 100644 index 00000000..d978b38d --- /dev/null +++ b/wqflask/wqflask/templates/ctl_results.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} +{% block title %}CTL results{% endblock %} + +{% block content %} +
+

CTL Results

+ {{request.form['trait_list'].split(',')|length}} phenotypes as input +
+{% endblock %} diff --git a/wqflask/wqflask/templates/ctl_setup.html b/wqflask/wqflask/templates/ctl_setup.html new file mode 100644 index 00000000..a5f1daf6 --- /dev/null +++ b/wqflask/wqflask/templates/ctl_setup.html @@ -0,0 +1,19 @@ +{% extends "base.html" %} +{% block title %}WCGNA analysis{% endblock %} + +{% block content %} +
+

CTL analysis parameters

+ {{request.form['trait_list'].split(',')|length}} phenotypes as input + + + +
+
+ +
+
+ + +
+{% endblock %} -- cgit v1.2.3 From 2cbdf59b98ae6061ef083ab29c30111d2ea1f853 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 17:59:47 +0100 Subject: Creating the analysis object and passing results to the results template --- wqflask/wqflask/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 5e8fb1fe..87ba8b32 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -48,6 +48,7 @@ from wqflask.correlation_matrix import show_corr_matrix from wqflask.correlation import corr_scatter_plot from wqflask.wgcna import wgcna_analysis +from wqflask.ctl import ctl_analysis from utility import temp_data from utility.tools import TEMPDIR @@ -204,7 +205,10 @@ def ctl_setup(): @app.route("/ctl_results", methods=('POST',)) def ctl_results(): print("In ctl, request.form is:", request.form) - return render_template("ctl_results.html", **result) # Display them using the template + ctl = ctl_analysis.CTL() # Start R, load the package and pointers and create the analysis + ctlA = ctl.run_analysis(request.form) # Start the analysis, a ctlA object should be a separate long running thread + result = ctl.process_results(ctlA) # After the analysis is finished store the result + return render_template("ctl_results.html", **result) # Display them using the template @app.route("/news") def news_route(): -- cgit v1.2.3 From a13baa5fb98d8167e70df7008d0d07b40a05a6b9 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 23:37:13 +0100 Subject: Adding the geno file parser from Zach --- wqflask/utility/genofile_parser.py | 96 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 wqflask/utility/genofile_parser.py diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py new file mode 100644 index 00000000..9dd7b08b --- /dev/null +++ b/wqflask/utility/genofile_parser.py @@ -0,0 +1,96 @@ +# CTL analysis for GN2 +# Author / Maintainer: Danny Arends + +from __future__ import print_function, division, absolute_import +import sys +import os +import glob +import traceback +import gzip + +import simplejson as json + +from pprint import pformat as pf + +class Marker(object): + def __init__(self): + self.name = None + self.chr = None + self.cM = None + self.Mb = None + self.genotypes = [] + + +class ConvertGenoFile(object): + + def __init__(self, input_file): + self.mb_exists = False + self.cm_exists = False + self.markers = [] + + self.latest_row_pos = None + self.latest_col_pos = None + + self.latest_row_value = None + self.latest_col_value = None + self.input_fh = open(input_file) + print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + self.haplotype_notation = { + '@mat': "3", + '@pat': "1", + '@het': "2", + '@unk': "NA" + } + self.configurations = {} + + def process_rows(self): + for self.latest_row_pos, row in enumerate(self.input_fh): + self.latest_row_value = row + # Take care of headers + if not row.strip(): + continue + if row.startswith('#'): + continue + if row.startswith('Chr'): + if 'Mb' in row.split(): + self.mb_exists = True + if 'cM' in row.split(): + self.cm_exists = True + continue + if row.startswith('@'): + key, _separater, value = row.partition(':') + key = key.strip() + value = value.strip() + if key in self.haplotype_notation: + self.configurations[value] = self.haplotype_notation[key] + continue + if not len(self.configurations): + raise EmptyConfigurations + yield row + + def process_csv(self): + for row_count, row in enumerate(self.process_rows()): + row_items = row.split("\t") + + this_marker = Marker() + this_marker.name = row_items[1] + this_marker.chr = row_items[0] + if self.cm_exists and self.mb_exists: + this_marker.cM = row_items[2] + this_marker.Mb = row_items[3] + genotypes = row_items[4:] + elif self.cm_exists: + this_marker.cM = row_items[2] + genotypes = row_items[3:] + elif self.mb_exists: + this_marker.Mb = row_items[2] + genotypes = row_items[3:] + else: + genotypes = row_items[2:] + for item_count, genotype in enumerate(genotypes): + if genotype.upper() in self.configurations: + this_marker.genotypes.append(self.configurations[genotype.upper()]) + else: + this_marker.genotypes.append("NA") + self.markers.append(this_marker.__dict__) + -- cgit v1.2.3 From 179b8eab7528db15f5c97733c94a5e24dbfed661 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 23:38:18 +0100 Subject: Using the geno parser to get/parse the genotypes for the dataset --- wqflask/wqflask/ctl/ctl_analysis.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 3159ad7f..8a2f1954 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -8,11 +8,14 @@ import rpy2.rinterface as ri from base.webqtlConfig import GENERATED_IMAGE_DIR from utility import webqtlUtil # Random number for the image +from utility import genofile_parser # genofile_parser import base64 import array +import csv from utility import helper_functions +from utility.tools import locate from rpy2.robjects.packages import importr utils = importr("utils") @@ -43,13 +46,17 @@ r_dev_off = ro.r["dev.off"] # Map the dev.off function class CTL(object): def __init__(self): print("Initialization of CTL") - #log = r_file("/tmp/genenetwork_wcgna.log", open = "wt") + #log = r_file("/tmp/genenetwork_ctl.log", open = "wt") #r_sink(log) # Uncomment the r_sink() commands to log output from stdout/stderr to a file #r_sink(log, type = "message") r_library("ctl") # Load CTL - Should only be done once, since it is quite expensive r_options(stringsAsFactors = False) print("Initialization of CTL done, package loaded in R session") - self.r_CTLscan = ro.r["CTLscan"] # Map the CTLscan function + self.r_CTLscan = ro.r["CTLscan"] # Map the CTLscan function + self.r_CTLsignificant = ro.r["CTLsignificant"] # Map the CTLsignificant function + self.r_lineplot = ro.r["ctl.lineplot"] # Map the ctl.lineplot function + self.r_CTLnetwork = ro.r["CTLnetwork"] # Map the CTLnetwork function + self.r_CTLprofiles = ro.r["CTLprofiles"] # Map the CTLprofiles function print("Obtained pointers to CTL functions") def run_analysis(self, requestform): @@ -57,18 +64,26 @@ class CTL(object): self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] print("Retrieved phenotype data from database", requestform['trait_list']) + helper_functions.get_trait_db_obs(self, self.trait_db_list) self.input = {} # self.input contains the phenotype values we need to send to R strains = [] # All the strains we have data for (contains duplicates) traits = [] # All the traits we have data for (should not contain duplicates) + genotypebasename = "" for trait in self.trait_list: traits.append(trait[0].name) + if genotypebasename == "": + genotypebasename = trait[1].group.name self.input[trait[0].name] = {} for strain in trait[0].data: strains.append(strain) self.input[trait[0].name][strain] = trait[0].data[strain].value + genofilelocation = locate(genotypebasename + ".geno", "genotype") + parser = genofile_parser.ConvertGenoFile(genofilelocation) + parser.process_csv() + print(parser.markers) self.results = {} sys.stdout.flush() -- cgit v1.2.3 From f5ce8d6abb8b6a48b5ef6af6444f0289327fd95b Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 22 Mar 2016 23:39:23 +0100 Subject: Minor changes to the CTL setup template --- wqflask/wqflask/templates/ctl_setup.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/ctl_setup.html b/wqflask/wqflask/templates/ctl_setup.html index a5f1daf6..070a97b4 100644 --- a/wqflask/wqflask/templates/ctl_setup.html +++ b/wqflask/wqflask/templates/ctl_setup.html @@ -10,7 +10,7 @@
- +
-- cgit v1.2.3 From 3d505d997511cd8f7b9f14510059cb2983edc6d4 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Wed, 23 Mar 2016 23:07:16 +0100 Subject: Parsing the names of the individuals, and coding H as -999 --- wqflask/utility/genofile_parser.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py index 9dd7b08b..67b84dc9 100644 --- a/wqflask/utility/genofile_parser.py +++ b/wqflask/utility/genofile_parser.py @@ -8,6 +8,7 @@ import glob import traceback import gzip + import simplejson as json from pprint import pformat as pf @@ -34,12 +35,12 @@ class ConvertGenoFile(object): self.latest_row_value = None self.latest_col_value = None self.input_fh = open(input_file) - print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + print("!!!!!!!!!!!!!!!!PARSER!!!!!!!!!!!!!!!!!!") self.haplotype_notation = { - '@mat': "3", - '@pat': "1", - '@het': "2", - '@unk': "NA" + '@mat': "1", + '@pat': "2", + '@het': "-999", + '@unk': "-999" } self.configurations = {} @@ -56,6 +57,8 @@ class ConvertGenoFile(object): self.mb_exists = True if 'cM' in row.split(): self.cm_exists = True + skip = 2 + self.cm_exists + self.mb_exists + self.individuals = row.split()[skip:] continue if row.startswith('@'): key, _separater, value = row.partition(':') @@ -88,9 +91,10 @@ class ConvertGenoFile(object): else: genotypes = row_items[2:] for item_count, genotype in enumerate(genotypes): - if genotype.upper() in self.configurations: - this_marker.genotypes.append(self.configurations[genotype.upper()]) + if genotype.upper().strip() in self.configurations: + this_marker.genotypes.append(self.configurations[genotype.upper().strip()]) else: + print("WARNING:", genotype.upper()) this_marker.genotypes.append("NA") self.markers.append(this_marker.__dict__) -- cgit v1.2.3 From 210d8109a4171ddf6ea6a3f70a1bfbea7b327722 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Wed, 23 Mar 2016 23:08:12 +0100 Subject: Adding code to do initial CTL mapping (working on the BXD) --- wqflask/wqflask/ctl/ctl_analysis.py | 99 ++++++++++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 22 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 8a2f1954..f998dc59 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -13,6 +13,10 @@ from utility import genofile_parser # genofile_parser import base64 import array import csv +import itertools + +from base import data_set +from base import trait as TRAIT from utility import helper_functions from utility.tools import locate @@ -26,9 +30,11 @@ r_options = ro.r["options"] # Map the options function r_read_csv = ro.r["read.csv"] # Map the read.csv function r_dim = ro.r["dim"] # Map the dim function r_c = ro.r["c"] # Map the c function +r_t = ro.r["t"] # Map the t function r_cat = ro.r["cat"] # Map the cat function r_paste = ro.r["paste"] # Map the paste function r_unlist = ro.r["unlist"] # Map the unlist function +r_head = ro.r["head"] # Map the unlist function r_unique = ro.r["unique"] # Map the unique function r_length = ro.r["length"] # Map the length function r_unlist = ro.r["unlist"] # Map the unlist function @@ -42,6 +48,12 @@ r_is_NA = ro.r["is.na"] # Map the is.na function r_file = ro.r["file"] # Map the file function r_png = ro.r["png"] # Map the png function for plotting r_dev_off = ro.r["dev.off"] # Map the dev.off function +r_save_image = ro.r["save.image"] # Map the save.image function +r_class = ro.r["class"] # Map the class function +r_save = ro.r["save"] # Map the save function +r_write_table = ro.r["write.table"] # Map the write.table function +r_as_data_frame = ro.r["as.data.frame"] # Map the write.table function +r_data_frame = ro.r["data.frame"] # Map the write.table function class CTL(object): def __init__(self): @@ -61,31 +73,73 @@ class CTL(object): def run_analysis(self, requestform): print("Starting CTL analysis on dataset") - self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] - print("Retrieved phenotype data from database", requestform['trait_list']) - - helper_functions.get_trait_db_obs(self, self.trait_db_list) - - self.input = {} # self.input contains the phenotype values we need to send to R - strains = [] # All the strains we have data for (contains duplicates) - traits = [] # All the traits we have data for (should not contain duplicates) - genotypebasename = "" - for trait in self.trait_list: - traits.append(trait[0].name) - if genotypebasename == "": - genotypebasename = trait[1].group.name - self.input[trait[0].name] = {} - for strain in trait[0].data: - strains.append(strain) - self.input[trait[0].name][strain] = trait[0].data[strain].value - - genofilelocation = locate(genotypebasename + ".geno", "genotype") + self.trait_db_list = [x for x in self.trait_db_list if x] + + datasetname = self.trait_db_list[0].split(":")[1] + dataset = data_set.create_dataset(datasetname) + + genofilelocation = locate(dataset.group.name + ".geno", "genotype") parser = genofile_parser.ConvertGenoFile(genofilelocation) parser.process_csv() - print(parser.markers) + + individuals = parser.individuals + markers = [] + markernames = [] + x = 1 + for marker in parser.markers: + markernames.append(marker["name"]) + markers.append(marker["genotypes"]) + if x == 1: + print marker["genotypes"] + + x = x +1 + + genotypes = list(itertools.chain(*markers)) + print(len(genotypes) / len(individuals), "==", len(parser.markers)) + + rGeno = r_t(ro.r.matrix(r_unlist(genotypes), nrow=len(markernames), ncol=len(individuals), dimnames = r_list(markernames, individuals), byrow=True)) + + print(r_dim(rGeno)) + #self.trait_names = [trait.split(':')[0].strip() for trait in self.trait_db_list] + #print(self.trait_names) + + + traits = [] + for trait in self.trait_db_list: + print("retrieving data for", trait) + if trait != "": + ts = trait.split(':') + gt = TRAIT.GeneralTrait(name = ts[0], dataset_name = ts[1]) + gt.retrieve_sample_data(individuals) + for ind in individuals: + if ind in gt.data.keys(): + traits.append(gt.data[ind].value) + else: + traits.append("-999") + + print len(traits) / len(individuals), "==", len(self.trait_db_list) + rPheno = r_t(ro.r.matrix(r_unlist(traits), nrow=len(self.trait_db_list), ncol=len(individuals), dimnames = r_list(self.trait_db_list, individuals), byrow=True)) + + rPheno = r_data_frame(rPheno) + rGeno = r_data_frame(rGeno) + + print(r_class(rPheno)) + print(r_class(rGeno)) + + + + r_write_table(rPheno, "~/pheno.csv") + r_write_table(rGeno, "~/geno.csv") + res = self.r_CTLscan(rGeno, rPheno) + self.results = {} - sys.stdout.flush() + self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" + self.results['imgloc'] = GENERATED_IMAGE_DIR + self.results['imgurl'] + r_png(self.results['imgloc'], width=1000, height=600) + self.r_lineplot(res, significance = 1) + r_dev_off() + # sys.stdout.flush() def render_image(self, results): print("pre-loading imgage results:", self.results['imgloc']) @@ -98,7 +152,8 @@ class CTL(object): def process_results(self, results): print("Processing CTL output") template_vars = {} - template_vars["input"] = self.input + template_vars["results"] = self.results + self.render_image(self.results) sys.stdout.flush() return(dict(template_vars)) -- cgit v1.2.3 From b1e258e046e40f3ab8b6dba045c866959875000e Mon Sep 17 00:00:00 2001 From: DannyArends Date: Wed, 23 Mar 2016 23:08:32 +0100 Subject: Adding the lineplot image to the output --- wqflask/wqflask/templates/ctl_results.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/wqflask/wqflask/templates/ctl_results.html b/wqflask/wqflask/templates/ctl_results.html index d978b38d..e2adbdfe 100644 --- a/wqflask/wqflask/templates/ctl_results.html +++ b/wqflask/wqflask/templates/ctl_results.html @@ -5,5 +5,13 @@

CTL Results

{{request.form['trait_list'].split(',')|length}} phenotypes as input + + + Embedded Image +
{% endblock %} -- cgit v1.2.3 From fa45eb93c9cf83ea3080c862aeabe512b4cffa2f Mon Sep 17 00:00:00 2001 From: DannyArends Date: Wed, 23 Mar 2016 23:16:35 +0100 Subject: Cleaning code, adding comments --- wqflask/wqflask/ctl/ctl_analysis.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index f998dc59..71c41135 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -76,6 +76,7 @@ class CTL(object): self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] self.trait_db_list = [x for x in self.trait_db_list if x] + # Get the name of the .geno file belonging to the first phenotype datasetname = self.trait_db_list[0].split(":")[1] dataset = data_set.create_dataset(datasetname) @@ -83,28 +84,20 @@ class CTL(object): parser = genofile_parser.ConvertGenoFile(genofilelocation) parser.process_csv() + # Create a genotype matrix individuals = parser.individuals markers = [] markernames = [] - x = 1 for marker in parser.markers: markernames.append(marker["name"]) markers.append(marker["genotypes"]) - if x == 1: - print marker["genotypes"] - x = x +1 - genotypes = list(itertools.chain(*markers)) print(len(genotypes) / len(individuals), "==", len(parser.markers)) rGeno = r_t(ro.r.matrix(r_unlist(genotypes), nrow=len(markernames), ncol=len(individuals), dimnames = r_list(markernames, individuals), byrow=True)) - print(r_dim(rGeno)) - #self.trait_names = [trait.split(':')[0].strip() for trait in self.trait_db_list] - #print(self.trait_names) - - + # Create a phenotype matrix traits = [] for trait in self.trait_db_list: print("retrieving data for", trait) @@ -118,28 +111,25 @@ class CTL(object): else: traits.append("-999") - print len(traits) / len(individuals), "==", len(self.trait_db_list) rPheno = r_t(ro.r.matrix(r_unlist(traits), nrow=len(self.trait_db_list), ncol=len(individuals), dimnames = r_list(self.trait_db_list, individuals), byrow=True)) + # Use a data frame to store the objects rPheno = r_data_frame(rPheno) rGeno = r_data_frame(rGeno) - print(r_class(rPheno)) - print(r_class(rGeno)) - - - - r_write_table(rPheno, "~/pheno.csv") - r_write_table(rGeno, "~/geno.csv") + # Perform the CTL scan res = self.r_CTLscan(rGeno, rPheno) + # Create an image for output self.results = {} self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" self.results['imgloc'] = GENERATED_IMAGE_DIR + self.results['imgurl'] r_png(self.results['imgloc'], width=1000, height=600) self.r_lineplot(res, significance = 1) r_dev_off() - # sys.stdout.flush() + + # Flush any output from R + sys.stdout.flush() def render_image(self, results): print("pre-loading imgage results:", self.results['imgloc']) -- cgit v1.2.3 From f7053ce4e687d5626acd42295b2ca90ebb4c7ade Mon Sep 17 00:00:00 2001 From: DannyArends Date: Thu, 24 Mar 2016 20:33:25 +0100 Subject: Updates to the CTL code adding the significant results to the result object --- wqflask/wqflask/ctl/ctl_analysis.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 71c41135..44bade5c 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -52,8 +52,10 @@ r_save_image = ro.r["save.image"] # Map the save.image function r_class = ro.r["class"] # Map the class function r_save = ro.r["save"] # Map the save function r_write_table = ro.r["write.table"] # Map the write.table function -r_as_data_frame = ro.r["as.data.frame"] # Map the write.table function -r_data_frame = ro.r["data.frame"] # Map the write.table function +r_read_table = ro.r["read.table"] # Map the read.table function +r_as_data_frame = ro.r["as.data.frame"] # Map the write.table function +r_data_frame = ro.r["data.frame"] # Map the write.table function +r_as_numeric = ro.r["as.numeric"] # Map the write.table function class CTL(object): def __init__(self): @@ -69,6 +71,7 @@ class CTL(object): self.r_lineplot = ro.r["ctl.lineplot"] # Map the ctl.lineplot function self.r_CTLnetwork = ro.r["CTLnetwork"] # Map the CTLnetwork function self.r_CTLprofiles = ro.r["CTLprofiles"] # Map the CTLprofiles function + self.r_CTLsignificant = ro.r["CTLsignificant"] # Map the CTLsignificant function print("Obtained pointers to CTL functions") def run_analysis(self, requestform): @@ -111,21 +114,32 @@ class CTL(object): else: traits.append("-999") - rPheno = r_t(ro.r.matrix(r_unlist(traits), nrow=len(self.trait_db_list), ncol=len(individuals), dimnames = r_list(self.trait_db_list, individuals), byrow=True)) + rPheno = r_t(ro.r.matrix(r_as_numeric(r_unlist(traits)), nrow=len(self.trait_db_list), ncol=len(individuals), dimnames = r_list(self.trait_db_list, individuals), byrow=True)) # Use a data frame to store the objects rPheno = r_data_frame(rPheno) rGeno = r_data_frame(rGeno) + r_write_table(rGeno, "~/outputGN/geno.csv") + r_write_table(rPheno, "~/outputGN/pheno.csv") + # Perform the CTL scan - res = self.r_CTLscan(rGeno, rPheno) + res = self.r_CTLscan(rGeno, rPheno, ncores = 6) + + # Get significant interactions + significant = self.r_CTLsignificant(res) + + print(significant[0]) # Create an image for output self.results = {} self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" self.results['imgloc'] = GENERATED_IMAGE_DIR + self.results['imgurl'] + self.results['ctlresult'] = significant + + # Create the lineplot r_png(self.results['imgloc'], width=1000, height=600) - self.r_lineplot(res, significance = 1) + self.r_lineplot(res) r_dev_off() # Flush any output from R -- cgit v1.2.3 From 1f32d6a71beec407ffdf0ef5b17240ee07d68939 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Thu, 24 Mar 2016 20:33:46 +0100 Subject: Working on the output template of CTL --- wqflask/wqflask/templates/ctl_results.html | 25 +++++++++++++++++++++++-- wqflask/wqflask/templates/ctl_setup.html | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/ctl_results.html b/wqflask/wqflask/templates/ctl_results.html index e2adbdfe..914c1d43 100644 --- a/wqflask/wqflask/templates/ctl_results.html +++ b/wqflask/wqflask/templates/ctl_results.html @@ -4,14 +4,35 @@ {% block content %}

CTL Results

- {{request.form['trait_list'].split(',')|length}} phenotypes as input - + {{(request.form['trait_list'].split(',')|length -1)}} phenotypes as input
+

Figure

Embedded Image +

Tabular results

+ + + significant CTL:
+ {% for r in range(results['ctlresult'][0]|length) %} + + {% for c in range(results['ctlresult']|length) %} + + {% endfor %} + + {% endfor %} +
traitmarkertraitLODdCor
+ {% if c > 2 %} + {{results['ctlresult'][c][r]|float|round(2)}} + {% else %} + {{results['ctlresult'][c][r]}} + {% endif %} +
+ + +
{% endblock %} diff --git a/wqflask/wqflask/templates/ctl_setup.html b/wqflask/wqflask/templates/ctl_setup.html index 070a97b4..0ba281c6 100644 --- a/wqflask/wqflask/templates/ctl_setup.html +++ b/wqflask/wqflask/templates/ctl_setup.html @@ -4,7 +4,7 @@ {% block content %}

CTL analysis parameters

- {{request.form['trait_list'].split(',')|length}} phenotypes as input + {{(request.form['trait_list'].split(',')|length -1)}} phenotypes as input
-- cgit v1.2.3 From 1053764068a50e2b95e0f0b07cb4111369ce8ff8 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Thu, 24 Mar 2016 22:08:44 +0100 Subject: Adding parameters, and passing them to the CTL algorithm --- wqflask/wqflask/ctl/ctl_analysis.py | 26 +++++++++++++----- wqflask/wqflask/templates/ctl_setup.html | 46 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 44bade5c..77e34143 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -79,6 +79,18 @@ class CTL(object): self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] self.trait_db_list = [x for x in self.trait_db_list if x] + print("strategy:", requestform.get("strategy")) + strategy = requestform.get("strategy") + + print("nperm:", requestform.get("nperm")) + nperm = int(requestform.get("nperm")) + + print("parametric:", requestform.get("parametric")) + parametric = bool(requestform.get("parametric")) + + print("significance:", requestform.get("significance")) + significance = float(requestform.get("significance")) + # Get the name of the .geno file belonging to the first phenotype datasetname = self.trait_db_list[0].split(":")[1] dataset = data_set.create_dataset(datasetname) @@ -120,26 +132,26 @@ class CTL(object): rPheno = r_data_frame(rPheno) rGeno = r_data_frame(rGeno) - r_write_table(rGeno, "~/outputGN/geno.csv") - r_write_table(rPheno, "~/outputGN/pheno.csv") + # Debug: Print the genotype and phenotype files to disk + #r_write_table(rGeno, "~/outputGN/geno.csv") + #r_write_table(rPheno, "~/outputGN/pheno.csv") # Perform the CTL scan - res = self.r_CTLscan(rGeno, rPheno, ncores = 6) + res = self.r_CTLscan(rGeno, rPheno, strategy = strategy, nperm = nperm, parametric = parametric, ncores = 6) # Get significant interactions - significant = self.r_CTLsignificant(res) - - print(significant[0]) + significant = self.r_CTLsignificant(res, significance = significance) # Create an image for output self.results = {} self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" self.results['imgloc'] = GENERATED_IMAGE_DIR + self.results['imgurl'] self.results['ctlresult'] = significant + self.results['requestform'] = requestform # Store the user specified parameters for the output page # Create the lineplot r_png(self.results['imgloc'], width=1000, height=600) - self.r_lineplot(res) + self.r_lineplot(res, significance = significance) r_dev_off() # Flush any output from R diff --git a/wqflask/wqflask/templates/ctl_setup.html b/wqflask/wqflask/templates/ctl_setup.html index 0ba281c6..9c0d7bea 100644 --- a/wqflask/wqflask/templates/ctl_setup.html +++ b/wqflask/wqflask/templates/ctl_setup.html @@ -8,11 +8,57 @@ + +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+ +
+ +
+
+
+
-- cgit v1.2.3 From 87a9aab9c830d267cb9c9adbb10856054e6e12cc Mon Sep 17 00:00:00 2001 From: DannyArends Date: Mon, 28 Mar 2016 18:32:27 +0200 Subject: Adding small figures per trait, so that users can see CTL and QTL for each individual phenotype --- wqflask/wqflask/ctl/ctl_analysis.py | 35 +++++++++++++++++++++++------- wqflask/wqflask/templates/ctl_results.html | 15 ++++++++++--- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 77e34143..45bc4c6e 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -69,9 +69,10 @@ class CTL(object): self.r_CTLscan = ro.r["CTLscan"] # Map the CTLscan function self.r_CTLsignificant = ro.r["CTLsignificant"] # Map the CTLsignificant function self.r_lineplot = ro.r["ctl.lineplot"] # Map the ctl.lineplot function + self.r_CTLsignificant = ro.r["CTLsignificant"] # Map the CTLsignificant function self.r_CTLnetwork = ro.r["CTLnetwork"] # Map the CTLnetwork function self.r_CTLprofiles = ro.r["CTLprofiles"] # Map the CTLprofiles function - self.r_CTLsignificant = ro.r["CTLsignificant"] # Map the CTLsignificant function + self.r_plotCTLobject = ro.r["plot.CTLobject"] # Map the CTLsignificant function print("Obtained pointers to CTL functions") def run_analysis(self, requestform): @@ -144,26 +145,44 @@ class CTL(object): # Create an image for output self.results = {} - self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" - self.results['imgloc'] = GENERATED_IMAGE_DIR + self.results['imgurl'] + self.results['imgurl1'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" + self.results['imgloc1'] = GENERATED_IMAGE_DIR + self.results['imgurl1'] + self.results['ctlresult'] = significant self.results['requestform'] = requestform # Store the user specified parameters for the output page # Create the lineplot - r_png(self.results['imgloc'], width=1000, height=600) + r_png(self.results['imgloc1'], width=1000, height=600) self.r_lineplot(res, significance = significance) r_dev_off() + n = 2 + for trait in self.trait_db_list: + # Create the QTL like CTL plots + self.results['imgurl' + str(n)] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" + self.results['imgloc' + str(n)] = GENERATED_IMAGE_DIR + self.results['imgurl' + str(n)] + r_png(self.results['imgloc' + str(n)], width=1000, height=600) + self.r_plotCTLobject(res, (n-1), significance = significance, main='Phenotype ' + trait) + r_dev_off() + n = n + 1 + # Flush any output from R sys.stdout.flush() - def render_image(self, results): - print("pre-loading imgage results:", self.results['imgloc']) - imgfile = open(self.results['imgloc'], 'rb') + def loadImage(self, path, name): + print("pre-loading imgage results:", self.results[path]) + imgfile = open(self.results[path], 'rb') imgdata = imgfile.read() imgB64 = imgdata.encode("base64") bytesarray = array.array('B', imgB64) - self.results['imgdata'] = bytesarray + self.results[name] = bytesarray + + def render_image(self, results): + self.loadImage("imgloc1", "imgdata1") + n = 2 + for trait in self.trait_db_list: + self.loadImage("imgloc" + str(n), "imgdata" + str(n)) + n = n + 1 def process_results(self, results): print("Processing CTL output") diff --git a/wqflask/wqflask/templates/ctl_results.html b/wqflask/wqflask/templates/ctl_results.html index 914c1d43..a5cb1c08 100644 --- a/wqflask/wqflask/templates/ctl_results.html +++ b/wqflask/wqflask/templates/ctl_results.html @@ -5,13 +5,22 @@

CTL Results

{{(request.form['trait_list'].split(',')|length -1)}} phenotypes as input
-

Figure

- +

Network Figure

+ Embedded Image +

CTL/QTL Plots for individual traits

+ {% for r in range(2, (request.form['trait_list'].split(',')|length +1)) %} + + Embedded Image + {% endfor %}

Tabular results

-- cgit v1.2.3 From 1c1654bb162ff005e925cea228ae8c6a57e0210b Mon Sep 17 00:00:00 2001 From: DannyArends Date: Mon, 28 Mar 2016 21:28:33 +0200 Subject: Changed the file stem of generated pictures to CTL --- wqflask/wqflask/ctl/ctl_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 45bc4c6e..d79d9bfe 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -145,7 +145,7 @@ class CTL(object): # Create an image for output self.results = {} - self.results['imgurl1'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" + self.results['imgurl1'] = webqtlUtil.genRandStr("CTLline_") + ".png" self.results['imgloc1'] = GENERATED_IMAGE_DIR + self.results['imgurl1'] self.results['ctlresult'] = significant @@ -159,7 +159,7 @@ class CTL(object): n = 2 for trait in self.trait_db_list: # Create the QTL like CTL plots - self.results['imgurl' + str(n)] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" + self.results['imgurl' + str(n)] = webqtlUtil.genRandStr("CTL_") + ".png" self.results['imgloc' + str(n)] = GENERATED_IMAGE_DIR + self.results['imgurl' + str(n)] r_png(self.results['imgloc' + str(n)], width=1000, height=600) self.r_plotCTLobject(res, (n-1), significance = significance, main='Phenotype ' + trait) -- cgit v1.2.3 From 245399f72caae48b4aa2823b6959e78118018709 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 29 Mar 2016 15:15:34 -0500 Subject: README --- doc/README.org | 16 ++++++++++------ doc/database.org | 2 -- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/README.org b/doc/README.org index 86c8faaa..1894d8d3 100644 --- a/doc/README.org +++ b/doc/README.org @@ -24,7 +24,6 @@ checked out version of the Guix package tree (this is required for reproducibility). * Source deployment - ** Install guix Deploying from source is also straightforward. Install GNU Guix using @@ -43,7 +42,7 @@ repositories: cd ~ mkdir genenetwork cd genenetwork -git clone https://github.com/genenetwork/guix-bioinformatics +git clone --branch gn-latest https://github.com/genenetwork/guix-bioinformatics git clone --recursive --branch gn-latest https://github.com/genenetwork/guix guix-gn-latest cd guix-gn-latest #+end_src bash @@ -57,11 +56,13 @@ version of the guix-daemon by compiling the guix repository. Follow ** Install GN2 #+begin_src bash -export GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ -cd ~/genenetwork/guix-gn-latest -./pre-inst-env guix package -i genenetwork2 +env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --fallback #+end_src bash +Note that you can use the genenetwork.org guix substitute caching +server at http://guix.genenetwork.org (which speeds up installs +significantly because all packages are pre-built). + Make a note of the paths with #+begin_src bash @@ -126,6 +127,10 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; +Note that if the mysql connection is not working, try connecting to +the IP address and check server firewall, hosts.allow and mysql IP +configuration. + ** Run your own copy of GN2 At some point you may want to fix the source code. Clone the GN2 @@ -149,7 +154,6 @@ and everything *should* work (note the full path to the settings file). This way we develop against the exact same dependency graph of software. - * Source deployment and other information on reproducibility See the document [[GUIX-Reproducible-from-source.org]]. diff --git a/doc/database.org b/doc/database.org index 6c079225..af4b30eb 100644 --- a/doc/database.org +++ b/doc/database.org @@ -1,6 +1,4 @@ - github Document reduction issue -- github bio-vcf issue - * GeneNetwork Database -- cgit v1.2.3 From 1017f7b45c8102793c6d75959ae1eeb680ed3544 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 29 Mar 2016 15:19:16 -0500 Subject: Main README --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d2d15781..ce435f61 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # GENENETWORK -Source code for the GeneNetwork2 server http://www.genenetwork.org/ +This repository contains the source code for the GeneNetwork server +http://www.genenetwork.org/ (version 2 aka GN2). ## Install @@ -11,13 +12,14 @@ developers). See the [installation docs](doc/README.org). ## License -This GeneNetwork2 source code is released under the Affero General +The GeneNetwork2 source code is released under the Affero General Public License 3 (AGPLv3). See also LICENSE.txt ## More information -For more information visit http://www.genenetwork.org/ and -http://en.wikipedia.org/wiki/Genenetwork +For more information visit http://www.genenetwork.org/ + +## Contact IRC on #genenetwork on irc.freenode.net. -- cgit v1.2.3 From 87cb8afc55a22f795ed2f0df42d9e952cdbaaa2f Mon Sep 17 00:00:00 2001 From: DannyArends Date: Tue, 29 Mar 2016 22:22:45 +0200 Subject: trasnform to transform --- wqflask/wqflask/static/new/css/corr_matrix.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/static/new/css/corr_matrix.css b/wqflask/wqflask/static/new/css/corr_matrix.css index 495ca28c..cd2b0a80 100755 --- a/wqflask/wqflask/static/new/css/corr_matrix.css +++ b/wqflask/wqflask/static/new/css/corr_matrix.css @@ -3,7 +3,7 @@ -moz-transform: rotate(-90deg); -ms-transform: rotate(-90deg); -o-transform: rotate(-90deg); - trasnform: rotate(-90deg); + transform: rotate(-90deg); color: #000 font-size: 22px; height: 50px; -- cgit v1.2.3 From aa47bf9d94c3f5232cabfd1bef2eb3b54241d076 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 4 Apr 2016 11:38:15 +0000 Subject: README: point to gn-latest SHA values --- README.md | 2 +- doc/README.org | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ce435f61..4fe739bf 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ developers). See the [installation docs](doc/README.org). ## License The GeneNetwork2 source code is released under the Affero General -Public License 3 (AGPLv3). See also LICENSE.txt +Public License 3 (AGPLv3). See [LICENSE.txt](LICENSE.txt). ## More information diff --git a/doc/README.org b/doc/README.org index 1894d8d3..5fffa108 100644 --- a/doc/README.org +++ b/doc/README.org @@ -14,14 +14,15 @@ * Introduction -Large system deployments tend to get very complex. In this document we -explain the GeneNetwork deployment system which is based on GNU Guix -(see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install -GN with all its files and dependencies. - -The official installation path is via a GNU Guix installation from a -checked out version of the Guix package tree (this is required for -reproducibility). +Large system deployments can get very complex. In this document we +explain the GeneNetwork version 2 (GN2) reproducible deployment system +which is based on GNU Guix (see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix +system can be used to install GN with all its files and dependencies. + +The official installation path is from a checked out version of the +main Guix package tree and that of the Genenetwork package +tree. Current supported versions can be found as the SHA values of +'gn-latest' branches of [[https://github.com/genenetwork/guix-bioinformatics/tree/gn-latest][Guix bioinformatics]] and [[https://github.com/genenetwork/guix/tree/gn-latest][GNU Guix main]]. * Source deployment ** Install guix -- cgit v1.2.3 From 551b57ca82abf77fad388cb3aead1c3f58fdc43b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 9 Apr 2016 10:43:52 +0000 Subject: Database information --- doc/README.org | 4 +- doc/database.org | 471 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 471 insertions(+), 4 deletions(-) diff --git a/doc/README.org b/doc/README.org index 5fffa108..ee54f781 100644 --- a/doc/README.org +++ b/doc/README.org @@ -43,8 +43,8 @@ repositories: cd ~ mkdir genenetwork cd genenetwork -git clone --branch gn-latest https://github.com/genenetwork/guix-bioinformatics -git clone --recursive --branch gn-latest https://github.com/genenetwork/guix guix-gn-latest +git clone --branch gn-latest https://github.com/genenetwork/guix-bioinformatics +git clone --branch gn-latest --recursive https://github.com/genenetwork/guix guix-gn-latest cd guix-gn-latest #+end_src bash diff --git a/doc/database.org b/doc/database.org index af4b30eb..e06ac1ff 100644 --- a/doc/database.org +++ b/doc/database.org @@ -1,12 +1,189 @@ - github Document reduction issue + * GeneNetwork Database +** Estimated table sizes + + +select table_name,round(((data_length + index_length) / 1024 / 1024), 2) `Size in MB` from information_schema.TABLES where table_schema = "db_webqtl" order by data_length; + ++-------------------------+------------+ +| table_name | Size in MB | ++-------------------------+------------+ +| ProbeSetData | 59358.80 | +| SnpAll | 15484.67 | +| ProbeData | 22405.44 | +| SnpPattern | 9177.05 | +| ProbeSetSE | 14551.02 | +| QuickSearch | 5972.86 | +| ProbeSetXRef | 4532.89 | +| LCorrRamin3 | 18506.53 | +| ProbeSE | 6263.83 | +| ProbeSet | 2880.21 | +| Probe | 2150.30 | +| GenoData | 3291.91 | +| CeleraINFO_mm6 | 989.80 | +| pubmedsearch | 1032.50 | +| ProbeXRef | 743.38 | +| GeneRIF_BASIC | 448.54 | +| BXDSnpPosition | 224.44 | +| EnsemblProbe | 133.66 | +| EnsemblProbeLocation | 105.49 | +| Genbank | 37.71 | +| TissueProbeSetData | 74.42 | +| AccessLog | 42.38 | +| GeneList | 34.11 | +| Geno | 33.90 | +| MachineAccessLog | 28.34 | +| IndelAll | 22.42 | +| PublishData | 22.54 | +| TissueProbeSetXRef | 14.73 | +| ProbeH2 | 13.26 | +| GenoXRef | 22.83 | +| TempData | 8.35 | +| GeneList_rn3 | 5.54 | +| GORef | 4.97 | +| Phenotype | 6.50 | +| temporary | 3.59 | +| InfoFiles | 3.32 | +| Publication | 3.42 | +| Homologene | 5.69 | +| Datasets | 2.31 | +| GeneList_rn33 | 2.61 | +| PublishSE | 4.71 | +| GeneRIF | 2.18 | +| Vlookup | 1.87 | +| H2 | 2.18 | +| PublishXRef | 2.18 | +| NStrain | 4.80 | +| IndelXRef | 2.91 | +| Strain | 1.07 | +| GeneMap_cuiyan | 0.51 | +| user_collection | 0.30 | +| CaseAttributeXRef | 0.44 | +| StrainXRef | 0.56 | +| GeneIDXRef | 0.77 | +| Docs | 0.17 | +| News | 0.17 | +| ProbeSetFreeze | 0.22 | +| GeneRIFXRef | 0.24 | +| Sample | 0.06 | +| login | 0.06 | +| user | 0.04 | +| TableFieldAnnotation | 0.05 | +| DatasetMapInvestigator | 0.05 | +| User | 0.04 | +| ProbeFreeze | 0.06 | +| TableComments | 0.02 | +| Investigators | 0.02 | +| DBList | 0.03 | +| Tissue | 0.02 | +| GeneChip | 0.01 | +| GeneCategory | 0.01 | +| SampleXRef | 0.01 | +| InbredSet | 0.01 | +| SnpAllele_to_be_deleted | 0.00 | +| Organizations | 0.01 | +| PublishFreeze | 0.00 | +| GenoFreeze | 0.00 | +| Chr_Length | 0.01 | +| SnpSource | 0.00 | +| AvgMethod | 0.00 | +| Species | 0.00 | +| Dataset_mbat | 0.00 | +| TissueProbeFreeze | 0.00 | +| EnsemblChip | 0.00 | +| TissueProbeSetFreeze | 0.01 | +| UserPrivilege | 0.00 | +| CaseAttribute | 0.00 | +| MappingMethod | 0.00 | +| DBType | 0.00 | +| InfoFilesUser_md5 | 0.00 | +| GenoCode | 0.00 | +| DatasetStatus | 0.00 | +| GeneChipEnsemblXRef | 0.00 | +| GenoSE | 0.00 | +| user_openids | 0.00 | +| roles_users | 0.00 | +| role | 0.00 | +| Temp | NULL | ++-------------------------+------------+ +97 rows in set, 1 warning (0.01 sec) + All *Data tables are large ** User access -GN1 uses access table and GN2 uses user table. +According to the meta data: + +This table tracks access time and IP addresses. Used for logging in +registered users and tracking cookies. + +# GN1 uses access table and GN2 uses user table (true/false?) + + select * from AccessLog limit 5; ++-------+---------------------+----------------+ +| id | accesstime | ip_address | ++-------+---------------------+----------------+ +| 12174 | 2003-10-28 02:17:41 | 130.120.104.71 | +| 12173 | 2003-10-28 02:16:27 | 130.120.104.71 | +| 3 | 2003-02-22 07:38:33 | 192.117.159.1 | +| 4 | 2003-02-22 07:49:13 | 192.117.159.1 | +| 5 | 2003-02-22 07:51:08 | 192.117.159.1 | ++-------+---------------------+----------------+ + +select * from AccessLog order by accesstime desc limit 5; ++---------+---------------------+---------------+ +| id | accesstime | ip_address | ++---------+---------------------+---------------+ +| 1025735 | 2016-02-08 14:23:29 | 100.43.81.157 | +| 1025734 | 2016-02-08 13:54:28 | 180.76.15.144 | +| 1025733 | 2016-02-08 13:43:37 | 66.249.65.217 | +| 1025732 | 2016-02-08 13:39:50 | 66.249.65.217 | +| 1025731 | 2016-02-08 13:15:46 | 66.249.65.217 | ++---------+---------------------+---------------+ + +Quite a few trait page hits: + +select count(*) from AccessLog; + ++----------+ +| count(*) | ++----------+ +| 1025685 | ++----------+ + +show indexes from AccessLog; ++-----------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | ++-----------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| AccessLog | 0 | PRIMARY | 1 | id | A | 1025685 | NULL | NULL | | BTREE | | | ++-----------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ + +This table is being used by both GN1 and GN2 from the trait pages! + +: grep -ir AccessLog *|grep -e "^gn1\|^gn2"|grep \.py|grep -v doc + +gn1/web/webqtl/showTrait/ShowTraitPage.py: query = "SELECT count(id) FROM AccessLog WHERE ip_address = %s and \ +gn1/web/webqtl/showTrait/ShowTraitPage.py: self.cursor.execute("insert into AccessLog(accesstime,ip_address) values(Now(),%s)" ,user_ip) +gn1/web/webqtl/textUI/cmdClass.py: query = """SELECT count(id) FROM AccessLog WHERE ip_address = %s AND UNIX_TIMESTAMP()-UNIX_TIMESTAMP(accesstime)<86400""" +gn1/web/webqtl/textUI/cmdClass.py: query = """INSERT INTO AccessLog(accesstime,ip_address) values(Now(),%s)""" +gn2/wqflask/wqflask/show_trait/show_trait_page.py: query = "SELECT count(id) FROM AccessLog WHERE ip_address = %s and \ +gn2/wqflask/wqflask/show_trait/show_trait_page.py: self.cursor.execute("insert into AccessLog(accesstime,ip_address) values(Now(),%s)", user_ip) + +When looking at the code in GN1 and GN2 it restricts the daily use of +the trait data page (set to 1,000 - whoever reaches that?). Unlike +mentioned in the schema description, this table does *not* keep track +of cookies. + +From the code it looks like GN2 uses a mixture of Redis and sqlalchemy +to keep track of logged in sessions (see +gn2/wqflask/wqflask/user_manager.py) and cookies through a user_uuid in +model.py. + +In gn2/wqflask/wqflask/templates/collections/view_anonymous.html it +show_trait_page appears to be loaded (need to check). ** AvgMethod @@ -189,10 +366,300 @@ NStrain = number of phenotype samples ProbesetFreeze contains all data, incl. metabolomic. -** Probe +** Phenotype + +This table contains names, full descriptions, and short symbols for +traits and phenotype used primarily in the Published Phenotypes +databases. + +Contains 10k rows, March 2016, of which 5000 are for the BXDs). + +| Id | Pre_publication_description | Post_publication_description | Original_description | Units | Pre_publication_abbreviation | Post_publication_abbreviation | Lab_code | Submitter | Owner | Authorized_Users | ++----+-----------------------------+----------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------+-------------------------------+----------+-------------+-------+------------------+ +| 1 | NULL | Hippocampus weight | Original post publication description: Hippocampus weight | Unknown | NULL | HPCWT | NULL | robwilliams | NULL | robwilliams | +| 2 | NULL | Cerebellum weight | Original post publication description: Cerebellum weight | mg | NULL | CBLWT | NULL | robwilliams | NULL | robwilliams | +| 3 | NULL | Interleukin 1 activity by peritoneal macrophages stimulated with 10 ug/ml lipopolysaccharide [units/100 ug protein] | Original post publication description: Interleukin 1 activity by peritoneal macrophages stimulated with 10 ug/ml lipopolysaccharide [units/100 ug protein] | units/100 ug protein | NULL | IL1Activity | NULL | robwilliams | NULL | robwilliams | +| 4 | NULL | Central nervous system, morphology: Cerebellum weight, whole, bilateral in adults of both sexes [mg] | Original post publication description: Cerebellum weight [mg] | mg | NULL | CBLWT2 | NULL | robwilliams | NULL | robwilliams | +| 5 | NULL | The coat color of 79 BXD RI strain | Original post publication description: The coat color of 79 BXD RI strain | Unknown | NULL | CoatColor | NULL | robwilliams | NULL | robwilliams | ++----+-----------------------------+----------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------+-------------------------------+----------+-------------+-------+------------------+ +5 rows in set (0.00 sec) + +** ProbeData + +Table with fine-grained probe level Affymetrix data only. Contains 1 +billion rows March 2016. This table may be deletable since it is only +used by the Probe Table display in GN1. Not used in GN2 +(double-check). + +In comparison the "ProbeSetData" table contains more molecular assay +data, including probe set data, RNA-seq data, proteomic data, and +metabolomic data. 2.5 billion rows March 2016. In comparison, +ProbeData contains data only for Affymetrix probe level data +(e.g. Exon array probes and M430 probes). + +"ProbeData.StrainId" should be "CaseId" or "SampleId". + +"ProbeData" should probably be "AssayData" or something more neutral. + +select * from ProbeData limit 2; ++--------+----------+---------+ +| Id | StrainId | value | ++--------+----------+---------+ +| 503636 | 42 | 11.6906 | +| 503636 | 43 | 11.4205 | ++--------+----------+---------+ +2 rows in set (0.00 sec) + +select count(*) from ProbeData limit 2; ++-----------+ +| count(*) | ++-----------+ +| 976753435 | ++-----------+ +1 row in set (0.00 sec) + +** ProbeSet + +Comment: PLEASE CHANGE TABLE NAME and rework fields carefully. This is +a terrible table but it works well (RWW March 2016). It is used in +combination with the crucial TRAIT DATA and ANALYSIS pages in GN1 and +GN2. It is also used by annotators using the UPDATE INFO AND DATA web +form to correct and update annotation. It is used by Arthur to enter +new annotation files and metadata for arrays, genes, proteins, +metabolites. The main problem with this table is that it is doing too +much work. + +Initially (2003) this table contained only Affymetrix ProbeSet data +for mouse (U74aV2 initially). Many other array platforms for different +species were added. At least four other major categories of molecular +assays have been added since about 2010. + +1. RNA-seq annotation and sequence data for transcripts using ENSEMBL + identifiers or NCBI NM_XXXXX and NR_XXXXX type identifiers + +2. Protein and peptide annotation and sequence data (see BXD Liver + Proteome data, SRM and SWATH type data) with identifiers such as + "abcb10_q9ji39_t311" for SRM data and "LLGNMIVIVLGHHLGKDFTPAAQAA" + for SWATH data where the latter is just the peptide fragment that + has been quantified. Data first entered in 2015 for work by Rudi + Aebersold and colleagues. + +3. Metabolite annotation and metadata (see BXD Liver Metabolome data) + with identifiers that are usually Mass charge ratios such as + "149.0970810_MZ" + +4. Epigenomic and methylome data (e.g. Human CANDLE Methylation data + with identifiers such as "cg24523000") + +It would make good sense to break this table into four or more types +of molecular assay metadata or annotation tables) (AssayRNA_Anno, +AssayProtein_Anno, AssayMetabolite_Anno, AssayEpigenome_Anno, +AssayMetagenome_Anno), since these assays will have many differences +in annotation content compared to RNAs. + +Some complex logic is used to update contents of this table when +annotators modify and correct the information (for example, updating +gene symbols). These features requested by Rob so that annotating one +gene symbol in one species would annotate all gene symbols in the same +species based on common NCBI GeneID number. For example, changing the +gene alias for one ProbeSet.Id will changing the list of aliases in +all instances with the same gene symbol. + +If the ProbeSet.BlatSeq (or is this ProbSetTargetSeq) is identical +between different ProbeSet.Ids then annotation is forced to be the +same even if the symbol or geneID is different. This "feature" was +implemented when we found many probe sets with identical sequence but +different annotations and identifiers. + + +select count(*) from ProbeSet limit 5; ++----------+ +| count(*) | ++----------+ +| 4351030 | ++----------+ + +| Id | ChipId | Name | TargetId | Symbol | description | Chr | Mb | alias | GeneId | GenbankId | SNP | BlatSeq | TargetSeq | UniGeneId | Strand_Probe | Strand_Gene | OMIM | comments | Probe_set_target_region | Probe_set_specificity | Probe_set_BLAT_score | Probe_set_Blat_Mb_start | Probe_set_Blat_Mb_end | Probe_set_strand | Probe_set_Note_by_RW | flag | Symbol_H | description_H | chromosome_H | MB_H | alias_H | GeneId_H | chr_num | name_num | Probe_Target_Description | RefSeq_TranscriptId | Chr_mm8 | Mb_mm8 | Probe_set_Blat_Mb_start_mm8 | Probe_set_Blat_Mb_end_mm8 | HomoloGeneID | Biotype_ENS | ProteinID | ProteinName | Flybase_Id | HMDB_ID | Confidence | ChEBI_ID | ChEMBL_ID | CAS_number | PubChem_ID | ChemSpider_ID | UNII_ID | EC_number | KEGG_ID | Molecular_Weight | Nugowiki_ID | Type | Tissue | PrimaryName | SecondaryNames | PeptideSequence | ++------+--------+----------+----------+--------+----------------------------------------------+------+-----------+----------+--------+-----------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+--------------+-------------+--------+----------+-------------------------+-----------------------+----------------------+-------------------------+-----------------------+------------------+----------------------+------+----------+---------------+--------------+------+---------+----------+---------+----------+--------------------------+---------------------+---------+-----------+-----------------------------+---------------------------+--------------+-------------+-----------+-------------+------------+---------+------------+----------+-----------+------------+------------+---------------+---------+-----------+---------+------------------+-------------+------+--------+-------------+----------------+-----------------+ +| 7282 | 1 | 93288_at | NULL | Arpc2 | actin related protein 2/3 complex, subunit 2 | 1 | 74.310961 | AK008777 | 76709 | AI835883 | 0 | CCGACTTCCTTAAGGTGCTCAACCGGACTGCTTGCTACTGGATAATCGTGAGGGATTCTCCATTTGGGTTCCATTTTGTACGAGTTTGGCAAATAACCTGCAGAAACGAGCTGTGCTTGCAAGGACTTGATAGTTCCTAATCCTTTTCCAAGCTGTTTGCTTTGCAATATGT | ccgacttccttaaggtgctcaaccgtnnnnnnccnannnnccnagaaaaaagaaatgaaaannnnnnnnnnnnnnnnnnnttcatcccgctaactcttgggaactgaggaggaagcgctgtcgaccgaagnntggactgcttgctactggataatcgtnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnntgagggattctccatttgggttccattttgtacgagtttggcaaataacctgcagaaacgagctgtgcttgcaaggacttgatagttcctaagaattanaanaaaaaaaanaanttccacttgatcaanttaattcccttttatttttcctccctcantccccttccttttccaagctgtttgctttgcaatatgt | Mm.337038 | + | | 604224 | | NULL | 8.45 | 169 | 74.310961 | 74.31466 | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | 1 | 93288 | NULL | XM_129773 | 1 | 74.197594 | 74.197594 | 74.201293 | 4187 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ++------+--------+----------+----------+--------+----------------------------------------------+------+-----------+----------+--------+-----------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+--------------+-------------+--------+----------+-------------------------+-----------------------+----------------------+-------------------------+-----------------------+------------------+----------------------+------+----------+---------------+--------------+------+---------+----------+---------+----------+--------------------------+---------------------+---------+-----------+-----------------------------+---------------------------+--------------+-------------+-----------+-------------+------------+---------+------------+----------+-----------+------------+------------+---------------+---------+-----------+---------+------------------+-------------+------+--------+-------------+----------------+-----------------+ +2 rows in set (0.00 sec) + + + + +** ProbeSetData Probedata - main molecular data. Probesets, metabolome, +Almost all important molecular assay data is in this table including +probe set data, RNA-seq data, proteomic data, and metabolomic +data. 2.5 billion rows March 2016. In comparison, ProbeData contains +data only for Affymetrix probe level data (e.g. Exon array probes and +M430 probes). + +select count(*) from ProbeSetData limit 5; ++---------------+ +| count(*) | ++---------------+ +| 2,510,566,472 | ++---------------+ + + +select * from ProbeSetData limit 5; ++----+----------+-------+ +| Id | StrainId | value | ++----+----------+-------+ +| 1 | 1 | 5.742 | +| 1 | 2 | 5.006 | +| 1 | 3 | 6.079 | +| 1 | 4 | 6.414 | +| 1 | 5 | 4.885 | ++----+----------+-------+ + +show indexes from ProbeSetData; ++--------------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | ++--------------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| ProbeSetData | 0 | DataId | 1 | Id | A | 34868978 | NULL | NULL | | BTREE | | | +| ProbeSetData | 0 | DataId | 2 | StrainId | A | 2510566472 | NULL | NULL | | BTREE | | | ++--------------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ + +select * from Strain limit 5; ++----+----------+----------+-----------+--------+-------+ +| Id | Name | Name2 | SpeciesId | Symbol | Alias | ++----+----------+----------+-----------+--------+-------+ +| 1 | B6D2F1 | B6D2F1 | 1 | NULL | NULL | +| 2 | C57BL/6J | C57BL/6J | 1 | B6J | NULL | +| 3 | DBA/2J | DBA/2J | 1 | D2J | NULL | +| 4 | BXD1 | BXD1 | 1 | NULL | NULL | +| 5 | BXD2 | BXD2 | 1 | NULL | NULL | ++----+----------+----------+-----------+--------+-------+ + +show indexes from Strain; ++--------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | ++--------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| Strain | 0 | PRIMARY | 1 | Id | A | 14368 | NULL | NULL | | BTREE | | | +| Strain | 0 | Name | 1 | Name | A | 14368 | NULL | NULL | YES | BTREE | | | +| Strain | 0 | Name | 2 | SpeciesId | A | 14368 | NULL | NULL | | BTREE | | | +| Strain | 1 | Symbol | 1 | Symbol | A | 14368 | NULL | NULL | YES | BTREE | | | ++--------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ + +A typical query may look like + +SELECT Strain.Name, ProbeSetData.value, ProbeSetSE.error, ProbeSetData.Id + FROM (ProbeSetData, ProbeSetFreeze, Strain, ProbeSet, ProbeSetXRef) + left join ProbeSetSE on + (ProbeSetSE.DataId = ProbeSetData.Id AND ProbeSetSE.StrainId = ProbeSetData.StrainId) + WHERE + ProbeSetFreeze.name = 'B139_K_1206_M' AND + ProbeSetXRef.ProbeSetId = ProbeSet.Id AND + ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND + ProbeSetXRef.DataId = ProbeSetData.Id AND + ProbeSetData.StrainId = Strain.Id + Order BY Strain.Name + ++-------+-------+-------+----------+ +| Name | value | error | Id | ++-------+-------+-------+----------+ +| SM001 | 38.3 | NULL | 25309550 | +| SM001 | 2.7 | NULL | 25309520 | +| SM001 | 20.3 | NULL | 25309507 | +| SM001 | 125.8 | NULL | 25309511 | +| SM001 | 8.2 | NULL | 25309534 | ++-------+-------+-------+----------+ +5 rows in set (22.28 sec) + +select * from ProbeSetFreeze limit 5; ++----+---------------+-------+-------------+---------------------------------+---------------------------------------------+-------------------------+------------+-----------+--------+-----------------+-----------------+-----------+ +| Id | ProbeFreezeId | AvgID | Name | Name2 | FullName | ShortName | CreateTime | OrderList | public | confidentiality | AuthorisedUsers | DataScale | ++----+---------------+-------+-------------+---------------------------------+---------------------------------------------+-------------------------+------------+-----------+--------+-----------------+-----------------+-----------+ +| 1 | 3 | 1 | Br_U_0803_M | BXDMicroArray_ProbeSet_August03 | UTHSC Brain mRNA U74Av2 (Aug03) MAS5 | Brain U74Av2 08/03 MAS5 | 2003-08-01 | NULL | 0 | 0 | NULL | log2 | +| 2 | 10 | 1 | Br_U_0603_M | BXDMicroArray_ProbeSet_June03 | UTHSC Brain mRNA U74Av2 (Jun03) MAS5 | Brain U74Av2 06/03 MAS5 | 2003-06-01 | NULL | 0 | 0 | NULL | log2 | +| 3 | 8 | 1 | Br_U_0303_M | BXDMicroArray_ProbeSet_March03 | UTHSC Brain mRNA U74Av2 (Mar03) MAS5 | Brain U74Av2 03/03 MAS5 | 2003-03-01 | NULL | 0 | 0 | NULL | log2 | +| 4 | 5 | 1 | Br_U_0503_M | BXDMicroArray_ProbeSet_May03 | UTHSC Brain mRNA U74Av2 (May03) MAS5 | Brain U74Av2 05/03 MAS5 | 2003-05-01 | NULL | 0 | 0 | NULL | log2 | +| 5 | 4 | 1 | HC_U_0303_M | GNFMicroArray_ProbeSet_March03 | GNF Hematopoietic Cells U74Av2 (Mar03) MAS5 | GNF U74Av2 03/03 MAS5 | 2003-03-01 | NULL | 0 | 0 | NULL | log2 | ++----+---------------+-------+-------------+---------------------------------+---------------------------------------------+-------------------------+------------+-----------+--------+-----------------+-----------------+-----------+ + + select * from ProbeSetXRef limit 5; ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ +| ProbeSetFreezeId | ProbeSetId | DataId | Locus_old | LRS_old | pValue_old | mean | se | Locus | LRS | pValue | additive | h2 | ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ +| 1 | 1 | 1 | 10.095.400 | 13.3971627898894 | 0.163 | 5.48794285714286 | 0.08525787814808819 | rs13480619 | 12.590069931048001 | 0.269 | -0.28515625 | NULL | +| 1 | 2 | 2 | D15Mit189 | 10.042057464356201 | 0.431 | 9.90165714285714 | 0.0374686634976217 | CEL-17_50896182 | 10.5970737900941 | 0.304 | -0.11678333333333299 | NULL | +| 1 | 3 | 3 | D5Mit139 | 5.43678531742749 | 0.993 | 7.83948571428571 | 0.0457583416912569 | rs13478499 | 6.0970532702754 | 0.988 | 0.112957489878542 | NULL | +| 1 | 4 | 4 | D1Mit511 | 9.87815279480766 | 0.483 | 8.315628571428569 | 0.0470396593931327 | rs6154379 | 11.774867551173099 | 0.286 | -0.157113725490196 | NULL | +| 1 | 5 | 5 | D16H21S16 | 10.191723834264499 | 0.528 | 9.19345714285714 | 0.0354801718293322 | rs4199265 | 10.923263374016202 | 0.468 | 0.11476470588235299 | NULL | ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ + + +Note that the following unlimited search is very slow: + +select max(value) from ProbeSetData; + ++------------+ +| max(value) | ++------------+ +| 26436006 | ++------------+ +1 row in set (2 min 16.31 sec) + +which is in some form is used in the search page, see [[https://github.com/genenetwork/genenetwork2_diet/blob/master/wqflask/wqflask/do_search.py#L811][the search code]]. + + +*** Improvements? + +Suggestions on the schema page: + +"StrainId" should be "CaseId" or "SampleId". + +"ProbeSetData" should probably be "AssayData" or something more neutral. + +*** Comments + +I think the ProbeSetData table should be generalized to a 'phenotypes' +table with an 'sample_id' column and a 'value' column. + +A new table 'samples' will link each sample against an 'experiment', +an 'individual' and which in turn can link to a 'strain'. + +Experiment is here in a wide sense, GTex can be one - I don't want to +use dataset ;) + +This means a (slight) reordering: + +phenotypes: (id), sample_id, value +samples: experiment_id, individual_id +experiments: name, version +individual: strain_id +strains: species_id +species: ... + +ProbeData is also interesting, because it has the same structure as +ProbeSetData, but only contains microarrays. This tables should be one +(when we clear up the cross-referencing) as they both contain +phenotype values. Both are large tables. + +PublishData is another phenotype table with values only which can be +merged into that same table. + +So we have phenotype data in 3 tables with exactly the same +layout. There is also TissueProbeSet*, but we'll ignore those for +now. I think we should merge these into one and have the sample ref +refer to the type of data (probeset, probe, metabolomics, +whatever). These are all phenotype values and by having them split +into different tables they won't play well when looking for +correlations. + +ProbeSet contains the metadata on the probes and should (eventually) +move into NoSQL. There is plenty redundancy in that table now. + +I know it is going to be a pain to reorganize the database, but if we +want to use it in the long run we are going to have to simplify it. + + + ** Publication and publishdata (all pheno) Phenotype pubs -- cgit v1.2.3 From 6d29f03f47f542a5e09943c9664e991f313d7c49 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 10:26:35 +0000 Subject: No longer ignore webqtlConfig.py - it should be genereic anyway --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 35be2d7b..4586a167 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ *~ web/new_genotypes/HSNIH.json wqflask/secure_server.py -wqflask/base/webqtlConfig.py build/ dist/ *.egg-info/ -- cgit v1.2.3 From 9164ffc23879a8f9338939c06c36d3d49c3c899a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 10:26:54 +0000 Subject: Add file --- wqflask/base/webqtlConfig.py | 84 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 wqflask/base/webqtlConfig.py diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py new file mode 100755 index 00000000..cdce119a --- /dev/null +++ b/wqflask/base/webqtlConfig.py @@ -0,0 +1,84 @@ +#########################################' +# Environment Variables - public +# +# Note: much of this needs to handled by the settings/environment +# scripts. But rather than migrating everything in one go, we'll +# take it a step at a time. First the hard coded paths get replaced +# with those in utility/tools.py +# +######################################### + +from utility.tools import mk_dir, assert_dir, flat_files, TEMPDIR + +#Debug Level +#1 for debug, mod python will reload import each time +DEBUG = 1 + +#USER privilege +USERDICT = {'guest':1,'user':2, 'admin':3, 'root':4} + +#minimum number of informative strains +KMININFORMATIVE = 5 + +#maximum number of traits for interval mapping +MULTIPLEMAPPINGLIMIT = 11 + +#maximum number of traits for correlation +MAXCORR = 100 + +#Daily download limit from one IP +DAILYMAXIMUM = 1000 + +#maximum LRS value +MAXLRS = 460.0 + +#temporary data life span +MAXLIFE = 86400 + +#MINIMUM Database public value +PUBLICTHRESH = 0 + +#NBCI address +NCBI_LOCUSID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" +UCSC_REFSEQ = "http://genome.cse.ucsc.edu/cgi-bin/hgGene?db=%s&hgg_gene=%s&hgg_chrom=chr%s&hgg_start=%s&hgg_end=%s" +GENBANK_ID = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=Nucleotide&cmd=search&doptcmdl=DocSum&term=%s" +OMIM_ID = "http://www.ncbi.nlm.nih.gov/omim/%s" +UNIGEN_ID = "http://www.ncbi.nlm.nih.gov/UniGene/clust.cgi?ORG=%s&CID=%s"; +HOMOLOGENE_ID = "http://www.ncbi.nlm.nih.gov/sites/entrez?Db=homologene&Cmd=DetailsSearch&Term=%s" +PUBMEDLINK_URL = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids=%s&dopt=Abstract" +UCSC_POS = "http://genome.ucsc.edu/cgi-bin/hgTracks?clade=mammal&org=%s&db=%s&position=chr%s:%s-%s&pix=800&Submit=submit" +UCSC_BLAT = 'http://genome.ucsc.edu/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' +UTHSC_BLAT = 'http://ucscbrowser.genenetwork.org/cgi-bin/hgBlat?org=%s&db=%s&type=0&sort=0&output=0&userSeq=%s' +UCSC_GENOME = "http://genome.ucsc.edu/cgi-bin/hgTracks?db=%s&position=chr%s:%d-%d&hgt.customText=http://web2qtl.utmem.edu:88/snp/chr%s" +ENSEMBLE_BLAT = 'http://www.ensembl.org/Mus_musculus/featureview?type=AffyProbe&id=%s' +DBSNP = 'http://www.ncbi.nlm.nih.gov/SNP/snp_ref.cgi?type=rs&rs=%s' +UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s&hgt.customText=http://gbic.biol.rug.nl/~ralberts/tracks/%s/%s" +GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit" +ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s" + +# HTMLPATH = GNROOT + 'genotype_files/' +# PYLMM_PATH +# IMGDIR = GNROOT + '/wqflask/wqflask/static/output/' + +# Temporary storage: +TMPDIR = mk_dir(TEMPDIR+'/gn2/') +CACHEDIR = mk_dir(TEMPDIR+'/cache/') +# We can no longer write into the git tree: +GENERATED_IMAGE_DIR = mk_dir(TMPDIR+'/generate/') +GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generate_text/') + +# Flat file directories +GENODIR = flat_files('genotype')+'/' +JSON_GENODIR = assert_dir(GENODIR+'json/') + +# SITENAME = 'GN' +# PORTADDR = "http://50.16.251.170" +# BASEHREF = '' + +INFOPAGEHREF = '/dbdoc/%s.html' +CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR' +SCRIPTFILE = 'main.py' + +# GLOSSARYFILE = "/glossary.html" +# REFRESHSTR = '' +# REFRESHDIR = '%s' + SCRIPTFILE +'?sid=%s' -- cgit v1.2.3 From 28ccde8e417b965710bbeef48d468a6b717c10a4 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 10:44:10 +0000 Subject: A few fixes to get the webserver running again --- doc/README.org | 11 ++++++++--- wqflask/utility/tools.py | 1 - wqflask/wqflask/marker_regression/gemma_mapping.py | 4 +--- wqflask/wqflask/marker_regression/marker_regression.py | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/README.org b/doc/README.org index ee54f781..3b96717f 100644 --- a/doc/README.org +++ b/doc/README.org @@ -134,12 +134,17 @@ configuration. ** Run your own copy of GN2 -At some point you may want to fix the source code. Clone the GN2 -repository from https://github.com/genenetwork/genenetwork2_diet +At some point you may want to fix the source code. Assuming you have +Guix and Genenetwork2 installed (as described above) clone the GN2 +repository from https://github.com/genenetwork/genenetwork2_diet + +Copy the paths into your terminal from (mainly so PYTHON_PATH and +R_LIBS_SITE are set) + +: guix package --search-paths Inside the repository: -: git clone : cd genenetwork2_diet : ./bin/genenetwork2 diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 9405a9c6..dd8c4a1e 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -5,7 +5,6 @@ import os import sys from wqflask import app - def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with get_valid_path is a function checking whether the diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index ae426621..8fb086c1 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -1,9 +1,7 @@ import os from base import webqtlConfig -from utility.tools import gemma_command - -GEMMA_PATH,GEMMA_COMMAND = gemma_command() +from utility.tools import GEMMA_COMMAND def run_gemma(this_dataset, samples, vals): """Generates p-values for each marker using GEMMA""" diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 39d7c78a..97949f93 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -1103,5 +1103,5 @@ def get_markers_from_csv(included_markers, p_values, group_name): return markers - if __name__ == '__main__': - import cPickle as pickle +if __name__ == '__main__': + import cPickle as pickle -- cgit v1.2.3 From d531829d6184f444cb9257fe2e220ef3b9f462ab Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 10:46:57 +0000 Subject: Fix typo --- wqflask/wqflask/show_trait/show_trait.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index e80cf191..79f1c867 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -159,7 +159,7 @@ class ShowTrait(object): def get_mapping_methods(self): '''Only display mapping methods when the dataset group's genotype file exists''' def check_plink_gemma(): - if (os.path.isfile(MAPPYING_PATH+"/"+self.dataset.group.name+".bed") and + if (os.path.isfile(MAPPING_PATH+"/"+self.dataset.group.name+".bed") and os.path.isfile(MAPPING_PATH+"/"+self.dataset.group.name+".map")): return True else: -- cgit v1.2.3 From 9001d7575cef0dcb0aaecfa15e8c242873fae01e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 26 Apr 2016 11:28:51 +0000 Subject: README: add environment paths info --- doc/README.org | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/README.org b/doc/README.org index 3b96717f..494e2a70 100644 --- a/doc/README.org +++ b/doc/README.org @@ -11,6 +11,8 @@ - [[#run-mysql-server][Run MySQL server]] - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] + - [[#trouble-shooting][Trouble shooting]] + - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] * Introduction @@ -138,8 +140,8 @@ At some point you may want to fix the source code. Assuming you have Guix and Genenetwork2 installed (as described above) clone the GN2 repository from https://github.com/genenetwork/genenetwork2_diet -Copy the paths into your terminal from (mainly so PYTHON_PATH and -R_LIBS_SITE are set) +Copy-paste the paths into your terminal (mainly so PYTHON_PATH and +R_LIBS_SITE are set) from the information given by guix: : guix package --search-paths @@ -164,3 +166,21 @@ software. See the document [[GUIX-Reproducible-from-source.org]]. +* Trouble shooting + +** ImportError: No module named jinja2 + +If you have all the Guix packages installed this error points out that +the environment variables are not set. Copy-paste the paths into your +terminal (mainly so PYTHON_PATH and R_LIBS_SITE are set) from the +information given by guix: + +: guix package --search-paths + +On one system: + +: export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +: export R_LIBS_SITE="$HOME/.guix-profile/site-library/" +: export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0" + +and perhaps a few more. -- cgit v1.2.3 From 95a797509caae727ae204861e5aa9402fdd9ecc4 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 26 Apr 2016 11:29:42 +0000 Subject: Interval mapping: fix font and image path --- wqflask/wqflask/marker_regression/marker_regression_gn1.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 11e312b2..97e4b934 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -42,6 +42,7 @@ from utility import webqtlUtil from utility import helper_functions from utility import Plot from wqflask.interval_analyst import GeneUtil +from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR, GENERATED_IMAGE_DIR ######################################### # Inteval Mapping Plot Page @@ -930,7 +931,7 @@ class MarkerRegression(object): bootScale = bootScale[:-1] + [highestPercent] bootOffset = 50*fontZoom - bootScaleFont=Font(ttf="verdana",size=13*fontZoom,bold=0) + bootScaleFont=pid.Font(ttf="verdana",size=13*fontZoom,bold=0) canvas.drawRect(canvas.size[0]-bootOffset,yZero-bootHeightThresh,canvas.size[0]-bootOffset-15*zoom,yZero,fillColor = pid.yellow) canvas.drawLine(canvas.size[0]-bootOffset+4, yZero, canvas.size[0]-bootOffset, yZero, color=pid.black) canvas.drawString('0%' ,canvas.size[0]-bootOffset+10,yZero+5,font=bootScaleFont,color=pid.black) @@ -2581,7 +2582,7 @@ class MarkerRegression(object): Plot.plotBar(myCanvas, perm_output, XLabel=self.LRS_LOD, YLabel='Frequency', title=' Histogram of Permutation Test') filename= webqtlUtil.genRandStr("Reg_") - myCanvas.save(webqtlConfig.IMGDIR+filename, format='gif') + myCanvas.save(GENERATED_IMAGE_DIR+filename, format='gif') return filename -- cgit v1.2.3 From ebf1db54525a6f344725717f83dfeb1f7b1ada18 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 26 Apr 2016 16:23:38 +0000 Subject: Test: main page and first search --- bin/test-website | 37 +++++++++++++++++++++++++++++++++++++ test/lib/main_web_functionality.rb | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100755 bin/test-website create mode 100644 test/lib/main_web_functionality.rb diff --git a/bin/test-website b/bin/test-website new file mode 100755 index 00000000..b766edd8 --- /dev/null +++ b/bin/test-website @@ -0,0 +1,37 @@ +#!/usr/bin/env ruby + + +USAGE = <0 + ARGV.shift + else + "http://localhost:5003" + end + +$stderr.print "Testing <",$host,">\n" + +require 'mechanize' +require 'minitest/spec' +require 'minitest/autorun' +require 'main_web_functionality' + + diff --git a/test/lib/main_web_functionality.rb b/test/lib/main_web_functionality.rb new file mode 100644 index 00000000..03c76d42 --- /dev/null +++ b/test/lib/main_web_functionality.rb @@ -0,0 +1,32 @@ +class MainWebFunctionality +end + +describe MainWebFunctionality do + before do + @agent = Mechanize.new + page = @agent.get($host) + # p page + @form = page.forms[1] + + end + + describe MainWebFunctionality do + it "The main page contains a search button" do + @form.buttons[0].value.must_equal "Search" + end + + # http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult + it "Search for MEAN=(15 16) LRS=(23 46)" do + @form.fields[2].value = "MEAN=(15 16) LRS=(23 46)" + @form.fields[3].value = "mouse" + @form.fields[4].value = "BXD" + @form.fields[5].value = "Hippocampus mRNA" + @form.fields[6].value = "HC_M2_0606_P" + @searchpage = @agent.submit(@form, @form.buttons.first) + probe_link = @searchpage.links.find { |l| l.text =~ /1435395_s_at/ } + probe_link.uri.to_s.must_equal "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" + end + end + +end + -- cgit v1.2.3 From c9c0b876318ae0c192a2895c59baeb394a77295e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 27 Apr 2016 09:40:24 +0000 Subject: Test: get to trait page --- bin/test-website | 3 +++ test/lib/main_web_functionality.rb | 36 +++++++++++++++++++----------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/bin/test-website b/bin/test-website index b766edd8..8efa37b1 100755 --- a/bin/test-website +++ b/bin/test-website @@ -32,6 +32,9 @@ $stderr.print "Testing <",$host,">\n" require 'mechanize' require 'minitest/spec' require 'minitest/autorun' + +# These are the actual testing modules + require 'main_web_functionality' diff --git a/test/lib/main_web_functionality.rb b/test/lib/main_web_functionality.rb index 03c76d42..4103d653 100644 --- a/test/lib/main_web_functionality.rb +++ b/test/lib/main_web_functionality.rb @@ -1,30 +1,32 @@ +# In these tests we navigate from the main page to search + class MainWebFunctionality end describe MainWebFunctionality do before do @agent = Mechanize.new - page = @agent.get($host) - # p page - @form = page.forms[1] - end describe MainWebFunctionality do - it "The main page contains a search button" do - @form.buttons[0].value.must_equal "Search" - end - - # http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult - it "Search for MEAN=(15 16) LRS=(23 46)" do - @form.fields[2].value = "MEAN=(15 16) LRS=(23 46)" - @form.fields[3].value = "mouse" - @form.fields[4].value = "BXD" - @form.fields[5].value = "Hippocampus mRNA" - @form.fields[6].value = "HC_M2_0606_P" - @searchpage = @agent.submit(@form, @form.buttons.first) - probe_link = @searchpage.links.find { |l| l.text =~ /1435395_s_at/ } + it "Get to trait page" do + page = @agent.get($host) + form = page.forms[1] + form.buttons[0].value.must_equal "Search" + # http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult + form.fields[2].value = "MEAN=(15 16) LRS=(23 46)" + form.fields[3].value = "mouse" + form.fields[4].value = "BXD" + form.fields[5].value = "Hippocampus mRNA" + form.fields[6].value = "HC_M2_0606_P" + search_page = @agent.submit(form, form.buttons.first) + probe_link = search_page.links.find { |l| l.text =~ /1435395_s_at/ } probe_link.uri.to_s.must_equal "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" + show_trait_page = probe_link.click + # Get to trait page for 1435395_s_at + form2 = show_trait_page.forms_with("trait_page")[0] + form2.fields[30].name.must_equal "value:DBA/2J" + form2.fields[30].value.must_equal "15.287" end end -- cgit v1.2.3 From 4172e8940d2656f9f2a4de6e72e6d114c317d1c0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 27 Apr 2016 11:25:53 +0000 Subject: test: test for broken links --- test/lib/main_web_functionality.rb | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/test/lib/main_web_functionality.rb b/test/lib/main_web_functionality.rb index 4103d653..d0a32835 100644 --- a/test/lib/main_web_functionality.rb +++ b/test/lib/main_web_functionality.rb @@ -6,13 +6,15 @@ end describe MainWebFunctionality do before do @agent = Mechanize.new + @agent.agent.http.ca_file = '/etc/ssl/certs/ca-certificates.crt' end describe MainWebFunctionality do it "Get to trait page" do page = @agent.get($host) form = page.forms[1] - form.buttons[0].value.must_equal "Search" + form.buttons[0].value.must_equal "Search" + # http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29&FormID=searchResult form.fields[2].value = "MEAN=(15 16) LRS=(23 46)" form.fields[3].value = "mouse" @@ -23,12 +25,25 @@ describe MainWebFunctionality do probe_link = search_page.links.find { |l| l.text =~ /1435395_s_at/ } probe_link.uri.to_s.must_equal "/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P" show_trait_page = probe_link.click + p show_trait_page + # Get to trait page for 1435395_s_at form2 = show_trait_page.forms_with("trait_page")[0] - form2.fields[30].name.must_equal "value:DBA/2J" - form2.fields[30].value.must_equal "15.287" + form2.fields[30].name.must_equal "variance:C57BL/6J" + # form2.fields[30].value.must_equal "15.287" + + # Test every link on the page to check if it's broken or not + show_trait_page.links.each do |link| + puts link.href + if link.href !~ /static\/dbdoc\/Hippocampus/ and link.href !~ /glossary.html|sample_r|grits.eecs.utk.edu|correlationAnnotation.html/ + # Fetch link, program will crash with exception if link is broken + linkpage = @agent.get(link.href) + puts "Link to #{link.href} is valid, response code #{linkpage.code}" + end + end + end + end - -end +end -- cgit v1.2.3 From 4d8da2a56ab315a28f0e296690f5ae5b09324615 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 28 Apr 2016 08:47:56 +0000 Subject: Changed the defunct GeneRIF URL to a working Wikipedia URL (which also points to the non-working NCBI link) --- wqflask/wqflask/templates/index_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 5d658bc7..5cc15682 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -143,7 +143,7 @@ highly expressed genes (15 to 16 log2 units) AND with peak LRS linkage between 23 and 46. -
  • RIF=mitochondrial searches RNA databases for +
  • RIF=mitochondrial searches RNA databases for GeneRIF links.
  • WIKI=nicotine searches -- cgit v1.2.3 From a46be3cb631b3e1c7f76593a65a0ae1692c72fda Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 28 Apr 2016 08:55:45 +0000 Subject: Test: splitting out static URL tests --- bin/test-website | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/bin/test-website b/bin/test-website index 8efa37b1..24274713 100755 --- a/bin/test-website +++ b/bin/test-website @@ -14,11 +14,30 @@ the root of the genenetwork2 source tree as, for example, For more information see http://genenetwork.org/ EOT +$stderr.print USAGE -libpath = File.dirname(File.dirname(__FILE__)) -$: << File.join(libpath,'test/lib') +require 'optparse' + +options = {} +opts = OptionParser.new do |o| + o.banner = "Usage: #{File.basename($0)} [options] URL" -$stderr.print USAGE + o.on('-l','--link-checker', 'Check for dead links') do + options[:link_checker] = true + end + + o.separator "" + o.on_tail('-h', '--help', 'display this help and exit') do + options[:show_help] = true + end +end + +opts.parse!(ARGV) + +if options[:show_help] + print opts + exit 1 +end $host = if ARGV.size>0 @@ -35,6 +54,13 @@ require 'minitest/autorun' # These are the actual testing modules -require 'main_web_functionality' +libpath = File.dirname(File.dirname(__FILE__)) +$: << File.join(libpath,'test/lib') +if options[:link_checker] + require 'link_checker' +else + require 'main_web_functionality' +end + -- cgit v1.2.3 From c2adf6b49626355576d3ea454d891307c8221f85 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 28 Apr 2016 08:55:57 +0000 Subject: Test: splitting out static URL tests --- test/lib/link_checker.rb | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 test/lib/link_checker.rb diff --git a/test/lib/link_checker.rb b/test/lib/link_checker.rb new file mode 100644 index 00000000..fb201887 --- /dev/null +++ b/test/lib/link_checker.rb @@ -0,0 +1,48 @@ + +class LinkChecker +end + +describe LinkChecker do + before do + @agent = Mechanize.new + @agent.agent.http.ca_file = '/etc/ssl/certs/ca-certificates.crt' + end + + it "Get to front page" do + page = @agent.get($host) + page.links.each do |link| + puts link.href + if link.href !~ /static\/dbdoc\/Hippocampus/ and link.href !~ /glossary.html|sample_r|grits.eecs.utk.edu|correlationAnnotation.html/ + # Fetch link, program will crash with exception if link is broken + linkpage = @agent.get(link.href) + puts "Link to #{link.href} is valid, response code #{linkpage.code}" + end + end + + end + + describe LinkChecker do + it "Get to trait page" do + page = @agent.get($host+'/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P') + p page + + # Get to trait page for 1435395_s_at + # form2 = show_trait_page.forms_with("trait_page")[0] + # form2.fields[30].name.must_equal "variance:C57BL/6J" + # form2.fields[30].value.must_equal "15.287" + + # Test every link on the page to check if it's broken or not + page.links.each do |link| + puts link.href + if link.href !~ /static\/dbdoc\/Hippocampus/ and link.href !~ /glossary.html|sample_r|grits.eecs.utk.edu|correlationAnnotation.html/ + # Fetch link, program will crash with exception if link is broken + linkpage = @agent.get(link.href) + puts "Link to #{link.href} is valid, response code #{linkpage.code}" + end + end + + end + + end + +end -- cgit v1.2.3 From 6e23be8f278e98386171f4702414224545decfe0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 28 Apr 2016 13:16:00 +0000 Subject: README --- doc/README.org | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/doc/README.org b/doc/README.org index 494e2a70..13f70ad8 100644 --- a/doc/README.org +++ b/doc/README.org @@ -13,6 +13,7 @@ - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] - [[#trouble-shooting][Trouble shooting]] - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] + - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]] * Introduction @@ -59,12 +60,14 @@ version of the guix-daemon by compiling the guix repository. Follow ** Install GN2 #+begin_src bash -env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --fallback +env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env \ + guix package -i genenetwork2 --fallback #+end_src bash Note that you can use the genenetwork.org guix substitute caching server at http://guix.genenetwork.org (which speeds up installs -significantly because all packages are pre-built). +significantly because all packages are pre-built). And/or use the Guix +mirror with option --substitute-urls=http://mirror.guixsd.org Make a note of the paths with @@ -147,13 +150,12 @@ R_LIBS_SITE are set) from the information given by guix: Inside the repository: -: cd genenetwork2_diet +: cd genenetwork2 : ./bin/genenetwork2 Will fire up your local repo http://localhost:5003/ using the -settings in ./etc/default_settings.py - -To override settings create your own from a copy of +settings in ./etc/default_settings.py. These settings may +not reflect your system. To override settings create your own from a copy of default_settings.py and pass it into GN2 with : ./bin/genenetwork2 $HOME/my_settings.py @@ -162,6 +164,13 @@ and everything *should* work (note the full path to the settings file). This way we develop against the exact same dependency graph of software. +If something is not working, take a hint from the settings file +that comes in the Guix installation. It sits in something like + +: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py + + + * Source deployment and other information on reproducibility See the document [[GUIX-Reproducible-from-source.org]]. @@ -184,3 +193,9 @@ On one system: : export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0" and perhaps a few more. +** ERROR: can not find directory $HOME/gn2_data + +The default settings file looks in your $HOME/gn2_data. Since these +files come with a Guix installation you should take a hint from the +values in the installed version of default_settings.py (see above in +this document). -- cgit v1.2.3 From f8960a0a668cf90b88a06c3c2f977062160a211e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 29 Apr 2016 03:29:44 -0500 Subject: Plotting: using cairo-png for headless environments --- wqflask/wqflask/ctl/ctl_analysis.py | 4 ++-- wqflask/wqflask/wgcna/wgcna_analysis.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index d79d9bfe..7a42b2f8 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -152,7 +152,7 @@ class CTL(object): self.results['requestform'] = requestform # Store the user specified parameters for the output page # Create the lineplot - r_png(self.results['imgloc1'], width=1000, height=600) + r_png(self.results['imgloc1'], width=1000, height=600, type='cairo-png') self.r_lineplot(res, significance = significance) r_dev_off() @@ -161,7 +161,7 @@ class CTL(object): # Create the QTL like CTL plots self.results['imgurl' + str(n)] = webqtlUtil.genRandStr("CTL_") + ".png" self.results['imgloc' + str(n)] = GENERATED_IMAGE_DIR + self.results['imgurl' + str(n)] - r_png(self.results['imgloc' + str(n)], width=1000, height=600) + r_png(self.results['imgloc' + str(n)], width=1000, height=600, type='cairo-png') self.r_plotCTLobject(res, (n-1), significance = significance, main='Phenotype ' + trait) r_dev_off() n = n + 1 diff --git a/wqflask/wqflask/wgcna/wgcna_analysis.py b/wqflask/wqflask/wgcna/wgcna_analysis.py index 6901712b..880a1cb2 100644 --- a/wqflask/wqflask/wgcna/wgcna_analysis.py +++ b/wqflask/wqflask/wgcna/wgcna_analysis.py @@ -128,7 +128,7 @@ class WGCNA(object): # The iconic WCGNA plot of the modules in the hanging tree self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" self.results['imgloc'] = GENERATED_IMAGE_DIR + self.results['imgurl'] - r_png(self.results['imgloc'], width=1000, height=600) + r_png(self.results['imgloc'], width=1000, height=600, type='cairo-png') mergedColors = self.r_labels2colors(network[1]) self.r_plotDendroAndColors(network[5][0], mergedColors, "Module colors", dendroLabels = False, hang = 0.03, addGuide = True, guideHang = 0.05) r_dev_off() -- cgit v1.2.3 From 51599a8d7b4abc11f9f6a658a2451f0ee0eacbe5 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 29 Apr 2016 09:48:45 +0000 Subject: Updated mailing list URL --- wqflask/wqflask/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index f25bf9f7..701af140 100755 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -151,7 +151,7 @@ -- cgit v1.2.3 From 4c206f9277561446f32dd01004ff3eae682cfd27 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 29 Apr 2016 10:01:25 +0000 Subject: README: on missing modules --- doc/README.org | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/README.org b/doc/README.org index 13f70ad8..10a77580 100644 --- a/doc/README.org +++ b/doc/README.org @@ -14,6 +14,7 @@ - [[#trouble-shooting][Trouble shooting]] - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]] + - [[#cant-run-a-module][Can't run a module]] * Introduction @@ -199,3 +200,10 @@ The default settings file looks in your $HOME/gn2_data. Since these files come with a Guix installation you should take a hint from the values in the installed version of default_settings.py (see above in this document). + +** Can't run a module + +In rare cases, development modules are not brought in with Guix +because no source code is available. This can lead to missing modules +on a running server. Please check with the authors when a module +is missing. -- cgit v1.2.3 From 9ceffdac4d39b0e2086975bd4aad65c2a7626824 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 29 Apr 2016 10:01:50 +0000 Subject: URL: removed _blank from first URL --- wqflask/wqflask/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 701af140..b4fdbd8e 100755 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -151,7 +151,7 @@ -- cgit v1.2.3 From 1c713642907317692bda4c148afe26d897f0bb82 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 11 May 2016 03:27:40 -0500 Subject: Nginx configuration example --- etc/nginx-genenetwork.conf | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 etc/nginx-genenetwork.conf diff --git a/etc/nginx-genenetwork.conf b/etc/nginx-genenetwork.conf new file mode 100644 index 00000000..9ae042b6 --- /dev/null +++ b/etc/nginx-genenetwork.conf @@ -0,0 +1,49 @@ +# nginx configuration example for GeneNetwork +# +# Simple port reverse proxy from port 80 to 5003 for domain + +user nobody; +worker_processes 4; +pid /var/run/nginx.pid; + +error_log /var/log/nginx/error.log; + +events { + worker_connections 4; +} + +http { + # include mime.types; + default_type application/octet-stream; + + sendfile on; + keepalive_timeout 65; + + server { + listen 80; + server_name localhost; + + location / { + root html; + index index.html index.htm; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root html; + } + + } + + server { + listen 80; + server_name test-gn2.genenetwork.org; + access_log logs/test-gn2.access.log; + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_pass http://127.0.0.1:5003; + } + } + +} -- cgit v1.2.3 From d4edc2787e3bac98924b6ca3d3c4b1acc44138bd Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 11 May 2016 03:31:22 -0500 Subject: nginx: add README info --- etc/nginx-genenetwork.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/nginx-genenetwork.conf b/etc/nginx-genenetwork.conf index 9ae042b6..b966fe6d 100644 --- a/etc/nginx-genenetwork.conf +++ b/etc/nginx-genenetwork.conf @@ -1,6 +1,8 @@ # nginx configuration example for GeneNetwork # # Simple port reverse proxy from port 80 to 5003 for domain +# +# See also the information in ./doc/README.org user nobody; worker_processes 4; -- cgit v1.2.3 From 38d0dddf1359c64e065a408a4f4b307434bc3584 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 11 May 2016 09:18:49 +0000 Subject: README: nginx --- doc/README.org | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/doc/README.org b/doc/README.org index 10a77580..9608548d 100644 --- a/doc/README.org +++ b/doc/README.org @@ -1,3 +1,4 @@ + #+TITLE: Installing GeneNetwork services * Table of Contents :TOC: @@ -10,6 +11,7 @@ - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] + - [[#set-up-nginx-port-forwarding][Set up nginx port forwarding]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] - [[#trouble-shooting][Trouble shooting]] - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] @@ -170,7 +172,44 @@ that comes in the Guix installation. It sits in something like : cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py +** Set up nginx port forwarding + +Install the webserver (as root) + +: guix package -i nginx + +The configuration examples can be found in the Guix store through + +: ls -l /root/.guix-profile/sbin/nginx +: lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx + +Use that path + +: ls /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/share/nginx/conf/ +: fastcgi.conf koi-win scgi_params +: fastcgi.conf.default mime.types scgi_params.default +: fastcgi_params mime.types.default uwsgi_params +: fastcgi_params.default nginx.conf uwsgi_params.default +: koi-utf nginx.conf.default win-utf + +And copy any relevant files to /etc/nginx. A configuration file for +GeneNetwork (reverse proxy) port forwarding can be found in the source +repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc +(still as root) + +: cp ./etc/nginx-genenetwork.conf /etc/nginx/ + +Make dirs + +: mkdir -p /var/spool/nginx/logs + +Add users + +: adduser nobody ; addgroup nobody + +Run nginx +: /root/.guix-profile/sbin/nginx -c /etc/nginx/nginx-genenetwork.conf -p /var/spool/nginx * Source deployment and other information on reproducibility -- cgit v1.2.3 From 7334000f8758fbdeca6885a1637b44663f4ea2a7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 06:24:06 +0000 Subject: Added IRC log --- doc/README.org | 415 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 409 insertions(+), 6 deletions(-) diff --git a/doc/README.org b/doc/README.org index 9608548d..9fa44442 100644 --- a/doc/README.org +++ b/doc/README.org @@ -68,9 +68,415 @@ env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env \ #+end_src bash Note that you can use the genenetwork.org guix substitute caching -server at http://guix.genenetwork.org (which speeds up installs -significantly because all packages are pre-built). And/or use the Guix -mirror with option --substitute-urls=http://mirror.guixsd.org +server at http://guix.genenetwork.org:8080 (which speeds up installs +significantly because all packages are pre-built). Here an IRC session +where we installed GN2 from scratch using GNU Guix and a download +of the test database: + +#+begin_src bash + time to get binary install sorted :) [07:03] + Guix is designed for distributed installation servers + we have one on guix.genenetwork.org + it contains all the prebuild packages + for GN + okay [07:04] + let's step back however [07:05] + I presume the environment is set with all guix package --search-paths + right? + yep + set to the ones in ~/.guix-profile/ + good, and you are in gn-latest-guix repo [07:06] + yep [07:07] + git log shows + +Author: David Thompson +Date: Sun Mar 27 21:20:19 2016 -0400 + + yes + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -A genenetwork2 [07:08] + shows + +genenetwork2 2.0-a8fcff4 out ../guix-bioinformatics/gn/packages/genenetwork.scm:144:2 +genenetwork2-database-small 1.0 out ../guix-bioinformatics/gn/packages/genenetwork.scm:270:4 +genenetwork2-files-small 1.0 out ../guix-bioinformatics/gn/packages/genenetwork.scm:228:4 + + yeah [07:09] + OK, we are in sync. This means we should be able to install the exact + same software + I need to start up my guix daemon - I usually run it in a screen + screen -S guix-daemon + hah, I don't have screen installed yet [07:11] + comes with guix ;) [07:12] + no worries, you can run it any way you want + $HOME/.guix-profile/bin/guix-daemon --build-users-group=guixbuild + then something's weird, because it says I don't have it + oh, you need to install it first [07:13] + guix package -A screen + screen 4.3.1 out gnu/packages/screen.scm:34:2 + but you can skip this install, for now + alright [07:14] + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -i genenetwork2 --dry-run + substitute: updating list of substitutes from + 'https://mirror.hydra.gnu.org'... 79.1% + you see that? + followed by [07:15] +substitute: updating list of substitutes from +'https://hydra.gnu.org'... 100.0% +The following derivations would be built: + /gnu/store/rk7nw0rjqqsha958m649wrykadx6mmhl-profile.drv + +/gnu/store/7b0qjybvfx8syzvfs7p5rdablwhbkbvs-module-import-compiled.drv + /gnu/store/cy9zahbbf23d3cqyy404lk9f50z192kp-module-import.drv + /gnu/store/ibdn603i8grf0jziy5gjsly34wx82lmk-gtk-icon-themes.drv + + which should have the same HASH values /gnu/store/7b0qjybvf... etc. + [07:16] + profile has a different hash + but the next ones? + they're the same + not sure why profile differs. Do you see the contact with + mirror.hydra.org? [07:17] + yeah + OK, that means you set the key correctly for that one :) + alright we are at the same state now. You can see most packages need + to be rebuild because they are no longer cached as binaries on hydra + [07:18] + things move fast... + hehe + let me also do the same on my laptop - which I have staged before + [07:19] + btw, to set the path I often do [07:20] + export + PATH="/home/wrk/.guix-profile/bin:/home/wrk/.guix-profile/sbin":$PATH + to keep things like 'screen' from Debian + Once past building guix itself that is normally OK [07:21] + ah, okay + will do that + the guix build requires certain versions of tools, so you don't want + to mix foreign tools in [07:23] + makes sense [07:24] + On my laptop I am trying the main updating list of substitutes from + 'http://hydra.gnu.org'... 10.5% [07:27] + it is a bit slow, but let's see if there is a difference with the + mirror + you can see there are two servers here. Actually with recent daemons, + if the mirror fails it will try the main server [07:28] + I documented the use of a caching server here [07:29] + https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org + this is exactly what we are doing now + alrighty [07:35] + To see if a remote server has a guix server running it should respond + [07:36] + lynx http://guix.genenetwork.org:8080 --dump + Resource not found: / + + you see that? + yes [07:37] + good. The main hydra server is too slow. So on my laptop I forced + using the mirror with [07:38] + env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix + package -i genenetwork2 --dry-run + --substitute-urls="http://mirror.hydra.gnu.org" + + the list looks the same to me [07:40] + me too + note that some packages will be built and some downloaded, right? + [07:41] + yes + atlas is actually a binary on my system [07:43] + I mean in that list + so, it should not build. Same as yours? + yeah, atlas and r-gtable are the ones to be downloaded + You should not have seen that error ;) + we should try and install it this way, try [07:44] + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -i genenetwork2 --cores=4 --max-jobs=4 --keep-going [07:46] + set CPUs and max-jobs to something sensible + Does your VM have multiple cores? + note you can always press Ctrl-C during install + it doesn't, I'll reboot it and give it another core [07:47] + Hey [07:48] + I'm here + Will be stepping away for some breakfast + Can you do the same as us + Can you see the irc log + Alright + Yes, I can + Please email me a copy in five minutes + user01: so when I use the GN server [07:56] + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -i genenetwork2 --dry-run + --substitute-urls=http://guix.genenetwork.org:8080 + I don't need to build anything [07:57] + (this won't work for you, yet) + to get it to work you need to 'trust' it [07:58] + but, first get the build going + I'll have a coffee while you and get building + yeah it's doing its thing now [08:01] + cool [08:02] + in a separate terminal you can try and install with the gn mirror + [08:05] + I'll send you the public key and you can paste it as said + https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org + [08:06] + alright + should be in the E-mail [08:09] + getting it working it kinda nasty since the server gives no feedback + it works when you see no more in the build list ;) [08:11] + btw, you can install software in parallel. Guix does that. + even the same packages + so keep building ;) + try and do this with Debian... + coffee for me [08:12] + the first build failed [08:15] + OK, Dennis fixed that one yesterday [08:27] + the problem is that sometime source tarballs disappear [08:28] + R is notorious for that + haha, that's inconvenient.. + well, it is good that Guix catches them + but we do not cache sources + binaries are cached - to some degree - so we don't have to rebuild + those [08:29] + time to use the guix cache at guix.genenetwork.org + try and install the key (it is in the E-mail) + and see what this lists [08:31] + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -i genenetwork2 + --substitute-urls=http://guix.genenetwork.org:8080 --dry-run + should be all binary installs + it's not.. [08:32] + if I remove --substitute-urls, the list changes, does that mean I + have the key set up correctly at least? [08:33] + dunno [08:35] + how many packages does it want to build? + should be zero + four + Ah, that is OK - those are default profile things + genenetwork2 is among the ones to be downloaded so [08:36] + remove --dry-run + yeah, good sign :) + we'll still hit a snag, but run it + should be fast + doing it [08:37] + it worked! [08:38] + I think [08:39] + heh [08:40] + you mean it is finished? + yep + type genenetwork2 + complains about not being able to connect to the database [08:41] + last snag :) + no database + well, we succeeded in installing a same-byte install of a very + complex system :) [08:42] + (always take time to congratulate yourself) + now we need to install mysql + hehe :) + this can be done throug guix or through debian [08:43] + the latter is a bit easier here, so let's do that + fun note: you can mix debian and guix + Follow instructions on [08:44] + + https://github.com/genenetwork/genenetwork2/tree/staging/doc#run-mysql-server + apt-get install mysql-common [08:45] + may do it + You can also install with guix, but I need to document that + btw your internet must be fast :) [08:46] + hehe it is ;) + when the database is installed [08:48] + be sure to set the password as instructed [08:50] + when mysql is set the genenetwork2 command should fire up the web + server on localhost:5003 [08:58] + btw my internet is way slower :) [09:00] + I'm back [09:04] + fixed router firmware upgrade problem + unbricking + tssk [09:07] + I'll never leave routers to update themselves again [09:08] + self-brick highway + Resuming [09:09] + auto-updates are evil + always switch them off + user02: can you install genenetwork like user has done? [09:10] + pretty well documented here now :) + Yes I can [09:11] + Already installed key + user02: you are getting binary packages only now? [09:13] + That's the sanest way to go now + seriously + everything should be pre-built from guix.genenetwork.org + you are downloading? + yes [09:15] + cool. Maybe an idea to set up a server + for your own use + Stuck at downloading preprocesscore + should not [09:24] + what does env GUIX_PACKAGE_PATH=../guix-bioinformatics/ + ./pre-inst-env guix package -i genenetwork2 + --substitute-urls="http://guix.genenetwork.org:8080" --dry-run + [09:25] + say for r-prepocesscore + download or build? + mine says download [09:26] + it only lists the derivatives to be built + nothing else happens [09:27] + OK, so there is a problem + your key may not be working + everything should be listed as 'to be download' [09:28] + Hmm + Ah + I know where I messed up + where? + I did add the key + However + (I am documenting) + I did not tell guix to trust it + yes + and there is another potential problem + Remember the documentation on installing guix? + You have to tell guix to trust the default key [09:29] + Right? + So in this case + read the IRC log + That step is mandatory + user01: how are you doing? + user02: + https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org#using-gnu-guix-archive + [09:30] + a little bit left on the db download + user02: you should see no more building + user02: another issue may be that you updated r-preprocesscore + package in guix-buinformatics [09:32] + all downstream packages will want to rebuild + no, not really + It's not even installed + checkout a branch of the the old version - make sure we are in synch + should be at + /gnu/store/y1f3r2xs3fhyadd46nd2aqbr2p9qv2ra-r-biocpreprocesscore-1.32.0 + [09:33] + + pjotrp: Possibly we should use the archive utility of Guix to do + deployment to avoid such out-of-sync differences :) [09:34] + maybe. I did not get archive to update profiles properly [09:37] + Also it is good that they get to understand guix + this way + carved in stone, eh [09:38] + Yeah, all good [09:39] + My mistake was skipping the guix archive part + Can we begin with the install? + It's telling me of derivatives that will be downloaded [09:40] + So we're good + Here goes + yeeha [09:42] + pjotrp, where is this guix.genenetwork.org located at? + Tennessee + It's...it's....sloooooooowwwwwwwwwwwwww + not from Europe + is it downloading at all? + It should be extended + Yes...like at 100KB/s [09:43] + tear-jerker + Verizon problems + who's the host? + I am getting 500Kb/s + UT + Guix's servers can run off more than one server, right? + I'd like to host that particular server here + For speed + yes + Sooner or later + It will be a necessity [09:45] + exactly what I am doing - this is our server + guix.genenetwork.org:8080 + All done installing [09:46] + what? + Now the databases + what do you mean by slow exactly? + Yes, it's installed + can you run genenetwork2 + setting variables + If I try running it now, it will fail as I don't have the DBs [09:47] + cool - you had a lot of prebuilt packages already + OK, follow the instructions I wrote above + now everything seems to be working for me :) + OK + user01: excellent! + you see a webserver? + yep, can connect to localhost:5003 [09:48] + So now you are running a guix copy of GN2 + you can see where it lives with `which genenetwork2` or ls -l + ~/.guix-profile/bin/genenetwork2 [09:49] + + /gnu/store/1kma5xszvzsvmbb4k699h7gvdncw901i-genenetwork2-2.0-a8fcff4/bin/genenetwork2 + it is a script + written by guix, open it [09:50] + inside it points to paths and our script at + + /gnu/store/1kma5xszvzsvmbb4k699h7gvdncw901i-genenetwork2-2.0-a8fcff4/bin/.genenetwork2-real + if you open that you can see how the webserver is started [09:51] + next step is to run a recent version of GN2 + okay [09:52] + See + https://github.com/genenetwork/genenetwork2/tree/staging/doc#run-your-own-copy-of-gn2 + but do not checkout that genetwork2_diet + we reverted to the main tree + clone git@github.com:genenetwork/genenetwork2.git [09:53] + instead and checkout the staging branch + that is effectively my branch [09:54] + when that is done you should be able to fire up the webserver from + there [09:55] + using ./bin/genenetwork2 + now installing DBs + Downloading + annoyingly the source tree is ~700Mb [09:56] + Can it also be done by installing the guix package + genenetwork2-database-small? + I changed it in the diet version to 8Mb, but I had to revert + I need to make my VM bigger... + user02: not ready [09:57] + ok + user01: sorry + user01: you could mount a local dir inside the VM for development + that would allow you to use MAC tools for editing + just an idea + yeah, I figure I'll do something like that + do you use emacs? [09:58] + yep + that can also run on remote files over ssh + that's an alternative + kudos for using emacs :), wdyt user03 + 79 minutes to go downloading the db + user02: sorry about that [09:59] + it is 2GB + user, you can also mount the directory via sshfs + Mac OSX runs OpenSSH + user02: sopa + You can therefore mount a directory outside the VM to the VM via + sshfs [10:00] + yes, 3 options now + That way, you can set up a VM only for it's logic + Apps + the OS it runs [10:01] + For data, let it reside on physical host accessible via sshfs + Use this Arch wiki reference: + https://wiki.archlinux.org/index.php/SSHFS + I edited that last somewhere in 2015, may have been updated since + then + alright, cool! [10:04] + user01: you are almost done [10:06] + I wrote an elixir package for guix :) + env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix + package -A elixir + --substitute-urls="http://guix.genenetwork.org:8080" [10:08] + elixir 1.2.3 out + ../guix-bioinformatics/gn/packages/elixir.scm:31:2 + + I am building it on guix.genenetwork.org right now [10:09] + nice [10:10] +#+end_src bash + +** Run GN2 Make a note of the paths with @@ -78,8 +484,6 @@ Make a note of the paths with ./pre-inst-env guix package --search-paths #+end_src bash -** Run GN2 - After setting the paths for the server #+begin_src bash @@ -196,7 +600,6 @@ And copy any relevant files to /etc/nginx. A configuration file for GeneNetwork (reverse proxy) port forwarding can be found in the source repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc (still as root) - : cp ./etc/nginx-genenetwork.conf /etc/nginx/ Make dirs -- cgit v1.2.3 From 470b110cd34c1f7abeaf3e3f0b0b3d3549775ae8 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 06:25:02 +0000 Subject: Added IRC log --- doc/README.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/README.org b/doc/README.org index 9fa44442..3c1fb910 100644 --- a/doc/README.org +++ b/doc/README.org @@ -73,7 +73,7 @@ significantly because all packages are pre-built). Here an IRC session where we installed GN2 from scratch using GNU Guix and a download of the test database: -#+begin_src bash +#+begin_src time to get binary install sorted :) [07:03] Guix is designed for distributed installation servers we have one on guix.genenetwork.org @@ -474,7 +474,7 @@ The following derivations would be built: I am building it on guix.genenetwork.org right now [10:09] nice [10:10] -#+end_src bash +#+end_src ** Run GN2 -- cgit v1.2.3 From 24d733635d1040985a5894e0adbe763eca8a5dae Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 03:07:21 -0500 Subject: Nginx configuration --- etc/nginx-genenetwork.conf | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/etc/nginx-genenetwork.conf b/etc/nginx-genenetwork.conf index b966fe6d..06fd3c57 100644 --- a/etc/nginx-genenetwork.conf +++ b/etc/nginx-genenetwork.conf @@ -1,9 +1,3 @@ -# nginx configuration example for GeneNetwork -# -# Simple port reverse proxy from port 80 to 5003 for domain -# -# See also the information in ./doc/README.org - user nobody; worker_processes 4; pid /var/run/nginx.pid; @@ -11,15 +5,15 @@ pid /var/run/nginx.pid; error_log /var/log/nginx/error.log; events { - worker_connections 4; + worker_connections 16; } http { - # include mime.types; - default_type application/octet-stream; + include mime.types; + # default_type application/octet-stream; - sendfile on; - keepalive_timeout 65; + # sendfile on; + keepalive_timeout 480; server { listen 80; @@ -34,18 +28,27 @@ http { location = /50x.html { root html; } - } server { listen 80; server_name test-gn2.genenetwork.org; access_log logs/test-gn2.access.log; + + proxy_connect_timeout 3000; + proxy_send_timeout 3000; + proxy_read_timeout 3000; + send_timeout 3000; + location / { - proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header Host $host; proxy_set_header Host $http_host; + # proxy_redirect off; + proxy_set_header Connection keep-alive; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; proxy_pass http://127.0.0.1:5003; } } - } -- cgit v1.2.3 From b63e4b5111269dcaf4a9b9eec7d387d8aecb3ba5 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 03:07:41 -0500 Subject: Fix paths for images --- wqflask/base/webqtlConfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index cdce119a..f62207e0 100755 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -64,8 +64,8 @@ ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?sp TMPDIR = mk_dir(TEMPDIR+'/gn2/') CACHEDIR = mk_dir(TEMPDIR+'/cache/') # We can no longer write into the git tree: -GENERATED_IMAGE_DIR = mk_dir(TMPDIR+'/generate/') -GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generate_text/') +GENERATED_IMAGE_DIR = mk_dir(TMPDIR+'/generated/') +GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generated_text/') # Flat file directories GENODIR = flat_files('genotype')+'/' -- cgit v1.2.3 From 8aa1e752ac2d576a6e46610b4fe13f335682f6ba Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 08:18:20 +0000 Subject: README: nginx configuration as reverse proxy --- doc/README.org | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/doc/README.org b/doc/README.org index 3c1fb910..345341e1 100644 --- a/doc/README.org +++ b/doc/README.org @@ -578,11 +578,38 @@ that comes in the Guix installation. It sits in something like ** Set up nginx port forwarding -Install the webserver (as root) +nginx can be used as a reverse proxy for GN2. For example, we want to +expose GN2 on port 80 while it is running on port 5003. Essentially +the configuration looks like + +#+begin_src js + server { + listen 80; + server_name test-gn2.genenetwork.org; + access_log logs/test-gn2.access.log; + + proxy_connect_timeout 3000; + proxy_send_timeout 3000; + proxy_read_timeout 3000; + send_timeout 3000; + + location / { + proxy_set_header Host $http_host; + proxy_set_header Connection keep-alive; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + proxy_pass http://127.0.0.1:5003; + } +} +#+end_src js + +Install the nginx webserver (as root) : guix package -i nginx -The configuration examples can be found in the Guix store through +The nginx example configuration examples can be found in the Guix +store through : ls -l /root/.guix-profile/sbin/nginx : lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx -- cgit v1.2.3 From 409848a4b3873fb0512ae17135b9794030eff024 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 03:21:20 -0500 Subject: WebqtlConfig.py: Constant handling --- wqflask/base/webqtlConfig.py | 7 +------ wqflask/wqflask/marker_regression/marker_regression_gn1.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index f62207e0..6890dacf 100755 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -71,14 +71,9 @@ GENERATED_TEXT_DIR = mk_dir(TMPDIR+'/generated_text/') GENODIR = flat_files('genotype')+'/' JSON_GENODIR = assert_dir(GENODIR+'json/') -# SITENAME = 'GN' -# PORTADDR = "http://50.16.251.170" -# BASEHREF = '' +PORTADDR = "http://50.16.251.170" INFOPAGEHREF = '/dbdoc/%s.html' CGIDIR = '/webqtl/' #XZ: The variable name 'CGIDIR' should be changed to 'PYTHONDIR' SCRIPTFILE = 'main.py' -# GLOSSARYFILE = "/glossary.html" -# REFRESHSTR = '' -# REFRESHDIR = '%s' + SCRIPTFILE +'?sid=%s' diff --git a/wqflask/wqflask/marker_regression/marker_regression_gn1.py b/wqflask/wqflask/marker_regression/marker_regression_gn1.py index 97e4b934..4460c06d 100644 --- a/wqflask/wqflask/marker_regression/marker_regression_gn1.py +++ b/wqflask/wqflask/marker_regression/marker_regression_gn1.py @@ -674,7 +674,7 @@ class MarkerRegression(object): fpText.write("Source: WebQTL, The GeneNetwork (%s)\n" % webqtlConfig.PORTADDR) # - fpText.write("Site: %s\n" % webqtlConfig.SITENAME) + fpText.write("Site: GN\n") fpText.write("Page: Map Viewer\n") fpText.write(time.strftime("Date and Time (US Center): %b %d, %Y at %I.%M %p\n", time.localtime())) fpText.write("Trait ID: %s\n" % self.this_trait.name) -- cgit v1.2.3 From 3706cf389a8ead64f02c2ed997fd3defbede9832 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 08:48:18 +0000 Subject: Removing ref. to HTMLPATH --- wqflask/base/webqtlConfig.py | 6 +++--- wqflask/wqflask/marker_regression/MarkerRegressionPage.py | 4 ++-- wqflask/wqflask/marker_regression/gemma_mapping.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 6890dacf..0358bcbf 100755 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -56,9 +56,9 @@ UCSC_RUDI_TRACK_URL = " http://genome.cse.ucsc.edu/cgi-bin/hgTracks?org=%s&db=%s GENOMEBROWSER_URL="http://ucscbrowser.genenetwork.org/cgi-bin/hgTracks?clade=mammal&org=Mouse&db=mm9&position=%s&hgt.suggest=&pix=800&Submit=submit" ENSEMBLETRANSCRIPT_URL="http://useast.ensembl.org/Mus_musculus/Lucene/Details?species=Mus_musculus;idx=Transcript;end=1;q=%s" -# HTMLPATH = GNROOT + 'genotype_files/' -# PYLMM_PATH -# IMGDIR = GNROOT + '/wqflask/wqflask/static/output/' +# The following paths are no longer in use! +# HTMLPATH is replaced by GENODIR +# IMGDIR is replaced by GENERATED_IMAGE_DIR # Temporary storage: TMPDIR = mk_dir(TEMPDIR+'/gn2/') diff --git a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py index 455fcf95..4c3391e5 100755 --- a/wqflask/wqflask/marker_regression/MarkerRegressionPage.py +++ b/wqflask/wqflask/marker_regression/MarkerRegressionPage.py @@ -72,7 +72,7 @@ class MarkerRegressionPage(templatePage): #automatically generate pheno txt file for PLINK self.genPhenoTxtFileForPlink(phenoFileName=plinkOutputFileName,RISetName=fd.RISet,probesetName=probesetName, valueDict=allTraitValueDict) # os.system full path is required for input and output files; specify missing value is -9999 - plink_command = '%splink/plink --noweb --ped %splink/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %splink/%s.map --pheno %s/%s.txt --pheno-name %s --missing-phenotype -9999 --out %s%s --assoc ' % (webqtlConfig.HTMLPATH, webqtlConfig.HTMLPATH, fd.RISet, webqtlConfig.HTMLPATH, fd.RISet, webqtlConfig.TMPDIR, plinkOutputFileName, probesetName, webqtlConfig.TMPDIR, plinkOutputFileName) + plink_command = '%splink/plink --noweb --ped %splink/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %splink/%s.map --pheno %s/%s.txt --pheno-name %s --missing-phenotype -9999 --out %s%s --assoc ' % (webqtlConfig.GENODIR, webqtlConfig.GENODIR, fd.RISet, webqtlConfig.GENODIR, fd.RISet, webqtlConfig.TMPDIR, plinkOutputFileName, probesetName, webqtlConfig.TMPDIR, plinkOutputFileName) os.system(plink_command) @@ -1597,7 +1597,7 @@ class MarkerRegressionPage(templatePage): # get strain name from ped file in order def getStrainNameFromPedFile(self, RISetName=''): - pedFileopen= open("%splink/%s.ped"%(webqtlConfig.HTMLPATH, RISetName),"r") + pedFileopen= open("%splink/%s.ped"%(webqtlConfig.GENODIR, RISetName),"r") line =pedFileopen.readline() strainNameList=[] diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index 8fb086c1..caea5802 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -11,7 +11,7 @@ def run_gemma(this_dataset, samples, vals): gen_pheno_txt_file(this_dataset, samples, vals) # Don't do this! - # os.chdir("{}gemma".format(webqtlConfig.HTMLPATH)) + # os.chdir("{}gemma".format(webqtlConfig.GENODIR)) # use GEMMA_RUN in the next one, create a unique temp file -- cgit v1.2.3 From 04afa563e6d53fe2a91ac2e6eb4af2f2fa5d5c3b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 08:53:42 +0000 Subject: Fixed path for IM bar plot --- wqflask/wqflask/templates/marker_regression_gn1.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/marker_regression_gn1.html b/wqflask/wqflask/templates/marker_regression_gn1.html index df948190..978ea55f 100644 --- a/wqflask/wqflask/templates/marker_regression_gn1.html +++ b/wqflask/wqflask/templates/marker_regression_gn1.html @@ -151,7 +151,7 @@ {% endif %} {% if nperm > 0 and permChecked == "ON" %}

    - +

    Total of {{ nperm }} permutations  Download Permutation Results
    -- cgit v1.2.3
    • traitmarkertraitLODdCor