From 847f911257e66be454f16443ac53c107064eb6aa Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:10:30 +0000 Subject: Docs --- doc/GUIX-Reproducible-from-source.org | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index b88eb9e8..871156ed 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -31,5 +31,12 @@ Next build guix (and run) following the instructions in [[https://github.com/pjo Once that is done we can add the guix-bioinformatics path with -: env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A slurm +: env GUIX_PACKAGE_PATH=../guix-bioinformatics command +such as + +#+begin_src sh :lang bash +gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A genenetwork +genenetwork1 1.0-d622c803b out ../guix-bioinformatics/gn/packages/bioinformatics.scm:163:2 +genenetwork2 2.0-9e9475053 out ../guix-bioinformatics/gn/packages/bioinformatics.scm:215:2 +#+end_src sh :lang bash -- cgit v1.2.3 From 94459e2cb31cbbd2f3f58c74e26dd762e981f7da Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:13:04 +0000 Subject: [PATCH 010/100] README --- doc/README.org | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index f6ab6a52..9206cd35 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,7 +3,6 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - [[#binary-deployment][Binary deployment]] - - [[#from-source-deployment][From source deployment]] * Introduction @@ -15,14 +14,3 @@ explain the GeneNetwork deployment system which is based on GNU Guix NYA -* From source deployment - -GNU Guix allows for [[https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org][reproducible deployment]] based on a checked out -Guix repository - use gn-stable for that: - -#+begin_src sh :lang bash -mkdir genenetwork -cd genenetwork -git checkout https://github.com/genenetwork/guix.git gn-stable-guix -git checkout https://github.com/genenetwork/guix-bioinformatics.git -#+end_src -- cgit v1.2.3 From e1cdb11c211394472e2f8cdd033fd0f93dbf5cea Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sat, 13 Feb 2016 11:22:02 +0100 Subject: Docs: Use ./doc dir instead of ./misc --- INSTALL.md | 2 +- doc/gn_installation_notes.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'doc') diff --git a/INSTALL.md b/INSTALL.md index b8c9c977..3ef2d5b3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -86,4 +86,4 @@ GN2 can locate PYLMM through PYLMM_PATH in setting.py (or in ENV) ## Other information -Check also the ./misc/ directory for settings +Check also the ./doc directory for settings diff --git a/doc/gn_installation_notes.txt b/doc/gn_installation_notes.txt index 584080f7..efea0309 100644 --- a/doc/gn_installation_notes.txt +++ b/doc/gn_installation_notes.txt @@ -96,7 +96,7 @@ Before installing from requirements.txt, install numpy separately: pip install numpy==1.7.0 (or whatever version we're using) Install from requirements.txt (after activating virtualenv): -pip install -r gene/misc/requirements.txt +pip install -r gene/doc/requirements.txt =========================================== @@ -343,4 +343,4 @@ python runserver.py To do full upgrade (as opposed to apt-get upgrade) sudo aptitude full-upgrade -=========================================== \ No newline at end of file +=========================================== -- cgit v1.2.3 From 73ef712f3fd02df7ffa84a2b35ee5648f689d31b Mon Sep 17 00:00:00 2001 From: pjotrp Date: Sun, 14 Feb 2016 10:33:05 +0100 Subject: README: add URL --- doc/README.org | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 9206cd35..8e0823de 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,7 +2,8 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#binary-deployment][Binary deployment]] + - [[#binary-deployment-wip][Binary deployment (WIP)]] + - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -10,7 +11,12 @@ Large system deployments tend to get very complex. In this document we explain the GeneNetwork deployment system which is based on GNU Guix (see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). -* Binary deployment +* Binary deployment (WIP) -NYA +GN can be deployed either as a binary tarball or as a GNU Guix +package. + +* Source deployment and other information on reproducibility + +See the document [[GUIX-Reproducible-from-source.org]]. -- cgit v1.2.3 From c76299fa981add7b8cf56126c8d7ffeb9a6c4034 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Tue, 16 Feb 2016 10:30:27 +0000 Subject: Successful startup of GN2 with MySQL --- bin/genenetwork2 | 8 +++++++- doc/README.org | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) (limited to 'doc') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index d0177045..a48e8b93 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -8,8 +8,14 @@ SCRIPT=$(readlink -f "$0") GN2_BASE_PATH=$(dirname $(dirname "$SCRIPT")) echo $GN2_BASE_PATH +settings=$1 +if [ -z $settings ]; then settings=$GN2_BASE_PATH/etc/default_settings.py ; fi export PYTHONPATH=$GN2_BASE_PATH/wqflask:$PYTHONPATH -export WQFLASK_SETTINGS=$GN2_BASE_PATH/etc/default_settings.py +export WQFLASK_SETTINGS=$settings + +# start the redis server +redis-server & cd $GN2_BASE_PATH/wqflask +echo "Starting with $settings" /usr/bin/env python runserver.py diff --git a/doc/README.org b/doc/README.org index 8e0823de..375a5d20 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,6 +3,8 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - [[#binary-deployment-wip][Binary deployment (WIP)]] + - [[#run-mysql-server][Run MySQL server]] + - [[#start-the-gn2-server][Start the GN2 server]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -16,6 +18,53 @@ explain the GeneNetwork deployment system which is based on GNU Guix GN can be deployed either as a binary tarball or as a GNU Guix package. +** Run MySQL server + +At this point we require the underlying distribution to install +and run mysqld. + +Download one of + +http://files.genenetwork.org/raw_database/ +https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip + +Check the md5sum. + +After installation inflate the database binary in the MySQL directory +(this is subject to change soon) + +: chown -R mysql:mysql db_webqtl_s/ +: chmod 700 db_webqtl_s/ +: chmod 660 db_webqtl_s/* + +restart MySQL service (mysqld). Login as root and + +: mysql> show databases; +: +--------------------+ +: | Database | +: +--------------------+ +: | information_schema | +: | db_webqtl_s | +: | mysql | +: | performance_schema | +: +--------------------+ + +Set permissions and match password in your settings file below: + +: mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; + +** Start the GN2 server + +Copy the default_settings.py file and modify + +: DB_URI = """mysql://gn2:mysql_password@localhost/db_webqtl""" +: SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password/db_webqtl' +: SECRET_KEY = 'secret_key' + +Start the server with your settings file + +: ./bin/genenetwork2 ~/my_gn2_settings.py + * Source deployment and other information on reproducibility See the document [[GUIX-Reproducible-from-source.org]]. -- cgit v1.2.3 From af7d0bca229f3ebaa80a16d1ce3a2bf1a8abd5df Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:37:55 +0000 Subject: [PATCH 023/100] WIP fixing all paths --- README.md | 2 + doc/GUIX-Reproducible-from-source.org | 14 +++- doc/README.org | 23 +++++- etc/default_settings.py | 15 ++-- wqflask/base/data_set.py | 3 +- wqflask/utility/tools.py | 82 ++++++++++------------ wqflask/wqflask/database.py | 5 +- .../wqflask/marker_regression/marker_regression.py | 6 +- wqflask/wqflask/show_trait/show_trait.py | 10 +-- 9 files changed, 94 insertions(+), 66 deletions(-) (limited to 'doc') diff --git a/README.md b/README.md index 3d95d05f..db495910 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ Elissa Chesler, Jintao Wang, Kenneth Manly, Robert W. Williams, and colleagues. Code and primary web service managed by Dr. Robert W. Williams and the University of Tennessee Health Science Center, Memphis TN, USA. +Join #genenetwork on irc.freenode.net. + Email labwilliams@gmail.com or rwilliams@uthsc.edu Older version available on SourceForge http://sourceforge.net/projects/genenetwork/ diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index 871156ed..4399ea26 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -4,6 +4,7 @@ - [[#introduction][Introduction]] - [[#binary-deployment][Binary deployment]] - [[#from-source-deployment][From source deployment]] + - [[#create-archive][Create archive]] * Introduction @@ -33,10 +34,21 @@ Once that is done we can add the guix-bioinformatics path with : env GUIX_PACKAGE_PATH=../guix-bioinformatics command -such as +So +#+begin_src sh :lang bash #+begin_src sh :lang bash gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A genenetwork genenetwork1 1.0-d622c803b out ../guix-bioinformatics/gn/packages/bioinformatics.scm:163:2 genenetwork2 2.0-9e9475053 out ../guix-bioinformatics/gn/packages/bioinformatics.scm:215:2 #+end_src sh :lang bash + +Install with + +#+begin_src sh :lang bash +gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -i genenetwork2 +#+end_src sh :lang bash + +* Create archive + +: env GUIX_PACKAGE_PATH=../../genenetwork/guix-bioinformatics/ ./pre-inst-env guix archive --export -r genenetwork2 > guix_gn2-2.0-9e9475053.nar diff --git a/doc/README.org b/doc/README.org index 375a5d20..d9730948 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,6 +3,7 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - [[#binary-deployment-wip][Binary deployment (WIP)]] + - [[#install-genenetwork-server][Install genenetwork server]] - [[#run-mysql-server][Run MySQL server]] - [[#start-the-gn2-server][Start the GN2 server]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] @@ -16,7 +17,27 @@ explain the GeneNetwork deployment system which is based on GNU Guix * Binary deployment (WIP) GN can be deployed either as a binary tarball or as a GNU Guix -package. +package. First install GNU Guix following the instructions of the +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]] and, update guix with a +'guix pull' and make guix visible in the path. More information +exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. + +With guix running you should be able to install python, for example. + +: guix package -i python2 + +This will make python appear in $HOME/.guix-profile/bin/python. Suggested +environment settings can be seen with + +: guix package --search-paths + +** Install genenetwork server + +Fetch the nar file and install it after adding the key with, for +example + +: cat signing-key.pub |.guix-profile/bin/guix archive --authorize +: guix archive --import < guix_gn2-2.0-9e9475053.nar ** Run MySQL server diff --git a/etc/default_settings.py b/etc/default_settings.py index 929bd687..48d3f66b 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -1,13 +1,14 @@ import os +import sys -LOGFILE = "/var/log/genenetwork/wqflask.log" +LOGFILE = "/tmp/genenetwork2.log" # This is needed because Flask turns key errors into a # 400 bad request response with no exception/log TRAP_BAD_REQUEST_ERRORS = True -DB_URI = "mysql://gn2:default@localhost/db_webqtl" -SQLALCHEMY_DATABASE_URI = 'mysql://gn2:default@localhost/db_webqtl' +DB_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" +SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password@localhost/db_webqtl_s' # http://pythonhosted.org/Flask-Security/configuration.html SECURITY_CONFIRMABLE = True @@ -22,7 +23,7 @@ SERVER_PORT = 5003 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' # Path overrides for Genenetwork -HOME=os.environ.get('HOME') -PYLMM_PATH = HOME+"/izip/git/opensource/python/pylmm_gn2/" -PLINK_PATH = HOME+"/.guix-profile/bin" -GEMMA_PATH = HOME+"/.guix-profile/bin" +GENENETWORK_FILES = "../../gn2_data" +PYLMM_RUN = os.popen("which pylmm_redis").read() +PLINK_RUN = os.popen("which plink2").read() +GEMMA_RUN = os.popen("which gemma").read() diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index e37a838f..0e5a3ac1 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -405,10 +405,11 @@ class DatasetGroup(object): #print("Cache not hit") from utility.tools import plink_command - PLINK_PATH,PLINK_COMMAND = plink_command() + PLINK_RUN = plink_command() geno_file_path = webqtlConfig.GENODIR+self.name+".geno" plink_file_path = PLINK_PATH+"/"+self.name+".fam" + # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES if os.path.isfile(plink_file_path): self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 0db195df..c3c9b292 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -9,22 +9,22 @@ import os import sys from wqflask import app -def get_setting(id,default,guess,find_path): +def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with get_valid_path is a function checking whether the - path points to an expected directory an returns the full path e.g. + path points to an expected directory and returns the full path to + the binary command guess = os.environ.get('HOME')+'/pylmm' - get_setting('PYLMM_PATH',default,guess,get_valid_path) + get_setting('PYLMM_PATH',guess) first tries the environment variable in +id+, next gets the Flask - app setting for the same +id+, next tries the path passed in with - +default+ and finally does an educated +guess+. + app setting for the same +id+ and finally does an educated + +guess+. In all, the environment overrides the others, next is the flask - setting, then the default and finally the guess (which is - $HOME/repo). A valid path is returned. If none is resolved an - exception is thrown. + setting, then the guess. A valid path to the binary command is + returned. If none is resolved an exception is thrown. Note that we do not use the system path. This is on purpose because it will mess up controlled (reproducible) deployment. The @@ -33,46 +33,36 @@ def get_setting(id,default,guess,find_path): different settings.py file (or setting the environment). """ - # ---- Check whether environment exists - path = find_path(os.environ.get(id)) - # ---- Check whether setting exists - setting = app.config.get(id) - if not path: - path = find_path(setting) - # ---- Check whether default exists - if not path: - path = find_path(default) - # ---- Guess directory - if not path: - guess = os.environ.get('HOME')+guess - if not setting: - setting = guess - path = find_path(guess) - if not path: - raise Exception(id+' '+setting+' path unknown or faulty (update settings.py?). '+id+' should point to the path') - return path - -def find_command(command,id1,default,guess): - def find_path(path): - """Test for a valid repository""" - if path: - sys.stderr.write("Trying "+id1+" in "+path+"\n") - binary = str.split(command)[0] - if path and os.path.isfile(path+'/'+binary): - return path + def valid(command): + if command: + sys.stderr.write("Found value "+command+"\n") + return command else: - None + return None + + # ---- Check whether environment exists + sys.stderr.write("Looking for "+command_id+"\n") + command = valid(os.environ.get(command_id)) + if not command: + # ---- Check whether setting exists in app + command = valid(app.config.get(command_id)) + if not command: + command = valid(guess) + if not command: + raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path') + return command - path = get_setting(id1,default,guess,find_path) - binary = path+'/'+command - sys.stderr.write("Found "+binary+"\n") - return path,binary +def pylmm_command(guess=None): + return get_setting("PYLMM_RUN",guess) -def pylmm_command(default=None): - return find_command('pylmm_gn2/lmm.py',"PYLMM_PATH",default,'/pylmm2') +def gemma_command(guess=None): + return get_setting("GEMMA_RUN",guess) -def gemma_command(default=None): - return find_command('gemma',"GEMMA_PATH",default,'/gemma') +def plink_command(guess=None): + return get_setting("PLINK_RUN",guess) -def plink_command(default=None): - return find_command('plink2',"PLINK_PATH",default,'/plink') +def flat_files(subdir=None): + base = get_setting("GENENETWORK_FILES") + if subdir: + return base+"/"+subdir + return base diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 159c5d6c..2f544d44 100755 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -24,8 +24,9 @@ def init_db(): # you will have to import them first before calling init_db() #import yourapplication.models import wqflask.model - print("Creating all..") + print("database.py: Creating all model metadata..") Base.metadata.create_all(bind=engine) - print("Done creating all...") + print("database.py: Done creating all model metadata...") + print("Point your browser at http://localhost:5003/") init_db() diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index a657510d..b0f5ed69 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -46,9 +46,9 @@ from wqflask.marker_regression import gemma_mapping # runs at startup, so a missing binary will balk before running the # service -GEMMA_PATH,GEMMA_COMMAND = gemma_command() -PYLMM_PATH,PYLMM_COMMAND = pylmm_command() -PLINK_PATH,PLINK_COMMAND = plink_command() +GEMMA_RUN = gemma_command() +PYLMM_RUN = pylmm_command() +PLINK_RUN = plink_command() # RQTL_PATH,RQTL_COMMAND = rqtl_command() diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 2d4c952a..458e48da 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -16,7 +16,8 @@ from base import webqtlConfig from base import webqtlCaseData from wqflask.show_trait.SampleList import SampleList from utility import webqtlUtil, Plot, Bunch, helper_functions -from utility.tools import pylmm_command, plink_command +# from utility.tools import plink_command +from utility.tools import flat_files from base.trait import GeneralTrait from base import data_set from dbFunction import webqtlDatabaseFunction @@ -24,8 +25,7 @@ from basicStatistics import BasicStatisticsFunctions from pprint import pformat as pf -PYLMM_PATH,PYLMM_COMMAND = pylmm_command() -PLINK_PATH,PLINK_COMMAND = plink_command() +MAPPING_FILES = flat_files("mapping") ############################################### # @@ -162,8 +162,8 @@ class ShowTrait(object): def get_mapping_methods(self): '''Only display mapping methods when the dataset group's genotype file exists''' def check_plink_gemma(): - if (os.path.isfile(PLINK_PATH+"/"+self.dataset.group.name+".bed") and - os.path.isfile(PLINK_PATH+"/"+self.dataset.group.name+".map")): + if (os.path.isfile(MAPPYING_FILES+"/"+self.dataset.group.name+".bed") and + os.path.isfile(MAPPING_FILES+"/"+self.dataset.group.name+".map")): return True else: return False -- cgit v1.2.3 From e1c42eaed65de9cd9a99cc4731f3ede66cfe94bb Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:17:33 +0000 Subject: docs: muck out outdated info --- doc/gn_installation_notes.txt | 346 -------------------------------------- doc/new_variable_names.txt | 6 - doc/notes.txt | 311 ---------------------------------- doc/notes_DA.txt | 10 -- doc/old/gn_installation_notes.txt | 346 ++++++++++++++++++++++++++++++++++++++ doc/old/notes.txt | 311 ++++++++++++++++++++++++++++++++++ doc/requirements.txt | 36 ---- doc/todo.txt | 2 - 8 files changed, 657 insertions(+), 711 deletions(-) delete mode 100644 doc/gn_installation_notes.txt delete mode 100644 doc/new_variable_names.txt delete mode 100644 doc/notes.txt delete mode 100644 doc/notes_DA.txt create mode 100644 doc/old/gn_installation_notes.txt create mode 100644 doc/old/notes.txt delete mode 100644 doc/requirements.txt delete mode 100644 doc/todo.txt (limited to 'doc') diff --git a/doc/gn_installation_notes.txt b/doc/gn_installation_notes.txt deleted file mode 100644 index efea0309..00000000 --- a/doc/gn_installation_notes.txt +++ /dev/null @@ -1,346 +0,0 @@ -UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration - -Enable UFW: -sudo ufw enable -Check status of UFW: -sudo ufw status verbose - -Limit ssh: -sudo ufw limit ssh - -Enable http, https: -sudo ufw allow http -sudo ufw allow https - -=========================================== - -http://evilrouters.net/2011/02/18/using-etckeeper-with-git-on-ubuntu/ - -Install etckeeper: -sudo apt-get install etckeeper - -Edit /etc/etckeeper/etckeeper.conf to comment out VCS="bzr" and comment in VCS="git" - -Initialize repository: -sudo etckeeper init - -Do initial commit: -sudo etckeeper commit "Initial commit." - -=========================================== - -Get git: - -sudo apt-get install git - -=========================================== - -Clone code from git repository: -git clone http://github.com/zsloan/genenetwork.git gene - -Pull from branch in git repository: -git pull origin flask(or whatever the branch is) - -**"apt-cache search" searches for text in the description for various packages, while -"apt-file search" searches for files within the packages** - -Search for package with a specified file that can be installed with apt-get -apt-file search _______ - -============================================ - -Create trash directory: -mkdir ~/.trash - -============================================ - -Using dpkg - -Check which system packages are already installed: -dpkg -l | less - -=========================================== - -Installing virtualenv: -sudo pip install virtualenv - -Tell virtualenv which directory to use: -virtualenv ve27 - -Start up virtual environment: -source ~/ve27/bin/activate - -=========================================== - -Install libmysqlclient-dev (Mysql-Python dependency) -sudo apt-get install libmysqlclient-dev - -Install python-dev (numpy dependency) -sudo apt-get install python-dev - -Install scipy dependencies: -sudo apt-get install libatlas-base-dev gfortran g++ - -=========================================== - -Install pip: -sudo apt-get install python-pip - -REMEMBER TO SOURCE VE BEFORE INSTALLING - -Comment out in requirements.txt: -Reaper -numarray - -Before installing from requirements.txt, install numpy separately: -pip install numpy==1.7.0 (or whatever version we're using) - -Install from requirements.txt (after activating virtualenv): -pip install -r gene/doc/requirements.txt - -=========================================== - -Installing QTL Reaper - -wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb -mv -v qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb qtlreaper-1.1.1.tar.gz -tar xvf qtlreaper-1.1.1.tar.gz (to unzip) -mkdir /home/zas1024/ve27/include/python2.7/Reaper -python setup.py install - -=========================================== - -Installing numarray 1.5.2 - -wget http://downloads.sourceforge.net/project/numpy/Old%20Numarray/1.5.2/numarray-1.5.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%2520Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload&ts=1358978306&use_mirror=superb-dca2 -mv -v numarray-1.5.2.tar.gz?r=http:%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%20Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload numarray-1.5.2.tar.gz -tar xvf numarray-1.5.2.tar.gz -sudo python setup.py install - -=========================================== - -Installing nginx - -sudo -s -nginx=stable # use nginx=development for latest development version -echo "deb http://ppa.launchpad.net/nginx/$nginx/ubuntu lucid main" > /etc/apt/sources.list.d/nginx-$nginx-lucid.list -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C300EE8C -apt-get update -apt-get install nginx - -Create configuration file in ~/gene/wqflask/other_config/nginx_conf/ (modeled off of the others) -Create symbolic link to config file in /etc/nginx/sites-enabled/: -ln -s dir_to_link_to linking_dir - -Run nginx: -sudo /usr/sbin/nginx - -============================================ - -Copy over zach_settings.py to /home/zas1024 directory (or whatever is home directory) - -export WQFLASK_SETTINGS=~/gene/wqflask/cfg/zach_settings.py (or wherever file is located) - -============================================ - -Install MySQL Server - -sudo apt-get install mysql-server - -mkdir /mnt/big -fdisk /dev/sdb -m: for help -n: new partion -... -w: write to table and exit - -Start MySQL server: -service mysql start - -Stop MySQL server: -service mysql stop - -Change root password: -mysql> UPDATE mysql.user SET Password=PASSWORD('your password') WHERE User='root'; - -Setup accounts in MySQL (first need to delete anonymous/non-root accounts): -#; use mysql; -#; select * from user; -#; delete from user where Host!="localhost"; -#; delete from user where User!="root"; -#; update user set Password = Password('yourpassword') where User='root'; -#; GRANT ALL ON *.* TO 'yourusername'@'%' IDENTIFIED BY 'yourpassword'; -#; select * from user; - -Use Percona to optimise my.cfg file: -https://tools.percona.com/wizard - -To check # CPUs: -grep -c processor /proc/cpuinfo - -To check memory, can use htop - -============================================ - -Check RSA key: -ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key - -03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) - -============================================ - -Installing redis server - -In the dls directory: -$ wget http://redis.googlecode.com/files/redis-2.6.14.tar.gz -$ tar xzf redis-2.6.14.tar.gz -$ cd redis-2.6.14 -$ make - -Run redis (from inside redis-2.6.14 directory): -src/redis-server - -For the rest of the installation instructions, refer to: -http://redis.io/topics/quickstart - -To continue installation (from the README): - -In order to install Redis binaries into /usr/local/bin just use: - - % make install - -Make install will just install binaries in your system, but will not configure -init scripts and configuration files in the appropriate place. This is not -needed if you want just to play a bit with Redis, but if you are installing -it the proper way for a production system, we have a script doing this -for Ubuntu and Debian systems: - - % cd utils - % ./install_server - -The script will ask you a few questions and will setup everything you need -to run Redis properly as a background daemon that will start again on -system reboots. - -In the redis config file (/etc/redis/6379.conf) uncomment the following line: -bind 127.0.0.1 - -This is so only the local computer has access - -============================================ - -Check RSA key: -ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key - -03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) - -============================================ - -Using Yolk - -Install Yolk: -pip install yolk - -Check packages installed in this virtual environment: -yolk -l - -Checks packages that have updates available: -yolk -U - -=========================================== - -Installing yaml - -Install libyaml-dev: -sudo apt-get install libyaml-dev - -Install yaml: -pip install pyyaml - -=========================================== - -Install MySQL Client - -To fix error "mysql_config not found" while installing packages with pip: -sudo apt-get install libmysqlclient-dev - -=========================================== - -Installing R - -sudo apt-get install r-base-dev - -=========================================== - -Installing rpy2 - -pip install rpy2 - -=========================================== - -Install Parallel Python (pp) - -wget http://www.parallelpython.com/downloads/pp/pp-1.6.3.tar.gz -tar xvf pp-1.6.3.tar.gz -python setup.py install - -========================================== - -Install requests - -pip install requests - -========================================== - -Install logging_tree - -pip install logging_tree - -========================================== - -Install colordiff (just to make it easier to view differences in files, not necessary for GN) - -sudo apt-get install colordiff - -========================================== - -Install NTP (network time protocol) - -sudo apt-get install ntp - -========================================== - -To get server running: - -!If having seemingly inexplicable problems with imports, make sure I've started the environment! - -Start up virtual environment: -source ~/ve27/bin/activate============== - -Install requests - -pip install requests - -======================= - -To set WQFLASK_SETTINGS environment variable: -export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) - -To change screen environment variable (if man not working or to get color, for example): -export TERM=screen - -To search for commands in history if necessary: -history | grep "(whatever is being searched for)" - -Run web server: -/usr/local/nginx/sbin/nginx - -Run server: -python runserver.py - -=========================================== - -To do full upgrade (as opposed to apt-get upgrade) -sudo aptitude full-upgrade - -=========================================== diff --git a/doc/new_variable_names.txt b/doc/new_variable_names.txt deleted file mode 100644 index c11c160e..00000000 --- a/doc/new_variable_names.txt +++ /dev/null @@ -1,6 +0,0 @@ -RISet/riset -> group -webqtlDataset.py -> data_set.py -webqtlDataset (class object) -> DataSet -database/db -> dataset/data_set -DataEditingPage -> show_trait.py/show_trait.html -webqtlTrait -> GeneralTrait \ No newline at end of file diff --git a/doc/notes.txt b/doc/notes.txt deleted file mode 100644 index f8ce2759..00000000 --- a/doc/notes.txt +++ /dev/null @@ -1,311 +0,0 @@ -Clone code from git repository: -git clone http://github.com/zsloan/genenetwork.git gene - -Pull from branch in git repository: -git pull origin flask(or whatever the branch is) - -Add all files in current directory branch to be staged to be committed -git add -A - -Git push to my github repository -git push https://github.com/zsloan/genenetwork.git master - -Get the default path being used when a command is typed: -which _______ - -Change default path python looks at when importing -export PYTHONPATH="/home/zas1024/gene/wqflask" - -Get the paths checked when a user types a command: -echo $PATH - -Get all environment variables: -set - - -**** apt-get is an Ubuntu system-wide package manager, while pip is a Python-only system -for python packages **** - -Upgrade system packages for security updates, etc: -apt-get update -apt-get dist-upgrade (can probably be used in place of apt-get upgrade) -apt-get upgrade - -**"apt-cache search" searches for text in the description for various packages, while -"apt-file search" searches for files within the packages** - -Show information about given result item returned by apt-cache search: -apt-cache show _______ - -Search for package with a specified file that can be installed with apt-get -apt-file search _______ - -============================================ - -Check which system packages are installed: -dpkg -l | less - -============================================ - -NEVER USE SUDO WITH PIP - -To fix error "mysql_config not found": -sudo apt-get install libmysqlclient-dev - -Create requirements.txt file: -pip freeze > requirements.txt (or whatever filename) - -Install from requirements.txt: -pip install -r gene/wqflask/requirements.txt -t ve27 - -============================================ - -To get server running: - -!If having seemingly inexplicable problems with imports, make sure I've started the environment! - -Note: Virtual environment is not activated when running as root - -Start up virtual environment: -source ~/ve27/bin/activate - -Deactivate virtual environment: -deactivate (just type deactivate) - -To set WQFLASK_SETTINGS environment variable: -export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) - -zach_settings.py is the local config file with the DB_URI -default_settings.py is the config file with the configuration information that would apply -to any running instance of GeneNetwork - -To change screen environment variable (if man not working or to get color, for example): -export TERM=screen - -To search for commands in history if necessary: -history | grep "(whatever is being searched for)" - -Reload web server: -/usr/sbin/nginx -s reload - -Run server: -python runserver.py - -Run sendmail.py -python send_mail.py - -=========================================== - -UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration - -Enable UFW: -sudo ufw enable -Check status of UFW: -sudo ufw status verbose - -=========================================== - -Update user password: -use mysql; -UPDATE user SET password=PASSWORD('whatever') WHERE user='webqtl'; -FLUSH PRIVILEGES; - -=========================================== - -Redis things - -Restart redis server: -sudo /etc/init.d/redis_6379 stop -sudo /etc/init.d/redis_6379 start - -=========================================== - -Run pylint: -pylint file_name - -For options: -pylint --help - -=========================================== - -Start screen session: - -If "no option -R" or running for first time: -byobu-select-backend -2. screen - -byobu -RD (to start) -control-a then :multiuser on -control-a then :acladd sam - -More info on sharing byobu screen with other users: -http://wiki.networksecuritytoolkit.org/nstwiki/index.php/HowTo_Share_A_Terminal_Session_Using_Screen - -control-a c to create channel - -type: screen -list for sessions -screen -r zas1024/25679.byobu - -or if only one: - -screen -r zas1024/ - -=========================================== - -Start up log: -Go to /tmp and tail -f flask_gn_log - -=========================================== - -Coffeescript Stuff: - -coffee -c (filename) -coffee -c -w . (to watch for changes and recompile in current directory; the "." is for current directory) -coffee --help (for information about setting options) - -=========================================== - -Unset ASKPASS when trying to git push - -unset SSH_ASKPASS - -=========================================== - -htop: Gives information on processes, cpu/memory load, etc -dstat: Also gives various system information, resource usage, etc -df: Reports file system disk space usage - -=========================================== - -ll h*: Finds items in the directory starting with h - -=========================================== - -du -hms * | sort -n : Gives size used by different directories --h: human readable --m: in megabytes (default) --s: summarize - -=========================================== - -rm -rfv - -=========================================== - -cp -a (archive; copies recursively and doesn't follow symbol links) - -i (interactive, prompts before overwrite) - -v (verbose) - -b (backup) - -mv (same as above, but with no -a) - -=========================================== - -Add user: -sudo adduser -Edit /etc/sudoers to give user root privileges - - -=========================================== - -tidyp - Improves/beautifies html code -tidyp -m -i -w 100 index_page.html - -=========================================== - -ps ax - View processes -ps ux (the "u" is for user) - -kill (process #) - -=========================================== - -Installing QTL Reaper - -wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb -tar xvf qtlreaper-1.1.1.tar.gz (to unzip) -python setup.py build -python setup.py install - -=========================================== - -Using Yolk - -Install Yolk: -pip install yolk - -Check packages installed in this virtual environment: -yolk -l - -Checks packages that have updates available: -yolk -U - -=========================================== - -Python stuff: - -Classes should always inherit "object" -To iterate through dictionary items: for X, Y in MyDictionary.items(): - -=========================================== - -Change import path from command line -export PYTHONPATH=~/gene/wqflask (or whatever) - -=========================================== - -Create symbolic link: -ln -s dir_to_link_to linking_dir - -=========================================== - -Reboot server -sudo reboot - -=========================================== - -Using grep: - -grep -ir (search string) (directory) - --i ignores case --r searches recursively through subdirectories of directory you specify - -=========================================== - -Command line arguments: - -Use argparse to deal with command line arguments (instead of argv or optparse) - -=========================================== - -Change owner/group: - -chown zas1024 somefile (change owner of somefile to zas1024) -chown zas1024: somefile (change owner and group of somefile to zas1024) -sudo chown root: somefile (change owner/group of somefile to root) - -Change permissions: - -chmod g-wr (remove write/read privileges for group) -chmod g+w (add write privileges for group) -chmod u+rwx (add read, write, and execute privileges for user) -etc. - -=========================================== - -MySQL Things - - -Create fulltext (MATCH AGAINST syntax) index: -CREATE FULLTEXT INDEX __________ ON Table (Column); - -If <4 character searches not working: -Add ft_min_word_len = 2 (to make te minimum characters 2) to config file -Rebuild relevant tables with fulltext indexes with "REPAIR TABLE _______ QUICK" - -=========================================== - -To delete all .pyc files (for example if there's some issue you're having trouble resolving): -find . -name '*.pyc' -delete -http://stackoverflow.com/questions/785519/how-do-i-remove-all-pyc-files-from-a-project - diff --git a/doc/notes_DA.txt b/doc/notes_DA.txt deleted file mode 100644 index 410e0182..00000000 --- a/doc/notes_DA.txt +++ /dev/null @@ -1,10 +0,0 @@ -Danny's notes about the genenetwork source - -Location of static files: - -Location of HTML templates: wqflask/wqflask/templates/ - -Entry point of the wqflask app: wqflask/wqflask/__init__.py - -Application routes: wqflask/wqflask/views.py - diff --git a/doc/old/gn_installation_notes.txt b/doc/old/gn_installation_notes.txt new file mode 100644 index 00000000..efea0309 --- /dev/null +++ b/doc/old/gn_installation_notes.txt @@ -0,0 +1,346 @@ +UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration + +Enable UFW: +sudo ufw enable +Check status of UFW: +sudo ufw status verbose + +Limit ssh: +sudo ufw limit ssh + +Enable http, https: +sudo ufw allow http +sudo ufw allow https + +=========================================== + +http://evilrouters.net/2011/02/18/using-etckeeper-with-git-on-ubuntu/ + +Install etckeeper: +sudo apt-get install etckeeper + +Edit /etc/etckeeper/etckeeper.conf to comment out VCS="bzr" and comment in VCS="git" + +Initialize repository: +sudo etckeeper init + +Do initial commit: +sudo etckeeper commit "Initial commit." + +=========================================== + +Get git: + +sudo apt-get install git + +=========================================== + +Clone code from git repository: +git clone http://github.com/zsloan/genenetwork.git gene + +Pull from branch in git repository: +git pull origin flask(or whatever the branch is) + +**"apt-cache search" searches for text in the description for various packages, while +"apt-file search" searches for files within the packages** + +Search for package with a specified file that can be installed with apt-get +apt-file search _______ + +============================================ + +Create trash directory: +mkdir ~/.trash + +============================================ + +Using dpkg + +Check which system packages are already installed: +dpkg -l | less + +=========================================== + +Installing virtualenv: +sudo pip install virtualenv + +Tell virtualenv which directory to use: +virtualenv ve27 + +Start up virtual environment: +source ~/ve27/bin/activate + +=========================================== + +Install libmysqlclient-dev (Mysql-Python dependency) +sudo apt-get install libmysqlclient-dev + +Install python-dev (numpy dependency) +sudo apt-get install python-dev + +Install scipy dependencies: +sudo apt-get install libatlas-base-dev gfortran g++ + +=========================================== + +Install pip: +sudo apt-get install python-pip + +REMEMBER TO SOURCE VE BEFORE INSTALLING + +Comment out in requirements.txt: +Reaper +numarray + +Before installing from requirements.txt, install numpy separately: +pip install numpy==1.7.0 (or whatever version we're using) + +Install from requirements.txt (after activating virtualenv): +pip install -r gene/doc/requirements.txt + +=========================================== + +Installing QTL Reaper + +wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb +mv -v qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb qtlreaper-1.1.1.tar.gz +tar xvf qtlreaper-1.1.1.tar.gz (to unzip) +mkdir /home/zas1024/ve27/include/python2.7/Reaper +python setup.py install + +=========================================== + +Installing numarray 1.5.2 + +wget http://downloads.sourceforge.net/project/numpy/Old%20Numarray/1.5.2/numarray-1.5.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%2520Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload&ts=1358978306&use_mirror=superb-dca2 +mv -v numarray-1.5.2.tar.gz?r=http:%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%20Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload numarray-1.5.2.tar.gz +tar xvf numarray-1.5.2.tar.gz +sudo python setup.py install + +=========================================== + +Installing nginx + +sudo -s +nginx=stable # use nginx=development for latest development version +echo "deb http://ppa.launchpad.net/nginx/$nginx/ubuntu lucid main" > /etc/apt/sources.list.d/nginx-$nginx-lucid.list +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C300EE8C +apt-get update +apt-get install nginx + +Create configuration file in ~/gene/wqflask/other_config/nginx_conf/ (modeled off of the others) +Create symbolic link to config file in /etc/nginx/sites-enabled/: +ln -s dir_to_link_to linking_dir + +Run nginx: +sudo /usr/sbin/nginx + +============================================ + +Copy over zach_settings.py to /home/zas1024 directory (or whatever is home directory) + +export WQFLASK_SETTINGS=~/gene/wqflask/cfg/zach_settings.py (or wherever file is located) + +============================================ + +Install MySQL Server + +sudo apt-get install mysql-server + +mkdir /mnt/big +fdisk /dev/sdb +m: for help +n: new partion +... +w: write to table and exit + +Start MySQL server: +service mysql start + +Stop MySQL server: +service mysql stop + +Change root password: +mysql> UPDATE mysql.user SET Password=PASSWORD('your password') WHERE User='root'; + +Setup accounts in MySQL (first need to delete anonymous/non-root accounts): +#; use mysql; +#; select * from user; +#; delete from user where Host!="localhost"; +#; delete from user where User!="root"; +#; update user set Password = Password('yourpassword') where User='root'; +#; GRANT ALL ON *.* TO 'yourusername'@'%' IDENTIFIED BY 'yourpassword'; +#; select * from user; + +Use Percona to optimise my.cfg file: +https://tools.percona.com/wizard + +To check # CPUs: +grep -c processor /proc/cpuinfo + +To check memory, can use htop + +============================================ + +Check RSA key: +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key + +03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) + +============================================ + +Installing redis server + +In the dls directory: +$ wget http://redis.googlecode.com/files/redis-2.6.14.tar.gz +$ tar xzf redis-2.6.14.tar.gz +$ cd redis-2.6.14 +$ make + +Run redis (from inside redis-2.6.14 directory): +src/redis-server + +For the rest of the installation instructions, refer to: +http://redis.io/topics/quickstart + +To continue installation (from the README): + +In order to install Redis binaries into /usr/local/bin just use: + + % make install + +Make install will just install binaries in your system, but will not configure +init scripts and configuration files in the appropriate place. This is not +needed if you want just to play a bit with Redis, but if you are installing +it the proper way for a production system, we have a script doing this +for Ubuntu and Debian systems: + + % cd utils + % ./install_server + +The script will ask you a few questions and will setup everything you need +to run Redis properly as a background daemon that will start again on +system reboots. + +In the redis config file (/etc/redis/6379.conf) uncomment the following line: +bind 127.0.0.1 + +This is so only the local computer has access + +============================================ + +Check RSA key: +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key + +03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) + +============================================ + +Using Yolk + +Install Yolk: +pip install yolk + +Check packages installed in this virtual environment: +yolk -l + +Checks packages that have updates available: +yolk -U + +=========================================== + +Installing yaml + +Install libyaml-dev: +sudo apt-get install libyaml-dev + +Install yaml: +pip install pyyaml + +=========================================== + +Install MySQL Client + +To fix error "mysql_config not found" while installing packages with pip: +sudo apt-get install libmysqlclient-dev + +=========================================== + +Installing R + +sudo apt-get install r-base-dev + +=========================================== + +Installing rpy2 + +pip install rpy2 + +=========================================== + +Install Parallel Python (pp) + +wget http://www.parallelpython.com/downloads/pp/pp-1.6.3.tar.gz +tar xvf pp-1.6.3.tar.gz +python setup.py install + +========================================== + +Install requests + +pip install requests + +========================================== + +Install logging_tree + +pip install logging_tree + +========================================== + +Install colordiff (just to make it easier to view differences in files, not necessary for GN) + +sudo apt-get install colordiff + +========================================== + +Install NTP (network time protocol) + +sudo apt-get install ntp + +========================================== + +To get server running: + +!If having seemingly inexplicable problems with imports, make sure I've started the environment! + +Start up virtual environment: +source ~/ve27/bin/activate============== + +Install requests + +pip install requests + +======================= + +To set WQFLASK_SETTINGS environment variable: +export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) + +To change screen environment variable (if man not working or to get color, for example): +export TERM=screen + +To search for commands in history if necessary: +history | grep "(whatever is being searched for)" + +Run web server: +/usr/local/nginx/sbin/nginx + +Run server: +python runserver.py + +=========================================== + +To do full upgrade (as opposed to apt-get upgrade) +sudo aptitude full-upgrade + +=========================================== diff --git a/doc/old/notes.txt b/doc/old/notes.txt new file mode 100644 index 00000000..f8ce2759 --- /dev/null +++ b/doc/old/notes.txt @@ -0,0 +1,311 @@ +Clone code from git repository: +git clone http://github.com/zsloan/genenetwork.git gene + +Pull from branch in git repository: +git pull origin flask(or whatever the branch is) + +Add all files in current directory branch to be staged to be committed +git add -A + +Git push to my github repository +git push https://github.com/zsloan/genenetwork.git master + +Get the default path being used when a command is typed: +which _______ + +Change default path python looks at when importing +export PYTHONPATH="/home/zas1024/gene/wqflask" + +Get the paths checked when a user types a command: +echo $PATH + +Get all environment variables: +set + + +**** apt-get is an Ubuntu system-wide package manager, while pip is a Python-only system +for python packages **** + +Upgrade system packages for security updates, etc: +apt-get update +apt-get dist-upgrade (can probably be used in place of apt-get upgrade) +apt-get upgrade + +**"apt-cache search" searches for text in the description for various packages, while +"apt-file search" searches for files within the packages** + +Show information about given result item returned by apt-cache search: +apt-cache show _______ + +Search for package with a specified file that can be installed with apt-get +apt-file search _______ + +============================================ + +Check which system packages are installed: +dpkg -l | less + +============================================ + +NEVER USE SUDO WITH PIP + +To fix error "mysql_config not found": +sudo apt-get install libmysqlclient-dev + +Create requirements.txt file: +pip freeze > requirements.txt (or whatever filename) + +Install from requirements.txt: +pip install -r gene/wqflask/requirements.txt -t ve27 + +============================================ + +To get server running: + +!If having seemingly inexplicable problems with imports, make sure I've started the environment! + +Note: Virtual environment is not activated when running as root + +Start up virtual environment: +source ~/ve27/bin/activate + +Deactivate virtual environment: +deactivate (just type deactivate) + +To set WQFLASK_SETTINGS environment variable: +export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) + +zach_settings.py is the local config file with the DB_URI +default_settings.py is the config file with the configuration information that would apply +to any running instance of GeneNetwork + +To change screen environment variable (if man not working or to get color, for example): +export TERM=screen + +To search for commands in history if necessary: +history | grep "(whatever is being searched for)" + +Reload web server: +/usr/sbin/nginx -s reload + +Run server: +python runserver.py + +Run sendmail.py +python send_mail.py + +=========================================== + +UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration + +Enable UFW: +sudo ufw enable +Check status of UFW: +sudo ufw status verbose + +=========================================== + +Update user password: +use mysql; +UPDATE user SET password=PASSWORD('whatever') WHERE user='webqtl'; +FLUSH PRIVILEGES; + +=========================================== + +Redis things + +Restart redis server: +sudo /etc/init.d/redis_6379 stop +sudo /etc/init.d/redis_6379 start + +=========================================== + +Run pylint: +pylint file_name + +For options: +pylint --help + +=========================================== + +Start screen session: + +If "no option -R" or running for first time: +byobu-select-backend +2. screen + +byobu -RD (to start) +control-a then :multiuser on +control-a then :acladd sam + +More info on sharing byobu screen with other users: +http://wiki.networksecuritytoolkit.org/nstwiki/index.php/HowTo_Share_A_Terminal_Session_Using_Screen + +control-a c to create channel + +type: screen -list for sessions +screen -r zas1024/25679.byobu + +or if only one: + +screen -r zas1024/ + +=========================================== + +Start up log: +Go to /tmp and tail -f flask_gn_log + +=========================================== + +Coffeescript Stuff: + +coffee -c (filename) +coffee -c -w . (to watch for changes and recompile in current directory; the "." is for current directory) +coffee --help (for information about setting options) + +=========================================== + +Unset ASKPASS when trying to git push + +unset SSH_ASKPASS + +=========================================== + +htop: Gives information on processes, cpu/memory load, etc +dstat: Also gives various system information, resource usage, etc +df: Reports file system disk space usage + +=========================================== + +ll h*: Finds items in the directory starting with h + +=========================================== + +du -hms * | sort -n : Gives size used by different directories +-h: human readable +-m: in megabytes (default) +-s: summarize + +=========================================== + +rm -rfv + +=========================================== + +cp -a (archive; copies recursively and doesn't follow symbol links) + -i (interactive, prompts before overwrite) + -v (verbose) + -b (backup) + +mv (same as above, but with no -a) + +=========================================== + +Add user: +sudo adduser +Edit /etc/sudoers to give user root privileges + + +=========================================== + +tidyp - Improves/beautifies html code +tidyp -m -i -w 100 index_page.html + +=========================================== + +ps ax - View processes +ps ux (the "u" is for user) + +kill (process #) + +=========================================== + +Installing QTL Reaper + +wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb +tar xvf qtlreaper-1.1.1.tar.gz (to unzip) +python setup.py build +python setup.py install + +=========================================== + +Using Yolk + +Install Yolk: +pip install yolk + +Check packages installed in this virtual environment: +yolk -l + +Checks packages that have updates available: +yolk -U + +=========================================== + +Python stuff: + +Classes should always inherit "object" +To iterate through dictionary items: for X, Y in MyDictionary.items(): + +=========================================== + +Change import path from command line +export PYTHONPATH=~/gene/wqflask (or whatever) + +=========================================== + +Create symbolic link: +ln -s dir_to_link_to linking_dir + +=========================================== + +Reboot server +sudo reboot + +=========================================== + +Using grep: + +grep -ir (search string) (directory) + +-i ignores case +-r searches recursively through subdirectories of directory you specify + +=========================================== + +Command line arguments: + +Use argparse to deal with command line arguments (instead of argv or optparse) + +=========================================== + +Change owner/group: + +chown zas1024 somefile (change owner of somefile to zas1024) +chown zas1024: somefile (change owner and group of somefile to zas1024) +sudo chown root: somefile (change owner/group of somefile to root) + +Change permissions: + +chmod g-wr (remove write/read privileges for group) +chmod g+w (add write privileges for group) +chmod u+rwx (add read, write, and execute privileges for user) +etc. + +=========================================== + +MySQL Things - + +Create fulltext (MATCH AGAINST syntax) index: +CREATE FULLTEXT INDEX __________ ON Table (Column); + +If <4 character searches not working: +Add ft_min_word_len = 2 (to make te minimum characters 2) to config file +Rebuild relevant tables with fulltext indexes with "REPAIR TABLE _______ QUICK" + +=========================================== + +To delete all .pyc files (for example if there's some issue you're having trouble resolving): +find . -name '*.pyc' -delete +http://stackoverflow.com/questions/785519/how-do-i-remove-all-pyc-files-from-a-project + diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index 39ee5652..00000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,36 +0,0 @@ -BeautifulSoup==3.2.1 -Flask==0.9 -Flask-Login==0.1.3 -Flask-Mail==0.7.6 -Flask-Principal==0.3.4 -Flask-SQLAlchemy==0.16 -Flask-Security==1.6.0 -Flask-WTF==0.8.3 -Jinja2==2.6 -MySQL-python==1.2.4 -PyYAML==3.10 -#Reaper==1.0 -Reindent==0.1.1 -SQLAlchemy==0.8.0 -WTForms==1.0.3 -Werkzeug==0.8.3 -apache-libcloud==0.12.3 -argparse==1.2.1 -blinker==1.2 -cairosvg==1.0.15 -itsdangerous==0.17 -logging-tree==1.2 -logilab-astng==0.24.3 -logilab-common==0.59.1 -#numarray==1.5.2 -numpy==1.7.0 -passlib==1.6.1 -pp==1.6.3 -pylint==0.27.0 -redis==2.7.2 -requests==1.1.0 -scipy==0.11.0 -simplejson==3.0.7 -wsgiref==0.1.2 -yolk==0.4.3 -XlsxWriter==0.7.2 diff --git a/doc/todo.txt b/doc/todo.txt deleted file mode 100644 index 1d781b13..00000000 --- a/doc/todo.txt +++ /dev/null @@ -1,2 +0,0 @@ -- Ask Rob about potentially recoding qtlreaper -- Ask Rob about Probe/cellid traits \ No newline at end of file -- cgit v1.2.3 From 791a80c3409327e5578dc36cc575169b8ee5b13b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 4 Mar 2016 10:41:14 +0000 Subject: README --- README.md | 5 ++++- doc/README.org | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'doc') diff --git a/README.md b/README.md index 6a69d6cd..d2d15781 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,10 @@ Source code for the GeneNetwork2 server http://www.genenetwork.org/ ## Install -See the [installation docs](doc/README.org) +The recommended installation is with GNU Guix which allows you to +deploy GN2 and dependencies as a self contained unit on any machine. +The database can be run separately as well as the source tree (for +developers). See the [installation docs](doc/README.org). ## License diff --git a/doc/README.org b/doc/README.org index d9730948..fd19141c 100644 --- a/doc/README.org +++ b/doc/README.org @@ -1,4 +1,4 @@ -#+TITLE: Installing GeneNetwork services with GNU Guix +#+TITLE: Installing GeneNetwork services * Table of Contents :TOC: - [[#introduction][Introduction]] @@ -12,7 +12,8 @@ Large system deployments tend to get very complex. In this document we explain the GeneNetwork deployment system which is based on GNU Guix -(see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). +(see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install +GN with all its files and dependencies. * Binary deployment (WIP) -- cgit v1.2.3 From b4376b2a4a1a03abec6020275253ad3c9f25a6e7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 5 Mar 2016 16:12:18 +0000 Subject: README --- doc/README.org | 168 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 144 insertions(+), 24 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index fd19141c..ecc9b799 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,10 +2,19 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#binary-deployment-wip][Binary deployment (WIP)]] - - [[#install-genenetwork-server][Install genenetwork server]] + - [[#binary-deployment][Binary deployment]] + - [[#install-guix-using-a-tar-ball][Install Guix using a tar ball]] + - [[#fix-locale][Fix locale]] + - [[#authorize-our-archives][Authorize our archives]] + - [[#download-and-install-the-gn2-archive][Download and install the GN2 archive]] + - [[#source-deployment][Source deployment]] + - [[#install-guix][Install guix]] + - [[#checkout-the-git-repositories][Checkout the git repositories]] + - [[#run-guix-daemon][Run guix-daemon]] + - [[#install-gn2][Install GN2]] + - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] - - [[#start-the-gn2-server][Start the GN2 server]] + - [[#other][Other]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -15,30 +24,136 @@ explain the GeneNetwork deployment system which is based on GNU Guix (see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install GN with all its files and dependencies. -* Binary deployment (WIP) +* Binary deployment + +Note binary deployment is not working pending a few improvements +to GNU Guix. See source deployment instead. + +** Install Guix using a tar ball GN can be deployed either as a binary tarball or as a GNU Guix package. First install GNU Guix following the instructions of the -[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]] and, update guix with a -'guix pull' and make guix visible in the path. More information -exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]]. -With guix running you should be able to install python, for example. +With guix-daemon running you should be able to install the hello +package: -: guix package -i python2 +: guix package -i hello -This will make python appear in $HOME/.guix-profile/bin/python. Suggested -environment settings can be seen with +** Fix locale -: guix package --search-paths +You may want to + +#+begin_src sh :lang bash +export GUIX_LOCPATH=$HOME/.guix-profile/lib/locale +export LC_ALL=en_US.utf8 +#+end_src sh :lang bash + +** Authorize our archives + +Next add our archive key to guix (as root): + +#+begin_src scheme +echo "(public-key + (ecc + (curve Ed25519) + (q #E9A95686D8437186302E07C7AB9BF3913F435026C2D389AF27D9C66FD6EBB649#) + ) + ) +"|guix archive --authorize +#+end_src scheme + +if you have trouble finding a suitable guix try + +: ls /gnu/store/*guix-*/bin/guix + +and you should be able to use this directly, e.g. + +: alias guix=/gnu/store/632msbms2yaldfnlrb5lbnlnmn9yjisw-guix-0.9.0/bin/guix +: guix --version + +** Download and install the GN2 archive + +Find the archive on + + http://files.genenetwork.org/software/ + +download and install with + +#+begin_src bash +guix archive --import < genenetwork2-data-hash.nar +#+end_src bash + +and you should see a list of packages installing, e.g. + +#+begin_src bash +importing path `/gnu/store/l1zs2drn3zdzl5ysjcmhibcpa35p9zfc-python2-mysqlclient-1.3.7' +importing path `/gnu/store/n7kfg4knibvblggy8ci2liscl7vz5wkg-python2-parallel-1.6.4' +importing path `/gnu/store/qvv16qwlq59gp5d07lwbf5n8ndsi3il3-python2-sqlalchemy-1.0.11' +importing path `/gnu/store/qw872mbmr9ir0a9drv9xw9pvjk05ywwy-python2-xlsxwriter-0.8.4' +importing path `/gnu/store/wc112m1xfy3p08v14bdzay2ki2rirdsm-pylmm-gn2-1.0-3c6d1cac8' +importing path `/gnu/store/zfkcy17c2ks3cd9ks14irdabqvmlfpyn-python2-flask-sqlalchemy-2.1' +importing path `/gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff4' +(etc.) +#+end_src bash + +The following packages need to be added and the R path set + +: export R_LIBS_SITE="/home/wrk/.guix-profile/site-library/" +: guix package -i /gnu/store/w0dqg9dshq53j8xhcnqgvnvms2s6y5k5-r-wgcna-1.49-425bc170cc0873ddbd414675ac40f6d4d724c7cb +: guix package -i /gnu/store/k60bdlm0v7xic88j2z5c1jb1jvc371mn-r-qtl-1.38-4 -** Install genenetwork server +You can add the last one to your profile -Fetch the nar file and install it after adding the key with, for -example +: guix package -i /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff +: export PATH=~/.guix-profile/bin:$PATH +: genenetwork2 -: cat signing-key.pub |.guix-profile/bin/guix archive --authorize -: guix archive --import < guix_gn2-2.0-9e9475053.nar + or run it directly with + +: /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff/bin/genenetwork2 + +* Source deployment + +** Install guix + +Deploying from source is also straightforward. Install GNU Guix using +a binary tar ball as described [[https://github.com/pjotrp/guix-notes][here]]. + +** Checkout the git repositories + +Check out the guix and guix-bioinformatics repositories: + +#+begin_src bash +cd ~ +mkdir genenetwork +cd genenetwork +git clone https://github.com/genenetwork/guix-bioinformatics +git clone --recursive --branch gn-latest https://github.com/genenetwork/guix guix-gn-latest +cd guix-gn-latest +#+end_src bash + +** Run guix-daemon + +At this point you may decide to create, install and run a recent +version of the guix-daemon by compiling the guix repository. Follow +[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. + +** Install GN2 + +#+begin_src bash +env GUIX_PACKAGE_PATH=../guix-bioinformatics/ \ + ./pre-inst-env guix package -i genenetwork2 +#+end_src bash + +** Run GN2 + +#+begin_src bash +export PATH=~/.guix-profile/bin:$PATH +genenetwork2 +#+end_src bash + +will start the default server which listens on port 5003. ** Run MySQL server @@ -75,17 +190,22 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; -** Start the GN2 server -Copy the default_settings.py file and modify +** Other + +Update guix with a 'guix pull' and make guix visible in the path. +More information exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. -: DB_URI = """mysql://gn2:mysql_password@localhost/db_webqtl""" -: SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password/db_webqtl' -: SECRET_KEY = 'secret_key' +With guix running you should be able to install python, for example. + +: guix package -i python2 + +This will make python appear in $HOME/.guix-profile/bin/python. Suggested +environment settings can be seen with + +: guix package --search-paths -Start the server with your settings file -: ./bin/genenetwork2 ~/my_gn2_settings.py * Source deployment and other information on reproducibility -- cgit v1.2.3 From d8b3b06ada09b7aeead3a179673c3121d786ad46 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 5 Mar 2016 16:30:34 +0000 Subject: README --- doc/README.org | 140 ++++++++++----------------------------------------------- 1 file changed, 23 insertions(+), 117 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index ecc9b799..adb545f6 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,19 +2,13 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#binary-deployment][Binary deployment]] - - [[#install-guix-using-a-tar-ball][Install Guix using a tar ball]] - - [[#fix-locale][Fix locale]] - - [[#authorize-our-archives][Authorize our archives]] - - [[#download-and-install-the-gn2-archive][Download and install the GN2 archive]] - [[#source-deployment][Source deployment]] - [[#install-guix][Install guix]] - [[#checkout-the-git-repositories][Checkout the git repositories]] - - [[#run-guix-daemon][Run guix-daemon]] - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] - - [[#other][Other]] + - [[#notes][Notes]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -24,94 +18,9 @@ explain the GeneNetwork deployment system which is based on GNU Guix (see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install GN with all its files and dependencies. -* Binary deployment - -Note binary deployment is not working pending a few improvements -to GNU Guix. See source deployment instead. - -** Install Guix using a tar ball - -GN can be deployed either as a binary tarball or as a GNU Guix -package. First install GNU Guix following the instructions of the -[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]]. - -With guix-daemon running you should be able to install the hello -package: - -: guix package -i hello - -** Fix locale - -You may want to - -#+begin_src sh :lang bash -export GUIX_LOCPATH=$HOME/.guix-profile/lib/locale -export LC_ALL=en_US.utf8 -#+end_src sh :lang bash - -** Authorize our archives - -Next add our archive key to guix (as root): - -#+begin_src scheme -echo "(public-key - (ecc - (curve Ed25519) - (q #E9A95686D8437186302E07C7AB9BF3913F435026C2D389AF27D9C66FD6EBB649#) - ) - ) -"|guix archive --authorize -#+end_src scheme - -if you have trouble finding a suitable guix try - -: ls /gnu/store/*guix-*/bin/guix - -and you should be able to use this directly, e.g. - -: alias guix=/gnu/store/632msbms2yaldfnlrb5lbnlnmn9yjisw-guix-0.9.0/bin/guix -: guix --version - -** Download and install the GN2 archive - -Find the archive on - - http://files.genenetwork.org/software/ - -download and install with - -#+begin_src bash -guix archive --import < genenetwork2-data-hash.nar -#+end_src bash - -and you should see a list of packages installing, e.g. - -#+begin_src bash -importing path `/gnu/store/l1zs2drn3zdzl5ysjcmhibcpa35p9zfc-python2-mysqlclient-1.3.7' -importing path `/gnu/store/n7kfg4knibvblggy8ci2liscl7vz5wkg-python2-parallel-1.6.4' -importing path `/gnu/store/qvv16qwlq59gp5d07lwbf5n8ndsi3il3-python2-sqlalchemy-1.0.11' -importing path `/gnu/store/qw872mbmr9ir0a9drv9xw9pvjk05ywwy-python2-xlsxwriter-0.8.4' -importing path `/gnu/store/wc112m1xfy3p08v14bdzay2ki2rirdsm-pylmm-gn2-1.0-3c6d1cac8' -importing path `/gnu/store/zfkcy17c2ks3cd9ks14irdabqvmlfpyn-python2-flask-sqlalchemy-2.1' -importing path `/gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff4' -(etc.) -#+end_src bash - -The following packages need to be added and the R path set - -: export R_LIBS_SITE="/home/wrk/.guix-profile/site-library/" -: guix package -i /gnu/store/w0dqg9dshq53j8xhcnqgvnvms2s6y5k5-r-wgcna-1.49-425bc170cc0873ddbd414675ac40f6d4d724c7cb -: guix package -i /gnu/store/k60bdlm0v7xic88j2z5c1jb1jvc371mn-r-qtl-1.38-4 - -You can add the last one to your profile - -: guix package -i /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff -: export PATH=~/.guix-profile/bin:$PATH -: genenetwork2 - - or run it directly with - -: /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff/bin/genenetwork2 +The official installation path is via a GNU Guix installation from a +checked out version of the Guix package tree (this is required for +reproducibility). * Source deployment @@ -120,9 +29,14 @@ You can add the last one to your profile Deploying from source is also straightforward. Install GNU Guix using a binary tar ball as described [[https://github.com/pjotrp/guix-notes][here]]. +If it works you should be able to install a package with + +: guix package -i hello + ** Checkout the git repositories -Check out the guix and guix-bioinformatics repositories: +Check out the two relevant guix and guix-bioinformatics git +repositories: #+begin_src bash cd ~ @@ -133,17 +47,20 @@ git clone --recursive --branch gn-latest https://github.com/genenetwork/guix gui cd guix-gn-latest #+end_src bash -** Run guix-daemon +*** Update the path -At this point you may decide to create, install and run a recent -version of the guix-daemon by compiling the guix repository. Follow -[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. +Now make sure the guix update path is inline with ours + +#+begin_src bash +mkdir -p ~/.config/guix +ln -s ~/genenetwork/guix-gn-latest ~/.config/guix/latest +#+end_src bash ** Install GN2 #+begin_src bash -env GUIX_PACKAGE_PATH=../guix-bioinformatics/ \ - ./pre-inst-env guix package -i genenetwork2 +export GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ +guix package -i genenetwork2 #+end_src bash ** Run GN2 @@ -190,22 +107,11 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; +** Notes -** Other - -Update guix with a 'guix pull' and make guix visible in the path. -More information exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. - -With guix running you should be able to install python, for example. - -: guix package -i python2 - -This will make python appear in $HOME/.guix-profile/bin/python. Suggested -environment settings can be seen with - -: guix package --search-paths - - +At some point you may decide to create, install and run a recent +version of the guix-daemon by compiling the guix repository. Follow +[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. * Source deployment and other information on reproducibility -- cgit v1.2.3 From 87442be548c8779bcc06693490b9e11ccba26dc5 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 5 Mar 2016 16:45:06 +0000 Subject: README --- doc/README.org | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index adb545f6..ec0054e4 100644 --- a/doc/README.org +++ b/doc/README.org @@ -5,10 +5,11 @@ - [[#source-deployment][Source deployment]] - [[#install-guix][Install guix]] - [[#checkout-the-git-repositories][Checkout the git repositories]] + - [[#update-guix][Update guix]] + - [[#update-the-path][Update the path]] - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] - - [[#notes][Notes]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -47,9 +48,15 @@ git clone --recursive --branch gn-latest https://github.com/genenetwork/guix gui cd guix-gn-latest #+end_src bash -*** Update the path +** Update guix -Now make sure the guix update path is inline with ours +At some point you may decide to create, install and run a recent +version of the guix-daemon by compiling the guix repository. Follow +[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. + +** Update the path + +Now make sure the guix update path is in line with ours #+begin_src bash mkdir -p ~/.config/guix @@ -60,7 +67,8 @@ ln -s ~/genenetwork/guix-gn-latest ~/.config/guix/latest #+begin_src bash export GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ -guix package -i genenetwork2 +cd ~/genenetwork/guix-gn-latest +./pre-inst guix package -i genenetwork2 #+end_src bash ** Run GN2 @@ -107,11 +115,6 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; -** Notes - -At some point you may decide to create, install and run a recent -version of the guix-daemon by compiling the guix repository. Follow -[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. * Source deployment and other information on reproducibility -- cgit v1.2.3 From a203e362ec57351f29274b32b7c80dc9392d591d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 6 Mar 2016 11:15:50 +0000 Subject: README --- doc/README.org | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index ec0054e4..efaec8a6 100644 --- a/doc/README.org +++ b/doc/README.org @@ -6,7 +6,6 @@ - [[#install-guix][Install guix]] - [[#checkout-the-git-repositories][Checkout the git repositories]] - [[#update-guix][Update guix]] - - [[#update-the-path][Update the path]] - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] @@ -54,21 +53,12 @@ At some point you may decide to create, install and run a recent version of the guix-daemon by compiling the guix repository. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. -** Update the path - -Now make sure the guix update path is in line with ours - -#+begin_src bash -mkdir -p ~/.config/guix -ln -s ~/genenetwork/guix-gn-latest ~/.config/guix/latest -#+end_src bash - ** Install GN2 #+begin_src bash export GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ cd ~/genenetwork/guix-gn-latest -./pre-inst guix package -i genenetwork2 +./pre-inst-env guix package -i genenetwork2 #+end_src bash ** Run GN2 -- cgit v1.2.3 From 3a34a3de80ce33afb7fb600620d693be26583462 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 6 Mar 2016 13:03:14 +0000 Subject: README --- doc/README.org | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index efaec8a6..85c6da1c 100644 --- a/doc/README.org +++ b/doc/README.org @@ -9,6 +9,7 @@ - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] + - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] * Introduction @@ -68,7 +69,8 @@ export PATH=~/.guix-profile/bin:$PATH genenetwork2 #+end_src bash -will start the default server which listens on port 5003. +will start the default server which listens on port 5003, i.e., +http://localhost:5003/. ** Run MySQL server @@ -105,6 +107,27 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; +** Run your own copy of GN2 + +At some point you may want to fix the source code. Clone the GN2 +repository from https://github.com/genenetwork/genenetwork2_diet + +Inside the repository: + +: cd genenetwork2_diet +: ./bin/genenetwork2 + +Will fire up your local repo http://localhost:5003/ using the +settings in ./etc/default_settings.py + +To override settings create your own from a copy of +default_settings.py and pass it into GN2 with + +: ./bin/genenetwork2 $HOME/my_settings.py + +and everything *should* work. This way we develop against +the exact same dependency graph of software. + * Source deployment and other information on reproducibility -- cgit v1.2.3 From 9b632ebbeb73de1369a49a712d614fd8a2b1ab30 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 6 Mar 2016 15:55:48 +0000 Subject: README --- doc/README.org | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 85c6da1c..7d4d7e2d 100644 --- a/doc/README.org +++ b/doc/README.org @@ -62,10 +62,29 @@ cd ~/genenetwork/guix-gn-latest ./pre-inst-env guix package -i genenetwork2 #+end_src bash +Make a note of the paths with + +#+begin_src bash +./pre-inst-env guix package --search-paths +#+end_src bash + ** Run GN2 +After setting the paths for the server + #+begin_src bash export PATH=~/.guix-profile/bin:$PATH +export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +export R_LIBS_SITE="$HOME/.guix-profile/site-library/" +export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0" +export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0" +export XDG_DATA_DIRS="$HOME/.guix-profile/share" +export GIO_EXTRA_MODULES="$HOME/.guix-profile/lib/gio/modules" +#+end_src bash + +run the main script (in ~/.guix-profile/bin) + +#+begin_src bash genenetwork2 #+end_src bash @@ -125,8 +144,9 @@ default_settings.py and pass it into GN2 with : ./bin/genenetwork2 $HOME/my_settings.py -and everything *should* work. This way we develop against -the exact same dependency graph of software. +and everything *should* work (note the full path to the settings +file). This way we develop against the exact same dependency graph of +software. * Source deployment and other information on reproducibility -- cgit v1.2.3 From bbbca36baf3739c15e7a276c22c1c37700992e60 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 16 Mar 2016 15:08:56 -0500 Subject: README --- doc/README.org | 1 + 1 file changed, 1 insertion(+) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 7d4d7e2d..86c8faaa 100644 --- a/doc/README.org +++ b/doc/README.org @@ -133,6 +133,7 @@ repository from https://github.com/genenetwork/genenetwork2_diet Inside the repository: +: git clone : cd genenetwork2_diet : ./bin/genenetwork2 -- cgit v1.2.3 From 3a3498c519eca58abec3dab08b8309cc15e00e7d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 21 Mar 2016 16:57:11 -0500 Subject: Docs: Database and Guix --- doc/GUIX-archive.org | 106 ++++++++++++++++++++++ doc/database.org | 245 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 351 insertions(+) create mode 100644 doc/GUIX-archive.org create mode 100644 doc/database.org (limited to 'doc') diff --git a/doc/GUIX-archive.org b/doc/GUIX-archive.org new file mode 100644 index 00000000..67ab5cd0 --- /dev/null +++ b/doc/GUIX-archive.org @@ -0,0 +1,106 @@ +* Binary deployment + +Note binary deployment is not working pending a few improvements +to GNU Guix. See source deployment instead. + +** Install Guix using a tar ball + +GN can be deployed either as a binary tarball or as a GNU Guix +package. First install GNU Guix following the instructions of the +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]]. + +With guix-daemon running you should be able to install the hello +package: + +: guix package -i hello + +** Fix locale + +You may want to + +#+begin_src sh :lang bash +export GUIX_LOCPATH=$HOME/.guix-profile/lib/locale +export LC_ALL=en_US.utf8 +#+end_src sh :lang bash + +** Authorize our archives + +Next add our archive key to guix (as root): + +#+begin_src scheme +echo "(public-key + (ecc + (curve Ed25519) + (q #E9A95686D8437186302E07C7AB9BF3913F435026C2D389AF27D9C66FD6EBB649#) + ) + ) +"|guix archive --authorize +#+end_src scheme + +if you have trouble finding a suitable guix try + +: ls /gnu/store/*guix-*/bin/guix + +and you should be able to use this directly, e.g. + +: alias guix=/gnu/store/632msbms2yaldfnlrb5lbnlnmn9yjisw-guix-0.9.0/bin/guix +: guix --version + +** Download and install the GN2 archive + +Find the archive on + + http://files.genenetwork.org/software/ + +download and install with + +#+begin_src bash +guix archive --import < genenetwork2-data-hash.nar +#+end_src bash + +and you should see a list of packages installing, e.g. + +#+begin_src bash +importing path `/gnu/store/l1zs2drn3zdzl5ysjcmhibcpa35p9zfc-python2-mysqlclient-1.3.7' +importing path `/gnu/store/n7kfg4knibvblggy8ci2liscl7vz5wkg-python2-parallel-1.6.4' +importing path `/gnu/store/qvv16qwlq59gp5d07lwbf5n8ndsi3il3-python2-sqlalchemy-1.0.11' +importing path `/gnu/store/qw872mbmr9ir0a9drv9xw9pvjk05ywwy-python2-xlsxwriter-0.8.4' +importing path `/gnu/store/wc112m1xfy3p08v14bdzay2ki2rirdsm-pylmm-gn2-1.0-3c6d1cac8' +importing path `/gnu/store/zfkcy17c2ks3cd9ks14irdabqvmlfpyn-python2-flask-sqlalchemy-2.1' +importing path `/gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff4' +(etc.) +#+end_src bash + +The following packages need to be added and the R path set + +: export R_LIBS_SITE="/home/wrk/.guix-profile/site-library/" +: guix package -i /gnu/store/w0dqg9dshq53j8xhcnqgvnvms2s6y5k5-r-wgcna-1.49-425bc170cc0873ddbd414675ac40f6d4d724c7cb +: guix package -i /gnu/store/k60bdlm0v7xic88j2z5c1jb1jvc371mn-r-qtl-1.38-4 + +You can add the last one to your profile + +: guix package -i /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff +: export PATH=~/.guix-profile/bin:$PATH +: genenetwork2 + + or run it directly with + +: /gnu/store/cgcjdiz1qylbc372gc3nda3372ihkpqb-genenetwork2-2.0-a8fcff/bin/genenetwork2 + + + +** Other + +Update guix with a 'guix pull' and make guix visible in the path. +More information exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. + +With guix running you should be able to install python, for example. + +: guix package -i python2 + +This will make python appear in $HOME/.guix-profile/bin/python. Suggested +environment settings can be seen with + +: guix package --search-paths + + diff --git a/doc/database.org b/doc/database.org new file mode 100644 index 00000000..6c079225 --- /dev/null +++ b/doc/database.org @@ -0,0 +1,245 @@ +- github Document reduction issue +- github bio-vcf issue + + +* GeneNetwork Database + +All *Data tables are large + +** User access + +GN1 uses access table and GN2 uses user table. + +** AvgMethod + +Probesetfreeze refers to AvgMethod + +** BXDSnPosition + +Snp table (all snps) + +Mapping in GN1 shows snps when you select a chromosome. + +** CaseAttribute(XRef) + +Metadata + +** CeleralINFO_mm6 + +? + +** Chr_Length + +Default mm9, column for mm8 + +** Dataset_mbat + +Menu for BXD (linkouts) + +** DatasetMapInvestigator + +Arthur? + +** DataSets + +Information/metadata + +** DatasetStatus + +Arthur private/public + +** DBList and DBType + +Hooked in API (URL encoding) + +** Docs + +GN2 only (see menu bar) + +** Ensembl* + +Probe information + +(will be deprecated) + +** Genbank + +Linkout and not important + +** GeneCategory + +Not important. GeneWiki notes function classification. + +Deprecate. + +** GeneChip + +** GeneIDXRef + +Interspecies gene comparison + +** GeneList + +Track info + +** Genlist_rn3(3) + +Rat list + +** GeneMap_cuiyan + +Link outs + +** GeneRIF + +Wiki info (nightly updated from NCBI) + +XRef should be foreign keys + +** Geno + +SNP or marker info + +** GenoCode + +Belongs to someone else + +** GenoData + +Allele info + +** GenoFreeze + +Big menu (Freeze refers to menu) + +** GenoSE + +SE standard err, not used + +** GenoXREF + +Very important. Key links between Geno, GenoData + +** GORef + +GO terms + +** H2 + +Heritability for probeset(?) + +** Homologene + +Homology, not used much + +** InbredSet + +Group in menu + +** Indelall, SnpAll, SnpPattern, SnpSource + +Indel Snp browser (variant browser Gn1) + +** Info* + +Infra system PhP + +Data Info button + +Infosystem users has separate entries + +Also Investigators, User, Organizations, + +** LCorrRamin3 + +Lit. Correlations Prof. Ramin + +** Login + +GN2 login info + +** MachineAccessLog + +Old + +** MappingMethod + +GN1 + +** News + +GN2 + +** NStrain + +pheno publishfreeze (menu) + xref (keys) + xref links to publish (pubmed), phenotype, pubishdata +geno genofreeze + xref (keys) + xref links to publish (pubmed), genotype, genodata +probeset/expr. probesetfreeze + xref (keys) + xref links to publish (pubmed), probeset, probesetdata +probe/expr. probefreeze + xref (keys) + xref links to publish (pubmed), probe, probedata + +Each dataset has 3 values (real value (1), number of samples (2), stderr (3)) + +NStrain = number of phenotype samples + +ProbesetFreeze contains all data, incl. metabolomic. + +** Probe + +Probedata - main molecular data. Probesets, metabolome, + +** Publication and publishdata (all pheno) + +Phenotype pubs + +** QuickSearch + +No longer used + +** role + +empty + +** Sample* + +No longer used + +** Species & Strain (should be sample) + +Menu + +** InbredSet + +Menu + +** TableComments + +Metadata on DB + +** Temp* + +User upload data + +** Tissue + +Menu - 3rd level + +** TissueP* + +Correlation tables + +** User collection + +User selection - retained + +** UserPrivilege + +** Vlookup + -- cgit v1.2.3 From 245399f72caae48b4aa2823b6959e78118018709 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 29 Mar 2016 15:15:34 -0500 Subject: README --- doc/README.org | 16 ++++++++++------ doc/database.org | 2 -- 2 files changed, 10 insertions(+), 8 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 86c8faaa..1894d8d3 100644 --- a/doc/README.org +++ b/doc/README.org @@ -24,7 +24,6 @@ checked out version of the Guix package tree (this is required for reproducibility). * Source deployment - ** Install guix Deploying from source is also straightforward. Install GNU Guix using @@ -43,7 +42,7 @@ repositories: cd ~ mkdir genenetwork cd genenetwork -git clone https://github.com/genenetwork/guix-bioinformatics +git clone --branch gn-latest https://github.com/genenetwork/guix-bioinformatics git clone --recursive --branch gn-latest https://github.com/genenetwork/guix guix-gn-latest cd guix-gn-latest #+end_src bash @@ -57,11 +56,13 @@ version of the guix-daemon by compiling the guix repository. Follow ** Install GN2 #+begin_src bash -export GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ -cd ~/genenetwork/guix-gn-latest -./pre-inst-env guix package -i genenetwork2 +env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --fallback #+end_src bash +Note that you can use the genenetwork.org guix substitute caching +server at http://guix.genenetwork.org (which speeds up installs +significantly because all packages are pre-built). + Make a note of the paths with #+begin_src bash @@ -126,6 +127,10 @@ Set permissions and match password in your settings file below: : mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; +Note that if the mysql connection is not working, try connecting to +the IP address and check server firewall, hosts.allow and mysql IP +configuration. + ** Run your own copy of GN2 At some point you may want to fix the source code. Clone the GN2 @@ -149,7 +154,6 @@ and everything *should* work (note the full path to the settings file). This way we develop against the exact same dependency graph of software. - * Source deployment and other information on reproducibility See the document [[GUIX-Reproducible-from-source.org]]. diff --git a/doc/database.org b/doc/database.org index 6c079225..af4b30eb 100644 --- a/doc/database.org +++ b/doc/database.org @@ -1,6 +1,4 @@ - github Document reduction issue -- github bio-vcf issue - * GeneNetwork Database -- cgit v1.2.3 From aa47bf9d94c3f5232cabfd1bef2eb3b54241d076 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 4 Apr 2016 11:38:15 +0000 Subject: README: point to gn-latest SHA values --- README.md | 2 +- doc/README.org | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'doc') diff --git a/README.md b/README.md index ce435f61..4fe739bf 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ developers). See the [installation docs](doc/README.org). ## License The GeneNetwork2 source code is released under the Affero General -Public License 3 (AGPLv3). See also LICENSE.txt +Public License 3 (AGPLv3). See [LICENSE.txt](LICENSE.txt). ## More information diff --git a/doc/README.org b/doc/README.org index 1894d8d3..5fffa108 100644 --- a/doc/README.org +++ b/doc/README.org @@ -14,14 +14,15 @@ * Introduction -Large system deployments tend to get very complex. In this document we -explain the GeneNetwork deployment system which is based on GNU Guix -(see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install -GN with all its files and dependencies. - -The official installation path is via a GNU Guix installation from a -checked out version of the Guix package tree (this is required for -reproducibility). +Large system deployments can get very complex. In this document we +explain the GeneNetwork version 2 (GN2) reproducible deployment system +which is based on GNU Guix (see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix +system can be used to install GN with all its files and dependencies. + +The official installation path is from a checked out version of the +main Guix package tree and that of the Genenetwork package +tree. Current supported versions can be found as the SHA values of +'gn-latest' branches of [[https://github.com/genenetwork/guix-bioinformatics/tree/gn-latest][Guix bioinformatics]] and [[https://github.com/genenetwork/guix/tree/gn-latest][GNU Guix main]]. * Source deployment ** Install guix -- cgit v1.2.3 From 551b57ca82abf77fad388cb3aead1c3f58fdc43b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 9 Apr 2016 10:43:52 +0000 Subject: Database information --- doc/README.org | 4 +- doc/database.org | 471 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 471 insertions(+), 4 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 5fffa108..ee54f781 100644 --- a/doc/README.org +++ b/doc/README.org @@ -43,8 +43,8 @@ repositories: cd ~ mkdir genenetwork cd genenetwork -git clone --branch gn-latest https://github.com/genenetwork/guix-bioinformatics -git clone --recursive --branch gn-latest https://github.com/genenetwork/guix guix-gn-latest +git clone --branch gn-latest https://github.com/genenetwork/guix-bioinformatics +git clone --branch gn-latest --recursive https://github.com/genenetwork/guix guix-gn-latest cd guix-gn-latest #+end_src bash diff --git a/doc/database.org b/doc/database.org index af4b30eb..e06ac1ff 100644 --- a/doc/database.org +++ b/doc/database.org @@ -1,12 +1,189 @@ - github Document reduction issue + * GeneNetwork Database +** Estimated table sizes + + +select table_name,round(((data_length + index_length) / 1024 / 1024), 2) `Size in MB` from information_schema.TABLES where table_schema = "db_webqtl" order by data_length; + ++-------------------------+------------+ +| table_name | Size in MB | ++-------------------------+------------+ +| ProbeSetData | 59358.80 | +| SnpAll | 15484.67 | +| ProbeData | 22405.44 | +| SnpPattern | 9177.05 | +| ProbeSetSE | 14551.02 | +| QuickSearch | 5972.86 | +| ProbeSetXRef | 4532.89 | +| LCorrRamin3 | 18506.53 | +| ProbeSE | 6263.83 | +| ProbeSet | 2880.21 | +| Probe | 2150.30 | +| GenoData | 3291.91 | +| CeleraINFO_mm6 | 989.80 | +| pubmedsearch | 1032.50 | +| ProbeXRef | 743.38 | +| GeneRIF_BASIC | 448.54 | +| BXDSnpPosition | 224.44 | +| EnsemblProbe | 133.66 | +| EnsemblProbeLocation | 105.49 | +| Genbank | 37.71 | +| TissueProbeSetData | 74.42 | +| AccessLog | 42.38 | +| GeneList | 34.11 | +| Geno | 33.90 | +| MachineAccessLog | 28.34 | +| IndelAll | 22.42 | +| PublishData | 22.54 | +| TissueProbeSetXRef | 14.73 | +| ProbeH2 | 13.26 | +| GenoXRef | 22.83 | +| TempData | 8.35 | +| GeneList_rn3 | 5.54 | +| GORef | 4.97 | +| Phenotype | 6.50 | +| temporary | 3.59 | +| InfoFiles | 3.32 | +| Publication | 3.42 | +| Homologene | 5.69 | +| Datasets | 2.31 | +| GeneList_rn33 | 2.61 | +| PublishSE | 4.71 | +| GeneRIF | 2.18 | +| Vlookup | 1.87 | +| H2 | 2.18 | +| PublishXRef | 2.18 | +| NStrain | 4.80 | +| IndelXRef | 2.91 | +| Strain | 1.07 | +| GeneMap_cuiyan | 0.51 | +| user_collection | 0.30 | +| CaseAttributeXRef | 0.44 | +| StrainXRef | 0.56 | +| GeneIDXRef | 0.77 | +| Docs | 0.17 | +| News | 0.17 | +| ProbeSetFreeze | 0.22 | +| GeneRIFXRef | 0.24 | +| Sample | 0.06 | +| login | 0.06 | +| user | 0.04 | +| TableFieldAnnotation | 0.05 | +| DatasetMapInvestigator | 0.05 | +| User | 0.04 | +| ProbeFreeze | 0.06 | +| TableComments | 0.02 | +| Investigators | 0.02 | +| DBList | 0.03 | +| Tissue | 0.02 | +| GeneChip | 0.01 | +| GeneCategory | 0.01 | +| SampleXRef | 0.01 | +| InbredSet | 0.01 | +| SnpAllele_to_be_deleted | 0.00 | +| Organizations | 0.01 | +| PublishFreeze | 0.00 | +| GenoFreeze | 0.00 | +| Chr_Length | 0.01 | +| SnpSource | 0.00 | +| AvgMethod | 0.00 | +| Species | 0.00 | +| Dataset_mbat | 0.00 | +| TissueProbeFreeze | 0.00 | +| EnsemblChip | 0.00 | +| TissueProbeSetFreeze | 0.01 | +| UserPrivilege | 0.00 | +| CaseAttribute | 0.00 | +| MappingMethod | 0.00 | +| DBType | 0.00 | +| InfoFilesUser_md5 | 0.00 | +| GenoCode | 0.00 | +| DatasetStatus | 0.00 | +| GeneChipEnsemblXRef | 0.00 | +| GenoSE | 0.00 | +| user_openids | 0.00 | +| roles_users | 0.00 | +| role | 0.00 | +| Temp | NULL | ++-------------------------+------------+ +97 rows in set, 1 warning (0.01 sec) + All *Data tables are large ** User access -GN1 uses access table and GN2 uses user table. +According to the meta data: + +This table tracks access time and IP addresses. Used for logging in +registered users and tracking cookies. + +# GN1 uses access table and GN2 uses user table (true/false?) + + select * from AccessLog limit 5; ++-------+---------------------+----------------+ +| id | accesstime | ip_address | ++-------+---------------------+----------------+ +| 12174 | 2003-10-28 02:17:41 | 130.120.104.71 | +| 12173 | 2003-10-28 02:16:27 | 130.120.104.71 | +| 3 | 2003-02-22 07:38:33 | 192.117.159.1 | +| 4 | 2003-02-22 07:49:13 | 192.117.159.1 | +| 5 | 2003-02-22 07:51:08 | 192.117.159.1 | ++-------+---------------------+----------------+ + +select * from AccessLog order by accesstime desc limit 5; ++---------+---------------------+---------------+ +| id | accesstime | ip_address | ++---------+---------------------+---------------+ +| 1025735 | 2016-02-08 14:23:29 | 100.43.81.157 | +| 1025734 | 2016-02-08 13:54:28 | 180.76.15.144 | +| 1025733 | 2016-02-08 13:43:37 | 66.249.65.217 | +| 1025732 | 2016-02-08 13:39:50 | 66.249.65.217 | +| 1025731 | 2016-02-08 13:15:46 | 66.249.65.217 | ++---------+---------------------+---------------+ + +Quite a few trait page hits: + +select count(*) from AccessLog; + ++----------+ +| count(*) | ++----------+ +| 1025685 | ++----------+ + +show indexes from AccessLog; ++-----------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | ++-----------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| AccessLog | 0 | PRIMARY | 1 | id | A | 1025685 | NULL | NULL | | BTREE | | | ++-----------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ + +This table is being used by both GN1 and GN2 from the trait pages! + +: grep -ir AccessLog *|grep -e "^gn1\|^gn2"|grep \.py|grep -v doc + +gn1/web/webqtl/showTrait/ShowTraitPage.py: query = "SELECT count(id) FROM AccessLog WHERE ip_address = %s and \ +gn1/web/webqtl/showTrait/ShowTraitPage.py: self.cursor.execute("insert into AccessLog(accesstime,ip_address) values(Now(),%s)" ,user_ip) +gn1/web/webqtl/textUI/cmdClass.py: query = """SELECT count(id) FROM AccessLog WHERE ip_address = %s AND UNIX_TIMESTAMP()-UNIX_TIMESTAMP(accesstime)<86400""" +gn1/web/webqtl/textUI/cmdClass.py: query = """INSERT INTO AccessLog(accesstime,ip_address) values(Now(),%s)""" +gn2/wqflask/wqflask/show_trait/show_trait_page.py: query = "SELECT count(id) FROM AccessLog WHERE ip_address = %s and \ +gn2/wqflask/wqflask/show_trait/show_trait_page.py: self.cursor.execute("insert into AccessLog(accesstime,ip_address) values(Now(),%s)", user_ip) + +When looking at the code in GN1 and GN2 it restricts the daily use of +the trait data page (set to 1,000 - whoever reaches that?). Unlike +mentioned in the schema description, this table does *not* keep track +of cookies. + +From the code it looks like GN2 uses a mixture of Redis and sqlalchemy +to keep track of logged in sessions (see +gn2/wqflask/wqflask/user_manager.py) and cookies through a user_uuid in +model.py. + +In gn2/wqflask/wqflask/templates/collections/view_anonymous.html it +show_trait_page appears to be loaded (need to check). ** AvgMethod @@ -189,10 +366,300 @@ NStrain = number of phenotype samples ProbesetFreeze contains all data, incl. metabolomic. -** Probe +** Phenotype + +This table contains names, full descriptions, and short symbols for +traits and phenotype used primarily in the Published Phenotypes +databases. + +Contains 10k rows, March 2016, of which 5000 are for the BXDs). + +| Id | Pre_publication_description | Post_publication_description | Original_description | Units | Pre_publication_abbreviation | Post_publication_abbreviation | Lab_code | Submitter | Owner | Authorized_Users | ++----+-----------------------------+----------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------+-------------------------------+----------+-------------+-------+------------------+ +| 1 | NULL | Hippocampus weight | Original post publication description: Hippocampus weight | Unknown | NULL | HPCWT | NULL | robwilliams | NULL | robwilliams | +| 2 | NULL | Cerebellum weight | Original post publication description: Cerebellum weight | mg | NULL | CBLWT | NULL | robwilliams | NULL | robwilliams | +| 3 | NULL | Interleukin 1 activity by peritoneal macrophages stimulated with 10 ug/ml lipopolysaccharide [units/100 ug protein] | Original post publication description: Interleukin 1 activity by peritoneal macrophages stimulated with 10 ug/ml lipopolysaccharide [units/100 ug protein] | units/100 ug protein | NULL | IL1Activity | NULL | robwilliams | NULL | robwilliams | +| 4 | NULL | Central nervous system, morphology: Cerebellum weight, whole, bilateral in adults of both sexes [mg] | Original post publication description: Cerebellum weight [mg] | mg | NULL | CBLWT2 | NULL | robwilliams | NULL | robwilliams | +| 5 | NULL | The coat color of 79 BXD RI strain | Original post publication description: The coat color of 79 BXD RI strain | Unknown | NULL | CoatColor | NULL | robwilliams | NULL | robwilliams | ++----+-----------------------------+----------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------+-------------------------------+----------+-------------+-------+------------------+ +5 rows in set (0.00 sec) + +** ProbeData + +Table with fine-grained probe level Affymetrix data only. Contains 1 +billion rows March 2016. This table may be deletable since it is only +used by the Probe Table display in GN1. Not used in GN2 +(double-check). + +In comparison the "ProbeSetData" table contains more molecular assay +data, including probe set data, RNA-seq data, proteomic data, and +metabolomic data. 2.5 billion rows March 2016. In comparison, +ProbeData contains data only for Affymetrix probe level data +(e.g. Exon array probes and M430 probes). + +"ProbeData.StrainId" should be "CaseId" or "SampleId". + +"ProbeData" should probably be "AssayData" or something more neutral. + +select * from ProbeData limit 2; ++--------+----------+---------+ +| Id | StrainId | value | ++--------+----------+---------+ +| 503636 | 42 | 11.6906 | +| 503636 | 43 | 11.4205 | ++--------+----------+---------+ +2 rows in set (0.00 sec) + +select count(*) from ProbeData limit 2; ++-----------+ +| count(*) | ++-----------+ +| 976753435 | ++-----------+ +1 row in set (0.00 sec) + +** ProbeSet + +Comment: PLEASE CHANGE TABLE NAME and rework fields carefully. This is +a terrible table but it works well (RWW March 2016). It is used in +combination with the crucial TRAIT DATA and ANALYSIS pages in GN1 and +GN2. It is also used by annotators using the UPDATE INFO AND DATA web +form to correct and update annotation. It is used by Arthur to enter +new annotation files and metadata for arrays, genes, proteins, +metabolites. The main problem with this table is that it is doing too +much work. + +Initially (2003) this table contained only Affymetrix ProbeSet data +for mouse (U74aV2 initially). Many other array platforms for different +species were added. At least four other major categories of molecular +assays have been added since about 2010. + +1. RNA-seq annotation and sequence data for transcripts using ENSEMBL + identifiers or NCBI NM_XXXXX and NR_XXXXX type identifiers + +2. Protein and peptide annotation and sequence data (see BXD Liver + Proteome data, SRM and SWATH type data) with identifiers such as + "abcb10_q9ji39_t311" for SRM data and "LLGNMIVIVLGHHLGKDFTPAAQAA" + for SWATH data where the latter is just the peptide fragment that + has been quantified. Data first entered in 2015 for work by Rudi + Aebersold and colleagues. + +3. Metabolite annotation and metadata (see BXD Liver Metabolome data) + with identifiers that are usually Mass charge ratios such as + "149.0970810_MZ" + +4. Epigenomic and methylome data (e.g. Human CANDLE Methylation data + with identifiers such as "cg24523000") + +It would make good sense to break this table into four or more types +of molecular assay metadata or annotation tables) (AssayRNA_Anno, +AssayProtein_Anno, AssayMetabolite_Anno, AssayEpigenome_Anno, +AssayMetagenome_Anno), since these assays will have many differences +in annotation content compared to RNAs. + +Some complex logic is used to update contents of this table when +annotators modify and correct the information (for example, updating +gene symbols). These features requested by Rob so that annotating one +gene symbol in one species would annotate all gene symbols in the same +species based on common NCBI GeneID number. For example, changing the +gene alias for one ProbeSet.Id will changing the list of aliases in +all instances with the same gene symbol. + +If the ProbeSet.BlatSeq (or is this ProbSetTargetSeq) is identical +between different ProbeSet.Ids then annotation is forced to be the +same even if the symbol or geneID is different. This "feature" was +implemented when we found many probe sets with identical sequence but +different annotations and identifiers. + + +select count(*) from ProbeSet limit 5; ++----------+ +| count(*) | ++----------+ +| 4351030 | ++----------+ + +| Id | ChipId | Name | TargetId | Symbol | description | Chr | Mb | alias | GeneId | GenbankId | SNP | BlatSeq | TargetSeq | UniGeneId | Strand_Probe | Strand_Gene | OMIM | comments | Probe_set_target_region | Probe_set_specificity | Probe_set_BLAT_score | Probe_set_Blat_Mb_start | Probe_set_Blat_Mb_end | Probe_set_strand | Probe_set_Note_by_RW | flag | Symbol_H | description_H | chromosome_H | MB_H | alias_H | GeneId_H | chr_num | name_num | Probe_Target_Description | RefSeq_TranscriptId | Chr_mm8 | Mb_mm8 | Probe_set_Blat_Mb_start_mm8 | Probe_set_Blat_Mb_end_mm8 | HomoloGeneID | Biotype_ENS | ProteinID | ProteinName | Flybase_Id | HMDB_ID | Confidence | ChEBI_ID | ChEMBL_ID | CAS_number | PubChem_ID | ChemSpider_ID | UNII_ID | EC_number | KEGG_ID | Molecular_Weight | Nugowiki_ID | Type | Tissue | PrimaryName | SecondaryNames | PeptideSequence | ++------+--------+----------+----------+--------+----------------------------------------------+------+-----------+----------+--------+-----------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+--------------+-------------+--------+----------+-------------------------+-----------------------+----------------------+-------------------------+-----------------------+------------------+----------------------+------+----------+---------------+--------------+------+---------+----------+---------+----------+--------------------------+---------------------+---------+-----------+-----------------------------+---------------------------+--------------+-------------+-----------+-------------+------------+---------+------------+----------+-----------+------------+------------+---------------+---------+-----------+---------+------------------+-------------+------+--------+-------------+----------------+-----------------+ +| 7282 | 1 | 93288_at | NULL | Arpc2 | actin related protein 2/3 complex, subunit 2 | 1 | 74.310961 | AK008777 | 76709 | AI835883 | 0 | CCGACTTCCTTAAGGTGCTCAACCGGACTGCTTGCTACTGGATAATCGTGAGGGATTCTCCATTTGGGTTCCATTTTGTACGAGTTTGGCAAATAACCTGCAGAAACGAGCTGTGCTTGCAAGGACTTGATAGTTCCTAATCCTTTTCCAAGCTGTTTGCTTTGCAATATGT | ccgacttccttaaggtgctcaaccgtnnnnnnccnannnnccnagaaaaaagaaatgaaaannnnnnnnnnnnnnnnnnnttcatcccgctaactcttgggaactgaggaggaagcgctgtcgaccgaagnntggactgcttgctactggataatcgtnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnntgagggattctccatttgggttccattttgtacgagtttggcaaataacctgcagaaacgagctgtgcttgcaaggacttgatagttcctaagaattanaanaaaaaaaanaanttccacttgatcaanttaattcccttttatttttcctccctcantccccttccttttccaagctgtttgctttgcaatatgt | Mm.337038 | + | | 604224 | | NULL | 8.45 | 169 | 74.310961 | 74.31466 | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | 1 | 93288 | NULL | XM_129773 | 1 | 74.197594 | 74.197594 | 74.201293 | 4187 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ++------+--------+----------+----------+--------+----------------------------------------------+------+-----------+----------+--------+-----------+------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+--------------+-------------+--------+----------+-------------------------+-----------------------+----------------------+-------------------------+-----------------------+------------------+----------------------+------+----------+---------------+--------------+------+---------+----------+---------+----------+--------------------------+---------------------+---------+-----------+-----------------------------+---------------------------+--------------+-------------+-----------+-------------+------------+---------+------------+----------+-----------+------------+------------+---------------+---------+-----------+---------+------------------+-------------+------+--------+-------------+----------------+-----------------+ +2 rows in set (0.00 sec) + + + + +** ProbeSetData Probedata - main molecular data. Probesets, metabolome, +Almost all important molecular assay data is in this table including +probe set data, RNA-seq data, proteomic data, and metabolomic +data. 2.5 billion rows March 2016. In comparison, ProbeData contains +data only for Affymetrix probe level data (e.g. Exon array probes and +M430 probes). + +select count(*) from ProbeSetData limit 5; ++---------------+ +| count(*) | ++---------------+ +| 2,510,566,472 | ++---------------+ + + +select * from ProbeSetData limit 5; ++----+----------+-------+ +| Id | StrainId | value | ++----+----------+-------+ +| 1 | 1 | 5.742 | +| 1 | 2 | 5.006 | +| 1 | 3 | 6.079 | +| 1 | 4 | 6.414 | +| 1 | 5 | 4.885 | ++----+----------+-------+ + +show indexes from ProbeSetData; ++--------------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | ++--------------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| ProbeSetData | 0 | DataId | 1 | Id | A | 34868978 | NULL | NULL | | BTREE | | | +| ProbeSetData | 0 | DataId | 2 | StrainId | A | 2510566472 | NULL | NULL | | BTREE | | | ++--------------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ + +select * from Strain limit 5; ++----+----------+----------+-----------+--------+-------+ +| Id | Name | Name2 | SpeciesId | Symbol | Alias | ++----+----------+----------+-----------+--------+-------+ +| 1 | B6D2F1 | B6D2F1 | 1 | NULL | NULL | +| 2 | C57BL/6J | C57BL/6J | 1 | B6J | NULL | +| 3 | DBA/2J | DBA/2J | 1 | D2J | NULL | +| 4 | BXD1 | BXD1 | 1 | NULL | NULL | +| 5 | BXD2 | BXD2 | 1 | NULL | NULL | ++----+----------+----------+-----------+--------+-------+ + +show indexes from Strain; ++--------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | ++--------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ +| Strain | 0 | PRIMARY | 1 | Id | A | 14368 | NULL | NULL | | BTREE | | | +| Strain | 0 | Name | 1 | Name | A | 14368 | NULL | NULL | YES | BTREE | | | +| Strain | 0 | Name | 2 | SpeciesId | A | 14368 | NULL | NULL | | BTREE | | | +| Strain | 1 | Symbol | 1 | Symbol | A | 14368 | NULL | NULL | YES | BTREE | | | ++--------+------------+----------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+ + +A typical query may look like + +SELECT Strain.Name, ProbeSetData.value, ProbeSetSE.error, ProbeSetData.Id + FROM (ProbeSetData, ProbeSetFreeze, Strain, ProbeSet, ProbeSetXRef) + left join ProbeSetSE on + (ProbeSetSE.DataId = ProbeSetData.Id AND ProbeSetSE.StrainId = ProbeSetData.StrainId) + WHERE + ProbeSetFreeze.name = 'B139_K_1206_M' AND + ProbeSetXRef.ProbeSetId = ProbeSet.Id AND + ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND + ProbeSetXRef.DataId = ProbeSetData.Id AND + ProbeSetData.StrainId = Strain.Id + Order BY Strain.Name + ++-------+-------+-------+----------+ +| Name | value | error | Id | ++-------+-------+-------+----------+ +| SM001 | 38.3 | NULL | 25309550 | +| SM001 | 2.7 | NULL | 25309520 | +| SM001 | 20.3 | NULL | 25309507 | +| SM001 | 125.8 | NULL | 25309511 | +| SM001 | 8.2 | NULL | 25309534 | ++-------+-------+-------+----------+ +5 rows in set (22.28 sec) + +select * from ProbeSetFreeze limit 5; ++----+---------------+-------+-------------+---------------------------------+---------------------------------------------+-------------------------+------------+-----------+--------+-----------------+-----------------+-----------+ +| Id | ProbeFreezeId | AvgID | Name | Name2 | FullName | ShortName | CreateTime | OrderList | public | confidentiality | AuthorisedUsers | DataScale | ++----+---------------+-------+-------------+---------------------------------+---------------------------------------------+-------------------------+------------+-----------+--------+-----------------+-----------------+-----------+ +| 1 | 3 | 1 | Br_U_0803_M | BXDMicroArray_ProbeSet_August03 | UTHSC Brain mRNA U74Av2 (Aug03) MAS5 | Brain U74Av2 08/03 MAS5 | 2003-08-01 | NULL | 0 | 0 | NULL | log2 | +| 2 | 10 | 1 | Br_U_0603_M | BXDMicroArray_ProbeSet_June03 | UTHSC Brain mRNA U74Av2 (Jun03) MAS5 | Brain U74Av2 06/03 MAS5 | 2003-06-01 | NULL | 0 | 0 | NULL | log2 | +| 3 | 8 | 1 | Br_U_0303_M | BXDMicroArray_ProbeSet_March03 | UTHSC Brain mRNA U74Av2 (Mar03) MAS5 | Brain U74Av2 03/03 MAS5 | 2003-03-01 | NULL | 0 | 0 | NULL | log2 | +| 4 | 5 | 1 | Br_U_0503_M | BXDMicroArray_ProbeSet_May03 | UTHSC Brain mRNA U74Av2 (May03) MAS5 | Brain U74Av2 05/03 MAS5 | 2003-05-01 | NULL | 0 | 0 | NULL | log2 | +| 5 | 4 | 1 | HC_U_0303_M | GNFMicroArray_ProbeSet_March03 | GNF Hematopoietic Cells U74Av2 (Mar03) MAS5 | GNF U74Av2 03/03 MAS5 | 2003-03-01 | NULL | 0 | 0 | NULL | log2 | ++----+---------------+-------+-------------+---------------------------------+---------------------------------------------+-------------------------+------------+-----------+--------+-----------------+-----------------+-----------+ + + select * from ProbeSetXRef limit 5; ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ +| ProbeSetFreezeId | ProbeSetId | DataId | Locus_old | LRS_old | pValue_old | mean | se | Locus | LRS | pValue | additive | h2 | ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ +| 1 | 1 | 1 | 10.095.400 | 13.3971627898894 | 0.163 | 5.48794285714286 | 0.08525787814808819 | rs13480619 | 12.590069931048001 | 0.269 | -0.28515625 | NULL | +| 1 | 2 | 2 | D15Mit189 | 10.042057464356201 | 0.431 | 9.90165714285714 | 0.0374686634976217 | CEL-17_50896182 | 10.5970737900941 | 0.304 | -0.11678333333333299 | NULL | +| 1 | 3 | 3 | D5Mit139 | 5.43678531742749 | 0.993 | 7.83948571428571 | 0.0457583416912569 | rs13478499 | 6.0970532702754 | 0.988 | 0.112957489878542 | NULL | +| 1 | 4 | 4 | D1Mit511 | 9.87815279480766 | 0.483 | 8.315628571428569 | 0.0470396593931327 | rs6154379 | 11.774867551173099 | 0.286 | -0.157113725490196 | NULL | +| 1 | 5 | 5 | D16H21S16 | 10.191723834264499 | 0.528 | 9.19345714285714 | 0.0354801718293322 | rs4199265 | 10.923263374016202 | 0.468 | 0.11476470588235299 | NULL | ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ + + +Note that the following unlimited search is very slow: + +select max(value) from ProbeSetData; + ++------------+ +| max(value) | ++------------+ +| 26436006 | ++------------+ +1 row in set (2 min 16.31 sec) + +which is in some form is used in the search page, see [[https://github.com/genenetwork/genenetwork2_diet/blob/master/wqflask/wqflask/do_search.py#L811][the search code]]. + + +*** Improvements? + +Suggestions on the schema page: + +"StrainId" should be "CaseId" or "SampleId". + +"ProbeSetData" should probably be "AssayData" or something more neutral. + +*** Comments + +I think the ProbeSetData table should be generalized to a 'phenotypes' +table with an 'sample_id' column and a 'value' column. + +A new table 'samples' will link each sample against an 'experiment', +an 'individual' and which in turn can link to a 'strain'. + +Experiment is here in a wide sense, GTex can be one - I don't want to +use dataset ;) + +This means a (slight) reordering: + +phenotypes: (id), sample_id, value +samples: experiment_id, individual_id +experiments: name, version +individual: strain_id +strains: species_id +species: ... + +ProbeData is also interesting, because it has the same structure as +ProbeSetData, but only contains microarrays. This tables should be one +(when we clear up the cross-referencing) as they both contain +phenotype values. Both are large tables. + +PublishData is another phenotype table with values only which can be +merged into that same table. + +So we have phenotype data in 3 tables with exactly the same +layout. There is also TissueProbeSet*, but we'll ignore those for +now. I think we should merge these into one and have the sample ref +refer to the type of data (probeset, probe, metabolomics, +whatever). These are all phenotype values and by having them split +into different tables they won't play well when looking for +correlations. + +ProbeSet contains the metadata on the probes and should (eventually) +move into NoSQL. There is plenty redundancy in that table now. + +I know it is going to be a pain to reorganize the database, but if we +want to use it in the long run we are going to have to simplify it. + + + ** Publication and publishdata (all pheno) Phenotype pubs -- cgit v1.2.3 From 28ccde8e417b965710bbeef48d468a6b717c10a4 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 10:44:10 +0000 Subject: A few fixes to get the webserver running again --- doc/README.org | 11 ++++++++--- wqflask/utility/tools.py | 1 - wqflask/wqflask/marker_regression/gemma_mapping.py | 4 +--- wqflask/wqflask/marker_regression/marker_regression.py | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index ee54f781..3b96717f 100644 --- a/doc/README.org +++ b/doc/README.org @@ -134,12 +134,17 @@ configuration. ** Run your own copy of GN2 -At some point you may want to fix the source code. Clone the GN2 -repository from https://github.com/genenetwork/genenetwork2_diet +At some point you may want to fix the source code. Assuming you have +Guix and Genenetwork2 installed (as described above) clone the GN2 +repository from https://github.com/genenetwork/genenetwork2_diet + +Copy the paths into your terminal from (mainly so PYTHON_PATH and +R_LIBS_SITE are set) + +: guix package --search-paths Inside the repository: -: git clone : cd genenetwork2_diet : ./bin/genenetwork2 diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 9405a9c6..dd8c4a1e 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -5,7 +5,6 @@ import os import sys from wqflask import app - def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with get_valid_path is a function checking whether the diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index ae426621..8fb086c1 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -1,9 +1,7 @@ import os from base import webqtlConfig -from utility.tools import gemma_command - -GEMMA_PATH,GEMMA_COMMAND = gemma_command() +from utility.tools import GEMMA_COMMAND def run_gemma(this_dataset, samples, vals): """Generates p-values for each marker using GEMMA""" diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 39d7c78a..97949f93 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -1103,5 +1103,5 @@ def get_markers_from_csv(included_markers, p_values, group_name): return markers - if __name__ == '__main__': - import cPickle as pickle +if __name__ == '__main__': + import cPickle as pickle -- cgit v1.2.3 From 9001d7575cef0dcb0aaecfa15e8c242873fae01e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 26 Apr 2016 11:28:51 +0000 Subject: README: add environment paths info --- doc/README.org | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 3b96717f..494e2a70 100644 --- a/doc/README.org +++ b/doc/README.org @@ -11,6 +11,8 @@ - [[#run-mysql-server][Run MySQL server]] - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] + - [[#trouble-shooting][Trouble shooting]] + - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] * Introduction @@ -138,8 +140,8 @@ At some point you may want to fix the source code. Assuming you have Guix and Genenetwork2 installed (as described above) clone the GN2 repository from https://github.com/genenetwork/genenetwork2_diet -Copy the paths into your terminal from (mainly so PYTHON_PATH and -R_LIBS_SITE are set) +Copy-paste the paths into your terminal (mainly so PYTHON_PATH and +R_LIBS_SITE are set) from the information given by guix: : guix package --search-paths @@ -164,3 +166,21 @@ software. See the document [[GUIX-Reproducible-from-source.org]]. +* Trouble shooting + +** ImportError: No module named jinja2 + +If you have all the Guix packages installed this error points out that +the environment variables are not set. Copy-paste the paths into your +terminal (mainly so PYTHON_PATH and R_LIBS_SITE are set) from the +information given by guix: + +: guix package --search-paths + +On one system: + +: export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +: export R_LIBS_SITE="$HOME/.guix-profile/site-library/" +: export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0" + +and perhaps a few more. -- cgit v1.2.3 From 6e23be8f278e98386171f4702414224545decfe0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 28 Apr 2016 13:16:00 +0000 Subject: README --- doc/README.org | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 494e2a70..13f70ad8 100644 --- a/doc/README.org +++ b/doc/README.org @@ -13,6 +13,7 @@ - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] - [[#trouble-shooting][Trouble shooting]] - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] + - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]] * Introduction @@ -59,12 +60,14 @@ version of the guix-daemon by compiling the guix repository. Follow ** Install GN2 #+begin_src bash -env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --fallback +env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env \ + guix package -i genenetwork2 --fallback #+end_src bash Note that you can use the genenetwork.org guix substitute caching server at http://guix.genenetwork.org (which speeds up installs -significantly because all packages are pre-built). +significantly because all packages are pre-built). And/or use the Guix +mirror with option --substitute-urls=http://mirror.guixsd.org Make a note of the paths with @@ -147,13 +150,12 @@ R_LIBS_SITE are set) from the information given by guix: Inside the repository: -: cd genenetwork2_diet +: cd genenetwork2 : ./bin/genenetwork2 Will fire up your local repo http://localhost:5003/ using the -settings in ./etc/default_settings.py - -To override settings create your own from a copy of +settings in ./etc/default_settings.py. These settings may +not reflect your system. To override settings create your own from a copy of default_settings.py and pass it into GN2 with : ./bin/genenetwork2 $HOME/my_settings.py @@ -162,6 +164,13 @@ and everything *should* work (note the full path to the settings file). This way we develop against the exact same dependency graph of software. +If something is not working, take a hint from the settings file +that comes in the Guix installation. It sits in something like + +: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py + + + * Source deployment and other information on reproducibility See the document [[GUIX-Reproducible-from-source.org]]. @@ -184,3 +193,9 @@ On one system: : export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0" and perhaps a few more. +** ERROR: can not find directory $HOME/gn2_data + +The default settings file looks in your $HOME/gn2_data. Since these +files come with a Guix installation you should take a hint from the +values in the installed version of default_settings.py (see above in +this document). -- cgit v1.2.3 From 4c206f9277561446f32dd01004ff3eae682cfd27 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 29 Apr 2016 10:01:25 +0000 Subject: README: on missing modules --- doc/README.org | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 13f70ad8..10a77580 100644 --- a/doc/README.org +++ b/doc/README.org @@ -14,6 +14,7 @@ - [[#trouble-shooting][Trouble shooting]] - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]] + - [[#cant-run-a-module][Can't run a module]] * Introduction @@ -199,3 +200,10 @@ The default settings file looks in your $HOME/gn2_data. Since these files come with a Guix installation you should take a hint from the values in the installed version of default_settings.py (see above in this document). + +** Can't run a module + +In rare cases, development modules are not brought in with Guix +because no source code is available. This can lead to missing modules +on a running server. Please check with the authors when a module +is missing. -- cgit v1.2.3 From 38d0dddf1359c64e065a408a4f4b307434bc3584 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 11 May 2016 09:18:49 +0000 Subject: README: nginx --- doc/README.org | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 10a77580..9608548d 100644 --- a/doc/README.org +++ b/doc/README.org @@ -1,3 +1,4 @@ + #+TITLE: Installing GeneNetwork services * Table of Contents :TOC: @@ -10,6 +11,7 @@ - [[#run-gn2][Run GN2]] - [[#run-mysql-server][Run MySQL server]] - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] + - [[#set-up-nginx-port-forwarding][Set up nginx port forwarding]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] - [[#trouble-shooting][Trouble shooting]] - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] @@ -170,7 +172,44 @@ that comes in the Guix installation. It sits in something like : cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py +** Set up nginx port forwarding + +Install the webserver (as root) + +: guix package -i nginx + +The configuration examples can be found in the Guix store through + +: ls -l /root/.guix-profile/sbin/nginx +: lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx + +Use that path + +: ls /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/share/nginx/conf/ +: fastcgi.conf koi-win scgi_params +: fastcgi.conf.default mime.types scgi_params.default +: fastcgi_params mime.types.default uwsgi_params +: fastcgi_params.default nginx.conf uwsgi_params.default +: koi-utf nginx.conf.default win-utf + +And copy any relevant files to /etc/nginx. A configuration file for +GeneNetwork (reverse proxy) port forwarding can be found in the source +repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc +(still as root) + +: cp ./etc/nginx-genenetwork.conf /etc/nginx/ + +Make dirs + +: mkdir -p /var/spool/nginx/logs + +Add users + +: adduser nobody ; addgroup nobody + +Run nginx +: /root/.guix-profile/sbin/nginx -c /etc/nginx/nginx-genenetwork.conf -p /var/spool/nginx * Source deployment and other information on reproducibility -- cgit v1.2.3 From 7334000f8758fbdeca6885a1637b44663f4ea2a7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 06:24:06 +0000 Subject: Added IRC log --- doc/README.org | 415 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 409 insertions(+), 6 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 9608548d..9fa44442 100644 --- a/doc/README.org +++ b/doc/README.org @@ -68,9 +68,415 @@ env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env \ #+end_src bash Note that you can use the genenetwork.org guix substitute caching -server at http://guix.genenetwork.org (which speeds up installs -significantly because all packages are pre-built). And/or use the Guix -mirror with option --substitute-urls=http://mirror.guixsd.org +server at http://guix.genenetwork.org:8080 (which speeds up installs +significantly because all packages are pre-built). Here an IRC session +where we installed GN2 from scratch using GNU Guix and a download +of the test database: + +#+begin_src bash + time to get binary install sorted :) [07:03] + Guix is designed for distributed installation servers + we have one on guix.genenetwork.org + it contains all the prebuild packages + for GN + okay [07:04] + let's step back however [07:05] + I presume the environment is set with all guix package --search-paths + right? + yep + set to the ones in ~/.guix-profile/ + good, and you are in gn-latest-guix repo [07:06] + yep [07:07] + git log shows + +Author: David Thompson +Date: Sun Mar 27 21:20:19 2016 -0400 + + yes + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -A genenetwork2 [07:08] + shows + +genenetwork2 2.0-a8fcff4 out ../guix-bioinformatics/gn/packages/genenetwork.scm:144:2 +genenetwork2-database-small 1.0 out ../guix-bioinformatics/gn/packages/genenetwork.scm:270:4 +genenetwork2-files-small 1.0 out ../guix-bioinformatics/gn/packages/genenetwork.scm:228:4 + + yeah [07:09] + OK, we are in sync. This means we should be able to install the exact + same software + I need to start up my guix daemon - I usually run it in a screen + screen -S guix-daemon + hah, I don't have screen installed yet [07:11] + comes with guix ;) [07:12] + no worries, you can run it any way you want + $HOME/.guix-profile/bin/guix-daemon --build-users-group=guixbuild + then something's weird, because it says I don't have it + oh, you need to install it first [07:13] + guix package -A screen + screen 4.3.1 out gnu/packages/screen.scm:34:2 + but you can skip this install, for now + alright [07:14] + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -i genenetwork2 --dry-run + substitute: updating list of substitutes from + 'https://mirror.hydra.gnu.org'... 79.1% + you see that? + followed by [07:15] +substitute: updating list of substitutes from +'https://hydra.gnu.org'... 100.0% +The following derivations would be built: + /gnu/store/rk7nw0rjqqsha958m649wrykadx6mmhl-profile.drv + +/gnu/store/7b0qjybvfx8syzvfs7p5rdablwhbkbvs-module-import-compiled.drv + /gnu/store/cy9zahbbf23d3cqyy404lk9f50z192kp-module-import.drv + /gnu/store/ibdn603i8grf0jziy5gjsly34wx82lmk-gtk-icon-themes.drv + + which should have the same HASH values /gnu/store/7b0qjybvf... etc. + [07:16] + profile has a different hash + but the next ones? + they're the same + not sure why profile differs. Do you see the contact with + mirror.hydra.org? [07:17] + yeah + OK, that means you set the key correctly for that one :) + alright we are at the same state now. You can see most packages need + to be rebuild because they are no longer cached as binaries on hydra + [07:18] + things move fast... + hehe + let me also do the same on my laptop - which I have staged before + [07:19] + btw, to set the path I often do [07:20] + export + PATH="/home/wrk/.guix-profile/bin:/home/wrk/.guix-profile/sbin":$PATH + to keep things like 'screen' from Debian + Once past building guix itself that is normally OK [07:21] + ah, okay + will do that + the guix build requires certain versions of tools, so you don't want + to mix foreign tools in [07:23] + makes sense [07:24] + On my laptop I am trying the main updating list of substitutes from + 'http://hydra.gnu.org'... 10.5% [07:27] + it is a bit slow, but let's see if there is a difference with the + mirror + you can see there are two servers here. Actually with recent daemons, + if the mirror fails it will try the main server [07:28] + I documented the use of a caching server here [07:29] + https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org + this is exactly what we are doing now + alrighty [07:35] + To see if a remote server has a guix server running it should respond + [07:36] + lynx http://guix.genenetwork.org:8080 --dump + Resource not found: / + + you see that? + yes [07:37] + good. The main hydra server is too slow. So on my laptop I forced + using the mirror with [07:38] + env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix + package -i genenetwork2 --dry-run + --substitute-urls="http://mirror.hydra.gnu.org" + + the list looks the same to me [07:40] + me too + note that some packages will be built and some downloaded, right? + [07:41] + yes + atlas is actually a binary on my system [07:43] + I mean in that list + so, it should not build. Same as yours? + yeah, atlas and r-gtable are the ones to be downloaded + You should not have seen that error ;) + we should try and install it this way, try [07:44] + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -i genenetwork2 --cores=4 --max-jobs=4 --keep-going [07:46] + set CPUs and max-jobs to something sensible + Does your VM have multiple cores? + note you can always press Ctrl-C during install + it doesn't, I'll reboot it and give it another core [07:47] + Hey [07:48] + I'm here + Will be stepping away for some breakfast + Can you do the same as us + Can you see the irc log + Alright + Yes, I can + Please email me a copy in five minutes + user01: so when I use the GN server [07:56] + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -i genenetwork2 --dry-run + --substitute-urls=http://guix.genenetwork.org:8080 + I don't need to build anything [07:57] + (this won't work for you, yet) + to get it to work you need to 'trust' it [07:58] + but, first get the build going + I'll have a coffee while you and get building + yeah it's doing its thing now [08:01] + cool [08:02] + in a separate terminal you can try and install with the gn mirror + [08:05] + I'll send you the public key and you can paste it as said + https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org + [08:06] + alright + should be in the E-mail [08:09] + getting it working it kinda nasty since the server gives no feedback + it works when you see no more in the build list ;) [08:11] + btw, you can install software in parallel. Guix does that. + even the same packages + so keep building ;) + try and do this with Debian... + coffee for me [08:12] + the first build failed [08:15] + OK, Dennis fixed that one yesterday [08:27] + the problem is that sometime source tarballs disappear [08:28] + R is notorious for that + haha, that's inconvenient.. + well, it is good that Guix catches them + but we do not cache sources + binaries are cached - to some degree - so we don't have to rebuild + those [08:29] + time to use the guix cache at guix.genenetwork.org + try and install the key (it is in the E-mail) + and see what this lists [08:31] + env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix + package -i genenetwork2 + --substitute-urls=http://guix.genenetwork.org:8080 --dry-run + should be all binary installs + it's not.. [08:32] + if I remove --substitute-urls, the list changes, does that mean I + have the key set up correctly at least? [08:33] + dunno [08:35] + how many packages does it want to build? + should be zero + four + Ah, that is OK - those are default profile things + genenetwork2 is among the ones to be downloaded so [08:36] + remove --dry-run + yeah, good sign :) + we'll still hit a snag, but run it + should be fast + doing it [08:37] + it worked! [08:38] + I think [08:39] + heh [08:40] + you mean it is finished? + yep + type genenetwork2 + complains about not being able to connect to the database [08:41] + last snag :) + no database + well, we succeeded in installing a same-byte install of a very + complex system :) [08:42] + (always take time to congratulate yourself) + now we need to install mysql + hehe :) + this can be done throug guix or through debian [08:43] + the latter is a bit easier here, so let's do that + fun note: you can mix debian and guix + Follow instructions on [08:44] + + https://github.com/genenetwork/genenetwork2/tree/staging/doc#run-mysql-server + apt-get install mysql-common [08:45] + may do it + You can also install with guix, but I need to document that + btw your internet must be fast :) [08:46] + hehe it is ;) + when the database is installed [08:48] + be sure to set the password as instructed [08:50] + when mysql is set the genenetwork2 command should fire up the web + server on localhost:5003 [08:58] + btw my internet is way slower :) [09:00] + I'm back [09:04] + fixed router firmware upgrade problem + unbricking + tssk [09:07] + I'll never leave routers to update themselves again [09:08] + self-brick highway + Resuming [09:09] + auto-updates are evil + always switch them off + user02: can you install genenetwork like user has done? [09:10] + pretty well documented here now :) + Yes I can [09:11] + Already installed key + user02: you are getting binary packages only now? [09:13] + That's the sanest way to go now + seriously + everything should be pre-built from guix.genenetwork.org + you are downloading? + yes [09:15] + cool. Maybe an idea to set up a server + for your own use + Stuck at downloading preprocesscore + should not [09:24] + what does env GUIX_PACKAGE_PATH=../guix-bioinformatics/ + ./pre-inst-env guix package -i genenetwork2 + --substitute-urls="http://guix.genenetwork.org:8080" --dry-run + [09:25] + say for r-prepocesscore + download or build? + mine says download [09:26] + it only lists the derivatives to be built + nothing else happens [09:27] + OK, so there is a problem + your key may not be working + everything should be listed as 'to be download' [09:28] + Hmm + Ah + I know where I messed up + where? + I did add the key + However + (I am documenting) + I did not tell guix to trust it + yes + and there is another potential problem + Remember the documentation on installing guix? + You have to tell guix to trust the default key [09:29] + Right? + So in this case + read the IRC log + That step is mandatory + user01: how are you doing? + user02: + https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org#using-gnu-guix-archive + [09:30] + a little bit left on the db download + user02: you should see no more building + user02: another issue may be that you updated r-preprocesscore + package in guix-buinformatics [09:32] + all downstream packages will want to rebuild + no, not really + It's not even installed + checkout a branch of the the old version - make sure we are in synch + should be at + /gnu/store/y1f3r2xs3fhyadd46nd2aqbr2p9qv2ra-r-biocpreprocesscore-1.32.0 + [09:33] + + pjotrp: Possibly we should use the archive utility of Guix to do + deployment to avoid such out-of-sync differences :) [09:34] + maybe. I did not get archive to update profiles properly [09:37] + Also it is good that they get to understand guix + this way + carved in stone, eh [09:38] + Yeah, all good [09:39] + My mistake was skipping the guix archive part + Can we begin with the install? + It's telling me of derivatives that will be downloaded [09:40] + So we're good + Here goes + yeeha [09:42] + pjotrp, where is this guix.genenetwork.org located at? + Tennessee + It's...it's....sloooooooowwwwwwwwwwwwww + not from Europe + is it downloading at all? + It should be extended + Yes...like at 100KB/s [09:43] + tear-jerker + Verizon problems + who's the host? + I am getting 500Kb/s + UT + Guix's servers can run off more than one server, right? + I'd like to host that particular server here + For speed + yes + Sooner or later + It will be a necessity [09:45] + exactly what I am doing - this is our server + guix.genenetwork.org:8080 + All done installing [09:46] + what? + Now the databases + what do you mean by slow exactly? + Yes, it's installed + can you run genenetwork2 + setting variables + If I try running it now, it will fail as I don't have the DBs [09:47] + cool - you had a lot of prebuilt packages already + OK, follow the instructions I wrote above + now everything seems to be working for me :) + OK + user01: excellent! + you see a webserver? + yep, can connect to localhost:5003 [09:48] + So now you are running a guix copy of GN2 + you can see where it lives with `which genenetwork2` or ls -l + ~/.guix-profile/bin/genenetwork2 [09:49] + + /gnu/store/1kma5xszvzsvmbb4k699h7gvdncw901i-genenetwork2-2.0-a8fcff4/bin/genenetwork2 + it is a script + written by guix, open it [09:50] + inside it points to paths and our script at + + /gnu/store/1kma5xszvzsvmbb4k699h7gvdncw901i-genenetwork2-2.0-a8fcff4/bin/.genenetwork2-real + if you open that you can see how the webserver is started [09:51] + next step is to run a recent version of GN2 + okay [09:52] + See + https://github.com/genenetwork/genenetwork2/tree/staging/doc#run-your-own-copy-of-gn2 + but do not checkout that genetwork2_diet + we reverted to the main tree + clone git@github.com:genenetwork/genenetwork2.git [09:53] + instead and checkout the staging branch + that is effectively my branch [09:54] + when that is done you should be able to fire up the webserver from + there [09:55] + using ./bin/genenetwork2 + now installing DBs + Downloading + annoyingly the source tree is ~700Mb [09:56] + Can it also be done by installing the guix package + genenetwork2-database-small? + I changed it in the diet version to 8Mb, but I had to revert + I need to make my VM bigger... + user02: not ready [09:57] + ok + user01: sorry + user01: you could mount a local dir inside the VM for development + that would allow you to use MAC tools for editing + just an idea + yeah, I figure I'll do something like that + do you use emacs? [09:58] + yep + that can also run on remote files over ssh + that's an alternative + kudos for using emacs :), wdyt user03 + 79 minutes to go downloading the db + user02: sorry about that [09:59] + it is 2GB + user, you can also mount the directory via sshfs + Mac OSX runs OpenSSH + user02: sopa + You can therefore mount a directory outside the VM to the VM via + sshfs [10:00] + yes, 3 options now + That way, you can set up a VM only for it's logic + Apps + the OS it runs [10:01] + For data, let it reside on physical host accessible via sshfs + Use this Arch wiki reference: + https://wiki.archlinux.org/index.php/SSHFS + I edited that last somewhere in 2015, may have been updated since + then + alright, cool! [10:04] + user01: you are almost done [10:06] + I wrote an elixir package for guix :) + env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix + package -A elixir + --substitute-urls="http://guix.genenetwork.org:8080" [10:08] + elixir 1.2.3 out + ../guix-bioinformatics/gn/packages/elixir.scm:31:2 + + I am building it on guix.genenetwork.org right now [10:09] + nice [10:10] +#+end_src bash + +** Run GN2 Make a note of the paths with @@ -78,8 +484,6 @@ Make a note of the paths with ./pre-inst-env guix package --search-paths #+end_src bash -** Run GN2 - After setting the paths for the server #+begin_src bash @@ -196,7 +600,6 @@ And copy any relevant files to /etc/nginx. A configuration file for GeneNetwork (reverse proxy) port forwarding can be found in the source repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc (still as root) - : cp ./etc/nginx-genenetwork.conf /etc/nginx/ Make dirs -- cgit v1.2.3 From 470b110cd34c1f7abeaf3e3f0b0b3d3549775ae8 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 06:25:02 +0000 Subject: Added IRC log --- doc/README.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 9fa44442..3c1fb910 100644 --- a/doc/README.org +++ b/doc/README.org @@ -73,7 +73,7 @@ significantly because all packages are pre-built). Here an IRC session where we installed GN2 from scratch using GNU Guix and a download of the test database: -#+begin_src bash +#+begin_src time to get binary install sorted :) [07:03] Guix is designed for distributed installation servers we have one on guix.genenetwork.org @@ -474,7 +474,7 @@ The following derivations would be built: I am building it on guix.genenetwork.org right now [10:09] nice [10:10] -#+end_src bash +#+end_src ** Run GN2 -- cgit v1.2.3 From 8aa1e752ac2d576a6e46610b4fe13f335682f6ba Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 13 May 2016 08:18:20 +0000 Subject: README: nginx configuration as reverse proxy --- doc/README.org | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 3c1fb910..345341e1 100644 --- a/doc/README.org +++ b/doc/README.org @@ -578,11 +578,38 @@ that comes in the Guix installation. It sits in something like ** Set up nginx port forwarding -Install the webserver (as root) +nginx can be used as a reverse proxy for GN2. For example, we want to +expose GN2 on port 80 while it is running on port 5003. Essentially +the configuration looks like + +#+begin_src js + server { + listen 80; + server_name test-gn2.genenetwork.org; + access_log logs/test-gn2.access.log; + + proxy_connect_timeout 3000; + proxy_send_timeout 3000; + proxy_read_timeout 3000; + send_timeout 3000; + + location / { + proxy_set_header Host $http_host; + proxy_set_header Connection keep-alive; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + proxy_pass http://127.0.0.1:5003; + } +} +#+end_src js + +Install the nginx webserver (as root) : guix package -i nginx -The configuration examples can be found in the Guix store through +The nginx example configuration examples can be found in the Guix +store through : ls -l /root/.guix-profile/sbin/nginx : lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx -- cgit v1.2.3 From a0fe8353ed71e6b2d59305614b18f0a79173677e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 17 May 2016 08:27:32 +0000 Subject: Update URL --- doc/README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 345341e1..278dd31c 100644 --- a/doc/README.org +++ b/doc/README.org @@ -548,7 +548,7 @@ configuration. At some point you may want to fix the source code. Assuming you have Guix and Genenetwork2 installed (as described above) clone the GN2 -repository from https://github.com/genenetwork/genenetwork2_diet +repository from https://github.com/genenetwork/genenetwork2. Copy-paste the paths into your terminal (mainly so PYTHON_PATH and R_LIBS_SITE are set) from the information given by guix: -- cgit v1.2.3 From b38772cad362be39d7af2125c66d20d9c20291d0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 24 May 2016 14:02:03 +0000 Subject: Docs: phenotypes --- doc/database.org | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/database.org b/doc/database.org index e06ac1ff..ef4c5ad3 100644 --- a/doc/database.org +++ b/doc/database.org @@ -1,7 +1,11 @@ - github Document reduction issue +* The small test database (2GB) -* GeneNetwork Database +The default install comes with a smaller database which includes a +number of the BSD's and the Human liver dataset (GSE9588). + +* GeneNetwork database ** Estimated table sizes @@ -487,7 +491,6 @@ select count(*) from ProbeSet limit 5; - ** ProbeSetData Probedata - main molecular data. Probesets, metabolome, @@ -708,3 +711,62 @@ User selection - retained ** Vlookup +* Fetching Data + +** Fetch phenotypes + +To get at phenotype data ProbeSetData is the main table (almost all +important molecular assay data is in this table including probe set +data, RNA-seq data, proteomic data, and metabolomic data. 2.5 billion +rows March 2016) + +select count(*) from ProbeSetData limit 5; ++---------------+ +| count(*) | ++---------------+ +| 2,510,566,472 | ++---------------+ + +select * from ProbeSetData limit 5; ++----+----------+-------+ +| Id | StrainId | value | ++----+----------+-------+ +| 1 | 1 | 5.742 | +| 1 | 2 | 5.006 | +| 1 | 3 | 6.079 | +| 1 | 4 | 6.414 | +| 1 | 5 | 4.885 | ++----+----------+-------+ + +This table is used in + +: wqflask/base/do_search.py +: wqflask/base/data_set.py +: wqflask/utility/AJAX_table.py +: wqflask/wqflask/correlation/show_corr_results.py + +In there we find 'ProbeSetData.Id = ProbeSetXRef.dataId'. + +select * from ProbeSetXRef limit 5; ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ +| ProbeSetFreezeId | ProbeSetId | DataId | Locus_old | LRS_old | pValue_old | mean | se | Locus | LRS | pValue | additive | h2 | ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ +| 1 | 1 | 1 | 10.095.400 | 13.3971627898894 | 0.163 | 5.48794285714286 | 0.08525787814808819 | rs13480619 | 12.590069931048001 | 0.269 | -0.28515625 | NULL | +| 1 | 2 | 2 | D15Mit189 | 10.042057464356201 | 0.431 | 9.90165714285714 | 0.0374686634976217 | CEL-17_50896182 | 10.5970737900941 | 0.304 | -0.11678333333333299 | NULL | +| 1 | 3 | 3 | D5Mit139 | 5.43678531742749 | 0.993 | 7.83948571428571 | 0.0457583416912569 | rs13478499 | 6.0970532702754 | 0.988 | 0.112957489878542 | NULL | +| 1 | 4 | 4 | D1Mit511 | 9.87815279480766 | 0.483 | 8.315628571428569 | 0.0470396593931327 | rs6154379 | 11.774867551173099 | 0.286 | -0.157113725490196 | NULL | +| 1 | 5 | 5 | D16H21S16 | 10.191723834264499 | 0.528 | 9.19345714285714 | 0.0354801718293322 | rs4199265 | 10.923263374016202 | 0.468 | 0.11476470588235299 | NULL | ++------------------+------------+--------+------------+--------------------+------------+-------------------+---------------------+-----------------+--------------------+--------+----------------------+------+ + +i.e., for Strain Id 1 (DataId) 1, the locus '10.095.400' has a +phenotype value of 5.742. + +GeneNetwork1 already has a limited REST interface, if you do + +: curl "http://robot.genenetwork.org/webqtl/mairobeset=1443823_s_at&db=HC_M2_0606_P" + +you get + +: ProbeSetID B6D2F1 C57BL/6J DBA/2J BXD1 BXD2 BXD5 BXD6 BXD8 BXD9 BXD11 BXD12 BXD13 BXD15 BXD16 BXD19 BXD20 BXD21 BXD22 BXD23 BXD24 BXD27 BXD28 BXD29 BXD31 BXD32 BXD33 BXD34 BXD38 BXD39 BXD40 BXD42 BXD67 BXD68 BXD43 BXD44 BXD45 BXD48 BXD50 BXD51 BXD55 BXD60 BXD61 BXD62 BXD63 BXD64 BXD65 BXD66 BXD69 BXD70 BXD73 BXD74 BXD75 BXD76 BXD77 BXD79 BXD73a BXD83 BXD84 BXD85 BXD86 BXD87 BXD89 BXD90 BXD65b BXD93 BXD94 A/J AKR/J C3H/HeJ C57BL/6ByJ CXB1 CXB2 CXB3 CXB4 CXB5 CXB6 CXB7 CXB8 CXB9 CXB10 CXB11 CXB12 CXB13 BXD48a 129S1/SvImJ BALB/cJ BALB/cByJ LG/J NOD/ShiLtJ PWD/PhJ BXD65a BXD98 BXD99 CAST/EiJ KK/HlJ WSB/EiJ NZO/HlLtJ PWK/PhJ D2B6F1 +: 1443823_s_at 15.251 15.626 14.716 15.198 14.918 15.057 15.232 14.968 14.87 15.084 15.192 14.924 15.343 15.226 15.364 15.36 14.792 14.908 15.344 14.948 15.08 15.021 15.176 15.14 14.796 15.443 14.636 14.921 15.22 15.62 14.816 15.39 15.428 14.982 15.05 15.13 14.722 14.636 15.242 15.527 14.825 14.416 15.125 15.362 15.226 15.176 15.328 14.895 15.141 15.634 14.922 14.764 15.122 15.448 15.398 15.089 14.765 15.234 15.302 14.774 14.979 15.212 15.29 15.012 15.041 15.448 14.34 14.338 14.809 15.046 14.816 15.232 14.933 15.255 15.21 14.766 14.8 15.506 15.749 15.274 15.599 15.673 14.651 14.692 14.552 14.563 14.164 14.546 15.044 14.695 15.162 14.772 14.645 15.493 14.75 14.786 15.003 15.148 15.221 + -- cgit v1.2.3 From 36dc2d077debe2c82f13b67887689bd11025cf1b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 25 May 2016 07:42:55 +0000 Subject: Docs: database reverse engineering --- doc/database.org | 87 +++++- doc/database.svg | 867 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 952 insertions(+), 2 deletions(-) create mode 100644 doc/database.svg (limited to 'doc') diff --git a/doc/database.org b/doc/database.org index ef4c5ad3..6e0f42c9 100644 --- a/doc/database.org +++ b/doc/database.org @@ -763,10 +763,93 @@ phenotype value of 5.742. GeneNetwork1 already has a limited REST interface, if you do -: curl "http://robot.genenetwork.org/webqtl/mairobeset=1443823_s_at&db=HC_M2_0606_P" +: curl "http://robot.genenetwork.org/webqtl/main.py?cmd=get&probeset=1443823_s_at&db=HC_M2_0606_P" -you get +we get : ProbeSetID B6D2F1 C57BL/6J DBA/2J BXD1 BXD2 BXD5 BXD6 BXD8 BXD9 BXD11 BXD12 BXD13 BXD15 BXD16 BXD19 BXD20 BXD21 BXD22 BXD23 BXD24 BXD27 BXD28 BXD29 BXD31 BXD32 BXD33 BXD34 BXD38 BXD39 BXD40 BXD42 BXD67 BXD68 BXD43 BXD44 BXD45 BXD48 BXD50 BXD51 BXD55 BXD60 BXD61 BXD62 BXD63 BXD64 BXD65 BXD66 BXD69 BXD70 BXD73 BXD74 BXD75 BXD76 BXD77 BXD79 BXD73a BXD83 BXD84 BXD85 BXD86 BXD87 BXD89 BXD90 BXD65b BXD93 BXD94 A/J AKR/J C3H/HeJ C57BL/6ByJ CXB1 CXB2 CXB3 CXB4 CXB5 CXB6 CXB7 CXB8 CXB9 CXB10 CXB11 CXB12 CXB13 BXD48a 129S1/SvImJ BALB/cJ BALB/cByJ LG/J NOD/ShiLtJ PWD/PhJ BXD65a BXD98 BXD99 CAST/EiJ KK/HlJ WSB/EiJ NZO/HlLtJ PWK/PhJ D2B6F1 : 1443823_s_at 15.251 15.626 14.716 15.198 14.918 15.057 15.232 14.968 14.87 15.084 15.192 14.924 15.343 15.226 15.364 15.36 14.792 14.908 15.344 14.948 15.08 15.021 15.176 15.14 14.796 15.443 14.636 14.921 15.22 15.62 14.816 15.39 15.428 14.982 15.05 15.13 14.722 14.636 15.242 15.527 14.825 14.416 15.125 15.362 15.226 15.176 15.328 14.895 15.141 15.634 14.922 14.764 15.122 15.448 15.398 15.089 14.765 15.234 15.302 14.774 14.979 15.212 15.29 15.012 15.041 15.448 14.34 14.338 14.809 15.046 14.816 15.232 14.933 15.255 15.21 14.766 14.8 15.506 15.749 15.274 15.599 15.673 14.651 14.692 14.552 14.563 14.164 14.546 15.044 14.695 15.162 14.772 14.645 15.493 14.75 14.786 15.003 15.148 15.221 +getTraitData is defined in the file [[https://github.com/genenetwork/genenetwork/blob/master/web/webqtl/textUI/cmdClass.py#L134][web/webqtl/textUI/cmdClass.py]]. +probe is None, so the code at line 199 is run + +query = "SELECT Strain.Name, %sData.value from %sData, Strain, %s, +%sXRef WHERE %s.Name = '%s' and %sXRef.%sId = %s.Id and +%sXRef.%sFreezeId = %d and %sXRef.DataId = %sData.Id and +%sData.StrainId = Strain.Id order by Strain.Id" % (prefix, prefix, +prefix, prefix, prefix, probeset,prefix, prefix, prefix, prefix, +prefix, dbId, prefix, prefix, prefix) + +where prefix is ProbeSet (one presumes). So, let's see if we can do this by hand + +SELECT Strain.Name, ProbeSetData.value from ProbeSetData, Strain, ProbeSet, +ProbeSetXRef WHERE ProbeSet.Name = '1443823_s_at' and ProbeSetXRef.ProbeSetId = ProbeSet.Id and +ProbeSetXRef.ProbeSetFreezeId = $dbid and ProbeSetXRef.DataId = ProbeSetData.Id and +ProbeSetData.StrainId = Strain.Id ORDER BY Strain.Id + +The $dbid is listed in the ProbeSetFreeze table, + +SELECT ProbeFreezeId,Name FROM ProbeSetFreeze WHERE Name='HC_M2_0606_P' limit 5; + ++---------------+--------------+ +| ProbeFreezeId | Name | ++---------------+--------------+ +| 30 | HC_M2_0606_P | ++---------------+--------------+ + +select id,name from ProbeSet WHERE Name = '1443823_s_at' limit 5; ++--------+--------------+ +| id | name | ++--------+--------------+ +| 106556 | 1443823_s_at | ++--------+--------------+ + +So + +query = "SELECT Strain.Name, %sData.value from %sData, Strain, %s, +%sXRef WHERE %s.Name = '%s' and %sXRef.%sId = %s.Id and +%sXRef.%sFreezeId = %d and %sXRef.DataId = %sData.Id and +%sData.StrainId = Strain.Id order by Strain.Id" % (prefix, prefix, +prefix, prefix, prefix, probeset,prefix, prefix, prefix, prefix, +prefix, dbId, prefix, prefix, prefix) + +CORRECT NAME: + +SELECT Strain.Name, ProbeSetData.value from ProbeSetData, Strain, +ProbeSet, ProbeSetXRef WHERE ProbeSet.Name = 'at_probe' and +ProbeSetXRef.ProbeSetId = ProbeSet.Id and +ProbeSetXRef.ProbeSetFreezeId = dbid and ProbeSetXRef.DataId = +ProbeSetData.Id and ProbeSetData.StrainId = Strain.Id order by +Strain.Id + + +select * from ProbeSetXRef WHERE probesetfreezeid=30 limit 5; ++------------------+------------+--------+------------+--------------------+------------+--------------------+--------------------+-----------------+--------------------+--------+-------------------+------+ +| ProbeSetFreezeId | ProbeSetId | DataId | Locus_old | LRS_old | pValue_old | mean | se | Locus | LRS | pValue | additive | h2 | ++------------------+------------+--------+------------+--------------------+------------+--------------------+--------------------+-----------------+--------------------+--------+-------------------+------+ +| 30 | 1 | 445962 | 01.059.350 | 7.1753152078069595 | 0.961 | 30.0646 | 1.79763935596594 | rs13475891 | 7.1753152078069204 | 0.973 | 4.71778125 | NULL | +| 30 | 2 | 445963 | D4Mit156 | 7.58944292943285 | 0.724 | 232.38328571428602 | 9.00278909374791 | CEL-4_118751423 | 7.57513435426218 | 0.793 | 25.2660951417004 | NULL | +| 30 | 3 | 445964 | D1Mit134 | 9.766065497826819 | 0.737 | 47.206 | 1.58413526287766 | mCV23431007 | 9.76606549782677 | 0.797 | -4.82405952380952 | NULL | +| 30 | 4 | 445965 | D1Mit155 | 18.0045829157241 | 0.033 | 132.29248571428602 | 4.37799472291842 | rs3689947 | 17.9365068406286 | 0.049 | -16.945619047619 | NULL | +| 30 | 5 | 445966 | D5Mit197 | 9.51068902627823 | 0.476 | 271.309971428571 | 7.4294268316065395 | rs6239372 | 10.4214974316601 | 0.41 | -25.6148045454546 | NULL | ++------------------+------------+--------+------------+--------------------+------------+--------------------+--------------------+-----------------+--------------------+--------+-------------------+------+ + +So, apparently ProbeSetFreezeID points to the database identifier in +ProbeSetFreeze which has the name of the 'DB'. OK, that kinda makes +sense now. Meanwhile Probeset.name points to the phenotype name. + +ProbeSetXRef binds these tables together. Finally there is the data in + +select * from ProbeSetData limit 5; ++----+----------+-------+ +| Id | StrainId | value | ++----+----------+-------+ +| 1 | 1 | 5.742 | +| 1 | 2 | 5.006 | +| 1 | 3 | 6.079 | +| 1 | 4 | 6.414 | +| 1 | 5 | 4.885 | ++----+----------+-------+ +5 rows in set (0.00 sec) + +linked by ProbeSetXRef.dataid. diff --git a/doc/database.svg b/doc/database.svg new file mode 100644 index 00000000..ae9d55a5 --- /dev/null +++ b/doc/database.svg @@ -0,0 +1,867 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + ProbesetData + + + id + strainid + + value + ProbeSetXRef + + ProbeSetFreezeId + ProbeSetID + DataId + pValue + + ProbesetFreeze + ProbeFreezeId + Name + + + + + + + + + + DB + + ProbeFreeze + id + ProbeFreezeId + TissueId + id + + = + + + + + Strain + id + name + + + ProbeSet + id + name + many infos + + + -- cgit v1.2.3 From 5925dc7199a9fc6766c0468f5625068d1bd9603f Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 25 May 2016 08:21:30 +0000 Subject: Docs: database reverse engineering --- doc/database.org | 16 +++++++++++++++- doc/database.svg | 21 ++++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) (limited to 'doc') diff --git a/doc/database.org b/doc/database.org index 6e0f42c9..2221c4fb 100644 --- a/doc/database.org +++ b/doc/database.org @@ -681,7 +681,21 @@ No longer used ** Species & Strain (should be sample) -Menu +select * from Species; ++----+-----------+----------------------+----------------+----------------------+-------------------------+------------+---------+ +| Id | SpeciesId | SpeciesName | Name | MenuName | FullName | TaxonomyId | OrderId | ++----+-----------+----------------------+----------------+----------------------+-------------------------+------------+---------+ +| 1 | 1 | Mouse | mouse | Mouse | Mus musculus | 10090 | 30 | +| 2 | 2 | Rat | rat | Rat | Rattus norvegicus | 10116 | 40 | +| 3 | 3 | Arabidopsis thaliana | arabidopsis | Arabidopsis thaliana | Arabidopsis thaliana | 3702 | 60 | +| 4 | 4 | Human | human | Human | Homo sapiens | 9606 | 10 | +| 5 | 5 | Barley | barley | Barley | Hordeum vulgare | 4513 | 70 | +| 6 | 6 | Drosophila | drosophila | Drosophila | Drosophila melanogaster | 7227 | 50 | +| 7 | 7 | Macaque monkey | macaque monkey | Macaque monkey | Macaca mulatta | 9544 | 20 | +| 8 | 8 | Soybean | soybean | Soybean | Soybean | 3847 | 80 | +| 9 | 9 | Tomato | tomato | Tomato | Tomato | 4081 | 90 | ++----+-----------+----------------------+----------------+----------------------+-------------------------+------------+---------+ + ** InbredSet diff --git a/doc/database.svg b/doc/database.svg index ae9d55a5..9ddc2f47 100644 --- a/doc/database.svg +++ b/doc/database.svg @@ -15,7 +15,7 @@ id="svg2" version="1.1" inkscape:version="0.91 r13725" - sodipodi:docname="drawing.svg"> + sodipodi:docname="database.svg"> image/svg+xml - + @@ -863,5 +863,20 @@ inkscape:connector-curvature="0" inkscape:transform-center-x="-32.453094" inkscape:transform-center-y="114.65633" /> + speciesid -- cgit v1.2.3 From 4ccb2411b78bcc5112316cef7192033d698b8b03 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 29 May 2016 16:51:45 +0000 Subject: Added JOSS paper --- bin/test-website | 9 +++++ doc/joss/2016/paper.bib | 36 ++++++++++++++++++++ doc/joss/2016/paper.json | 23 +++++++++++++ doc/joss/2016/paper.md | 84 +++++++++++++++++++++++++++++++++++++++++++++++ doc/joss/2016/qtl.png | Bin 0 -> 146924 bytes doc/joss/2016/qtl2.png | Bin 0 -> 375505 bytes 6 files changed, 152 insertions(+) create mode 100644 doc/joss/2016/paper.bib create mode 100644 doc/joss/2016/paper.json create mode 100644 doc/joss/2016/paper.md create mode 100644 doc/joss/2016/qtl.png create mode 100644 doc/joss/2016/qtl2.png (limited to 'doc') diff --git a/bin/test-website b/bin/test-website index 2585b465..aeab459b 100755 --- a/bin/test-website +++ b/bin/test-website @@ -25,6 +25,10 @@ opts = OptionParser.new do |o| o.on('-l','--link-checker', 'Check for dead links') do options[:link_checker] = true end + + o.on('-n','--navigation-test', 'Check for navigation test results') do + options[:navigation_test] = true + end o.separator "" o.on_tail('-h', '--help', 'display this help and exit') do @@ -62,3 +66,8 @@ if options[:link_checker] else require 'main_web_functionality' end + +if options[:navigation_test] + require 'NavigationTest' +end + diff --git a/doc/joss/2016/paper.bib b/doc/joss/2016/paper.bib new file mode 100644 index 00000000..73a88227 --- /dev/null +++ b/doc/joss/2016/paper.bib @@ -0,0 +1,36 @@ +@article{WGCNA:2008, + author = {Langfelder, P. and Horvath, S.}, + title = {{WGCNA: an R package for weighted correlation network analysis}}, + journal = {BMC Bioinformatics}, + year = {2008}, + volume = {9}, + pages = {559}, + doi = {10.1186/1471-2105-9-559}, + url = {http://www.ncbi.nlm.nih.gov/pubmed/19114008}, + abstract = {BACKGROUND: Correlation networks are increasingly being used in bioinformatics applications. For example, weighted gene co-expression network analysis is a systems biology method for describing the correlation patterns among genes across microarray samples. Weighted correlation network analysis (WGCNA) can be used for finding clusters (modules) of highly correlated genes, for summarizing such clusters using the module eigengene or an intramodular hub gene, for relating modules to one another and to external sample traits (using eigengene network methodology), and for calculating module membership measures. Correlation networks facilitate network based gene screening methods that can be used to identify candidate biomarkers or therapeutic targets. These methods have been successfully applied in various biological contexts, e.g. cancer, mouse genetics, yeast genetics, and analysis of brain imaging data. While parts of the correlation network methodology have been described in separate publications, there is a need to provide a user-friendly, comprehensive, and consistent software implementation and an accompanying tutorial. RESULTS: The WGCNA R software package is a comprehensive collection of R functions for performing various aspects of weighted correlation network analysis. The package includes functions for network construction, module detection, gene selection, calculations of topological properties, data simulation, visualization, and interfacing with external software. Along with the R package we also present R software tutorials. While the methods development was motivated by gene expression data, the underlying data mining approach can be applied to a variety of different settings. CONCLUSION: The WGCNA package provides R functions for weighted correlation network analysis, e.g. co-expression network analysis of gene expression data. The R package along with its source code and additional material are freely available at http://www.genetics.ucla.edu/labs/horvath/CoexpressionNetwork/Rpackages/WGCNA.} +} + +@article{Wang:2016, + author = {Wang, X. and Pandey, A. K. and Mulligan, M. K. and Williams, E. G. and Mozhui, K. and Li, Z. and Jovaisaite, V. and Quarles, L. D. and Xiao, Z. and Huang, J. and Capra, J. A. and Chen, Z. and Taylor, W. L. and Bastarache, L. and Niu, X. and Pollard, K. S. and Ciobanu, D. C. and Reznik, A. O. and Tishkov, A. V. and Zhulin, I. B. and Peng, J. and Nelson, S. F. and Denny, J. C. and Auwerx, J. and Lu, L. and Williams, R. W.}, + title = {{Joint mouse-human phenome-wide association to test gene function and disease risk}}, + journal = {Nat Commun}, + year = {2016}, + volume = {7}, + pages = {10464}, + doi = {10.1038/ncomms10464}, + url = {http://www.ncbi.nlm.nih.gov/pubmed/26833085}, + abstract = {Phenome-wide association is a novel reverse genetic strategy to analyze genome-to-phenome relations in human clinical cohorts. Here we test this approach using a large murine population segregating for approximately 5 million sequence variants, and we compare our results to those extracted from a matched analysis of gene variants in a large human cohort. For the mouse cohort, we amassed a deep and broad open-access phenome consisting of approximately 4,500 metabolic, physiological, pharmacological and behavioural traits, and more than 90 independent expression quantitative trait locus (QTL), transcriptome, proteome, metagenome and metabolome data sets--by far the largest coherent phenome for any experimental cohort (www.genenetwork.org). We tested downstream effects of subsets of variants and discovered several novel associations, including a missense mutation in fumarate hydratase that controls variation in the mitochondrial unfolded protein response in both mouse and Caenorhabditis elegans, and missense mutations in Col6a5 that underlies variation in bone mineral density in both mouse and human.} +} + +@article{Lippert:2011, + author = {Lippert, C. and Listgarten, J. and Liu, Y. and Kadie, C. M. and Davidson, R. I. and Heckerman, D.}, + title = {{FaST linear mixed models for genome-wide association studies}}, + journal = {Nat Methods}, + year = {2011}, + volume = {8}, + number = {10}, + pages = {833-835}, + doi = {10.1038/nmeth.1681}, + url = {http://www.ncbi.nlm.nih.gov/pubmed/21892150}, + abstract = {We describe factored spectrally transformed linear mixed models (FaST-LMM), an algorithm for genome-wide association studies (GWAS) that scales linearly with cohort size in both run time and memory use. On Wellcome Trust data for 15,000 individuals, FaST-LMM ran an order of magnitude faster than current efficient algorithms. Our algorithm can analyze data for 120,000 individuals in just a few hours, whereas current algorithms fail on data for even 20,000 individuals (http://mscompbio.codeplex.com/).} +} diff --git a/doc/joss/2016/paper.json b/doc/joss/2016/paper.json new file mode 100644 index 00000000..c3c02156 --- /dev/null +++ b/doc/joss/2016/paper.json @@ -0,0 +1,23 @@ +{ + "@context": "https://raw.githubusercontent.com/mbjones/codemeta/master/codemeta.jsonld", + "@type": "Code", + "author": [ + { + "@id": "0000-0002-9623-3401", + "@type": "Person", + "email": "jakevdp@uw.edu", + "name": "Jake VanderPlas", + "affiliation": "University of Washington eScience Institute" + } + ], + "identifier": "https://zenodo.org/record/50995#.Vyp9DBUrJBw", + "codeRepository": "http://github.com/jakevdp/mst_clustering", + "datePublished": "2016-05-04", + "dateModified": "2016-05-04", + "dateCreated": "2016-05-04", + "description": "Clustering via Euclidean Minimum Spanning Trees", + "keywords": "machine learning", + "license": "BSD", + "title": "mst_clustering", + "version": "v1.0" +} diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md new file mode 100644 index 00000000..81ec2b72 --- /dev/null +++ b/doc/joss/2016/paper.md @@ -0,0 +1,84 @@ +--- +title: 'GeneNetwork: framework for web-based genetics' +tags: + - bioinformatics + - genetics + - genomics +authors: + - name: Zachary Sloan + orcid: 0000-0002-8099-1363 + affiliation: University of Tennessee Health Science Center, USA + - name: Danny Arends + orcid: 0000-0001-8738-0162 + affiliation: Humboldt Universityl, Berlin, Germany + - name: Karl W. Broman + orcid: 0000-0002-4914-6671 + affiliation: University of Wisconsin, USA + - name: Arthur Centeno + orcid: 0000-0003-3142-2081 + affiliation: University of Tennessee Health Science Center, USA + - name: Nick Furlotte + orcid: ? + - name: Harm Nijveen + orcid: 0000-0002-9167-4945 + affiliation: Wageningen University, The Netherlands + - name: Lei Yan + orcid: 0000-0001-5259-3379 + affiliation: University of Tennessee Health Science Center, USA + - name: Xiang Zhou + orcid: 0000-0002-4331-7599 + affiliation: University of Michigan + - name: Robert W. WIlliams + orcid: 0000-0001-8924-4447 + affiliation: University of Tennessee Health Science Center, USA + - name: Pjotr Prins + orcid: orcid.org/0000-0002-8021-9162 + affiliation: University Medical Center Utrecht, The Netherlands + affiliation: University of Tennessee Health Science Center, USA +date: 29 May 2016 +bibliography: paper.bib +--- + +# Summary + +GeneNetwork (GN) is a free and open source (FOSS) framework for web +based genetics that can be deployed anywhere. GN allows biologists to +upload experimental data and map phenotypes interactively against +genotypes using tools, such as R/QTL [@mqm paper] mapping, interval +mapping for model organisms and pylmm; an implementation of FaST-LMM +[@Lippert:2011] which is suitable for human populations and outbred +crosses, such as the mouse diversity outcross. Interactive D3 graphics +are included from R/qtlcharts and presentation-ready figures can be +generated. Recently we have added functionality for phenotype +correlation [@Wang:2016] and network analysis [@WGCNA:2008]. + +-![Mouse LMM mapping example](qtl2.png) + +GN is written in python and javascript and contains a rich set of +tools and libraries that can be written in any computer language. A +full list of included software can be found in +[guix-bioinformatics](https://github.com/genenetwork/guix-bioinformatics/blob/master/gn/packages/genenetwork.scm). To +make it easy to install GN locally in a byte reproducible way, +including all dependencies and a 2GB MySQL test database (the full +database is 160GB and growing), GN is packaged with +[GNU Guix](https://www.gnu.org/software/guix/), as described +[here](https://github.com/genenetwork/genenetwork2/blob/staging/doc/README.org). +GNU Guix deployment makes it feasible to deploy and rebrand GN +anywhere. + +# Future work + +More mapping tools will be added, including support for Genome-wide +Efficient Mixed Model Association (GEMMA). The +[Biodiallance genome browser](http://www.biodalliance.org/) is being +added as a Google Summer of Code project with special tracks related +to QTL mapping and network analysis. Faster LMM solutions are being +worked on, including GPU support. + +A REST interface is being added so that data can be uploaded to a +server, analysis run remotely on high performance hardware, and +results downloaded and used for further analysis. This feature will +allow biologist-programmers to use R and python on their computer and +execute computations on GN enabled servers. + +# References diff --git a/doc/joss/2016/qtl.png b/doc/joss/2016/qtl.png new file mode 100644 index 00000000..995a2739 Binary files /dev/null and b/doc/joss/2016/qtl.png differ diff --git a/doc/joss/2016/qtl2.png b/doc/joss/2016/qtl2.png new file mode 100644 index 00000000..e0b684ef Binary files /dev/null and b/doc/joss/2016/qtl2.png differ -- cgit v1.2.3 From 42f7955c8619ba43766a3b13626d554c2f3d8399 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 29 May 2016 16:57:28 +0000 Subject: JOSS: add MQM citation and fix typo --- doc/joss/2016/paper.bib | 14 ++++++++++++++ doc/joss/2016/paper.md | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.bib b/doc/joss/2016/paper.bib index 73a88227..34c0fd05 100644 --- a/doc/joss/2016/paper.bib +++ b/doc/joss/2016/paper.bib @@ -34,3 +34,17 @@ url = {http://www.ncbi.nlm.nih.gov/pubmed/21892150}, abstract = {We describe factored spectrally transformed linear mixed models (FaST-LMM), an algorithm for genome-wide association studies (GWAS) that scales linearly with cohort size in both run time and memory use. On Wellcome Trust data for 15,000 individuals, FaST-LMM ran an order of magnitude faster than current efficient algorithms. Our algorithm can analyze data for 120,000 individuals in just a few hours, whereas current algorithms fail on data for even 20,000 individuals (http://mscompbio.codeplex.com/).} } + +@article{Arends:2010, + author = {Arends, D. and Prins, P. and Jansen, R. C. and Broman, K. W.}, + title = {{R/qtl: high-throughput multiple QTL mapping}}, + journal = {Bioinformatics}, + year = {2010}, + volume = {26}, + number = {23}, + pages = {2990-2992}, + doi = {10.1093/bioinformatics/btq565}, + url = {http://www.ncbi.nlm.nih.gov/pubmed/20966004}, + abstract = {MOTIVATION: R/qtl is free and powerful software for mapping and exploring quantitative trait loci (QTL). R/qtl provides a fully comprehensive range of methods for a wide range of experimental cross types. We recently added multiple QTL mapping (MQM) to R/qtl. MQM adds higher statistical power to detect and disentangle the effects of multiple linked and unlinked QTL compared with many other methods. MQM for R/qtl adds many new features including improved handling of missing data, analysis of 10,000 s of molecular traits, permutation for determining significance thresholds for QTL and QTL hot spots, and visualizations for cis-trans and QTL interaction effects. MQM for R/qtl is the first free and open source implementation of MQM that is multi-platform, scalable and suitable for automated procedures and large genetical genomics datasets. AVAILABILITY: R/qtl is free and open source multi-platform software for the statistical language R, and is made available under the GPLv3 license. R/qtl can be installed from http://www.rqtl.org/. R/qtl queries should be directed at the mailing list, see http://www.rqtl.org/list/. CONTACT: kbroman@biostat.wisc.edu.}, + +} diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 81ec2b72..42fcd500 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -41,10 +41,10 @@ bibliography: paper.bib # Summary -GeneNetwork (GN) is a free and open source (FOSS) framework for web -based genetics that can be deployed anywhere. GN allows biologists to -upload experimental data and map phenotypes interactively against -genotypes using tools, such as R/QTL [@mqm paper] mapping, interval +GeneNetwork (GN) is a free and open source (FOSS) framework for +web-based genetics that can be deployed anywhere. GN allows biologists +to upload experimental data and map phenotypes interactively against +genotypes using tools, such as R/QTL [@Arends:2010] mapping, interval mapping for model organisms and pylmm; an implementation of FaST-LMM [@Lippert:2011] which is suitable for human populations and outbred crosses, such as the mouse diversity outcross. Interactive D3 graphics -- cgit v1.2.3 From 76a15627f15f9612702cc135747bb383c5a776b1 Mon Sep 17 00:00:00 2001 From: Arfon Smith Date: Sun, 29 May 2016 21:16:35 -0500 Subject: Fixing paper metadata --- doc/joss/2016/paper.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 42fcd500..38db3ca7 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -18,7 +18,6 @@ authors: orcid: 0000-0003-3142-2081 affiliation: University of Tennessee Health Science Center, USA - name: Nick Furlotte - orcid: ? - name: Harm Nijveen orcid: 0000-0002-9167-4945 affiliation: Wageningen University, The Netherlands @@ -33,8 +32,7 @@ authors: affiliation: University of Tennessee Health Science Center, USA - name: Pjotr Prins orcid: orcid.org/0000-0002-8021-9162 - affiliation: University Medical Center Utrecht, The Netherlands - affiliation: University of Tennessee Health Science Center, USA + affiliation: University Medical Center Utrecht, The Netherlands, University of Tennessee Health Science Center, USA date: 29 May 2016 bibliography: paper.bib --- -- cgit v1.2.3 From 7cc0899e46d2a5538b2646191daba6987d9ed16f Mon Sep 17 00:00:00 2001 From: DannyArends Date: Mon, 30 May 2016 09:39:00 +0200 Subject: Fixing my affiliation --- doc/joss/2016/paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 38db3ca7..bc63e835 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -10,7 +10,7 @@ authors: affiliation: University of Tennessee Health Science Center, USA - name: Danny Arends orcid: 0000-0001-8738-0162 - affiliation: Humboldt Universityl, Berlin, Germany + affiliation: Humboldt University, Berlin, Germany - name: Karl W. Broman orcid: 0000-0002-4914-6671 affiliation: University of Wisconsin, USA -- cgit v1.2.3 From 1912e8dd100cf97bd706e8c475dc9d09c1c3e56f Mon Sep 17 00:00:00 2001 From: DannyArends Date: Mon, 30 May 2016 09:40:55 +0200 Subject: Removing paper.json (unneeded) --- doc/joss/2016/paper.json | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 doc/joss/2016/paper.json (limited to 'doc') diff --git a/doc/joss/2016/paper.json b/doc/joss/2016/paper.json deleted file mode 100644 index c3c02156..00000000 --- a/doc/joss/2016/paper.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "@context": "https://raw.githubusercontent.com/mbjones/codemeta/master/codemeta.jsonld", - "@type": "Code", - "author": [ - { - "@id": "0000-0002-9623-3401", - "@type": "Person", - "email": "jakevdp@uw.edu", - "name": "Jake VanderPlas", - "affiliation": "University of Washington eScience Institute" - } - ], - "identifier": "https://zenodo.org/record/50995#.Vyp9DBUrJBw", - "codeRepository": "http://github.com/jakevdp/mst_clustering", - "datePublished": "2016-05-04", - "dateModified": "2016-05-04", - "dateCreated": "2016-05-04", - "description": "Clustering via Euclidean Minimum Spanning Trees", - "keywords": "machine learning", - "license": "BSD", - "title": "mst_clustering", - "version": "v1.0" -} -- cgit v1.2.3 From 7e3e07c1e3fb150a2d37088a8b250ef028c162ba Mon Sep 17 00:00:00 2001 From: DannyArends Date: Mon, 30 May 2016 09:42:01 +0200 Subject: Consistency in the orchid IDs --- doc/joss/2016/paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index bc63e835..02fc6a65 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -31,7 +31,7 @@ authors: orcid: 0000-0001-8924-4447 affiliation: University of Tennessee Health Science Center, USA - name: Pjotr Prins - orcid: orcid.org/0000-0002-8021-9162 + orcid: 0000-0002-8021-9162 affiliation: University Medical Center Utrecht, The Netherlands, University of Tennessee Health Science Center, USA date: 29 May 2016 bibliography: paper.bib -- cgit v1.2.3 From e241d27ccd3302632fec52043fc8b42125f4cfd7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 2 Jun 2016 09:40:22 +0200 Subject: Paper: addressing https://github.com/openjournals/joss-reviews/issues/25 1st par: "experimental data" could be fleshed out in 2-3 words for the biologist reader (is it SNPs, microarray data, NGS, Y2H or manually counted individuals?) --- doc/joss/2016/paper.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 02fc6a65..a3eb4477 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -41,14 +41,17 @@ bibliography: paper.bib GeneNetwork (GN) is a free and open source (FOSS) framework for web-based genetics that can be deployed anywhere. GN allows biologists -to upload experimental data and map phenotypes interactively against -genotypes using tools, such as R/QTL [@Arends:2010] mapping, interval -mapping for model organisms and pylmm; an implementation of FaST-LMM -[@Lippert:2011] which is suitable for human populations and outbred -crosses, such as the mouse diversity outcross. Interactive D3 graphics -are included from R/qtlcharts and presentation-ready figures can be -generated. Recently we have added functionality for phenotype -correlation [@Wang:2016] and network analysis [@WGCNA:2008]. +to upload high-throughput experimental data, such as expression data +from microarrays and RNA-seq, and also `classic' phenotypes, such as +disease phenotypes. These phenotypes can be mapped interactively +against genotypes using embedded tools, such as R/QTL [@Arends:2010] +mapping, interval mapping for model organisms and pylmm; an +implementation of FaST-LMM [@Lippert:2011] which is more suitable for +human populations and outbred crosses, such as the mouse diversity +outcross. Interactive D3 graphics are included from R/qtlcharts and +presentation-ready figures can be generated. Recently we have added +functionality for phenotype correlation [@Wang:2016] and network +analysis [@WGCNA:2008]. -![Mouse LMM mapping example](qtl2.png) -- cgit v1.2.3 From 9669b09bdd8885acae07e05cb878e575b1b2c6e1 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 2 Jun 2016 09:43:58 +0200 Subject: Paper: addressing https://github.com/openjournals/joss-reviews/issues/25 2nd par: "can be found in ___ guix-bioinformatics", please insert 'the article', 'the package', 'the lab fridge labeled'.. whichever is aproppriate ;-) --- doc/joss/2016/paper.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index a3eb4477..62daf177 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -57,7 +57,8 @@ analysis [@WGCNA:2008]. GN is written in python and javascript and contains a rich set of tools and libraries that can be written in any computer language. A -full list of included software can be found in +full list of included software can be found in the package named +`genenetwork2' and defined in [guix-bioinformatics](https://github.com/genenetwork/guix-bioinformatics/blob/master/gn/packages/genenetwork.scm). To make it easy to install GN locally in a byte reproducible way, including all dependencies and a 2GB MySQL test database (the full -- cgit v1.2.3 From 3788a3ae75d8260c167818786aeeeb20b22fb305 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 2 Jun 2016 09:45:41 +0200 Subject: Paper: addressing https://github.com/openjournals/joss-reviews/issues/25 last par: "Python" is spelled with a capital 'P' --- doc/joss/2016/paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 62daf177..9ac323d2 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -80,7 +80,7 @@ worked on, including GPU support. A REST interface is being added so that data can be uploaded to a server, analysis run remotely on high performance hardware, and results downloaded and used for further analysis. This feature will -allow biologist-programmers to use R and python on their computer and +allow biologist-programmers to use R and Python on their computer and execute computations on GN enabled servers. # References -- cgit v1.2.3 From 1aefe07b557b97b32c69ce926f90fd7f577ad7d9 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 8 Jun 2016 06:54:22 +0000 Subject: Docs: added installation recipe --- doc/README.org | 155 +++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 135 insertions(+), 20 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 278dd31c..f49143af 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,9 +3,12 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] + - [[#quick-installation-recipe][Quick installation recipe]] + - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]] + - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]] + - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] + - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]] - [[#source-deployment][Source deployment]] - - [[#install-guix][Install guix]] - - [[#checkout-the-git-repositories][Checkout the git repositories]] - [[#update-guix][Update guix]] - [[#install-gn2][Install GN2]] - [[#run-gn2][Run GN2]] @@ -30,20 +33,69 @@ main Guix package tree and that of the Genenetwork package tree. Current supported versions can be found as the SHA values of 'gn-latest' branches of [[https://github.com/genenetwork/guix-bioinformatics/tree/gn-latest][Guix bioinformatics]] and [[https://github.com/genenetwork/guix/tree/gn-latest][GNU Guix main]]. -* Source deployment -** Install guix +* Quick installation recipe + +This is a recipe for quick and dirty installation of GN2. For +convenience everything is installed as root, though in reality only +GNU Guix has to be installed as root. I tested this recipe on a fresh +install of Debian 8.3.0 (in KVM) though it should work on any modern +Linux distribution (including CentOS). For more elaborate installation +instructions see [[#source-deployment][Source deployment]]. + +** Step 1: Install GNU Guix + +Fetch the GNU Guix binary from [[https://www.gnu.org/software/guix/download/][here]] (middle panel) and follow +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html][instructions]]. Essentially download and unpack the tar ball (which +creates directories in /gnu and /var/guix), add build users and group +(Guix builds software as unpriviliged users) and run the Guix daemon +after fixing the paths (aka known as the 'profile'). + +You have succeeded and you have set the [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-the-key][set the key]] (getting +permission to download binaries), you should be able to install the +hello package using binary install + +: export PATH=~/.guix-profile/bin:$PATH +: guix pull +: guix package -i hello --dry-run -Deploying from source is also straightforward. Install GNU Guix using -a binary tar ball as described [[https://github.com/pjotrp/guix-notes][here]]. +Which should show something like -If it works you should be able to install a package with +: The following files would be downloaded: +: /gnu/store/zby49aqfbd9w9br4l52mvb3y6f9vfv22-hello-2.10 +: ... + +where 'downloaded' means binary installs. The actual installation +command of 'hello' is : guix package -i hello +: hello +: Hello, world! + +If you see things building it means that Guix is not yet properly +installed and up-to-date, i.e., the key is missing or you need to do a +'guix pull'. + +If you need more help we have another writeup in [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#binary-installation][guix-notes]]. To get +rid of the locale warning see [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-locale][set-locale]]. + +Probably a good idea to update guix to the latest + +: guix pull +: guix package -i guix +: guix --version +: guix (GNU Guix) 0.10.0 -** Checkout the git repositories +and then best restart the daemon (see above). -Check out the two relevant guix and guix-bioinformatics git -repositories: +** Step 2: Checkout the GN2 git repositories + +To fixate the software dependency graph GN2 uses git repositories of +Guix packages. First install git if it is missing + +: guix package -i git +: export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt + +check out the git repsitories (gn-latest branch) #+begin_src bash cd ~ @@ -54,24 +106,87 @@ git clone --branch gn-latest --recursive https://github.com/genenetwork/guix gui cd guix-gn-latest #+end_src bash +** Step 3: Authorize the GN Guix server + +GN2 has its own GNU Guix binary distribution server. To trust it you have +to add the following key + +#+begin_src scheme +(public-key + (ecc + (curve Ed25519) + (q #11217788B41ADC8D5B8E71BD87EF699C65312EC387752899FE9C888856F5C769#) + ) +) +#+end_src + +by pasting it into the command + +: guix archive --authorize + +and hit Ctrl-D. + +Now you can use the substitute server to install GN2 binaries. + +** Step 4: Install and run GN2 + +Since this is a quick and dirty install we are going to override the +GNU Guix package path by pointing the package path to our repository: + +: rm /root/.config/guix/latest +: ln -s ~/genenetwork/guix-gn-latest/ /root/.config/guix/latest + +Now check whether you can find the GN2 package with + +: env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -A genenetwork2 +: genenetwork2 2.0-a8fcff4 out gn/packages/genenetwork.scm:144:2 + +(ignore the source file newer then ... messages, this is caused by the +/root/.config/guix/latest override). + +And install with + +: env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -i genenetwork2 --substitute-urls="http://guix.genenetwork.org:8080 https://mirror.guixsd.org" + +After installation you should be able to run genenetwork2 + +: genenetwork2 + +It will complain that the database is missing. See the section on [[#run-mysql-server][Run +MySQL server]] for downloading and installing a MySQL GN2 +database. After installing the database restart genenetwork2 and point +your browser at http://localhost:5003/. + +End of recipe! + +* Source deployment + +This section gives a more elaborate instruction for installing GN2 +from source. + +First execute above 3 steps: + + - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]] + - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]] + - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] + ** Update guix -At some point you may decide to create, install and run a recent -version of the guix-daemon by compiling the guix repository. Follow -[[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. +Instead of Step 4 we now compile Guix from scratch. + +Create, install and run a recent version of the guix-daemon by +compiling the guix repository. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. ** Install GN2 #+begin_src bash -env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env \ - guix package -i genenetwork2 --fallback +env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --substitute-urls="http://guix.genenetwork.org:8080 https://mirror.guixsd.org" #+end_src bash -Note that you can use the genenetwork.org guix substitute caching -server at http://guix.genenetwork.org:8080 (which speeds up installs -significantly because all packages are pre-built). Here an IRC session -where we installed GN2 from scratch using GNU Guix and a download -of the test database: +Note the use of ./pre-inst-env here! + +Here an IRC session where we installed GN2 from scratch using GNU Guix +and a download of the test database: #+begin_src time to get binary install sorted :) [07:03] -- cgit v1.2.3 From 7e210c2b8a3f0e4525dcfafcdca4e95ef7a33690 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 8 Jun 2016 09:47:40 +0000 Subject: Docs: updated installation recipe --- doc/README.org | 97 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 37 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index f49143af..36b7d8f2 100644 --- a/doc/README.org +++ b/doc/README.org @@ -42,58 +42,58 @@ install of Debian 8.3.0 (in KVM) though it should work on any modern Linux distribution (including CentOS). For more elaborate installation instructions see [[#source-deployment][Source deployment]]. +Note that GN2 consists of an approx. 4GB installation including +database. + ** Step 1: Install GNU Guix Fetch the GNU Guix binary from [[https://www.gnu.org/software/guix/download/][here]] (middle panel) and follow -[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html][instructions]]. Essentially download and unpack the tar ball (which -creates directories in /gnu and /var/guix), add build users and group -(Guix builds software as unpriviliged users) and run the Guix daemon -after fixing the paths (aka known as the 'profile'). +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html][instructions]]. Essentially you have to download and unpack the tar ball +(which creates directories in /gnu and /var/guix), add build users and +group (Guix builds software as unpriviliged users) and run the Guix +daemon after fixing the paths (also known as the 'profile'). -You have succeeded and you have set the [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-the-key][set the key]] (getting -permission to download binaries), you should be able to install the -hello package using binary install +Once you have succeeded, you have to [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-the-key][set the key]] (getting permission +to download binaries from the GNU server) and you should be able to +install the hello package using binary packages (no building) -: export PATH=~/.guix-profile/bin:$PATH -: guix pull -: guix package -i hello --dry-run +#+begin_src bash +export PATH=~/.guix-profile/bin:$PATH +guix pull +guix package -i hello --dry-run +#+end_src Which should show something like : The following files would be downloaded: : /gnu/store/zby49aqfbd9w9br4l52mvb3y6f9vfv22-hello-2.10 : ... +#+end_src -where 'downloaded' means binary installs. The actual installation -command of 'hello' is +means binary installs. The actual installation command of 'hello' is -: guix package -i hello -: hello -: Hello, world! +#+begin_src bash +guix package -i hello +hello + Hello, world! +#+end_src -If you see things building it means that Guix is not yet properly -installed and up-to-date, i.e., the key is missing or you need to do a -'guix pull'. +If you actually see things building it means that Guix is not yet +properly installed and up-to-date, i.e., the key is missing or you +need to do a 'guix pull'. Press Ctrl-C to interrupt. If you need more help we have another writeup in [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#binary-installation][guix-notes]]. To get rid of the locale warning see [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#set-locale][set-locale]]. -Probably a good idea to update guix to the latest - -: guix pull -: guix package -i guix -: guix --version -: guix (GNU Guix) 0.10.0 - -and then best restart the daemon (see above). - ** Step 2: Checkout the GN2 git repositories To fixate the software dependency graph GN2 uses git repositories of Guix packages. First install git if it is missing -: guix package -i git -: export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt +#+begin_src bash +guix package -i git +export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt +#+end_src check out the git repsitories (gn-latest branch) @@ -122,7 +122,9 @@ to add the following key by pasting it into the command -: guix archive --authorize +#+begin_src bash +guix archive --authorize +#+end_src and hit Ctrl-D. @@ -133,24 +135,45 @@ Now you can use the substitute server to install GN2 binaries. Since this is a quick and dirty install we are going to override the GNU Guix package path by pointing the package path to our repository: -: rm /root/.config/guix/latest -: ln -s ~/genenetwork/guix-gn-latest/ /root/.config/guix/latest +#+begin_src bash +rm /root/.config/guix/latest +ln -s ~/genenetwork/guix-gn-latest/ /root/.config/guix/latest +#+end_src Now check whether you can find the GN2 package with -: env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -A genenetwork2 -: genenetwork2 2.0-a8fcff4 out gn/packages/genenetwork.scm:144:2 +#+begin_src bash +env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -A genenetwork2 + genenetwork2 2.0-a8fcff4 out gn/packages/genenetwork.scm:144:2 +#+end_src (ignore the source file newer then ... messages, this is caused by the /root/.config/guix/latest override). And install with -: env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ guix package -i genenetwork2 --substitute-urls="http://guix.genenetwork.org:8080 https://mirror.guixsd.org" +#+begin_src bash +env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ \ + guix package -i genenetwork2 \ + --substitute-urls="http://guix.genenetwork.org:8080 https://mirror.guixsd.org" \ + --fallback +#+end_src + +Note that you may (currently) get an error: -After installation you should be able to run genenetwork2 +: guix substitute: error: download from 'http://guix.genenetwork.org:8080/nar/sqd3q1xq5fsbga00bwhghi9shi7xdaac-gtk+-3.18.2' failed: 404, "Not Found" -: genenetwork2 +which can be fixed with using the --fallback switch, or install + +: guix package -i gtk+@3.18.2 + +and restart the genenetwork2 install. + +After installation you should be able to run genenetwork2. + +#+begin_src bash +genenetwork2 +#+end_src It will complain that the database is missing. See the section on [[#run-mysql-server][Run MySQL server]] for downloading and installing a MySQL GN2 -- cgit v1.2.3 From b88843ee70fa2976674c9b2b5721af18f570978d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 8 Jun 2016 10:04:45 +0000 Subject: Docs: updated installation recipe --- doc/README.org | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 36b7d8f2..952f406f 100644 --- a/doc/README.org +++ b/doc/README.org @@ -42,7 +42,7 @@ install of Debian 8.3.0 (in KVM) though it should work on any modern Linux distribution (including CentOS). For more elaborate installation instructions see [[#source-deployment][Source deployment]]. -Note that GN2 consists of an approx. 4GB installation including +Note that GN2 consists of an approx. 5 GB installation including database. ** Step 1: Install GNU Guix @@ -169,7 +169,16 @@ which can be fixed with using the --fallback switch, or install and restart the genenetwork2 install. -After installation you should be able to run genenetwork2. +After installation you should be able to run genenetwork2 after updating +the Guix suggested environment vars. Check the ouput of + +#+begin_src bash +guix package --search-paths +export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages" +export R_LIBS_SITE="/root/.guix-profile/site-library/" +#+end_src + +and copy-paste the exports into the terminal befor running: #+begin_src bash genenetwork2 -- cgit v1.2.3 From 1484054f2a8cdc94d1fccc0c6914c04f5145a2fa Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 8 Jun 2016 10:11:47 +0000 Subject: Docs: updated installation recipe --- doc/README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 952f406f..8a26c0f3 100644 --- a/doc/README.org +++ b/doc/README.org @@ -187,7 +187,7 @@ genenetwork2 It will complain that the database is missing. See the section on [[#run-mysql-server][Run MySQL server]] for downloading and installing a MySQL GN2 database. After installing the database restart genenetwork2 and point -your browser at http://localhost:5003/. +your browser at [[http://localhost:5003/]]. End of recipe! -- cgit v1.2.3 From fbd60cd5fccdd6e664865a74fd1a4433fc1ec55d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 8 Jun 2016 10:23:53 +0000 Subject: JOSS: added orcid for Nick Furlotte --- doc/joss/2016/paper.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 9ac323d2..6b640836 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -17,7 +17,8 @@ authors: - name: Arthur Centeno orcid: 0000-0003-3142-2081 affiliation: University of Tennessee Health Science Center, USA - - name: Nick Furlotte + - name: Nicholas Furlotte + orcid: 0000-0002-9096-6276 - name: Harm Nijveen orcid: 0000-0002-9167-4945 affiliation: Wageningen University, The Netherlands -- cgit v1.2.3 From 76895b944eb03011123b35e88570691dde60e9ff Mon Sep 17 00:00:00 2001 From: Dennis E. Mungai Date: Wed, 8 Jun 2016 16:09:37 +0300 Subject: Fix a typo Fixed: Before, substituted.--- doc/README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 8a26c0f3..20240c43 100644 --- a/doc/README.org +++ b/doc/README.org @@ -178,7 +178,7 @@ export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages" export R_LIBS_SITE="/root/.guix-profile/site-library/" #+end_src -and copy-paste the exports into the terminal befor running: +and copy-paste the exports into the terminal before running: #+begin_src bash genenetwork2 -- cgit v1.2.3 From cf0be1402a6d14b9aa969f68bb9ef731749c6fe7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 9 Jun 2016 08:23:03 +0000 Subject: Docs: expanding on GN2 source install and moved IRC log to the bottom --- doc/README.org | 450 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 237 insertions(+), 213 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 8a26c0f3..86aa9aa9 100644 --- a/doc/README.org +++ b/doc/README.org @@ -8,18 +8,19 @@ - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]] - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]] + - [[#run-mysql-server][Run MySQL server]] - [[#source-deployment][Source deployment]] - - [[#update-guix][Update guix]] - - [[#install-gn2][Install GN2]] - - [[#run-gn2][Run GN2]] - - [[#run-mysql-server][Run MySQL server]] - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] - [[#set-up-nginx-port-forwarding][Set up nginx port forwarding]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] + - [[#update-to-recent-guix][Update to recent guix]] + - [[#install-gn2][Install GN2]] + - [[#run-gn2][Run GN2]] - [[#trouble-shooting][Trouble shooting]] - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]] - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]] - [[#cant-run-a-module][Can't run a module]] + - [[#irc-session][IRC session]] * Introduction @@ -184,41 +185,265 @@ and copy-paste the exports into the terminal befor running: genenetwork2 #+end_src -It will complain that the database is missing. See the section on [[#run-mysql-server][Run -MySQL server]] for downloading and installing a MySQL GN2 +It will complain that the database is missing. See the next section on +running MySQL server for downloading and installing a MySQL GN2 database. After installing the database restart genenetwork2 and point your browser at [[http://localhost:5003/]]. -End of recipe! +End of the GN2 installation recipe! + +* Run MySQL server + +At this point we require the underlying distribution to install and +run mysqld. Currently we have two databases for deployment, +'db_webqtl_s' is the small testing database containing experiments +from BXD mice and 'db_webqtl_plant' which contains all plant related +material. + +Download one database from + +http://files.genenetwork.org/raw_database/ +https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip + +Check the md5sum. + +After installation inflate the database binary in the MySQL directory +(this installation path is subject to change soon) + +: chown -R mysql:mysql db_webqtl_s/ +: chmod 700 db_webqtl_s/ +: chmod 660 db_webqtl_s/* + +restart MySQL service (mysqld). Login as root and + +: mysql> show databases; +: +--------------------+ +: | Database | +: +--------------------+ +: | information_schema | +: | db_webqtl_s | +: | mysql | +: | performance_schema | +: +--------------------+ + +Set permissions and match password in your settings file below: + +: mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; + +Note that if the mysql connection is not working, try connecting to +the IP address and check server firewall, hosts.allow and mysql IP +configuration. + +Note for the plant database you can rename it to db_webqtl_s, or +change the settings in etc/default_settings.py to match your path. * Source deployment This section gives a more elaborate instruction for installing GN2 from source. -First execute above 3 steps: +First execute above 4 steps: - [[#step-1-install-gnu-guix][Step 1: Install GNU Guix]] - [[#step-2-checkout-the-gn2-git-repositories][Step 2: Checkout the GN2 git repositories]] - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] + - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]] + + +** Run your own copy of GN2 + +At some point you may want to fix the source code. Assuming you have +Guix and Genenetwork2 installed (as described above) clone the GN2 +repository from https://github.com/genenetwork/genenetwork2. + +Copy-paste the paths into your terminal (mainly so PYTHON_PATH and +R_LIBS_SITE are set) from the information given by guix: + +: guix package --search-paths + +Inside the repository: + +: cd genenetwork2 +: ./bin/genenetwork2 + +Will fire up your local repo http://localhost:5003/ using the +settings in ./etc/default_settings.py. These settings may +not reflect your system. To override settings create your own from a copy of +default_settings.py and pass it into GN2 with + +: ./bin/genenetwork2 $HOME/my_settings.py + +and everything *should* work (note the full path to the settings +file). This way we develop against the exact same dependency graph of +software. + +If something is not working, take a hint from the settings file +that comes in the Guix installation. It sits in something like + +: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py + +** Set up nginx port forwarding + +nginx can be used as a reverse proxy for GN2. For example, we want to +expose GN2 on port 80 while it is running on port 5003. Essentially +the configuration looks like + +#+begin_src js + server { + listen 80; + server_name test-gn2.genenetwork.org; + access_log logs/test-gn2.access.log; + + proxy_connect_timeout 3000; + proxy_send_timeout 3000; + proxy_read_timeout 3000; + send_timeout 3000; + + location / { + proxy_set_header Host $http_host; + proxy_set_header Connection keep-alive; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + proxy_pass http://127.0.0.1:5003; + } +} +#+end_src js -** Update guix +Install the nginx webserver (as root) + +: guix package -i nginx + +The nginx example configuration examples can be found in the Guix +store through + +: ls -l /root/.guix-profile/sbin/nginx +: lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx + +Use that path + +: ls /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/share/nginx/conf/ +: fastcgi.conf koi-win scgi_params +: fastcgi.conf.default mime.types scgi_params.default +: fastcgi_params mime.types.default uwsgi_params +: fastcgi_params.default nginx.conf uwsgi_params.default +: koi-utf nginx.conf.default win-utf -Instead of Step 4 we now compile Guix from scratch. +And copy any relevant files to /etc/nginx. A configuration file for +GeneNetwork (reverse proxy) port forwarding can be found in the source +repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc +(still as root) +: cp ./etc/nginx-genenetwork.conf /etc/nginx/ + +Make dirs + +: mkdir -p /var/spool/nginx/logs + +Add users + +: adduser nobody ; addgroup nobody + +Run nginx + +: /root/.guix-profile/sbin/nginx -c /etc/nginx/nginx-genenetwork.conf -p /var/spool/nginx + +* Source deployment and other information on reproducibility + +See the document [[GUIX-Reproducible-from-source.org]]. + +** Update to recent guix + +We now compile Guix from scratch. Create, install and run a recent version of the guix-daemon by -compiling the guix repository. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully. +compiling the guix repository you have installed with git in +step 2. Follow [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][these]] steps carefully after + +: cd ~/genenetwork/guix-gn-latest + +Make sure to restart the guix daemon and run guix client from this +directory. ** Install GN2 +Reinstall genenetwork2 using the new tree + #+begin_src bash env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ ./pre-inst-env guix package -i genenetwork2 --substitute-urls="http://guix.genenetwork.org:8080 https://mirror.guixsd.org" #+end_src bash Note the use of ./pre-inst-env here! +Actually, it should be the same installation as in step 4, so nothing +gets downloaded. + +** Run GN2 + +Make a note of the paths with + +#+begin_src bash +./pre-inst-env guix package --search-paths +#+end_src bash + +After setting the paths for the server + +#+begin_src bash +export PATH=~/.guix-profile/bin:$PATH +export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +export R_LIBS_SITE="$HOME/.guix-profile/site-library/" +export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0" +export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0" +export XDG_DATA_DIRS="$HOME/.guix-profile/share" +export GIO_EXTRA_MODULES="$HOME/.guix-profile/lib/gio/modules" +#+end_src bash + +run the main script (in ~/.guix-profile/bin) + +#+begin_src bash +genenetwork2 +#+end_src bash + +will start the default server which listens on port 5003, i.e., +http://localhost:5003/. + +OK, we are where we were before with step 4. Only difference is that we +used our own compiled guix server. + +* Trouble shooting + +** ImportError: No module named jinja2 + +If you have all the Guix packages installed this error points out that +the environment variables are not set. Copy-paste the paths into your +terminal (mainly so PYTHON_PATH and R_LIBS_SITE are set) from the +information given by guix: + +: guix package --search-paths + +On one system: + +: export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" +: export R_LIBS_SITE="$HOME/.guix-profile/site-library/" +: export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0" + +and perhaps a few more. +** ERROR: can not find directory $HOME/gn2_data + +The default settings file looks in your $HOME/gn2_data. Since these +files come with a Guix installation you should take a hint from the +values in the installed version of default_settings.py (see above in +this document). + +** Can't run a module + +In rare cases, development modules are not brought in with Guix +because no source code is available. This can lead to missing modules +on a running server. Please check with the authors when a module +is missing. +* IRC session + Here an IRC session where we installed GN2 from scratch using GNU Guix -and a download of the test database: +and a download of the test database. #+begin_src time to get binary install sorted :) [07:03] @@ -622,204 +847,3 @@ The following derivations would be built: I am building it on guix.genenetwork.org right now [10:09] nice [10:10] #+end_src - -** Run GN2 - -Make a note of the paths with - -#+begin_src bash -./pre-inst-env guix package --search-paths -#+end_src bash - -After setting the paths for the server - -#+begin_src bash -export PATH=~/.guix-profile/bin:$PATH -export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" -export R_LIBS_SITE="$HOME/.guix-profile/site-library/" -export GUIX_GTK3_PATH="$HOME/.guix-profile/lib/gtk-3.0" -export GI_TYPELIB_PATH="$HOME/.guix-profile/lib/girepository-1.0" -export XDG_DATA_DIRS="$HOME/.guix-profile/share" -export GIO_EXTRA_MODULES="$HOME/.guix-profile/lib/gio/modules" -#+end_src bash - -run the main script (in ~/.guix-profile/bin) - -#+begin_src bash -genenetwork2 -#+end_src bash - -will start the default server which listens on port 5003, i.e., -http://localhost:5003/. - -** Run MySQL server - -At this point we require the underlying distribution to install -and run mysqld. - -Download one of - -http://files.genenetwork.org/raw_database/ -https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip - -Check the md5sum. - -After installation inflate the database binary in the MySQL directory -(this is subject to change soon) - -: chown -R mysql:mysql db_webqtl_s/ -: chmod 700 db_webqtl_s/ -: chmod 660 db_webqtl_s/* - -restart MySQL service (mysqld). Login as root and - -: mysql> show databases; -: +--------------------+ -: | Database | -: +--------------------+ -: | information_schema | -: | db_webqtl_s | -: | mysql | -: | performance_schema | -: +--------------------+ - -Set permissions and match password in your settings file below: - -: mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; - -Note that if the mysql connection is not working, try connecting to -the IP address and check server firewall, hosts.allow and mysql IP -configuration. - -** Run your own copy of GN2 - -At some point you may want to fix the source code. Assuming you have -Guix and Genenetwork2 installed (as described above) clone the GN2 -repository from https://github.com/genenetwork/genenetwork2. - -Copy-paste the paths into your terminal (mainly so PYTHON_PATH and -R_LIBS_SITE are set) from the information given by guix: - -: guix package --search-paths - -Inside the repository: - -: cd genenetwork2 -: ./bin/genenetwork2 - -Will fire up your local repo http://localhost:5003/ using the -settings in ./etc/default_settings.py. These settings may -not reflect your system. To override settings create your own from a copy of -default_settings.py and pass it into GN2 with - -: ./bin/genenetwork2 $HOME/my_settings.py - -and everything *should* work (note the full path to the settings -file). This way we develop against the exact same dependency graph of -software. - -If something is not working, take a hint from the settings file -that comes in the Guix installation. It sits in something like - -: cat ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py - -** Set up nginx port forwarding - -nginx can be used as a reverse proxy for GN2. For example, we want to -expose GN2 on port 80 while it is running on port 5003. Essentially -the configuration looks like - -#+begin_src js - server { - listen 80; - server_name test-gn2.genenetwork.org; - access_log logs/test-gn2.access.log; - - proxy_connect_timeout 3000; - proxy_send_timeout 3000; - proxy_read_timeout 3000; - send_timeout 3000; - - location / { - proxy_set_header Host $http_host; - proxy_set_header Connection keep-alive; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; - proxy_pass http://127.0.0.1:5003; - } -} -#+end_src js - -Install the nginx webserver (as root) - -: guix package -i nginx - -The nginx example configuration examples can be found in the Guix -store through - -: ls -l /root/.guix-profile/sbin/nginx -: lrwxrwxrwx 3 root guixbuild 66 Dec 31 1969 /root/.guix-profile/sbin/nginx -> /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/sbin/nginx - -Use that path - -: ls /gnu/store/g0wrcl5z27rmk5b52rldzvk1bzzbnz2l-nginx-1.8.1/share/nginx/conf/ -: fastcgi.conf koi-win scgi_params -: fastcgi.conf.default mime.types scgi_params.default -: fastcgi_params mime.types.default uwsgi_params -: fastcgi_params.default nginx.conf uwsgi_params.default -: koi-utf nginx.conf.default win-utf - -And copy any relevant files to /etc/nginx. A configuration file for -GeneNetwork (reverse proxy) port forwarding can be found in the source -repository under ./etc/nginx-genenetwork.conf. Copy this file to /etc -(still as root) -: cp ./etc/nginx-genenetwork.conf /etc/nginx/ - -Make dirs - -: mkdir -p /var/spool/nginx/logs - -Add users - -: adduser nobody ; addgroup nobody - -Run nginx - -: /root/.guix-profile/sbin/nginx -c /etc/nginx/nginx-genenetwork.conf -p /var/spool/nginx - -* Source deployment and other information on reproducibility - -See the document [[GUIX-Reproducible-from-source.org]]. - -* Trouble shooting - -** ImportError: No module named jinja2 - -If you have all the Guix packages installed this error points out that -the environment variables are not set. Copy-paste the paths into your -terminal (mainly so PYTHON_PATH and R_LIBS_SITE are set) from the -information given by guix: - -: guix package --search-paths - -On one system: - -: export PYTHONPATH="$HOME/.guix-profile/lib/python2.7/site-packages" -: export R_LIBS_SITE="$HOME/.guix-profile/site-library/" -: export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0" - -and perhaps a few more. -** ERROR: can not find directory $HOME/gn2_data - -The default settings file looks in your $HOME/gn2_data. Since these -files come with a Guix installation you should take a hint from the -values in the installed version of default_settings.py (see above in -this document). - -** Can't run a module - -In rare cases, development modules are not brought in with Guix -because no source code is available. This can lead to missing modules -on a running server. Please check with the authors when a module -is missing. -- cgit v1.2.3 From 47cf69f2f2fc8243b94f17387547c1fc12fb60ab Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 9 Jun 2016 09:07:34 +0000 Subject: Docs: add SVG graph --- doc/README.org | 2 +- doc/joss/2016/paper.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index bcc8a65e..681357ef 100644 --- a/doc/README.org +++ b/doc/README.org @@ -24,7 +24,7 @@ * Introduction -Large system deployments can get very complex. In this document we +Large system deployments can get very [[http://biobeat.org/gn2.svg][complex]]. In this document we explain the GeneNetwork version 2 (GN2) reproducible deployment system which is based on GNU Guix (see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install GN with all its files and dependencies. diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 6b640836..7c6f76cc 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -69,6 +69,8 @@ database is 160GB and growing), GN is packaged with GNU Guix deployment makes it feasible to deploy and rebrand GN anywhere. +-![GN2 dependency graph](http://biobeat.org/gn2.svg) + # Future work More mapping tools will be added, including support for Genome-wide -- cgit v1.2.3 From ed482493f2fa907632bacffac7a1938e9656b0d5 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 9 Jun 2016 09:37:30 +0000 Subject: Doc: moved graph into README --- doc/README.org | 5 +++++ doc/joss/2016/paper.md | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 681357ef..a36d04bb 100644 --- a/doc/README.org +++ b/doc/README.org @@ -34,6 +34,11 @@ main Guix package tree and that of the Genenetwork package tree. Current supported versions can be found as the SHA values of 'gn-latest' branches of [[https://github.com/genenetwork/guix-bioinformatics/tree/gn-latest][Guix bioinformatics]] and [[https://github.com/genenetwork/guix/tree/gn-latest][GNU Guix main]]. +Graph of all runtime dependencies as installed by GNU Guix. + +-![GN2 dependency graph](http://biobeat.org/gn2.svg) + + * Quick installation recipe This is a recipe for quick and dirty installation of GN2. For diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 7c6f76cc..6b640836 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -69,8 +69,6 @@ database is 160GB and growing), GN is packaged with GNU Guix deployment makes it feasible to deploy and rebrand GN anywhere. --![GN2 dependency graph](http://biobeat.org/gn2.svg) - # Future work More mapping tools will be added, including support for Genome-wide -- cgit v1.2.3 From 65e8217f02c5828e86c523fc98f757b05e1ba24c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 9 Jun 2016 09:45:38 +0000 Subject: Doc: moved graph into README --- doc/README.org | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index a36d04bb..1fffd32a 100644 --- a/doc/README.org +++ b/doc/README.org @@ -36,7 +36,8 @@ tree. Current supported versions can be found as the SHA values of Graph of all runtime dependencies as installed by GNU Guix. --![GN2 dependency graph](http://biobeat.org/gn2.svg) +#+ATTR_HTML: :title GN2_graph +http://biobeat.org/gn2.svg]] * Quick installation recipe -- cgit v1.2.3 From b989351195cb0ecb14f26773b4ab547558d1e662 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 9 Jun 2016 09:47:00 +0000 Subject: Doc: moved graph into README --- doc/README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 1fffd32a..7b410305 100644 --- a/doc/README.org +++ b/doc/README.org @@ -37,7 +37,7 @@ tree. Current supported versions can be found as the SHA values of Graph of all runtime dependencies as installed by GNU Guix. #+ATTR_HTML: :title GN2_graph -http://biobeat.org/gn2.svg]] +[[http://biobeat.org/gn2.svg]] * Quick installation recipe -- cgit v1.2.3 From f5778c492dfb05ba413405d8b0e758b886c573f9 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 9 Jun 2016 09:49:10 +0000 Subject: Doc: moved graph to a lower section --- doc/README.org | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index 7b410305..fc64983e 100644 --- a/doc/README.org +++ b/doc/README.org @@ -9,6 +9,7 @@ - [[#step-3-authorize-the-gn-guix-server][Step 3: Authorize the GN Guix server]] - [[#step-4-install-and-run-gn2-][Step 4: Install and run GN2 ]] - [[#run-mysql-server][Run MySQL server]] + - [[#gn2-dependency-graph][GN2 Dependency Graph]] - [[#source-deployment][Source deployment]] - [[#run-your-own-copy-of-gn2][Run your own copy of GN2]] - [[#set-up-nginx-port-forwarding][Set up nginx port forwarding]] @@ -34,11 +35,8 @@ main Guix package tree and that of the Genenetwork package tree. Current supported versions can be found as the SHA values of 'gn-latest' branches of [[https://github.com/genenetwork/guix-bioinformatics/tree/gn-latest][Guix bioinformatics]] and [[https://github.com/genenetwork/guix/tree/gn-latest][GNU Guix main]]. -Graph of all runtime dependencies as installed by GNU Guix. - -#+ATTR_HTML: :title GN2_graph -[[http://biobeat.org/gn2.svg]] - +For a full view of runtime dependencies as defined by GNU Guix, see +the [[#gn2-dependency-graph][GN2 Dependency Graph]]. * Quick installation recipe @@ -243,6 +241,13 @@ configuration. Note for the plant database you can rename it to db_webqtl_s, or change the settings in etc/default_settings.py to match your path. +* GN2 Dependency Graph + +Graph of all runtime dependencies as installed by GNU Guix. + +#+ATTR_HTML: :title GN2_graph +[[http://biobeat.org/gn2.svg]] + * Source deployment This section gives a more elaborate instruction for installing GN2 -- cgit v1.2.3 From a04689a8d86d4e61ded3916bcf5d64e66b94c3ce Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 10 Jun 2016 09:54:33 +0000 Subject: Docs: the caching server got fixed by http://bugs.gnu.org/21888 --- doc/README.org | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) (limited to 'doc') diff --git a/doc/README.org b/doc/README.org index fc64983e..fbb3e1ad 100644 --- a/doc/README.org +++ b/doc/README.org @@ -164,18 +164,11 @@ env GUIX_PACKAGE_PATH=~/genenetwork/guix-bioinformatics/ \ --fallback #+end_src -Note that you may (currently) get an error: - -: guix substitute: error: download from 'http://guix.genenetwork.org:8080/nar/sqd3q1xq5fsbga00bwhghi9shi7xdaac-gtk+-3.18.2' failed: 404, "Not Found" - -which can be fixed with using the --fallback switch, or install - -: guix package -i gtk+@3.18.2 - -and restart the genenetwork2 install. +Note: the order of the substitute url's may make a difference in speed +(put the one first that is fastest for your location and time of day). After installation you should be able to run genenetwork2 after updating -the Guix suggested environment vars. Check the ouput of +the Guix suggested environment vars. Check the output of #+begin_src bash guix package --search-paths @@ -183,7 +176,7 @@ export PYTHONPATH="/root/.guix-profile/lib/python2.7/site-packages" export R_LIBS_SITE="/root/.guix-profile/site-library/" #+end_src -and copy-paste the exports into the terminal before running: +and copy-paste the listed exports into the terminal before running: #+begin_src bash genenetwork2 -- cgit v1.2.3 From 312f051074a8509ba85b6961fe1f04d4111a7288 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 14 Jun 2016 06:43:47 +0000 Subject: Adding documentation --- README.md | 8 +- doc/Architecture.org | 15 ++ doc/README.org | 4 + doc/old/gn_installation_notes.txt | 346 -------------------------------------- doc/old/notes.txt | 311 ---------------------------------- 5 files changed, 24 insertions(+), 660 deletions(-) create mode 100644 doc/Architecture.org delete mode 100644 doc/old/gn_installation_notes.txt delete mode 100644 doc/old/notes.txt (limited to 'doc') diff --git a/README.md b/README.md index f42a2333..8221ad65 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,12 @@ testing using [mechanize](https://github.com/genenetwork/genenetwork2/tree/maste ## Documentation User documentation can be found -[here](http://gn2.genenetwork.org/help). The database schema is -shared with GN1 and described +[here](http://gn2.genenetwork.org/help). The architecture of the +software stack is described [here](./doc/Architecture.org). The +database schema is (still) shared with GN1 and currently described [here](http://www.genenetwork.org/webqtl/main.py?FormID=schemaShowPage). Software -documentation is being expanded in the [source code repository](https://github.com/genenetwork/genenetwork2/tree/master/doc). +documentation is being expanded in the +[source code repository](https://github.com/genenetwork/genenetwork2/tree/master/doc). ## Contributing diff --git a/doc/Architecture.org b/doc/Architecture.org new file mode 100644 index 00000000..c263d3b9 --- /dev/null +++ b/doc/Architecture.org @@ -0,0 +1,15 @@ +* GeneNetwork Architecture + +** Introduction + +This document describes the architecture of GN2. Because GN2 is +evolving, only a high-level overview is given here. + +** Webserver + +The webserver is built on [[http://flask.pocoo.org/][Python flask]] and this GN2 source code can be +found on [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask][github]] in the wqflask directory. The routing tables are +defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/views.py][views.py]]. For example the main page is loaded from a +template named [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/templates/index_page.htm][index_page.html]] in the [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask/templates][templates]] directory. In the +template you can find get the form gets filled by a Javascript +routine defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/static/new/javascript/dataset_select_menu.js][data_select_menu.js]]. diff --git a/doc/README.org b/doc/README.org index fbb3e1ad..3754dbbd 100644 --- a/doc/README.org +++ b/doc/README.org @@ -25,6 +25,10 @@ * Introduction +If you want to understand the architecture of GN2 read +[[Architecture.org]]. The rest of this document is mostly on deployment +of GN2. + Large system deployments can get very [[http://biobeat.org/gn2.svg][complex]]. In this document we explain the GeneNetwork version 2 (GN2) reproducible deployment system which is based on GNU Guix (see also Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix diff --git a/doc/old/gn_installation_notes.txt b/doc/old/gn_installation_notes.txt deleted file mode 100644 index efea0309..00000000 --- a/doc/old/gn_installation_notes.txt +++ /dev/null @@ -1,346 +0,0 @@ -UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration - -Enable UFW: -sudo ufw enable -Check status of UFW: -sudo ufw status verbose - -Limit ssh: -sudo ufw limit ssh - -Enable http, https: -sudo ufw allow http -sudo ufw allow https - -=========================================== - -http://evilrouters.net/2011/02/18/using-etckeeper-with-git-on-ubuntu/ - -Install etckeeper: -sudo apt-get install etckeeper - -Edit /etc/etckeeper/etckeeper.conf to comment out VCS="bzr" and comment in VCS="git" - -Initialize repository: -sudo etckeeper init - -Do initial commit: -sudo etckeeper commit "Initial commit." - -=========================================== - -Get git: - -sudo apt-get install git - -=========================================== - -Clone code from git repository: -git clone http://github.com/zsloan/genenetwork.git gene - -Pull from branch in git repository: -git pull origin flask(or whatever the branch is) - -**"apt-cache search" searches for text in the description for various packages, while -"apt-file search" searches for files within the packages** - -Search for package with a specified file that can be installed with apt-get -apt-file search _______ - -============================================ - -Create trash directory: -mkdir ~/.trash - -============================================ - -Using dpkg - -Check which system packages are already installed: -dpkg -l | less - -=========================================== - -Installing virtualenv: -sudo pip install virtualenv - -Tell virtualenv which directory to use: -virtualenv ve27 - -Start up virtual environment: -source ~/ve27/bin/activate - -=========================================== - -Install libmysqlclient-dev (Mysql-Python dependency) -sudo apt-get install libmysqlclient-dev - -Install python-dev (numpy dependency) -sudo apt-get install python-dev - -Install scipy dependencies: -sudo apt-get install libatlas-base-dev gfortran g++ - -=========================================== - -Install pip: -sudo apt-get install python-pip - -REMEMBER TO SOURCE VE BEFORE INSTALLING - -Comment out in requirements.txt: -Reaper -numarray - -Before installing from requirements.txt, install numpy separately: -pip install numpy==1.7.0 (or whatever version we're using) - -Install from requirements.txt (after activating virtualenv): -pip install -r gene/doc/requirements.txt - -=========================================== - -Installing QTL Reaper - -wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb -mv -v qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb qtlreaper-1.1.1.tar.gz -tar xvf qtlreaper-1.1.1.tar.gz (to unzip) -mkdir /home/zas1024/ve27/include/python2.7/Reaper -python setup.py install - -=========================================== - -Installing numarray 1.5.2 - -wget http://downloads.sourceforge.net/project/numpy/Old%20Numarray/1.5.2/numarray-1.5.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%2520Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload&ts=1358978306&use_mirror=superb-dca2 -mv -v numarray-1.5.2.tar.gz?r=http:%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%20Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload numarray-1.5.2.tar.gz -tar xvf numarray-1.5.2.tar.gz -sudo python setup.py install - -=========================================== - -Installing nginx - -sudo -s -nginx=stable # use nginx=development for latest development version -echo "deb http://ppa.launchpad.net/nginx/$nginx/ubuntu lucid main" > /etc/apt/sources.list.d/nginx-$nginx-lucid.list -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C300EE8C -apt-get update -apt-get install nginx - -Create configuration file in ~/gene/wqflask/other_config/nginx_conf/ (modeled off of the others) -Create symbolic link to config file in /etc/nginx/sites-enabled/: -ln -s dir_to_link_to linking_dir - -Run nginx: -sudo /usr/sbin/nginx - -============================================ - -Copy over zach_settings.py to /home/zas1024 directory (or whatever is home directory) - -export WQFLASK_SETTINGS=~/gene/wqflask/cfg/zach_settings.py (or wherever file is located) - -============================================ - -Install MySQL Server - -sudo apt-get install mysql-server - -mkdir /mnt/big -fdisk /dev/sdb -m: for help -n: new partion -... -w: write to table and exit - -Start MySQL server: -service mysql start - -Stop MySQL server: -service mysql stop - -Change root password: -mysql> UPDATE mysql.user SET Password=PASSWORD('your password') WHERE User='root'; - -Setup accounts in MySQL (first need to delete anonymous/non-root accounts): -#; use mysql; -#; select * from user; -#; delete from user where Host!="localhost"; -#; delete from user where User!="root"; -#; update user set Password = Password('yourpassword') where User='root'; -#; GRANT ALL ON *.* TO 'yourusername'@'%' IDENTIFIED BY 'yourpassword'; -#; select * from user; - -Use Percona to optimise my.cfg file: -https://tools.percona.com/wizard - -To check # CPUs: -grep -c processor /proc/cpuinfo - -To check memory, can use htop - -============================================ - -Check RSA key: -ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key - -03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) - -============================================ - -Installing redis server - -In the dls directory: -$ wget http://redis.googlecode.com/files/redis-2.6.14.tar.gz -$ tar xzf redis-2.6.14.tar.gz -$ cd redis-2.6.14 -$ make - -Run redis (from inside redis-2.6.14 directory): -src/redis-server - -For the rest of the installation instructions, refer to: -http://redis.io/topics/quickstart - -To continue installation (from the README): - -In order to install Redis binaries into /usr/local/bin just use: - - % make install - -Make install will just install binaries in your system, but will not configure -init scripts and configuration files in the appropriate place. This is not -needed if you want just to play a bit with Redis, but if you are installing -it the proper way for a production system, we have a script doing this -for Ubuntu and Debian systems: - - % cd utils - % ./install_server - -The script will ask you a few questions and will setup everything you need -to run Redis properly as a background daemon that will start again on -system reboots. - -In the redis config file (/etc/redis/6379.conf) uncomment the following line: -bind 127.0.0.1 - -This is so only the local computer has access - -============================================ - -Check RSA key: -ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key - -03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) - -============================================ - -Using Yolk - -Install Yolk: -pip install yolk - -Check packages installed in this virtual environment: -yolk -l - -Checks packages that have updates available: -yolk -U - -=========================================== - -Installing yaml - -Install libyaml-dev: -sudo apt-get install libyaml-dev - -Install yaml: -pip install pyyaml - -=========================================== - -Install MySQL Client - -To fix error "mysql_config not found" while installing packages with pip: -sudo apt-get install libmysqlclient-dev - -=========================================== - -Installing R - -sudo apt-get install r-base-dev - -=========================================== - -Installing rpy2 - -pip install rpy2 - -=========================================== - -Install Parallel Python (pp) - -wget http://www.parallelpython.com/downloads/pp/pp-1.6.3.tar.gz -tar xvf pp-1.6.3.tar.gz -python setup.py install - -========================================== - -Install requests - -pip install requests - -========================================== - -Install logging_tree - -pip install logging_tree - -========================================== - -Install colordiff (just to make it easier to view differences in files, not necessary for GN) - -sudo apt-get install colordiff - -========================================== - -Install NTP (network time protocol) - -sudo apt-get install ntp - -========================================== - -To get server running: - -!If having seemingly inexplicable problems with imports, make sure I've started the environment! - -Start up virtual environment: -source ~/ve27/bin/activate============== - -Install requests - -pip install requests - -======================= - -To set WQFLASK_SETTINGS environment variable: -export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) - -To change screen environment variable (if man not working or to get color, for example): -export TERM=screen - -To search for commands in history if necessary: -history | grep "(whatever is being searched for)" - -Run web server: -/usr/local/nginx/sbin/nginx - -Run server: -python runserver.py - -=========================================== - -To do full upgrade (as opposed to apt-get upgrade) -sudo aptitude full-upgrade - -=========================================== diff --git a/doc/old/notes.txt b/doc/old/notes.txt deleted file mode 100644 index f8ce2759..00000000 --- a/doc/old/notes.txt +++ /dev/null @@ -1,311 +0,0 @@ -Clone code from git repository: -git clone http://github.com/zsloan/genenetwork.git gene - -Pull from branch in git repository: -git pull origin flask(or whatever the branch is) - -Add all files in current directory branch to be staged to be committed -git add -A - -Git push to my github repository -git push https://github.com/zsloan/genenetwork.git master - -Get the default path being used when a command is typed: -which _______ - -Change default path python looks at when importing -export PYTHONPATH="/home/zas1024/gene/wqflask" - -Get the paths checked when a user types a command: -echo $PATH - -Get all environment variables: -set - - -**** apt-get is an Ubuntu system-wide package manager, while pip is a Python-only system -for python packages **** - -Upgrade system packages for security updates, etc: -apt-get update -apt-get dist-upgrade (can probably be used in place of apt-get upgrade) -apt-get upgrade - -**"apt-cache search" searches for text in the description for various packages, while -"apt-file search" searches for files within the packages** - -Show information about given result item returned by apt-cache search: -apt-cache show _______ - -Search for package with a specified file that can be installed with apt-get -apt-file search _______ - -============================================ - -Check which system packages are installed: -dpkg -l | less - -============================================ - -NEVER USE SUDO WITH PIP - -To fix error "mysql_config not found": -sudo apt-get install libmysqlclient-dev - -Create requirements.txt file: -pip freeze > requirements.txt (or whatever filename) - -Install from requirements.txt: -pip install -r gene/wqflask/requirements.txt -t ve27 - -============================================ - -To get server running: - -!If having seemingly inexplicable problems with imports, make sure I've started the environment! - -Note: Virtual environment is not activated when running as root - -Start up virtual environment: -source ~/ve27/bin/activate - -Deactivate virtual environment: -deactivate (just type deactivate) - -To set WQFLASK_SETTINGS environment variable: -export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) - -zach_settings.py is the local config file with the DB_URI -default_settings.py is the config file with the configuration information that would apply -to any running instance of GeneNetwork - -To change screen environment variable (if man not working or to get color, for example): -export TERM=screen - -To search for commands in history if necessary: -history | grep "(whatever is being searched for)" - -Reload web server: -/usr/sbin/nginx -s reload - -Run server: -python runserver.py - -Run sendmail.py -python send_mail.py - -=========================================== - -UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration - -Enable UFW: -sudo ufw enable -Check status of UFW: -sudo ufw status verbose - -=========================================== - -Update user password: -use mysql; -UPDATE user SET password=PASSWORD('whatever') WHERE user='webqtl'; -FLUSH PRIVILEGES; - -=========================================== - -Redis things - -Restart redis server: -sudo /etc/init.d/redis_6379 stop -sudo /etc/init.d/redis_6379 start - -=========================================== - -Run pylint: -pylint file_name - -For options: -pylint --help - -=========================================== - -Start screen session: - -If "no option -R" or running for first time: -byobu-select-backend -2. screen - -byobu -RD (to start) -control-a then :multiuser on -control-a then :acladd sam - -More info on sharing byobu screen with other users: -http://wiki.networksecuritytoolkit.org/nstwiki/index.php/HowTo_Share_A_Terminal_Session_Using_Screen - -control-a c to create channel - -type: screen -list for sessions -screen -r zas1024/25679.byobu - -or if only one: - -screen -r zas1024/ - -=========================================== - -Start up log: -Go to /tmp and tail -f flask_gn_log - -=========================================== - -Coffeescript Stuff: - -coffee -c (filename) -coffee -c -w . (to watch for changes and recompile in current directory; the "." is for current directory) -coffee --help (for information about setting options) - -=========================================== - -Unset ASKPASS when trying to git push - -unset SSH_ASKPASS - -=========================================== - -htop: Gives information on processes, cpu/memory load, etc -dstat: Also gives various system information, resource usage, etc -df: Reports file system disk space usage - -=========================================== - -ll h*: Finds items in the directory starting with h - -=========================================== - -du -hms * | sort -n : Gives size used by different directories --h: human readable --m: in megabytes (default) --s: summarize - -=========================================== - -rm -rfv - -=========================================== - -cp -a (archive; copies recursively and doesn't follow symbol links) - -i (interactive, prompts before overwrite) - -v (verbose) - -b (backup) - -mv (same as above, but with no -a) - -=========================================== - -Add user: -sudo adduser -Edit /etc/sudoers to give user root privileges - - -=========================================== - -tidyp - Improves/beautifies html code -tidyp -m -i -w 100 index_page.html - -=========================================== - -ps ax - View processes -ps ux (the "u" is for user) - -kill (process #) - -=========================================== - -Installing QTL Reaper - -wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb -tar xvf qtlreaper-1.1.1.tar.gz (to unzip) -python setup.py build -python setup.py install - -=========================================== - -Using Yolk - -Install Yolk: -pip install yolk - -Check packages installed in this virtual environment: -yolk -l - -Checks packages that have updates available: -yolk -U - -=========================================== - -Python stuff: - -Classes should always inherit "object" -To iterate through dictionary items: for X, Y in MyDictionary.items(): - -=========================================== - -Change import path from command line -export PYTHONPATH=~/gene/wqflask (or whatever) - -=========================================== - -Create symbolic link: -ln -s dir_to_link_to linking_dir - -=========================================== - -Reboot server -sudo reboot - -=========================================== - -Using grep: - -grep -ir (search string) (directory) - --i ignores case --r searches recursively through subdirectories of directory you specify - -=========================================== - -Command line arguments: - -Use argparse to deal with command line arguments (instead of argv or optparse) - -=========================================== - -Change owner/group: - -chown zas1024 somefile (change owner of somefile to zas1024) -chown zas1024: somefile (change owner and group of somefile to zas1024) -sudo chown root: somefile (change owner/group of somefile to root) - -Change permissions: - -chmod g-wr (remove write/read privileges for group) -chmod g+w (add write privileges for group) -chmod u+rwx (add read, write, and execute privileges for user) -etc. - -=========================================== - -MySQL Things - - -Create fulltext (MATCH AGAINST syntax) index: -CREATE FULLTEXT INDEX __________ ON Table (Column); - -If <4 character searches not working: -Add ft_min_word_len = 2 (to make te minimum characters 2) to config file -Rebuild relevant tables with fulltext indexes with "REPAIR TABLE _______ QUICK" - -=========================================== - -To delete all .pyc files (for example if there's some issue you're having trouble resolving): -find . -name '*.pyc' -delete -http://stackoverflow.com/questions/785519/how-do-i-remove-all-pyc-files-from-a-project - -- cgit v1.2.3 From 75c7a0828625e3aaf6326e91fadf20df96207fc3 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 16 Jun 2016 17:15:39 +0000 Subject: JOSS: fix typo and amend URL --- doc/joss/2016/paper.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/doc/joss/2016/paper.md b/doc/joss/2016/paper.md index 6b640836..12b3b5d0 100644 --- a/doc/joss/2016/paper.md +++ b/doc/joss/2016/paper.md @@ -28,7 +28,7 @@ authors: - name: Xiang Zhou orcid: 0000-0002-4331-7599 affiliation: University of Michigan - - name: Robert W. WIlliams + - name: Robert W. Williams orcid: 0000-0001-8924-4447 affiliation: University of Tennessee Health Science Center, USA - name: Pjotr Prins @@ -65,7 +65,7 @@ make it easy to install GN locally in a byte reproducible way, including all dependencies and a 2GB MySQL test database (the full database is 160GB and growing), GN is packaged with [GNU Guix](https://www.gnu.org/software/guix/), as described -[here](https://github.com/genenetwork/genenetwork2/blob/staging/doc/README.org). +[here](https://github.com/genenetwork/genenetwork2/blob/master/doc/README.org). GNU Guix deployment makes it feasible to deploy and rebrand GN anywhere. -- cgit v1.2.3