From af7d0bca229f3ebaa80a16d1ce3a2bf1a8abd5df Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 20 Apr 2016 08:37:55 +0000 Subject: [PATCH 023/100] WIP fixing all paths --- README.md | 2 + doc/GUIX-Reproducible-from-source.org | 14 +++- doc/README.org | 23 +++++- etc/default_settings.py | 15 ++-- wqflask/base/data_set.py | 3 +- wqflask/utility/tools.py | 82 ++++++++++------------ wqflask/wqflask/database.py | 5 +- .../wqflask/marker_regression/marker_regression.py | 6 +- wqflask/wqflask/show_trait/show_trait.py | 10 +-- 9 files changed, 94 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 3d95d05f..db495910 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ Elissa Chesler, Jintao Wang, Kenneth Manly, Robert W. Williams, and colleagues. Code and primary web service managed by Dr. Robert W. Williams and the University of Tennessee Health Science Center, Memphis TN, USA. +Join #genenetwork on irc.freenode.net. + Email labwilliams@gmail.com or rwilliams@uthsc.edu Older version available on SourceForge http://sourceforge.net/projects/genenetwork/ diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index 871156ed..4399ea26 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -4,6 +4,7 @@ - [[#introduction][Introduction]] - [[#binary-deployment][Binary deployment]] - [[#from-source-deployment][From source deployment]] + - [[#create-archive][Create archive]] * Introduction @@ -33,10 +34,21 @@ Once that is done we can add the guix-bioinformatics path with : env GUIX_PACKAGE_PATH=../guix-bioinformatics command -such as +So +#+begin_src sh :lang bash #+begin_src sh :lang bash gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A genenetwork genenetwork1 1.0-d622c803b out ../guix-bioinformatics/gn/packages/bioinformatics.scm:163:2 genenetwork2 2.0-9e9475053 out ../guix-bioinformatics/gn/packages/bioinformatics.scm:215:2 #+end_src sh :lang bash + +Install with + +#+begin_src sh :lang bash +gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -i genenetwork2 +#+end_src sh :lang bash + +* Create archive + +: env GUIX_PACKAGE_PATH=../../genenetwork/guix-bioinformatics/ ./pre-inst-env guix archive --export -r genenetwork2 > guix_gn2-2.0-9e9475053.nar diff --git a/doc/README.org b/doc/README.org index 375a5d20..d9730948 100644 --- a/doc/README.org +++ b/doc/README.org @@ -3,6 +3,7 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - [[#binary-deployment-wip][Binary deployment (WIP)]] + - [[#install-genenetwork-server][Install genenetwork server]] - [[#run-mysql-server][Run MySQL server]] - [[#start-the-gn2-server][Start the GN2 server]] - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]] @@ -16,7 +17,27 @@ explain the GeneNetwork deployment system which is based on GNU Guix * Binary deployment (WIP) GN can be deployed either as a binary tarball or as a GNU Guix -package. +package. First install GNU Guix following the instructions of the +[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]] and, update guix with a +'guix pull' and make guix visible in the path. More information +exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]]. + +With guix running you should be able to install python, for example. + +: guix package -i python2 + +This will make python appear in $HOME/.guix-profile/bin/python. Suggested +environment settings can be seen with + +: guix package --search-paths + +** Install genenetwork server + +Fetch the nar file and install it after adding the key with, for +example + +: cat signing-key.pub |.guix-profile/bin/guix archive --authorize +: guix archive --import < guix_gn2-2.0-9e9475053.nar ** Run MySQL server diff --git a/etc/default_settings.py b/etc/default_settings.py index 929bd687..48d3f66b 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -1,13 +1,14 @@ import os +import sys -LOGFILE = "/var/log/genenetwork/wqflask.log" +LOGFILE = "/tmp/genenetwork2.log" # This is needed because Flask turns key errors into a # 400 bad request response with no exception/log TRAP_BAD_REQUEST_ERRORS = True -DB_URI = "mysql://gn2:default@localhost/db_webqtl" -SQLALCHEMY_DATABASE_URI = 'mysql://gn2:default@localhost/db_webqtl' +DB_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" +SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password@localhost/db_webqtl_s' # http://pythonhosted.org/Flask-Security/configuration.html SECURITY_CONFIRMABLE = True @@ -22,7 +23,7 @@ SERVER_PORT = 5003 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' # Path overrides for Genenetwork -HOME=os.environ.get('HOME') -PYLMM_PATH = HOME+"/izip/git/opensource/python/pylmm_gn2/" -PLINK_PATH = HOME+"/.guix-profile/bin" -GEMMA_PATH = HOME+"/.guix-profile/bin" +GENENETWORK_FILES = "../../gn2_data" +PYLMM_RUN = os.popen("which pylmm_redis").read() +PLINK_RUN = os.popen("which plink2").read() +GEMMA_RUN = os.popen("which gemma").read() diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index e37a838f..0e5a3ac1 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -405,10 +405,11 @@ class DatasetGroup(object): #print("Cache not hit") from utility.tools import plink_command - PLINK_PATH,PLINK_COMMAND = plink_command() + PLINK_RUN = plink_command() geno_file_path = webqtlConfig.GENODIR+self.name+".geno" plink_file_path = PLINK_PATH+"/"+self.name+".fam" + # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES if os.path.isfile(plink_file_path): self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 0db195df..c3c9b292 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -9,22 +9,22 @@ import os import sys from wqflask import app -def get_setting(id,default,guess,find_path): +def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with get_valid_path is a function checking whether the - path points to an expected directory an returns the full path e.g. + path points to an expected directory and returns the full path to + the binary command guess = os.environ.get('HOME')+'/pylmm' - get_setting('PYLMM_PATH',default,guess,get_valid_path) + get_setting('PYLMM_PATH',guess) first tries the environment variable in +id+, next gets the Flask - app setting for the same +id+, next tries the path passed in with - +default+ and finally does an educated +guess+. + app setting for the same +id+ and finally does an educated + +guess+. In all, the environment overrides the others, next is the flask - setting, then the default and finally the guess (which is - $HOME/repo). A valid path is returned. If none is resolved an - exception is thrown. + setting, then the guess. A valid path to the binary command is + returned. If none is resolved an exception is thrown. Note that we do not use the system path. This is on purpose because it will mess up controlled (reproducible) deployment. The @@ -33,46 +33,36 @@ def get_setting(id,default,guess,find_path): different settings.py file (or setting the environment). """ - # ---- Check whether environment exists - path = find_path(os.environ.get(id)) - # ---- Check whether setting exists - setting = app.config.get(id) - if not path: - path = find_path(setting) - # ---- Check whether default exists - if not path: - path = find_path(default) - # ---- Guess directory - if not path: - guess = os.environ.get('HOME')+guess - if not setting: - setting = guess - path = find_path(guess) - if not path: - raise Exception(id+' '+setting+' path unknown or faulty (update settings.py?). '+id+' should point to the path') - return path - -def find_command(command,id1,default,guess): - def find_path(path): - """Test for a valid repository""" - if path: - sys.stderr.write("Trying "+id1+" in "+path+"\n") - binary = str.split(command)[0] - if path and os.path.isfile(path+'/'+binary): - return path + def valid(command): + if command: + sys.stderr.write("Found value "+command+"\n") + return command else: - None + return None + + # ---- Check whether environment exists + sys.stderr.write("Looking for "+command_id+"\n") + command = valid(os.environ.get(command_id)) + if not command: + # ---- Check whether setting exists in app + command = valid(app.config.get(command_id)) + if not command: + command = valid(guess) + if not command: + raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path') + return command - path = get_setting(id1,default,guess,find_path) - binary = path+'/'+command - sys.stderr.write("Found "+binary+"\n") - return path,binary +def pylmm_command(guess=None): + return get_setting("PYLMM_RUN",guess) -def pylmm_command(default=None): - return find_command('pylmm_gn2/lmm.py',"PYLMM_PATH",default,'/pylmm2') +def gemma_command(guess=None): + return get_setting("GEMMA_RUN",guess) -def gemma_command(default=None): - return find_command('gemma',"GEMMA_PATH",default,'/gemma') +def plink_command(guess=None): + return get_setting("PLINK_RUN",guess) -def plink_command(default=None): - return find_command('plink2',"PLINK_PATH",default,'/plink') +def flat_files(subdir=None): + base = get_setting("GENENETWORK_FILES") + if subdir: + return base+"/"+subdir + return base diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 159c5d6c..2f544d44 100755 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -24,8 +24,9 @@ def init_db(): # you will have to import them first before calling init_db() #import yourapplication.models import wqflask.model - print("Creating all..") + print("database.py: Creating all model metadata..") Base.metadata.create_all(bind=engine) - print("Done creating all...") + print("database.py: Done creating all model metadata...") + print("Point your browser at http://localhost:5003/") init_db() diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index a657510d..b0f5ed69 100644 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -46,9 +46,9 @@ from wqflask.marker_regression import gemma_mapping # runs at startup, so a missing binary will balk before running the # service -GEMMA_PATH,GEMMA_COMMAND = gemma_command() -PYLMM_PATH,PYLMM_COMMAND = pylmm_command() -PLINK_PATH,PLINK_COMMAND = plink_command() +GEMMA_RUN = gemma_command() +PYLMM_RUN = pylmm_command() +PLINK_RUN = plink_command() # RQTL_PATH,RQTL_COMMAND = rqtl_command() diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 2d4c952a..458e48da 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -16,7 +16,8 @@ from base import webqtlConfig from base import webqtlCaseData from wqflask.show_trait.SampleList import SampleList from utility import webqtlUtil, Plot, Bunch, helper_functions -from utility.tools import pylmm_command, plink_command +# from utility.tools import plink_command +from utility.tools import flat_files from base.trait import GeneralTrait from base import data_set from dbFunction import webqtlDatabaseFunction @@ -24,8 +25,7 @@ from basicStatistics import BasicStatisticsFunctions from pprint import pformat as pf -PYLMM_PATH,PYLMM_COMMAND = pylmm_command() -PLINK_PATH,PLINK_COMMAND = plink_command() +MAPPING_FILES = flat_files("mapping") ############################################### # @@ -162,8 +162,8 @@ class ShowTrait(object): def get_mapping_methods(self): '''Only display mapping methods when the dataset group's genotype file exists''' def check_plink_gemma(): - if (os.path.isfile(PLINK_PATH+"/"+self.dataset.group.name+".bed") and - os.path.isfile(PLINK_PATH+"/"+self.dataset.group.name+".map")): + if (os.path.isfile(MAPPYING_FILES+"/"+self.dataset.group.name+".bed") and + os.path.isfile(MAPPING_FILES+"/"+self.dataset.group.name+".map")): return True else: return False -- cgit v1.2.3