[PATCH 023/100] WIP fixing all paths

9 files changed, 94 insertions, 66 deletions

diff --git a/README.md b/README.md
index 3d95d05f..db495910 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,8 @@ Elissa Chesler, Jintao Wang, Kenneth Manly, Robert W. Williams, and colleagues.
 Code and primary web service managed by Dr. Robert W. Williams and the University of Tennessee Health Science Center,
 Memphis TN, USA. 
 
+Join #genenetwork on irc.freenode.net.
+
 Email labwilliams@gmail.com or rwilliams@uthsc.edu
 
 Older version available on SourceForge  http://sourceforge.net/projects/genenetwork/
diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org
index 871156ed..4399ea26 100644
--- a/doc/GUIX-Reproducible-from-source.org
+++ b/doc/GUIX-Reproducible-from-source.org
@@ -4,6 +4,7 @@
  - [[#introduction][Introduction]]
  - [[#binary-deployment][Binary deployment]]
  - [[#from-source-deployment][From source deployment]]
+ - [[#create-archive][Create archive]]
 
 * Introduction
 
@@ -33,10 +34,21 @@ Once that is done we can add the guix-bioinformatics path with
 
 : env GUIX_PACKAGE_PATH=../guix-bioinformatics command
 
-such as 
+So
 
 #+begin_src sh   :lang bash
+#+begin_src sh   :lang bash
 gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A genenetwork
 genenetwork1    1.0-d622c803b   out     ../guix-bioinformatics/gn/packages/bioinformatics.scm:163:2
 genenetwork2    2.0-9e9475053   out     ../guix-bioinformatics/gn/packages/bioinformatics.scm:215:2
 #+end_src sh   :lang bash
+
+Install with
+
+#+begin_src sh   :lang bash
+gn-stable-guix$ env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -i genenetwork2
+#+end_src sh   :lang bash
+
+* Create archive
+
+: env GUIX_PACKAGE_PATH=../../genenetwork/guix-bioinformatics/ ./pre-inst-env guix archive --export -r genenetwork2 > guix_gn2-2.0-9e9475053.nar
diff --git a/doc/README.org b/doc/README.org
index 375a5d20..d9730948 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -3,6 +3,7 @@
 * Table of Contents                                                     :TOC:
  - [[#introduction][Introduction]]
  - [[#binary-deployment-wip][Binary deployment (WIP)]]
+   - [[#install-genenetwork-server][Install genenetwork server]]
    - [[#run-mysql-server][Run MySQL server]]
    - [[#start-the-gn2-server][Start the GN2 server]]
  - [[#source-deployment-and-other-information-on-reproducibility][Source deployment and other information on reproducibility]]
@@ -16,7 +17,27 @@ explain the GeneNetwork deployment system which is based on GNU Guix
 * Binary deployment (WIP)
 
 GN can be deployed either as a binary tarball or as a GNU Guix
-package.
+package. First install GNU Guix following the instructions of the
+[[https://www.gnu.org/software/guix/manual/html_node/Binary-Installation.html#Binary-Installation][binary installation]] using a tar ball from [[https://www.gnu.org/software/guix/download/][here]] and, update guix with a 
+'guix pull' and make guix visible in the path.  More information
+exists also in my [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org][guix-notes]].
+
+With guix running you should be able to install python, for example.
+
+: guix package -i python2
+
+This will make python appear in $HOME/.guix-profile/bin/python. Suggested
+environment settings can be seen with
+
+: guix package --search-paths
+
+** Install genenetwork server
+
+Fetch the nar file and install it after adding the key with, for
+example
+
+: cat signing-key.pub |.guix-profile/bin/guix archive --authorize
+: guix archive --import < guix_gn2-2.0-9e9475053.nar
 
 ** Run MySQL server
 
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 929bd687..48d3f66b 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -1,13 +1,14 @@
 import os
+import sys
 
-LOGFILE = "/var/log/genenetwork/wqflask.log"
+LOGFILE = "/tmp/genenetwork2.log"
 
 # This is needed because Flask turns key errors into a
 # 400 bad request response with no exception/log
 TRAP_BAD_REQUEST_ERRORS = True
 
-DB_URI = "mysql://gn2:default@localhost/db_webqtl"
-SQLALCHEMY_DATABASE_URI = 'mysql://gn2:default@localhost/db_webqtl'
+DB_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s"
+SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password@localhost/db_webqtl_s'
 
 # http://pythonhosted.org/Flask-Security/configuration.html
 SECURITY_CONFIRMABLE = True
@@ -22,7 +23,7 @@ SERVER_PORT = 5003
 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc'
 
 # Path overrides for Genenetwork
-HOME=os.environ.get('HOME')
-PYLMM_PATH = HOME+"/izip/git/opensource/python/pylmm_gn2/"
-PLINK_PATH = HOME+"/.guix-profile/bin"
-GEMMA_PATH = HOME+"/.guix-profile/bin"
+GENENETWORK_FILES = "../../gn2_data"
+PYLMM_RUN = os.popen("which pylmm_redis").read()
+PLINK_RUN = os.popen("which plink2").read()
+GEMMA_RUN = os.popen("which gemma").read()
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index e37a838f..0e5a3ac1 100755
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -405,10 +405,11 @@ class DatasetGroup(object):
             #print("Cache not hit")
 
             from utility.tools import plink_command
-            PLINK_PATH,PLINK_COMMAND = plink_command()
+            PLINK_RUN = plink_command()
 
             geno_file_path = webqtlConfig.GENODIR+self.name+".geno"
             plink_file_path = PLINK_PATH+"/"+self.name+".fam"
+            # @FIXME PJOTR/ZACH: .fam files should go into FLATFILES
 
             if os.path.isfile(plink_file_path):
                 self.samplelist = get_group_samplelists.get_samplelist("plink", plink_file_path)
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 0db195df..c3c9b292 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -9,22 +9,22 @@ import os
 import sys
 from wqflask import app
 
-def get_setting(id,default,guess,find_path):
+def get_setting(command_id,guess=None):
     """Resolve a setting from the environment or the global settings in
     app.config, with get_valid_path is a function checking whether the
-    path points to an expected directory an returns the full path e.g.
+    path points to an expected directory and returns the full path to
+    the binary command
 
       guess = os.environ.get('HOME')+'/pylmm'
-      get_setting('PYLMM_PATH',default,guess,get_valid_path)
+      get_setting('PYLMM_PATH',guess)
 
     first tries the environment variable in +id+, next gets the Flask
-    app setting for the same +id+, next tries the path passed in with
-    +default+ and finally does an educated +guess+.
+    app setting for the same +id+ and finally does an educated
+    +guess+.
 
     In all, the environment overrides the others, next is the flask
-    setting, then the default and finally the guess (which is
-    $HOME/repo). A valid path is returned. If none is resolved an
-    exception is thrown.
+    setting, then the guess. A valid path to the binary command is
+    returned. If none is resolved an exception is thrown.
 
     Note that we do not use the system path. This is on purpose
     because it will mess up controlled (reproducible) deployment. The
@@ -33,46 +33,36 @@ def get_setting(id,default,guess,find_path):
     different settings.py file (or setting the environment).
 
     """
-    # ---- Check whether environment exists
-    path = find_path(os.environ.get(id))
-    # ---- Check whether setting exists
-    setting = app.config.get(id)
-    if not path:
-        path = find_path(setting)
-    # ---- Check whether default exists
-    if not path:
-        path = find_path(default)
-    # ---- Guess directory
-    if not path:
-        guess = os.environ.get('HOME')+guess
-        if not setting:
-            setting = guess
-        path = find_path(guess)
-    if not path:
-        raise Exception(id+' '+setting+' path unknown or faulty (update settings.py?). '+id+' should point to the path')
-    return path
-
-def find_command(command,id1,default,guess):
-    def find_path(path):
-        """Test for a valid repository"""
-        if path:
-            sys.stderr.write("Trying "+id1+" in "+path+"\n")
-        binary = str.split(command)[0]
-        if path and os.path.isfile(path+'/'+binary):
-            return path
+    def valid(command):
+        if command:
+            sys.stderr.write("Found value "+command+"\n")
+            return command
         else:
-            None
+            return None
+    
+    # ---- Check whether environment exists
+    sys.stderr.write("Looking for "+command_id+"\n")
+    command = valid(os.environ.get(command_id))
+    if not command:
+        # ---- Check whether setting exists in app
+        command = valid(app.config.get(command_id))
+        if not command:
+            command = valid(guess)
+            if not command:
+                raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path')
+    return command
 
-    path = get_setting(id1,default,guess,find_path)
-    binary = path+'/'+command
-    sys.stderr.write("Found "+binary+"\n")
-    return path,binary
+def pylmm_command(guess=None):
+    return get_setting("PYLMM_RUN",guess)
 
-def pylmm_command(default=None):
-    return find_command('pylmm_gn2/lmm.py',"PYLMM_PATH",default,'/pylmm2')
+def gemma_command(guess=None):
+    return get_setting("GEMMA_RUN",guess)
 
-def gemma_command(default=None):
-    return find_command('gemma',"GEMMA_PATH",default,'/gemma')
+def plink_command(guess=None):
+    return get_setting("PLINK_RUN",guess)
 
-def plink_command(default=None):
-    return find_command('plink2',"PLINK_PATH",default,'/plink')
+def flat_files(subdir=None):
+    base = get_setting("GENENETWORK_FILES")
+    if subdir:
+        return base+"/"+subdir
+    return base
diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py
index 159c5d6c..2f544d44 100755
--- a/wqflask/wqflask/database.py
+++ b/wqflask/wqflask/database.py
@@ -24,8 +24,9 @@ def init_db():
     # you will have to import them first before calling init_db()
     #import yourapplication.models
     import wqflask.model
-    print("Creating all..")
+    print("database.py: Creating all model metadata..")
     Base.metadata.create_all(bind=engine)
-    print("Done creating all...")
+    print("database.py: Done creating all model metadata...")
+    print("Point your browser at http://localhost:5003/")
     
 init_db()
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index a657510d..b0f5ed69 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -46,9 +46,9 @@ from wqflask.marker_regression import gemma_mapping
 # runs at startup, so a missing binary will balk before running the
 # service
 
-GEMMA_PATH,GEMMA_COMMAND = gemma_command()
-PYLMM_PATH,PYLMM_COMMAND = pylmm_command()
-PLINK_PATH,PLINK_COMMAND = plink_command()
+GEMMA_RUN = gemma_command()
+PYLMM_RUN = pylmm_command()
+PLINK_RUN = plink_command()
 # RQTL_PATH,RQTL_COMMAND   = rqtl_command()
 
 
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index 2d4c952a..458e48da 100755
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -16,7 +16,8 @@ from base import webqtlConfig
 from base import webqtlCaseData
 from wqflask.show_trait.SampleList import SampleList
 from utility import webqtlUtil, Plot, Bunch, helper_functions
-from utility.tools import pylmm_command, plink_command
+# from utility.tools import plink_command
+from utility.tools import flat_files
 from base.trait import GeneralTrait
 from base import data_set
 from dbFunction import webqtlDatabaseFunction
@@ -24,8 +25,7 @@ from basicStatistics import BasicStatisticsFunctions
 
 from pprint import pformat as pf
 
-PYLMM_PATH,PYLMM_COMMAND = pylmm_command()
-PLINK_PATH,PLINK_COMMAND = plink_command()
+MAPPING_FILES = flat_files("mapping")
 
 ###############################################
 #
@@ -162,8 +162,8 @@ class ShowTrait(object):
     def get_mapping_methods(self):
         '''Only display mapping methods when the dataset group's genotype file exists'''
         def check_plink_gemma():
-            if (os.path.isfile(PLINK_PATH+"/"+self.dataset.group.name+".bed") and
-                os.path.isfile(PLINK_PATH+"/"+self.dataset.group.name+".map")):
+            if (os.path.isfile(MAPPYING_FILES+"/"+self.dataset.group.name+".bed") and
+                os.path.isfile(MAPPING_FILES+"/"+self.dataset.group.name+".map")):
                 return True
             else:
                 return False