Merge branch 'main' of /home/git/public/gn-machines into main

6 files changed, 64 insertions, 22 deletions

diff --git a/README.org b/README.org
index 892ab23..e4985da 100644
--- a/README.org
+++ b/README.org
@@ -3,7 +3,11 @@ containers.
 
 The git repo lives at https://git.genenetwork.org/gn-machines/
 
-* GeneNetwork development container
+For philosophy and (KISS) incremental development containers, see:
+
+=> https://issues.genenetwork.org/topics/systems/debug-and-developing-code-with-genenetwork-system-container
+
+* GeneNetwork development container (aka CI/CD)
 
 The GeneNetwork development container is currently run on /tux02/. It runs
 continuous integration and continuous deployment services for
@@ -31,6 +35,7 @@ To build and install the container, you will need the
 guix-bioinformatics channel. Once guix-bioinformatics is pulled and
 available, on /tux01/, run
 #+begin_src shell
+
   $ ./virtuoso-deploy.sh
 #+end_src
 
diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm
index a403f21..65e8e22 100644
--- a/genenetwork/services/genenetwork.scm
+++ b/genenetwork/services/genenetwork.scm
@@ -57,6 +57,7 @@
             genenetwork-configuration-port           ; external port
             genenetwork-configuration-gn2-port       ; internal port
             genenetwork-configuration-gn3-port       ; internal port
+            genenetwork-configuration-gn-guile-port  ; aka gn4 internal port (may be external)
             genenetwork-configuration-auth-db        ; RW auth DB
             genenetwork-configuration-xapian-db      ; RO search index, unless you want to regenerate inside VM
             genenetwork-configuration-genotype-files ; RO genotype files
@@ -150,6 +151,8 @@
                   (default "https://genenetwork.org"))
   (sessions-dir gn-uploader-sessions-dir
                 (default "/var/genenetwork/sessions/gn-uploader"))
+  (sqlite-databases-directory gn-uploader-sqlite-databases-directory
+                              (default "/var/genenetwork/sqlite/gn-uploader"))
   (log-level gn-uploader-configuration-log-level
              (default 'warning)
              (sanitize sanitize-log-level)))
@@ -370,6 +373,8 @@ object."
                                        ("GENENETWORK_FILES" ,genotype-files)
                                        ("GN3_LOCAL_URL" ,(string-append "http://localhost:"
                                                                         (number->string gn3-port)))
+                                       ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4
+                                                                        (number->string gn-guile-port) "/" ))
                                        ("GN_SERVER_URL" ,(string-append "https://" server-name "/api3/"))
                                        ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/"))
                                        ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript"))
@@ -386,6 +391,8 @@ object."
                                     (configuration-file-gexp
                                      `(("AUTH_DB" ,auth-db)
                                        ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/"))
+                                       ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4
+                                                                        (number->string gn-guile-port) "/"))
                                        ("DATA_DIR" ,gn3-data-directory)
                                        ("SOURCE_DIR" ,gn-sourcecode-directory)
                                        ("SPARQL_ENDPOINT" ,sparql-endpoint)
@@ -534,6 +541,7 @@ object."
                              (port gn-auth-port))))
              (wsgi-app-module "gn_auth:create_app()")
              (workers 20)
+             (timeout 1200)
              (environment-variables
               (list (environment-variable
                      (name "GN_AUTH_CONF")
@@ -613,7 +621,7 @@ a @code{<genenetwork-configuration>} record."
                                (build-xapian-index-cron-gexp config))
                #:user "root")))
 
-(define (gn-guile-gexp gn-guile-port)
+(define (gn-guile-gexp gn-guile-port) 
   (with-imported-modules '((guix build utils))
     #~(begin
         (use-modules (guix build utils))
@@ -684,7 +692,7 @@ a @code{<genenetwork-configuration>} record."
 
 (define (gn-uploader-activation config)
   (match-record config <gn-uploader-configuration>
-    (secrets data-directory sessions-dir)
+    (secrets data-directory sessions-dir sqlite-databases-directory)
     (with-imported-modules '((guix build utils))
       #~(begin
           (use-modules (guix build utils))
@@ -695,6 +703,8 @@ a @code{<genenetwork-configuration>} record."
                              (passwd:gid (getpw "gunicorn-gn-uploader"))))
                     (append (list #$secrets)
                             (find-files #$sessions-dir
+                                        #:directories? #t)
+                            (find-files #$sqlite-databases-directory
                                         #:directories? #t)))
           ;; Set owner-only permissions on secrets files.
           (for-each (lambda (file)
@@ -711,7 +721,7 @@ a @code{<genenetwork-configuration>} record."
 
 (define (gn-uploader-gunicorn-app config)
   (match-record config <gn-uploader-configuration>
-    (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir)
+    (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory)
     ;; If we mapped only the mysqld.sock socket file, it would break
     ;; when the external mysqld server is restarted.
     (let* ((database-mapping (file-system-mapping
@@ -726,7 +736,8 @@ a @code{<genenetwork-configuration>} record."
                                                                                 "/uploads"))
                                                ("AUTH_SERVER_URL" ,auth-server-url)
                                                ("GN2_SERVER_URL" ,gn2-server-url)
-                                               ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir)))))
+                                               ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir)
+                                               ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db"))))))
            (gn-uploader-profile (profile
                                  (content (package->development-manifest gn-uploader))
                                  (allow-collisions? #t)))
@@ -738,6 +749,7 @@ a @code{<genenetwork-configuration>} record."
                              (port port))))
              (wsgi-app-module "scripts.qcapp_wsgi:app")
              (workers 20)
+             (timeout 1200)
              (environment-variables
               (list (environment-variable
                      (name "UPLOADER_CONF")
@@ -771,6 +783,10 @@ a @code{<genenetwork-configuration>} record."
                              (file-system-mapping
                               (source sessions-dir)
                               (target source)
+                              (writable? #t))
+                             (file-system-mapping
+                              (source sqlite-databases-directory)
+                              (target source)
                               (writable? #t))))
              (extra-cli-arguments
               (list "--log-level"
diff --git a/production-deploy.sh b/production-deploy.sh
index a88fcb8..7cd1cc7 100755
--- a/production-deploy.sh
+++ b/production-deploy.sh
@@ -26,21 +26,23 @@ container_script=$(guix system container \
                         --network \
                         --load-path=. \
                         --verbosity=3 \
-                        --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \
-                        --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \
-                        --share=/export2/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \
-                        --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \
-                        --share=/export2/guix-containers/genenetwork/var/log=/var/log \
-                        --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \
-                        --share=/export2/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \
-                        --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \
-                        --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \
+                        --share=/export/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \
+                        --share=/export/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \
+                        --share=/export/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \
+                        --share=/export/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \
+                        --share=/export/guix-containers/genenetwork/var/log=/var/log \
+                        --share=/export/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \
+                        --share=/export/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \
+                        --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-auth=/var/lib/genenetwork/sqlite/gn-auth \
+                        --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/genenetwork3=/var/lib/genenetwork/sqlite/genenetwork3 \
                         --share=/var/run/mysqld=/run/mysqld \
-			--share=/export/data/gn-docs/ \
-                        --share=/export2/guix-containers/genenetwork/tmp=/opt/gn/tmp \
-                        --expose=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \
-                        --share=/export2/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \
-                        --share=/export2/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \
+			--share=/export/guix-containers/genenetwork/var/lib/gn-docs.git=/var/lib/gn-docs.git \
+                        --share=/export/guix-containers/genenetwork/tmp=/opt/gn/tmp \
+                        --expose=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \
+                        --share=/export/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \
+                        --share=/export/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \
+                        --share=/export/guix-containers/genenetwork/var/lib/genenetwork/uploader=/var/lib/genenetwork/uploader \
+                        --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-uploader=/var/lib/genenetwork/sqlite/gn-uploader \
                         production.scm)
 
 echo $container_script
diff --git a/production.scm b/production.scm
index 9e629f0..ffa75da 100644
--- a/production.scm
+++ b/production.scm
@@ -84,9 +84,22 @@
                              (gn2-secrets "/etc/genenetwork/genenetwork2")
                              (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py")
                              (gn-auth-secrets "/etc/genenetwork/gn-auth")
-                             (auth-db "/var/lib/genenetwork-sqlite/auth.db")
-                             (llm-db-path "/var/lib/genenetwork-gnqa/llm.db")
+                             (auth-db "/var/lib/genenetwork/sqlite/gn-auth/auth.db")
+                             (llm-db-path "/var/lib/genenetwork/sqlite/genenetwork3/llm.db")
                              (gn3-alias-server-port 9800)
                              (gn-tmpdir "/opt/gn/tmp")
+                             (gn-doc-git-checkout "/var/lib/gn-docs.git")
+                             (log-level 'debug)))
+                   (service gn-uploader-service-type
+                            (gn-uploader-configuration
+                             (server-name "uploader.genenetwork.org")
+                             (port 9897)
+                             (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py")
+                             (sql-uri
+                              "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8")
+                             (data-directory "/var/lib/genenetwork/uploader/data")
+                             (auth-server-url "https://auth.genenetwork.org/")
+                             (gn2-server-url "https://genenetwork.org")
+                             (sqlite-databases-directory "/var/lib/genenetwork/sqlite/gn-uploader")
                              (log-level 'debug)))
                    %base-services)))
diff --git a/public-sparql-deploy.sh b/public-sparql-deploy.sh
index bc4348d..bd8b938 100755
--- a/public-sparql-deploy.sh
+++ b/public-sparql-deploy.sh
@@ -26,6 +26,7 @@ container_script=$(guix system container \
 			--share=/export/guix-containers/public-sparql/tmp=/tmp \
 			--share=/export/guix-containers/public-sparql/var/log=/var/log \
 			--share=/export/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \
+                        --share=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \
                         public-sparql.scm)
 
 echo $container_script
diff --git a/public-sparql.scm b/public-sparql.scm
index 4f5ad13..4603cec 100644
--- a/public-sparql.scm
+++ b/public-sparql.scm
@@ -20,6 +20,7 @@
 (use-modules (gnu)
              (gn services databases)
              (gnu services web)
+             ((gnu packages admin) #:select (shepherd))
              (forge nginx)
              (forge socket))
 
@@ -50,13 +51,17 @@ SPARQL endpoint is listening on."
                (targets (list "/dev/sdX"))))
   (file-systems %base-file-systems)
   (users %base-user-accounts)
+  (sudoers-file
+   (mixed-text-file "sudoers"
+                    "@include " %sudoers-specification
+                    "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n"))
   (packages %base-packages)
   (services (cons* (service virtuoso-service-type
                             (virtuoso-configuration
                              (server-port %virtuoso-port)
                              (http-server-port %sparql-port)
 			     (number-of-buffers 4000000)
-			     (dirs-allowed (list "/var/lib/virtuoso"))
+			     (dirs-allowed (list "/export/data/virtuoso"))
 			     (maximum-dirty-buffers 3000000)
                              (database-file "/var/lib/virtuoso/public-virtuoso.db")
                              (transaction-file "/var/lib/virtuoso/public-virtuoso.trx")))