diff options
| -rw-r--r-- | genenetwork-development.scm | 288 |
1 files changed, 150 insertions, 138 deletions
diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 3afa63d..08e50eb 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -123,7 +123,9 @@ be imported into G-expressions." (gn-libs-repository genenetwork-configuration-gn-libs-repository (default "https://git.genenetwork.org/gn-libs")) (gn-guile-repository genenetwork-configuration-gn-libs-repository - (default "https://git.genenetwork.org/gn-guile")) + (default "https://git.genenetwork.org/gn-guile")) + (repositories-checkout-directory genenetwork-repositories-checkout-directory + (default "/home/genenetwork")) (gn2-port genenetwork-configuration-gn2-port (default 8082)) (gn3-port genenetwork-configuration-gn3-port @@ -491,7 +493,7 @@ genenetwork3 source from the latest commit of @var{project}." "Return a G-expression that runs the latest genenetwork2 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files gn-guile-port) + (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files gn-guile-port repositories-checkout-directory) (with-packages (list coreutils git-minimal gunicorn nss-certs) (with-imported-modules '((guix build utils)) #~(begin @@ -512,77 +514,81 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "TERM" "xterm-256color") ;; Clone the latest genenetwork2 ;; repositories. - (with-directory-excursion - "/home/genenetwork" - (when (file-exists? "/home/genenetwork/genenetwork2") - (delete-file-recursively "/home/genenetwork/genenetwork2")) - (invoke "git" "clone" "--depth" "1" #$gn2-repository)) - - ;; Override the genenetwork3 used by genenetwork2. - (setenv "GN3_PYTHONPATH" "/home/genenetwork/genenetwork3") - ;; Set other environment variables required by - ;; genenetwork2. - (setenv "GN2_PROFILE" #$(profile - (content (package->development-manifest genenetwork2)) - (allow-collisions? #t))) - (setenv "REQUESTS_CA_BUNDLE" (string-append - (getenv "GN2_PROFILE") - "/etc/ssl/certs/ca-certificates.crt")) - (setenv "PYTHONPATH" (string-append - "/home/genenetwork/gn-libs:" - (getenv "GN3_PYTHONPATH") - ":" - (string-append - (getenv "GN2_PROFILE") - "/lib/python3.11/site-packages"))) - (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) - (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) - (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) - (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) - (setenv "GENENETWORK_FILES" #$genotype-files) - (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) - (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) - (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) - (setenv "HOME" "/home/genenetwork") - (setenv - "GN2_SETTINGS" - #$(mixed-text-file "gn2.conf" - "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" - "AI_SEARCH_ENABLED=True\n" - "TEST_FEATURE_SWITCH=True\n" - "GN3_LOCAL_URL=\"" - (string-append "http://localhost:" - (number->string gn3-port)) - "\"\n" - "GN_GUILE_SERVER_URL=\"" - (string-append "http://localhost:" - (number->string gn-guile-port)) - "\"\n" - "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" - "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" - "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" - "SSL_PRIVATE_KEY=\"" gn2-secrets "/gn2-ssl-private-key.pem\"\n" - "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) - - ;; Start genenetwork2. - (with-directory-excursion - "/home/genenetwork/genenetwork2" - (show-head-commit) - (invoke #$(file-append gunicorn "/bin/gunicorn") - "--bind" (string-append "0.0.0.0:" (number->string #$gn2-port)) - "--workers" "20" - "--keep-alive" "6000" - "--max-requests" "100" - "--max-requests-jitter" "30" - "--timeout" "1200" - "--log-level" "debug" - "gn2.wsgi"))))))) + (let ((gn2-checkout (string-append #$repositories-checkout-directory "/genenetwork2")) + (gn3-checkout (string-append #$repositories-checkout-directory "/genenetwork3")) + (gn-libs-checkout (string-append #$repositories-checkout-directory "/gn-libs"))) + (with-directory-excursion + #$repositories-checkout-directory + (when (file-exists? gn2-checkout) + (delete-file-recursively gn2-checkout)) + (invoke "git" "clone" "--depth" "1" #$gn2-repository)) + + ;; Override the genenetwork3 used by genenetwork2. + (setenv "GN3_PYTHONPATH" gn3-checkout) + ;; Set other environment variables required by + ;; genenetwork2. + (setenv "GN2_PROFILE" #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN2_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "PYTHONPATH" (string-append + gn-libs-checkout + ":" + (getenv "GN3_PYTHONPATH") + ":" + (string-append + (getenv "GN2_PROFILE") + "/lib/python3.11/site-packages"))) + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" #$genotype-files) + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/home/genenetwork") + (setenv + "GN2_SETTINGS" + #$(mixed-text-file "gn2.conf" + "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"" + (string-append "http://localhost:" + (number->string gn3-port)) + "\"\n" + "GN_GUILE_SERVER_URL=\"" + (string-append "http://localhost:" + (number->string gn-guile-port)) + "\"\n" + "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SSL_PRIVATE_KEY=\"" gn2-secrets "/gn2-ssl-private-key.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) + + ;; Start genenetwork2. + (with-directory-excursion + gn2-checkout + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "--bind" (string-append "0.0.0.0:" (number->string #$gn2-port)) + "--workers" "20" + "--keep-alive" "6000" + "--max-requests" "100" + "--max-requests-jitter" "30" + "--timeout" "1200" + "--log-level" "debug" + "gn2.wsgi")))))))) (define (genenetwork3-cd-gexp config) "Return a G-expression that runs the latest genenetwork3 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path lmdb-data-path gn-libs-repository) + (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path lmdb-data-path gn-libs-repository repositories-checkout-directory) (with-manifest (package->development-manifest genenetwork3) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -624,35 +630,37 @@ server described by CONFIG, a <genenetwork-configuration> object." (getenv "GN3_PROFILE") "/bin/Rscript")) - (with-directory-excursion - "/home/genenetwork" - ;; Clone the latest genenetwork3 repository. - (when (file-exists? "genenetwork3") - (delete-file-recursively "genenetwork3")) - (invoke "git" "clone" "--depth" "1" #$gn3-repository) - (when (file-exists? "gn-libs") - (delete-file-recursively "gn-libs")) - (invoke "git" "clone" "--depth" "1" #$gn-libs-repository)) - (setenv "PYTHONPATH" "/home/genenetwork/gn-libs:$PYTHONPATH") - (with-directory-excursion - "/home/genenetwork/genenetwork3" - (show-head-commit) - ;; Run genenetwork3. - (invoke #$(file-append gunicorn "/bin/gunicorn") - "-b" #$(string-append "localhost:" (number->string gn3-port)) - "--workers" "8" - "gn3.app:create_app()" - ;; gunicorn's default 30 second timeout is - ;; insufficient for Fahamu AI endpoints and - ;; results in worker timeout errors. - "--timeout" "1200" - "--log-level" "debug")))))))) + (let ((gn3-checkout (string-append #$repositories-checkout-directory "/genenetwork3")) + (gn-libs-checkout (string-append #$repositories-checkout-directory "/gn-libs"))) + (with-directory-excursion + #$repositories-checkout-directory + ;; Clone the latest genenetwork3 repository. + (when (file-exists? "genenetwork3") + (delete-file-recursively "genenetwork3")) + (invoke "git" "clone" "--depth" "1" #$gn3-repository) + (when (file-exists? "gn-libs") + (delete-file-recursively "gn-libs")) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository)) + (setenv "PYTHONPATH" (string-append gn-libs-checkout ":$PYTHONPATH")) + (with-directory-excursion + gn3-checkout + (show-head-commit) + ;; Run genenetwork3. + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn3-port)) + "--workers" "8" + "gn3.app:create_app()" + ;; gunicorn's default 30 second timeout is + ;; insufficient for Fahamu AI endpoints and + ;; results in worker timeout errors. + "--timeout" "1200" + "--log-level" "debug"))))))))) (define (gn-auth-cd-gexp config) "Return a G-expression that runs the latest gn-auth development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn-auth-repository gn-libs-repository gn-auth-port auth-db-path gn-auth-secrets) + (gn-auth-repository gn-libs-repository gn-auth-port auth-db-path gn-auth-secrets repositories-checkout-directory) (with-manifest (package->development-manifest gn-auth) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -673,47 +681,50 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) - (with-directory-excursion - "/home/genenetwork/" - ;; Clone the latest gn-auth repository. - (when (file-exists? "/home/genenetwork/gn-auth") - (delete-file-recursively "/home/genenetwork/gn-auth")) - (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) - (when (file-exists? "/home/genenetwork/gn-libs") - (delete-file-recursively "/home/genenetwork/gn-libs")) - (invoke "git" "clone" "--depth" "1" #$gn-libs-repository)) - - ;; Configure gn-auth. - (setenv "GN_AUTH_PROFILE" #$(profile - (content (package->development-manifest gn-auth)) - (allow-collisions? #t))) - (setenv "REQUESTS_CA_BUNDLE" (string-append - (getenv "GN_AUTH_PROFILE") - "/etc/ssl/certs/ca-certificates.crt")) - (setenv "PYTHONPATH" (string-append - "/home/genenetwork/gn-libs:" - (string-append - (getenv "GN_AUTH_PROFILE") - ":" - "/lib/python3.11/site-packages"))) - (setenv "GN_AUTH_CONF" - #$(mixed-text-file "gn-auth.conf" - "AUTH_DB=\"" auth-db-path "\"\n" - "GN_AUTH_SECRETS=\"" gn-auth-secrets "/gn-auth-secrets.py\"\n" - "CLIENTS_SSL_PUBLIC_KEYS_DIR=\"" gn-auth-secrets "/clients-public-keys\"\n" - "SSL_PRIVATE_KEY=\"" gn-auth-secrets "/gn-auth-ssl-private-key.pem\"\n")) - (setenv "HOME" "/tmp") - (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") - ;; Run gn-auth. - (with-directory-excursion "/home/genenetwork/gn-auth" - (show-head-commit) - (invoke #$(file-append gunicorn "/bin/gunicorn") - "-b" #$(string-append "localhost:" (number->string gn-auth-port)) - "--workers" "8" - "--log-level" "debug" - "gn_auth.wsgi:app")))))))) - -(define (gn-guile-gexp gn-guile-port) + (let ((gn-auth-checkout (string-append #$repositories-checkout-directory "/gn-auth")) + (gn-libs-checkout (string-append #$repositories-checkout-directory "/gn-libs"))) + (with-directory-excursion + #$repositories-checkout-directory + ;; Clone the latest gn-auth repository. + (when (file-exists? gn-auth-checkout) + (delete-file-recursively gn-auth-checkout)) + (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) + (when (file-exists? gn-libs-checkout) + (delete-file-recursively gn-libs-checkout)) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository)) + + ;; Configure gn-auth. + (setenv "GN_AUTH_PROFILE" #$(profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN_AUTH_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "PYTHONPATH" (string-append + gn-libs-checkout + ":" + (string-append + (getenv "GN_AUTH_PROFILE") + ":" + "/lib/python3.11/site-packages"))) + (setenv "GN_AUTH_CONF" + #$(mixed-text-file "gn-auth.conf" + "AUTH_DB=\"" auth-db-path "\"\n" + "GN_AUTH_SECRETS=\"" gn-auth-secrets "/gn-auth-secrets.py\"\n" + "CLIENTS_SSL_PUBLIC_KEYS_DIR=\"" gn-auth-secrets "/clients-public-keys\"\n" + "SSL_PRIVATE_KEY=\"" gn-auth-secrets "/gn-auth-ssl-private-key.pem\"\n")) + (setenv "HOME" "/tmp") + (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") + ;; Run gn-auth. + (with-directory-excursion gn-auth-checkout + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn-auth-port)) + "--workers" "8" + "--log-level" "debug" + "gn_auth.wsgi:app"))))))))) + +(define (gn-guile-gexp gn-guile-port repositories-checkout-directory) (with-packages (list coreutils git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -752,10 +763,11 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "REQUESTS_CA_BUNDLE" (getenv "SSL_CERT_FILE")) (setenv "SPARQL-ENDPOINT" "http://localhost:9082/sparql/") (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) - (let ((current-repo-path "/home/genenetwork/gn-docs")) + (let ((current-repo-path (string-append #$repositories-checkout-directory "/gn-docs")) + (gn-guile-checkout (string-append #$repositories-checkout-directory "/gn-guile"))) (setenv "CURRENT_REPO_PATH" current-repo-path) (with-directory-excursion - "/home/genenetwork" + #$repositories-checkout-directory ;; All edits go to the current-repo-path; so we need it to ;; be persistent. (unless (file-exists? current-repo-path) @@ -768,7 +780,7 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "clone" "--depth" "1" "https://git.genenetwork.org/gn-guile" "gn-guile") (with-directory-excursion "gn-guile" - (setenv "HOME" "/home/genenetwork/gn-guile") + (setenv "HOME" gn-guile-checkout) ;; why? (show-head-commit) (invoke #$(file-append gn-guile "/bin/gn-guile") "--port" @@ -784,7 +796,7 @@ server described by CONFIG, a <genenetwork-configuration> object." "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path lmdb-data-path gn-doc-git-checkout gn-guile-port) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path lmdb-data-path gn-doc-git-checkout gn-guile-port repositories-checkout-directory) (list (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -800,7 +812,7 @@ described by CONFIG, a <genenetwork-configuration> object." #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-guile" - (gn-guile-gexp gn-guile-port)) + (gn-guile-gexp gn-guile-port repositories-checkout-directory)) #:name "gn-guile-pola-wrapper" #:preserved-environment-variables (map first gn-guile-settings) |