diff options
| -rw-r--r-- | genenetwork-local-container.scm | 302 | ||||
| -rwxr-xr-x | genenetwork-local-container.sh | 34 |
2 files changed, 236 insertions, 100 deletions
diff --git a/genenetwork-local-container.scm b/genenetwork-local-container.scm index 3942383..8dc02c7 100644 --- a/genenetwork-local-container.scm +++ b/genenetwork-local-container.scm @@ -23,13 +23,13 @@ (use-modules (gnu) (genenetwork services genenetwork) (gnu build linux-container) - ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs)) + ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs gn-uploader)) ((gnu packages admin) #:select (shepherd shadow)) ((gnu packages certs) #:select (nss-certs)) ((gnu packages bash) #:select (bash)) ((gnu packages check) #:select (python-pylint python-hypothesis)) ((gnu packages python-check) #:select (python-mypy python-mypy-extensions)) - ((gnu packages python-web) #:select (gunicorn python-flask)) + ((gnu packages python-web) #:select (gunicorn python-flask python-flask-cors)) ((gnu packages version-control) #:select (git-minimal)) ((gn packages guile) #:select (gn-guile)) (gn services databases) @@ -76,57 +76,69 @@ (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout - (default "/var/lib/gn-docs"))) + (default "/var/lib/gn-docs")) + (sql-uri genenetwork-configuration-sql-uri + (default + "mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock&charset=utf8")) + (gn-uploader-data-directory genenetwork-configuration-gn-uploader-data-directory + (default "/var/lib/gn-uploader/data")) + (gn-uploader-port genenetwork-configuration-gn-uploader-port + (default 8085)) + (gn-uploader-secrets genenetwork-configuration-gn-uploader-configuration-secrets + (default "/etc/genenetwork/conf/gn-uploader/secrets.py")) + (gn-uploader-sessions-dir genenetwork-configuration-gn-uploader-sessions-dir + (default "/var/lib/gn-uploader/sessions"))) + (define (genenetwork2-gexp config) "Return a G-expression that runs the latest genenetwork2 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn2-secrets gn3-port gn-auth-port genotype-files) - (with-packages - (list coreutils git-minimal gunicorn nss-certs) - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils) - (ice-9 ftw) - (ice-9 match)) + (sql-uri gn2-port gn2-secrets gn3-port gn-auth-port genotype-files) + (with-packages + (list coreutils git-minimal gunicorn nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 ftw) + (ice-9 match)) - ;; Override the genenetwork3 used by genenetwork2. - (setenv "GN3_PYTHONPATH" "/genenetwork3") - (setenv "GN2_PROFILE" #$(profile - (content (package->development-manifest genenetwork2)) - (allow-collisions? #t))) - (unless (file-exists? "/etc/genenetwork/conf/gn2.conf") - (with-output-to-file "/etc/genenetwork/conf/gn2.conf" - (lambda () - (display #$(string-append - "GN2_SECRETS=\"" gn2-secrets "\"\n" - "AI_SEARCH_ENABLED=True\n" - "TEST_FEATURE_SWITCH=True\n" - "GN3_LOCAL_URL=\"http://localhost:" (number->string gn3-port) "\"\n" - "GN_SERVER_URL=\"http://localhost:" (number->string gn3-port) "/api/\"\n" - "AUTH_SERVER_URL=\"http://localhost:" (number->string gn-auth-port) "\"\n" - "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock&charset=utf8\"\n" - "SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn2/private.pem\"\n" - "AUTH_SERVER_SSL_PUBLIC_KEY=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/gn-auth.pem\"\n")) - ;; We actually set here twice so that we can refer to it - ;; when running things manually. - (display "GN2_PROFILE=\"") - (display #$(file-append (profile - (content (package->development-manifest genenetwork2)) - (allow-collisions? #t)) - "\"\n"))))) - (setenv "GN2_SETTINGS" "/etc/genenetwork/conf/gn2.conf") - ;; Start genenetwork2. - (with-directory-excursion "/genenetwork2" - (invoke #$(file-append bash "/bin/sh") - "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-dev"))))))) + ;; Override the genenetwork3 used by genenetwork2. + (setenv "GN3_PYTHONPATH" "/genenetwork3") + (setenv "GN2_PROFILE" #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t))) + (unless (file-exists? "/etc/genenetwork/conf/gn2.conf") + (with-output-to-file "/etc/genenetwork/conf/gn2.conf" + (lambda () + (display #$(string-append + "GN2_SECRETS=\"" gn2-secrets "\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"http://localhost:" (number->string gn3-port) "\"\n" + "GN_SERVER_URL=\"http://localhost:" (number->string gn3-port) "/api/\"\n" + "AUTH_SERVER_URL=\"http://localhost:" (number->string gn-auth-port) "\"\n" + "SQL_URI=\"" sql-uri "\"\n" + "SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn2/private.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/gn-auth.pem\"\n")) + ;; We actually set here twice so that we can refer to it + ;; when running things manually. + (display "GN2_PROFILE=\"") + (display #$(file-append (profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t)) + "\"\n"))))) + (setenv "GN2_SETTINGS" "/etc/genenetwork/conf/gn2.conf") + ;; Start genenetwork2. + (with-directory-excursion "/genenetwork2" + (invoke #$(file-append bash "/bin/sh") + "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-dev"))))))) (define (genenetwork3-gexp config) "Return a G-expression that runs the latest genenetwork3 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn3-port gn3-secrets gn-auth-port sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) + (sql-uri gn3-port gn3-secrets gn-auth-port sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) (with-manifest (package->development-manifest genenetwork3) (with-imported-modules '((guix build utils)) #~(begin @@ -161,7 +173,7 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "GN3_CONF" "/etc/genenetwork/conf/gn3.conf") (setenv "GN3_SECRETS" #$gn3-secrets) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock") + (setenv "SQL_URI" sql-uri) (setenv "FLASK_ENV" "development") (setenv "FLASK_DEBUG" "1") @@ -175,7 +187,7 @@ server described by CONFIG, a <genenetwork-configuration> object." "Return a G-expression that runs the latest gn-auth development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn-auth-port auth-db-path gn-auth-secrets) + (sql-uri gn-auth-port auth-db-path gn-auth-secrets) (with-manifest (package->development-manifest gn-auth) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -183,18 +195,18 @@ server described by CONFIG, a <genenetwork-configuration> object." (use-modules (guix build utils)) ;; Configure gn-auth. (unless (file-exists? "/etc/genenetwork/conf/gn-auth.conf") - (with-output-to-file "/etc/genenetwork/conf/gn-auth.conf" - (lambda () - (display #$(string-append - "LOGLEVEL=\"DEBUG\"\n" - "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock\"\n" - "AUTH_DB=\"" auth-db-path "\"\n" - "GN_AUTH_SECRETS=\"" gn-auth-secrets "\"\n")) - (display "GN_AUTH_PROFILE=\"") - (display #$(file-append (profile - (content (package->development-manifest gn-auth)) - (allow-collisions? #t)) - "\"\n"))))) + (with-output-to-file "/etc/genenetwork/conf/gn-auth.conf" + (lambda () + (display #$(string-append + "LOGLEVEL=\"DEBUG\"\n" + "SQL_URI=\"" sql-uri "\"\n" + "AUTH_DB=\"" auth-db-path "\"\n" + "GN_AUTH_SECRETS=\"" gn-auth-secrets "\"\n")) + (display "GN_AUTH_PROFILE=\"") + (display #$(file-append (profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t)) + "\"\n"))))) (setenv "GN_AUTH_CONF" "/etc/genenetwork/conf/gn-auth.conf") (setenv "HOME" "/tmp") (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") @@ -205,43 +217,104 @@ server described by CONFIG, a <genenetwork-configuration> object." "--workers" "8" "gn_auth.wsgi:app")))))))) -(define (genenetwork-activation config) +(define (gn-uploader-gexp config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn3-port gn-auth-port auth-db-path gn-auth-secrets gn-doc-git-checkout) - (with-imported-modules '((guix build utils)) + (sql-uri gn-uploader-port gn-uploader-data-directory + gn-uploader-secrets gn-auth-port + gn2-port gn-uploader-sessions-dir) + (with-manifest + (package->development-manifest gn-uploader) + (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils) (ice-9 ftw)) - ;; Set ownership of files. - (for-each (lambda (file) - (when (eq? (stat:type (stat file)) 'directory) - (chmod file #o755)) - (chown file - (passwd:uid (getpw "genenetwork")) - (passwd:gid (getpw "genenetwork")))) - (append - '("/etc/genenetwork/conf" "/genenetwork2/flask_session") - (find-files #$(dirname gn-auth-secrets) - #:directories? #t) - (find-files #$(dirname gn2-secrets) - #:directories? #t) - (find-files #$(dirname gn3-secrets) - #:directories? #t) - (find-files #$(dirname auth-db-path) - #:directories? #t) - (find-files #$(dirname gn-doc-git-checkout) - #:directories? #t))) - ;; Prevent other users from reading secret files. - (for-each (lambda (file) - (chmod file #o600)) - (append - (find-files #$gn-auth-secrets - #:directories? #f) - (find-files #$gn2-secrets - #:directories? #f) - (find-files #$gn3-secrets - #:directories? #f))))))) + (unless (file-exists? "/etc/genenetwork/conf/gn-uploader.conf") + (with-output-to-file "/etc/genenetwork/conf/gn-uploader.conf" + (lambda () + (display #$(string-append + "UPLOADER_SECRETS=\"" gn-uploader-secrets "\"\n" + "SQL_URI=\"" sql-uri "\"\n" + "ASYNCHRONOUS_JOBS_SQLITE_DB=\"/var/genenetwork/gn-uploader.db\"\n" + "UPLOAD_FOLDER=\"" gn-uploader-data-directory "\"\n" + "AUTH_SERVER_URL=\"http://localhost:" (number->string gn-auth-port) "\"\n" + "GN2_SERVER_URL=\"http://localhost:" (number->string gn2-port) "\"\n" + "SESSION_FILESYSTEM_CACHE_PATH=\"" gn-uploader-sessions-dir "\"\n")) + (display "GN_UPLOADER_PROFILE=\"") + (display #$(file-append (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t)) + "\"\n"))))) + ;; Run gn-uploader + (with-directory-excursion "/gn-uploader" + ;; Configure Uploader + (setenv "UPLOADER_CONF" "/etc/genenetwork/conf/gn-uploader.conf") + (setenv "FLASK_APP" "scripts/qcapp_wsgi.py") + (setenv "FLASK_ENV" "development") + (setenv "FLASK_DEBUG" "1") + (setenv "ASYNCHRONOUS_JOBS_SQLITE_DB" "/var/genenetwork/gn-uploader.db") + (setenv "GN_UPLOADER_ENVIRONMENT" + #$(profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (invoke #$(file-append (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t)) + "/bin/flask") + "run" + #$(string-append "--port=" (number->string gn-uploader-port))))))))) + +(define (genenetwork-activation config) + (match-record + config <genenetwork-configuration> + (gn2-secrets + gn3-secrets gn3-port gn-auth-port + auth-db-path gn-auth-secrets gn-doc-git-checkout + gn-uploader-data-directory + gn-uploader-secrets + gn-uploader-sessions-dir) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 ftw)) + ;; Set ownership of files. + (for-each (lambda (file) + (when (eq? (stat:type (stat file)) 'directory) + (chmod file #o755)) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (append + '("/etc/genenetwork/conf" "/genenetwork2/flask_session" + "/gn-uploader/flask_session") + (find-files #$(dirname gn-auth-secrets) + #:directories? #t) + (find-files #$(dirname gn2-secrets) + #:directories? #t) + (find-files #$(dirname gn3-secrets) + #:directories? #t) + (find-files #$(dirname gn-uploader-secrets) + #:directories? #t) + (find-files #$(dirname auth-db-path) + #:directories? #t) + (find-files #$(dirname gn-doc-git-checkout) + #:directories? #t) + (find-files #$(dirname gn-uploader-data-directory) + #:directories? #t) + (find-files #$(dirname gn-uploader-sessions-dir) + #:directories? #t))) + ;; Prevent other users from reading secret files. + (for-each (lambda (file) + (chmod file #o600)) + (append + (find-files #$gn-auth-secrets + #:directories? #f) + (find-files #$gn2-secrets + #:directories? #f) + (find-files #$gn3-secrets + #:directories? #f) + (find-files #$gn-uploader-secrets + #:directories? #f))))))) (define (gn-guile-gexp gn-guile-port) (with-imported-modules '((guix build utils)) @@ -261,7 +334,15 @@ server described by CONFIG, a <genenetwork-configuration> object." "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port) + (gn2-port + gn3-port gn-auth-port genotype-files + data-directory xapian-db-path gn2-secrets + auth-db-path gn-auth-secrets + llm-db-path gn-doc-git-checkout + gn-guile-port + gn-uploader-port + gn-uploader-data-directory + gn-uploader-sessions-dir) (list (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -424,6 +505,49 @@ described by CONFIG, a <genenetwork-configuration> object." #:log-file "/var/log/genenetwork3.log")) (stop #~(make-kill-destructor))) (shepherd-service + (documentation "Run gn-uploader") + (provision '(gn-uploader)) + (requirement '(networking)) + (start #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-uploader" + (gn-uploader-gexp config)) + #:name "gn-uploader-pola-wrapper" + #:mappings (list (file-system-mapping + (source "/gn-uploader") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source "/var/genenetwork/gn-uploader.db") + (target source) + (writable? #t)) + (file-system-mapping + (source "/etc/genenetwork/conf") + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-data-directory) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-sessions-dir) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-uploader-port)) + #:user "genenetwork" + #:group "genenetwork" + #:log-file "/var/log/gn-uploader.log")) + (stop #~(make-kill-destructor))) + (shepherd-service (documentation "Run gn-auth development server.") (provision '(gn-auth)) (requirement '(networking)) diff --git a/genenetwork-local-container.sh b/genenetwork-local-container.sh index 5a81e61..0108390 100755 --- a/genenetwork-local-container.sh +++ b/genenetwork-local-container.sh @@ -33,6 +33,8 @@ SYSTEM_DIRECTORIES=( "$BASE_DIR/var/lib/redis" "$BASE_DIR/var/lib/virtuoso" "$BASE_DIR/var/lib/data" + "$BASE_DIR/var/lib/gn-uploader/data" + "$BASE_DIR/var/lib/gn-uploader/sessions" "$BASE_DIR/var/lib/xapian" "$BASE_DIR/var/genenetwork/genotype-files/genotype/json" "$BASE_DIR/var/lib/genenetwork-sqlite" @@ -46,15 +48,18 @@ GN_PROJECTS=( "gn-transform-databases:https://git.genenetwork.org/gn-transform-databases/" "gn-auth:https://git.genenetwork.org/gn-auth" "gn-docs:https://git.genenetwork.org/gn-docs" + "gn-uploader:https://git.genenetwork.org/gn-uploader" ) # File mappings: source -> destination declare -A FILE_MAPPINGS=( ["etc/auth.db"]="$BASE_DIR/var/genenetwork/auth.db" + ["etc/gn-uploader.db"]="$BASE_DIR/var/genenetwork/gn-uploader.db" ["etc/llm.db"]="$BASE_DIR/var/lib/genenetwork-sqlite/llm.db" ["etc/gn2-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn2/secrets.py" ["etc/gn3-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn3/secrets.py" ["etc/gn-auth-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn-auth/secrets.py" + ["etc/gn-uploader-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn-uploader/secrets.py" ) CONTAINER_SCM="genenetwork-local-container.scm" @@ -197,18 +202,22 @@ if [ "$1" = "--init-container" ]; then fi done - FLASK_SESSION="$BASE_DIR/genenetwork2/flask_session" - log "INFO" "Checking FLASK_SESSION directory: $FLASK_SESSION" - if [ ! -d "$FLASK_SESSION" ]; then - log "INFO" "Creating FLASK_SESSION directory: $FLASK_SESSION" - if ! mkdir -p "$FLASK_SESSION"; then - log "ERROR" "Failed to create FLASK_SESSION directory: $FLASK_SESSION" - exit 1 + FLASK_SESSIONS=( + "gn-uploader:$BASE_DIR/gn-uploader/flask_session" + "genenetwork2:$BASE_DIR/genenetwork2/flask_session" + ) + for flask_session in "${FLASK_SESSIONS[@]}"; do + IFS=':' read -r project session_dir <<< "$flask_session" + if [ ! -d "$session_dir" ]; then + log "INFO" "Creating FLASK_SESSION directory: $session_dir" + if ! mkdir -p "$session_dir"; then + log "ERROR" "Failed to create FLASK_SESSION directory: $session_dir" + exit 1 + fi + else + log "DEBUG" "FLASK_SESSION directory already exists: $session_dir" fi - else - log "DEBUG" "FLASK_SESSION directory already exists: $FLASK_SESSION" - fi - + done # Verify container SCM file exists if [ ! -f "$CONTAINER_SCM" ]; then @@ -226,8 +235,11 @@ if [ "$1" = "--init-container" ]; then "--share=$BASE_DIR/var/lib/redis=/var/lib/redis" "--share=$BASE_DIR/var/lib/virtuoso=/var/lib/virtuoso" "--share=$BASE_DIR/var/lib/data=/var/lib/data" + "--share=$BASE_DIR/var/lib/gn-uploader/data=/var/lib/gn-uploader/data" + "--share=$BASE_DIR/var/lib/gn-uploader/sessions=/var/lib/gn-uploader/sessions" "--share=$BASE_DIR/genenetwork2=/genenetwork2" "--share=$BASE_DIR/genenetwork3=/genenetwork3" + "--share=$BASE_DIR/gn-uploader=/gn-uploader" "--share=$BASE_DIR/gn-auth=/gn-auth" "--share=$BASE_DIR/var/lib/xapian=/var/lib/xapian" "--share=$BASE_DIR/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite" |