//! Layered configuration, mirroring GeneNetwork3's convention: //! //! ```text //! built-in defaults -> GN4_CONF file -> GN4_SECRETS file -> environment //! ``` //! //! GN4_CONF and GN4_SECRETS are paths to files of `KEY=VALUE` lines //! (# comments and blank lines allowed). Splitting secrets into their //! own file lets GN4_CONF be checked into a repo while GN4_SECRETS //! stays out of it. Environment variables always win, so ad-hoc //! overrides never need a file edit. use std::collections::HashMap; /// The GeneNetwork4 configuration. pub struct Config { /// mysql://user:password@host/database. Same SQL_URI config key /// GeneNetwork3 uses, so the two services can share configuration. pub sql_uri: String, /// Whether the database is required (`production`) or optional /// (`dev`). Set via the GN4_DB_MODE key. `production` (the default) /// refuses to start when the database is unreachable; `dev` starts /// loudly without it, with DB-backed routes disabled. See /// `DbMode` and src/main.rs. pub db_mode: DbMode, /// Base URL of the GeneNetwork3 API, used by proxy endpoints. /// Must end with a slash, e.g. http://127.0.0.1:5003/api/. pub gn3_url: String, /// Seconds before an upstream gn3 call is aborted. gn3 sets no /// timeout of its own (flask app.run()), so this is the only /// guard against a stalled upstream. pub gn3_timeout_secs: u64, /// Directory of the xapian search index queried by the native /// search engine. Same XAPIAN_DB_PATH config key GeneNetwork3 uses /// (gn3 reads it at gn3/api/search.py:295), so the two services can /// share configuration. Empty by default: the forwarding engine /// needs no index, and the native engine reports a local /// misconfiguration if it is asked to search without one. pub xapian_db_path: String, /// Which engine backs /search: "forward" (default, proxy gn3) or /// "native" (query the xapian index directly). Set via the /// SEARCH_ENGINE key; opt-in so flipping is a one-line config /// change and never a deployment. pub search_engine: String, /// TCP port to bind. pub port: u16, /// Base URL of the gn-auth server, e.g. "http://127.0.0.1:8081/". /// Same AUTH_SERVER_URL config key GeneNetwork3 uses. See /// docs/gn-auth.md. pub auth_server_url: String, /// Address to bind the TCP listener to (see src/main.rs). Defaults /// to "0.0.0.0" (every interface); set to "127.0.0.1" to only /// accept local connections. pub bind: String, /// Whether POST /get-test-token is registered at all /// (GENENETWORK4_TEST_TOKEN; on by default). When off the route is /// absent: a 404, and not in /openapi.json either. See /// `parse_switch` and src/routes/dev.rs. pub test_token_enabled: bool, /// One access-log line per request on stderr, naming the caller /// (`GN4_ACCESS_LOG`, default on). The line is one compact JSON object: /// method, path, status, took_ms, request_id and `user` when the token /// identified one. Turn it off where the proxy's own access log is the /// record of truth and this would only duplicate it. pub access_log: bool, /// gn-auth OAuth2 client id/secret and dev user credentials used by /// POST /get-test-token's password-grant call. Default to gn-auth's own /// well-known development fixtures (`flask init-dev-clients`, /// `gn_auth/wsgi.py`), so this works against an unmodified dev /// gn-auth instance with no extra configuration. pub dev_token_client_id: String, pub dev_token_client_secret: String, pub dev_token_username: String, pub dev_token_password: String, /// The gemma-wrapper executable to invoke for POST /compute/gemma. /// Same GEMMA_WRAPPER_CMD config key GeneNetwork3 uses. Defaults to /// "gemma-wrapper", resolved via PATH. pub gemma_wrapper_cmd: String, /// Where the short (`gn4ai_`) API tokens live: one SQLite database, /// `GN4_API_TOKENS_DB`, default `tokens.sqlite` in the working /// directory. See src/tokens.rs. The file is chmod 0600 on open -- /// it is a secret store, and it must not be in the repository. pub api_tokens_db: String, /// Directory gemma-wrapper uses to cache K matrices and association /// results between calls (its own --cache-dir switch). Defaults to /// a "gn4-gemma" directory under the system temp dir. pub gemma_cache_dir: String, /// The ravanan executable to invoke for POST /compute/gemma/ravanan /// (see src/gemma.rs's compute_ravanan). Defaults to "ravanan", /// resolved via PATH. pub ravanan_cmd: String, /// The ccwl (https://ccwl.systemreboot.net/) executable used to /// compile `gemma_ravanan_workflow` to CWL before each ravanan run /// (see src/gemma.rs's compute_ravanan). Defaults to "ccwl", /// resolved via PATH. pub ccwl_cmd: String, /// Path to the ccwl source POST /compute/gemma/ravanan compiles to /// CWL and runs under ravanan. Not a CWL file itself -- the /// compiled .cwl is a generated, throwaway artifact (written under /// `ravanan_cache_dir` on every call), never checked into the repo. /// Defaults to that path relative to the working directory the /// server is started from, same convention as `rqtl_wrapper_path`. pub gemma_ravanan_workflow: String, /// Path to the Guix manifest ravanan resolves its jobs' commands /// against (its own --guix-manifest switch, required -- ravanan /// refuses to run without one). Defaults to that path relative to /// the working directory the server is started from, same /// convention as `gemma_ravanan_workflow`. pub gemma_ravanan_manifest: String, /// Directory POST /compute/gemma/ravanan uses for phenotype files, /// run logs, and ravanan's own --store and --output-directory. /// Separate from `gemma_cache_dir` so the two runners don't share /// state. Defaults to a "gn4-gemma-ravanan" directory under the /// system temp dir. With `ravanan_batch` set to slurm this directory /// must be on a filesystem the cluster's compute nodes can see (see /// `ravanan_scratch`). pub ravanan_cache_dir: String, /// How ravanan runs each step: locally (`single-machine`, the /// default) or as jobs on a Slurm cluster (`slurm-api`, submitting /// through slurmrestd's REST API). Set via RAVANAN_BATCH_SYSTEM; /// see `RavananBatch` and docs/ravanan-slurm.md. pub ravanan_batch: RavananBatch, /// Directory ravanan stages job scratch space in (its own --scratch /// switch). Only meaningful for the slurm backend, where it must be /// a path every compute node can write; defaults to /// `ravanan_cache_dir` when the backend is slurm. Empty means "do /// not pass --scratch" (single-machine's own default). pub ravanan_scratch: String, /// Path to scripts/rqtl_wrapper.R (copied here from GeneNetwork3) /// for POST /compute/rqtl. Defaults to that path relative to the /// working directory the server is started from. pub rqtl_wrapper_path: String, /// Path to scripts/qtl_fitqtl.R, which POST /compute/fitqtl runs to get /// effect sizes for named markers. Defaults to that path relative to /// the working directory the server is started from. pub fitqtl_script_path: String, /// Path to scripts/qtl2_scan.R, which POST /compute/qtl2 runs. pub qtl2_script_path: String, /// Directory POST /compute/qtl2 keeps converted crosses and scan /// output in. Defaults to a "gn4-qtl2" directory under the temp dir. pub qtl2_cache_dir: String, /// Directory POST /compute/rqtl uses for phenotype files, run logs /// and the "gn3" subdirectory the R script itself writes its /// output into (see src/rqtl.rs). Defaults to a "gn4-rqtl" /// directory under the system temp dir. pub rqtl_cache_dir: String, /// LMDB directory holding Wikidata gene aliases (see src/wikidata.rs), /// keyed by species and symbol. `WIKIDATA_CACHE_DIR`. pub wikidata_cache_dir: String, /// The SPARQL endpoint those aliases come from, `WIKIDATA_ENDPOINT`. pub wikidata_endpoint: String, /// How long a cached answer is trusted, in seconds /// (`WIKIDATA_TTL_SECONDS`, default seven days). Past it the value is /// refetched; if Wikidata is unreachable the old value is served and /// marked stale. pub wikidata_ttl: i64, /// Directory holding one SQLite file per user's baskets (`BASKET_DIR`), /// named by a hash of the token's subject. See src/baskets.rs. pub basket_dir: String, /// The script that draws a trait by genotype class (`GENOTYPE_PLOTTER`). pub genotype_plotter: String, /// The BNW driver for `/compute/bnw` (`BNW_CMD`): the headless /// `bin/bnw` from the gn-bnw repository, which runs BNW's C programs /// `BNW_BIN_DIR` overrides where those compiled programs live, for a /// deployment that installs the driver and the binaries separately. pub bnw_cmd: String, /// Directory holding the compiled BNW programs (`BNW_BIN_DIR`); empty /// means "beside the driver", which is how `build.sh` lays them out. pub bnw_bin_dir: String, /// Where the per-request matrices are written (`BNW_CACHE_DIR`). pub bnw_cache_dir: String, /// The script `/plot/bnw` runs to draw a network (`BNW_PLOTTER`). pub bnw_plotter: String, /// Script that draws the scree plot of a correlation-matrix PCA (PCA_PLOTTER). pub pca_plotter: String, /// Script that draws the clustered heatmap of a correlation matrix (HEATMAP_PLOTTER). pub heatmap_plotter: String, /// Directory holding BNW's worked example datasets (`BNW_EXAMPLES_DIR`), /// served by `/compute/bnw/examples`. Empty means the endpoint reports /// that no examples are configured. pub bnw_examples_dir: String, /// The `julia` binary for BulkLMM (`BULKLMM_CMD`), the project its /// packages are instantiated into (`BULKLMM_ENV`), the depot they live in /// (`BULKLMM_DEPOT`), the driver script (`BULKLMM_SCRIPT`), the threads it /// may use (`BULKLMM_THREADS`) and where its scratch files go /// (`BULKLMM_CACHE_DIR`). pub bulklmm_cmd: String, pub bulklmm_env: String, pub bulklmm_depot: String, pub bulklmm_script: String, pub bulklmm_threads: u32, /// Independent marker blocks scanned in parallel (`BULKLMM_BLOCKS`, /// default 4). `1` is a plain serial scan; the LODs and thresholds are /// identical either way. pub bulklmm_blocks: u32, pub bulklmm_cache_dir: String, /// The script that draws a BulkLMM scan (`BULKLMM_PLOTTER`). pub bulklmm_plotter: String, /// Directory holding the genotype files (`GENOTYPE_FILES`): GN2's /// layout -- `genotype/.json` for the version list, /// `genotype/.geno`, `genotype/bimbam/_geno.txt` and /// `_snps.txt`, and `parents_and_f1s.json`. See src/genotypes.rs and /// docs/genotypes.md. pub genotype_files_dir: String, /// Interpreter used to run the plotter scripts (`PLOT_PYTHON`, /// default `python3`). /// Whether the connection pool pings a connection taken from it and /// replaces it when the ping fails (`SQL_CHECK_HEALTH`, default /// `true`). Costs one round trip per pooled checkout; turning it off /// means a connection the server closed surfaces as a failed request /// instead of being replaced. pub sql_check_health: bool, /// How many `/compute/rqtl` scans may run at once /// (`RQTL_MAX_CONCURRENT`, default 8), one per caller. Fixed when the /// gate is built, i.e. at the first compute request. pub rqtl_max_concurrent: usize, pub plot_python: String, /// `scripts/plot-rqtl.py`: turns a /compute/rqtl response into an SVG /// Manhattan plot (`RQTL_PLOTTER`). pub rqtl_plotter: String, /// `scripts/plot-gemma.py`, for /compute/gemma responses /// (`GEMMA_PLOTTER`). pub gemma_plotter: String, } impl Config { /// Load configuration from all layers, highest priority last. pub fn load() -> Config { let mut values = HashMap::::new(); // Layer 1: built-in defaults. values.insert( "SQL_URI".to_string(), "mysql://webqtlout:webqtlout@127.0.0.1/db_webqtl".to_string(), ); // Preserve today's behaviour: with no GN4_DB_MODE set, gn4 still // starts when the database is unreachable (it just logs loudly // and answers DB-backed routes as 503). Flipping this default to // `production` (refuse to start) is a consumer-facing behaviour // change and needs owner sign-off -- see REPORT.md. values.insert("GN4_DB_MODE".to_string(), "dev".to_string()); values.insert("GENENETWORK4_PORT".to_string(), "3000".to_string()); values.insert( "AUTH_SERVER_URL".to_string(), "http://127.0.0.1:8081/".to_string(), ); values.insert("GENENETWORK4_BIND".to_string(), "0.0.0.0".to_string()); values.insert("GN4_ACCESS_LOG".to_string(), "true".to_string()); values.insert("GENENETWORK4_TEST_TOKEN".to_string(), "true".to_string()); values.insert( "GN_AUTH_DEV_CLIENT_ID".to_string(), "0bbfca82-d73f-4bd4-a140-5ae7abb4a64d".to_string(), ); values.insert( "GN_AUTH_DEV_CLIENT_SECRET".to_string(), "yadabadaboo".to_string(), ); values.insert( "GN_AUTH_DEV_USERNAME".to_string(), "test@development.user".to_string(), ); values.insert("GN_AUTH_DEV_PASSWORD".to_string(), "testpasswd".to_string()); values.insert( "GN3_URL".to_string(), "http://127.0.0.1:5003/api/".to_string(), ); values.insert("GN3_TIMEOUT_SECS".to_string(), "30".to_string()); values.insert("SQL_CHECK_HEALTH".to_string(), "true".to_string()); values.insert("RQTL_MAX_CONCURRENT".to_string(), "8".to_string()); values.insert("PLOT_PYTHON".to_string(), "python3".to_string()); values.insert( "RQTL_PLOTTER".to_string(), "scripts/plot-rqtl.py".to_string(), ); values.insert( "GEMMA_PLOTTER".to_string(), "scripts/plot-gemma.py".to_string(), ); values.insert( "GENOTYPE_FILES".to_string(), "/home/wrk/tmp/ai-root/factory/gn4-data/genotype-files".to_string(), ); values.insert("XAPIAN_DB_PATH".to_string(), String::new()); values.insert("SEARCH_ENGINE".to_string(), "forward".to_string()); values.insert("GN4_API_TOKENS_DB".to_string(), "tokens.sqlite".to_string()); values.insert("GEMMA_WRAPPER_CMD".to_string(), "gemma-wrapper".to_string()); values.insert( "GEMMA_CACHE_DIR".to_string(), std::env::temp_dir() .join("gn4-gemma") .to_string_lossy() .into_owned(), ); values.insert("RAVANAN_CMD".to_string(), "ravanan".to_string()); values.insert("CCWL_CMD".to_string(), "ccwl".to_string()); values.insert( "GEMMA_RAVANAN_WORKFLOW".to_string(), "cwl/gemma/gemma-loco.scm".to_string(), ); values.insert( "GEMMA_RAVANAN_MANIFEST".to_string(), "cwl/gemma/manifest.scm".to_string(), ); values.insert( "RAVANAN_CACHE_DIR".to_string(), std::env::temp_dir() .join("gn4-gemma-ravanan") .to_string_lossy() .into_owned(), ); values.insert( "RAVANAN_BATCH_SYSTEM".to_string(), "single-machine".to_string(), ); // Only read when RAVANAN_BATCH_SYSTEM=slurm-api; left blank so // the single-machine default stays a zero-configuration path. values.insert("RAVANAN_SLURM_API_ENDPOINT".to_string(), String::new()); values.insert("RAVANAN_SLURM_JWT".to_string(), String::new()); values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new()); values.insert("RAVANAN_SLURM_NICE".to_string(), String::new()); values.insert("RAVANAN_SCRATCH".to_string(), String::new()); values.insert( "RQTL_WRAPPER_PATH".to_string(), "scripts/rqtl_wrapper.R".to_string(), ); values.insert( "FITQTL_SCRIPT_PATH".to_string(), "scripts/qtl_fitqtl.R".to_string(), ); values.insert( "QTL2_SCRIPT_PATH".to_string(), "scripts/qtl2_scan.R".to_string(), ); values.insert( "QTL2_CACHE_DIR".to_string(), std::env::temp_dir() .join("gn4-qtl2") .to_string_lossy() .into_owned(), ); values.insert( "RQTL_CACHE_DIR".to_string(), std::env::temp_dir() .join("gn4-rqtl") .to_string_lossy() .into_owned(), ); // Wikidata aliases (src/wikidata.rs): an LMDB cache and the SPARQL // endpoint it refreshes from. Seven days, because an alias list is // not news. values.insert( "WIKIDATA_CACHE_DIR".to_string(), std::env::temp_dir() .join("gn4-wikidata") .to_string_lossy() .into_owned(), ); values.insert( "WIKIDATA_ENDPOINT".to_string(), "https://query.wikidata.org/sparql".to_string(), ); values.insert("WIKIDATA_TTL_SECONDS".to_string(), "604800".to_string()); // BulkLMM.jl (src/bulklmm.rs): the Julia binary, the project its // packages are instantiated into, and the depot those packages live // in. `guix.scm` supplies the binary; the depot is built once. values.insert("BNW_CMD".to_string(), "bnw".to_string()); values.insert("BNW_BIN_DIR".to_string(), String::new()); values.insert("BNW_EXAMPLES_DIR".to_string(), String::new()); values.insert( "BNW_PLOTTER".to_string(), "scripts/plot-bnw.py".to_string(), ); values.insert( "BNW_CACHE_DIR".to_string(), std::env::temp_dir() .join("gn4-bnw") .to_string_lossy() .into_owned(), ); values.insert( "BULKLMM_CMD".to_string(), "/gnu/store/4w6ivx7b27djzg16js6mkks1n4dvwq0f-julia-1.8.5/bin/julia".to_string(), ); values.insert("BULKLMM_ENV".to_string(), "/tmp/gj-env".to_string()); values.insert( "BULKLMM_DEPOT".to_string(), format!("{}/.julia", std::env::var("HOME").unwrap_or_else(|_| "/tmp".into())), ); values.insert( "BULKLMM_SCRIPT".to_string(), "scripts/bulklmm_scan.jl".to_string(), ); values.insert("BULKLMM_THREADS".to_string(), "4".to_string()); // Independent marker blocks scanned in parallel: a BulkLMM `scan()` is // serial in Julia, so this is where a single-trait scan gets its // parallelism. 1 is a plain serial scan and gives the same answer. values.insert("BULKLMM_BLOCKS".to_string(), "4".to_string()); values.insert( "BULKLMM_PLOTTER".to_string(), "scripts/plot-bulklmm.py".to_string(), ); values.insert( "BULKLMM_CACHE_DIR".to_string(), std::env::temp_dir() .join("gn4-bulklmm") .to_string_lossy() .into_owned(), ); values.insert( "GENOTYPE_PLOTTER".to_string(), "scripts/plot-genotypes.py".to_string(), ); values.insert( "PCA_PLOTTER".to_string(), "scripts/plot-pca.py".to_string(), ); values.insert( "HEATMAP_PLOTTER".to_string(), "scripts/plot-heatmap.py".to_string(), ); // Per-user baskets: one SQLite file each (src/baskets.rs). Never // MariaDB -- a user's edited column is theirs. values.insert( "BASKET_DIR".to_string(), std::env::temp_dir() .join("gn4-baskets") .to_string_lossy() .into_owned(), ); // Layers 2 and 3: config files, in increasing priority. for env_key in ["GN4_CONF", "GN4_SECRETS"] { if let Ok(path) = std::env::var(env_key) { for (key, value) in load_file(&path) { values.insert(key, value); } } } // Layer 4: environment wins over everything. for key in [ "SQL_URI", "GN4_DB_MODE", "GENENETWORK4_PORT", "AUTH_SERVER_URL", "GENENETWORK4_BIND", "GN4_ACCESS_LOG", "GENENETWORK4_TEST_TOKEN", "GN_AUTH_DEV_CLIENT_ID", "GN_AUTH_DEV_CLIENT_SECRET", "GN_AUTH_DEV_USERNAME", "GN_AUTH_DEV_PASSWORD", "GN3_URL", "GN3_TIMEOUT_SECS", "GENOTYPE_FILES", "SQL_CHECK_HEALTH", "RQTL_MAX_CONCURRENT", "PLOT_PYTHON", "RQTL_PLOTTER", "GEMMA_PLOTTER", "XAPIAN_DB_PATH", "SEARCH_ENGINE", "GN4_API_TOKENS_DB", "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", "RAVANAN_CMD", "CCWL_CMD", "GEMMA_RAVANAN_WORKFLOW", "GEMMA_RAVANAN_MANIFEST", "RAVANAN_CACHE_DIR", "RAVANAN_BATCH_SYSTEM", "RAVANAN_SLURM_API_ENDPOINT", "RAVANAN_SLURM_JWT", "RAVANAN_SLURM_PARTITION", "RAVANAN_SLURM_NICE", "RAVANAN_SCRATCH", "RQTL_WRAPPER_PATH", "FITQTL_SCRIPT_PATH", "QTL2_SCRIPT_PATH", "QTL2_CACHE_DIR", "RQTL_CACHE_DIR", "WIKIDATA_CACHE_DIR", "WIKIDATA_ENDPOINT", "WIKIDATA_TTL_SECONDS", "BASKET_DIR", "GENOTYPE_PLOTTER", "PCA_PLOTTER", "HEATMAP_PLOTTER", "BNW_CMD", "BNW_BIN_DIR", "BNW_CACHE_DIR", "BNW_PLOTTER", "BNW_EXAMPLES_DIR", "BULKLMM_CMD", "BULKLMM_ENV", "BULKLMM_DEPOT", "BULKLMM_SCRIPT", "BULKLMM_THREADS", "BULKLMM_BLOCKS", "BULKLMM_CACHE_DIR", "BULKLMM_PLOTTER", ] { if let Ok(value) = std::env::var(key) { values.insert(key.to_string(), value); } } let gn3_url = normalize_base_url(values["GN3_URL"].clone()); Config { sql_uri: values["SQL_URI"].clone(), db_mode: parse_db_mode(&values["GN4_DB_MODE"]), gn3_url, gn3_timeout_secs: values["GN3_TIMEOUT_SECS"] .parse() .expect("GN3_TIMEOUT_SECS must be a number"), xapian_db_path: values["XAPIAN_DB_PATH"].clone(), search_engine: values["SEARCH_ENGINE"].clone(), port: values["GENENETWORK4_PORT"] .parse() .expect("GENENETWORK4_PORT must be a number"), auth_server_url: values["AUTH_SERVER_URL"].clone(), bind: values["GENENETWORK4_BIND"].clone(), test_token_enabled: parse_switch(&values["GENENETWORK4_TEST_TOKEN"]), access_log: parse_switch(&values["GN4_ACCESS_LOG"]), dev_token_client_id: values["GN_AUTH_DEV_CLIENT_ID"].clone(), dev_token_client_secret: values["GN_AUTH_DEV_CLIENT_SECRET"].clone(), dev_token_username: values["GN_AUTH_DEV_USERNAME"].clone(), dev_token_password: values["GN_AUTH_DEV_PASSWORD"].clone(), api_tokens_db: values["GN4_API_TOKENS_DB"].clone(), gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(), gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(), ravanan_cmd: values["RAVANAN_CMD"].clone(), ccwl_cmd: values["CCWL_CMD"].clone(), gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(), gemma_ravanan_manifest: values["GEMMA_RAVANAN_MANIFEST"].clone(), ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(), ravanan_batch: parse_ravanan_batch(&values), ravanan_scratch: values["RAVANAN_SCRATCH"].clone(), rqtl_wrapper_path: values["RQTL_WRAPPER_PATH"].clone(), fitqtl_script_path: values["FITQTL_SCRIPT_PATH"].clone(), qtl2_script_path: values["QTL2_SCRIPT_PATH"].clone(), qtl2_cache_dir: values["QTL2_CACHE_DIR"].clone(), rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(), wikidata_cache_dir: values["WIKIDATA_CACHE_DIR"].clone(), wikidata_endpoint: values["WIKIDATA_ENDPOINT"].clone(), wikidata_ttl: values["WIKIDATA_TTL_SECONDS"].parse().unwrap_or(604800), basket_dir: values["BASKET_DIR"].clone(), genotype_plotter: values["GENOTYPE_PLOTTER"].clone(), bnw_cmd: values["BNW_CMD"].clone(), bnw_bin_dir: values["BNW_BIN_DIR"].clone(), bnw_cache_dir: values["BNW_CACHE_DIR"].clone(), bnw_plotter: values["BNW_PLOTTER"].clone(), pca_plotter: values["PCA_PLOTTER"].clone(), heatmap_plotter: values["HEATMAP_PLOTTER"].clone(), bnw_examples_dir: values["BNW_EXAMPLES_DIR"].clone(), bulklmm_cmd: values["BULKLMM_CMD"].clone(), bulklmm_env: values["BULKLMM_ENV"].clone(), bulklmm_depot: values["BULKLMM_DEPOT"].clone(), bulklmm_script: values["BULKLMM_SCRIPT"].clone(), bulklmm_threads: values["BULKLMM_THREADS"].parse().unwrap_or(4), bulklmm_blocks: values["BULKLMM_BLOCKS"].parse().unwrap_or(4), bulklmm_cache_dir: values["BULKLMM_CACHE_DIR"].clone(), bulklmm_plotter: values["BULKLMM_PLOTTER"].clone(), genotype_files_dir: values["GENOTYPE_FILES"].clone(), sql_check_health: parse_switch(&values["SQL_CHECK_HEALTH"]), rqtl_max_concurrent: values["RQTL_MAX_CONCURRENT"] .parse() .ok() .filter(|permits| *permits > 0) .unwrap_or(8), plot_python: values["PLOT_PYTHON"].clone(), rqtl_plotter: values["RQTL_PLOTTER"].clone(), gemma_plotter: values["GEMMA_PLOTTER"].clone(), } } } /// Reads the GN4_DB_MODE setting, failing closed: only the exact /// values `dev` and `production` (any case, trimmed) are accepted; /// anything else -- including a typo like "prodution" -- refuses to /// start rather than silently picking a mode. The empty string (an /// explicitly blank key) is likewise refused. fn parse_db_mode(value: &str) -> DbMode { match value.trim().to_ascii_lowercase().as_str() { "dev" => DbMode::Dev, "production" => DbMode::Production, other => panic!("GN4_DB_MODE must be \"dev\" or \"production\", not {other:?}"), } } /// Whether the database is required for the process to serve. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum DbMode { /// The database is optional: gn4 starts without a reachable /// database, logging loudly and serving DB-backed routes as 503. Dev, /// The database is required: an unreachable/unconfigured database /// at startup is a misconfiguration and the process refuses to /// start (exit non-zero). Production, } /// Reads an on/off setting, failing closed: only a clearly affirmative /// value ("true", "1", "yes", "on", any case) is on; "false", "0", "no", /// "off", the empty string and anything unrecognised (a typo like /// "flase") are off. For a switch guarding a credential-issuing endpoint, /// a typo must not leave it running. fn parse_switch(value: &str) -> bool { matches!( value.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes" | "on" ) } /// How ravanan is asked to run each step of a workflow. /// /// `single-machine` (the default) keeps the previous behaviour: ravanan /// forks each step locally. `slurm-api` submits every step to a Slurm /// cluster through slurmrestd's REST API instead, so the CWL scatter /// really does fan out over the cluster's nodes. /// /// Selected with RAVANAN_BATCH_SYSTEM; the slurm-specific keys are /// `RAVANAN_SLURM_API_ENDPOINT` (e.g. http://127.0.0.1:6820), /// `RAVANAN_SLURM_JWT` (path to a *file* holding the JWT -- ravanan's /// `--slurm-jwt` takes a file, not the token), `RAVANAN_SLURM_PARTITION` /// and `RAVANAN_SLURM_NICE`. See docs/ravanan-slurm.md. #[derive(Clone, PartialEq, Eq, Debug)] pub enum RavananBatch { SingleMachine, SlurmApi { /// slurmrestd base URL, e.g. http://127.0.0.1:6820. endpoint: String, /// Path to a file containing the JWT ravanan authenticates with. jwt_file: String, partition: Option, nice: Option, }, } impl RavananBatch { /// The ravanan command-line switches this backend adds. /// /// `ravanan` takes `--slurm-api-endpoint`, `--slurm-jwt`, /// `--slurm-partition`, `--slurm-nice` and `--scratch`. The last one /// only matters for slurm (the directory the compute nodes stage job /// scratch in), so `scratch` is empty for single-machine. pub fn ravanan_args(&self, scratch: &str) -> Vec { let mut args = Vec::new(); match self { RavananBatch::SingleMachine => { if !scratch.is_empty() { args.push(format!("--scratch={scratch}")); } } RavananBatch::SlurmApi { endpoint, jwt_file, partition, nice, } => { args.push("--batch-system=slurm-api".to_string()); args.push(format!("--slurm-api-endpoint={endpoint}")); args.push(format!("--slurm-jwt={jwt_file}")); if let Some(partition) = partition { args.push(format!("--slurm-partition={partition}")); } if let Some(nice) = nice { args.push(format!("--slurm-nice={nice}")); } if !scratch.is_empty() { args.push(format!("--scratch={scratch}")); } } } args } } /// Reads RAVANAN_BATCH_SYSTEM, failing closed: only `single-machine` and /// `slurm-api` (any case, trimmed) are accepted, so a typo cannot /// silently leave jobs running locally. Selecting slurm-api without an /// endpoint or a JWT file is likewise refused at startup -- ravanan /// would fail every step, and a clear refusal here beats 500s later. fn parse_ravanan_batch(values: &HashMap) -> RavananBatch { let system = values["RAVANAN_BATCH_SYSTEM"].trim().to_ascii_lowercase(); match system.as_str() { "" | "single-machine" => RavananBatch::SingleMachine, "slurm-api" => { let endpoint = values["RAVANAN_SLURM_API_ENDPOINT"].trim(); if endpoint.is_empty() { panic!( "RAVANAN_BATCH_SYSTEM=slurm-api needs RAVANAN_SLURM_API_ENDPOINT \ (e.g. http://127.0.0.1:6820)" ); } let jwt_file = values["RAVANAN_SLURM_JWT"].trim(); if jwt_file.is_empty() { panic!( "RAVANAN_BATCH_SYSTEM=slurm-api needs RAVANAN_SLURM_JWT \ (path to a file containing the JWT)" ); } let partition = Some(values["RAVANAN_SLURM_PARTITION"].trim().to_string()) .filter(|p| !p.is_empty()); let nice = Some(values["RAVANAN_SLURM_NICE"].trim().to_string()) .filter(|n| !n.is_empty()) .map(|n| { n.parse().unwrap_or_else(|_| { panic!("RAVANAN_SLURM_NICE must be an integer, not {n:?}") }) }); RavananBatch::SlurmApi { endpoint: endpoint.to_string(), jwt_file: jwt_file.to_string(), partition, nice, } } other => panic!( "RAVANAN_BATCH_SYSTEM must be \"single-machine\" or \"slurm-api\", not {other:?}" ), } } /// The proxy appends "search/" to this base; without a trailing slash /// the result would be ".../apisearch/". Operators may set either form. fn normalize_base_url(mut url: String) -> String { if !url.ends_with('/') { url.push('/'); } url } /// Parse a `KEY=VALUE` file into a map. Missing/unreadable files are /// ignored so a stale GN4_CONF path never stops the server from /// starting on defaults. fn load_file(path: &str) -> HashMap { let Ok(contents) = std::fs::read_to_string(path) else { return HashMap::new(); }; let mut map = HashMap::new(); for line in contents.lines() { let line = line.trim(); if line.is_empty() || line.starts_with('#') { continue; } if let Some((key, value)) = line.split_once('=') { map.insert(key.trim().to_string(), value.trim().to_string()); } } map } #[cfg(test)] mod tests { use super::*; use std::sync::{Mutex, MutexGuard}; /// These tests read and write process-global environment variables, /// and `Config::load()` reads *all* of them: a test that sets /// `RAVANAN_*` while another loads the config makes that other one /// fail on an unrelated assertion (the file's own warning about /// env-var mutation, one level up). This serialises the module. /// Poisoning is ignored: one failing test should not cascade. fn env_guard() -> MutexGuard<'static, ()> { static LOCK: Mutex<()> = Mutex::new(()); LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) } /// Defaults apply when nothing overrides them. Skips (rather than /// flakes) if the surrounding environment happens to set any of the /// knobs this test asserts on. #[test] fn defaults_apply_with_no_files_or_env() { let _env = env_guard(); for key in [ "SQL_URI", "GN4_DB_MODE", "GN4_CONF", "GN4_SECRETS", "GENENETWORK4_PORT", "AUTH_SERVER_URL", "GENENETWORK4_BIND", "GENENETWORK4_TEST_TOKEN", "GN_AUTH_DEV_CLIENT_ID", "GN_AUTH_DEV_CLIENT_SECRET", "GN_AUTH_DEV_USERNAME", "GN_AUTH_DEV_PASSWORD", "GN3_URL", "GN3_TIMEOUT_SECS", "XAPIAN_DB_PATH", "SEARCH_ENGINE", "GN4_API_TOKENS_DB", "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", "RAVANAN_CMD", "CCWL_CMD", "GEMMA_RAVANAN_WORKFLOW", "GEMMA_RAVANAN_MANIFEST", "RAVANAN_CACHE_DIR", "RAVANAN_BATCH_SYSTEM", "RAVANAN_SLURM_API_ENDPOINT", "RAVANAN_SLURM_JWT", "RAVANAN_SLURM_PARTITION", "RAVANAN_SLURM_NICE", "RAVANAN_SCRATCH", "RQTL_WRAPPER_PATH", "RQTL_CACHE_DIR", "WIKIDATA_CACHE_DIR", "WIKIDATA_ENDPOINT", "WIKIDATA_TTL_SECONDS", "BASKET_DIR", "GENOTYPE_PLOTTER", "BULKLMM_CMD", "BULKLMM_ENV", "BULKLMM_DEPOT", "BULKLMM_SCRIPT", "BULKLMM_THREADS", "BULKLMM_BLOCKS", "BULKLMM_CACHE_DIR", "BULKLMM_PLOTTER", ] { if std::env::var(key).is_ok() { eprintln!("skipping: {key} is set in the environment"); return; } } let config = Config::load(); assert!(config.sql_uri.contains("db_webqtl")); assert_eq!( config.db_mode, DbMode::Dev, "no-DB start preserved; the production flip needs sign-off" ); assert_eq!(config.port, 3000); assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper"); assert!(config.gemma_cache_dir.ends_with("gn4-gemma")); assert_eq!(config.ravanan_cmd, "ravanan"); assert_eq!(config.ccwl_cmd, "ccwl"); assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.scm"); assert_eq!(config.gemma_ravanan_manifest, "cwl/gemma/manifest.scm"); assert!(config.ravanan_cache_dir.ends_with("gn4-gemma-ravanan")); assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R"); assert!(config.rqtl_cache_dir.ends_with("gn4-rqtl")); assert_eq!(config.auth_server_url, "http://127.0.0.1:8081/"); assert_eq!(config.bind, "0.0.0.0"); assert!(config.test_token_enabled, "on by default"); assert_eq!(config.dev_token_username, "test@development.user"); // Search-engine wiring defaults: forwarding stays the default // engine and no index path is configured out of the box. assert_eq!(config.search_engine, "forward"); assert_eq!(config.xapian_db_path, ""); } #[test] fn db_mode_accepts_only_dev_and_production() { let _env = env_guard(); assert_eq!(parse_db_mode("dev"), DbMode::Dev); assert_eq!(parse_db_mode(" DEV "), DbMode::Dev); assert_eq!(parse_db_mode("production"), DbMode::Production); assert_eq!(parse_db_mode("Production"), DbMode::Production); // Fail-closed: a typo or a blank value must refuse, not default. for bad in ["prodution", "prod", "", "none", "devmode"] { let refused = std::panic::catch_unwind(|| parse_db_mode(bad)).is_err(); assert!(refused, "{bad:?} must be refused"); } } #[test] fn a_switch_is_on_only_for_a_clearly_affirmative_value() { let _env = env_guard(); for on in ["true", "TRUE", "True", "1", "yes", "ON", " on "] { assert!(parse_switch(on), "{on:?}"); } for off in [ "false", "False", "0", "no", "off", "", "flase", "maybe", "2", "tru", ] { assert!(!parse_switch(off), "{off:?}"); } } #[test] fn load_file_parses_key_value_lines() { let path = std::env::temp_dir().join(format!("gn4-test-{}", std::process::id())); std::fs::write( &path, "# comment\n\nSQL_URI = mysql://u:p@h/db\nGENENETWORK4_PORT=4000\n", ) .unwrap(); let values = load_file(path.to_str().unwrap()); std::fs::remove_file(&path).unwrap(); assert_eq!(values["SQL_URI"], "mysql://u:p@h/db"); assert_eq!(values["GENENETWORK4_PORT"], "4000"); assert_eq!(values.len(), 2, "comments and blank lines are skipped"); } #[test] fn load_file_missing_file_yields_empty_map() { assert!(load_file("/nonexistent/gn4.conf").is_empty()); } #[test] fn normalize_base_url_appends_missing_slash() { assert_eq!( normalize_base_url("http://h:5003/api".into()), "http://h:5003/api/" ); assert_eq!( normalize_base_url("http://h:5003/api/".into()), "http://h:5003/api/" ); } /// The new search-engine keys follow the same layering as the gn3 /// keys they sit next to: environment beats the built-in default. /// One sequential test function mutating the process environment, /// per the rule documented in tests/search_engine.rs. #[test] fn search_engine_keys_follow_env_over_default_layering() { let _env = env_guard(); let restore = |key: &str, previous: Option| unsafe { match previous { Some(value) => std::env::set_var(key, value), None => std::env::remove_var(key), } }; let prior_path = std::env::var("XAPIAN_DB_PATH").ok(); let prior_engine = std::env::var("SEARCH_ENGINE").ok(); unsafe { std::env::remove_var("XAPIAN_DB_PATH"); std::env::remove_var("SEARCH_ENGINE"); } let config = Config::load(); assert_eq!(config.xapian_db_path, ""); assert_eq!(config.search_engine, "forward"); unsafe { std::env::set_var("XAPIAN_DB_PATH", "/srv/xapian-index"); std::env::set_var("SEARCH_ENGINE", "native"); } let config = Config::load(); assert_eq!(config.xapian_db_path, "/srv/xapian-index"); assert_eq!(config.search_engine, "native"); restore("XAPIAN_DB_PATH", prior_path); restore("SEARCH_ENGINE", prior_engine); } /// A values map with every ravanan batch key present (empty means /// unset), so the panic assertions below fail for the reason under /// test rather than on a missing HashMap entry. fn ravanan_values(system: &str, endpoint: &str, jwt: &str, nice: &str) -> HashMap { let mut values = HashMap::new(); values.insert("RAVANAN_BATCH_SYSTEM".to_string(), system.to_string()); values.insert( "RAVANAN_SLURM_API_ENDPOINT".to_string(), endpoint.to_string(), ); values.insert("RAVANAN_SLURM_JWT".to_string(), jwt.to_string()); values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new()); values.insert("RAVANAN_SLURM_NICE".to_string(), nice.to_string()); values } /// The ravanan backend is single-machine unless asked for slurm, and /// single-machine adds no switches of its own beyond --scratch. #[test] fn ravanan_batch_defaults_to_single_machine() { let _env = env_guard(); let mut values = HashMap::new(); values.insert("RAVANAN_BATCH_SYSTEM".to_string(), String::new()); assert_eq!(parse_ravanan_batch(&values), RavananBatch::SingleMachine); assert_eq!( RavananBatch::SingleMachine.ravanan_args(""), Vec::::new() ); assert_eq!( RavananBatch::SingleMachine.ravanan_args("/scratch"), vec!["--scratch=/scratch".to_string()] ); } /// Selecting slurm builds exactly the switches ravanan's slurm-api /// backend needs, and nothing when they are optional and unset. #[test] fn ravanan_batch_slurm_args() { let _env = env_guard(); let mut values = HashMap::new(); values.insert("RAVANAN_BATCH_SYSTEM".to_string(), " Slurm-API ".to_string()); values.insert( "RAVANAN_SLURM_API_ENDPOINT".to_string(), "http://127.0.0.1:6820".to_string(), ); values.insert( "RAVANAN_SLURM_JWT".to_string(), "/etc/gn4/slurm-jwt".to_string(), ); values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new()); values.insert("RAVANAN_SLURM_NICE".to_string(), String::new()); let batch = parse_ravanan_batch(&values); assert_eq!( batch, RavananBatch::SlurmApi { endpoint: "http://127.0.0.1:6820".to_string(), jwt_file: "/etc/gn4/slurm-jwt".to_string(), partition: None, nice: None, } ); assert_eq!( batch.ravanan_args("/srv/gn4-ravanan"), vec![ "--batch-system=slurm-api".to_string(), "--slurm-api-endpoint=http://127.0.0.1:6820".to_string(), "--slurm-jwt=/etc/gn4/slurm-jwt".to_string(), "--scratch=/srv/gn4-ravanan".to_string(), ] ); values.insert("RAVANAN_SLURM_PARTITION".to_string(), "debug".to_string()); values.insert("RAVANAN_SLURM_NICE".to_string(), "100".to_string()); assert_eq!( parse_ravanan_batch(&values).ravanan_args(""), vec![ "--batch-system=slurm-api".to_string(), "--slurm-api-endpoint=http://127.0.0.1:6820".to_string(), "--slurm-jwt=/etc/gn4/slurm-jwt".to_string(), "--slurm-partition=debug".to_string(), "--slurm-nice=100".to_string(), ] ); } /// A typo in the backend name, or slurm without an endpoint or a JWT /// file, refuses to start rather than silently running jobs locally /// or failing every step at request time. #[test] #[should_panic(expected = "RAVANAN_BATCH_SYSTEM must be")] fn ravanan_batch_rejects_unknown_backend() { let _env = env_guard(); parse_ravanan_batch(&ravanan_values("slrum-api", "http://h:6820", "/jwt", "")); } #[test] #[should_panic(expected = "RAVANAN_SLURM_API_ENDPOINT")] fn ravanan_batch_slurm_requires_endpoint() { let _env = env_guard(); parse_ravanan_batch(&ravanan_values("slurm-api", "", "/jwt", "")); } #[test] #[should_panic(expected = "RAVANAN_SLURM_JWT")] fn ravanan_batch_slurm_requires_jwt_file() { let _env = env_guard(); parse_ravanan_batch(&ravanan_values("slurm-api", "http://127.0.0.1:6820", "", "")); } #[test] #[should_panic(expected = "RAVANAN_SLURM_NICE must be an integer")] fn ravanan_batch_rejects_non_integer_nice() { let _env = env_guard(); parse_ravanan_batch(&ravanan_values("slurm-api", "http://127.0.0.1:6820", "/jwt", "high")); } /// The new keys layer like the rest: environment beats the default. #[test] fn ravanan_batch_keys_follow_env_over_default_layering() { let _env = env_guard(); let keys = [ "RAVANAN_BATCH_SYSTEM", "RAVANAN_SLURM_API_ENDPOINT", "RAVANAN_SLURM_JWT", "RAVANAN_SCRATCH", ]; let prior: Vec> = keys.iter().map(|k| std::env::var(k).ok()).collect(); let restore = |key: &str, previous: Option| unsafe { match previous { Some(value) => std::env::set_var(key, value), None => std::env::remove_var(key), } }; unsafe { std::env::remove_var("RAVANAN_BATCH_SYSTEM"); std::env::remove_var("RAVANAN_SLURM_API_ENDPOINT"); std::env::remove_var("RAVANAN_SLURM_JWT"); std::env::remove_var("RAVANAN_SCRATCH"); } let config = Config::load(); assert_eq!(config.ravanan_batch, RavananBatch::SingleMachine); assert_eq!(config.ravanan_scratch, ""); unsafe { std::env::set_var("RAVANAN_BATCH_SYSTEM", "slurm-api"); std::env::set_var("RAVANAN_SLURM_API_ENDPOINT", "http://slurm:6820"); std::env::set_var("RAVANAN_SLURM_JWT", "/etc/gn4/jwt"); std::env::set_var("RAVANAN_SCRATCH", "/srv/scratch"); } let config = Config::load(); assert_eq!( config.ravanan_batch, RavananBatch::SlurmApi { endpoint: "http://slurm:6820".to_string(), jwt_file: "/etc/gn4/jwt".to_string(), partition: None, nice: None, } ); assert_eq!(config.ravanan_scratch, "/srv/scratch"); for (key, previous) in keys.iter().zip(prior) { restore(key, previous); } } }