diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-15 01:22:23 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-15 01:22:23 +0000 |
| commit | 4dbb553d5e4e48a785bbb5f4b97a84836eef296a (patch) | |
| tree | d8a74d4a32d88e71c98b6460618d1e6a63017fc0 | |
| parent | 2900aa07dfbb857382088ab4148f29dfd04f0123 (diff) | |
| download | genenetwork4-4dbb553d5e4e48a785bbb5f4b97a84836eef296a.tar.gz | |
correlation: vendor the engine's git dependency
Cargo.toml pins gn-rust-correlation 12c2a37 from the genenetwork forge (refs/heads/main there; GitHub still sits at 473ad82). The lock grows by exactly the four packages this brings in -- correlation_rust, GSL, GSL-sys, assert_approx_eq -- and nothing else moves. Following the xapian-rs precedent for a git dependency, since guix's cargo build is offline and the registry replacement only knows crates.io: - rust-package.scm gains the git origin as a bare `origin' (not a crate-source: unpack-rust-crates copies a *directory* input into guix-vendor/, so a checkout is exactly what it wants), plus crate sources for the three new registry crates; - guix.scm lists them and adds a second source-replacement stanza beside xapian's, keyed `git+<url>?rev=<rev>` because cargo resolves a git source by the exact id in Cargo.lock; - both git checkouts get the empty `.cargo-checksum.json` stub; - gsl is a native input: the p-values come from GSL's t-distribution through rgsl, found with pkg-config at build time. One fix in the vendoring phase itself: the gawk that back-fills package checksums named vendor directories from the lock entry verbatim, which is right for lowercase crates with no underscores and wrong for `GSL` and `GSL-sys` (crate-source lowercases, via crate-name->package-name). It lowercases now, so the two agree. Cargo cannot fetch this repo at all -- its HEAD is dangling, so cargo's "fetch HEAD and every head" fails where guix's fetch-by-sha does not. Generating the lock therefore needed CARGO_NET_OFFLINE with the crate pre-cloned into ~/.cargo/git/db; nothing about that affects the build, which vendors the checkout.
| -rw-r--r-- | Cargo.lock | 38 | ||||
| -rw-r--r-- | Cargo.toml | 1 | ||||
| -rw-r--r-- | guix.scm | 52 | ||||
| -rw-r--r-- | rust-package.scm | 29 |
4 files changed, 110 insertions, 10 deletions
diff --git a/Cargo.lock b/Cargo.lock index e5ccd59..f895a6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,6 +3,26 @@ version = 4 [[package]] +name = "GSL" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9becaf6d7d1ba36a457288e661fa6a0472e8328629276f45369eafcd48ef1ce" +dependencies = [ + "GSL-sys", + "paste", +] + +[[package]] +name = "GSL-sys" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577670dcc0720995dc39f04c438595eaae8ccc27f4aafd3e572dd408d01bd9d" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] name = "addr2line" version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -235,6 +255,12 @@ dependencies = [ ] [[package]] +name = "assert_approx_eq" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c07dab4369547dbe5114677b33fbbf724971019f3818172d59a97a61c774ffd" + +[[package]] name = "atoi" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -658,6 +684,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] +name = "correlation_rust" +version = "0.1.0" +source = "git+https://git.genenetwork.org/correlation_rust?rev=12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6#12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6" +dependencies = [ + "GSL", + "assert_approx_eq", + "serde", + "serde_json", +] + +[[package]] name = "cpufeatures" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1036,6 +1073,7 @@ name = "genenetwork4" version = "0.1.0" dependencies = [ "axum", + "correlation_rust", "duckdb", "gnauth-macros", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index 72df08f..0733e4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] axum = "0.8.9" +correlation_rust = { git = "https://git.genenetwork.org/correlation_rust", rev = "12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6", version = "0.1.0" } duckdb = "1.10506.0" gnauth-macros = { path = "gnauth-macros" } jsonwebtoken = "9.3.1" diff --git a/guix.scm b/guix.scm index 7fea85e..c5067d4 100644 --- a/guix.scm +++ b/guix.scm @@ -30,6 +30,7 @@ (gnu packages python) (gnu packages search) ; xapian: the native engine links it (pinned ; to 1.4.x below, see xapian-1.4) + (gnu packages maths) ; gsl: p-values in the correlation engine (rgsl) (gnu packages statistics) ; r: POST /compute/rqtl (gnu packages cran) ; r-qtl, r-optparse, r-stringi, r-stringr: ditto (gnu packages version-control) @@ -150,18 +151,38 @@ replace-with = 'vendored-sources' ;; The JSON quotes are built via %c so the awk ;; program needs no escaped literals. (invoke "gawk" - "/^name = / { name = $0; sub(/^[^\"]*\"/, \"\", name); sub(/\".*$/, \"\", name); gsub(/_/, \"-\", name) }\n/^version = / { ver = $0; sub(/^[^\"]*\"/, \"\", ver); sub(/\".*$/, \"\", ver) }\n/^checksum = / { chk = $0; sub(/^[^\"]*\"/, \"\", chk); sub(/\".*$/, \"\", chk);\n q = sprintf(\"%c\", 34);\n file = \"guix-vendor/rust-\" name \"-\" ver \".tar.gz/.cargo-checksum.json\";\n print \"{\" q \"files\" q \":{},\" q \"package\" q \":\" q chk q \"}\" > file }" + "/^name = / { name = $0; sub(/^[^\"]*\"/, \"\", name); sub(/\".*$/, \"\", name); name = tolower(name); gsub(/_/, \"-\", name) }\n/^version = / { ver = $0; sub(/^[^\"]*\"/, \"\", ver); sub(/\".*$/, \"\", ver) }\n/^checksum = / { chk = $0; sub(/^[^\"]*\"/, \"\", chk); sub(/\".*$/, \"\", chk);\n q = sprintf(\"%c\", 34);\n file = \"guix-vendor/rust-\" name \"-\" ver \".tar.gz/.cargo-checksum.json\";\n print \"{\" q \"files\" q \":{},\" q \"package\" q \":\" q chk q \"}\" > file }" "Cargo.lock") - ;; The git checkout carries no checksum in the - ;; lock; stub it (empty files map = no per-file + ;; The git checkouts carry no checksum in the lock; + ;; stub each (empty files map = no per-file ;; verification, the same trust level as above). - (let ((xapian-dir - (car (find-files vendor-dir "^rust-xapian-rs" - #:directories? #t)))) - (call-with-output-file - (string-append xapian-dir "/.cargo-checksum.json") - (lambda (port) - (display "{\"files\":{}}" port))))))) + (for-each + (lambda (pattern) + (let ((dir (car (find-files vendor-dir pattern + #:directories? #t)))) + (call-with-output-file + (string-append dir "/.cargo-checksum.json") + (lambda (port) + (display "{\"files\":{}}" port))))) + (list "^rust-xapian-rs" "^rust-correlation_rust"))))) + + ;; The correlation engine is a second git dependency, so it + ;; needs its own source-replacement stanza: cargo resolves a + ;; git source by the exact id in Cargo.lock. + (add-after 'patch-cargo-checksums 'vendor-correlation-crate + (lambda* (#:key source #:allow-other-keys) + (let* ((rev "12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6") + (git-url "https://git.genenetwork.org/correlation_rust") + (port (open-file ".cargo/config" "a" + #:encoding "utf-8"))) + (format port " +[source.\"git+~a?rev=~a\"] +git = \"~a\" +rev = \"~a\" +replace-with = 'vendored-sources' +" + git-url rev git-url rev) + (close-port port)))) ;; The install phase must pass --locked: `cargo install' ;; re-resolves the dependency graph and, for a ;; source-replaced GIT dependency, refuses to trust the @@ -234,6 +255,9 @@ replace-with = 'vendored-sources' (native-inputs (list git-minimal python + ;; The correlation engine computes p-values through rgsl, and + ;; gsl-sys finds GSL with pkg-config at build time. + gsl ;; autocxx-bindgen must use libclang 13, not 22: clang 22 ;; mis-parses Xapian 1.4.29 and the generated C++ ;; instantiates the abstract Xapian::Weight (and emits a @@ -533,6 +557,14 @@ replace-with = 'vendored-sources' rust-unicode-linebreak-0.1.5 rust-unicode-width-0.1.14 rust-xapian-rs-0.3.0.70c5eac + ;; The correlation engine's git dependency, for + ;; /compute/correlation: rust-correlation_rust-0.1.0.12c2a37 is + ;; its git origin (vendored the same way as xapian-rs). + rust-correlation_rust-0.1.0.12c2a37 + ;; and the three crates it pulls in (the p-values come from GSL): + rust-gsl-6.0.0 + rust-gsl-sys-3.0.0 + rust-assert-approx-eq-1.1.0 ;; Added for the embedded stores (LMDB, DuckDB, SQLite): rust-ahash-0.8.12 rust-android-system-properties-0.1.6 diff --git a/rust-package.scm b/rust-package.scm index aab55ce..6b1c80c 100644 --- a/rust-package.scm +++ b/rust-package.scm @@ -97,6 +97,10 @@ (crate-source "atoi" "2.0.0" "0a05h42fggmy7h0ajjv6m7z72l924i7igbx13hk9d8pyign9k3gj")) +(define-public rust-assert-approx-eq-1.1.0 + (crate-source "assert_approx_eq" "1.1.0" + "1zagfwfad5wssmr830gk3489f97ppczv6xs627jxniwm6ssdl1rw")) + (define-public rust-atomic-waker-1.1.2 (crate-source "atomic-waker" "1.1.2" "1h5av1lw56m0jf0fd3bchxq8a30xv0b4wv8s4zkp4s0i7mfvs18m")) @@ -415,6 +419,16 @@ (crate-source "futures-util" "0.3.34" "1g3r9ghzq7c2fh34lis43i72xavk9p84npgfwgb5vfpqcwjajl0d")) +(define-public rust-gsl-6.0.0 + ;; Reached through the correlation engine (rgsl -> gsl-sys); the p-values + ;; are computed by GSL's t-distribution, so this is not optional. + (crate-source "GSL" "6.0.0" + "1kpiivagrsk9ags7d4k2521jwix0lqgnd3i8ayj3dfniszvcmgn9")) + +(define-public rust-gsl-sys-3.0.0 + (crate-source "GSL-sys" "3.0.0" + "17dx066l1pbjwp9syjkzqb6fiajyb4wc814zqdfrj807rh6nfxs5")) + (define-public rust-generic-array-0.14.7 (crate-source "generic-array" "0.14.7" "16lyyrzrljfq424c3n8kfwkqihlimmsg5nhshbbp48np3yjrqr45")) @@ -1440,6 +1454,21 @@ (crate-source "writeable" "0.6.4" "1p3r4s4wbf3dksfpj3xyrn7id5p0f7r74mj6qx6ngjfd6cm2vn1s")) +(define-public rust-correlation_rust-0.1.0.12c2a37 + ;; Git dependency of /compute/correlation (Cargo.toml pins this rev). + ;; guix.scm vendors it as a directory source, the same way as + ;; rust-xapian-rs below; see its 'patch-cargo-checksums phase for the + ;; cargo source replacement. 12c2a37 is the commit that makes + ;; `Spearman' public, so a caller holding two value vectors can compute + ;; the rank correlation without going through `Compute' -- the + ;; in-memory path docs/correlations.md asks for. + (origin + (method git-fetch) + (uri (git-reference (url "https://git.genenetwork.org/correlation_rust") + (commit "12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6"))) + (file-name (git-file-name "rust-correlation_rust" "0.1.0.12c2a37")) + (sha256 (base32 "15wxzbzf7l4qvfdjd5y51ymjg1knvy74xndlzdhh9gn53sm1a20b")))) + (define-public rust-xapian-rs-0.3.0.70c5eac ;; Git dependency of the native search engine (Cargo.toml pins this ;; rev). guix.scm vendors it as a directory source; see its |
