about summary refs log tree commit diff
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-15 01:22:23 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-15 01:22:23 +0000
commit4dbb553d5e4e48a785bbb5f4b97a84836eef296a (patch)
treed8a74d4a32d88e71c98b6460618d1e6a63017fc0
parent2900aa07dfbb857382088ab4148f29dfd04f0123 (diff)
downloadgenenetwork4-4dbb553d5e4e48a785bbb5f4b97a84836eef296a.tar.gz
correlation: vendor the engine's git dependency
Cargo.toml pins gn-rust-correlation 12c2a37 from the genenetwork forge
(refs/heads/main there; GitHub still sits at 473ad82).  The lock grows by
exactly the four packages this brings in -- correlation_rust, GSL,
GSL-sys, assert_approx_eq -- and nothing else moves.

Following the xapian-rs precedent for a git dependency, since guix's
cargo build is offline and the registry replacement only knows crates.io:

- rust-package.scm gains the git origin as a bare `origin' (not a
  crate-source: unpack-rust-crates copies a *directory* input into
  guix-vendor/, so a checkout is exactly what it wants), plus crate
  sources for the three new registry crates;
- guix.scm lists them and adds a second source-replacement stanza beside
  xapian's, keyed `git+<url>?rev=<rev>` because cargo resolves a git source
  by the exact id in Cargo.lock;
- both git checkouts get the empty `.cargo-checksum.json` stub;
- gsl is a native input: the p-values come from GSL's t-distribution
  through rgsl, found with pkg-config at build time.

One fix in the vendoring phase itself: the gawk that back-fills package
checksums named vendor directories from the lock entry verbatim, which is
right for lowercase crates with no underscores and wrong for `GSL` and
`GSL-sys` (crate-source lowercases, via crate-name->package-name).  It
lowercases now, so the two agree.

Cargo cannot fetch this repo at all -- its HEAD is dangling, so cargo's
"fetch HEAD and every head" fails where guix's fetch-by-sha does not.
Generating the lock therefore needed CARGO_NET_OFFLINE with the crate
pre-cloned into ~/.cargo/git/db; nothing about that affects the build,
which vendors the checkout.
-rw-r--r--Cargo.lock38
-rw-r--r--Cargo.toml1
-rw-r--r--guix.scm52
-rw-r--r--rust-package.scm29
4 files changed, 110 insertions, 10 deletions
diff --git a/Cargo.lock b/Cargo.lock
index e5ccd59..f895a6c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3,6 +3,26 @@
 version = 4
 
 [[package]]
+name = "GSL"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9becaf6d7d1ba36a457288e661fa6a0472e8328629276f45369eafcd48ef1ce"
+dependencies = [
+ "GSL-sys",
+ "paste",
+]
+
+[[package]]
+name = "GSL-sys"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4577670dcc0720995dc39f04c438595eaae8ccc27f4aafd3e572dd408d01bd9d"
+dependencies = [
+ "libc",
+ "pkg-config",
+]
+
+[[package]]
 name = "addr2line"
 version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -235,6 +255,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "assert_approx_eq"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c07dab4369547dbe5114677b33fbbf724971019f3818172d59a97a61c774ffd"
+
+[[package]]
 name = "atoi"
 version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -658,6 +684,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
+name = "correlation_rust"
+version = "0.1.0"
+source = "git+https://git.genenetwork.org/correlation_rust?rev=12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6#12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6"
+dependencies = [
+ "GSL",
+ "assert_approx_eq",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
 name = "cpufeatures"
 version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1036,6 +1073,7 @@ name = "genenetwork4"
 version = "0.1.0"
 dependencies = [
  "axum",
+ "correlation_rust",
  "duckdb",
  "gnauth-macros",
  "http-body-util",
diff --git a/Cargo.toml b/Cargo.toml
index 72df08f..0733e4b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2024"
 
 [dependencies]
 axum = "0.8.9"
+correlation_rust = { git = "https://git.genenetwork.org/correlation_rust", rev = "12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6", version = "0.1.0" }
 duckdb = "1.10506.0"
 gnauth-macros = { path = "gnauth-macros" }
 jsonwebtoken = "9.3.1"
diff --git a/guix.scm b/guix.scm
index 7fea85e..c5067d4 100644
--- a/guix.scm
+++ b/guix.scm
@@ -30,6 +30,7 @@
   (gnu packages python)
   (gnu packages search)     ; xapian: the native engine links it (pinned
                             ; to 1.4.x below, see xapian-1.4)
+  (gnu packages maths)      ; gsl: p-values in the correlation engine (rgsl)
   (gnu packages statistics) ; r: POST /compute/rqtl
   (gnu packages cran)       ; r-qtl, r-optparse, r-stringi, r-stringr: ditto
   (gnu packages version-control)
@@ -150,18 +151,38 @@ replace-with = 'vendored-sources'
                      ;; The JSON quotes are built via %c so the awk
                      ;; program needs no escaped literals.
                      (invoke "gawk"
-                             "/^name = / { name = $0; sub(/^[^\"]*\"/, \"\", name); sub(/\".*$/, \"\", name); gsub(/_/, \"-\", name) }\n/^version = / { ver = $0; sub(/^[^\"]*\"/, \"\", ver); sub(/\".*$/, \"\", ver) }\n/^checksum = / { chk = $0; sub(/^[^\"]*\"/, \"\", chk); sub(/\".*$/, \"\", chk);\n  q = sprintf(\"%c\", 34);\n  file = \"guix-vendor/rust-\" name \"-\" ver \".tar.gz/.cargo-checksum.json\";\n  print \"{\" q \"files\" q \":{},\" q \"package\" q \":\" q chk q \"}\" > file }"
+                             "/^name = / { name = $0; sub(/^[^\"]*\"/, \"\", name); sub(/\".*$/, \"\", name); name = tolower(name); gsub(/_/, \"-\", name) }\n/^version = / { ver = $0; sub(/^[^\"]*\"/, \"\", ver); sub(/\".*$/, \"\", ver) }\n/^checksum = / { chk = $0; sub(/^[^\"]*\"/, \"\", chk); sub(/\".*$/, \"\", chk);\n  q = sprintf(\"%c\", 34);\n  file = \"guix-vendor/rust-\" name \"-\" ver \".tar.gz/.cargo-checksum.json\";\n  print \"{\" q \"files\" q \":{},\" q \"package\" q \":\" q chk q \"}\" > file }"
                              "Cargo.lock")
-                     ;; The git checkout carries no checksum in the
-                     ;; lock; stub it (empty files map = no per-file
+                     ;; The git checkouts carry no checksum in the lock;
+                     ;; stub each (empty files map = no per-file
                      ;; verification, the same trust level as above).
-                     (let ((xapian-dir
-                            (car (find-files vendor-dir "^rust-xapian-rs"
-                                             #:directories? #t))))
-                       (call-with-output-file
-                           (string-append xapian-dir "/.cargo-checksum.json")
-                         (lambda (port)
-                           (display "{\"files\":{}}" port)))))))
+                     (for-each
+                      (lambda (pattern)
+                        (let ((dir (car (find-files vendor-dir pattern
+                                                    #:directories? #t))))
+                          (call-with-output-file
+                              (string-append dir "/.cargo-checksum.json")
+                            (lambda (port)
+                              (display "{\"files\":{}}" port)))))
+                      (list "^rust-xapian-rs" "^rust-correlation_rust")))))
+
+               ;; The correlation engine is a second git dependency, so it
+               ;; needs its own source-replacement stanza: cargo resolves a
+               ;; git source by the exact id in Cargo.lock.
+               (add-after 'patch-cargo-checksums 'vendor-correlation-crate
+                 (lambda* (#:key source #:allow-other-keys)
+                   (let* ((rev "12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6")
+                          (git-url "https://git.genenetwork.org/correlation_rust")
+                          (port (open-file ".cargo/config" "a"
+                                           #:encoding "utf-8")))
+                     (format port "
+[source.\"git+~a?rev=~a\"]
+git = \"~a\"
+rev = \"~a\"
+replace-with = 'vendored-sources'
+"
+                             git-url rev git-url rev)
+                     (close-port port))))
                ;; The install phase must pass --locked: `cargo install'
                ;; re-resolves the dependency graph and, for a
                ;; source-replaced GIT dependency, refuses to trust the
@@ -234,6 +255,9 @@ replace-with = 'vendored-sources'
     (native-inputs
      (list git-minimal
            python
+           ;; The correlation engine computes p-values through rgsl, and
+           ;; gsl-sys finds GSL with pkg-config at build time.
+           gsl
            ;; autocxx-bindgen must use libclang 13, not 22: clang 22
            ;; mis-parses Xapian 1.4.29 and the generated C++
            ;; instantiates the abstract Xapian::Weight (and emits a
@@ -533,6 +557,14 @@ replace-with = 'vendored-sources'
            rust-unicode-linebreak-0.1.5
            rust-unicode-width-0.1.14
            rust-xapian-rs-0.3.0.70c5eac
+           ;; The correlation engine's git dependency, for
+           ;; /compute/correlation: rust-correlation_rust-0.1.0.12c2a37 is
+           ;; its git origin (vendored the same way as xapian-rs).
+           rust-correlation_rust-0.1.0.12c2a37
+           ;; and the three crates it pulls in (the p-values come from GSL):
+           rust-gsl-6.0.0
+           rust-gsl-sys-3.0.0
+           rust-assert-approx-eq-1.1.0
            ;; Added for the embedded stores (LMDB, DuckDB, SQLite):
            rust-ahash-0.8.12
            rust-android-system-properties-0.1.6
diff --git a/rust-package.scm b/rust-package.scm
index aab55ce..6b1c80c 100644
--- a/rust-package.scm
+++ b/rust-package.scm
@@ -97,6 +97,10 @@
   (crate-source "atoi" "2.0.0"
                 "0a05h42fggmy7h0ajjv6m7z72l924i7igbx13hk9d8pyign9k3gj"))
 
+(define-public rust-assert-approx-eq-1.1.0
+  (crate-source "assert_approx_eq" "1.1.0"
+                "1zagfwfad5wssmr830gk3489f97ppczv6xs627jxniwm6ssdl1rw"))
+
 (define-public rust-atomic-waker-1.1.2
   (crate-source "atomic-waker" "1.1.2"
                 "1h5av1lw56m0jf0fd3bchxq8a30xv0b4wv8s4zkp4s0i7mfvs18m"))
@@ -415,6 +419,16 @@
   (crate-source "futures-util" "0.3.34"
                 "1g3r9ghzq7c2fh34lis43i72xavk9p84npgfwgb5vfpqcwjajl0d"))
 
+(define-public rust-gsl-6.0.0
+  ;; Reached through the correlation engine (rgsl -> gsl-sys); the p-values
+  ;; are computed by GSL's t-distribution, so this is not optional.
+  (crate-source "GSL" "6.0.0"
+                "1kpiivagrsk9ags7d4k2521jwix0lqgnd3i8ayj3dfniszvcmgn9"))
+
+(define-public rust-gsl-sys-3.0.0
+  (crate-source "GSL-sys" "3.0.0"
+                "17dx066l1pbjwp9syjkzqb6fiajyb4wc814zqdfrj807rh6nfxs5"))
+
 (define-public rust-generic-array-0.14.7
   (crate-source "generic-array" "0.14.7"
                 "16lyyrzrljfq424c3n8kfwkqihlimmsg5nhshbbp48np3yjrqr45"))
@@ -1440,6 +1454,21 @@
   (crate-source "writeable" "0.6.4"
                 "1p3r4s4wbf3dksfpj3xyrn7id5p0f7r74mj6qx6ngjfd6cm2vn1s"))
 
+(define-public rust-correlation_rust-0.1.0.12c2a37
+  ;; Git dependency of /compute/correlation (Cargo.toml pins this rev).
+  ;; guix.scm vendors it as a directory source, the same way as
+  ;; rust-xapian-rs below; see its 'patch-cargo-checksums phase for the
+  ;; cargo source replacement.  12c2a37 is the commit that makes
+  ;; `Spearman' public, so a caller holding two value vectors can compute
+  ;; the rank correlation without going through `Compute' -- the
+  ;; in-memory path docs/correlations.md asks for.
+  (origin
+    (method git-fetch)
+    (uri (git-reference (url "https://git.genenetwork.org/correlation_rust")
+                        (commit "12c2a37c3ef3a09f1ab0c5ea69b31b433b15f9f6")))
+    (file-name (git-file-name "rust-correlation_rust" "0.1.0.12c2a37"))
+    (sha256 (base32 "15wxzbzf7l4qvfdjd5y51ymjg1knvy74xndlzdhh9gn53sm1a20b"))))
+
 (define-public rust-xapian-rs-0.3.0.70c5eac
   ;; Git dependency of the native search engine (Cargo.toml pins this
   ;; rev).  guix.scm vendors it as a directory source; see its