about summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore1
-rw-r--r--.guix-channel8
-rw-r--r--.guix/modules/gn-guile.scm34
-rw-r--r--README.md43
-rwxr-xr-xgn-guile.sh3
-rw-r--r--gn/data/dataset.scm61
-rw-r--r--gn/data/genotype.scm1
-rw-r--r--gn/data/strains.scm2
-rw-r--r--gn/db/mysql.scm14
-rw-r--r--gn/db/sources/wikidata.scm50
-rw-r--r--gn/db/sparql.scm59
-rw-r--r--gn/runner/gemma.scm15
l---------[-rw-r--r--]guix.scm80
-rw-r--r--manifest.scm24
-rwxr-xr-xscripts/lmdb-publishdata-export.scm229
-rwxr-xr-xscripts/precompute/list-traits-to-compute.scm4
-rw-r--r--web/README.md1
-rw-r--r--web/css/gn-template-style.css39
-rw-r--r--web/templates/genenetwork.scm18
-rw-r--r--web/view/brand/aging.scm7
-rw-r--r--web/view/brand/msk.scm2
-rw-r--r--web/view/doc.scm2
-rw-r--r--web/view/view.scm16
-rw-r--r--web/webserver.scm74
24 files changed, 659 insertions, 128 deletions
diff --git a/.gitignore b/.gitignore
index 55bfd36..5f81cf8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@ BXD.*
 pheno.txt
 GWA.json
 K.json
+.aider*
diff --git a/.guix-channel b/.guix-channel
new file mode 100644
index 0000000..6e0982a
--- /dev/null
+++ b/.guix-channel
@@ -0,0 +1,8 @@
+(channel
+ (version 0)
+ (directory ".guix/modules")
+ (dependencies
+  (channel
+   (name guix-bioinformatics)
+   (url "https://git.genenetwork.org/guix-bioinformatics")
+   (branch "master"))))
\ No newline at end of file
diff --git a/.guix/modules/gn-guile.scm b/.guix/modules/gn-guile.scm
new file mode 100644
index 0000000..03f2b14
--- /dev/null
+++ b/.guix/modules/gn-guile.scm
@@ -0,0 +1,34 @@
+;; To use this file to build HEAD of gn-guile:
+;;
+;;   guix build -f guix.scm
+;;
+;; To get a development container
+;;
+;;   guix shell -C -D -f guix.scm
+;;
+
+(define-module (gn-guile)
+  #:use-module ((gn packages guile) #:select (gn-guile) #:prefix gn:)
+  #:use-module (guix gexp)
+  #:use-module (guix utils)
+  #:use-module (guix packages)
+  #:use-module (guix git-download))
+
+(define %source-dir (dirname (dirname (current-source-directory))))
+
+(define vcs-file?
+  (or (git-predicate %source-dir)
+      (const #t)))
+
+(define-public gn-guile
+  (package
+    (inherit gn:gn-guile)
+    (source
+     (local-file "../.."
+                 "gn-guile-checkout"
+                 #:recursive? #t
+                 #:select? vcs-file?))))
+
+;; Add definition for tests should you need it, here.
+
+gn-guile
diff --git a/README.md b/README.md
index d3326bd..923b64e 100644
--- a/README.md
+++ b/README.md
@@ -25,31 +25,60 @@ The current repository lives at
 git clone tux02.genenetwork.org:/home/git/public/gn-guile
 ```
 
-GNU Guile allows you to develop against a live running web server using emacs-geiser. To try this fire up the web server from the `web` directory as
+or
+
+```
+git remote add gn git.genenetwork.org:/home/git/public/gn-guile
+```
+
+GNU Guile allows you to develop against a live running web server using emacs-geiser on port 1970. To try this fire up the web server from the `web` directory as
 
 ```sh
-guix shell --container --network --file=guix.scm -- guile -L . --fresh-auto-compile --listen=1970 -e main web/webserver.scm 8091
+guix shell -L ~/guix-bioinformatics -m manifest.scm --container --network --file=guix.scm -- guile -L . --fresh-auto-compile --listen=1970 -e main web/webserver.scm 8091
 ```
 
+By default the root points to the API:
+
+curl http://127.0.0.1:8091
+
+We also have some services
+
+curl http://127.0.0.1:8091/home/msk
+curl http://127.0.0.1:8091/home/aging
+
 The `--container` option runs the code in an isolated container, and the `--network` option connects that container's networking to the host to allow you to access the running service.
 
+If you get an error `no code for module (gn packages guile)` add the appropriate guix-bioinformatics repo to the load path (`-L` switch). Note that, instead of hard-wiring guix-bioinformatics, the recommended way is to use a guix channel as defined in [guix-channel](./.guix-channel).
+
+We recommend checking the Guix documentation for manifests, channels and guix.scm definitions.
+
+To run a standalone server you should run without the listener on port 1970:
+
+```
+guix shell -L ~/guix-bioinformatics -m manifest.scm --container --network --file=guix.scm -- guile -L . --fresh-auto-compile -e main web/webserver.scm 8091
+```
+
 ## Welcome to the world of interactive Lisp programming
 
 Next fire up emacs with `emacs-geiser-guile` and connect to the running web server with `M-x geiser-connect` and the port `1970`. Now you can not only inspect procedures, but also update any prodedure on the live server using `C-M-x` on code, call and get updated output from the webserver! No need to save/reload files and all that.
 
 Note that you may have to try different versions of guile+emacs to succeed.
 
+Also these days it may be a better bet to use Andrew Tropin's alternatives ares and arei that are very powerful alternatives to geiser (with a more common lisp type interactive experience):
+
+=> https://git.sr.ht/~abcdw/guile-ares-rs
+=> https://git.sr.ht/~abcdw/emacs-arei
+
+
 # Tools
 
 Some tooling and scripts that run independently are stored in `./scripts`.
 
+# Forwarding a MySQL port
 
+You may want to forward a mysql port if there is no DB locally you can do something like:
 
-# Development
-
-```
-git remote add gn git.genenetwork.org:/home/git/public/gn-guile
-```
+ssh -L 3306:127.0.0.1:3306 -f -N tux02.genenetwork.org
 
 # Topics
 
diff --git a/gn-guile.sh b/gn-guile.sh
new file mode 100755
index 0000000..9341b26
--- /dev/null
+++ b/gn-guile.sh
@@ -0,0 +1,3 @@
+#! @SHELL@
+
+guile -e main web/webserver.scm "$@"
diff --git a/gn/data/dataset.scm b/gn/data/dataset.scm
index c28cf25..afe75ba 100644
--- a/gn/data/dataset.scm
+++ b/gn/data/dataset.scm
@@ -4,14 +4,21 @@
   #:use-module (ice-9 iconv)
   #:use-module (ice-9 receive)
   #:use-module (ice-9 string-fun)
+  #:use-module (srfi srfi-1)
   #:use-module (dbi dbi)
   #:use-module (gn db mysql)
+  #:use-module (gn data genotype)
   #:use-module (gn data group)
   #:use-module (gn util convert)
   #:use-module (web gn-uri)
+  #:use-module (rnrs base) ; for assert
 
   #:export (
             dataset-name
+            get-bxd-publish-list
+            get-bxd-publish-values-list
+            get-bxd-publish-name-value-dict
+            get-bxd-publish-dataid-name-value-dict
             ))
 
 (define (get-dataset db probesetfreeze-id)
@@ -22,3 +29,57 @@
 
 (define (dataset-name db probesetfreeze-id)
   (assoc-ref (get-dataset db probesetfreeze-id) "Name"))
+
+(define (get-dataid-from-publishxrefid id)
+  "Get the internal dataid from publishxref - which is the same as used in the GN2 web interface"
+  (call-with-db
+   (lambda (db)
+     (let [(query (string-append "SELECT Id,PhenotypeId,DataId FROM PublishXRef WHERE Id=" id " AND InbredSetId=1 LIMIT 1"))]
+       (dbi-query db query)
+       (pk (int-to-string (assoc-ref (get-row db) "DataId")))))))
+
+(define (get-bxd-publish-list)
+  (call-with-db
+   (lambda (db)
+     (let [(query "SELECT Id,PhenotypeId,DataId FROM PublishXRef WHERE InbredSetId=1")]
+       (dbi-query db query)
+       (get-rows db '())))))
+
+(define* (get-bxd-publish-values-list dataid #:optional used-for-mapping?)
+  "Returns dict of name values , e.g. [{\"Name\":\"C57BL/6J\",\"value\":9.136},{\"Name\":\"DBA/2J\",\"value\":4.401},{\"Name\":\"BXD9\",\"value\":4.36}, ... used-for-mapping? skips the founders and maybe other unmappable inds. Note, currently unused."
+  (call-with-db
+   (lambda (db)
+     (let [(query (string-append "SELECT Strain.Name, PublishData.value FROM Strain, PublishData WHERE PublishData.Id=" dataid " and Strain.Id=StrainID;"))]
+       (dbi-query db query)
+       (if used-for-mapping?
+           (remove null? (pk (get-rows-apply db
+                                             (lambda (r)
+                                               (if (string-contains (assoc-ref r "Name") "BXD")
+                                                   `(("Name" . ,(assoc-ref r "Name")) ("value" . ,(assoc-ref r "value")))
+                                                   '() ) ;; return empty on no match
+                                               ) '())))
+           (get-rows db '())
+           )))))
+
+(define* (get-bxd-publish-dataid-name-value-dict dataid #:optional used-for-mapping?)
+  "Returns dict of name values, e.g. (((\"C57BL/6J\" . 9.136) (\"DBA/2J\" . 4.401) (\"BXD9\" . 4.36) ... used-for-mapping? skips the founders and maybe other unmappable inds."
+  (call-with-db
+   (lambda (db)
+     (let [(query (string-append "SELECT Strain.Name, PublishData.value FROM Strain, PublishData WHERE PublishData.Id=" dataid " and Strain.Id=StrainID;"))]
+       (dbi-query db query)
+       (if used-for-mapping?
+           (remove null? (pk (get-rows-apply db
+                                             (lambda (r)
+                                               (if (string-contains (assoc-ref r "Name") "BXD")
+                                                   `(,(assoc-ref r "Name") . ,(assoc-ref r "value"))
+                                                   '() ) ;; return empty on no match
+                                               ) '())))
+           (remove null? (pk (get-rows-apply db
+                                             (lambda (r)
+                                               `(,(assoc-ref r "Name") . ,(assoc-ref r "value"))
+                                               ) '())))
+           )))))
+
+(define* (get-bxd-publish-name-value-dict id #:optional used-for-mapping?)
+  "Same as above function, but starting from data id"
+  (get-bxd-publish-dataid-name-value-dict (get-dataid-from-publishxrefid id) used-for-mapping?))
diff --git a/gn/data/genotype.scm b/gn/data/genotype.scm
index c7cb63c..5574382 100644
--- a/gn/data/genotype.scm
+++ b/gn/data/genotype.scm
@@ -16,6 +16,7 @@
             ))
 
 (define (geno-inds-bxd fn)
+  "Returns information from GN's BXD.json, note it fetches the first geno file info, now BXD.8.geno"
   (let [(js (call-with-input-file fn
               (lambda (port)
                 (json->scm port))))]
diff --git a/gn/data/strains.scm b/gn/data/strains.scm
index e5f839b..07b69ff 100644
--- a/gn/data/strains.scm
+++ b/gn/data/strains.scm
@@ -25,7 +25,7 @@
   "Return assoc list of tuples of strain id+names:
    ((4 . BXD1) (5 . BXD2) (6 . BXD5) (7 . BXD6)...
 
-used-for-mapping? will say whether the strains/individuals are used for mapping. Always True, FIXME
+optional key used-for-mapping? will say whether the strains/individuals are used for mapping.
 "
   (call-with-db
    (lambda (db)
diff --git a/gn/db/mysql.scm b/gn/db/mysql.scm
index ccd414a..8da7b60 100644
--- a/gn/db/mysql.scm
+++ b/gn/db/mysql.scm
@@ -32,22 +32,26 @@
     ;; (display "===> OPENING DB")
     ;; (newline)
     (let [(db (dbi-open "mysql" "webqtlout:webqtlout:db_webqtl:tcp:127.0.0.1:3306"))]
-      (ensure db)
+      (ensure db "Can't open connection")
       db
     )))
 
 (define (call-with-db thunk)
-  (thunk (db-open)))
+  (let* [(db (db-open))
+         (result (thunk db))]
+    (dbi-close db)
+    result))
 
-(define (ensure db)
+(define (ensure db msg1)
   "Use DBI-style handle to report an error. On error the program will stop."
   (match (dbi-get_status db)
     ((stat . msg) (if (= stat 0)
                      #t
                      (begin
-                       (display msg)
+                       (display "SQL Connection ERROR! ")
+                       (display (string-append msg1 " - " msg)
                        (newline)
-                       (assert stat))))))
+                       (assert #f)))))))
 
 (define (has-result? db)
   "Return #t or #f if result is valid"
diff --git a/gn/db/sources/wikidata.scm b/gn/db/sources/wikidata.scm
index 7397426..954ce93 100644
--- a/gn/db/sources/wikidata.scm
+++ b/gn/db/sources/wikidata.scm
@@ -1,10 +1,38 @@
 #!
 
-Wikidata queries
+Wikidata queries, initially lifted over from the gn3 gene-alias code (that was written in Racket).
 
+Note you can take a SPARQL query and push it into https://query.wikidata.org/. E.g. generate a query and
+copy paste into the query service:
+
+scheme@(guile-user) [3]> (display (wikidata-query-geneids "Shh"))
+```
+SELECT DISTINCT ?wikidata_id
+            WHERE {
+              ?wikidata_id wdt:P31 wd:Q7187;
+                           wdt:P703 ?species .
+              VALUES (?species) { (wd:Q15978631 ) ( wd:Q83310 ) ( wd:Q184224 ) } .
+              ?wikidata_id rdfs:label "Shh"@en .
+              }
+```
+
+It is possible to run queries through curl with
+
+```
+curl -G https://query.wikidata.org/sparql -H "Accept: application/json; charset=utf-8" --data-urlencode query="
+    SELECT DISTINCT ?alias
+             WHERE {
+                     wd:Q24420953 rdfs:label ?name ;
+                         skos:altLabel ?alias .
+                     FILTER(LANG(?name) = \"en\" && LANG(?alias) = \"en\").
+                   }"
+```
 !#
 
 (define-module (gn db sources wikidata)
+  #:export (wikidata-query-geneids
+            wikidata-query-gene-aliases
+            )
 )
 
 (define ps-encoded-by "ps:P702")
@@ -14,16 +42,24 @@ Wikidata queries
 (define wd-mouse "wd:Q83310")
 (define wd-rat "wd:Q184224")
 (define wd-gene "wd:Q7187")
+(define wd-shh-rat "wd:Q24420953")
 
-(define (wikidata_query_geneids gene_name)
-  "Return the wikidata identifiers pointing to genes of listed species"
+(define (wikidata-query-geneids gene_name)
+  "SPARQL query to get the wikidata identifiers pointing to genes of listed species, e.g. 'Shh'"
   (string-append
      "SELECT DISTINCT ?wikidata_id
             WHERE {
               ?wikidata_id " wdt-instance-of " " wd-gene ";
                            " wdt-in-taxon " ?species .
               VALUES (?species) { (" wd-human " ) ( " wd-mouse" ) ( " wd-rat" ) } .
-              ?wikidata_id rdfs:label \"" gene_name "\"@en .
-        }
-"
-              ))
+              ?wikidata_id rdfs:label \"" gene_name "\"@en .}"))
+
+(define (wikidata-query-gene-aliases wikidata_id)
+  "SPARQL query to get a list of gene aliases based on a wikidata identifier, e.g. for Q24420953. This
+version supports the expanded id only, so <http://www.wikidata.org/entity/Q24420953> including the <,>."
+  (string-append
+      "SELECT DISTINCT ?stripped_alias
+             WHERE { " wikidata_id " rdfs:label ?name ;
+                         skos:altLabel ?alias .
+                         BIND (STR(?alias)  AS ?stripped_alias) .
+                     FILTER(LANG(?name) = \"en\" && LANG(?alias) = \"en\").}"))
diff --git a/gn/db/sparql.scm b/gn/db/sparql.scm
index f03389b..bd7a306 100644
--- a/gn/db/sparql.scm
+++ b/gn/db/sparql.scm
@@ -8,18 +8,19 @@ the case.
 !#
 
 (define-module (gn db sparql)
-  #:use-module (json)
-  #:use-module (ice-9 match)
+  #:use-module (gn cache memoize)
+  #:use-module (gn db sources wikidata)
   #:use-module (ice-9 format)
   #:use-module (ice-9 iconv)
+  #:use-module (ice-9 match)
   #:use-module (ice-9 receive)
   #:use-module (ice-9 string-fun)
+  #:use-module (json)
+  #:use-module (srfi srfi-1)
   #:use-module (web client)
+  #:use-module (web gn-uri)
   #:use-module (web request)
   #:use-module (web uri)
-  #:use-module (gn cache memoize)
-  #:use-module (gn db sources wikidata)
-  #:use-module (web gn-uri)
 
   #:export (memo-sparql-species
             memo-sparql-species-meta
@@ -27,6 +28,8 @@ the case.
             sparql-groups-meta
             sparql-group-info
             memo-sparql-wd-species-info
+            memo-sparql-wd-gene-aliases
+            memo-sparql-wd-geneids
             compile-species
             compile-groups-meta
             get-rows
@@ -73,7 +76,9 @@ PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
 
 (define (sparql-tsv endpoint-url query)
   "Execute raw SPARQL query returning response as a UTF8 string, e.g.
-(tsv->scm (sparql-tsv (wd-sparql-endpoint-url) \"wd:Q158695\"))
+(tsv->scm (sparql-tsv (wd-sparql-endpoint-url) \"wd:Q158695\")).
+
+Note this procedure works for wikidata, but not for gn!
 "
   ; GET /sparql?query=SELECT%20DISTINCT%20%2A%20where%20%7B%0A%20%20wd%3AQ158695%20wdt%3AP225%20%3Fo%20.%0A%7D%20limit%205 HTTP/2
   (receive (response-status response-body)
@@ -93,7 +98,9 @@ PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
   (unpack "bindings" (unpack "results" response)))
 
 (define (sparql-scm endpoint-url query)
-  "Return dual S-exp 'resultset' of varnames and results"
+  "Return dual S-exp 'resultset' of varnames and results.
+
+Note this procedure works for GN, but does not yet work for wikidata"
   (let ((response (json-string->scm
                    (sparql-exec endpoint-url (gn-sparql-prefix query)))))
    (values (sparql-names response) (sparql-results response))))
@@ -161,9 +168,47 @@ SELECT DISTINCT ?taxon ?ncbi ?descr where {
 
 ")))
 
+(define (flatten lst)
+  (cond ((null? lst) '())
+        ((pair? lst) (append (flatten (car lst)) (flatten (cdr lst))))
+        (else (list lst))))
+
+(define (remove-quotes s)
+  (substring s 1 (- (string-length s) 1)))
+
 (define memo-sparql-wd-species-info
   (memoize sparql-wd-species-info))
 
+(define (sparql-wd-geneids gene-name)
+  "Return a list of expanded wikidata ids, e.g.
+(\"<http://www.wikidata.org/entity/Q14860079>\" \"<http://www.wikidata.org/entity/Q24420953>\")"
+  (receive (type values)
+      (tsv->scm (sparql-tsv (wd-sparql-endpoint-url) (wikidata-query-geneids gene-name)))
+    (map (lambda (item) (car item)) values) ;; flatten list
+    ))
+
+(define memo-sparql-wd-geneids
+  (memoize sparql-wd-geneids))
+
+(define (sparql-wd-gene-aliases geneids)
+  "Returns a flattened and dedpulicated list of geneids with
+(sparql-wd-gene-aliases '(\"Q14860079\" \"Q24420953\"))
+"
+  (let* ([aliases
+         (map (lambda (geneid)
+                (receive (type values)
+                    (tsv->scm (sparql-tsv (wd-sparql-endpoint-url) (wikidata-query-gene-aliases (pk geneid))))
+                  (map (lambda (item) (car item)) values) ;; flatten list))
+                  )
+                ) geneids)]
+         [rm-quotes-aliases (map (lambda (s) (remove-quotes s)) (flatten aliases))]
+         )
+    (delete-duplicates rm-quotes-aliases)))
+
+(define memo-sparql-wd-gene-aliases
+  (memoize sparql-wd-gene-aliases))
+
+
 #!
 gn:Mus_musculus rdf:type gnc:species .
 gn:Mus_musculus gnt:name "Mouse" .
diff --git a/gn/runner/gemma.scm b/gn/runner/gemma.scm
index 9a5c0fc..c577305 100644
--- a/gn/runner/gemma.scm
+++ b/gn/runner/gemma.scm
@@ -10,11 +10,24 @@
   #:use-module (rnrs base)
 
   #:export (
-            write-pheno-file
+            gemma-pheno-txt
             invoke-gemma-wrapper-loco
             run-gemma
             ))
 
+(define (gemma-pheno-txt family traits)
+  "Return a list of values for GEMMA"
+  (assert (string=? family "BXD")) ; only supported right now
+  (define bxd-inds (geno-inds-bxd "BXD.json"))
+  (assert (= 235 (length bxd-inds)))
+  (map (lambda (ind)
+         (let [(value (assoc-ref traits ind))]
+           (if value
+               (format #f "~a" value)
+               "NA\n")
+           ))
+       bxd-inds))
+
 (define (write-pheno-file fn traits)
   (define bxd-inds (geno-inds-bxd "BXD.json"))
   (assert (= 235 (length bxd-inds)))
diff --git a/guix.scm b/guix.scm
index 1cde2e0..973f44f 100644..120000
--- a/guix.scm
+++ b/guix.scm
@@ -1,79 +1 @@
-;; To use this file to build HEAD of gn-guile:
-;;
-;;   guix build -f guix.scm
-;;
-;; To get a development container
-;;
-;;   guix shell -C -D -f guix.scm
-;;
-
-(define-module (gn-guile-package)
-  #:use-module ((guix licenses) #:prefix license:)
-  #:use-module (guix gexp)
-  #:use-module (guix packages)
-  #:use-module (guix git-download)
-  #:use-module (guix build-system guile)
-  #:use-module (guix utils)
-  #:use-module (gnu packages build-tools)
-  #:use-module  (gnu packages package-management)
-  #:use-module ((gnu packages bash) #:select (bash-minimal))
-  #:use-module ((gnu packages bioinformatics) #:select (gemma))
-  #:use-module ((gnu packages certs) #:select (nss-certs))
-  #:use-module ((gnu packages guile) #:select (guile-json-4 guile-3.0 guile-readline))
-  #:use-module ((gnu packages guile-xyz) #:select (guile-dbi guile-dbd-mysql guile-fibers guile-redis guile-hashing guile-commonmark))
-  #:use-module ((gnu packages parallel) #:select (parallel))
-  #:use-module ((gnu packages perl) #:select (perl))
-  #:use-module ((gnu packages tls) #:select (guile-gnutls openssl))
-  #:use-module (srfi srfi-1)
-  #:use-module (ice-9 popen)
-  #:use-module (ice-9 rdelim))
-
-(define %source-dir (dirname (current-filename)))
-
-(define %git-commit
-    (read-string (open-pipe "git show HEAD | head -1 | cut -d ' ' -f 2" OPEN_READ)))
-
-(define-public gn-guile-git
-  (package
-    (name "gn-guile-git")
-    (version (git-version "4.0.0-" "HEAD" %git-commit))
-    (source (local-file %source-dir
-			#:recursive? #t
-			#:select? (or (git-predicate (dirname (current-source-directory)))
-                                      (const #t))))
-    (build-system guile-build-system)
-    (propagated-inputs
-     (list guile-3.0 guile-dbi guile-dbd-mysql guile-fibers guile-gnutls guile-readline
-	   guile-commonmark guile-redis openssl nss-certs gemma parallel guile-hashing
-           guile-json-4))
-    (arguments
-     (list
-      #:phases
-      #~(modify-phases %standard-phases
-	  ;; When using the guile-build-system, guild doesn't
-	  ;; correctly set the GUILE_LOAD_PATH for the various guile
-	  ;; packages in the build phase leading to build failures.
-	  (add-before 'build 'augment-GUILE_LOAD_PATH
-	    (lambda* (#:key outputs #:allow-other-keys)
-	      (let* ((guile-version (target-guile-effective-version))
-		     (guile-dbi-scm (string-append #$guile-dbi "/share/guile/site/2.2")))
-		;; guild uses this: "\\.(scm|sls)$" regexp to try and
-		;; compile all scm files in this repository.  We don't
-		;; need to compile guix.scm
-		(delete-file "guix.scm")
-		(setenv "GUILE_LOAD_PATH"
-			(string-append
-			 guile-dbi-scm ":"
-			 (format
-			  #f "~{~a:~}"
-			  (map (lambda (package)
-				 (format #f "~a/share/guile/site/~a"
-					 package guile-version))
-			       (list #$guile-fibers #$guile-commonmark #$guile-json-4 #$guile-hashing)))
-			 #$(getenv "GUILE_LOAD_PATH")))))))))
-    (home-page "https://git.genenetwork.com/gn-guile")
-    (synopsis "Next generation GN code in guile")
-    (description "Use of guile.")
-    (license license:gpl3)))
-
-gn-guile-git
+.guix/modules/gn-guile.scm
\ No newline at end of file
diff --git a/manifest.scm b/manifest.scm
new file mode 100644
index 0000000..bc8f699
--- /dev/null
+++ b/manifest.scm
@@ -0,0 +1,24 @@
+;; please do not remove
+(specifications->manifest
+ '("coreutils"
+   "guile"
+   "guile-dbi"
+   "guile-dbd-mysql"
+   "guile-fibers"
+   "guile-json"
+   "guile-gnutls"
+   "guile-readline"
+   "guile-redis"
+   "openssl"
+   "nss-certs"
+   "gemma"
+   "parallel"
+   "tar"
+   "xz"
+   "python"
+   "python-lmdb"
+   "python-cffi"
+   "guile-gcrypt"
+   "guile-hashing"
+   "time"
+   "gemma-gn2"))
diff --git a/scripts/lmdb-publishdata-export.scm b/scripts/lmdb-publishdata-export.scm
new file mode 100755
index 0000000..8427112
--- /dev/null
+++ b/scripts/lmdb-publishdata-export.scm
@@ -0,0 +1,229 @@
+#! /usr/bin/env guile
+!#
+;; To run this script run:
+;;
+;; $ guix shell guile guile-hashing guile3-dbi guile3-dbd-mysql \
+;; guile-lmdb -- guile lmdb-publishdata-export.scm
+;; conn.scm
+;;
+;; Example conn.scm:
+;; ((sql-username . "webqtlout")
+;;  (sql-password . "xxxx")
+;;  (sql-database . "db_webqtl")
+;;  (sql-host . "localhost")
+;;  (sql-port . 3306)
+;;  (output-dir . "/tmp/data")
+;;  (log-file . "/tmp/export.log"))
+
+(use-modules (dbi dbi)
+             (rnrs bytevectors)
+             (system foreign)
+             (ice-9 match)
+             (srfi srfi-1)
+             (srfi srfi-26)
+	     (srfi srfi-43)
+             (rnrs io ports)
+             (hashing md5)
+             ((lmdb lmdb) #:prefix mdb:)
+             (ice-9 format)
+             (ice-9 exceptions)
+	     (json)
+             (logging logger)
+             (logging rotating-log)
+             (logging port-log)
+             (oop goops))
+
+
+;; Set up logging
+(define* (setup-logging #:key (log-file "lmdb-dump-log"))
+  "Initialize the logging system with rotating file logs and error port output.
+   Creates a new logger, adds rotating and error port handlers,
+   sets it as the default logger, and opens the log for writing."
+  (let ((lgr (make <logger>))
+	(rotating (make <rotating-log>
+		    #:num-files 3
+		    #:size-limit 1024
+		    #:file-name log-file))
+	(err (make <port-log> #:port (current-error-port))))
+    ;; add the handlers to our logger
+    (add-handler! lgr rotating)
+    (add-handler! lgr err)
+    ;; make this the application's default logger
+    (set-default-logger! lgr)
+    (open-log! lgr)))
+
+(define (shutdown-logging)
+  "Properly shutdown the logging system.
+   Flushes any pending log messages, closes the log handlers,
+   and removes the default logger reference."
+  (flush-log)  ;; since no args, it uses the default
+  (close-log!) ;; since no args, it uses the default
+  (set-default-logger! #f))
+
+(define (call-with-database backend connection-string proc)
+  "Execute PROC with an open database connection.  BACKEND is the
+database type (e.g. \"mysql\").  CONNECTION-STRING is the database
+connection string."
+  (let ((db #f))
+    (dynamic-wind
+      (lambda ()
+        (set! db (dbi-open backend connection-string)))
+      (cut proc db)
+      (lambda ()
+        (when db
+          (dbi-close db))))))
+
+(define (call-with-target-database connection-settings proc)
+  "Connect to the target database using CONNECTION-SETTINGS and execute
+PROC."
+  (call-with-database "mysql" (string-join
+                               (list (assq-ref connection-settings 'sql-username)
+                                     (assq-ref connection-settings 'sql-password)
+                                     (assq-ref connection-settings 'sql-database)
+                                     "tcp"
+                                     (assq-ref connection-settings 'sql-host)
+                                     (number->string
+                                      (assq-ref connection-settings 'sql-port)))
+                               ":")
+                      proc))
+
+(define* (lmdb-save path key value)
+  "Save a NUM with KEY to PATH."
+  (mdb:with-env-and-txn
+     (path) (env txn)
+     (let ((dbi (mdb:dbi-open txn #f 0)))
+       (mdb:put txn dbi key
+		(if (number? value)
+                    (number->string value)
+		    value)))))
+
+(define (sql-exec db statement)
+  "Execute an SQL STATEMENT on database connection DB.  Throws an error
+if the statement execution fails."
+  (dbi-query db statement)
+  (database-check-status db))
+
+(define (sql-fold proc init db statement)
+  "Fold over SQL query results."
+  (sql-exec db statement)
+  (let loop ((result init))
+    (let ((row (dbi-get_row db)))
+      (if row
+          (loop (proc row result))
+          result))))
+
+(define (sql-for-each proc db statement)
+  "Apply PROC to each row returned by STATEMENT."
+  (sql-fold (lambda (row _)
+              (proc row))
+            #f db statement))
+
+(define (sql-map proc db statement)
+  "Map PROC over rows returned by STATEMENT."
+  (sql-fold (lambda (row result)
+              (cons (proc row) result))
+            (list) db statement))
+
+(define (sql-find db statement)
+  (sql-exec db statement)
+  (dbi-get_row db))
+
+(define (database-check-status db)
+  "Check the status of the last database operation on DB.  Throws an
+error if the status code is non-zero."
+  (match (dbi-get_status db)
+    ((code . str)
+     (unless (zero? code)
+       (error str)))))
+
+(define* (save-dataset-values settings)
+  "Main function to extract and save dataset values.  Queries the
+database for datasets and their values, computes MD5 hashes for
+dataset-trait combinations, and saves strain values to LMDB files in
+/export5/lmdb-data-hashes/."
+  (dynamic-wind
+    (lambda ()
+      (setup-logging #:log-file (assq-ref settings 'log-file))
+      (log-msg 'INFO "Starting dataset value extraction"))
+    (lambda ()
+      (call-with-target-database
+       settings
+       (lambda (db)
+         (sql-for-each
+          (lambda (row)
+	    (match row
+	      ((("Name" . dataset-name)
+		("Id" . trait-id))
+			    (let* ((data-query (format #f "SELECT
+JSON_ARRAYAGG(JSON_ARRAY(Strain.Name, PublishData.Value)) AS data,
+ MD5(JSON_ARRAY(Strain.Name, PublishData.Value)) as md5hash
+FROM
+    PublishData
+    INNER JOIN Strain ON PublishData.StrainId = Strain.Id
+    INNER JOIN PublishXRef ON PublishData.Id = PublishXRef.DataId
+    INNER JOIN PublishFreeze ON PublishXRef.InbredSetId = PublishFreeze.InbredSetId
+LEFT JOIN PublishSE ON
+    PublishSE.DataId = PublishData.Id AND
+    PublishSE.StrainId = PublishData.StrainId
+LEFT JOIN NStrain ON
+    NStrain.DataId = PublishData.Id AND
+    NStrain.StrainId = PublishData.StrainId
+WHERE
+    PublishFreeze.Name = \"~a\" AND
+    PublishXRef.Id = ~a AND
+    PublishFreeze.public > 0 AND
+    PublishData.value IS NOT NULL AND
+    PublishFreeze.confidentiality < 1
+ORDER BY
+    LENGTH(Strain.Name), Strain.Name" dataset-name trait-id)))
+			      (match (call-with-target-database
+				      settings
+				      (lambda (db2) (sql-find db2 data-query)))
+				((("data" . data)
+				  ("md5hash" . md5-hash))
+				 (let* ((trait-name (format #f "~a~a" dataset-name trait-id))
+					(base-dir (assq-ref settings 'output-dir))
+					(out (format #f "~a-~a" trait-name
+						     (substring md5-hash 0 12)))
+					(out-dir (format #f "~a/~a" base-dir out)))
+				   (log-msg
+				    'INFO (format #f "Writing ~a to: ~a" trait-name out-dir))
+				   (unless (file-exists? out-dir)
+				     (mkdir out-dir))
+				   (lmdb-save (format #f "~a/index" base-dir) trait-name out)
+				   (vector-for-each
+				    (lambda (_ x)
+				      (match x
+					(#(strain value)
+					 (lmdb-save out-dir strain value))))
+				    (json-string->scm data)))))))))
+          db
+          "SELECT DISTINCT PublishFreeze.Name, PublishXRef.Id FROM
+PublishData INNER JOIN Strain ON PublishData.StrainId = Strain.Id
+INNER JOIN PublishXRef ON PublishData.Id = PublishXRef.DataId
+INNER JOIN PublishFreeze ON PublishXRef.InbredSetId = PublishFreeze.InbredSetId
+LEFT JOIN PublishSE ON
+    PublishSE.DataId = PublishData.Id AND
+    PublishSE.StrainId = PublishData.StrainId
+LEFT JOIN NStrain ON
+    NStrain.DataId = PublishData.Id AND
+    NStrain.StrainId = PublishData.StrainId
+WHERE
+    PublishFreeze.public > 0 AND
+    PublishFreeze.confidentiality < 1
+ORDER BY
+    PublishFreeze.Id, PublishXRef.Id"))))
+    (lambda ()
+      (shutdown-logging))))
+
+(define main
+  (match-lambda*
+    ((_ connection-settings-file)
+     (save-dataset-values (call-with-input-file connection-settings-file
+			    read)))
+    ((arg0 _ ...)
+     (display (format "Usage: ~a CONNECTION-SETTINGS-FILE~%" arg0)
+	      (current-error-port))
+     (exit #f))))
+
+(apply main (command-line))
diff --git a/scripts/precompute/list-traits-to-compute.scm b/scripts/precompute/list-traits-to-compute.scm
index 9f900d1..102a6fa 100755
--- a/scripts/precompute/list-traits-to-compute.scm
+++ b/scripts/precompute/list-traits-to-compute.scm
@@ -15,6 +15,10 @@ You may want to forward a mysql port if there is no DB locally
 
     ssh -L 3306:127.0.0.1:3306 -f -N tux02.genenetwork.org
 
+ignore IPv6 message:
+
+    bind [::1]:3306: Cannot assign requested address
+
 test connection with mysql client:
 
     mysql -uwebqtlout -pwebqtlout -A -h 127.0.0.1 -P 3306 db_webqtl -e "show tables;"
diff --git a/web/README.md b/web/README.md
new file mode 100644
index 0000000..fc7e158
--- /dev/null
+++ b/web/README.md
@@ -0,0 +1 @@
+Run the webserver from one directory up.
diff --git a/web/css/gn-template-style.css b/web/css/gn-template-style.css
new file mode 100644
index 0000000..38893c6
--- /dev/null
+++ b/web/css/gn-template-style.css
@@ -0,0 +1,39 @@
+* {
+    box-sizing: border-box;
+}
+
+body {
+    margin: 0.7em;
+    display: grid;
+    grid-template-columns: 9fr 1fr;
+    grid-gap: 20px;
+
+    font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+    font-style: normal;
+    font-size: 20px;
+}
+
+#header {
+    grid-column-start: 1;
+    grid-column-end: 3;
+
+    background-color: #336699;
+    color: #FFFFFF;
+    border-radius: 3px;
+    min-height: 30px;
+}
+
+#header #header-text {
+    padding-left: 0.2em;
+}
+
+#main {
+    grid-column-start: 1;
+    grid-column-end: 2;
+
+    max-width: 650px;
+}
+
+#main img {
+    max-width: 650px;
+}
diff --git a/web/templates/genenetwork.scm b/web/templates/genenetwork.scm
new file mode 100644
index 0000000..64e9852
--- /dev/null
+++ b/web/templates/genenetwork.scm
@@ -0,0 +1,18 @@
+(define-module (web templates genenetwork)
+  #:use-module (web view markdown)
+
+  #:export (default-gn-template))
+
+(define* (default-gn-template path #:optional (title "Default Page Template"))
+  "Render `PATH' with a default template and styling that fits in with
+ GeneNetwork's look and feel."
+  `(html
+    (head
+     (meta (@ (charset "UTF-8")))
+     (meta (@ (name "viewport") (content "width=device-width, initial-scale=1.0")))
+     (title ,title)
+     (link (@ (rel "stylesheet") (type "text/css")
+              (href "/css/gn-template-style.css"))))
+    (body
+     (header (@ (id "header")) (span (@ (id "header-text")) "GeneNetwork"))
+     (main (@ (id "main")) ,(markdown-github->sxml path)))))
diff --git a/web/view/brand/aging.scm b/web/view/brand/aging.scm
index 19db4d7..f1c48c9 100644
--- a/web/view/brand/aging.scm
+++ b/web/view/brand/aging.scm
@@ -45,15 +45,14 @@
           ))
       ,@head)
      (body
-      ;; (header (p "TEST"))
       (main (@ (class "container"))
        (h1 ,title)
        (article
-               (img (@ (src "/static/images/ole-farmer.jpg") (alt "ol farmer by hohumhobo is licensed under CC BY 2.0") (width "400") (align "right")))
-,info)
+       ;;        (img (@ (src "/static/images/ole-farmer.jpg") (alt "ol farmer by hohumhobo is licensed under CC BY 2.0") (width "400") (align "right")))
+        ,info)
       (footer
        (hr)
-       (p "Copyright © 2005-2023 "
+       (p "Copyright © 2005-2025 "
           (a (@ (href "https://genenetwork.org/")) "GeneNetwork Webservices") " | GeneNetwork and this website runs fully on free software. See status and download the "
           (a (@ (href "https://ci.genenetwork.org/"))
              "source code") ".")))
diff --git a/web/view/brand/msk.scm b/web/view/brand/msk.scm
index 69c1253..4cbcec4 100644
--- a/web/view/brand/msk.scm
+++ b/web/view/brand/msk.scm
@@ -51,7 +51,7 @@
         (p ,info)
         (footer
        (hr)
-       (p "Copyright © 2005-2023 "
+       (p "Copyright © 2005-2025 "
           (a (@ (href "https://genenetwork.org/")) "GeneNetwork Webservices") " | GeneNetwork and this website runs fully on free software. See status and download the "
           (a (@ (href "https://ci.genenetwork.org/"))
              "source code") ".")))
diff --git a/web/view/doc.scm b/web/view/doc.scm
index 71112eb..cec4400 100644
--- a/web/view/doc.scm
+++ b/web/view/doc.scm
@@ -44,7 +44,7 @@
            ,(scm->json-string body #:pretty #t))
            ; (p ,(parse-html "<b>some raw really <i>text</i> here</b>"))
            (footer
-            (p "Copyright © 2005—2023 by the GeneNetwork community with a touch of " (span (@ (class "lambda")) "λ") "!")
+            (p "Copyright © 2005—2025 by the GeneNetwork community with a touch of " (span (@ (class "lambda")) "λ") "!")
             (p "This is free software. Download the "
                (a (@ (href "https://ci.genenetwork.org/"))
                   "source code") "."))
diff --git a/web/view/view.scm b/web/view/view.scm
index 4584cf8..a7592ad 100644
--- a/web/view/view.scm
+++ b/web/view/view.scm
@@ -10,11 +10,12 @@
   #:use-module (web view markdown)
   #:use-module (web view brand msk)
   #:use-module (web view brand aging)
+  #:use-module (web templates genenetwork)
 
   #:export (view-brand))
 
 
-(define (view-aging)
+(define (view-aging-home)
   (aging-html #:info
               `(
 		,(markdown-github->sxml "genenetwork/gn-docs/general/brand/aging/home.md")
@@ -44,7 +45,18 @@ data to benefit from the power of integrated datasets, please contact:")
 
 (define* (view-brand path)
   (match path
-    ("aging" (view-aging))
+    ("aging/um-het3" (aging-html #:info
+              `(,(markdown-github->sxml "genenetwork/gn-docs/general/brand/aging/home.md"))))
+    ("aging/UM-HET3" (aging-html #:info
+              `(,(markdown-github->sxml "genenetwork/gn-docs/general/brand/aging/home.md"))))
+    ("aging/UMHET-3" (aging-html #:info
+              `(,(markdown-github->sxml "genenetwork/gn-docs/general/brand/aging/home.md"))))
+    ("aging/umhet-3" (aging-html #:info
+              `(,(markdown-github->sxml "genenetwork/gn-docs/general/brand/aging/home.md"))))
+    ("aging" (view-aging-home))
+    ("gnqa" (default-gn-template
+                   "genenetwork/gn-docs/general/brand/gnqa/gnqa.md"
+                   "GeneNetwork Question and Answer System"))
     ( _ (msk-html #:info
             `(
 	      ,(markdown-github->sxml "genenetwork/gn-docs/general/brand/msk/home.md")
diff --git a/web/webserver.scm b/web/webserver.scm
index 6f40614..0c0bdd1 100644
--- a/web/webserver.scm
+++ b/web/webserver.scm
@@ -7,6 +7,7 @@
              (ice-9 exceptions)
              (srfi srfi-1)
              (srfi srfi-11)
+             (srfi srfi-13)
              (srfi srfi-19)
              (srfi srfi-26)
              (rnrs io ports)
@@ -16,17 +17,24 @@
              (web request)
              (web response)
              (web uri)
-             (fibers web server)
+             (web server)
              (gn cache memoize)
              (web gn-uri)
              (gn db sparql)
+             (gn data dataset)
              (gn data species)
              (gn data group)
+             (gn runner gemma)
              (web sxml)
              (web view view)
              (web view doc)
              (web view markdown))
 
+(define (get-extension filename)
+  (let ((dot-pos (string-rindex filename #\.)))
+    (if dot-pos
+        (substring filename dot-pos) "")))
+
 (define +current-repo-path+
   (getenv "CURRENT_REPO_PATH"))
 
@@ -56,6 +64,20 @@ otherwise search for set/group data"
     (if taxoninfo taxoninfo
         (cdr (get-group-data id)))))
 
+(define (get-bxd-publish)
+  "Return a list of published datasets by their record ID. We add the dataset ID and phenotype ID for quick reference"
+  (list->vector (get-bxd-publish-list)))
+
+(define* (get-bxd-publish-dataid-values dataid #:optional used-for-mapping?)
+  (get-bxd-publish-dataid-name-value-dict dataid used-for-mapping?))
+
+(define* (get-bxd-publish-values dataid #:optional used-for-mapping?)
+  (get-bxd-publish-name-value-dict dataid used-for-mapping?))
+
+(define (get-gene-aliases genename)
+  "Return a vector of aliases for genename."
+  (list->vector (memo-sparql-wd-gene-aliases (memo-sparql-wd-geneids genename))))
+
 (define (not-found2 request)
   (values (build-response #:code 404)
           (string-append "Resource X not found: "
@@ -79,7 +101,7 @@ otherwise search for set/group data"
     ("html" text/html)))
 
 (define (file-extension file-name)
-  (last (string-split file-name #\.)))
+  (last (string-split file-name #\.))) ;; FIXME: does not handle files with multiple dots
 
 (define* (render-static-image file-name
                               #:key (extra-headers '()))
@@ -87,9 +109,9 @@ otherwise search for set/group data"
          (modified (and stat
                         (make-time time-utc 0
                                    (stat:mtime stat)))))
-    (list `((content-type ,(assoc-ref file-mime-types
-				      (file-extension file-name)))
-            (last-modified ,(time-utc->date modified)))
+    (list `((content-type . ,(assoc-ref file-mime-types
+				        (file-extension file-name)))
+            (last-modified . ,(time-utc->date modified)))
           (call-with-input-file file-name
             get-bytevector-all))))
 
@@ -100,9 +122,9 @@ otherwise search for set/group data"
          (modified (and stat
                         (make-time time-utc 0
                                    (stat:mtime stat)))))
-    (list `((content-type ,(assoc-ref file-mime-types
-				      (file-extension path)))
-            (last-modified ,(time-utc->date modified)))
+    (list `((content-type . ,(assoc-ref file-mime-types
+				        (file-extension path)))
+            (last-modified . ,(time-utc->date modified)))
           (call-with-input-file path
             get-bytevector-all))))
 
@@ -123,6 +145,11 @@ otherwise search for set/group data"
         (lambda (port)
           (sxml->html (view-brand path) port))))
 
+(define (render-string str)
+  (list '((content-type application/txt))
+        (lambda (port)
+          (put-string port str))))
+
 (define (render-json json)
   (list '((content-type application/json))
         (lambda (port)
@@ -224,13 +251,31 @@ otherwise search for set/group data"
     (('GET "version")
      (render-json get-version))
     (('GET "css" fn)
-     (render-static-file (string-append "css/" fn)))
+     (render-static-file (string-append (dirname (current-filename)) "/css/" fn)))
     (('GET "map" fn)
-     (render-static-file (string-append "css/" fn)))
+     (render-static-file (string-append (dirname (current-filename)) "/css/" fn)))
     (('GET "static" "images" fn)
-     (render-static-image (string-append "static/images/" fn)))
+     (render-static-image (string-append (dirname (current-filename)) "/static/images/" fn)))
     (('GET "home" path)
      (render-brand path)) ; branding route for /home/aging, /home/msk etc
+    (('GET "home" "aging" path)
+     (render-brand (string-append "aging/" path))) ; branding route subs of /home/aging/...
+    (('GET "dataset" "bxd-publish" "list")
+     (render-json (get-bxd-publish)))
+    (('GET "dataset" "bxd-publish" "dataid" "values" page)
+     (match (get-extension page)
+       (".json"
+        (render-json (get-bxd-publish-dataid-values (basename page ".json"))))
+       (else    (display "ERROR: unknown file extension"))))
+    (('GET "dataset" "bxd-publish" "values" page)
+     (match (get-extension page)
+       (".json"
+        (render-json (get-bxd-publish-values (basename page ".json"))))
+       ;; (".tsv"  (render-string "TEST1\nTEST2"))
+       ;; (".gemma" (render-string (string-join (gemma-pheno-txt "BXD" (get-bxd-publish-values (basename page ".gemma"))) "")))
+       (else    (display "ERROR: unknown file extension"))))
+    (('GET "dataset" "bxd-publish" "mapping" "values" (string-append dataid ".json"))
+     (render-json (get-bxd-publish-values dataid #t)))
     (('GET "doc" "species.html")
      (render-doc "doc" "species.html"
                  (get-species-meta)))
@@ -247,6 +292,8 @@ otherwise search for set/group data"
     (('GET "doc" path ... page)
      ;; serve documents from /doc/
      (render-doc path page))
+    (('GET "gene" "aliases" genename)
+     (render-json (get-gene-aliases genename)))
     (('GET "species.json")
      (render-json (get-species-data)))
     (('GET "species.meta.json")
@@ -303,8 +350,9 @@ otherwise search for set/group data"
   ;; only way to update the handler reference held by the web server
   ;; would be to restart the web server.
   (run-server (cut handler <> <>)
-              #:addr (inet-pton AF_INET address)
-              #:port port))
+              'http
+              (list #:addr (inet-pton AF_INET address)
+                    #:port port)))
 
 (define (main args)
   (write (string-append "Starting Guile REST API " get-version " server!"))
generated by cgit-pink 1.4.1 (git 2.36.1) at 2025-12-22 18:43:55 +0000