Merge branch 'master' of https://gitlab.com/genenetwork/guix-bioinformatics

24 files changed, 2116 insertions, 603 deletions

diff --git a/.guix-channel b/.guix-channel
index ff9ab9e..e42b9ad 100644
--- a/.guix-channel
+++ b/.guix-channel
@@ -5,17 +5,19 @@
    (name guix-past)
    (url "https://gitlab.inria.fr/guix-hpc/guix-past")
    (introduction
-    (make-channel-introduction
-     "0c119db2ea86a389769f4d2b9c6f5c41c027e336"
-     (openpgp-fingerprint
+    (channel-introduction
+     (version 0)
+     (commit "0c119db2ea86a389769f4d2b9c6f5c41c027e336")
+     (signer
       "3CE4 6455 8A84 FDC6 9DB4  0CFB 090B 1199 3D9A EBB5"))))
   (channel
    (name guix)
    (url "https://git.savannah.gnu.org/git/guix.git")
    (branch "master")
-   (commit "5b35626374783c019ba0911b612e1385f238dfbe")
+   (commit "d4e9ad2d42dddf06a8f1b01c9f49fee16c592d75")
    (introduction
-    (make-channel-introduction
-     "9edb3f66fd807b096b48283debdcddccfea34bad"
-     (openpgp-fingerprint
+    (channel-introduction
+     (version 0)
+     (commit "9edb3f66fd807b096b48283debdcddccfea34bad")
+     (signer
       "BBB0 2DDF 2CEA F6A8 0D1D  E643 A2A0 6DF2 A33A 54FA"))))))
diff --git a/README.org b/README.org
index 2731ee9..c5936fa 100644
--- a/README.org
+++ b/README.org
@@ -1,5 +1,7 @@
 * guix-bioinformatics
 
+IMPORTANT: this repository has moved to https://git.genenetwork.org/guix-bioinformatics/!
+
 Bioinformatics packages for GNU Guix that are used in
 https://genenetwork.org/ and some other places.  See [[https://gitlab.com/pjotrp/guix-notes/blob/master/HACKING.org][Guix notes]] for
 installing and hacking GNU Guix. Other channels of bioinformatics
@@ -143,6 +145,13 @@ and then run:
 
 : guix package --install-from-file=that-file.scm
 
+* Substitute server
+
+We run our own substitution server. Add the key to your machine as
+root with
+
+: guix archive --authorize < tux02-guix-substitutions-public-key.txt
+: guix build -L ~/guix-bioinformatics/ -L ~/guix-past/modules/ genenetwork2 --substitute-urls="https://ci.guix.gnu.org https://bordeaux.guix.gnu.org https://guix.genenetwork.org" --dry-run
 
 * LICENSE
 
diff --git a/gn/packages/bioinformatics.scm b/gn/packages/bioinformatics.scm
index 2d1b55f..db420b7 100644
--- a/gn/packages/bioinformatics.scm
+++ b/gn/packages/bioinformatics.scm
@@ -23,7 +23,6 @@
   #:use-module (gn packages java)
   #:use-module (gn packages ocaml)
   #:use-module (gn packages python)
-  #:use-module (gn packages twint)
   #:use-module (gnu packages algebra)
   #:use-module (gnu packages assembly)
   #:use-module (gnu packages autotools)
@@ -33,6 +32,7 @@
   #:use-module (gnu packages bioinformatics)
   #:use-module (gnu packages boost)
   #:use-module (gnu packages bootstrap)
+  #:use-module (gnu packages c)
   #:use-module (gnu packages check)
   #:use-module (gnu packages cmake)
   #:use-module (gnu packages compression)
@@ -54,6 +54,7 @@
   #:use-module (gnu packages java)
   #:use-module (gnu packages jemalloc)
   #:use-module (gnu packages linux)
+  #:use-module (gnu packages llvm)
   #:use-module (gnu packages machine-learning)
   #:use-module (gnu packages maths)
   #:use-module (gnu packages mpi)
@@ -460,7 +461,7 @@ reads.")
     (description "Variant detection in massively parallel sequencing data.")
     ;; Free for non-commercial use by academic, government, and
     ;; non-profit/not-for-profit institutions
-    (license license:non-copyleft)))
+    (license (license:non-copyleft "file:///LICENSE"))))
 
 (define-public edirect-gn
   (deprecated-package "edirect-gn" edirect))
@@ -487,7 +488,7 @@ reads.")
     (arguments
      `(#:install-source? #f
        #:cargo-inputs
-       (("rust-clap" ,rust-clap-3.1)
+       (("rust-clap" ,rust-clap-3)
         ("rust-rustc-hash" ,rust-rustc-hash-1)
         ("rust-regex" ,rust-regex-1)
         ("rust-handlegraph" ,rust-handlegraph-0.7)
@@ -510,6 +511,305 @@ reads.")
 collapses them into a non-redundant graph structure.")
     (license license:expat)))
 
+(define-public gafpack
+  (let ((commit "ad31875b6914d964c6fd72d1bf334f0843538fb6")     ; November 10, 2022
+        (revision "1"))
+    (package
+      (name "gafpack")
+      (version (git-version "0.0.0" revision commit))
+      (source
+        (origin
+          (method git-fetch)
+          (uri (git-reference
+                 (url "https://github.com/ekg/gafpack")
+                 (commit commit)))
+          (file-name (git-file-name name version))
+          (sha256
+           (base32 "0di2psh0ls7jlbnqs7k71p55f73pn23a09k1h3ril7gwjcrzr3rk"))))
+      (build-system cargo-build-system)
+      (arguments
+       `(#:install-source? #f
+         #:cargo-inputs
+         (("rust-clap" ,rust-clap-4)
+          ("rust-gfa" ,rust-gfa-0.10))))
+      (home-page "https://github.com/ekg/gafpack")
+      (synopsis "Convert variation graph alignments to coverage maps over nodes")
+      (description
+       "Gafpack converts alignments to pangenome variation graphs to coverage
+maps useful in haplotype-based genotyping.")
+      (license license:expat))))
+
+(define-public agc-for-pgr-tk
+  (let ((commit "453c0afdc54b4aa00fa8e97a63f196931fdb81c4") ; April 26, 2022
+        (revision "1"))
+    (package
+      (name "agc")
+      (version (git-version "2.0" revision commit))
+      (source
+        (origin
+          (method git-fetch)
+          (uri (git-reference
+                 (url "https://github.com/cschin/agc")
+                 (commit commit)))
+          (file-name (git-file-name name version))
+          (sha256
+           (base32 "1v5s79rl38dcyy5h1lykbp6clcbqq9winn533j54y49q1jp8chix"))
+          (snippet
+           #~(begin
+               (use-modules (guix build utils))
+               ;; Copy the two radul files we can't find a replacement for:
+               ;; https://github.com/refresh-bio/RADULS
+               (mkdir "keep-libs")
+               (rename-file "libs/raduls.h" "keep-libs/raduls.h")
+               (rename-file "libs/libraduls.a" "keep-libs/libraduls.a")
+               (delete-file-recursively "libs")
+               (rename-file "keep-libs" "libs")
+
+               (delete-file-recursively "py_agc_api/pybind11-2.8.1")
+               (substitute* '("makefile" "makefile.release")
+                 (("-mavx") "")
+                 (("-m64") "")
+                 (("\\$\\(AGC_LIBS_DIR)\\/mimalloc/\\$\\(LIB_ALLOC\\)")
+                  "$(pkg-config --cflags --libs mimalloc) /usr/lib/libmimalloc.so")
+                 (("\\$\\(AGC_LIBS_DIR)\\/\\$\\(LIB_ZLIB\\)")
+                  "$(pkg-config --cflags --libs zlib) /usr/lib/libz.so")
+                 (("\\$\\(AGC_LIBS_DIR)\\/\\$\\(LIB_ZSTD\\)")
+                  "$(pkg-config --cflags --libs libzstd) /usr/lib/libzstd.so")
+                 (("^PYBIND11_LIB = .*") "PYBIND11_LIB = /usr/include/pybind11")
+                 (("\\$\\(PYBIND11_LIB\\)/include") "$(PYBIND11_LIB)"))
+               (substitute* (find-files "src" "\\.(h|cpp)$")
+                 (("../../libs/ketopt.h") "ketopt.h")
+                 (("../../libs/zlib.h") "zlib.h")
+                 (("../../libs/zstd.h") "zstd.h"))))))
+      (build-system gnu-build-system)
+      (arguments
+       `(#:tests? #f                    ; No tests.
+         #:phases
+         (modify-phases %standard-phases
+           (delete 'configure)          ; No configure script.
+           (add-after 'unpack 'adjust-sources
+             (lambda* (#:key inputs #:allow-other-keys)
+               (let ((mimalloc (assoc-ref inputs "mimalloc")))
+                 (substitute* '("makefile" "makefile.release")
+                   (("/usr/include/pybind11")
+                    (search-input-directory inputs "/include/pybind11"))
+                   (("/usr/lib/libmimalloc.so")
+                    (search-input-file inputs "/lib/libmimalloc.so"))
+                   (("/usr/lib/libz.so")
+                    (search-input-file inputs "/lib/libz.so"))
+                   (("/usr/lib/libzstd.so")
+                    (search-input-file inputs "/lib/libzstd.so"))
+                   (("pkg-config") ,(pkg-config-for-target))))))
+           (replace 'install
+             (lambda* (#:key outputs #:allow-other-keys)
+               (let* ((out (assoc-ref outputs "out"))
+                      (include (string-append out "/include/")))
+                 (install-file "agc" (string-append out "/bin"))
+                 (install-file "libagc.so" (string-append out "/lib"))
+                 (mkdir-p (string-append include "app"))
+                 (mkdir-p (string-append include "core"))
+                 (mkdir-p (string-append include "lib-cxx"))
+                 (with-directory-excursion "src"
+                   (for-each
+                     (lambda (file)
+                       (copy-file file (string-append include file)))
+                     (find-files "." "\\.h$")))))))))
+      (native-inputs
+       (list minimap2                   ; for ketopt.h
+             pkg-config))
+      (inputs
+       (list mimalloc
+             python
+             pybind11
+             zlib
+             (list zstd "lib")))
+      (home-page "https://github.com/cschin/agc")
+      (synopsis "Assembled Genomes Compressor")
+      (description
+       "@acronym{Assembled Genomes Compressor, AGC} is a tool designed to
+compress collections of de-novo assembled genomes.  It can be used for various
+types of datasets: short genomes (viruses) as well as long (humans).")
+      (license license:expat))))
+
+(define-public pgr-tk
+  (package
+    (name "pgr-tk")
+    (version "0.3.6")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/Sema4-Research/pgr-tk")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "0vm1k63v91zd0pfbg2zmwskajylz8xg83m63qxwaiwny5f4y6f1j"))
+        (snippet
+         #~(begin
+             (use-modules (guix build utils))
+             (substitute* (find-files "." "Cargo.toml")
+               ;; Only use the major+minor version to decrease the number of
+               ;; special version crates.
+               (("(.*= \")([[:digit:]]+\\.[[:digit:]]+)\\.[[:digit:]]+(\".*)"
+                 _ name version tail)
+                (string-append name version tail))
+               ;; Then fix the version string for the actual package.
+               (("^version = \".*")
+                (string-append "version = \"" #$version "\"\n")))))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:install-source? #f
+       #:cargo-test-flags
+       (list "--release" "--"
+             "--skip=get_aln_segements"
+             "--skip=get_shmmr_dots"
+             "--skip=AGCFile"
+             "--skip=SeqIndexDB")
+       #:cargo-inputs
+       (("rust-bindgen" ,rust-bindgen-0.58)
+        ("rust-bgzip" ,rust-bgzip-0.2)
+        ("rust-byteorder" ,rust-byteorder-1)
+        ("rust-clap" ,rust-clap-3)
+        ("rust-cuckoofilter" ,rust-cuckoofilter-0.5)
+        ("rust-flate2" ,rust-flate2-1)
+        ("rust-libc" ,rust-libc-0.2)
+        ("rust-log" ,rust-log-0.4)
+        ("rust-petgraph" ,rust-petgraph-0.6)
+        ("rust-pyo3" ,rust-pyo3-0.14)
+        ("rust-rayon" ,rust-rayon-1)
+        ("rust-regex" ,rust-regex-1)
+        ("rust-rustc-hash" ,rust-rustc-hash-1)
+        ("rust-serde" ,rust-serde-1)
+        ("rust-serde-json" ,rust-serde-json-1)
+        ("rust-simple-logger" ,rust-simple-logger-1))
+       #:phases
+       (modify-phases %standard-phases
+         (add-after 'unpack 'insert-wfa-source
+           (lambda* (#:key inputs #:allow-other-keys)
+             (copy-recursively (assoc-ref inputs "wfa-src")
+                               "rs-wfa/WFA")))
+         (add-after 'unpack 'adjust-source
+           (lambda* (#:key inputs #:allow-other-keys)
+             (substitute* '("pgr-bin/build.rs"
+                            "pgr-db/build.rs"
+                            "pgr-tk/build.rs")
+               (("git") "ls")
+               (("bioconda") "Guix"))
+             ;; Build with zlib, not zlib-ng
+             (substitute* '("pgr-bin/Cargo.toml"
+                            "pgr-db/Cargo.toml")
+               (("zlib-ng-compat") "zlib"))
+             ;; Don't look for agc to be bundled.
+             (substitute* "pgr-db/wrapper.h"
+               (("../agc/src/lib-cxx/agc-api.h") "lib-cxx/agc-api.h"))
+             (substitute* "pgr-db/build.rs"
+               ((".*panic!\\(\"Error.*") ""))
+             (mkdir-p "target/release")
+             (symlink (search-input-file inputs "/bin/agc")
+                      "target/release/agc")
+             (symlink (search-input-file inputs "/lib/libagc.so")
+                      "target/release/libagc")))
+         (replace 'install
+           (lambda* (#:key outputs #:allow-other-keys)
+             (let ((out (assoc-ref outputs "out")))
+               (with-directory-excursion "target/release"
+                 (install-file "libpgrtk.so" (string-append out "/lib"))
+                 (for-each
+                   (lambda (file)
+                     (install-file file (string-append out "/bin")))
+                   (list "pgr-filter"
+                         "pgr-mdb"
+                         "pgr-multifilter"
+                         "pgr-probe-match"
+                         "pgr-shmmr-pair-count")))))))))
+    (inputs
+     (list agc-for-pgr-tk
+           clang
+           python
+           zlib
+           (list zstd "lib")))
+    (native-inputs
+     `(("pkg-config" ,pkg-config)
+       ("wfa-src"
+        ,(origin
+           (method git-fetch)
+           (uri (git-reference
+                  ;; forPYO3 branch, 14-03-2021
+                  (url "https://github.com/cschin/WFA")
+                  (commit "1f8c8d2905ed482cd2d306a1676d60c2a45cb098")))
+           (file-name "wfa-for-pgr-tk")
+           (sha256
+            (base32 "19h1cjp2bdlcfq5c6rsbk8bc0f8zn64b471dhj4xlfxd1prv2dpk"))))))
+    (home-page "https://github.com/Sema4-Research/pgr-tk")
+    (synopsis "Pangenome Research Tool Kit")
+    (description
+     "PGR-TK provides pangenome assembly management, query and
+@acronym{Minimizer Anchored Pangenome, MAP} Graph Generation.  It is a project
+to provide Python and Rust libraries to facilitate pangenomics analysis.
+Several algorithms and data structures used for the Peregrine Genome Assembler
+are useful for Pangenomics analysis as well.  This repo takes those algorithms
+and data structure, combining other handy 3rd party tools to expose them as a
+library in Python (with Rust code for those computing parts that need
+performance.)")
+    (license (license:non-copyleft
+               "file:///LICENSE"
+               "CC-BY-NC-SA 4.0"))))
+
+(define-public graph-genotyper
+  (let ((commit "e7cc6b43a5b1f389d76bf9aac7f2ee02f92caeaf") ; October 17, 2022
+        (revision "13"))
+    (package
+      (name "graph-genotyper")
+      (version (git-version "0.0.0" revision commit))
+      (source (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/davidebolo1993/graph_genotyper")
+               (commit commit)))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "1l8yjpkqamiqr1q5i7vr5z04aba7skpbcwyc9dx5fiklvljjfhcx"))))
+      (build-system copy-build-system)
+      (arguments
+       `(#:install-plan
+         '(("genotype.py" "bin/")
+           ("genotype.sh" "bin/"))
+         #:phases
+         (modify-phases %standard-phases
+           (add-after 'install 'wrap-genotype
+             (lambda* (#:key inputs outputs #:allow-other-keys)
+               (let ((out (assoc-ref outputs "out")))
+                 (wrap-script (string-append out "/bin/genotype.sh")
+                  `("GUIX_PYTHONPATH" ":" prefix (,(getenv "GUIX_PYTHONPATH")))
+                  `("PATH" ":" prefix
+                    ,(map (lambda (file-name)
+                            (string-append (assoc-ref inputs file-name) "/bin"))
+                          (list "gafpack"
+                                "odgi"
+                                "python"
+                                "samtools"
+                                "vg"))))))))))
+      (inputs
+       (list gafpack
+             guile-3.0
+             odgi
+             python
+             python-numpy
+             python-pandas
+             python-scipy
+             samtools
+             vg))
+      (home-page "https://bitbucket.org/jana_ebler")
+      (synopsis "Genotyping based on k-mers and pangenome graphs")
+      (description
+       "This package provides a genotyper for various types of genetic variants
+(such as SNPs, indels and structural variants).  Genotypes are computed based on
+read k-mer counts and a panel of known haplotypes.  A description of the method
+can be found @url{https://www.biorxiv.org/content/10.1101/2020.11.11.378133v1,
+here}.")
+      (license (license:non-copyleft
+                 "No license listed")))))
+
 (define-public pangenie
   (let ((commit "e779076827022d1416ab9fabf99a03d8f4725956") ; September 2, 2021 from phasing-tests branch
         (revision "2"))
@@ -1490,20 +1790,9 @@ reads, also called read-based phasing or haplotype assembly.  It is especially
 suitable for long reads, but works also well with short reads.")
     (license license:expat)))
 
-(define-public python-pytest-runner-2
-  (package
-    (inherit python-pytest-runner)
-    (version "2.12.2")
-    (source (origin
-              (method url-fetch)
-              (uri (pypi-uri "pytest-runner" version))
-              (sha256
-               (base32
-                "11ivjj9hfphkv4yfb2g74av4yy86y8gcbf7gbif0p1hcdfnxg3w6"))))))
-
 (define-public bh20-seq-resource
-  (let ((commit "ae4cb3c2cf7103bbc84f52618bb755d7ce25775b")
-        (revision "3"))
+  (let ((commit "2ae71911cd87ce4f2eabdff21e538267b3270d45")
+        (revision "4"))
     (package
       (name "bh20-seq-resource")
       (version (git-version "1.0" revision commit))
@@ -1514,36 +1803,45 @@ suitable for long reads, but works also well with short reads.")
                        (commit commit)))
                 (file-name (git-file-name name version))
                 (sha256
-                 (base32 "1k0gsz4yc8l5znanzd094g2jp40ksbpa9667zr31ayrjx6labz02"))
+                 (base32 "1k6cc88hrcm77jwpdk2084q0zirv2vlbz3c07nmpbhk1lhqk5x0n"))
                 (modules '((guix build utils)))
                 (snippet
                  '(begin
-                    (substitute* "setup.py"
-                      (("py-dateutil") "python-dateutil"))
-                    #t))))
+                    (delete-file "gittaggers.py")))))
       (build-system python-build-system)
+      (arguments
+       (list
+         #:tests? #f    ; Tests can't find pytest
+         #:phases
+         #~(modify-phases %standard-phases
+             (add-after 'unpack 'patch-program-calls
+               (lambda* (#:key inputs #:allow-other-keys)
+                 (substitute* "bh20sequploader/qc_fasta.py"
+                   (("\"minimap2\"")
+                    (string-append "\"" (search-input-file
+                                          inputs "/bin/minimap2")
+                                   "\""))))))))
       (propagated-inputs
-       `(("python-arvados-python-client" ,python-arvados-python-client)
-         ("python-dateutil" ,python-dateutil)
-         ("python-flask" ,python-flask)
-         ("python-magic" ,python-magic)
-         ("python-pyyaml" ,python-pyyaml)
-         ("python-pycurl" ,python-pycurl)
-         ("python-pyshex" ,python-pyshex)
-         ("python-redis" ,python-redis)
-         ("python-ruaml.yaml" ,python38-ruaml.yaml-0.15.76)
-         ("clustalw" ,clustalw)
-         ("python-schema-salad" ,python-schema-salad)
-         ("python-twint" ,python-twint)
-         ;; and for the service
-         ("python" ,python)
-         ("gunicorn" ,gunicorn)))
+       (list python-arvados-python-client
+             python-schema-salad
+             python-magic
+             python-pyshex
+             python-pyshexc-0.7
+             python-py-dateutil
+
+             ;; for the web
+             python-flask
+             python-pyyaml
+             python-redis
+
+             ;; and for the service
+             python
+             gunicorn))
+      (inputs
+       (list minimap2))
       (native-inputs
-       `(("git" ,(@ (gnu packages version-control) git))
-         ("python-oauth2client" ,python-oauth2client)
-         ("python-pytest" ,python-pytest-4)
-         ("python-pytest-runner" ,python-pytest-runner-2)
-         ("python-uritemplate" ,python-uritemplate)))
+       (list python-pytest-4                ; < 6
+             python-pytest-runner-4))       ; < 5
       (home-page "https://github.com/pubseq/bh20-seq-resource")
       (synopsis
        "Tool to upload SARS-CoV-19 sequences and service to kick off analysis")
@@ -1553,6 +1851,18 @@ it to upload the genomes of SARS-CoV-2 samples to make them publicly and freely
 available to other researchers.")
       (license license:asl2.0))))
 
+;; This version has no profile collisions.
+(define-public bh20-seq-resource-for-service
+  (package
+    ;(inherit (fix-profile-collisions-for-bh20 bh20-seq-resource))
+    (inherit
+      ((package-input-rewriting/spec
+        `(("python-google-api-core" . ,(const python-google-api-core-1))
+          ("python-google-auth" . ,(const python-google-auth-1))
+          ("python-pyparsing" . ,(const python-pyparsing-2.4.7))))
+       bh20-seq-resource))
+    (properties `((hidden? . #t)))))
+
 (define-public python-scanpy-git
   (let ((commit "590d42309f9ed6550d7b887039990edfc1ac7648") ; April 22, 2020
         (revision "1"))
@@ -1593,6 +1903,56 @@ available to other researchers.")
                  (delete-file "scanpy/tests/test_pca.py")
                  #t)))))))))
 
+;; TODO: Unbundle everything
+(define-public odgi
+  (package
+    (name "odgi")
+    (version "0.8.1")
+    (source (origin
+              (method url-fetch)
+              (uri (string-append "https://github.com/pangenome/odgi/releases"
+                                  "/download/v" version
+                                  "/odgi-v" version ".tar.gz"))
+              (sha256
+               (base32 "175083pb9hp0vn9a00hbxlayyk5a5j8p52yq5qfmbnfvndisbmbv"))
+              (snippet
+               #~(begin
+                   (use-modules (guix build utils))
+                   (substitute* "CMakeLists.txt"
+                     (("-march=native") "")
+                     (("-msse4\\.2") ""))
+                   (delete-file-recursively "deps/pybind11")
+                   (delete-file-recursively "deps/sdsl-lite")))))
+    (build-system cmake-build-system)
+    (native-inputs
+     (list pkg-config))
+    (inputs
+     (list jemalloc
+           libdivsufsort
+           pybind11
+           python
+           sdsl-lite))
+    (home-page "https://github.com/vgteam/odgi")
+    (synopsis "Optimized Dynamic Genome/Graph Implementation")
+    (description "@acronym{Optimized Dynamic Genome/Graph Implementation, odgi}
+provides an efficient and succinct dynamic DNA sequence graph model, as well as
+a host of algorithms that allow the use of such graphs in bioinformatic
+analyses.
+
+Careful encoding of graph entities allows odgi to efficiently compute and
+transform pangenomes with minimal overheads.  @command{odgi} implements a
+dynamic data structure that leveraged multi-core CPUs and can be updated on the
+fly.
+
+The edges and path steps are recorded as deltas between the current node id and
+the target node id, where the node id corresponds to the rank in the global
+array of nodes.  Graphs built from biological data sets tend to have local
+partial order and, when sorted, the deltas be small.  This allows them to be
+compressed with a variable length integer representation, resulting in a small
+in-memory footprint at the cost of packing and unpacking.")
+    (properties '((tunable? . #t)))
+    (license license:expat)))
+
 (define-public vg
   (package
     (name "vg")
@@ -1826,7 +2186,7 @@ available to other researchers.")
     (inputs
      `(("boost" ,boost)
        ("cairo" ,cairo)
-       ("curl" ,curl-minimal)
+       ("curl" ,curl)
        ("elfutils" ,elfutils)
        ("fastahack" ,fastahack)
        ("htslib" ,htslib)
@@ -2055,7 +2415,7 @@ The Genome Browser itself does not draw conclusions; rather, it collates all
 relevant information in one location, leaving the exploration and interpretation
 to the user.")
     (license (list
-               license:bsd-0    ; kent/src/{utils,lib,inc,tabStorm,parasol,hg/ausoSql,hg/autoXml}
+               ;; license:bsd-0    ; kent/src/{utils,lib,inc,tabStorm,parasol,hg/ausoSql,hg/autoXml}
                license:bsd-3    ; these two for bundled htslib-1.3
                license:expat
                (license:non-copyleft
diff --git a/gn/packages/crates-io.scm b/gn/packages/crates-io.scm
index 1dc4f65..e500d86 100644
--- a/gn/packages/crates-io.scm
+++ b/gn/packages/crates-io.scm
@@ -7,11 +7,145 @@
   #:use-module (guix build-system cargo)
   #:use-module (gnu packages crates-graphics)
   #:use-module (gnu packages crates-io)
-  #:use-module (gnu packages maths))
+  #:use-module (gnu packages maths)
+  #:use-module (gnu packages python))
+
+(define-public rust-bgzip-0.2
+  (package
+    (name "rust-bgzip")
+    (version "0.2.1")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "bgzip" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "1kssq4hp8csg27rhggabpfiyn9xp5rh5b8al63dghk11vqs7hk5j"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs
+       (("rust-flate2" ,rust-flate2-1)
+        ("rust-thiserror" ,rust-thiserror-1))
+       #:cargo-development-inputs
+       (("rust-clap" ,rust-clap-2)
+        ("rust-csv" ,rust-csv-1)
+        ("rust-tempfile" ,rust-tempfile-3))))
+    (home-page "https://github.com/informationsea/bgzip-rs")
+    (synopsis "Rust implementation of bgzip")
+    (description "Rust implementation of bgzip")
+    (license license:expat)))
+
+(define-public rust-boomphf-0.5
+  (package
+    (name "rust-boomphf")
+    (version "0.5.9")
+    (source
+      (origin
+        (method url-fetch)
+        (uri (crate-uri "boomphf" version))
+        (file-name
+         (string-append name "-" version ".tar.gz"))
+        (sha256
+         (base32 "0braniw72g9yq5006sfgc1g8d4317bb524c694jw6nggizrvg3sf"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs
+       (("rust-crossbeam-utils" ,rust-crossbeam-utils-0.8)
+        ("rust-log" ,rust-log-0.4)
+        ("rust-rayon" ,rust-rayon-1)
+        ("rust-serde" ,rust-serde-1)
+        ("rust-wyhash" ,rust-wyhash-0.5))
+       #:cargo-development-inputs
+       (("rust-bencher" ,rust-bencher-0.1)
+        ("rust-quickcheck" ,rust-quickcheck-1))))
+    (home-page "https://github.com/10XGenomics/rust-boomphf")
+    (synopsis "Scalable and Efficient Minimal Perfect Hash Functions")
+    (description "This package provides a Rust implementation of
+@url{https://arxiv.org/abs/1702.03154, fast and scalable minimal perfect hashing
+for massive key sets}.  It generates an @acronym{MPHF, minimal perfect hash
+functions} for a collection of hashable objects.")
+    (license license:expat)))
+
+(define-public rust-clap-lex-0.3
+  (package
+    (name "rust-clap-lex")
+    (version "0.3.0")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "clap-lex" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "1a4dzbnlxiamfsn0pnkhn7n9bdfjh66j9fxm6mmr7d227vvrhh8d"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs (("rust-os-str-bytes" ,rust-os-str-bytes-6))))
+    (home-page "https://github.com/clap-rs/clap/tree/master/clap_lex")
+    (synopsis "Minimal, flexible command line parser")
+    (description "Minimal, flexible command line parser")
+    (license (list license:expat license:asl2.0))))
+
+(define-public rust-cuckoofilter-0.5
+  (package
+    (name "rust-cuckoofilter")
+    (version "0.5.0")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "cuckoofilter" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "067fkr9dc118rqddr72xdldq05d31yyipvvyrmj9yrrik52ah45q"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:skip-build? #t     ; Not packaging dependencies
+       #:cargo-inputs
+       (("rust-byteorder" ,rust-byteorder-1)
+        ;("rust-clippy" ,rust-clippy-0.0.302)
+        ;("rust-farmhash" ,rust-farmhash-1)
+        ("rust-fnv" ,rust-fnv-1)
+        ("rust-rand" ,rust-rand-0.7)
+        ("rust-serde" ,rust-serde-1)
+        ("rust-serde-bytes" ,rust-serde-bytes-0.11)
+        ("rust-serde-derive" ,rust-serde-derive-1))
+       #:cargo-development-inputs
+       (("rust-serde-json" ,rust-serde-json-1))))
+    (home-page "http://axiom.co")
+    (synopsis "Cuckoo Filter: Practically Better Than Bloom")
+    (description "Cuckoo Filter: Practically Better Than Bloom")
+    (license license:expat)))
+
+(define-public rust-gfa-0.6
+  (package
+    (name "rust-gfa")
+    (version "0.6.2")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "gfa" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "0ghmy4r0324s6vvmj9nmh326346nkwm7nybnpcpswnjvf02b85gw"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs
+       (("rust-bstr" ,rust-bstr-0.2)
+        ("rust-bytemuck" ,rust-bytemuck-1)
+        ("rust-lazy-static" ,rust-lazy-static-1)
+        ("rust-nom" ,rust-nom-5)
+        ("rust-regex" ,rust-regex-1)
+        ("rust-serde" ,rust-serde-1))
+        #:cargo-development-inputs
+        (("rust-criterion" ,rust-criterion-0.3))))
+    (home-page "https://github.com/chfi/rs-gfa")
+    (synopsis
+     "Library for working with graphs in the GFA (Graphical Fragment Assembly) format")
+    (description
+     "Library for working with graphs in the GFA (Graphical Fragment Assembly) format")
+    (license license:expat)))
 
 (define-public rust-handlegraph-0.7
   (package
-    (inherit rust-handlegraph-0.3)
     (name "rust-handlegraph")
     (version "0.7.0-alpha.9")
     (source
@@ -23,6 +157,7 @@
         (sha256
          (base32
           "1frlcdwhycjvizb0gfb0v36vxjdi0jxagl2l2v6dzdjxpaawv9rs"))))
+    (build-system cargo-build-system)
     (arguments
      `(#:cargo-inputs
        (("rust-anyhow" ,rust-anyhow-1)
@@ -36,7 +171,206 @@
         ("rust-succinct" ,rust-succinct-0.5))
        #:cargo-development-inputs
        (("rust-quickcheck" ,rust-quickcheck-0.9)
-        ("rust-rand" ,rust-rand-0.7))))))
+        ("rust-rand" ,rust-rand-0.7))))
+    (home-page "https://github.com/chfi/rs-handlegraph")
+    (synopsis "Library for use in variation graphs")
+    (description
+     "This package provides a Rust implementation of VG handle graph.")
+    (license license:expat)))
+
+(define-public rust-handlegraph-0.3
+  (package
+    (inherit rust-handlegraph-0.7)
+    (name "rust-handlegraph")
+    (version "0.3.0")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "handlegraph" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "1sj100w4lpj7798pws85qrfrzsily5hhzh6j118rwf56sgic1yml"))))
+    (arguments
+     `(#:cargo-inputs
+       (("rust-bstr" ,rust-bstr-0.2)
+        ("rust-gfa" ,rust-gfa-0.6))))))
+
+(define-public rust-kstring-1
+  (package
+    (name "rust-kstring")
+    (version "1.1.0")
+    (source
+      (origin
+        (method url-fetch)
+        (uri (crate-uri "kstring" version))
+        (file-name (string-append name "-" version ".tar.gz"))
+        (sha256
+         (base32 "1r4n9fa5scikqvl736nxghcfa6s3b07xz61w43hyzs2qb3wmd3nk"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(;#:skip-build? #t     ; Uses unstable features.
+       #:cargo-inputs
+       (("rust-document-features" ,rust-document-features-0.2)
+        ("rust-serde" ,rust-serde-1)
+        ("rust-static-assertions" ,rust-static-assertions-1))
+       #:cargo-development-inputs
+       (("rust-criterion" ,rust-criterion-0.3)
+        ("rust-proptest" ,rust-proptest-1))))
+    (home-page "https://github.com/cobalt-org/kstring")
+    (synopsis "String optimized for map keys")
+    (description "Key String provides a Rust package optimized for map keys.")
+    (license (list license:expat license:asl2.0))))
+
+(define-public rust-pyo3-0.14
+  (package
+    (name "rust-pyo3")
+    (version "0.14.5")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "pyo3" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "1j70b9bkncgpgnfxjxyvp4mk40rp55lk6qmacxm5c2k78y9hy41m"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs
+       (("rust-cfg-if" ,rust-cfg-if-1)
+        ("rust-hashbrown" ,rust-hashbrown-0.11)
+        ("rust-indexmap" ,rust-indexmap-1)
+        ("rust-indoc" ,rust-indoc-0.3)
+        ("rust-inventory" ,rust-inventory-0.1)
+        ("rust-libc" ,rust-libc-0.2)
+        ("rust-num-bigint" ,rust-num-bigint-0.4)
+        ("rust-num-complex" ,rust-num-complex-0.4)
+        ("rust-parking-lot" ,rust-parking-lot-0.11)
+        ("rust-paste" ,rust-paste-0.1)
+        ("rust-pyo3-build-config" ,rust-pyo3-build-config-0.14)
+        ("rust-pyo3-macros" ,rust-pyo3-macros-0.14)
+        ("rust-serde" ,rust-serde-1)
+        ("rust-unindent" ,rust-unindent-0.1))
+       #:cargo-development-inputs
+       (("rust-assert-approx-eq" ,rust-assert-approx-eq-1)
+        ("rust-criterion" ,rust-criterion-0.3)
+        ("rust-proptest" ,rust-proptest-0.10)
+        ("rust-rustversion" ,rust-rustversion-1)
+        ("rust-serde-json" ,rust-serde-json-1)
+        ("rust-trybuild" ,rust-trybuild-1))))
+    (inputs (list python))
+    (home-page "https://github.com/pyo3/pyo3")
+    (synopsis "Rust bindings for the Python interpreter")
+    (description
+     "This package provides Rust bindings for Python, including tools for
+creating native Python extension modules.  Running and interacting with
+Python code from a Rust binary is also supported.")
+    (license license:asl2.0)))
+
+(define-public rust-pyo3-build-config-0.14
+  (package
+    (name "rust-pyo3-build-config")
+    (version "0.14.5")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "pyo3-build-config" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "13vkcl49b2x81azb613ss256k8pazrfc4fy4ny8pzgdciirn2afi"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs (("rust-once-cell" ,rust-once-cell-1))))
+    (home-page "https://github.com/pyo3/pyo3")
+    (synopsis "Build configuration for the PyO3 ecosystem")
+    (description "Build configuration for the PyO3 ecosystem")
+    (license license:asl2.0)))
+
+(define-public rust-pyo3-macros-0.14
+  (package
+    (name "rust-pyo3-macros")
+    (version "0.14.5")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "pyo3-macros" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "1a4fh24c5q85f31n2rwbqrai2bjprf9kzh6xvpgj8j3hblhwa2zw"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs
+       (("rust-pyo3-macros-backend" ,rust-pyo3-macros-backend-0.14)
+        ("rust-quote" ,rust-quote-1)
+        ("rust-syn" ,rust-syn-1))))
+    (home-page "https://github.com/pyo3/pyo3")
+    (synopsis "Proc macros for PyO3 package")
+    (description "Proc macros for PyO3 package")
+    (license license:asl2.0)))
+
+(define-public rust-pyo3-macros-backend-0.14
+  (package
+    (name "rust-pyo3-macros-backend")
+    (version "0.14.5")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "pyo3-macros-backend" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "1ps068jqyq1275zxxbzn6hyz9lkfz35az8waj6mzlji2jg2kyqki"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs
+       (("rust-proc-macro2" ,rust-proc-macro2-1)
+        ("rust-pyo3-build-config" ,rust-pyo3-build-config-0.14)
+        ("rust-quote" ,rust-quote-1)
+        ("rust-syn" ,rust-syn-1))))
+    (home-page "https://github.com/pyo3/pyo3")
+    (synopsis "Code generation for PyO3 package")
+    (description "Code generation for PyO3 package")
+    (license license:asl2.0)))
+
+(define-public rust-quick-csv-0.1
+  (package
+    (name "rust-quick-csv")
+    (version "0.1.6")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "quick-csv" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "131k8zzlplk2h62wz813jbvm0sk7v3mixwhhq34y9lmp3mqbgx7d"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs (("rust-rustc-serialize" ,rust-rustc-serialize-0.3))))
+    (home-page "https://github.com/tafia/quick-csv")
+    (synopsis "quick csv reader and decoder")
+    (description "quick csv reader and decoder")
+    (license license:expat)))
+
+(define-public rust-succinct-0.5
+  (package
+    (name "rust-succinct")
+    (version "0.5.2")
+    (source
+      (origin
+        (method url-fetch)
+        (uri (crate-uri "succinct" version))
+        (file-name
+         (string-append name "-" version ".tar.gz"))
+        (sha256
+         (base32 "0654c9gq50x7djyf25zbzz3d2pc4x3z21wmjj3qbr6d9h4hbd63p"))))
+    (build-system cargo-build-system)
+    (arguments
+     `(#:cargo-inputs
+       (("rust-byteorder" ,rust-byteorder-1)
+        ("rust-num-traits" ,rust-num-traits-0.2))
+       #:cargo-development-inputs
+       (("rust-quickcheck" ,rust-quickcheck-0.9))))
+    (home-page "https://github.com/tov/succinct-rs")
+    (synopsis "Succinct data structures for Rust")
+    (description "This package provides succinct data structures for Rust.")
+    (license (list license:expat license:asl2.0))))
 
 (define-public rust-clap-for-jrep
   (package
@@ -141,42 +475,42 @@ or any combination.")
       (description "Rust CLI tools for manipulation of Jupyter Notebooks.")
       (license #f)))) ; There is no license.
 
-;; replace fields with those from upstream
-(define-public rust-clap-3.1
+(define-public rust-clap-4
   (package
     (name "rust-clap")
-    (version "3.1.6")
-    (source
-      (origin
-        (method url-fetch)
-        (uri (crate-uri "clap" version))
-        (file-name (string-append name "-" version ".tar.gz"))
-        (sha256
-         (base32 "08q1hkksfixybnrwrpm44xq028wbn9yr2hnzrax9hihyq8v39jfq"))))
+    (version "4.0.9")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "clap" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "1w0dxqzrh449s9l2k8g66pdsff02599bwi5mh0gny3227kcpsq1h"))))
     (build-system cargo-build-system)
     (arguments
-     `(#:cargo-inputs
+     `(#:skip-build? #t     ; Not all inputs packaged.
+       #:cargo-inputs
        (("rust-atty" ,rust-atty-0.2)
         ("rust-backtrace" ,rust-backtrace-0.3)
         ("rust-bitflags" ,rust-bitflags-1)
-        ("rust-clap-derive" ,rust-clap-derive-3.1)
-        ("rust-indexmap" ,rust-indexmap-1)
-        ("rust-lazy-static" ,rust-lazy-static-1)
-        ("rust-os-str-bytes" ,rust-os-str-bytes-6)
-        ("rust-regex" ,rust-regex-1)
+        ("rust-clap-derive" ,rust-clap-derive-4)
+        ("rust-clap-lex" ,rust-clap-lex-0.3)
+        ("rust-once-cell" ,rust-once-cell-1)
         ("rust-strsim" ,rust-strsim-0.10)
         ("rust-termcolor" ,rust-termcolor-1)
         ("rust-terminal-size" ,rust-terminal-size-0.1)
-        ("rust-textwrap" ,rust-textwrap-0.15)
         ("rust-unicase" ,rust-unicase-2)
-        ("rust-yaml-rust" ,rust-yaml-rust-0.4))
-       #:cargo-development-inputs
-       (("rust-criterion" ,rust-criterion-0.3)
-        ("rust-lazy-static" ,rust-lazy-static-1)
-        ("rust-regex" ,rust-regex-1)
-        ("rust-rustversion" ,rust-rustversion-1)
-        ("rust-trybuild" ,rust-trybuild-1)
-        ("rust-trycmd" ,rust-trycmd-0.12))))
+        ("rust-unicode-width" ,rust-unicode-width-0.1))
+       ;#:cargo-development-inputs
+       ;(("rust-humantime" ,rust-humantime-2)
+       ; ("rust-rustversion" ,rust-rustversion-1)
+       ; ("rust-shlex" ,rust-shlex-1)
+       ; ("rust-snapbox" ,rust-snapbox-0.4)
+       ; ("rust-static-assertions" ,rust-static-assertions-1)
+       ; ("rust-trybuild" ,rust-trybuild-1)
+       ; ("rust-trycmd" ,rust-trycmd-0.13)
+       ; ("rust-unic-emoji-char" ,rust-unic-emoji-char-0.9))
+       ))
     (home-page "https://github.com/clap-rs/clap")
     (synopsis
       "A simple to use, efficient, and full-featured Command Line Argument Parser")
@@ -185,19 +519,17 @@ or any combination.")
       Argument Parser")
     (license (list license:expat license:asl2.0))))
 
-;; ready to upstream, WITH rust-clap-derive
-;; replace fields with those from upstream.
-(define-public rust-clap-derive-3.1
+(define-public rust-clap-derive-4
   (package
     (name "rust-clap-derive")
-    (version "3.1.4")
-    (source
-      (origin
-        (method url-fetch)
-        (uri (crate-uri "clap-derive" version))
-        (file-name (string-append name "-" version ".tar.gz"))
-        (sha256
-         (base32 "05mz2y6k73wc1gvv9r4mllfqslzvlwkvx77lk7769ag1xlwd15fs"))))
+    (version "4.0.9")
+    (source (origin
+              (method url-fetch)
+              (uri (crate-uri "clap-derive" version))
+              (file-name (string-append name "-" version ".tar.gz"))
+              (sha256
+               (base32
+                "02zhbbmyz3dpy9ml6xfp7i8p3ffj1djvkdnkg6gr6d0s5r4hg8x4"))))
     (build-system cargo-build-system)
     (arguments
      `(#:cargo-inputs
@@ -213,40 +545,6 @@ or any combination.")
       "Parse command line argument by defining a struct, derive crate.")
     (license (list license:expat license:asl2.0))))
 
-(define-public rust-textwrap-0.15
-  (package
-    (name "rust-textwrap")
-    (version "0.15.0")
-    (source
-      (origin
-        (method url-fetch)
-        (uri (crate-uri "textwrap" version))
-        (file-name (string-append name "-" version ".tar.gz"))
-        (sha256
-         (base32 "1yw513k61lfiwgqrfvsjw1a5wpvm0azhpjr2kr0jhnq9c56is55i"))))
-    (build-system cargo-build-system)
-    (arguments
-     `(#:skip-build? #t     ; Not all inputs packaged
-       ;#:tests? #f          ; Skip tests for now
-       #:cargo-inputs
-       (("rust-hyphenation" ,rust-hyphenation-0.8)
-        ("rust-smawk" ,rust-smawk-0.3)
-        ("rust-terminal-size" ,rust-terminal-size-0.1)
-        ("rust-unicode-linebreak" ,rust-unicode-linebreak-0.1)
-        ("rust-unicode-width" ,rust-unicode-width-0.1))
-       #:cargo-development-inputs
-       (("rust-criterion" ,rust-criterion-0.3)
-        ("rust-lipsum" ,rust-lipsum-0.8)
-        ("rust-termion" ,rust-termion-1)
-        ;("rust-unic-emoji-char" ,rust-unic-emoji-char-0.9)
-        ("rust-version-sync" ,rust-version-sync-0.9))))
-    (home-page "https://github.com/mgeisler/textwrap")
-    (synopsis
-      "Powerful library for word wrapping, indenting, and dedenting strings")
-    (description
-      "Powerful library for word wrapping, indenting, and dedenting strings")
-    (license license:expat)))
-
 (define-public rust-trycmd-0.12
   (package
     (name "rust-trycmd")
@@ -260,9 +558,7 @@ or any combination.")
          (base32 "1rwa5nzq8c5zg7lqmpkf7hyib415yxshd9amp911y8w1zss4s38p"))))
     (build-system cargo-build-system)
     (arguments
-     `(;#:skip-build? #t     ; Not all inputs at correct versions?
-       ;#:tests? #f          ; Skip tests for now
-       #:cargo-inputs
+     `(#:cargo-inputs
        (("rust-backtrace" ,rust-backtrace-0.3)
         ("rust-concolor" ,rust-concolor-0.0.8)
         ("rust-content-inspector" ,rust-content-inspector-0.2)
@@ -302,7 +598,7 @@ or any combination.")
          (base32 "0wx4wd849bmkqj0gdi041gmpfpvlyhy2ha4zpin69yw9d9npl8cl"))))
     (build-system cargo-build-system)
     (arguments
-     `(;#:skip-build? #t     ; Not all inputs packaged
+     `(#:skip-build? #t     ; Not all inputs packaged
        ;#:tests? #f          ; Skip tests for now
        #:cargo-inputs
        (("rust-combine" ,rust-combine-4)
@@ -371,7 +667,7 @@ or any combination.")
     (license license:gpl3+)))
 
 (define-public gn-rust-correlation
-  (let ((commit "fd85c0f75e03f170b97fcedba26317e0832d087e")
+  (let ((commit "a22381e9830bac9a6848449279d258397217e251")
         (revision "0"))
     (package
       (name "gn-rust-correlation")
@@ -385,7 +681,7 @@ or any combination.")
           (file-name (git-file-name name version))
           (sha256
            (base32
-            "164z2abp8vassxi8bpf68wgiqjmafd6hzrkz4hj3wdhf4djiplrr"))))
+            "1cb197p8zmgxymzs9b7bygs0kg67809z6gc2svza909sxr9ac5ps"))))
       (build-system cargo-build-system)
       (native-inputs
        (list gsl))
diff --git a/gn/packages/gemini.scm b/gn/packages/gemini.scm
deleted file mode 100644
index 9cefbeb..0000000
--- a/gn/packages/gemini.scm
+++ /dev/null
@@ -1,82 +0,0 @@
-(define-module (gn packages gemini)
-  #:use-module (gnu packages autotools)
-  #:use-module (gnu packages gettext)
-  #:use-module (gnu packages guile)
-  #:use-module (gnu packages guile-xyz)
-  #:use-module ((gnu packages skribilo) #:prefix guix:)
-  #:use-module (guix build-system gnu)
-  #:use-module (guix gexp)
-  #:use-module (guix packages)
-  #:use-module (guix git-download)
-  #:use-module ((guix licenses) #:prefix license:))
-
-(define-public skribilo-latest
-  (let ((commit "621eb1945aec8f26f5aee4bdf896f2434e145182")
-        (revision "1"))
-    (package
-      (inherit guix:skribilo)
-      (name "skribilo")
-      (version (git-version "0.9.5" revision commit))
-      (source (origin
-                (method git-fetch)
-                (uri (git-reference
-                      (url "https://git.systemreboot.net/skribilo")
-                      (commit commit)))
-                (file-name (git-file-name name version))
-                (sha256
-                 (base32
-                  "16rdcvszl9x183y32hjdwns0lkrvkmwd2fsshymspb12k4cxj6i4"))))
-      (native-inputs
-       `(("autoconf" ,autoconf)
-         ("automake" ,automake)
-         ("gettext" ,gnu-gettext)
-         ,@(package-native-inputs guix:skribilo))))))
-
-(define-public tissue
-  (let ((commit "6d6285d071132960835f848a1703faaea2356937")
-        (revision "3"))
-    (package
-      (name "tissue")
-      (version (git-version "0.1.0" revision commit))
-      (source (origin
-                (method git-fetch)
-                (uri (git-reference
-                      (url "https://git.systemreboot.net/tissue")
-                      (commit commit)))
-                (file-name (git-file-name name version))
-                (sha256
-                 (base32
-                  "1dlcy7m4gz1vmklyny4mxky9822q5hjc4qdmn42yf2qvh8xy62g5"))))
-      (build-system gnu-build-system)
-      (arguments
-       (list #:make-flags #~(list (string-append "prefix=" #$output))
-             #:modules `(((guix build guile-build-system)
-                          #:select (target-guile-effective-version))
-                         ,@%gnu-build-system-modules)
-             #:phases
-             (with-imported-modules '((guix build guile-build-system))
-               #~(modify-phases %standard-phases
-                   (replace 'patch-source-shebangs
-                     (lambda* (#:key inputs #:allow-other-keys)
-                       (substitute* "bin/tissue"
-                         (("^exec guile")
-                          (string-append "exec " (search-input-file inputs "/bin/guile"))))))
-                   (delete 'configure)
-                   (add-after 'install 'wrap
-                     (lambda* (#:key inputs outputs #:allow-other-keys)
-                       (let ((out (assoc-ref outputs "out"))
-                             (effective-version (target-guile-effective-version)))
-                         (wrap-program (string-append out "/bin/tissue")
-                           `("GUILE_LOAD_PATH" prefix
-                             (,(string-append out "/share/guile/site/" effective-version)
-                              ,(getenv "GUILE_LOAD_PATH")))
-                           `("GUILE_LOAD_COMPILED_PATH" prefix
-                             (,(string-append out "/lib/guile/" effective-version "/site-ccache")
-                              ,(getenv "GUILE_LOAD_COMPILED_PATH")))))))))))
-      (inputs (list guile-3.0 guile-filesystem guile-git guile-xapian))
-      (propagated-inputs
-       (list skribilo-latest))
-      (home-page "https://tissue.systemreboot.net")
-      (synopsis "Text based issue tracker")
-      (description "tissue is a text based issue tracker.")
-      (license license:gpl3+))))
diff --git a/gn/packages/gemma.scm b/gn/packages/gemma.scm
index e73a40d..e8c5194 100644
--- a/gn/packages/gemma.scm
+++ b/gn/packages/gemma.scm
@@ -27,7 +27,7 @@
 (define-public gemma-gn2 ; guix candidate - currently uses generic
                          ; openblas version and genenetwork github repo
 
-  (let ((commit "71553f5e5626e1d791b5be24c84ea6b17ae81cc7"))
+  (let ((commit "8cd4cdb23ecae8387c942ae5c203c5d61e137376"))
   (package
     (name "gemma-gn2")
     (version (string-append "0.98.5-" (string-take commit 7)))
@@ -39,8 +39,9 @@
              (file-name (string-append name "-" version "-checkout"))
              (sha256
               (base32
-               "11farc7smvkrrkvkbvh26i3sycdzwxrbgj536s3478v8j6iiwijp"))))
+            "06csrh8rgcjcl52ljyynizb15gy84gcd6glzl0gfkawmxy6am9n4"))))
     (inputs `(
+              ("catch" ,catch2)
               ("gsl" ,gsl)
               ("shunit2-old" ,shunit2-old)
               ("openblas" ,openblas)
diff --git a/gn/packages/genenetwork.scm b/gn/packages/genenetwork.scm
index 18b77e4..36d3ae6 100644
--- a/gn/packages/genenetwork.scm
+++ b/gn/packages/genenetwork.scm
@@ -27,8 +27,10 @@
   #:use-module (gnu packages golang)
   #:use-module (gnu packages graph)
   #:use-module (gnu packages graphviz)
+  #:use-module (gnu packages machine-learning)
   #:use-module (gnu packages parallel)
   #:use-module (gnu packages python)
+  #:use-module (gnu packages python-build)
   #:use-module (gnu packages python-check)
   #:use-module (gnu packages python-crypto)
   #:use-module (gnu packages python-science)
@@ -41,9 +43,7 @@
   #:use-module (gnu packages statistics)
   #:use-module (gnu packages textutils)
   #:use-module (gnu packages version-control)
-  ; #:use-module (gnu packages vim)
   #:use-module (gnu packages web)
-  ; #:use-module (gnu packages wget)
   #:use-module (gnu packages xml)
   #:use-module (gn packages bioinformatics)
   #:use-module (gn packages crates-io)
@@ -51,8 +51,8 @@
   #:use-module (gn packages javascript)
   #:use-module (gn packages python)
   #:use-module (gn packages statistics)
-  #:use-module (gn packages twint)
   #:use-module (gn packages web)
+  #:use-module (gn packages python-web)
   #:use-module (srfi srfi-1))
 
 
@@ -100,11 +100,20 @@
       (description "Reimplementation of genenetwork/QTLReaper in Rust")
       (license #f))))
 
+; Tests on the upstream python-pengouin package are broken. So, we
+; create this temporary workaround.
+(define python-pingouin-without-tests
+ (package
+   (inherit python-pingouin)
+   (arguments
+    (substitute-keyword-arguments (package-arguments python-pingouin)
+      ((#:tests? _ #f) #f)))))
+
 (define-public genenetwork3
-  (let ((commit "fe1b8be86b65346724f8f78ab9e5d897e0c480b0"))
+  (let ((commit "f52247c15f3694f3dd5fd0fd79c3e15376137e07"))
     (package
       (name "genenetwork3")
-      (version (git-version "0.1.0" "2" commit))
+      (version (git-version "0.1.0" "3" commit))
       (source
        (origin
          (method git-fetch)
@@ -114,7 +123,9 @@
          (file-name (git-file-name name version))
          (sha256
           (base32
-           "1i7g2c3irp5rr2b8vb7xm9v1hjb5vssc92s2lq910qszd53pdzgn"))))
+           "0ac0dr8dny65x4xvm8gw6ap3g8g0j933ipy9116idcws31rk2adk"))))
+      (inputs
+       (list python-click))
       (native-inputs
        (list python-hypothesis
              python-mypy
@@ -127,7 +138,8 @@
              python-wrapper
              csvdiff
              gn-rust-correlation
-             python-bcrypt
+             python-bcrypt ;; Replace use of bcrypt with argon below
+	     python-argon2-cffi
              python-flask
              python-flask-cors
              ;; Not working in Python > 3.8
@@ -136,12 +148,18 @@
              python-mysqlclient
              python-numpy
              python-pandas
-             python-pingouin
+             ;; python-pingouin << build failing
+             python-pingouin-without-tests
              python-plotly
+             python-scikit-learn
+             python-pymonad
              python-redis
              python-requests
              python-scipy
+	     python-authlib
              python-sparqlwrapper
+	     python-email-validator
+             python-xapian-bindings
              r-optparse
              r-qtl
              r-rjson
@@ -149,15 +167,23 @@
              r-wgcna
              r-ctl
              rust-qtlreaper
-	     diffutils))
+	     diffutils
+	     yoyo-migrations))
       (build-system python-build-system)
+      (arguments
+       (list #:phases
+             #~(modify-phases %standard-phases
+                 (replace 'check
+                   (lambda* (#:key tests? #:allow-other-keys)
+                     (when tests?
+                       (invoke "pytest" "-k" "unit_test")))))))
       (home-page "https://github.com/genenetwork/genenetwork3")
       (synopsis "GeneNetwork3 API for data science and machine learning.")
       (description "GeneNetwork3 API for data science and machine learning.")
       (license license:agpl3+))))
 
 (define-public genenetwork2
-  (let ((commit "13289325fb22c4bc10b52414fb9755e7911795f3"))
+  (let ((commit "bfe557dc1e537dc78a82a30817ecf2ca3004d978"))
     (package
       (name "genenetwork2")
       (version (git-version "3.11" "2" commit))
@@ -169,7 +195,7 @@
                 (file-name (string-append name "-" version))
                 (sha256
                  (base32
-                  "1ny0ix9h7r52lb2qhf2hpfqijckhkw4jr1wp53qfzrd84lf4pii4"))))
+                  "1bn0j0fpk4hcicgfird62x5wq2n6lj4rs1ggw69dcxyf4qdxbk5d"))))
       (native-inputs
        (list graphviz))
       (propagated-inputs
@@ -221,13 +247,13 @@
          ("python-simplejson" ,python-simplejson)
          ("python-markdown" ,python-markdown)
          ("python-rdflib" ,python-rdflib)
-         ("python-twint" ,python-twint)
+	 ("python-authlib" ,python-authlib)
+	 ("python-flask-session" ,python-flask-session)
          ;; TODO: Get rid of Python R bindings
          ("python-rpy2" ,python-rpy2)
          ("python-beautifulsoup4" ,python-beautifulsoup4)
          ;; Disable for now. Build fails on Penguin2
          ;; ("python-flask-socketio" ,python-flask-socketio)
-         ("python-xapian-bindings" ,python-xapian-bindings)
          ("python-xlsxwriter" ,python-xlsxwriter)
          ;; All the external js dependencies
          ("javascript-twitter-post-fetcher" ,javascript-twitter-post-fetcher)
diff --git a/gn/packages/julia.scm b/gn/packages/julia.scm
index 6dddd4b..31cc29c 100644
--- a/gn/packages/julia.scm
+++ b/gn/packages/julia.scm
@@ -23,19 +23,19 @@
 (define S specification->package)
 
 (define-public julia-visuals
-  (let ((commit "e7d670eb045a9f8e3a839476dc166318da7fe9dc")
-        (revision "1"))
+  (let ((commit "e8e2b601f40a76c8f20f0ddfe80c56257dd9a294")
+        (revision "2"))
     (package
       (name "julia-visuals")
       (version (git-version "0.0.0" revision commit))
       (source (origin
                 (method git-fetch)
                 (uri (git-reference
-                       (url "https://github.com/sens/visuals")
-                       (commit commit)))
+                      (url "https://github.com/sens/visuals")
+                      (commit commit)))
                 (file-name (git-file-name name version))
                 (sha256
-                 (base32 "15hshm5qrig5qbj02xy4ji79kfc72n93nna5nvxkhvb8gw3vvx07"))))
+                 (base32 "0lm9yhk0mq5cvvkcbsgcjc1y7fzhr8qz2nxn38cy1zdxd8vfknsx"))))
       (build-system julia-build-system)
       (arguments
        `(#:tests? #f    ; no test suite
@@ -53,38 +53,66 @@
                (let ((out (assoc-ref outputs "out")))
                  ;; Copied from the Dockerfile.
                  (for-each
-                   (lambda (file)
-                     (copy-recursively file (string-append out "/" file)))
-                   (list "plutoserver"
-                         "environment.yml"
-                         "setup.py"
-                         "runpluto.sh"
-                         "notebooks"
-                         "Project.toml"
-                         "Manifest.toml")))))
+                  (lambda (file)
+                    (copy-recursively file (string-append out "/" file)))
+                  (list "plutoserver"
+                        "environment.yml"
+                        "setup.py"
+                        "runpluto.sh"
+                        "notebooks"
+                        "Project.toml")))))
            (add-after 'install 'wrap-program
              (lambda* (#:key inputs outputs #:allow-other-keys)
                (let ((out (assoc-ref outputs "out")))
                  ;; Do we need to wrap this with PYTHONPATH too?
                  (wrap-script (string-append out "/runpluto.sh")
-                   `("PATH" ":" prefix (,(string-append (assoc-ref inputs "julia") "/bin")
+                  `("PATH" ":" prefix (,(string-append (assoc-ref inputs "julia") "/bin")
                                         ,(string-append (assoc-ref inputs "coreutils") "/bin")))
-                   `("JULIA_LOAD_PATH" ":" prefix (,(getenv "JULIA_LOAD_PATH")))))))
+                  `("JULIA_LOAD_PATH" ":" prefix (,(getenv "JULIA_LOAD_PATH")))))))
+           (add-after 'install 'create-run-program
+             (lambda* (#:key inputs outputs #:allow-other-keys)
+               (let ((out (assoc-ref outputs "out")))
+                 (with-output-to-file (string-append out "/runpluto")
+                   (lambda ()
+                     (format #t "#!~a --no-auto-compile
+!#
+(setenv \"JULIA_LOAD_PATH\" \"~a\")
+(setenv \"PATH\" \"~a\")
+(zero? (system*
+         \"~a\"
+         \"--project=/home/jovyan\"
+         \"--optimize=0\"
+         \"-e\" \"import Pluto;
+         Pluto.run(
+            host=\\\"0.0.0.0\\\",
+            port=4343,
+            launch_browser=false,
+            require_secret_for_open_links=false;
+            require_secret_for_access=false)\"))\n"
+                             (search-input-file inputs "/bin/guile")
+                             (getenv "JULIA_LOAD_PATH")
+                             (dirname (search-input-file inputs "/bin/yes"))
+                             (search-input-file inputs "/bin/julia"))))
+                   (chmod (string-append out "/runpluto") #o555))))
            (replace 'precompile
              (lambda _
                (invoke "julia" "-e" "\"import Pkg; Pkg.instantiate(); Pkg.status(); Pkg.precompile()\""))))))
+
       (propagated-inputs
        `(;; from setup.py
          ("python-jupyter-server-proxy"
           ,(@ (gn packages python) python-jupyter-server-proxy-1))))
+
       (inputs
-       `(("julia-distributions" ,julia-distributions)
-         ("julia-latexstrings" ,julia-latexstrings)
-         ("julia-optim" ,julia-optim)
-         ("julia-plots" ,julia-plots)
-         ("julia-pluto" ,julia-pluto)
-         ("julia-plutoui" ,julia-plutoui)
-         ("guile" ,(@ (gnu packages guile) guile-3.0))))    ; for wrap-script
+       (list julia-distributions
+             ;julia-interactiveutils    ; stdlib
+             julia-latexstrings
+             ;julia-markdown            ; stdlib
+             julia-optim
+             julia-plots
+             julia-pluto
+             julia-plutoui
+             (@ (gnu packages guile) guile-3.0)))   ; for wrap-script
       (home-page "https://github.com/sens/visuals")
       (synopsis "Visualizations using Pluto.jl notebooks")
       (description "Visualizations using Pluto.jl notebooks.")
@@ -262,7 +290,7 @@ distributed computing.")
 (define-public julia-distributions
   (package
     (name "julia-distributions")
-    (version "0.25.11")
+    (version "0.25.80")
     (source
       (origin
         (method git-fetch)
@@ -271,12 +299,14 @@ distributed computing.")
                (commit (string-append "v" version))))
         (file-name (git-file-name name version))
         (sha256
-         (base32 "0n5xgdpzrpb4s0g23rjggk7c7x8677hbhq0sam7xbw9mn2w79m7n"))))
+         (base32 "0nqlnkh8grxfm8d1mivi7dnrvb31bznj9s540a10d2v396ikfggn"))))
     (build-system julia-build-system)
     (arguments
      `(#:tests? #f))        ; Some failed tests
     (propagated-inputs
-     `(("julia-fillarrays" ,julia-fillarrays)
+     `(("julia-chainrulescore" ,julia-chainrulescore)
+       ("julia-densityinterface" ,julia-densityinterface)
+       ("julia-fillarrays" ,julia-fillarrays)
        ("julia-pdmats" ,julia-pdmats)
        ("julia-quadgk" ,julia-quadgk)
        ("julia-specialfunctions" ,julia-specialfunctions)
@@ -284,9 +314,11 @@ distributed computing.")
        ("julia-statsfuns" ,julia-statsfuns)))
     (native-inputs
      `(("julia-calculus" ,julia-calculus)
+       ("julia-chainrulestestutils" ,julia-chainrulestestutils)
        ("julia-finitedifferences" ,julia-finitedifferences)
        ("julia-forwarddiff" ,julia-forwarddiff)
        ("julia-json" ,julia-json)
+       ("julia-offsetarrays" ,julia-offsetarrays)
        ("julia-stablerngs" ,julia-stablerngs)
        ("julia-staticarrays" ,julia-staticarrays)))
     (home-page "https://github.com/JuliaStats/Distributions.jl")
@@ -303,6 +335,31 @@ properties
 @end enumerate")
     (license license:expat)))
 
+;; ready to upstream
+(define-public julia-densityinterface
+  (package
+    (name "julia-densityinterface")
+    (version "0.4.0")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaMath/DensityInterface.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "10yr69lndh4jdyhjnpm421zvbw8v48bimxjawz05lqkd7k4w4lw6"))))
+    (build-system julia-build-system)
+    (propagated-inputs
+     (list julia-inversefunctions))
+    (native-inputs
+     (list julia-documenter))
+    (home-page "https://github.com/JuliaMath/DensityInterface.jl")
+    (synopsis "Interface for mathematical/statistical densities")
+    (description "This package defines an interface for mathematical/statistical
+densities and objects associated with a density in Julia.")
+    (license license:expat)))
+
 (define-public julia-plots
   (package
     (name "julia-plots")
@@ -369,40 +426,40 @@ properties
 (define-public julia-pluto
   (package
     (name "julia-pluto")
-    (version "0.15.1")
+    (version "0.19.9")
     (source
       (origin
         (method git-fetch)
         (uri (git-reference
                (url "https://github.com/fonsp/Pluto.jl")
-               (commit (string-append "v" version))))
+               (commit (string-append "v" version "-src"))))
         (file-name (git-file-name name version))
         (sha256
-         (base32 "1jsvqi33rsj8izm9pb0r4gjzb5xd01dxri8xp95h84kd0rdliirr"))))
+         (base32 "0h9sz4mpf3a4k0f5fblbb6j07wdhrnarxajrn0wz6zsq6w30x6yj"))))
     (build-system julia-build-system)
     (arguments
-     `(#:tests? #f      ; Test suite fails to load HTTP.jl.
+     `(#:tests? #f      ; Test suite tries to download the package registry.
        #:phases
        (modify-phases %standard-phases
          (add-after 'link-depot 'dont-check-for-upgrades
            (lambda _
-             (substitute* "frontend/components/Welcome.js"
-               (("local_index !== -1") "false"))))
-         (add-after 'link-depot 'skip-network-tests
-           (lambda _
-             (substitute* "test/runtests.jl"
-               ;; Attempts to update the package registry.
-               ((".*Basic.jl.*") "")))))))
+             (substitute* "frontend/components/welcome/Welcome.js"
+               ((".*new_update_message.*") "")))))))
     (propagated-inputs
-     `(("julia-configurations" ,julia-configurations)
-       ("julia-fuzzycompletions" ,julia-fuzzycompletions)
-       ("julia-http" ,julia-http)
-       ("julia-msgpack" ,julia-msgpack)
-       ("julia-tableiointerface" ,julia-tableiointerface)
-       ("julia-tables" ,julia-tables)))
+     (list julia-configurations
+           julia-fuzzycompletions
+           julia-http
+           julia-hypertextliteral
+           julia-mimes
+           julia-msgpack
+           julia-precompilesignatures
+           julia-relocatablefolders
+           julia-tables
+           julia-uris))
     (native-inputs
-     `(("julia-dataframes" ,julia-dataframes)
-       ("julia-offsetarrays" ,julia-offsetarrays)))
+     (list julia-dataframes
+           julia-offsetarrays
+           julia-timeroutputs))
     (home-page "https://github.com/fonsp/Pluto.jl")
     (synopsis "Simple reactive notebooks for Julia")
     (description "A Pluto notebook is made up of small blocks of Julia code
@@ -412,11 +469,10 @@ placed in arbitrary order - intelligent syntax analysis figures out the
 dependencies between them and takes care of execution.")
     (license license:expat)))
 
-;; ready to upstream, wait on Pluto.jl?
 (define-public julia-plutoui
   (package
     (name "julia-plutoui")
-    (version "0.7.9")
+    (version "0.7.51")
     (source
       (origin
         (method git-fetch)
@@ -425,24 +481,601 @@ dependencies between them and takes care of execution.")
                (commit (string-append "v" version))))
         (file-name (git-file-name name version))
         (sha256
-         (base32
-          "0p159b4m0nxbz36ll5kf082vb806n2f26ma145pbhp749aldzplp"))))
+         (base32 "0hqaa8wx7mia3krdwhj2yf8aa4a8h4r09j16dxn7nyc0zcz8hgb2"))))
     (build-system julia-build-system)
     (propagated-inputs
-     `(("julia-json" ,julia-json)
-       ("julia-reexport" ,julia-reexport)
-       ("julia-suppressor" ,julia-suppressor)))
+     (list julia-abstractplutodingetjes
+           julia-colortypes
+           julia-fixedpointnumbers
+           julia-hyperscript
+           julia-hypertextliteral
+           julia-iocapture
+           julia-json
+           julia-mimes
+           julia-reexport
+           julia-uris))
     (home-page "https://github.com/fonsp/PlutoUI.jl")
     (synopsis "Helper package for Julia Pluto")
     (description "This package helps to make @code{html\"<input>\"} a bit more
 native to Julia.  Use it with the @code{@@bind} macro in Pluto.")
+    (license license:unlicense)))
+
+(define-public julia-http-1.5
+  (package
+    (inherit julia-http)
+    (name "julia-http")
+    (version "1.5.5")
+    (source
+     (origin
+       (method git-fetch)
+       (uri (git-reference
+             (url "https://github.com/JuliaWeb/HTTP.jl")
+             (commit (string-append "v" version))))
+       (file-name (git-file-name name version))
+       (sha256
+        (base32 "0qcfixsq65g8hdimygam7cd8nvcz6w7nzkkjk98mvf65dcby4593"))))
+    (arguments
+     (list
+      #:phases
+      #~(modify-phases %standard-phases
+          (add-before 'install 'disable-network-tests
+            (lambda _
+              #;(substitute* "test/runtests.jl"
+                (("\"async.jl") "# \"async.jl")
+                (("\"client.jl") "# \"client.jl"))
+              #;(substitute* "test/aws4.jl"
+                (("@testset.*HTTP.request with AWS authentication.*" all)
+                 (string-append all "return\n")))
+              #;(substitute* "test/insert_layers.jl"
+                (("@testset.*Inserted final layer runs handler.*" all)
+                 (string-append all "return\n")))
+              #;(substitute* "test/multipart.jl"
+                (("@testset \"Setting of Content-Type.*" all)
+                 (string-append all "return\n"))
+                (("@testset \"Deprecation of .*" all)
+                 (string-append all "return\n")))
+              #;(substitute* "test/websockets.jl"
+                (("@testset.*External Host.*" all)
+                 (string-append all "return\n")))
+              #;(substitute* "test/messages.jl"
+                (("@testset.*Read methods.*" all)
+                 (string-append all "return\n"))
+                (("@testset.*Body - .*" all)
+                 (string-append all "return\n"))
+                (("@testset.*Write to file.*" all)
+                 (string-append all "return\n")))
+              #;(substitute* "test/cookies.jl"
+                (("@testset.*Set-Cookie casing.*" all)
+                 (string-append all "return\n")))
+#t)))))
+    (propagated-inputs
+     (list julia-codeczlib
+           julia-inifile
+           julia-loggingextras
+           julia-mbedtls
+           julia-openssl
+           julia-simplebufferstream
+           julia-uris))
+    ;; required for tests
+    (native-inputs
+     (list julia-bufferedstreams
+           julia-json))))
+
+(define-public julia-simplebufferstream
+  (package
+    (name "julia-simplebufferstream")
+    (version "1.1.0")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaPackaging/SimpleBufferStream.jl")
+               ;; First commit after 1.1.0 with a license
+               (commit "80c9854d5d9ea921da6f619624989fa30e83b8be")))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "05c4d73ki4cp38g66ljxwbl2d0dni3w05r8xsd6g1v63x2rqqbgn"))))
+    (build-system julia-build-system)
+    (arguments
+     (list
+       #:phases
+       #~(modify-phases %standard-phases
+           (add-after 'link-depot 'adjust-tests
+             (lambda _
+               (substitute* "test/runtests.jl"
+                 ;; Tests fail when build machine is too *fast*.
+                 (("0\\.01") "0.001")
+                 ;; Don't create circular dependencies with http
+                 (("using HTTP.*") "")
+                 (("@testset.*HTTP.jl.*" all)
+                  (string-append all "return\n"))))))))
+    (home-page "https://github.com/JuliaPackaging/SimpleBufferStream.jl")
+    (synopsis "What Base.BufferStream should be")
+    (description "This is what I wish Base.BufferStream was.")
+    (license license:expat)))
+
+(define-public julia-openssl
+  (package
+    (name "julia-openssl")
+    (version "1.4.1")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaWeb/OpenSSL.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "1s2h4qh9y3alhkf18p4pjqp77mvsb47qagmk68pq0wsx8r3hzhzx"))))
+    (build-system julia-build-system)
+    (arguments
+     (list
+      #:phases
+      #~(modify-phases %standard-phases
+          (add-before 'install 'disable-network-tests
+            (lambda _
+              (substitute* "test/runtests.jl"
+                ;; No /etc/ssl/certs/ca-certificates.crt in the build environment.
+                (("@testset.*(ReadPEMCert|X509Certificate|X509Store).*" all)
+                 (string-append all "return\n"))
+                ;; No network connection
+                (("@testset.*(HttpsConnect|ClosedStream|NoCloseStream).*" all)
+                 (string-append all "return\n"))
+                ;; undefined symbol.
+                ;; XXX: THIS BREAKS THE PACKAGE!
+                (("@testset.*ErrorTaskTLS.*" all)
+                 (string-append all "return\n"))))))))
+    (propagated-inputs
+     (list julia-bitflags
+           julia-openssl-jll))
+    (home-page "https://github.com/JuliaWeb/OpenSSL.jl")
+    (synopsis "Openssl Julia bindings")
+    (description "This package provides Openssl Julia bindings.")
+    (license license:expat)))
+
+#;(define-public julia-openssl-jll
+  (package
+    (name "julia-openssl-jll")
+    (version "3.0.8+0")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaBinaryWrappers/OpenSSL_jll.jl")
+               (commit (string-append "OpenSSL-v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "1d9kcx7a3yv3rgkggq7h436sfjafr66pq8d0lmlcjxdpl46hx3j7"))))
+    (build-system julia-build-system)
+    (arguments
+     '(#:tests? #f ; no runtests
+       #:phases
+       (modify-phases %standard-phases
+         (add-after 'link-depot 'override-binary-path
+           (lambda* (#:key inputs #:allow-other-keys)
+             (map
+               (lambda (wrapper)
+                 (substitute* wrapper
+                   (("lib64") "lib")
+                   (("generate_wrapper_header.*")
+                   (string-append
+                    "generate_wrapper_header(\"OpenSSL\", \""
+                    (assoc-ref inputs "openssl") "\")\n"))))
+               ;; There's a Julia file for each platform, override them all
+               (find-files "src/wrappers/" "\\.jl$")))))))
+    (inputs
+     (list (@ (gnu packages tls) openssl)))
+    (propagated-inputs
+     (list julia-jllwrappers))
+    (home-page "https://github.com/JuliaBinaryWrappers/OpenSSL_jll.jl")
+    (synopsis "Openssl library wrappers")
+    (description "This package provides a wrapper for openssl.")
+    (license license:expat)))
+
+#;(define-public julia-openssl-jll-1.1
+  (package
+    (name "julia-openssl-jll")
+    (version "1.1.21+0")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaBinaryWrappers/OpenSSL_jll.jl")
+               (commit (string-append "OpenSSL-v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "05wgsvy0iviffkcq4w1dg8gaafyxknnk26c1gpcgma5yhfia8yfs"))))
+    (build-system julia-build-system)
+    (arguments
+     '(#:tests? #f ; no runtests
+       #:phases
+       (modify-phases %standard-phases
+         (add-after 'link-depot 'override-binary-path
+           (lambda* (#:key inputs #:allow-other-keys)
+             (map
+               (lambda (wrapper)
+                 (substitute* wrapper
+                   (("generate_wrapper_header.*")
+                   (string-append
+                    "generate_wrapper_header(\"OpenSSL\", \""
+                    (assoc-ref inputs "openssl") "\")\n"))))
+               ;; There's a Julia file for each platform, override them all
+               (find-files "src/wrappers/" "\\.jl$")))))))
+    (inputs
+     (list (@ (gnu packages tls) openssl-1.1)))
+    (propagated-inputs
+     (list julia-jllwrappers))
+    (home-page "https://github.com/JuliaBinaryWrappers/OpenSSL_jll.jl")
+    (synopsis "Openssl library wrappers")
+    (description "This package provides a wrapper for openssl.")
+    (license license:expat)))
+
+(define-public julia-bitflags
+  (package
+    (name "julia-bitflags")
+    (version "0.1.7")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/jmert/BitFlags.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "1wphfpwqm7bsff68d6h9yhhxrh1lq2shyl0afbzp0h5czf6qi2xr"))))
+    (build-system julia-build-system)
+    (home-page "https://github.com/jmert/BitFlags.jl")
+    (synopsis "Enum-like type for bit flag option values")
+    (description "@code{BitFlag.jl} provides an @code{Enum}-like type for bit
+flag option values.")
+    (license license:expat)))
+
+(define-public julia-relocatablefolders
+  (package
+    (name "julia-relocatablefolders")
+    (version "0.3.0")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaPackaging/RelocatableFolders.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "116f98y2w5cqkv2fvddmv54422cpby6d7q1dndgqh2rdlkpb44iw"))))
+    (build-system julia-build-system)
+    (propagated-inputs
+     (list julia-scratch))
+    (home-page "https://github.com/JuliaPackaging/RelocatableFolders.jl")
+    (synopsis "Reference packages in their project directory")
+    (description "An alternative to the @code{@@__DIR__} macro.  Packages that
+wish to reference paths in their project directory run into issues with
+relocatability when used in conjunction with @code{PackageCompiler}.")
+    (license license:expat)))
+
+(define-public julia-scratch
+  (package
+    (name "julia-scratch")
+    (version "1.1.1")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaPackaging/Scratch.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "0s7dmqsskq5vy7sk6pxng5vrznrn6msg7xf8a9kj3a03b617pgap"))))
+    (build-system julia-build-system)
+    (arguments
+     (list #:tests? #f))        ; Tries to download the Julia registry.
+    (home-page "https://github.com/JuliaPackaging/Scratch.jl")
+    (synopsis "Scratch spaces for all your persistent mutable data needs")
+    (description "This repository implements the scratch spaces API for
+package-specific mutable containers of data.  These spaces can contain
+datasets, text, binaries, or any other kind of data that would be convenient to
+store in a location specific to your package.  As compared to Artifacts, these
+containers of data are mutable.  Because the scratch space location on disk is
+not very user-friendly, scratch spaces should, in general, not be used for a
+storing files that the user must interact with through a file browser.  In that
+event, packages should simply write out to disk at a location given by the
+user.  Scratch spaces are designed for data caches that are completely managed
+by a package and should be removed when the package itself is uninstalled.")
+    (license license:expat)))
+
+(define-public julia-hypertextliteral
+  (package
+    (name "julia-hypertextliteral")
+    (version "0.9.4")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaPluto/HypertextLiteral.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "06nzgxrl1aac9bqb37l2lak2aicp9h7fd1ijcva9pq80z0d74a8a"))))
+    (build-system julia-build-system)
+    (arguments (list #:tests? #f))      ; Disable tests until all inputs are pacakged.
+    (propagated-inputs
+     (list julia-tricks))
+    (native-inputs
+     (list julia-documenter
+           ;julia-faker
+           julia-hyperscript
+           ;julia-narrativetest
+           ))
+    (home-page "https://juliapluto.github.io/HypertextLiteral.jl/stable/")
+    (synopsis "Julia library for the string interpolation of HTML and SVG")
+    (description "HypertextLiteral is a Julia package for generating HTML, SVG,
+and other SGML tagged content.  It works similar to Julia string interpolation,
+only that it tracks hypertext escaping needs and provides handy conversions
+dependent upon context.")
+    (license license:expat)))
+
+;; This can be removed with julia-1.10
+(define-public julia-tricks
+  (package
+    (name "julia-tricks")
+    (version "0.1.7")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/oxinabox/Tricks.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "0870hiqpl14wpr2v16wl6fw89r469yjrchv6gn4pfw2irw3nkjip"))))
+    (build-system julia-build-system)
+    (home-page "https://github.com/oxinabox/Tricks.jl")
+    (synopsis "Cunning tricks though the julia compiler internals")
+    (description "Tricks.jl is an particularly cunning package that does tricks
+with the Julia edge system.")
+    (license license:expat)))
+
+(define-public julia-registryinstances
+  (package
+    (name "julia-registryinstances")
+    (version "0.1.0")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/GunnarFarneback/RegistryInstances.jl")
+               (commit "2796d959014475bc19e2dfa174179cdf02642d28")))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "09926dy2s4wcml6s2hrbq1j1r1456d61fvk5fma4sbr9qsvpyyd0"))))
+    (build-system julia-build-system)
+    (arguments
+     (list #:tests? #f))        ; TODO:   Got exception outside of a @test
+    (propagated-inputs
+     (list julia-lazilyinitializedfields))
+    (home-page "https://github.com/GunnarFarneback/RegistryInstances.jl")
+    (synopsis "Access the information in installed Julia registries")
+    (description "Julia's package manager stores package metadata in registries,
+which consist of TOML files in a directory structure.")
+    (license license:expat)))
+
+(define-public julia-lazilyinitializedfields
+  (package
+    (name "julia-lazilyinitializedfields")
+    (version "1.2.1")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/KristofferC/LazilyInitializedFields.jl")
+               (commit "53ed8cbe78b2048105a0e0b355294e7f024e3d14")))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "1jwpka27d873cyn0vxrg7h7ns7fmlqlyx5h38cw6zvfbdhkrmi09"))))
+    (build-system julia-build-system)
+    (native-inputs
+     (list julia-documenter))
+    (home-page "https://github.com/KristofferC/LazilyInitializedFields.jl")
+    (synopsis "Handle lazily initialized fields")
+    (description "A package for handling lazily initialized fields.")
+    (license license:expat)))
+
+(define-public julia-mimes
+  (package
+    (name "julia-mimes")
+    (version "0.1.4")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaWeb/MIMEs.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "0vgc2q2mgbrm16px7cbqsrga9l99djlb1ayg5k1djb0mbnprjajk"))))
+    (build-system julia-build-system)
+    (home-page "https://github.com/JuliaWeb/MIMEs.jl")
+    (synopsis "MIME information: filetype, encoding, gzip")
+    (description "A small package to transform between file extensions and MIME
+types, with bonus features.")
+    (license license:expat)))
+
+(define-public julia-loggingextras
+  (package
+    (name "julia-loggingextras")
+    (version "1.0.0")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaLogging/LoggingExtras.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "0mc0bbr2wsa809q74hg4npdw2xcni4xl40zz50i6djwnxq88yl07"))))
+    (build-system julia-build-system)
+    (home-page "https://github.com/JuliaLogging/LoggingExtras.jl")
+    (synopsis "Composable Loggers for the Julia Logging StdLib")
+    (description "LoggingExtras is designs around allowing you to build
+arbitrarily complicated systems for \"log plumbing\".  That is to say basically
+routing logged information to different places.  It is built around the idea of
+simple parts which are composed together, to allow for powerful and flexible
+definition of your logging system.")
+    (license license:expat)))
+
+(define-public julia-precompilesignatures
+  (package
+    (name "julia-precompilesignatures")
+    (version "3.0.3")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/rikhuijzer/PrecompileSignatures.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "11rjdx8n3q10iis7ryzbvlvgdpi6kd2h53qp7lac02yx6rsgsfx1"))))
+    (build-system julia-build-system)
+    (arguments
+     (list #:tests? #f))    ; cycle with Pluto.jl.
+    ;(native-inputs
+    ; (list julia-pluto))
+    (home-page "https://github.com/rikhuijzer/PrecompileSignatures.jl")
+    (synopsis "Generate precompile directives by reading method signatures")
+    (description "This package reads all method signatures in a package and
+generates precompile directives for any concrete signature that it can find.")
+    (license license:expat)))
+
+(define-public julia-precompiletools
+  (package
+    (name "julia-precompiletools")
+    (version "1.1.2")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaLang/PrecompileTools.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "124qzflwnn34d8x8vz3cmj2m4a55mg5qf8i8jdcwyyrnag3si7zr"))))
+    (build-system julia-build-system)
+    (arguments
+     (list #:tests? #f))        ; Tries to download the Julia registry.
+    (propagated-inputs
+     (list julia-preferences))
+    (home-page "https://github.com/JuliaLang/PrecompileTools.jl")
+    (synopsis "Reduce time-to-first-execution of Julia code")
+    (description "PrecompileTools allows you to reduce the latency of the first
+execution of Julia code.  It is applicable for package developers and for
+\"ordinary users\" in their personal workflows.")
+    (license license:expat)))
+
+(define-public julia-timeroutputs
+  (package
+    (name "julia-timeroutputs")
+    (version "0.5.23")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/KristofferC/TimerOutputs.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "031m7d837cw4c7iz4arrm1a1ppqihhj5jsldvm7z1bc72jxgfrcv"))))
+    (build-system julia-build-system)
+    (propagated-inputs
+     (list julia-exprtools))
+    (home-page "https://github.com/KristofferC/TimerOutputs.jl")
+    (synopsis "Formatted output of timed sections in Julia")
+    (description "TimerOutputs is a small Julia package that is used to generate
+formatted output from timings made in different sections of a program.  It's
+main functionality is the @code{@@timeit} macro, similar to the @code{@@time}
+macro in Base except one also assigns a label to the code section being timed.
+Multiple calls to code sections with the same label (and in the same \"scope\")
+will accumulate the data for that label.  After the program has executed, it is
+possible to print a nicely formatted table presenting how much time,
+allocations and number of calls were made in each section.  The output can be
+customized as to only show the things you are interested in.")
+    (license license:expat)))
+
+(define-public julia-memoize
+  (package
+    (name "julia-memoize")
+    (version "0.4.4")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaCollections/Memoize.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "0mfavihgdmh6zzjx5lzbigmgsa50pa9ik9gk2gq9wam26prkmkba"))))
+    (build-system julia-build-system)
+    (propagated-inputs
+     (list julia-macrotools))
+    (home-page "https://github.com/JuliaCollections/Memoize.jl")
+    (synopsis "Memoize macro for Julia")
+    (description "Easy memoization for Julia.")
+    (license license:bsd-0)))
+
+(define-public julia-abstractplutodingetjes
+  (package
+    (name "julia-abstractplutodingetjes")
+    (version "1.1.4")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaPluto/AbstractPlutoDingetjes.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "03grnv26n3qdadb2awv3d0rs655l5hhmh7h43v949za9aaaz89j3"))))
+    (build-system julia-build-system)
+    (arguments
+     (list #:tests? #f))        ; No tests for this package.
+    (home-page "https://docs.juliahub.com/AbstractPlutoDingetjes/UHbnu/")
+    (synopsis "Abstract package for creating widgets in Pluto.jl")
+    (description "An abstract package to be implemented by packages/people who
+create widgets (or other dingetjes) for Pluto.")
+    (license license:unlicense)))
+
+(define-public julia-hyperscript
+  (package
+    (name "julia-hyperscript")
+    (version "0.0.4")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaWeb/Hyperscript.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "0flap3a9p6fr768kjfk1lkyfgjqcyhzvzl7in7gjszfy04pqj55w"))))
+    (build-system julia-build-system)
+    (home-page "https://github.com/JuliaWeb/Hyperscript.jl")
+    (synopsis "Lightweight DOM representation for Julia")
+    (description "Hyperscript is a package for working with HTML, SVG, and CSS
+in Julia.  When using this library you automatically get:
+@enumerate
+@item A concise DSL for writing HTML, SVG, and CSS.
+@item Flexible ways to combine DOM pieces together into larger components.
+@item Safe and automatic HTML-escaping.
+@item Lightweight and optional support for scoped CSS.
+@item Lightweight and optional support for CSS unit arithmetic.
+@end enumerate")
     (license license:expat)))
 
 ;; ready to upstream
 (define-public julia-statsfuns
   (package
     (name "julia-statsfuns")
-    (version "0.9.8")
+    (version "0.9.18")
     (source
       (origin
         (method git-fetch)
@@ -451,21 +1084,19 @@ native to Julia.  Use it with the @code{@@bind} macro in Pluto.")
                (commit (string-append "v" version))))
         (file-name (git-file-name name version))
         (sha256
-         (base32 "1zl46p9gbx9xkjnnpd45csshqvq2i94mxw10karpr8xkx8msyk3k"))))
+         (base32 "1y71gz4skp6hxw8k5vjbjayplxmdfh3m3yjfw4ggi0azav6c9hrk"))))
     (build-system julia-build-system)
-    (arguments
-     `(#:phases
-       (modify-phases %standard-phases
-         (add-after 'link-depot 'adjust-test-suite
-           (lambda _
-             (substitute* "test/misc.jl"
-               (("test logmvbeta\\(1") "test_nowarn logmvbeta(1")))))))
     (propagated-inputs
-     (list julia-logexpfunctions
+     (list julia-chainrulescore
+           julia-inversefunctions
+           julia-irrationalconstants
+           julia-logexpfunctions
+           julia-reexport
            julia-rmath
            julia-specialfunctions))
     (native-inputs
-     (list julia-forwarddiff))
+     (list julia-chainrulestestutils
+           julia-forwarddiff))
     (home-page "https://github.com/JuliaStats/StatsFuns.jl")
     (synopsis "Mathematical functions related to statistics")
     (description "This package provides a collection of mathematical constants
@@ -473,6 +1104,29 @@ and numerical functions for statistical computing.")
     (license license:expat)))
 
 ;; ready to upstream
+(define-public julia-inversefunctions
+  (package
+    (name "julia-inversefunctions")
+    (version "0.1.8")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/JuliaMath/InverseFunctions.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "05g9f6i735x7syfr56l4yf4fy71kgdisjc6cfxi4jkf46iq86a69"))))
+    (build-system julia-build-system)
+    (native-inputs
+     (list julia-documenter))
+    (home-page "https://github.com/JuliaMath/InverseFunctions.jl")
+    (synopsis "Interface for function inversion in Julia")
+    (description
+     "This package provides an interface to invert functions in Julia.")
+    (license license:expat)))
+
+;; ready to upstream
 (define-public rmath-for-julia-rmath-jll
   ;; More recent commits fix various build issues
   (let ((commit "5c5dfd6baca358103fbb47cc03dc0ecee04fb1ff")
@@ -691,6 +1345,40 @@ polynomials.")
                      AbstractInterval, along with its subtypes Interval and AnchoredInterval, and also Bound.")
     (license license:expat)))
 
+(define-public julia-infinity
+  (package
+    (name "julia-infinity")
+    (version "0.2.4")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/cjdoris/Infinity.jl")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name name version))
+        (sha256
+         (base32 "1941lwvrdjnrynigzixxin3chpg1ba6xplvcwc89x0f6z658hwmm"))))
+    (build-system julia-build-system)
+    (arguments
+     (list
+      #:tests? #f           ; TODO: Fix tests!
+      #:phases
+      #~(modify-phases %standard-phases
+          (add-after 'link-depot 'remove-timezones.jl
+            (lambda _
+              (substitute* "test/runtests.jl"
+                (("using TimeZones.*") "")
+                ((".*infextendedtime.*") "")))))))
+    (propagated-inputs
+     (list julia-requires))
+    (native-inputs
+     (list julia-compat))
+    (home-page "https://docs.juliahub.com/Infinity/")
+    (synopsis "Representation of infinity in Julia")
+    (description "This package provides representations for infinity and
+negative infinity in Julia.")
+    (license license:expat)))
+
 ;; TODO: There is talk upstream about separating out the timezone data into a
 ;; separate package which can allow this to actually be packaged in a sane way.
 ;; As of 1.7.1 there are 257 items in Artifact.toml
diff --git a/gn/packages/lisp.scm b/gn/packages/lisp.scm
index 50ef3b5..111ee52 100644
--- a/gn/packages/lisp.scm
+++ b/gn/packages/lisp.scm
@@ -55,8 +55,6 @@
            (base32 "0n6m3aqvdfnsrhlhqjcy72d1i55lbkjg13ij5c7vw003p1n78wxi"))
           (file-name (git-file-name name commit))))
       (build-system asdf-build-system/sbcl)
-      (arguments
-       `(#:asd-files '("defstar.asd")))
       (home-page "https://github.com/lisp-maintainers/defstar")
       (synopsis "Type declarations for defun et all")
       (description
@@ -89,8 +87,6 @@ with a star added at the end.")
            (base32 "0zgx4ymyzvfg44z36yr4l87cd9mprajd7sycr2zc67ab6330rynf"))
           (file-name (git-file-name name commit))))
       (build-system asdf-build-system/sbcl)
-      (arguments
-       `(#:asd-files '("2am.asd")))
       (home-page "https://gitlab.common-lisp.net/dkochmanski/2am")
       (synopsis "Small testing framework based on 1am")
       (description
@@ -175,8 +171,6 @@ fringilla labore. Tellus penatibus arcu ligula nisi.")
            (base32 "1blmrb4c9gsxj87scz74z1s8w9d1w2r48fyxj0y1sw3vr6bsbb8f"))
           (file-name (git-file-name name commit))))
       (build-system asdf-build-system/sbcl)
-      (arguments
-       `(#:asd-files '("fare-memoization.asd")))
       (native-inputs
         (list sbcl-hu.dwim.stefil))
       (inputs
@@ -212,8 +206,6 @@ with the same arguments again, rather than re-doing the computation.")
            (base32 "08s53zj3mcx82kszp1bg2vsb4kydvkc70kj4hpq9h1l5a1wh44cy"))
           (file-name (git-file-name name commit))))
       (build-system asdf-build-system/sbcl)
-      (arguments
-       `(#:asd-files '("meta.asd")))
       (inputs
         (list sbcl-named-readtables))
       (home-page "https://gitlab.common-lisp.net/frideau/meta")
@@ -245,8 +237,6 @@ generators.")
            (base32 "056qi6vw9bk19s42mapyg55mimhhvhlwgny080v9mhv4fhnqi196"))
           (file-name (git-file-name name commit))))
       (build-system asdf-build-system/sbcl)
-      (arguments
-       `(#:asd-files '("scribble.asd")))
       (native-inputs
         (list sbcl-babel))
       (inputs
diff --git a/gn/packages/machine-learning.scm b/gn/packages/machine-learning.scm
index fbaffb2..8f9f1f0 100644
--- a/gn/packages/machine-learning.scm
+++ b/gn/packages/machine-learning.scm
@@ -11,18 +11,6 @@
     (arguments
      (substitute-keyword-arguments (package-arguments tensorflow)
        ((#:substitutable? _ #f) #f)
-       ;((#:phases phases)
-       ; `(modify-phases ,phases
-       ;    (add-after 'unpack 'hardcode-multicore-usage
-       ;      (lambda _
-       ;        (substitute* "tensorflow/core/protobuf/config.proto"
-       ;          ;(("num_threads = 1") "num_threads = 28") ; 56/2 for penguin2
-       ;          ;(("intra_op_parallelism_threads = 2")
-       ;          ; "intra_op_parallelism_threads = 0")
-       ;          ;(("inter_op_parallelism_threads = 5")
-       ;          ; "inter_op_parallelism_threads = 0")
-       ;          )
-       ;        #t))))
        ((#:configure-flags flags)
         `(cons
            "-Dtensorflow_OPTIMIZE_FOR_NATIVE_ARCH=ON"
diff --git a/gn/packages/notebooks.scm b/gn/packages/notebooks.scm
index 9a22b73..89f1360 100644
--- a/gn/packages/notebooks.scm
+++ b/gn/packages/notebooks.scm
@@ -59,8 +59,7 @@ instance.")
           (file-name (git-file-name name commit))))
       (build-system asdf-build-system/sbcl)
       (arguments
-        `(#:tests? #f ; There are no tests.
-          #:asd-files '("nb.asd")))
+        `(#:tests? #f)) ; There are no tests.
       (inputs
         (list sbcl-ningle
               sbcl-clack
diff --git a/gn/packages/pangenome.scm b/gn/packages/pangenome.scm
index 69e3d81..c66b594 100644
--- a/gn/packages/pangenome.scm
+++ b/gn/packages/pangenome.scm
@@ -1,23 +1,12 @@
-;; Bioinformatics module
+;; Pangenome module
 
 (define-module (gn packages pangenome)
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix packages)
-  #:use-module (guix utils)
   #:use-module (guix download)
-  #:use-module (guix git-download)
   #:use-module (guix build-system cargo)
-  #:use-module (guix build-system gnu)
-  #:use-module (guix graph)
-  #:use-module (guix scripts graph)
-  #:use-module (guix store)
-  #:use-module (guix gexp)
-  #:use-module (gnu packages)
-  #:use-module (gnu packages base)
   #:use-module (gnu packages crates-io)
-  #:use-module (gnu packages rust)
-  #:use-module (gnu packages version-control)
-  #:use-module (srfi srfi-1))
+  #:use-module (gn packages crates-io))
 
 
 (define-public gfautil
diff --git a/gn/packages/python-web.scm b/gn/packages/python-web.scm
new file mode 100644
index 0000000..88478a5
--- /dev/null
+++ b/gn/packages/python-web.scm
@@ -0,0 +1,40 @@
+(define-module (gn packages python-web)
+  ;; core guix packages
+  #:use-module (guix packages)
+  #:use-module (guix download)
+  #:use-module (guix build-system python)
+  #:use-module ((guix licenses) #:prefix license:)
+  ;; packages modules
+  #:use-module (gnu packages django)
+  #:use-module (gnu packages databases)
+  #:use-module (gnu packages python-web)
+  #:use-module (gnu packages python-crypto))
+
+(define-public python-authlib
+  (package
+   (name "python-authlib")
+   (version "1.2.0")
+   (source (origin
+            (method url-fetch)
+            (uri (pypi-uri "Authlib" version))
+            (sha256
+             (base32
+              "178ycfypdv1hy4zjm09rmysxakmwch2n6a5wypwmx4d5hc4fi8sg"))))
+   (build-system python-build-system)
+   (native-inputs (list python-httpx
+			python-flask
+			python-django
+			python-werkzeug
+			python-starlette
+			python-sqlalchemy
+			python-pycryptodomex))
+   (propagated-inputs (list python-cryptography))
+   (arguments `(#:tests? #f)) ;; temporarily deactivate tests
+   (home-page "https://authlib.org/")
+   (synopsis
+    "The ultimate Python library in building OAuth and OpenID Connect servers and clients.")
+   (description
+    "The ultimate Python library in building OAuth and OpenID Connect servers and
+clients. It is designed from low level specifications implementations to high
+level frameworks integrations, to meet the needs of everyone.")
+   (license license:bsd-3)))
diff --git a/gn/packages/python.scm b/gn/packages/python.scm
index d9c4511..510546d 100644
--- a/gn/packages/python.scm
+++ b/gn/packages/python.scm
@@ -9,8 +9,11 @@
   #:use-module (gnu packages image)
   #:use-module (gnu packages libffi)
   #:use-module (gnu packages linux)
+  #:use-module (gnu packages maths)
   #:use-module (gnu packages monitoring)
   #:use-module (gnu packages pcre)
+  #:use-module (gnu packages pkg-config)
+  #:use-module (gnu packages protobuf)
   #:use-module (gnu packages python)
   #:use-module (gnu packages python-build)
   #:use-module (gnu packages python-check)
@@ -29,6 +32,7 @@
   #:use-module (guix packages)
   #:use-module (guix download)
   #:use-module (guix git-download)
+  #:use-module (guix gexp)
   #:use-module (guix utils)
   #:use-module (guix build-system python)
   #:use-module (srfi srfi-1))
@@ -251,12 +255,6 @@ Python 3 support.")
   (package-with-python2 python-avro))
 
 
-; env IPFS_PATH=/export/ipfs/ ipfs add -r htmlgen/
-; added QmUD9LMJTE8q5wYkUyAwLdz2QCGXWt457iFnyYQAGfsN3j htmlgen/htmlgen-2.2.2-gn.tar.gz
-; added QmZLWsPHLFTU3hWAfdTwj3RXFrS8Ma7KEixne1suWuYqeG htmlgen
-; penguin2:~/tmp$ env IPFS_PATH=/export/ipfs/ ipfs pin add -r QmZLWsPHLFTU3hWAfdTwj3RXFrS8Ma7KEixne1suWuYqeG
-; pinned QmZLWsPHLFTU3hWAfdTwj3RXFrS8Ma7KEixne1suWuYqeG recursively
-
 (define-public python2-htmlgen-gn ; guix obsolete
   (package
     (name "python2-htmlgen-gn")
@@ -265,7 +263,7 @@ Python 3 support.")
               (method url-fetch)
               ;; http://files.genenetwork.org/software/contrib/htmlgen-2.2.2-gn.tar.gz
               (uri (string-append
-                     "http://ipfs.genenetwork.org/ipfs/QmZLWsPHLFTU3hWAfdTwj3RXFrS8Ma7KEixne1suWuYqeG/htmlgen-" version "-gn.tar.gz"))
+                     "https://files.genenetwork.org/software/htmlgen-2.2.2-gn.tar.gz"))
               (sha256
                (base32
                 "1lwsk56rymhrma46cbyh3g64ksmq1vsih3qkrc2vh0lpba825y7r"))
@@ -360,13 +358,6 @@ Python 3 support.")
     (description #f)
     (license #f)))
 
-; penguin2:~/tmp$ env IPFS_PATH=/export/ipfs/ ipfs add -r Imaging/
-; added QmV8Rew1re8gBTLsaqMU4bd7euFUPEpjiD572mtoz6KhPn Imaging/Imaging-1.1.6-gn.tar.gz
-; added QmdkzQpVMLZVtywpYesynt9c7H8w7hHZRYKq8woN7stfpD Imaging
-; env IPFS_PATH=/export/ipfs/ ipfs pin add -r QmdkzQpVMLZVtywpYesynt9c7H8w7hHZRYKq8woN7stfpD
-; pinned QmdkzQpVMLZVtywpYesynt9c7H8w7hHZRYKq8woN7stfpD recursively
-
-
 (define-public python2-pil1-gn ; guix obsolete
   (package
     (name "python2-pil1") ; works with GN2
@@ -427,11 +418,6 @@ capabilities to the Python interpreter.")
                "file://README"
                "See 'README' in the distribution."))))
 
-;  agrigento:~/izip/git/opensource/genenetwork$ scp ./contrib/piddle-1.0.15-gn.tgz penguin2.genenetwork.org
-; penguin2:~$ env IPFS_PATH=/export/ipfs/ ipfs add piddle-1.0.15-gn.tgz
-; added QmSMptV2VALL2s7igqRqKJ8ALNvhqFRUYVG54kEF7ac6ve piddle-1.0.15-gn.tgz
-; penguin2:~$ env IPFS_PATH=/export/ipfs/ ipfs pin add -r QmSMptV2VALL2s7igqRqKJ8ALNvhqFRUYVG54kEF7ac6ve
-; pinned QmSMptV2VALL2s7igqRqKJ8ALNvhqFRUYVG54kEF7ac6ve recursively
 
 (define-public python2-piddle-gn ; guix obsolete
   (package
@@ -440,7 +426,7 @@ capabilities to the Python interpreter.")
     (source (origin
      (method url-fetch)
      (uri (string-append
-           "http://ipfs.genenetwork.org/ipfs/QmeKcMb8AdwZNUcAaTASVpZ39ipwJn8eBoqqDfoCzQYmNk/piddle-" version ".tgz"))
+           "https://files.genenetwork.org/software/piddle-1.0.15-gn.tgz"))
      (sha256
       (base32
        "05gjnn31v7p0kh58qixrpcizcxqf3b7zv4a5kk8nsmqwgxh0c6gq"))))
@@ -707,8 +693,7 @@ clusters (computers connected via network).")
 (define GN1-thirdparty-sources
   (origin
     (method url-fetch/tarbomb)
-    ;; ipfs get QmTPwYT2pehdxdG1TiHEzVzLgbeuhJ4utXShuz3twA84AB
-    (uri "file:///gnu/store/p33a2sh3x2nhiiphdw9nly80njg6p8fi-thirdparty.tgz")
+    (uri "https://files.genenetwork.org/software/thirdparty.tgz")
     (file-name "GN1-thirdparty")
     (sha256
      (base32
@@ -851,37 +836,68 @@ spreadsheets without the need for COM objects.")
          (base32
           "1lg1klrczvzfan89y3bl9ykrknl3nb01vvai37fkww24apzyibjf"))))))
 
+(define-public python-py-dateutil
+  (package
+    (name "python-py-dateutil")
+    (version "2.2")
+    (source (origin
+              (method url-fetch)
+              (uri (pypi-uri "py-dateutil" version))
+              (sha256
+               (base32
+                "0j5hyhn2yqwyapbhvdvw14a0ydhdl6ddw95nii091iarf6hjryky"))))
+    (build-system python-build-system)
+    (arguments
+     (list
+       #:tests? #f          ; Package is unmaintained since ~2014
+       #:phases
+       #~(modify-phases %standard-phases
+           (replace 'check
+             (lambda* (#:key tests? #:allow-other-keys)
+               (when tests?
+                 (invoke "python" "test.py")))))))
+    (propagated-inputs (list python-six))
+    (home-page "https://bitbucket.org/cld/dateutil")
+    (synopsis "Extensions to the standard Python datetime module")
+    (description "Extensions to the standard Python datetime module")
+    (license license:bsd-3)))
+
 (define-public python-arvados-python-client
   (package
     (name "python-arvados-python-client")
-    (version "2.0.2")
+    (version "2.5.0")
     (source
       (origin
         (method url-fetch)
         (uri (pypi-uri "arvados-python-client" version))
         (sha256
-         (base32
-          "19l4w6m5426x5k2kick630dh2jx26j16ycs2nhbfgr4cd43d29y4"))))
+         (base32 "1j08aykj0v2z2bqwr5nfnbjgc1yzdnfdafcnxbf2jbwqh8kx7zc9"))
+        (modules '((guix build utils)))
+        (snippet
+         '(begin
+            (substitute* "setup.py"
+              ;; Don't set a maximum version of pycurl.
+              (("(pycurl >=([[:digit:]]+\\.?)+),.*" _ pycurl)
+               (string-append pycurl "',\n")))))))
     (build-system python-build-system)
     (arguments
      `(#:tests? #f))    ; tests not included?
     (propagated-inputs
-     `(("python-ciso8601" ,python-ciso8601)
-       ("python-future" ,python-future)
-       ;("python-google-api-python-client" ,python-google-api-python-client)
-       ("python-google-api-client" ,python-google-api-client)
-       ("python-httplib2" ,python-httplib2)
-       ("python-pycurl" ,python-pycurl)
-       ("python-ruaml.yaml" ,python38-ruaml.yaml-0.15.76)
-       ("python-setuptools" ,python-setuptools)
-       ("python-oauth2client" ,python-oauth2client)
-       ("python-uritemplate" ,python-uritemplate)
-       ("python-ws4py" ,python-ws4py)))
+     (list python-ciso8601
+           python-future
+           python-google-api-client
+           python-google-api-core-1
+           python-google-auth-1
+           python-httplib2
+           python-protobuf
+           python-pycurl
+           python-pyparsing-2.4.7       ; < 3
+           python-ruamel.yaml
+           python-ws4py))
     (native-inputs
-     `(("python-mock" ,python-mock)
-       ("python-pbr" ,python-pbr-1.6.0)
-       ("python-pyyaml" ,python-pyyaml)
-       ))
+     (list python-mock
+           python-pbr-1.6.0
+           python-pyyaml))
     (home-page "https://arvados.org")
     (synopsis "Arvados client library")
     (description "This package provides the arvados module, an API client for
@@ -899,46 +915,35 @@ server.")
         (method url-fetch)
         (uri (pypi-uri "PyShEx" version))
         (sha256
-         (base32
-          "1fy664bh6hpmr4cf49fwwxng36kv7s6b2986hbv0cqcypc4ri2cs"))))
+         (base32 "1fy664bh6hpmr4cf49fwwxng36kv7s6b2986hbv0cqcypc4ri2cs"))))
     (arguments
-     '(#:phases
+     '(#:tests? #f          ; Tests try to use the internet.
+       #:phases
        (modify-phases %standard-phases
-         (add-after 'unpack 'patch-source
+         (add-after 'unpack 'drop-rdflib-namespace-rdfnamespace
            (lambda _
-             (substitute* "requirements.txt"
-               ((">=.*") "\n"))
-             #t))
+             ;; _RDFNamespace is in rdflib-5
+             ;; https://github.com/hsolbrig/PyShEx/commit/d138a5b4f2249212f2141b7b9bbaff6696ee9415
+             (substitute* "pyshex/prefixlib.py"
+               (("XMLNS, _RDFNamespace") "XMLNS"))))
          (replace 'check
            (lambda* (#:key inputs outputs tests? #:allow-other-keys)
-             (if tests?
-               (begin
-                 (delete-file "tests/test_cli/test_evaluate.py")
-                 (delete-file "tests/test_cli/test_sparql_options.py")
-                 (delete-file "tests/test_issues/test_fhir.py")
-                 (delete-file "tests/test_issues/test_issue_30.py")
-                 (delete-file "tests/test_pyshex_utils/test_schema_loader.py")
-                 (delete-file "tests/test_shex_manifest/test_basics.py")
-                 (delete-file "tests/test_shextest_validation/test_manifest_shex_json.py")
-                 (delete-file "tests/test_shextest_validation/test_manifest_shex_shexc.py")
-                 (delete-file "tests/test_support_libraries/test_prefixlib.py")
-                 (delete-file "tests/test_utils/test_manifest.py")
-                 (delete-file "tests/test_utils/test_sparql_query.py")
-                 (delete-file "tests/test_utils/test_n3_mapper.py")
-                 (invoke "python" "-m" "unittest"))
-               #t))))))
+             (when tests?
+               (add-installed-pythonpath inputs outputs)
+               (setenv "SKIP_EXTERNAL_URLS" "true")
+               (invoke "python" "-m" "unittest")))))))
     (build-system python-build-system)
     (propagated-inputs
-     `(("python-cfgraph" ,python-cfgraph)
-       ("python-pyshexc" ,python-pyshexc)
-       ("python-rdflib" ,python-rdflib)
-       ("python-pbr" ,python-pbr)
-       ("python-rdflib-jsonld" ,python-rdflib-jsonld)
-       ("python-requests" ,python-requests)
-       ("python-shexjsg" ,python-shexjsg)
-       ("python-sparql-slurper" ,python-sparql-slurper)
-       ("python-sparqlwrapper" ,python-sparqlwrapper)
-       ("python-urllib3" ,python-urllib3)))
+     (list python-cfgraph
+           python-pyshexc-0.7
+           python-rdflib
+           python-requests
+           python-shexjsg
+           python-sparql-slurper
+           python-sparqlwrapper
+           python-urllib3))
+    (native-inputs
+     (list python-pbr python-unittest2))
     (home-page "https://github.com/hsolbrig/PyShEx")
     (synopsis "Python ShEx Implementation")
     (description "This package provides a python ShEx Implementation.")
@@ -947,44 +952,97 @@ server.")
 (define-public python-pyshexc
   (package
     (name "python-pyshexc")
-    (version "0.5.4")
+    (version "0.9.1")
     (source
       (origin
         (method url-fetch)
         (uri (pypi-uri "PyShExC" version))
         (sha256
-         (base32
-          "0hyhmc971gh25ja34j9hbkr7dg9n3jfin8668cqzjmcpjvb1jnil"))))
+         (base32 "1lq4lf0nal1v1d3vbyrr1hfkhmhphy06dyqhyw7b5zls9dfrga9m"))))
     (build-system python-build-system)
     (arguments
-     '(#:tests? #f  ; Tests aren't included in release tarball.
+     '(#:tests? #f  ; It isn't clear how the tests expect to succeed.
        #:phases
        (modify-phases %standard-phases
          (replace 'check
            (lambda* (#:key inputs outputs tests? #:allow-other-keys)
-             (if tests?
-               (begin (add-installed-pythonpath inputs outputs)
-                      (substitute* "tests/test_basic_parser.py"
-                        (("BasicParserTestCase.repo_url.*")
-                         (string-append "BasicParserTestCase.repo_url = \""
-                                        (assoc-ref inputs "test-suite")
-                                        "/schemas\"\n")))
-                      (with-directory-excursion "tests"
-                        (invoke "python" "build_test_harness.py")
-                        (invoke "python" "test_basic_parser.py")
-                        (invoke "python" "test_issue_2.py")
-                        (invoke "python" "test_shexr.py")))
-               #t))))))
+             (when tests?
+               (add-installed-pythonpath inputs outputs)
+               (invoke "python" "-m" "unittest" "discover" "-s" "tests")))))))
     (propagated-inputs
-     `(("python-antlr4-python3-runtime" ,python-antlr4-python3-runtime)
-       ("python-jsonasobj" ,python-jsonasobj)
-       ("python-pyjsg" ,python-pyjsg)
-       ("python-rdflib" ,python-rdflib)
-       ("python-rdflib-jsonld" ,python-rdflib-jsonld)
-       ("python-requests" ,python-requests)))
+     (list python-antlr4-python3-runtime
+           python-chardet
+           python-jsonasobj
+           python-pyjsg
+           python-rdflib-shim
+           python-shexjsg))
+    (native-inputs
+     (list python-pbr
+           python-requests
+           python-yadict-compare))
+    (home-page "https://github.com/shexSpec/grammar-python-antlr")
+    (synopsis "Python ShExC Parser")
+    (description "This package converts the @dfn{Shape Expression Compact}
+(ShExC) into @dfn{Python JSON Schema Binding} (pyjsg) objects.")
+    (license license:asl2.0)))
+
+(define-public python-pyshexc-0.7
+  (package
+    (inherit python-pyshexc)
+    (name "python-pyshexc")
+    (version "0.7.0")
+    (source
+      (origin
+        (method git-fetch)
+        (uri (git-reference
+               (url "https://github.com/shexSpec/grammar")
+               (commit (string-append "v" version))))
+        (file-name (git-file-name "shexspec-grammar" version))
+        (sha256
+         (base32 "08wsknjq8zyi3hk01w6yl6z8jld63l6l64n43gjhj0jphbm4s9iz"))
+        (modules '((guix build utils)))
+        (snippet
+         '(begin
+            (substitute* "parsers/python/setup.cfg"
+              (("python-requires = 3") "python_requires = >=3.6"))))))
+    (arguments
+     `(#:tests? #f  ; Tests try to access the internet.
+       #:phases
+       (modify-phases %standard-phases
+         (add-after 'unpack 'chdir
+           (lambda _
+             (chdir "parsers/python")
+             (setenv "PBR_VERSION" ,version)))
+         (replace 'check
+           (lambda* (#:key inputs outputs tests? #:allow-other-keys)
+             (when tests?
+               (add-installed-pythonpath inputs outputs)
+               (substitute* "tests/__init__.py"
+                 (("schemas_base =.*")
+                  (string-append "schemas_base = \""
+                                 (assoc-ref inputs "test-suite")
+                                 "/schemas\"\n")))
+               (substitute* "tests/test_shexr.py"
+                 (("https://raw.githubusercontent.com/shexSpec/shexTest/master/validation/manifest")
+                  (string-append (assoc-ref inputs "test-suite")
+                                 "/validation/manifest")))
+               (substitute* "tests/test_issue_9.py"
+                 (("https://raw.githubusercontent.com/shexSpec/shexTest/master/schemas/1dotNS2.shex")
+                  (search-input-file inputs "/schemas/1dotNS2.shex")))
+               (symlink (assoc-ref inputs "test-suite")
+                        "../../../shexTest")
+               (invoke "python" "-m" "unittest" "discover" "-s" "tests")))))))
+    (propagated-inputs
+     (list python-antlr4-python3-runtime
+           python-certifi
+           python-jsonasobj
+           python-rdflib-shim
+           python-requests
+           python-shexjsg))
     (native-inputs
-     `(("python-yadict-compare" ,python-yadict-compare)
-       ("python-shexjsg" ,python-shexjsg-min)
+     `(("python-pbr" ,python-pbr)
+       ("python-unittest2" ,python-unittest2)
+       ("python-yadict-compare" ,python-yadict-compare)
        ("test-suite"
         ,(origin
            (method git-fetch)
@@ -995,23 +1053,18 @@ server.")
            (sha256
             (base32
              "1x788nyrwycfr55wbg0ay6mc8mi6wwsg81h614rx9pw6rvrsppps"))))))
-    (home-page "https://github.com/shexSpec/grammar/tree/master/parsers/python")
-    (synopsis "Python ShExC Parser")
-    (description "This package converts the @dfn{Shape Expression Compact}
-(ShExC) into @dfn{Python JSON Schema Binding} (pyjsg) objects.")
-    (license license:asl2.0)))
+    (home-page "https://github.com/shexSpec/grammar/tree/master/parsers/python")))
 
 (define-public python-antlr4-python3-runtime
   (package
     (name "python-antlr4-python3-runtime")
-    (version "4.7.1")
+    (version "4.9.3")
     (source
       (origin
         (method url-fetch)
         (uri (pypi-uri "antlr4-python3-runtime" version))
         (sha256
-         (base32
-          "1lrzmagawmavyw1n1z0qarvs2jmbnbv0p89dah8g7klj8hnbf9hv"))))
+         (base32 "06w8fz73rk8vzjz9rydfk56g4mbqpyl81yhypc14jab886dlc97j"))))
     (build-system python-build-system)
     (home-page "https://www.antlr.org/")
     (synopsis "ANTLR runtime for Python")
@@ -1051,84 +1104,39 @@ treats name/value pairs as first class attributes whenever possible.")
 (define-public python-shexjsg
   (package
     (name "python-shexjsg")
-    (version "0.6.5")
+    (version "0.8.2")
     (source
       (origin
         (method url-fetch)
         (uri (pypi-uri "ShExJSG" version))
         (sha256
-         (base32
-          "1nn69sl5j949qy21nl5gr56cxfhmml1vng08hayxqfj6vn3ay3gg"))))
+         (base32 "1dnhpk6n6vzadkv13y7r6y2mi1pzv4y19vmxh91k9ykpqngn4ypi"))))
     (build-system python-build-system)
     (arguments
      '(#:phases
        (modify-phases %standard-phases
-         (add-after 'unpack 'patch-source
-           (lambda _
-             (substitute* '("requirements.txt"
-                            "requirements-dev.txt")
-              (("pyshexc.*") "") ; no loops
-               (("==.*") "\n"))
-             #t))
          (replace 'check
            (lambda* (#:key inputs outputs tests? #:allow-other-keys)
              (if tests?
                (begin (add-installed-pythonpath inputs outputs)
-                      (substitute* '("tests/test_shexc.py"
-                                     "tests/test_shexj.py")
-                        (("shexTestRepository =.*")
-                         (string-append "shexTestRepository = \""
-                                        (assoc-ref inputs "test-suite")
-                                        "/schemas\"\n")))
-                      (invoke "python" "-m" "unittest"))
-               #t))))))
+                      ;; Tries to download files from the internet.
+                      (substitute* "tests/test_shexj.py"
+                        (("skipIf\\(False") "skipIf(True"))
+                      (invoke "python" "-m" "unittest"))))))))
     (propagated-inputs
-     `(("python-antlr4-python3-runtime" ,python-antlr4-python3-runtime)
-       ("python-certifi" ,python-certifi)
-       ("python-chardet" ,python-chardet)
-       ("python-idna" ,python-idna)
-       ("python-isodate" ,python-isodate)
-       ("python-pyjsg" ,python-pyjsg)
-       ("python-requests" ,python-requests)
-       ("python-urllib3" ,python-urllib3)))
+     (list python-pyjsg))
     (native-inputs
-     `(("python-jsonasobj" ,python-jsonasobj)
-       ("python-pbr" ,python-pbr)
-       ("python-pyparsing" ,python-pyparsing)
-       ("python-pyshexc" ,python-pyshexc)
-       ("python-rdflib" ,python-rdflib)
-       ("python-rdflib-jsonld" ,python-rdflib-jsonld)
-       ("python-six" ,python-six)
-       ("python-yadict-compare" ,python-yadict-compare)
-       ("test-suite"
-        ,(origin
-           (method git-fetch)
-           (uri (git-reference
-                  (url "https://github.com/shexSpec/shexTest")
-                  (commit "v2.0.2")))
-           (file-name (git-file-name name version))
-           (sha256
-            (base32
-             "1x788nyrwycfr55wbg0ay6mc8mi6wwsg81h614rx9pw6rvrsppps"))))))
+     (list python-jsonasobj
+           python-pbr
+           python-rdflib-shim
+           python-requests
+           python-yadict-compare))
     (home-page "https://github.com/hsolbrig/ShExJSG")
     (synopsis "Astract Syntax Tree for the ShEx 2.0 language")
     (description "This package provides an astract syntax tree for the
 ShEx 2.0 language.")
     (license license:cc0)))
 
-;; Lets use this one for tests.
-(define python-shexjsg-min
-  (package
-    (inherit python-shexjsg)
-    (name "python-shexjsg")
-    (arguments
-     (substitute-keyword-arguments (package-arguments python-shexjsg)
-       ((#:tests? _ #t) #f)))
-    (native-inputs
-     `(,@(alist-delete "python-pyshexc"
-                       (package-native-inputs python-shexjsg))))
-    (properties `((hidden? . #t)))))
-
 (define-public python-yadict-compare
   (package
     (name "python-yadict-compare")
@@ -1160,31 +1168,17 @@ handles recursion and lists.")
 (define-public python-pyjsg
   (package
     (name "python-pyjsg")
-    (version "0.9.2")
+    (version "0.11.10")
     (source
       (origin
-        (method git-fetch)
-        (uri (git-reference
-               ;; Releases aren't tagged in the repository.
-               (url "https://github.com/hsolbrig/pyjsg")
-               (commit "9b2b8fa8e3b8448abe70b09f804a79f0f31b32b7")))
-        (file-name (git-file-name name version))
+        (method url-fetch)
+        (uri (pypi-uri "PyJSG" version))
         (sha256
-         (base32
-          "0fhpvb6i6xhyd6hnwknw0y2k33cb7iwj07g009lw96r580vprxs4"))))
+         (base32 "1ylbx2pc06qsvb8cqnr8nysvmw55f8nkm05ybcwjpyik53zy7mjb"))))
     (build-system python-build-system)
     (arguments
      '(#:phases
        (modify-phases %standard-phases
-         (add-after 'unpack 'patch-source
-           (lambda* (#:key inputs #:allow-other-keys)
-             (for-each (lambda (file)
-                         (make-file-writable file))
-                       (find-files "." "."))
-             (substitute* "tests_standalone_2/test_xsfacet.py"
-               (("\\.\\.', '\\.\\.', '\\.\\.', 'shexSpec', 'shexTest")
-                (assoc-ref inputs "test-suite")))
-             #t))
          ;; From tox.ini
          (replace 'check
            (lambda* (#:key tests? #:allow-other-keys)
@@ -1193,30 +1187,20 @@ handles recursion and lists.")
                  (lambda (dir)
                    (invoke "python" "-m" "unittest" "discover" "-s" dir))
                  '("tests/test_issues"
-                   "tests/test_basics"
+                   ;"tests/test_basics"
                    "tests/test_jsglib"
                    "tests/test_parser_impl"
                    "tests/test_python_generator"
                    "tests_standalone"
-                   "tests_standalone_2"))
-               #t))))))
+                   "tests_standalone_2"))))))))
     (propagated-inputs
-     `(("python-antlr4-python3-runtime" ,python-antlr4-python3-runtime)
-       ("python-jsonasobj" ,python-jsonasobj)
-       ("python-requests" ,python-requests)))
+     (list python-antlr4-python3-runtime
+           python-jsonasobj))
     (native-inputs
-     `(("python-unittest2" ,python-unittest2)
-       ("python-yadict-compare" ,python-yadict-compare)
-       ("test-suite"
-        ,(origin
-           (method git-fetch)
-           (uri (git-reference
-                  (url "https://github.com/shexSpec/shexTest")
-                  (commit "v2.0.2")))
-           (file-name (git-file-name name version))
-           (sha256
-            (base32
-             "1x788nyrwycfr55wbg0ay6mc8mi6wwsg81h614rx9pw6rvrsppps"))))))
+     (list python-pbr
+           python-requests
+           python-unittest2
+           python-yadict-compare))
     (home-page "https://github.com/hsolbrig/pyjsg")
     (synopsis "Python JSON Schema Grammar bindings")
     (description
@@ -1245,30 +1229,37 @@ handles recursion and lists.")
     (description "This package provides a SPARQL Slurper for rdflib.")
     (license license:asl2.0)))
 
-(define-public python38-ruaml.yaml-0.15.76 ;; no longer in use 
+(define-public python-sparqlslurper
   (package
-    (inherit python-ruamel.yaml)
-    (name "python-ruamel.yaml")
-    (version "0.15.76")
-    (source
-      (origin
-        (method url-fetch)
-        (uri (pypi-uri "ruamel.yaml" "0.15.78"))
-        (sha256
-         (base32
-          "0pwxgrma6k47kvsphqz5yrhwnfrhwsrhn6sjp8p21s91wdgkqyc5"))))
+    (name "python-sparqlslurper")
+    (version "0.5.1")
+    (source (origin
+              (method url-fetch)
+              (uri (pypi-uri "sparqlslurper" version))
+              (sha256
+               (base32 "0ipma74dr5jvsxwaa9al147mn9vv3v5r9lb9hajm4qgwcjqfp0lj"))))
+    (build-system python-build-system)
     (arguments
-     `(#:tests? #f  ; suprise test failures
+     '(#:tests? #f  ; Tests try to use the internet.
        #:phases
        (modify-phases %standard-phases
-         ;; For some unknown reason we need: ruamel.yaml<=0.15.77,>=0.15.54
-         ;; But 0.15.78 is the first which builds with python-3.8.
-         (add-after 'unpack 'patch-source
-           (lambda _
-             (substitute* "__init__.py"
-               (("0\\.15\\.78") "0.15.76")
-               (("15, 78") "15, 76"))
-             #t)))))))
+         (replace 'check
+           (lambda* (#:key inputs outputs tests? #:allow-other-keys)
+             (when tests?
+               (add-installed-pythonpath inputs outputs)
+               (invoke "python" "-m" "unittest")))))))
+    (propagated-inputs
+     (list python-rdflib-shim
+           python-sparqlwrapper))
+    (native-inputs
+     (list python-pbr python-unittest2))
+    (home-page "http://github.com/hsolbrig/sparqlslurper")
+    (synopsis "SPARQL Slurper for rdflib")
+    (description "This package provides an implementation of a
+@code{rdflibGraph} that acts as a cache for a SPARQL endpoint.  SPARQL Slurper
+translates the @code{Graph.triples()} function into the corresponding SPARQL
+query and resolves it against an endpoint.")
+    (license license:cc0)))
 
 (define-public python2-ruamel.ordereddict
   (package
@@ -1299,6 +1290,22 @@ handles recursion and lists.")
 sorted order.")
     (license license:expat)))
 
+(define-public python-pytest-5
+  (package
+    (inherit python-pytest)
+    (name "python-pytest")
+    (version "5.4.3")
+    (source (origin
+              (method url-fetch)
+              (uri (pypi-uri "pytest" version))
+              (sha256
+               (base32
+                "1n67lk8iwlsmfdm8663k8l7isllg1xd3n9p1yla7885szhdk6ybr"))))
+    (build-system python-build-system)
+    (native-inputs
+     (modify-inputs (package-native-inputs python-pytest)
+                    (prepend python-argcomplete python-requests)))))
+
 (define-public python-pytest-4
   (package
     (inherit python-pytest)
@@ -1315,9 +1322,33 @@ sorted order.")
        ((#:tests? _ #f) #f)))
     (native-inputs
      `(("python-argcomplete" ,python-argcomplete)
+       ("python-atomicwrites" ,python-atomicwrites)
        ("python-requests" ,python-requests)
        ,@(package-native-inputs python-pytest)))))
 
+(define-public python-pytest-runner-4
+  (package
+    (inherit python-pytest-runner)
+    (name "python-pytest-runner")
+    (version "4.5.1")
+    (source (origin
+              (method url-fetch)
+              (uri (pypi-uri "pytest-runner" version))
+              (sha256
+               (base32
+                "1vzilbayx5mznsdm1r258m3616374p6kvhsbj4j6238j9djkvjyi"))))))
+
+(define-public python-pytest-runner-2
+  (package
+    (inherit python-pytest-runner)
+    (version "2.12.2")
+    (source (origin
+              (method url-fetch)
+              (uri (pypi-uri "pytest-runner" version))
+              (sha256
+               (base32
+                "11ivjj9hfphkv4yfb2g74av4yy86y8gcbf7gbif0p1hcdfnxg3w6"))))))
+
 (define-public python-pandas-plink
   (package
     (name "python-pandas-plink")
@@ -1784,3 +1815,66 @@ file format spec.")
       (license:fsf-free (string-append "https://web.archive.org/web/20190211105114/"
                                        "http://www.repoze.org/LICENSE.txt")))))
 
+(define-public python-rdflib-shim
+  (package
+    (name "python-rdflib-shim")
+    (version "1.0.3")
+    (source (origin
+              (method url-fetch)
+              (uri (pypi-uri "rdflib_shim" version))
+              (sha256
+               (base32
+                "03gwsjcinbyyqrhs2jfhs6mr7j69dfn5djihd0mv9al654gd2mfr"))
+              (modules '((guix build utils)))
+              (snippet
+               '(begin
+                  (substitute* "requirements.txt"
+                    (("rdflib-jsonld==0.6.1") "rdflib-jsonld"))))))
+    (build-system python-build-system)
+    (arguments
+     '(#:phases
+       (modify-phases %standard-phases
+         (replace 'check
+           (lambda* (#:key inputs outputs tests? #:allow-other-keys)
+             (when tests?
+               (add-installed-pythonpath inputs outputs)
+               (invoke "python" "-m" "unittest" "discover" "-s" "tests")))))))
+    (propagated-inputs
+     (list python-rdflib python-rdflib-jsonld))
+    (native-inputs
+     (list python-pbr))
+    (home-page "http://hsolbrig.github.io/rdflib-shim")
+    (synopsis "Shim for rdflib 5 and 6 incompatibilities")
+    (description "Shim for rdflib 5 and 6 incompatibilities")
+    (license license:cc0)))
+
+(define-public python-h5py-2
+  (package
+    (name "python-h5py")
+    (version "2.10.0")
+    (source (origin
+              (method url-fetch)
+              (uri (pypi-uri "h5py" version))
+              (sha256
+               (base32
+                "0baipzv8n93m0dq0riyi8rfhzrjrfrfh8zqhszzp1j2xjac2fhc4"))))
+    (build-system python-build-system)
+    (arguments
+     `(#:tests? #f ;no test target
+       #:phases (modify-phases %standard-phases
+                  (add-after 'unpack 'fix-hdf5-paths
+                    (lambda* (#:key inputs #:allow-other-keys)
+                      (setenv "HDF5_DIR"
+                              (assoc-ref inputs "hdf5")))))))
+    (propagated-inputs (list python-six python-numpy))
+    (inputs (list hdf5-1.10))
+    (native-inputs (list python-cython python-pkgconfig pkg-config))
+    (home-page "https://www.h5py.org/")
+    (synopsis "Read and write HDF5 files from Python")
+    (description
+     "The h5py package provides both a high- and low-level interface to the
+HDF5 library from Python.  The low-level interface is intended to be a
+complete wrapping of the HDF5 API, while the high-level component supports
+access to HDF5 files, datasets and groups using established Python and NumPy
+concepts.")
+    (license license:bsd-3)))
diff --git a/gn/packages/python24.scm b/gn/packages/python24.scm
index 35cd91e..381b1ef 100644
--- a/gn/packages/python24.scm
+++ b/gn/packages/python24.scm
@@ -140,7 +140,15 @@ pre-defined variants."
     (name "python24-pil")
     (arguments
      (substitute-keyword-arguments (package-arguments python2-pil1)
-       ((#:python _) python-2.4)))))
+       ((#:python _) python-2.4)
+       ((#:phases phases '%standard-phases)
+        `(modify-phases ,phases
+           (replace 'add-install-to-pythonpath
+             (lambda* (#:key inputs outputs #:allow-other-keys)
+               (setenv "PYTHONPATH"
+                       (string-append (site-packages inputs outputs) ":"
+                                      (getenv "PYTHONPATH")))))
+           (delete 'sanity-check))))))) ; Not applicable to python-2.4
 
 (define-public python24-piddle
   (package
@@ -148,7 +156,15 @@ pre-defined variants."
     (name "python24-piddle")
     (arguments
      (substitute-keyword-arguments (package-arguments python2-piddle-gn)
-       ((#:python _) python-2.4)))
+       ((#:python _) python-2.4)
+       ((#:phases phases '%standard-phases)
+        `(modify-phases ,phases
+           (replace 'add-install-to-pythonpath
+             (lambda* (#:key inputs outputs #:allow-other-keys)
+               (setenv "PYTHONPATH"
+                       (string-append (site-packages inputs outputs) ":"
+                                      (getenv "PYTHONPATH")))))
+           (delete 'sanity-check)))))   ; Not applicable to python-2.4
     (native-inputs `(("python24-setuptools" ,python24-setuptools)))
     (propagated-inputs
      `(("python24-pil" ,python24-pil)))))
@@ -182,7 +198,7 @@ pre-defined variants."
 (define GN1-thirdparty-sources
   (origin
     (method url-fetch/tarbomb)
-    (uri "http://ipfs.genenetwork.org/ipfs/QmTPwYT2pehdxdG1TiHEzVzLgbeuhJ4utXShuz3twA84AB/thirdparty.tgz")
+    (uri "https://files.genenetwork.org/software/thirdparty.tgz")
     (file-name "GN1-thirdparty")
     (sha256
      (base32
@@ -200,13 +216,18 @@ pre-defined variants."
        (modify-phases %standard-phases
          (delete 'build)
          (delete 'check)
+         (replace 'add-install-to-pythonpath
+           (lambda* (#:key inputs outputs #:allow-other-keys)
+             (setenv "PYTHONPATH"
+                     (string-append (site-packages inputs outputs) ":"
+                                    (getenv "PYTHONPATH")))))
+         (delete 'sanity-check)     ; Not applicable to python-2.4
          (replace 'install
            (lambda* (#:key outputs #:allow-other-keys)
              (let* ((out     (assoc-ref outputs "out"))
                     (sitedir (string-append out "/lib/python2.4/site-packages/json/")))
                (mkdir-p sitedir)
-               (copy-recursively "thirdparty/json" sitedir)
-               #t))))))
+               (copy-recursively "thirdparty/json" sitedir)))))))
     (home-page "")
     (synopsis "")
     (description "")
@@ -224,13 +245,18 @@ pre-defined variants."
        (modify-phases %standard-phases
          (delete 'build)
          (delete 'check)
+         (replace 'add-install-to-pythonpath
+           (lambda* (#:key inputs outputs #:allow-other-keys)
+             (setenv "PYTHONPATH"
+                     (string-append (site-packages inputs outputs) ":"
+                                    (getenv "PYTHONPATH")))))
+         (delete 'sanity-check)     ; Not applicable to python-2.4
          (replace 'install
            (lambda* (#:key outputs #:allow-other-keys)
              (let* ((out     (assoc-ref outputs "out"))
                     (sitedir (string-append out "/lib/python2.4/site-packages/svg/")))
                (mkdir-p sitedir)
-               (copy-recursively "thirdparty/svg" sitedir)
-               #t))))))
+               (copy-recursively "thirdparty/svg" sitedir)))))))
     (home-page "")
     (synopsis "")
     (description "")
@@ -248,6 +274,12 @@ pre-defined variants."
        (modify-phases %standard-phases
          (delete 'build)
          (delete 'check)
+         (replace 'add-install-to-pythonpath
+           (lambda* (#:key inputs outputs #:allow-other-keys)
+             (setenv "PYTHONPATH"
+                     (string-append (site-packages inputs outputs) ":"
+                                    (getenv "PYTHONPATH")))))
+         (delete 'sanity-check)     ; Not applicable to python-2.4
          (replace 'install
            (lambda* (#:key outputs #:allow-other-keys)
              (let* ((out     (assoc-ref outputs "out"))
@@ -322,7 +354,13 @@ pre-defined variants."
        (modify-phases %standard-phases
          (add-after 'unpack 'change-directory
            (lambda _
-             (chdir "thirdparty/pp-1.5.7") #t)))))
+             (chdir "thirdparty/pp-1.5.7")))
+         (replace 'add-install-to-pythonpath
+           (lambda* (#:key inputs outputs #:allow-other-keys)
+             (setenv "PYTHONPATH"
+                     (string-append (site-packages inputs outputs) ":"
+                                    (getenv "PYTHONPATH")))))
+         (delete 'sanity-check))))  ; Not applicable to python-2.4
     (home-page "")
     (synopsis "")
     (description "")
@@ -346,9 +384,15 @@ pre-defined variants."
      `(#:python ,python-2.4
        #:phases
        (modify-phases %standard-phases
+         (replace 'add-install-to-pythonpath
+           (lambda* (#:key inputs outputs #:allow-other-keys)
+             (setenv "PYTHONPATH"
+                     (string-append (site-packages inputs outputs) ":"
+                                    (getenv "PYTHONPATH")))))
+         (delete 'sanity-check)     ; Not applicable to python-2.4
          (add-before 'check 'pre-check
            (lambda* (#:key inputs tests? #:allow-other-keys)
-             (if tests?
+             (when tests?
                (begin
                  (mkdir-p "/tmp/mysqld")
                  (call-with-output-file "/tmp/my.cnf"
@@ -368,8 +412,7 @@ pre-defined variants."
                  (sleep 5)
                  (invoke "mysqladmin" "-S" "/tmp/mysqld/mysql.sock" "variables")
                  (invoke "mysql" "-S" "/tmp/mysqld/mysql.sock"
-                         "-e" "'create database mysqldb_test charset utf8;'"))
-               #t))))
+                         "-e" "'create database mysqldb_test charset utf8;'"))))))
        #:tests? #f))    ; TODO: Run the test suite
     (native-inputs
      `(("mysql" ,mysql-5.0) ; Best supported version according to the README.
@@ -461,18 +504,13 @@ algorithm can be found in Gablonsky's
 thesis}.")
     (license license:expat)))
 
-; env IPFS_PATH=/export/ipfs/ ipfs add direct.so
-; added QmYUZiuAP6DJeubu69JqvRWSsn53qCZCS3FkRWgTowtWkA direct.so
-; penguin2:~/tmp$ env IPFS_PATH=/export/ipfs/ ipfs pin add QmYUZiuAP6DJeubu69JqvRWSsn53qCZCS3FkRWgTowtWkA
-; pinned QmYUZiuAP6DJeubu69JqvRWSsn53qCZCS3FkRWgTowtWkA recursively
-
 (define-public python24-direct-gn
   (package
     (name "python24-direct-gn")
     (version "GN")
     (source (origin
               (method url-fetch)
-              (uri "http://ipfs.genenetwork.org/ipfs/QmYUZiuAP6DJeubu69JqvRWSsn53qCZCS3FkRWgTowtWkA")
+              (uri "https://files.genenetwork.org/software/direct.so")
               (file-name "direct.so")
               (sha256
                (base32
diff --git a/gn/packages/quality-control.scm b/gn/packages/quality-control.scm
index 7bc60ef..f0a9c2f 100644
--- a/gn/packages/quality-control.scm
+++ b/gn/packages/quality-control.scm
@@ -59,8 +59,7 @@
           (file-name (git-file-name name commit))))
       (build-system asdf-build-system/sbcl)
       (arguments
-        `(#:tests? #f ; Need to fix tests.
-          #:asd-files '("qc.asd")))
+        `(#:tests? #f)) ; Need to fix tests.
       (native-inputs
         (list sbcl-1am))
       (inputs
diff --git a/gn/packages/ratspub.scm b/gn/packages/ratspub.scm
index 5f6f7b5..726bc0b 100644
--- a/gn/packages/ratspub.scm
+++ b/gn/packages/ratspub.scm
@@ -17,6 +17,7 @@
   #:use-module (gnu packages python-xyz)
   #:use-module (gn packages javascript)
   #:use-module (gn packages machine-learning)
+  #:use-module (gn packages python)
   #:use-module (gn packages web))
 
 (define-public ratspub
@@ -148,6 +149,13 @@ Association Studies}} catalog are also included in the search.  These
 gene-keyword relationships are presented as an interactive graph and a table.")
     (license license:expat)))
 
+(define use-corrected-inputs
+  (package-input-rewriting/spec
+    ;; Tensorflow-native provides much improved speeds. python-h5py@2 provides
+    ;; compatibility with our version of tensorflow.
+    `(("tensorflow" . ,(const tensorflow-native))
+      ("python-h5py" . ,(const python-h5py-2)))))
+
 (define-public ratspub-with-tensorflow-native
   (package
     (inherit
@@ -262,7 +270,7 @@ if __name__ == '__main__':
 (define-public genecup
   (package
     (name "genecup")
-    (version "1.6")
+    (version "1.8")
     (source (origin
               (method git-fetch)
               (uri (git-reference
@@ -270,7 +278,7 @@ if __name__ == '__main__':
                      (commit (string-append "v" version))))
               (file-name (git-file-name name version))
               (sha256
-               (base32 "050yg4hvjs9rghw41q35xdd2c2rzh9sl4sfwkk3am150mgf2p0nh"))))
+               (base32 "0clnlnj6cl816yj3nalh1h6hsvnh3sjq2g84x1kmj1zkqkw184ri"))))
     (build-system python-build-system)
     (arguments
      `(#:tests? #f  ; no test suite
@@ -399,11 +407,11 @@ concepts and a list of keywords for each concept.")
 (define-public genecup-with-tensorflow-native
   (package
     (inherit
-      (tensowflow-native-instead-of-tensorflow genecup))
+      (use-corrected-inputs genecup))
     (name "genecup-with-tensorflow-native")))
 
 (define-public genecup-latest-with-tensorflow-native
   (package
     (inherit
-      (tensowflow-native-instead-of-tensorflow genecup-master))
+      (use-corrected-inputs genecup-master))
     (name "genecup-latest-with-tensorflow-native")))
diff --git a/gn/packages/ruby.scm b/gn/packages/ruby.scm
index ca0339f..e2c4164 100644
--- a/gn/packages/ruby.scm
+++ b/gn/packages/ruby.scm
@@ -127,7 +127,7 @@ and comma separated table files - useful for Excel sheets and SQL/RDF
 output")
    (home-page
     "http://github.com/pjotrp/bioruby-table")
-   (license expat)))
+   (license license:expat)))
 
 (define-public ruby-ntlm-http
   (package
@@ -148,7 +148,8 @@ output")
     (description
      "Ruby/NTLM HTTP provides NTLM authentication over http.")
     (home-page "http://www.mindflowsolutions.net")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-webrobots
   (package
@@ -171,7 +172,8 @@ output")
      "This library helps write robots.txt compliant web robots in Ruby.
 ")
     (home-page "https://github.com/knu/webrobots")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-mechanize ; guix maybe ready
   (package
@@ -213,7 +215,7 @@ submitted.  Mechanize also keeps track of the sites that you have visited as
 a history.")
   (home-page
    "http://docs.seattlerb.org/mechanize/")
-  (license expat)))
+  (license license:expat)))
 
 (define-public ruby-elasticsearch-transport
 (package
@@ -240,7 +242,7 @@ a history.")
 ")
   (home-page
     "https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch-transport")
-  (license #f)))
+  (license (license:non-copyleft "will fill in later"))))
 
 (define-public ruby-elasticsearch-api
 (package
@@ -266,7 +268,7 @@ a history.")
 ")
   (home-page
     "https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch-api")
-  (license #f)))
+  (license (license:non-copyleft "will fill in later"))))
 
 (define-public ruby-elasticsearch
   (package
@@ -295,7 +297,7 @@ a history.")
 ")
    (home-page
     "http://github.com/elasticsearch/elasticsearch-ruby")
-   (license #f)))
+   (license (license:non-copyleft "will fill in later"))))
 
 
 
@@ -1720,7 +1722,8 @@ and HTML without having to deal with character set issues.")
     acts as a drop-in replacement for Rails `auto_link`
 ")
     (home-page "https://github.com/vmg/rinku")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-sidekiq
   (package
@@ -1745,7 +1748,8 @@ and HTML without having to deal with character set issues.")
     (description
       "Simple, efficient background processing for Ruby.")
     (home-page "https://sidekiq.org")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-mini-scheduler
   (package
@@ -2007,7 +2011,8 @@ and HTML without having to deal with character set issues.")
     (synopsis "")
     (description "")
     (home-page "")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public generate-ninja   ; or 'gn'
   (let ((commit "dfcbc6fed0a8352696f92d67ccad54048ad182b3")
@@ -2476,7 +2481,8 @@ Use rake-compiler-dock to enter an interactive shell session or add a task to yo
     (description
       "The CSV library provides a complete interface to CSV files and data.  It offers tools to enable you to read and write to and from Strings or IO objects, as needed.")
     (home-page "https://github.com/ruby/csv")
-    (license (list #f #f))))
+    (license (list (license:non-copyleft "will fill in later")
+                   (license:non-copyleft "will fill in later")))))
 
 (define-public ruby-maxminddb
   (package
@@ -2690,7 +2696,8 @@ Use rake-compiler-dock to enter an interactive shell session or add a task to yo
     (description
       "HTTP Accept, Accept-Charset, Accept-Encoding, and Accept-Language for Ruby/Rack")
     (home-page "http://mjijackson.github.com/rack-accept")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-mustermann
  (package
@@ -2948,7 +2955,8 @@ Use rake-compiler-dock to enter an interactive shell session or add a task to yo
     (description
       "Run test/unit tests by line number.  Metal!")
     (home-page "https://github.com/qrush/m")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-allocation-stats
   (package
@@ -3027,7 +3035,8 @@ Use rake-compiler-dock to enter an interactive shell session or add a task to yo
     (description
       "FakeWeb is a helper for faking web requests in Ruby.  It works at a global level, without modifying code or writing extensive stubs.")
     (home-page "http://github.com/chrisk/fakeweb")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-simple-oauth
   (package
@@ -3364,7 +3373,8 @@ Psych also knows how to serialize and de-serialize most Ruby objects to and from
       "   Kwalify is a parser, schema validator, and data binding tool for YAML and JSON.
 ")
     (home-page "http://www.kuwata-lab.com/kwalify/")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-reek
   (package
@@ -4375,7 +4385,8 @@ non-blocking I/O methods for Ruby without raising exceptions on
 EAGAIN and EINPROGRESS.
 ")
     (home-page "https://yhbt.net/kgio/")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-unicorn
   (package
@@ -4405,7 +4416,8 @@ advantage of features in Unix/Unix-like kernels.  Slow clients should
 only be served by placing a reverse proxy capable of fully buffering
 both the the request and response in between unicorn and slow clients.")
     (home-page "https://yhbt.net/unicorn/")
-    (license (list #f #f))))
+    (license (list (license:non-copyleft "will fill in later")
+                   (license:non-copyleft "will fill in later")))))
 
 (define-public ruby-pry-rails
   (package
@@ -4463,7 +4475,8 @@ both the the request and response in between unicorn and slow clients.")
     (description
       "CSS flipper for right-to-left processing.  A Ruby port of https://github.com/ded/r2")
     (home-page "https://github.com/mzsanford/R2rb")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-openssl
   (package
@@ -5146,7 +5159,8 @@ public-key signature system described in RFC 8032.")
     (description
       "Annotates Rails/ActiveRecord Models, routes, fixtures, and others based on the database schema.")
     (home-page "https://github.com/ctran/annotate_models")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-flamegraph
   (package
@@ -5303,7 +5317,7 @@ specify.")
     (description
       "This package provides a small gem that attempts to guess and then force encoding of HTML documents for Ruby 1.9")
     (home-page "https://github.com/cantino/guess_html_encoding")
-   (license expat)))
+   (license license:expat)))
 
 (define-public ruby-ruby-readability
   (package
@@ -5332,7 +5346,8 @@ specify.")
     (description
       "Port of arc90's readability project to ruby")
     (home-page "https://github.com/cantino/ruby-readability")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 ;; TODO: 0.3.3 never finishes the install phase with ruby-2.6
 ;; TODO: Unbundle cppjieba
@@ -5614,7 +5629,8 @@ with processing Chinese text.")
 \"test/unit/rr\".
 ")
     (home-page "https://github.com/test-unit/test-unit-rr")
-    (license #f)))
+    (license (license:non-copyleft
+               "will fill in later"))))
 
 (define-public ruby-aws-sdk-kinesis
   (package
@@ -5955,3 +5971,25 @@ Simple gem that adds various color methods to String class, and can be used as f
   (home-page
     "http://github.com/pjotrp/regressiontest")
   (license license:expat)))
+
+(define-public ruby-lmdb
+  (package
+   (name "ruby-lmdb")
+   (version "0.6.1")
+   (source (origin
+            (method url-fetch)
+            (uri (rubygems-uri "lmdb" version))
+            (sha256
+             (base32
+              "0yzlf56ynj545ln5dxl422rpi8r6h11cr6rag5hk2qpbkzs6l9c2"))))
+   (build-system ruby-build-system)
+   (native-inputs
+    `(("ruby-rake" ,ruby-rake)
+       ("ruby-rake-compiler" ,ruby-rake-compiler)
+       ("ruby-rspec" ,ruby-rspec)      ("ruby-rspec" ,ruby-rspec)))
+   (arguments
+    `(#:tests? #f ; There are no tests.
+   ))   (synopsis "lmdb is a Ruby binding to OpenLDAP Lightning MDB.")
+   (description "lmdb is a Ruby binding to OpenLDAP Lightning MDB.")
+   (home-page "https://github.com/doriantaylor/rb-lmdb")
+   (license license:expat)))
diff --git a/gn/past/genenetwork1.scm b/gn/past/genenetwork1.scm
index c557ee7..717aff6 100644
--- a/gn/past/genenetwork1.scm
+++ b/gn/past/genenetwork1.scm
@@ -44,7 +44,6 @@
   #:use-module (past packages python)
   #:use-module (past packages web)
   #:use-module (gn packages bioinformatics)
-  #:use-module (gn packages twint)
   #:use-module (gn packages databases)
   #:use-module (gn packages elixir)
   #:use-module (gn packages gemma)
@@ -110,15 +109,22 @@ location of a putative QTL.")
        `(#:python ,python-2.4
          #:phases
          (modify-phases %standard-phases
+           (replace 'add-install-to-pythonpath
+             (lambda* (#:key inputs outputs #:allow-other-keys)
+               (setenv "PYTHONPATH"
+                       (string-append (site-packages inputs outputs) ":"
+                                      (getenv "PYTHONPATH")))))
+           (delete 'sanity-check)       ; Not applicable to python-2.4
            (add-after 'unpack 'make-max-markername-size-larger
              (lambda _
                (substitute* "Src/dataset.c"
-                 (("512") "2048"))
-               #t))
+                 (("512") "2048"))))
            (replace 'check
-             (lambda* (#:key inputs outputs #:allow-other-keys)
-               (add-installed-pythonpath inputs outputs)
-               (invoke "python" "test/runtest.py"))))))
+             (lambda* (#:key tests? inputs outputs #:allow-other-keys)
+               (when tests?
+                 (setenv "PYTHONPATH" (string-append (site-packages inputs outputs) ":"
+                                                     (getenv "PYTHONPATH")))
+                 (invoke "python" "test/runtest.py")))))))
       (native-inputs
        `(("python24-setuptools" ,python24-setuptools)))
       (home-page "http://qtlreaper.sourceforge.net/")
diff --git a/gn/services/bh20-seq-resource-container.scm b/gn/services/bh20-seq-resource-container.scm
index cb77cdb..e0eccf7 100644
--- a/gn/services/bh20-seq-resource-container.scm
+++ b/gn/services/bh20-seq-resource-container.scm
@@ -5,6 +5,7 @@
              (guix modules)
              ((guix packages) #:select (package-source))
              (guix records)
+             (guix build-system python) ; for python-version from guix/build/python-build-system.scm
              (ice-9 match))
 (use-service-modules shepherd)
 (use-package-modules compression python python-web)
@@ -14,7 +15,7 @@
   make-covid19-pubseq-configuration
   covid19-pubseq-configuration?
   (package          covid19-pubseq-configuration-package    ; package
-                    (default bh20-seq-resource))
+                    (default bh20-seq-resource-for-service))
   (deploy-directory covid19-pubseq-deploy-directory         ; string
                     (default "/srv/http"))
   (port             covid19-pubseq-configuration-port       ; string
@@ -58,9 +59,10 @@
                           #:directory #$deploy-directory
                           #:log-file "/var/log/covid19-pubseq.log"
                           #:environment-variables
+                          ;(let (pyversion (python-version (@ (gnu packages python) python)))
                           '("TMPDIR=/export/tmp"
-                            ;; TODO: Don't hardcode python version!
-                            "PYTHONPATH=/run/current-system/profile/lib/python3.8/site-packages")
+                            "PYTHONPATH=/run/current-system/profile/lib/python3.9/site-packages")
+                          ;  (string-append "PYTHONPATH=/run/current-system/profile/lib/python" pyversion "/site-packages")))
                           ;#:mappings
                           ;(list (file-system-mapping
                           ;        (source "/export/tmp")
@@ -68,7 +70,7 @@
                           ;        (writable? #t))
                           ;      (file-system-mapping
                           ;        ;; TODO: Don't hardcode python version!
-                          ;        (source "/run/current-system/profile/lib/python3.8/site-packages")
+                          ;        (source "/run/current-system/profile/lib/python3.9/site-packages")
                           ;        (target source)))
                           ))
                (stop  #~(make-kill-destructor))))))))
@@ -104,3 +106,5 @@
   (packages '())
 
   (services (list (service covid19-pubseq-service-type))))
+
+;; guix system container /home/shepherd/guix-bioinformatics/gn/services/bh20-seq-resource-container.scm --share=/export/tmp=/export/tmp --network
diff --git a/gn/services/databases.scm b/gn/services/databases.scm
index 2b2b225..937aa9e 100644
--- a/gn/services/databases.scm
+++ b/gn/services/databases.scm
@@ -33,6 +33,8 @@
              (default "localhost"))
   (server-port virtuoso-configuration-server-port
                (default 1111))
+  (dirs-allowed virtuoso-dirs-allowed
+                (default "/var/genenetwork/virtuoso-data"))
   (number-of-buffers virtuoso-configuration-number-of-buffers
                      (default #f))
   (maximum-dirty-buffers virtuoso-configuration-maximum-dirty-buffers
@@ -84,6 +86,9 @@
                               (format port "ServerPort = ~a:~a~%"
                                       #$(virtuoso-configuration-server-ip config)
                                       #$(virtuoso-configuration-server-port config)))
+                            (when #$(virtuoso-dirs-allowed config)
+                                  (format port "DirsAllowed = ~a~%"
+                                          #$(virtuoso-dirs-allowed config)))
                             (when #$(virtuoso-configuration-number-of-buffers config)
                                   (format port "NumberOfBuffers = ~a~%"
                                           #$(virtuoso-configuration-number-of-buffers config)))
diff --git a/gn/services/genome-browser.scm b/gn/services/genome-browser.scm
index 2d14a42..e350674 100644
--- a/gn/services/genome-browser.scm
+++ b/gn/services/genome-browser.scm
@@ -130,7 +130,7 @@
                            (program (file-append ucsc-genome-browser "/bin/gfServer"))
                            (arguments
                              '("gfServer" "dynserver" "/gbdb")))))))
-          (syslog-service)  ; needed by inetd
+          (service syslog-service-type)  ; needed by inetd
           (service httpd-service-type
                    (httpd-configuration
                      (config
diff --git a/gn/services/pluto.scm b/gn/services/pluto.scm
index d137314..720b769 100644
--- a/gn/services/pluto.scm
+++ b/gn/services/pluto.scm
@@ -44,16 +44,20 @@
                (modules '((gnu build shepherd)
                           (gnu system file-systems)))
                (start #~(make-forkexec-constructor/container
-                          (list #$(file-append package "/runpluto.sh") #$port)
+                          ;(list #$(file-append package "/runpluto.sh") #$port)
+                          (list #$(file-append package "/runpluto"))
                           #:log-file "/var/log/pluto.log"
                           #:user "julia"
                           #:group "julia"
-                          ;; This needs to exist. Unclear why.
+                          ;; This prevents the service from using /root as $HOME.
                           #:environment-variables '()
                           #:mappings (list (file-system-mapping
                                              (source "/home/jovyan")
                                              (target source)
-                                             (writable? #t)))))
+                                             (writable? #t))
+                                           (file-system-mapping
+                                             (source "/etc/ssl")
+                                             (target source)))))
                (stop  #~(make-kill-destructor))))))))
 
 (define pluto-service-type
@@ -86,6 +90,10 @@
   ;; No firmware for VMs.
   (firmware '())
   (packages (list nss-certs))
+  ;; For testing
+  ;(packages (cons* nss-certs %base-packages))
+
+  (setuid-programs '())
 
   (services (list (service pluto-service-type
                            (pluto-configuration
diff --git a/tux02-guix-substitutions-public-key.txt b/tux02-guix-substitutions-public-key.txt
new file mode 100644
index 0000000..ee65299
--- /dev/null
+++ b/tux02-guix-substitutions-public-key.txt
@@ -0,0 +1,7 @@
+(public-key 
+ (ecc 
+  (curve Ed25519)
+  (q #9F56EAB5CE37AA15693C31F451140588240F259676C137E31C0CA70EC4D1B534#)
+  )
+ )
+