aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--gn/packages/bioinformatics.scm317
-rw-r--r--gn/packages/bnw.scm140
-rw-r--r--gn/packages/cran.scm29
-rw-r--r--gn/packages/cwl.scm77
-rw-r--r--gn/packages/graphviz.scm9
-rw-r--r--gn/packages/javascript.scm24
-rw-r--r--gn/packages/julia.scm198
-rw-r--r--gn/packages/jupyterhub.scm1
-rw-r--r--gn/packages/maths.scm1
-rw-r--r--gn/packages/python.scm193
-rw-r--r--gn/packages/ratspub.scm213
-rw-r--r--gn/services/archive-pubmed.service13
-rw-r--r--gn/services/archive-pubmed.timer9
-rw-r--r--gn/services/bnw-README4
-rw-r--r--gn/services/bnw-container.scm4
-rw-r--r--gn/services/bnw.service2
-rw-r--r--gn/services/genenetwork.scm26
-rw-r--r--keras-auc-optimizer.patch1133
18 files changed, 2062 insertions, 331 deletions
diff --git a/gn/packages/bioinformatics.scm b/gn/packages/bioinformatics.scm
index 8897752..056bc3f 100644
--- a/gn/packages/bioinformatics.scm
+++ b/gn/packages/bioinformatics.scm
@@ -14,6 +14,8 @@
#:use-module (guix build-system trivial)
#:use-module (guix build-system waf)
#:use-module (gnu packages)
+ #:use-module (gn packages python)
+ #:use-module (gnu packages bioconductor)
#:use-module (gnu packages bioinformatics)
#:use-module (gnu packages boost)
#:use-module (gnu packages check)
@@ -22,6 +24,7 @@
#:use-module (gnu packages datastructures)
#:use-module (gnu packages fontutils)
#:use-module (gnu packages gcc)
+ #:use-module (gnu packages graphviz)
#:use-module (gnu packages imagemagick)
#:use-module (gnu packages jemalloc)
#:use-module (gnu packages maths)
@@ -31,9 +34,13 @@
#:use-module (gnu packages protobuf)
#:use-module (gnu packages python)
#:use-module (gnu packages python-science)
+ #:use-module (gnu packages python-web)
#:use-module (gnu packages python-xyz)
+ #:use-module (gnu packages rdf)
#:use-module (gnu packages readline)
- #:use-module (gnu packages statistics))
+ #:use-module (gnu packages serialization)
+ #:use-module (gnu packages statistics)
+ #:use-module (gnu packages time))
(define-public contra
(package
@@ -415,92 +422,7 @@ reads.")
(license license:non-copyleft)))
(define-public edirect-gn
- (package
- (inherit edirect)
- (name "edirect-gn")
- (arguments
- (substitute-keyword-arguments (package-arguments edirect)
- ((#:phases phases)
- `(modify-phases ,phases
- ; (replace 'build
- ; (lambda* (#:key inputs #:allow-other-keys)
- ; (let ((go (string-append (assoc-ref inputs "go") "/bin/go")))
- ; (invoke go "build" "xtract.go"))))
- (add-after 'unpack 'patch-programs
- (lambda* (#:key inputs #:allow-other-keys)
- (let ((gzip (assoc-ref inputs "gzip")))
- (substitute* '("index-bioc"
- "pm-index"
- "pm-invert"
- "pm-stash"
- "rchive.go"
- "run-ncbi-converter")
- (("gunzip") (string-append gzip "/bin/gunzip")))
- (substitute* (find-files "." "^e")
- (("exec perl") "exec"))
- (substitute* '("xtract" "rchive")
- ;; or add current directory to PATH
- ((".*PATH.*") "")))
- #t))
- (replace 'install
- (lambda* (#:key inputs outputs #:allow-other-keys)
- (let ((bin (string-append (assoc-ref outputs "out") "/bin"))
- (xtract.linux (assoc-ref inputs "xtract.Linux"))
- (rchive.linux (assoc-ref inputs "rchive.Linux")))
- (for-each
- (lambda (file)
- (install-file file bin))
- '("archive-pubmed" "asp-cp" "asp-ls" "download-pubmed"
- "edirect.pl" "efetch" "epost" "esearch" "fetch-pubmed"
- "ftp-cp" "ftp-ls" "has-asp" "pm-prepare" "pm-refresh"
- "pm-stash" "rchive" "xtract"))
- (copy-file xtract.linux (string-append bin "/xtract.Linux"))
- (copy-file rchive.linux (string-append bin "/rchive.Linux"))
- (chmod (string-append bin "/xtract.Linux") #o555)
- (chmod (string-append bin "/rchive.Linux") #o555))
- #t))
- (replace 'wrap-program
- (lambda* (#:key outputs #:allow-other-keys)
- ;; Make sure 'edirect.pl' finds all perl inputs at runtime.
- (let ((out (assoc-ref outputs "out"))
- (path (getenv "PERL5LIB")))
- (for-each
- (lambda (file)
- (wrap-program (string-append out "/bin/" file)
- `("PERL5LIB" ":" prefix (,path))))
- '("edirect.pl" "asp-ls" "ftp-cp" "ftp-ls")))
- #t))))))
- (inputs
- `(("gzip" ,gzip)
- ,@(package-inputs edirect)))
- (native-inputs
- `(
- ;("go" ,go)
- ("xtract.Linux"
- ,(origin
- (method url-fetch)
- (uri (string-append "ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/"
- "/xtract.Linux")) ;; March 10, 2016
- (sha256
- (base32
- "15yhhh8kfipk12rhzabap81ys8wgj0khn0mp8p7zwqhq028fwj0l"))))
- ("rchive.Linux"
- ,(origin
- (method url-fetch)
- (uri (string-append "ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/"
- "/rchive.Linux")) ;; November 14, 2017
- (sha256
- (base32
- "0hl8zj1md9xbmaj0pv99rjyisw8w74rirw97xwqk47dz8v8ml338"))))))
- (native-search-paths
- ;; Ideally this should be set for LWP somewhere.
- (list (search-path-specification
- (variable "PERL_LWP_SSL_CA_FILE")
- (file-type 'regular)
- (separator #f)
- (files '("/etc/ssl/certs/ca-certificates.crt")))))
- ;; Due to the precompiled binaries we download:
- (supported-systems '("x86_64-linux"))))
+ (deprecated-package "edirect-gn" edirect))
;; TODO: Unbundle zlib, bamtools, tclap
(define-public sniffles
@@ -856,19 +778,19 @@ interest, and this app can provide values and figures for applicants to use.")
(license license:gpl3))))
(define-public singlecellrshiny
- (let ((commit "8061dcb477ba355de77d3e4fd3a15cf3267b56af")
- (revision "1"))
+ (let ((commit "bdca74f4819d11e8fe7b15d9ab91b853f6542f7a")
+ (revision "3"))
(package
(name "singlecellrshiny")
(version (git-version "0.0.0" revision commit))
(source (origin
(method git-fetch)
(uri (git-reference
- (url "https://github.com/syousefi/singleCellRshiny.git")
+ (url "https://github.com/genenetwork/singleCellRshiny")
(commit commit)))
(file-name (git-file-name name version))
(sha256
- (base32 "1pd8a9jx6ijjggsifvq66madx31h29rah5pmz4kdzfzb4fskpqz1"))))
+ (base32 "1rxj933s9p9r7358vnp15f7ag6c0j65r4hgr8kyirfhmp1i8xdlw"))))
(build-system trivial-build-system)
(arguments
`(#:modules ((guix build utils))
@@ -880,16 +802,20 @@ interest, and this app can provide values and figures for applicants to use.")
(app (string-append out "/bin/" ,name))
(Rbin (string-append (assoc-ref %build-inputs "r-min")
"/bin/Rscript"))
+ (top1001 (assoc-ref %build-inputs "RobTop1001.csv"))
+ (celltypes (assoc-ref %build-inputs "CellTypes_RGC_Master_08Dec2018.csv"))
+ (800-H1 (assoc-ref %build-inputs "800-H1-H20-RNA-Seq.csv"))
(source (assoc-ref %build-inputs "source")))
(copy-recursively source targetdir)
+ (substitute* (string-append targetdir "/app.R")
+ ;; As seen in https://github.com/genenetwork/singleCellRshiny/commit/6b2a344dd0d02f65228ad8c350bac0ced5850d05.patch
+ (("library\\(DT\\)") "library(DT)\nlibrary(multtest)"))
(substitute* (string-append targetdir "/global.R")
- (("800-H1-H20-RNA-Seq-SingleCell-Retina-OMRF-03-29-19_FPKM_v2_SiamakPlay.csv")
- "shinyRappToyDataset_SiamakPlay.csv")
- ;; Comment out the two unreferenced files for now
- (("^rgc.*") "")
- ;(("CellTypes_RGC_Master_08Dec2018.csv") "")
- ;(("RobTop1001.csv") "")
- )
+ (("800-H1-H20-RNA-Seq-SingleCell-Retina-OMRF-03-29-19_FPKM_v2_SiamakPlay.csv") 800-H1)
+ (("CellTypes_RGC_Master_08Dec2018.csv") celltypes)
+ (("RobTop1001.csv") top1001)
+ ;; As seen in https://github.com/genenetwork/singleCellRshiny/commit/6b2a344dd0d02f65228ad8c350bac0ced5850d05.patch
+ (("dim\\(sc.object.1") "dim(sc.object"))
(mkdir-p (string-append out "/bin"))
(call-with-output-file app
(lambda (port)
@@ -901,20 +827,41 @@ runApp(launch.browser=0, port=4208)~%\n"
Rbin targetdir)))
(chmod app #o555)
#t))))
- (native-inputs `(("source" ,source)))
(inputs
- `(("r-min" ,r-minimal)))
+ `(("r-min" ,r-minimal)
+ ("RobTop1001.csv"
+ ,(origin
+ (method url-fetch)
+ (uri "https://archive.org/download/celltypesrgcmaster08dec2018/RobTop1001.csv")
+ (file-name "RobTop1001.csv")
+ (sha256
+ (base32 "0pa73kc1p8417sjvvvhp9xsbh2h8g7h85pnmm16mnv4wjalhq0gn"))))
+ ("CellTypes_RGC_Master_08Dec2018.csv"
+ ,(origin
+ (method url-fetch)
+ (uri "https://archive.org/download/celltypesrgcmaster08dec2018/CellTypes_RGC_Master_08Dec2018.csv")
+ (file-name "CellTypes_RGC_Master_08Dec2018.csv")
+ (sha256
+ (base32 "0y411968np1f5g21iym9xc9yj5c1jsn94rpkwkxh9pw2z43gvghn"))))
+ ("800-H1-H20-RNA-Seq.csv"
+ ,(origin
+ (method url-fetch)
+ (uri "https://archive.org/download/celltypesrgcmaster08dec2018/800-H1-H20-RNA-Seq-SingleCell-Retina-OMRF-03-29-19_FPKM_v2_SiamakPlay.csv")
+ (file-name "800-H1-H20-RNA-Seq-SingleCell-Retina-OMRF-03-29-19_FPKM_v2_SiamakPlay.csv")
+ (sha256
+ (base32 "1b1y4lfs8drypm04i1rypbmk67rdqgs27nfh05pwnv3sja2nanam"))))))
(propagated-inputs
`(("r" ,r)
("r-dt" ,r-dt)
+ ("r-multtest" ,r-multtest)
("r-seurat" ,r-seurat)
("r-shiny" ,r-shiny)))
- (home-page "http://rn6err.opar.io/")
+ (home-page "http://singlecell.opar.io/")
(synopsis "RNA sequencing data analysis")
(description
"This is the R-Shiny programs to run some basic single cell RNA sequencing
(scRNA-seq) data analysis.")
- (license license:gpl3))))
+ (license license:agpl3))))
(define-public seqwish
(package
@@ -1193,3 +1140,169 @@ here}.")
(synopsis "Efficient sequence alignment of full genomes")
(description "MUMmer is a versatil alignment tool for DNA and protein sequences.")
(license license:artistic2.0)))
+
+(define-public grocsvs
+ (let ((commit "ecd956a65093a0b2c41849050e4512d46fecea5d")
+ (revision "1"))
+ (package
+ (name "grocsvs")
+ (version (git-version "0.2.6.1" revision commit))
+ (source (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/grocsvs/grocsvs")
+ (commit commit)))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32 "14505725gr7qxc17cxxf0k6lzcwmgi64pija4mwf29aw70qn35cc"))))
+ (build-system python-build-system)
+ (arguments
+ `(#:python ,python-2)) ; Only python-2 supported.
+ (inputs
+ `(("python2-admiral" ,python2-admiral)
+ ("python2-h5py" ,python2-h5py)
+ ("python2-ipython-cluster-helper" ,python2-ipython-cluster-helper)
+ ("python2-networkx" ,python2-networkx)
+ ("python2-psutil" ,python2-psutil)
+ ("python2-pandas" ,python2-pandas)
+ ("python2-pybedtools" ,python2-pybedtools)
+ ("python2-pyfaidx" ,python2-pyfaidx)
+ ("python2-pygraphviz" ,python2-pygraphviz)
+ ("python2-pysam" ,python2-pysam)
+ ("python2-scipy" ,python2-scipy)))
+ (home-page "https://github.com/grocsvs/grocsvs")
+ (synopsis "Genome-wide reconstruction of complex structural variants")
+ (description
+ "@dfn{Genome-wide Reconstruction of Complex Structural Variants}
+(GROC-SVs), is a software pipeline for identifying large-scale structural
+variants, performing sequence assembly at the breakpoints, and reconstructing
+the complex structural variants using the long-fragment information from the
+10x Genomics platform.")
+ (license license:expat))))
+
+(define-public diagnostic-slider
+ (let ((commit "514d65d4982133e4869e578c5553fced4c6d506c")
+ (revision "1"))
+ (package
+ (name "diagnostic-slider")
+ (version (git-version "0.0.0" revision commit))
+ (source (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/sens/diagnostic-slider")
+ (commit commit)))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32 "04g8if32g8syg6v0bd3jjn05i3d394nx8i3ccl0883p8mlmdvlmx"))))
+ (build-system trivial-build-system)
+ (arguments
+ `(#:modules ((guix build utils))
+ #:builder
+ (begin
+ (use-modules (guix build utils))
+ (let* ((out (assoc-ref %outputs "out"))
+ (targetdir (string-append out "/share/" ,name))
+ (app (string-append out "/bin/" ,name))
+ (Rbin (string-append (assoc-ref %build-inputs "r-min")
+ "/bin/Rscript"))
+ (source (assoc-ref %build-inputs "source")))
+ (copy-recursively source targetdir)
+ (mkdir-p (string-append out "/bin"))
+ (call-with-output-file app
+ (lambda (port)
+ (format port
+"#!~a
+library(shiny)
+setwd(\"~a\")
+runApp(launch.browser=0, port=4206)~%\n"
+ Rbin targetdir)))
+ (chmod app #o555)
+ #t))))
+ (native-inputs
+ `(("source" ,source)))
+ (inputs
+ `(("r-min" ,r-minimal)))
+ (propagated-inputs
+ `(("r" ,r)
+ ("r-shiny" ,r-shiny)))
+ (home-page "https://github.com/sens/diagnostic-slider")
+ (synopsis "")
+ (description
+ "")
+ (license #f))))
+
+(define-public bh20-seq-resource
+ (let ((commit "bbca5ac9b2538e410efe3e09651f87e5573145de")
+ (revision "2"))
+ (package
+ (name "bh20-seq-resource")
+ (version (git-version "1.0" revision commit))
+ (source (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/arvados/bh20-seq-resource")
+ (commit commit)))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32 "1kkysjgmhp7kfb17470ik821p9djsidyqmkbjvv37jx2w9pvw31z"))))
+ (build-system python-build-system)
+ (inputs
+ `(("python-arvados-python-client" ,python-arvados-python-client)
+ ("python-flask" ,python-flask)
+ ("python-magic" ,python-magic)
+ ("python-pyyaml" ,python-pyyaml)
+ ("python-schema-salad" ,python-schema-salad)))
+ (native-inputs
+ `(("git" ,(@ (gnu packages version-control) git))
+ ("python-oauth2client" ,python-oauth2client)
+ ("python-pytest" ,python-pytest)
+ ("python-pytest-runner" ,python-pytest-runner)
+ ("python-uritemplate" ,python-uritemplate)))
+ (home-page "https://github.com/arvados/bh20-seq-resource")
+ (synopsis
+ "Tool to upload SARS-CoV-19 sequences and service to kick off analysis")
+ (description "This repository provides a sequence uploader for the
+COVID-19 Virtual Biohackathon's Public Sequence Resource project. You can use
+it to upload the genomes of SARS-CoV-2 samples to make them publicly and freely
+available to other researchers.")
+ (license license:asl2.0))))
+
+(define-public python-scanpy-git
+ (let ((commit "590d42309f9ed6550d7b887039990edfc1ac7648") ; April 22, 2020
+ (revision "1"))
+ (package
+ (inherit python-scanpy)
+ (name "python-scanpy-git")
+ (version (git-version "1.4.6" revision commit))
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/theislab/scanpy")
+ (commit commit)))
+ (file-name (git-file-name "python-scanpy" version))
+ (sha256
+ (base32 "0z3pk9vh4b7fqq7fs262i6z0pm1dnn6bf49a4r7r73k6gjj6namd"))))
+ (arguments
+ (substitute-keyword-arguments (package-arguments python-scanpy)
+ ((#:phases phases)
+ `(modify-phases ,phases
+ (add-before 'build 'fix-build
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ (let ((out (assoc-ref outputs "out"))
+ (pyv (python-version (assoc-ref inputs "python"))))
+ (substitute* "setup.py"
+ (("use_scm_version=True") "use_scm_version=False"))
+ (substitute* "scanpy/__init__.py"
+ (("__version__.*")
+ (string-append "__version__ = '" ,version "'\n")))
+ (mkdir-p
+ (string-append out "/lib/python" pyv "/site-packages"))
+ (setenv "PYTHONPATH"
+ (string-append out
+ "/lib/python" pyv "/site-packages/:"
+ (getenv "PYTHONPATH"))))
+ ;; These tests fail on this git revision
+ (delete-file "scanpy/tests/test_neighbors_key_added.py")
+ (delete-file "scanpy/tests/test_pca.py")
+ #t)))))))))
diff --git a/gn/packages/bnw.scm b/gn/packages/bnw.scm
index 5c97eab..043077d 100644
--- a/gn/packages/bnw.scm
+++ b/gn/packages/bnw.scm
@@ -14,7 +14,7 @@
(define-public bnw
(let ((commit "eb6b002b924694808384f1a8d7c6d1121806ae04")
- (revision "5"))
+ (revision "6"))
(package
(name "bnw")
(version (git-version "1.22" revision commit)) ; June 28, 2019
@@ -86,59 +86,101 @@
(("rmdir ") (string-append (which "rmdir") " "))
(("wc ") (string-append (which "wc") " ")))
#t)))
- ;(add-after 'patch-source-shebangs 'replace-javascript
- ; (lambda* (#:key inputs #:allow-other-keys)
- ; (let (
- ; (jquery (assoc-ref inputs "jquery"))
- ; (awesome (assoc-ref inputs "awesome"))
- ; (cyto (assoc-ref inputs "cytoscape"))
- ; (cyto2 (assoc-ref inputs "cytoscape-2"))
- ; (cs-dagre (assoc-ref inputs "cyto-dagre"))
- ; (d3js (assoc-ref inputs "d3js"))
- ; (d3js-multi (assoc-ref inputs "d3js-multi"))
- ; (dagre (assoc-ref inputs "dagre"))
- ; (lodash (assoc-ref inputs "lodash"))
- ; (canvas-toblob (assoc-ref inputs "canvas-toblob"))
- ; (filesaver (assoc-ref inputs "filesaver"))
- ; (panzoom (assoc-ref inputs "panzoom"))
- ; (js-path "/share/genenetwork2/javascript/")
- ; )
- ;(substitute* "sourcecodes/layout_cyto.php"
- ; (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape/3.7.1/cytoscape.min.js")
- ; (string-append cyto js-path "cytoscape/cytoscape.min.js"))
- ; (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape/2.7.29/cytoscape.min.js")
- ; (string-append cyto2 js-path "cytoscape/cytoscape.min.js"))
- ; (("http://spades.bioinf.spbau.ru/~alla/graph_viewer/js/cytoscape-dagre.js")
- ; (string-append cs-dagre js-path "cytoscape-dagre/cytoscape-dagre.js"))
- ; (("https://unpkg.com/dagre@0.7.4/dist/dagre.js")
- ; (string-append dagre js-path "dagre/dagre.js"))
- ; (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape-panzoom/2.5.3/cytoscape.js-panzoom.css")
- ; (string-append panzoom js-path "cytoscape-panzoom/cytoscape.js-panzoom.css"))
- ; (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape-panzoom/2.5.3/cytoscape-panzoom.js")
- ; (string-append panzoom js-path "cytoscape-panzoom/cytoscape-panzoom.js"))
- ; (("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.0.3/css/font-awesome.css")
- ; (string-append awesome "/share/web/font-awesome/css/font-awesome.css"))
- ; (("https://ajax.googleapis.com/ajax/libs/jquery/3.1.0/jquery.min.js")
- ; (string-append jquery "/share/web/jquery/jquery.min.js"))
- ; (("https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.10/lodash.js")
- ; (string-append lodash js-path "lodash/lodash.js")))
- ;(substitute* '("sourcecodes/layout_svg_wt.php"
- ; "sourcecodes/layout_svg_no.php")
- ; (("http://d3js.org/d3.v4.min.js")
- ; (string-append d3js js-path "d3js/d3.min.js"))
- ; (("http://d3js.org/d3-selection-multi.v1.js")
- ; (string-append d3js-multi js-path "d3js-multi/d3-selection-multi.js"))
- ; (("https://cdn.rawgit.com/eligrey/canvas-toBlob.js/f1a01896135ab378aa5c0118eadd81da55e698d8/canvas-toBlob.js")
- ; (string-append canvas-toblob js-path "canvas-toBlob/canvas-toBlob.js"))
- ; (("https://cdn.rawgit.com/eligrey/FileSaver.js/e9d941381475b5df8b7d7691013401e171014e89/FileSaver.min.js")
- ; (string-append filesaver js-path "filesaver/filesaver.js")))
- ; )
- ; #t))
+ (add-after 'patch-source-shebangs 'replace-javascript
+ (lambda* (#:key inputs #:allow-other-keys)
+ (let ((jquery (assoc-ref inputs "jquery"))
+ (awesome (assoc-ref inputs "awesome"))
+ (cyto (assoc-ref inputs "cytoscape"))
+ (cyto2 (assoc-ref inputs "cytoscape-2"))
+ (cs-dagre (assoc-ref inputs "cyto-dagre"))
+ (d3js (assoc-ref inputs "d3js"))
+ (d3js-multi (assoc-ref inputs "d3js-multi"))
+ (dagre (assoc-ref inputs "dagre"))
+ (lodash (assoc-ref inputs "lodash"))
+ (canvas-toblob (assoc-ref inputs "canvas-toblob"))
+ (filesaver (assoc-ref inputs "filesaver"))
+ (panzoom (assoc-ref inputs "panzoom"))
+ (js-path "/share/genenetwork2/javascript/"))
+ (substitute* "sourcecodes/layout_cyto.php"
+ (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape/3.7.1/cytoscape.min.js")
+ "/javascript/cytoscape.min.js")
+ (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape/2.7.29/cytoscape.min.js")
+ "/javascript/cytoscape2.min.js")
+ (("http://spades.bioinf.spbau.ru/~alla/graph_viewer/js/cytoscape-dagre.js")
+ "/javascript/cytoscape-dagre.js")
+ (("https://unpkg.com/dagre@0.7.4/dist/dagre.js")
+ "/javascript/dagre.js")
+ (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape-panzoom/2.5.3/cytoscape.js-panzoom.css")
+ "/javascript/cytoscape.js-panzoom.css")
+ (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape-panzoom/2.5.3/cytoscape-panzoom.js")
+ "/javascript/cytoscape-panzoom.js")
+ (("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.0.3/css/font-awesome.css")
+ "/javascript/font-awesome.css")
+ (("https://ajax.googleapis.com/ajax/libs/jquery/3.1.0/jquery.min.js")
+ "/javascript/jquery.min.js")
+ (("https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.10/lodash.js")
+ "/javascript/lodash.js"))
+ (substitute* '("sourcecodes/layout_svg_wt.php"
+ "sourcecodes/layout_svg_no.php")
+ (("http://d3js.org/d3.v4.min.js")
+ "/javascript/d3.min.js")
+ (("http://d3js.org/d3-selection-multi.v1.js")
+ "/javascript/d3-selection-multi.js")
+ (("https://cdn.rawgit.com/eligrey/canvas-toBlob.js/f1a01896135ab378aa5c0118eadd81da55e698d8/canvas-toBlob.js")
+ "/javascript/canvas-toBlob.js")
+ (("https://cdn.rawgit.com/eligrey/FileSaver.js/e9d941381475b5df8b7d7691013401e171014e89/FileSaver.min.js")
+ "/javascript/filesaver.js")))
+ #t))
(replace 'install
(lambda* (#:key outputs #:allow-other-keys)
(let ((out (assoc-ref outputs "out")))
(copy-recursively "." out))
#t))
+ (add-after 'install 'install-javascript-libraries
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ (let ((out (assoc-ref outputs "out"))
+ (jquery (assoc-ref inputs "jquery"))
+ (awesome (assoc-ref inputs "awesome"))
+ (cyto (assoc-ref inputs "cytoscape"))
+ (cyto2 (assoc-ref inputs "cytoscape-2"))
+ (cs-dagre (assoc-ref inputs "cyto-dagre"))
+ (d3js (assoc-ref inputs "d3js"))
+ (d3js-multi (assoc-ref inputs "d3js-multi"))
+ (dagre (assoc-ref inputs "dagre"))
+ (lodash (assoc-ref inputs "lodash"))
+ (canvas-toblob (assoc-ref inputs "canvas-toblob"))
+ (filesaver (assoc-ref inputs "filesaver"))
+ (panzoom (assoc-ref inputs "panzoom"))
+ (js-path "/share/genenetwork/javascript/")
+ (js-path2 "/share/genenetwork2/javascript/"))
+ (mkdir-p (string-append out "/javascript"))
+ (symlink (string-append (string-append cyto2 js-path2 "cytoscape/cytoscape.min.js"))
+ (string-append out "/javascript/cytoscape.min.js"))
+ (symlink (string-append (string-append cyto js-path2 "cytoscape/cytoscape.min.js"))
+ (string-append out "/javascript/cytoscape2.min.js"))
+ (symlink (string-append cs-dagre js-path2 "cytoscape-dagre/cytoscape-dagre.js")
+ (string-append out "/javascript/cytoscape-dagre.js"))
+ (symlink (string-append dagre js-path2 "dagre/dagre.js")
+ (string-append out "/javascript/dagre.js"))
+ (symlink (string-append panzoom js-path2 "cytoscape-panzoom/cytoscape.js-panzoom.css")
+ (string-append out "/javascript/cytoscape.js-panzoom.css"))
+ (symlink (string-append panzoom js-path2 "cytoscape-panzoom/cytoscape-panzoom.js")
+ (string-append out "/javascript/cytoscape-panzoom.js"))
+ (symlink (string-append awesome "/share/web/font-awesomecss/font-awesome.css")
+ (string-append out "/javascript/font-awesome.css"))
+ (symlink (string-append jquery "/share/web/jquery/jquery.min.js")
+ (string-append out "/javascript/jquery.min.js"))
+ (symlink (string-append lodash js-path2 "lodash/lodash.js")
+ (string-append out "/javascript/lodash.js"))
+ (symlink (string-append d3js js-path "d3js/d3.min.js")
+ (string-append out "/javascript/d3.min.js"))
+ (symlink (string-append d3js-multi js-path "d3js-multi/d3-selection-multi.js")
+ (string-append out "/javascript/d3-selection-multi.js"))
+ (symlink (string-append canvas-toblob js-path "canvas-toblob/canvas-toBlob.js")
+ (string-append out "/javascript/canvas-toBlob.js"))
+ (symlink (string-append filesaver js-path2 "filesaver/FileSaver.js")
+ (string-append out "/javascript/filesaver.js"))
+ #t)))
(add-after 'install 'make-files-executable
(lambda* (#:key outputs #:allow-other-keys)
(let ((out (assoc-ref outputs "out")))
diff --git a/gn/packages/cran.scm b/gn/packages/cran.scm
new file mode 100644
index 0000000..d92f9f3
--- /dev/null
+++ b/gn/packages/cran.scm
@@ -0,0 +1,29 @@
+(define-module (gn packages cran)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix packages)
+ #:use-module (guix download)
+ #:use-module (guix build-system r)
+ #:use-module (gnu packages))
+
+(define-public r-tictoc
+ (package
+ (name "r-tictoc")
+ (version "1.0")
+ (source
+ (origin
+ (method url-fetch)
+ (uri (cran-uri "tictoc" version))
+ (sha256
+ (base32
+ "1zp2n8k2ax2jjw89dsri268asmm5ry3ijf32wbca5ji231y0knj7"))))
+ (build-system r-build-system)
+ (home-page "http://github.com/collectivemedia/tictoc")
+ (synopsis "Functions for timing R scripts")
+ (description
+ "This package provides the timing functions @code{tic} and @code{toc} that
+can be nested. One can record all timings while a complex script is running,
+and examine the values later. It is also possible to instrument the timing call
+with custom callbacks. In addition, this package provides class 'Stack',
+implemented as a vector, and class 'List', implemented as a list, both of whic
+support operations 'push', 'pop', 'first', 'last' and 'clear'.")
+ (license license:asl2.0)))
diff --git a/gn/packages/cwl.scm b/gn/packages/cwl.scm
index 6e0c909..be531f6 100644
--- a/gn/packages/cwl.scm
+++ b/gn/packages/cwl.scm
@@ -61,7 +61,7 @@
("python-mock" ,python-mock)
("python-subprocess32" ,python-subprocess32)
("python-ruamel.yaml" ,python-ruamel.yaml)
- ("python-cachecontrol" ,python-cachecontrol)
+ ("python-cachecontrol" ,python-cachecontrol-0.11)
("python-lxml" ,python-lxml)
("python-mypy-extensions" ,python-mypy-extensions)
("python-mistune" ,python-mistune)
@@ -89,78 +89,3 @@
(description
"Common workflow language reference implementation")
(license license:asl2.0))))
-
-(define-public python-cachecontrol
- (package
- (name "python-cachecontrol")
- (version "0.11.7")
- (source
- (origin
- (method url-fetch)
- ;; Pypi does not have tests.
- (uri (string-append
- "https://github.com/ionrock/cachecontrol/archive/v"
- version ".tar.gz"))
- (file-name (string-append name "-" version ".tar.gz"))
- (sha256
- (base32
- "1yfhwihx1b1xjsx0r19va2m0r2s91im03x4d7pwzp87368f2lkkp"))))
- (build-system python-build-system)
- (arguments
- `(#:tests? #f)) ;; Recent version breaks on cherrypy
- (native-inputs
- `(("python-pytest" ,python-pytest)
- ("python-redis" ,python-redis)
- ("python-webtest" ,python-webtest)
- ("python-mock" ,python-mock)))
- (propagated-inputs
- `(("python-requests" ,python-requests)
- ("python-lockfile" ,python-lockfile)))
- (home-page "https://github.com/ionrock/cachecontrol")
- (synopsis "The httplib2 caching algorithms for use with requests")
- (description "CacheControl is a port of the caching algorithms in
-@code{httplib2} for use with @code{requests} session objects.")
- (license license:asl2.0)))
-
-
-(define-public python-schema-salad
- (let ((commit "eb85c3d49b99b7643e8a12248e2dc05504910c1e"))
- (package
- (name "python-schema-salad")
- (version "3.0.20181129082112")
- (source
- (origin
- ; (method url-fetch)
- ; (uri (pypi-uri "schema-salad" version))
- (method git-fetch)
- (uri (git-reference
- (url "https://github.com/genenetwork/schema_salad.git") ;; my repo for Python3.7
- (commit commit)))
- (file-name (git-file-name name (string-append version "-" (string-take commit 7))))
- (sha256
- (base32
- "174f224zzjr0nbjlq3ypciyfijnibasysrgjswvx8yhan2dizlhr"))))
- (build-system python-build-system)
- (arguments `(#:tests? #f)) ;; CWL includes no tests.
- (inputs
- `(("python-cython" ,python-cython)
- ("python-setuptools" ,python-setuptools)
- ("python-rdflib-jsonld" ,python-rdflib-jsonld)
- ("python-mistune" ,python-mistune)))
- (propagated-inputs
- `(("python-rdflib" ,python-rdflib)
- ("python-avro" ,python-avro)
- ("python-pyyaml" ,python-pyyaml)
- ("python-requests" ,python-requests)
- ("python-shellescape" ,python-shellescape)
- ))
- (home-page
- "https://github.com/common-workflow-language/common-workflow-language")
- (synopsis
- "Schema Annotations for Linked Avro Data (SALAD)")
- (description
- "Schema Annotations for Linked Avro Data (SALAD)")
- (license license:asl2.0))))
-
-; (define-public python2-schema-salad
-; (package-with-python2 python-schema-salad))
diff --git a/gn/packages/graphviz.scm b/gn/packages/graphviz.scm
index 81e745c..33b15a1 100644
--- a/gn/packages/graphviz.scm
+++ b/gn/packages/graphviz.scm
@@ -3,7 +3,6 @@
#:use-module (guix packages)
#:use-module (guix download)
#:use-module (guix utils)
- #:use-module (gn packages python24)
#:use-module (gnu packages gl)
#:use-module (gnu packages graphviz)
#:use-module (gnu packages gtk)
@@ -28,18 +27,16 @@
"18bzyg17ni0lpcd2g5dhan8fjv3vzkjym38jq8vm42did5p9j47l"))))
;; TODO: unbundle libraries?
(arguments
- `(#:configure-flags '("--enable-python24=yes")
+ `(#:configure-flags '("--enable-swig=no")
,@(substitute-keyword-arguments (package-arguments graphviz)
((#:phases phases)
`(modify-phases ,phases
(delete 'move-docs) ; one output
(delete 'move-guile-bindings))))))
(inputs
- ;; TODO: Add(?) perl, guile@1.8, gtk@2, lua5.1, tcl8.[3-6], rsvg
+ ;; TODO(?): Add perl, guile@1.8, gtk@2, lua5.1, tcl8.[3-6], rsvg, python-2.4
`(("gdk-pixbuf" ,gdk-pixbuf)
- ;("ghostscript" ,ghostscript)
("freeglut" ,freeglut)
- ("python2.4" ,python-2.4)
,@(fold alist-delete (package-inputs graphviz)
- '("libjpeg" "guile"))))
+ '("libjpeg" "guile" "swig"))))
(license license:cpl1.0)))
diff --git a/gn/packages/javascript.scm b/gn/packages/javascript.scm
index 79a8060..95d4154 100644
--- a/gn/packages/javascript.scm
+++ b/gn/packages/javascript.scm
@@ -1121,3 +1121,27 @@ widgets, and themes built on top of the jQuery JavaScript Library.")
(name "js-jquery-ui")
(arguments `(#:javascript-files '("ui/jquery-ui.js")))
(build-system minify-build-system)))
+
+(define-public js-popper
+ (package
+ (name "js-popper")
+ (version "2.0.6")
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/popperjs/popper-core")
+ (commit (string-append "v" version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ "0lzy981p9nja2l3xa2zvals6q31v3bzpxxa85yn9pm7wkj3vglf2"))))
+ (build-system minify-build-system)
+ (arguments
+ `(#:javascript-files '("src/popper.js")))
+ (home-page "https://popper.js.org/")
+ (synopsis "Tooltip and popover positioning engine")
+ (description
+ "Given an element, such as a button, and a tooltip element describing it,
+Popper will automatically put the tooltip in the right place near the button.")
+ (license license:expat)))
diff --git a/gn/packages/julia.scm b/gn/packages/julia.scm
new file mode 100644
index 0000000..b55efde
--- /dev/null
+++ b/gn/packages/julia.scm
@@ -0,0 +1,198 @@
+(define-module (gn packages julia)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix utils)
+ #:use-module (guix packages)
+ #:use-module (guix download)
+ #:use-module (guix git-download)
+ #:use-module (guix build-system julia)
+ #:use-module (gn packages cran)
+ #:use-module (gnu packages bioinformatics)
+ #:use-module (gnu packages compression)
+ #:use-module (gnu packages cran)
+ #:use-module (gnu packages statistics)
+ #:use-module (ice-9 match))
+
+(define-public julia-lmgpu
+ (let ((commit "e9e95b5fa46f1905ca1ff32a3684a2616a7e482c")
+ (revision "1"))
+ (package
+ (name "julia-lmgpu")
+ (version (git-version "0.1.1" revision commit))
+ (source (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/ChelseaTrotter/LMGPU.jl")
+ (commit commit)))
+ ;(file-name (git-file-name name version))
+ (file-name "LMGPU")
+ (sha256
+ (base32
+ "1ddx2np1lakw1l2dclpcaihxd0fcj6bjxsvaxr6g5brxjqk5j7b1"))))
+ (build-system julia-build-system)
+ (arguments
+ `(#:phases
+ (modify-phases %standard-phases
+ ;; This is a super ugly hack. Some JULIA environment variable should
+ ;; be tuned so it can find the artifact directory.
+ (add-after 'unpack 'symlink-zlib-into-artifact-directory
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ (let ((julia-dir (string-append (assoc-ref outputs "out")
+ "/share/julia")))
+ (mkdir-p julia-dir)
+ (symlink
+ (string-append (assoc-ref inputs "julia-zlib-jll")
+ "/share/julia/artifacts")
+ (string-append julia-dir "/artifacts")))
+ #t))
+ (add-after 'precompile 'check
+ (lambda* (#:key outputs #:allow-other-keys)
+ (let* ((out (assoc-ref outputs "out"))
+ (builddir (string-append out "/share/julia/")))
+ (setenv "JULIA_LOAD_PATH"
+ (string-append builddir "packages/" ":"
+ (or (getenv "JULIA_LOAD_PATH")
+ "")))
+ (setenv "HOME" (getcwd))
+ (invoke "julia" "test/runtests.jl")))))))
+ (native-inputs
+ `(("r" ,r-minimal)
+ ("r-mice" ,r-mice)
+ ("r-qtl2" ,r-qtl2)
+ ("r-tictoc" ,r-tictoc)
+ ("r-tidyverse" ,r-tidyverse)))
+ (propagated-inputs
+ `(("julia-zipfile" ,julia-zipfile)))
+ (home-page "https://github.com/ChelseaTrotter/LMGPU.jl")
+ (synopsis "")
+ (description "")
+ (license license:expat))))
+
+(define-public julia-lmgpu-myapp
+ (package
+ (inherit julia-lmgpu)
+ (name "julia-lmgpu-myapp")
+ (source
+ (origin (inherit (package-source julia-lmgpu))
+ (file-name "MyApp")))
+ (arguments
+ (substitute-keyword-arguments (package-arguments julia-lmgpu)
+ ((#:phases phases)
+ `(modify-phases ,phases
+ (add-after 'unpack 'change-directory
+ (lambda _
+ (chdir "bin/MyApp") #t))))))
+ (propagated-inputs
+ `(("julia-lmgpu" ,julia-lmgpu)
+ ,@(package-propagated-inputs julia-lmgpu)))
+ (native-inputs
+ `(("julia-packagecompiler" ,julia-packagecompiler)))))
+
+(define-public julia-zipfile
+ (package
+ (name "julia-zipfile")
+ (version "0.9.1")
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/fhs/ZipFile.jl")
+ (commit (string-append "v" version))))
+ ;(file-name (git-file-name name version))
+ (file-name "ZipFile")
+ (sha256
+ (base32
+ "1fpvlhfqg5kgq5vchlf8dyc73r6dzki0dz7plddc3bnr0ld00rlw"))))
+ (build-system julia-build-system)
+ (arguments
+ `(#:phases
+ (modify-phases %standard-phases
+ ;; This is a super ugly hack. Some JULIA environment variable should
+ ;; be tuned so it can find the artifact directory.
+ (add-after 'unpack 'symlink-zlib-into-artifact-directory
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ (let ((julia-dir (string-append (assoc-ref outputs "out")
+ "/share/julia")))
+ (mkdir-p julia-dir)
+ (symlink
+ (string-append (assoc-ref inputs "julia-zlib-jll")
+ "/share/julia/artifacts")
+ (string-append julia-dir "/artifacts")))
+ #t)))))
+ (propagated-inputs
+ `(("julia-zlib-jll" ,julia-zlib-jll)))
+ (home-page "https://github.com/fhs/ZipFile.jl")
+ (synopsis "Read/Write ZIP archives in Julia")
+ (description "This module provides support for reading and writing ZIP
+archives in Julia.")
+ (license license:expat)))
+
+(define-public julia-zlib-jll
+ (package
+ (name "julia-zlib-jll")
+ (version "1.2.11+9")
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/JuliaBinaryWrappers/Zlib_jll.jl")
+ (commit (string-append "Zlib-v" version))))
+ ;(file-name (git-file-name name version))
+ (file-name "Zlib_jll")
+ (sha256
+ (base32
+ "0m9n8dp4bwhkyjag1szmhz02k0bxzm4ka2ia2jh8crnd1qi8w9dz"))))
+ (build-system julia-build-system)
+ (arguments
+ `(#:phases
+ (modify-phases %standard-phases
+ (add-after 'unpack 'symlink-zlib-into-artifact-directory
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ (let ((artifacts (string-append (assoc-ref outputs "out")
+ "/share/julia/artifacts")))
+ (mkdir-p artifacts)
+ (symlink
+ (assoc-ref inputs "zlib")
+ ;; from git-tree-sha1 in Artifacts.toml
+ (string-append
+ artifacts
+ ,(match (%current-system)
+ ("x86_64-linux" "/7846a2956a213715c2c76632f3461cef87d9d545")
+ ("i686-linux" "/c8456cbd00982236828623bbc63f21b9b7b03821")
+ ("armhf-linux" "/748c38025b5596a5005a87ac2b9476603cf8615b")
+ ("aarch64-linux" "/3dd0c7cd5424c8746a1a32034ba1b10458f20b3b")
+ (_ "/UNSUPPORTED")))))
+ #t)))))
+ (native-inputs
+ `(("zlib" ,zlib)))
+ (home-page "https://github.com/JuliaBinaryWrappers/Zlib_jll.jl")
+ (synopsis "Autogenerated package constructed using BinaryBuilder.jl")
+ (description "This is an autogenerated package constructed using
+@url{https://github.com/JuliaPackaging/BinaryBuilder.jl, BinaryBuilder.jl}.")
+ (license license:expat)))
+
+(define-public julia-packagecompiler
+ (package
+ (name "julia-packagecompiler")
+ (version "1.1.1")
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/JuliaLang/PackageCompiler.jl")
+ (commit (string-append "v" version))))
+ ;(file-name (git-file-name name version))
+ (file-name "PackageCompiler")
+ (sha256
+ (base32
+ "1s9xc17i308fdpyvkz1w6qb1h7yncdr2jgk1szfvygxd6yzkv1b4"))))
+ (build-system julia-build-system)
+ (home-page "https://github.com/JuliaLang/PackageCompiler.jl")
+ (synopsis "Compile your Julia Package")
+ (description "PackageCompiler is a Julia package with two main purposes:
+@itemize
+@item Creating custom sysimages for reduced latency when working locally with
+packages that has a high startup time.
+@item Creating \"apps\" which are a bundle of files including an executable that
+can be sent and run on other machines without Julia being installed on that machine.
+@end itemize")
+ (license license:expat)))
diff --git a/gn/packages/jupyterhub.scm b/gn/packages/jupyterhub.scm
index 5b69b5b..43da8ec 100644
--- a/gn/packages/jupyterhub.scm
+++ b/gn/packages/jupyterhub.scm
@@ -15,6 +15,7 @@
#:use-module (gnu packages python-crypto)
#:use-module (gnu packages python-web)
#:use-module (gnu packages python-xyz)
+ #:use-module (gnu packages rpc)
#:use-module (gnu packages serialization)
#:use-module (gnu packages time)
#:use-module (gn packages node))
diff --git a/gn/packages/maths.scm b/gn/packages/maths.scm
index 9e15400..1c1c1ae 100644
--- a/gn/packages/maths.scm
+++ b/gn/packages/maths.scm
@@ -30,6 +30,7 @@
((#:configure-flags cf)
`(cons "--enable-docs=no" ; docs fail to build
,cf))
+ ((#:tests? _ #f) #f) ; tests hang
((#:phases phases)
`(modify-phases ,phases
(add-after 'unpack 'patch-configure-script
diff --git a/gn/packages/python.scm b/gn/packages/python.scm
index 60558a2..7e4786f 100644
--- a/gn/packages/python.scm
+++ b/gn/packages/python.scm
@@ -3,6 +3,7 @@
#:use-module (gnu packages)
#:use-module (gnu packages attr)
#:use-module (gnu packages base)
+ #:use-module (gnu packages bioinformatics)
#:use-module (gnu packages check)
#:use-module (gnu packages compression)
#:use-module (gnu packages databases)
@@ -30,6 +31,7 @@
#:use-module (gnu packages python-xyz)
#:use-module (gnu packages rdf)
#:use-module (gnu packages readline)
+ #:use-module (gnu packages serialization)
#:use-module (gnu packages statistics)
#:use-module (gnu packages tcl)
#:use-module (gnu packages tex)
@@ -228,25 +230,6 @@ until a value is returned.")
"A library for W3C Provenance Data Model supporting PROV-JSON, PROV-XML and PROV-O (RDF)")
(license license:expat)))
-(define-public python-typing-extensions; guix candidate
- (package
- (name "python-typing-extensions")
- (version "3.6.6")
- (source
- (origin
- (method url-fetch)
- (uri (pypi-uri "typing_extensions" version))
- (sha256
- (base32
- "07vhddjnd3mhdijyc3s0mwi9jgfjp3rr056nxqiavydbvkrvgrsi"))))
- (build-system python-build-system)
- (home-page "https://pypi.python.org/pypi/typing_extensions")
- (synopsis "Python typing_extensions.")
- (description
- "Python typing_extensions.")
- (license license:gpl2))
-)
-
(define-public python-subprocess32 ; guix candidate
(package
(name "python-subprocess32")
@@ -322,39 +305,6 @@ until a value is returned.")
(define-public python2-xlsxwriter
(package-with-python2 python-xlsxwriter))
-(define-public python-rdflib-jsonld ; guix ready
- (package
- (name "python-rdflib-jsonld")
- (version "0.4.0")
- (source
- (origin
- (method url-fetch)
- (uri (pypi-uri "rdflib-jsonld" version))
- (sha256
- (base32
- "0bdw2pbjmpy1l4p6slsjn54bqy6crk5hk4san84xxirgd9w78iql"))))
- (build-system python-build-system)
- (inputs
- `(("python-setuptools" ,python-setuptools)))
- (propagated-inputs
- `(("python-rdflib" ,python-rdflib)
- ("python-isodate" ,python-isodate)
- ("python-pyparsing" ,python-pyparsing)
- ("python-html5lib" ,python-html5lib)
- ("python-nose" ,python-nose)
-))
- (home-page
- "https://github.com/RDFLib/rdflib-jsonld")
- (synopsis
- "rdflib extension adding JSON-LD parser and serializer")
- (description
- "rdflib extension adding JSON-LD parser and serializer")
- (license license:bsd-3)))
-
-(define-public python2-rdflib-jsonld
- (package-with-python2 python-rdflib-jsonld))
-
-
(define-public python-rserve
(package
(name "python-rserve")
@@ -1094,3 +1044,142 @@ spreadsheets without the need for COM objects.")
(synopsis "")
(description "")
(license license:bsd-4)))
+
+(define-public python-admiral
+ (package
+ (name "python-admiral")
+ (version "0.2")
+ (source
+ (origin
+ (method url-fetch)
+ (uri (pypi-uri "admiral" version))
+ (sha256
+ (base32
+ "1b2zjgyz94ld5wr7s4cm4x5sxijx3w0dmd7r2cq1s8iqjzz6rd1x"))))
+ (build-system python-build-system)
+ (arguments '(#:tests? #f)) ; No tests
+ (propagated-inputs
+ `(("python-humanfriendly" ,python-humanfriendly)))
+ (home-page "https://github.com/nspies/admiral")
+ (synopsis
+ "Simple python high-performance computing cluster batch submission")
+ (description
+ "Simple python high-performance computing cluster batch submission.")
+ (license #f))) ; No license in repository.
+
+(define-public python2-admiral
+ (package-with-python2 python-admiral))
+
+(define-public python-cachecontrol-0.11
+ (package
+ (inherit python-cachecontrol)
+ (name "python-cachecontrol")
+ (version "0.11.7")
+ (source
+ (origin
+ (method url-fetch)
+ (uri (pypi-uri "CacheControl" version))
+ (sha256
+ (base32
+ "07jsfhlbcwgqg6ayz8nznzaqg5rmxqblbzxz1qvg5wc44pcjjy4g"))))))
+
+(define-public python-ruamel.yaml-0.15
+ (package
+ (inherit python-ruamel.yaml)
+ (name "python-ruamel.yaml")
+ (version "0.15.77")
+ (source
+ (origin
+ (method url-fetch)
+ (uri (pypi-uri "ruamel.yaml" version))
+ (sha256
+ (base32
+ "1mhzxkkiv3xmr9izrgk78x7f1r5gi8kd5ac7z3vn7j00q1ydn6da"))))))
+
+(define-public python-pbr-1.6.0
+ (package
+ (inherit python-pbr)
+ (name "python-pbr")
+ (version "1.6.0")
+ (source
+ (origin
+ (method url-fetch)
+ (uri (pypi-uri "pbr" version))
+ (sha256
+ (base32
+ "1lg1klrczvzfan89y3bl9ykrknl3nb01vvai37fkww24apzyibjf"))))))
+
+(define-public python-arvados-python-client
+ (package
+ (name "python-arvados-python-client")
+ (version "2.0.2")
+ (source
+ (origin
+ (method url-fetch)
+ (uri (pypi-uri "arvados-python-client" version))
+ (sha256
+ (base32
+ "19l4w6m5426x5k2kick630dh2jx26j16ycs2nhbfgr4cd43d29y4"))))
+ (build-system python-build-system)
+ (arguments
+ `(#:tests? #f)) ; tests not included?
+ (propagated-inputs
+ `(("python-ciso8601" ,python-ciso8601)
+ ("python-future" ,python-future)
+ ;("python-google-api-python-client" ,python-google-api-python-client)
+ ("python-google-api-client" ,python-google-api-client)
+ ("python-httplib2" ,python-httplib2)
+ ("python-pycurl" ,python-pycurl)
+ ("python-ruamel.yaml" ,python-ruamel.yaml-0.15)
+ ("python-setuptools" ,python-setuptools)
+ ("python-ws4py" ,python-ws4py)))
+ (native-inputs
+ `(("python-mock" ,python-mock)
+ ("python-oauth2client" ,python-oauth2client)
+ ("python-pbr" ,python-pbr-1.6.0)
+ ("python-pyyaml" ,python-pyyaml)
+ ("python-uritemplate" ,python-uritemplate)))
+ (home-page "https://arvados.org")
+ (synopsis "Arvados client library")
+ (description "This package provides the arvados module, an API client for
+Arvados. It also includes higher-level functions to help you write Crunch
+scripts, and command-line tools to store and retrieve data in the Keep storage
+server.")
+ (license license:asl2.0)))
+
+(define-public python-schema-salad
+ (package
+ (name "python-schema-salad")
+ (version "5.0.20200416112825")
+ (source
+ (origin
+ (method url-fetch)
+ (uri (pypi-uri "schema-salad" version))
+ (sha256
+ (base32
+ "1pm6q266qrw4r0w0vnzhsvqgk5j8b3q61hxg99awhgpjsmcvkmsz"))))
+ (build-system python-build-system)
+ (propagated-inputs
+ `(("python-cachecontrol" ,python-cachecontrol-0.11)
+ ("python-lockfile" ,python-lockfile)
+ ("python-mistune" ,python-mistune)
+ ("python-rdflib" ,python-rdflib)
+ ("python-rdflib-jsonld" ,python-rdflib-jsonld)
+ ("python-requests" ,python-requests)
+ ("python-ruamel.yaml" ,python-ruamel.yaml)
+ ("python-setuptools" ,python-setuptools)
+ ("python-typing-extensions" ,python-typing-extensions)))
+ (native-inputs
+ `(("python-pytest" ,python-pytest)
+ ("python-pytest-runner" ,python-pytest-runner)))
+ (home-page "https://github.com/common-workflow-language/schema_salad")
+ (synopsis "Schema Annotations for Linked Avro Data (SALAD)")
+ (description
+ "Salad is a schema language for describing JSON or YAML structured linked
+data documents. Salad schema describes rules for preprocessing, structural
+validation, and hyperlink checking for documents described by a Salad schema.
+Salad supports rich data modeling with inheritance, template specialization,
+object identifiers, object references, documentation generation, code
+generation, and transformation to RDF. Salad provides a bridge between document
+and record oriented data modeling and the Semantic Web.")
+ (license license:asl2.0)))
diff --git a/gn/packages/ratspub.scm b/gn/packages/ratspub.scm
index 29e1313..b664799 100644
--- a/gn/packages/ratspub.scm
+++ b/gn/packages/ratspub.scm
@@ -1,11 +1,16 @@
(define-module (gn packages ratspub)
#:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix utils)
+ #:use-module (gnu packages)
#:use-module (guix packages)
#:use-module (guix git-download)
#:use-module (guix build-system python)
#:use-module (gnu packages admin)
- #:use-module (gn packages bioinformatics)
+ #:use-module (gnu packages bioinformatics)
#:use-module (gn packages javascript)
+ #:use-module (gnu packages machine-learning)
+ #:use-module (gnu packages python)
+ #:use-module (gnu packages python-crypto)
#:use-module (gnu packages python-web)
#:use-module (gnu packages python-xyz)
#:use-module (gn packages web))
@@ -13,19 +18,27 @@
(define-public ratspub
(package
(name "ratspub")
- (version "0.1")
+ (version "0.3.1")
(source (origin
- (method git-fetch)
- (uri (git-reference
- (url "https://github.com/chen42/ratspub.git")
- (commit (string-append "v" version))))
- (file-name (git-file-name name version))
- (sha256
- (base32
- "0cm9g38fxpa52458mdmhzhghj5c7b8l3k1b764zs9hdrki5s7wi7"))))
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/chen42/ratspub.git")
+ (commit (string-append "v" version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ "1ii3721mqd3dbpjkhqi7yqjd9bqcf0g19kdbb8265pmbfjjsg164"))
+ (modules '((guix build utils)))
+ (snippet
+ '(begin (substitute* "server.py"
+ ;; Keep the service running on port 4200
+ (("4201") "4200")
+ ;; Backport to python-keras-2.2.4
+ (("learning_rate") "lr") )
+ #t))))
(build-system python-build-system)
(arguments
- `(#:tests? #f ; no test suite
+ `(#:tests? #f ; no test suite
#:phases
(modify-phases %standard-phases
(delete 'configure)
@@ -34,12 +47,24 @@
(lambda* (#:key inputs outputs #:allow-other-keys)
(let ((out (assoc-ref outputs "out"))
(inetutils (assoc-ref inputs "inetutils")))
- (substitute* "templates/cytoscape.html"
+ (substitute* '("templates/cytoscape.html"
+ "templates/tableview.html"
+ "templates/tableview0.html"
+ "templates/userarchive.html")
(("script src=.*")
"script src=\"/static/cytoscape.min.js\"></script>\n"))
(substitute* "templates/layout.html"
- (("https://stackpath.bootstrapcdn.com/bootstrap/.*")
- "/static/bootstrap.min.css\">\n"))
+ (("https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css.*")
+ "/static/bootstrap.min.css\">\n")
+ (("https://.*.bootstrapcdn.com/bootstrap/4.*/js/bootstrap.min.js.*")
+ "/static/bootstrap.min.js\"></script>\n")
+ (("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css")
+ "/static/font-awesome.min.css")
+ (("https://code.jquery.com/jquery-3.2.1.slim.min.js.*")
+ "/static/jquery.slim.min.js\"></script>\n")
+ ;(("https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js.*")
+ ; "/static/popper.min.js\"></script>\n")
+ )
(substitute* "ratspub.py"
(("hostname") (string-append inetutils "/bin/hostname"))))
#t))
@@ -51,31 +76,58 @@
(add-after 'install 'install-javascript
(lambda* (#:key inputs outputs #:allow-other-keys)
(let ((out (assoc-ref outputs "out"))
+ (awesome (assoc-ref inputs "font-awesome"))
+ (bootstrap (assoc-ref inputs "bootstrap"))
(cytoscape (assoc-ref inputs "cytoscape"))
- (bootstrap (assoc-ref inputs "bootstrap")))
+ (jquery (assoc-ref inputs "jquery"))
+ ;(js-popper (assoc-ref inputs "js-popper"))
+ )
+ (symlink (string-append awesome
+ "/share/web/font-awesomecss/font-awesome.min.css")
+ (string-append out "/static/font-awesome.min.css"))
+ (symlink (string-append bootstrap
+ "/share/web/bootstrap/css/bootstrap.min.css")
+ (string-append out "/static/bootstrap.min.css"))
+ (symlink (string-append bootstrap
+ "/share/web/bootstrap/js/bootstrap.min.js")
+ (string-append out "/static/bootstrap.min.js"))
(symlink (string-append cytoscape
"/share/genenetwork2/javascript/cytoscape/cytoscape.min.js")
(string-append out "/static/cytoscape.min.js"))
- (symlink (string-append bootstrap
- "/share/web/bootstrap/css/bootstrap.min.css")
- (string-append out "/static/bootstrap.min.css")))
+ (symlink (string-append jquery
+ "/share/web/jquery/jquery.slim.min.js")
+ (string-append out "/static/jquery.slim.min.js"))
+ ;(symlink (string-append js-popper
+ ; "/share/web/popper/popper.min.js")
+ ; (string-append out "/static/popper.min.js"))
+ )
#t))
(add-after 'install 'wrap-executable
(lambda* (#:key inputs outputs #:allow-other-keys)
(let ((out (assoc-ref outputs "out"))
(path (getenv "PYTHONPATH")))
(wrap-program (string-append out "/server.py")
- `("PATH" ":" prefix (,(dirname (which "edirect.pl"))))
- `("PYTHONPATH" ":" prefix (,path))))
+ `("PATH" ":" prefix (,(dirname (which "edirect.pl"))
+ ,(dirname (which "dirname"))
+ ,(dirname (which "grep"))
+ ,(dirname (which "sed"))))
+ `("PYTHONPATH" ":" prefix (,path))))
#t)))))
(inputs
- `(("edirect" ,edirect-gn)
+ `(("edirect" ,edirect)
("inetutils" ,inetutils)
- ("python-flask" ,python-flask)
- ("python-nltk" ,python-nltk)))
+ ("python-bcrypt" ,python-bcrypt)
+ ("python-flask-sqlalchemy" ,python-flask-sqlalchemy)
+ ("python-keras" ,python-keras-for-ratspub)
+ ("python-nltk" ,python-nltk)
+ ("tensorflow" ,tensorflow)))
(native-inputs
- `(("cytoscape" ,javascript-cytoscape)
- ("bootstrap" ,web-bootstrap)))
+ `(("bootstrap" ,web-bootstrap)
+ ("cytoscape" ,javascript-cytoscape)
+ ("font-awesome" ,web-font-awesome)
+ ("jquery" ,web-jquery)
+ ;("js-popper" ,js-popper) ; empty output
+ ))
(home-page "http://rats.pub/")
(synopsis "Relationship with Addiction Through Searches of PubMed")
(description
@@ -86,4 +138,113 @@ the question \"What do we know about these genes and addiction?\". Data from
@acronym{EBI GWAS, European Bioinformatics Institute Genome-Wide Association
Studies} catalog are also included in the search to better answer this
question.")
- (license #f)))
+ (license license:expat)))
+
+;; We want a copy of python-keras with the AUC optimizer backported.
+;; We skip the tests because we "test in production".
+;; That's a lie. The test suite just takes a long time to run.
+(define-public python-keras-for-ratspub
+ (hidden-package
+ (package
+ (inherit python-keras)
+ (source
+ (origin
+ (inherit (package-source python-keras))
+ (patches (search-patches "keras-auc-optimizer.patch"))))
+ (arguments
+ (substitute-keyword-arguments (package-arguments python-keras)
+ ((#:phases phases)
+ `(modify-phases ,phases
+ (delete 'check))))))))
+
+(define-public hrdp-project
+ (package
+ (name "hrdp-project")
+ (version "0.1")
+ (source (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/noderboarder/hrdp-project")
+ (commit (string-append "v" version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ "1ag7jm43p35yh0cqcn53wg4iw7sgfypw10mxq5klxvhgj3r6cf7i"))))
+ (build-system python-build-system)
+ (arguments
+ `(#:tests? #f ; no test suite
+ #:phases
+ (modify-phases %standard-phases
+ (delete 'configure)
+ (delete 'build)
+ (add-after 'unpack 'patch-sources
+ (lambda _
+ (substitute* "./app/templates/layout.html"
+ (("https://.*.bootstrapcdn.com/bootstrap/4.*/css/bootstrap.min.css.*")
+ "/static/bootstrap.min.css\">\n")
+ (("https://.*.bootstrapcdn.com/bootstrap/4.*/js/bootstrap.min.js.*")
+ "/static/bootstrap.min.js\"></script>\n")
+ (("https://code.jquery.com/jquery-3.*.slim.min.js.*")
+ "/static/jquery.slim.min.js\"></script>\n")
+ ;(("https://cdn.jsdelivr.net/npm/popper.js@1.16.0/dist/umd/popper.min.js.*")
+ ; "/static/popper.min.js\"></script>\n")
+ )
+ #t))
+ (replace 'install
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ (let ((out (assoc-ref outputs "out"))
+ (python (assoc-ref inputs "python")))
+ (delete-file "main.py")
+ (with-output-to-file "main.py"
+ (lambda ()
+ (format #t "#!~a/bin/python
+from app import create_app
+
+app = create_app()
+
+if __name__ == '__main__':
+ app.run(debug=True, port=4222)~%"
+ python)))
+ (chmod "main.py" #o555)
+ (copy-recursively "." out))
+ #t))
+ (add-after 'install 'install-javascript
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ (let ((out (assoc-ref outputs "out"))
+ (bootstrap (assoc-ref inputs "bootstrap"))
+ (jquery (assoc-ref inputs "jquery"))
+ ;(js-popper (assoc-ref inputs "js-popper"))
+ )
+ (symlink (string-append bootstrap
+ "/share/web/bootstrap/css/bootstrap.min.css")
+ (string-append out "/app/static/bootstrap.min.css"))
+ (symlink (string-append bootstrap
+ "/share/web/bootstrap/js/bootstrap.min.js")
+ (string-append out "/app/static/bootstrap.min.js"))
+ (symlink (string-append jquery
+ "/share/web/jquery/jquery.slim.min.js")
+ (string-append out "/app/static/jquery.slim.min.js"))
+ ;(symlink (string-append js-popper
+ ; "/share/web/popper/popper.min.js")
+ ; (string-append out "/static/popper.min.js"))
+ )
+ #t))
+ (add-after 'install 'wrap-executable
+ (lambda* (#:key inputs outputs #:allow-other-keys)
+ (let ((out (assoc-ref outputs "out"))
+ (path (getenv "PYTHONPATH")))
+ (wrap-program (string-append out "/main.py")
+ `("PYTHONPATH" ":" prefix (,path))))
+ #t)))))
+ (inputs
+ `(("python" ,python)
+ ("python-flask-sqlalchemy" ,python-flask-sqlalchemy)))
+ (native-inputs
+ `(("bootstrap" ,web-bootstrap)
+ ("jquery" ,web-jquery)
+ ;("js-popper" ,js-popper) ; empty output
+ ))
+ (home-page "https://github.com/noderboarder/hrdp-project")
+ (synopsis "")
+ (description "")
+ (license license:expat)))
diff --git a/gn/services/archive-pubmed.service b/gn/services/archive-pubmed.service
new file mode 100644
index 0000000..fd00b71
--- /dev/null
+++ b/gn/services/archive-pubmed.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Download PubMed Archives
+After=network-online.target
+Wants=network-online.target
+AssertPathExists=/export2/PubMed
+
+[Service]
+ExecStart=/usr/local/guix-profiles/ratspub/bin/archive-pubmed
+Environment="EDIRECT_PUBMED_MASTER=/export2/PubMed" "PERL_LWP_SSL_CA_FILE=/etc/ssl/certs/ca-certificates.crt" "PATH=/usr/local/guix-profiles/ratspub/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/usr/sbin"
+User=hchen
+
+[Install]
+WantedBy=multi-user.target
diff --git a/gn/services/archive-pubmed.timer b/gn/services/archive-pubmed.timer
new file mode 100644
index 0000000..4bb27ba
--- /dev/null
+++ b/gn/services/archive-pubmed.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Daily PubMed downloads
+
+[Timer]
+OnBootSec=5h
+OnUnitInactiveSec=15h
+
+[Install]
+WantedBy=default.target
diff --git a/gn/services/bnw-README b/gn/services/bnw-README
index ab7de53..d70b7fc 100644
--- a/gn/services/bnw-README
+++ b/gn/services/bnw-README
@@ -6,8 +6,8 @@ $ cd guix-bioinformatics && git pull
# systemctl enable --now bnw.service
for testing changes to the container, you'll want to do the following (or thereabouts):
-$ guix system container gn/services/GN1-container.scm --share=/home/bnw/server=/srv/http --network
-$ sudo -E $(guix system container gn/services/GN1-container.scm --share=/home/bnw/server=/srv/http --network)
+$ guix system container gn/servicebnwN1-container.scm --network
+$ sudo -E $(guix system container gn/services/bnw-container.scm --network)
for running the service:
see included bnw.service
diff --git a/gn/services/bnw-container.scm b/gn/services/bnw-container.scm
index 8463a75..39da38d 100644
--- a/gn/services/bnw-container.scm
+++ b/gn/services/bnw-container.scm
@@ -21,10 +21,6 @@
(match-lambda
(($ <bnw-configuration> package deploy-directory port)
#~(begin
- (use-modules (guix build utils))
- (when (directory-exists? #$deploy-directory)
- ;; Not 'delete-file-recursively' because the directory might be empty.
- (system* "rm" "-r" #$(string-append deploy-directory "/*")))
(mkdir-p #$deploy-directory)
(copy-recursively #$package #$deploy-directory)
(invoke #$(file-append coreutils "/bin/chmod") "a+w"
diff --git a/gn/services/bnw.service b/gn/services/bnw.service
index 5ba8f3f..6ac99a1 100644
--- a/gn/services/bnw.service
+++ b/gn/services/bnw.service
@@ -3,7 +3,7 @@ Description=BNW web server
Wants=guix-daemon.service
[Service]
-Environment="BNW_COMMAND=$(/bin/su -l bnw -c '/var/guix/profiles/per-user/bnw/current-guix/bin/guix system container /home/bnw/guix-bioinformatics/gn/services/bnw-container.scm --share=/home/bnw/server=/srv/http --share=/home/bnw/server/var-log=/var/log --network')"
+Environment="BNW_COMMAND=$(/bin/su -l bnw -c '/var/guix/profiles/per-user/bnw/current-guix/bin/guix system container /home/bnw/guix-bioinformatics/gn/services/bnw-container.scm --share=/home/bnw/server/var-log=/var/log --network')"
ExecStart=/bin/bash -c '${BNW_COMMAND}'
[Install]
diff --git a/gn/services/genenetwork.scm b/gn/services/genenetwork.scm
index 0978cd2..4693152 100644
--- a/gn/services/genenetwork.scm
+++ b/gn/services/genenetwork.scm
@@ -2,13 +2,13 @@
(use-modules (gnu)
(gn packages genenetwork)
- (gn packages python)
+ (gn packages python24)
(gn packages web))
(use-service-modules web)
(use-package-modules python)
(define %mod-python-path
- (file-append mod-python "/lib/python2.7/site-packages"))
+ (file-append mod-python "/lib/python2.4/site-packages"))
(operating-system
(host-name "genenetwork")
@@ -22,15 +22,15 @@
;; No firmware for VMs
(firmware '())
- (packages (cons* python-2
+ (packages (cons* python-2.4
mod-python
- python2-qtlreaper
- python2-htmlgen-gn
- python2-json-GN1
- python2-piddle
- python2-pyx-GN1
- python2-pyxlwriter
- python2-svg-GN1
+ python24-qtlreaper
+ ;python24-htmlgen-gn
+ python24-json-GN1
+ python24-piddle
+ python24-pyx-GN1
+ python24-pyxlwriter
+ python24-svg-GN1
%base-packages))
(services (list (service httpd-service-type
@@ -38,7 +38,7 @@
(config
(httpd-config-file
(server-name "www.genenetwork.org")
- (document-root (file-append genenetwork "/web"))
+ (document-root (file-append genenetwork1 "/web"))
(listen '("8811"))
(modules (cons*
(httpd-module
@@ -46,8 +46,8 @@
(file (file-append mod-python "/modules/mod_python.so")))
%default-httpd-modules))
(extra-config (list "\
-PythonPath \"sys.path+['" %mod-python-path "', '" (file-append genenetwork "/web/webqtl") "']\"
-<Directory " (file-append genenetwork "/web/webqtl") ">
+PythonPath \"sys.path+['" %mod-python-path "', '" (file-append genenetwork1 "/web/webqtl") "']\"
+<Directory " (file-append genenetwork1 "/web/webqtl") ">
SetHandler python-program
PythonHandler mod_python.publisher
PythonAutoReload Off
diff --git a/keras-auc-optimizer.patch b/keras-auc-optimizer.patch
new file mode 100644
index 0000000..bbc6924
--- /dev/null
+++ b/keras-auc-optimizer.patch
@@ -0,0 +1,1133 @@
+From 901159da45695da24a5206125910f02fc50169ce Mon Sep 17 00:00:00 2001
+From: Efraim Flashner <efraim@flashner.co.il>
+Date: Thu, 23 Apr 2020 15:50:37 +0300
+Subject: [PATCH] add keras metrics
+
+---
+ keras/backend/tensorflow_backend.py | 12 +
+ keras/metrics.py | 584 ++++++++++++++++++++++++++++
+ keras/utils/__init__.py | 2 +
+ keras/utils/losses_utils.py | 177 +++++++++
+ keras/utils/metrics_utils.py | 278 +++++++++++++
+ 5 files changed, 1053 insertions(+)
+ create mode 100644 keras/utils/losses_utils.py
+ create mode 100644 keras/utils/metrics_utils.py
+
+diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
+index bcb8be0..a2870f5 100644
+--- a/keras/backend/tensorflow_backend.py
++++ b/keras/backend/tensorflow_backend.py
+@@ -4453,3 +4453,15 @@ def local_conv2d(inputs, kernel, kernel_size, strides, output_shape, data_format
+ else:
+ output = permute_dimensions(output, (2, 0, 1, 3))
+ return output
++
++#get_graph = tf_keras_backend.get_graph
++
++#def is_symbolic(x):
++# return isinstance(x, tf.Tensor) and hasattr(x, 'op')
++
++def size(x, name=None):
++# if is_symbolic(x):
++# with get_graph().as_default():
++# return tf.size(x)
++ return tf.size(x, name=name)
++
+diff --git a/keras/metrics.py b/keras/metrics.py
+index 8e3df1f..8f57910 100644
+--- a/keras/metrics.py
++++ b/keras/metrics.py
+@@ -4,8 +4,12 @@ from __future__ import absolute_import
+ from __future__ import division
+ from __future__ import print_function
+
++import abc
+ import six
++import types
++
+ from . import backend as K
++from .engine.base_layer import Layer
+ from .losses import mean_squared_error
+ from .losses import mean_absolute_error
+ from .losses import mean_absolute_percentage_error
+@@ -19,10 +23,201 @@ from .losses import binary_crossentropy
+ from .losses import kullback_leibler_divergence
+ from .losses import poisson
+ from .losses import cosine_proximity
++from .utils import losses_utils
++from .utils import metrics_utils
+ from .utils.generic_utils import deserialize_keras_object
+ from .utils.generic_utils import serialize_keras_object
+
+
++@six.add_metaclass(abc.ABCMeta)
++class Metric(Layer):
++ """Encapsulates metric logic and state.
++
++ Standalone usage:
++ ```python
++ m = SomeMetric(...)
++ for input in ...:
++ m.update_state(input)
++ m.result()
++ ```
++
++ Usage with the `compile` API:
++ ```python
++ model.compile(optimizer='rmsprop',
++ loss=keras.losses.categorical_crossentropy,
++ metrics=[keras.metrics.CategoricalAccuracy()])
++ ```
++
++ To be implemented by subclasses:
++ * `__init__()`: All state variables should be created in this method by
++ calling `self.add_weight()` like: `self.var = self.add_weight(...)`
++ * `update_state()`: Has all updates to the state variables like:
++ self.var.assign_add(...).
++ * `result()`: Computes and returns a value for the metric
++ from the state variables.
++ """
++
++ def __init__(self, name=None, dtype=None, **kwargs):
++ super(Metric, self).__init__(name=name, dtype=dtype, **kwargs)
++ self.stateful = True # All metric layers are stateful.
++ self.built = True
++ self.dtype = K.floatx() if dtype is None else dtype
++
++ def __new__(cls, *args, **kwargs):
++ obj = super(Metric, cls).__new__(cls)
++ update_state_fn = obj.update_state
++
++ obj.update_state = types.MethodType(
++ metrics_utils.update_state_wrapper(update_state_fn), obj)
++ return obj
++
++ def __call__(self, *args, **kwargs):
++ """Accumulates statistics and then computes metric result value."""
++ update_op = self.update_state(*args, **kwargs)
++ return self.result()
++
++ def get_config(self):
++ """Returns the serializable config of the metric."""
++ return {'name': self.name, 'dtype': self.dtype}
++
++ def reset_states(self):
++ """Resets all of the metric state variables.
++ This function is called between epochs/steps,
++ when a metric is evaluated during training.
++ """
++ K.batch_set_value([(v, 0) for v in self.weights])
++
++ @abc.abstractmethod
++ def update_state(self, *args, **kwargs):
++ """Accumulates statistics for the metric. """
++ raise NotImplementedError('Must be implemented in subclasses.')
++
++ @abc.abstractmethod
++ def result(self):
++ """Computes and returns the metric value tensor.
++ Result computation is an idempotent operation that simply calculates the
++ metric value using the state variables.
++ """
++ raise NotImplementedError('Must be implemented in subclasses.')
++
++ # For use by subclasses #
++ def add_weight(self,
++ name,
++ shape=(),
++ initializer=None,
++ dtype=None):
++ """Adds state variable. Only for use by subclasses."""
++ return super(Metric, self).add_weight(
++ name=name,
++ shape=shape,
++ dtype=self.dtype if dtype is None else dtype,
++ trainable=False,
++ initializer=initializer)
++
++ # End: For use by subclasses ###
++
++
++class Reduce(Metric):
++ """Encapsulates metrics that perform a reduce operation on the values."""
++
++ def __init__(self, reduction, name, dtype=None):
++ """Creates a `Reduce` instance.
++ # Arguments
++ reduction: a metrics `Reduction` enum value.
++ name: string name of the metric instance.
++ dtype: (Optional) data type of the metric result.
++ """
++ super(Reduce, self).__init__(name=name, dtype=dtype)
++ self.reduction = reduction
++ self.total = self.add_weight('total', initializer='zeros')
++ if reduction in [metrics_utils.Reduction.SUM_OVER_BATCH_SIZE,
++ metrics_utils.Reduction.WEIGHTED_MEAN]:
++ self.count = self.add_weight('count', initializer='zeros')
++
++ def update_state(self, values, sample_weight=None):
++ """Accumulates statistics for computing the reduction metric.
++ For example, if `values` is [1, 3, 5, 7] and reduction=SUM_OVER_BATCH_SIZE,
++ then the value of `result()` is 4. If the `sample_weight` is specified as
++ [1, 1, 0, 0] then value of `result()` would be 2.
++ # Arguments
++ values: Per-example value.
++ sample_weight: Optional weighting of each example. Defaults to 1.
++ """
++ values = K.cast(values, self.dtype)
++ if sample_weight is not None:
++ sample_weight = K.cast(sample_weight, self.dtype)
++ # Update dimensions of weights to match with values if possible.
++ values, _, sample_weight = losses_utils.squeeze_or_expand_dimensions(
++ values, sample_weight=sample_weight)
++
++ # Broadcast weights if possible.
++ sample_weight = losses_utils.broadcast_weights(sample_weight, values)
++ values = values * sample_weight
++
++ value_sum = K.sum(values)
++ update_total_op = K.update_add(self.total, value_sum)
++
++ # Exit early if the reduction doesn't have a denominator.
++ if self.reduction == metrics_utils.Reduction.SUM:
++ return update_total_op
++
++ # Update `count` for reductions that require a denominator.
++ if self.reduction == metrics_utils.Reduction.SUM_OVER_BATCH_SIZE:
++ num_values = K.cast(K.size(values), self.dtype)
++ elif self.reduction == metrics_utils.Reduction.WEIGHTED_MEAN:
++ if sample_weight is None:
++ num_values = K.cast(K.size(values), self.dtype)
++ else:
++ num_values = K.sum(sample_weight)
++ else:
++ raise NotImplementedError(
++ 'reduction [%s] not implemented' % self.reduction)
++
++ with K.control_dependencies([update_total_op]):
++ return K.update_add(self.count, num_values)
++
++ def result(self):
++ if self.reduction == metrics_utils.Reduction.SUM:
++ return self.total
++ elif self.reduction in [
++ metrics_utils.Reduction.WEIGHTED_MEAN,
++ metrics_utils.Reduction.SUM_OVER_BATCH_SIZE
++ ]:
++ return self.total / self.count
++ else:
++ raise NotImplementedError(
++ 'reduction [%s] not implemented' % self.reduction)
++
++
++class Sum(Reduce):
++ """Computes the (weighted) sum of the given values.
++
++ For example, if values is [1, 3, 5, 7] then the sum is 16.
++ If the weights were specified as [1, 1, 0, 0] then the sum would be 4.
++
++ This metric creates one variable, `total`, that is used to compute the sum of
++ `values`. This is ultimately returned as `sum`.
++ If `sample_weight` is `None`, weights default to 1. Use `sample_weight` of 0
++ to mask values.
++
++ Standalone usage:
++ ```python
++ m = keras.metrics.Sum()
++ m.update_state([1, 3, 5, 7])
++ m.result()
++ ```
++ """
++
++ def __init__(self, name='sum', dtype=None):
++ """Creates a `Sum` instance.
++ # Arguments
++ name: (Optional) string name of the metric instance.
++ dtype: (Optional) data type of the metric result.
++ """
++ super(Sum, self).__init__(reduction=metrics_utils.Reduction.SUM,
++ name=name, dtype=dtype)
++
++
+ def binary_accuracy(y_true, y_pred):
+ return K.mean(K.equal(y_true, K.round(y_pred)), axis=-1)
+
+@@ -49,6 +244,395 @@ def sparse_top_k_categorical_accuracy(y_true, y_pred, k=5):
+ return K.mean(K.in_top_k(y_pred, K.cast(K.flatten(y_true), 'int32'), k),
+ axis=-1)
+
++class SensitivitySpecificityBase(Metric):
++ """Abstract base class for computing sensitivity and specificity.
++
++ For additional information about specificity and sensitivity, see the
++ following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity
++ """
++
++ def __init__(self, value, num_thresholds=200, name=None, dtype=None):
++ super(SensitivitySpecificityBase, self).__init__(name=name, dtype=dtype)
++ if num_thresholds <= 0:
++ raise ValueError('`num_thresholds` must be > 0.')
++ self.value = value
++ self.true_positives = self.add_weight(
++ 'true_positives',
++ shape=(num_thresholds,),
++ initializer='zeros')
++ self.true_negatives = self.add_weight(
++ 'true_negatives',
++ shape=(num_thresholds,),
++ initializer='zeros')
++ self.false_positives = self.add_weight(
++ 'false_positives',
++ shape=(num_thresholds,),
++ initializer='zeros')
++ self.false_negatives = self.add_weight(
++ 'false_negatives',
++ shape=(num_thresholds,),
++ initializer='zeros')
++
++ # Compute `num_thresholds` thresholds in [0, 1]
++ if num_thresholds == 1:
++ self.thresholds = [0.5]
++ else:
++ thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
++ for i in range(num_thresholds - 2)]
++ self.thresholds = [0.0] + thresholds + [1.0]
++
++ def update_state(self, y_true, y_pred, sample_weight=None):
++ return metrics_utils.update_confusion_matrix_variables(
++ {
++ metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
++ metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives,
++ metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,
++ metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,
++ },
++ y_true,
++ y_pred,
++ thresholds=self.thresholds,
++ sample_weight=sample_weight)
++
++ def reset_states(self):
++ num_thresholds = len(self.thresholds)
++ K.batch_set_value(
++ [(v, np.zeros((num_thresholds,))) for v in self.variables])
++
++
++class SensitivityAtSpecificity(SensitivitySpecificityBase):
++ """Computes the sensitivity at a given specificity.
++
++ `Sensitivity` measures the proportion of actual positives that are correctly
++ identified as such (tp / (tp + fn)).
++ `Specificity` measures the proportion of actual negatives that are correctly
++ identified as such (tn / (tn + fp)).
++
++ This metric creates four local variables, `true_positives`, `true_negatives`,
++ `false_positives` and `false_negatives` that are used to compute the
++ sensitivity at the given specificity. The threshold for the given specificity
++ value is computed and used to evaluate the corresponding sensitivity.
++
++ If `sample_weight` is `None`, weights default to 1.
++ Use `sample_weight` of 0 to mask values.
++
++ For additional information about specificity and sensitivity, see the
++ following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity
++
++ Usage with the compile API:
++
++ ```python
++ model = keras.Model(inputs, outputs)
++ model.compile(
++ 'sgd',
++ loss='mse',
++ metrics=[keras.metrics.SensitivityAtSpecificity()])
++ ```
++
++ # Arguments
++ specificity: A scalar value in range `[0, 1]`.
++ num_thresholds: (Optional) Defaults to 200. The number of thresholds to
++ use for matching the given specificity.
++ name: (Optional) string name of the metric instance.
++ dtype: (Optional) data type of the metric result.
++ """
++
++ def __init__(self, specificity, num_thresholds=200, name=None, dtype=None):
++ if specificity < 0 or specificity > 1:
++ raise ValueError('`specificity` must be in the range [0, 1].')
++ self.specificity = specificity
++ self.num_thresholds = num_thresholds
++ super(SensitivityAtSpecificity, self).__init__(
++ specificity, num_thresholds=num_thresholds, name=name, dtype=dtype)
++
++ def result(self):
++ # Calculate specificities at all the thresholds.
++ specificities = K.switch(
++ K.greater(self.true_negatives + self.false_positives, 0),
++ (self.true_negatives / (self.true_negatives + self.false_positives)),
++ K.zeros_like(self.thresholds))
++
++ # Find the index of the threshold where the specificity is closest to the
++ # given specificity.
++ min_index = K.argmin(
++ K.abs(specificities - self.value), axis=0)
++ min_index = K.cast(min_index, 'int32')
++
++ # Compute sensitivity at that index.
++ return K.switch(
++ K.greater((self.true_positives[min_index] +
++ self.false_negatives[min_index]), 0),
++ (self.true_positives[min_index] /
++ (self.true_positives[min_index] + self.false_negatives[min_index])),
++ K.zeros_like(self.true_positives[min_index]))
++
++ def get_config(self):
++ config = {
++ 'num_thresholds': self.num_thresholds,
++ 'specificity': self.specificity
++ }
++ base_config = super(SensitivityAtSpecificity, self).get_config()
++ return dict(list(base_config.items()) + list(config.items()))
++
++
++class AUC(Metric):
++ """Computes the approximate AUC (Area under the curve) via a Riemann sum.
++
++ This metric creates four local variables, `true_positives`, `true_negatives`,
++ `false_positives` and `false_negatives` that are used to compute the AUC.
++ To discretize the AUC curve, a linearly spaced set of thresholds is used to
++ compute pairs of recall and precision values. The area under the ROC-curve is
++ therefore computed using the height of the recall values by the false positive
++ rate, while the area under the PR-curve is the computed using the height of
++ the precision values by the recall.
++
++ This value is ultimately returned as `auc`, an idempotent operation that
++ computes the area under a discretized curve of precision versus recall values
++ (computed using the aforementioned variables). The `num_thresholds` variable
++ controls the degree of discretization with larger numbers of thresholds more
++ closely approximating the true AUC. The quality of the approximation may vary
++ dramatically depending on `num_thresholds`. The `thresholds` parameter can be
++ used to manually specify thresholds which split the predictions more evenly.
++
++ For best results, `predictions` should be distributed approximately uniformly
++ in the range [0, 1] and not peaked around 0 or 1. The quality of the AUC
++ approximation may be poor if this is not the case. Setting `summation_method`
++ to 'minoring' or 'majoring' can help quantify the error in the approximation
++ by providing lower or upper bound estimate of the AUC.
++
++ If `sample_weight` is `None`, weights default to 1.
++ Use `sample_weight` of 0 to mask values.
++
++ Usage with the compile API:
++
++ ```python
++ model = keras.Model(inputs, outputs)
++ model.compile('sgd', loss='mse', metrics=[keras.metrics.AUC()])
++ ```
++
++ # Arguments
++ num_thresholds: (Optional) Defaults to 200. The number of thresholds to
++ use when discretizing the roc curve. Values must be > 1.
++ curve: (Optional) Specifies the name of the curve to be computed, 'ROC'
++ [default] or 'PR' for the Precision-Recall-curve.
++ summation_method: (Optional) Specifies the Riemann summation method used
++ (https://en.wikipedia.org/wiki/Riemann_sum): 'interpolation' [default],
++ applies mid-point summation scheme for `ROC`. For PR-AUC, interpolates
++ (true/false) positives but not the ratio that is precision (see Davis
++ & Goadrich 2006 for details); 'minoring' that applies left summation
++ for increasing intervals and right summation for decreasing intervals;
++ 'majoring' that does the opposite.
++ name: (Optional) string name of the metric instance.
++ dtype: (Optional) data type of the metric result.
++ thresholds: (Optional) A list of floating point values to use as the
++ thresholds for discretizing the curve. If set, the `num_thresholds`
++ parameter is ignored. Values should be in [0, 1]. Endpoint thresholds
++ equal to {-epsilon, 1+epsilon} for a small positive epsilon value will
++ be automatically included with these to correctly handle predictions
++ equal to exactly 0 or 1.
++ """
++
++ def __init__(self,
++ num_thresholds=200,
++ curve='ROC',
++ summation_method='interpolation',
++ name=None,
++ dtype=None,
++ thresholds=None):
++ # Validate configurations.
++ if (isinstance(curve, metrics_utils.AUCCurve) and
++ curve not in list(metrics_utils.AUCCurve)):
++ raise ValueError('Invalid curve: "{}". Valid options are: "{}"'.format(
++ curve, list(metrics_utils.AUCCurve)))
++ if isinstance(
++ summation_method,
++ metrics_utils.AUCSummationMethod) and summation_method not in list(
++ metrics_utils.AUCSummationMethod):
++ raise ValueError(
++ 'Invalid summation method: "{}". Valid options are: "{}"'.format(
++ summation_method, list(metrics_utils.AUCSummationMethod)))
++
++ # Update properties.
++ if thresholds is not None:
++ # If specified, use the supplied thresholds.
++ self.num_thresholds = len(thresholds) + 2
++ thresholds = sorted(thresholds)
++ else:
++ if num_thresholds <= 1:
++ raise ValueError('`num_thresholds` must be > 1.')
++
++ # Otherwise, linearly interpolate (num_thresholds - 2) thresholds in
++ # (0, 1).
++ self.num_thresholds = num_thresholds
++ thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
++ for i in range(num_thresholds - 2)]
++
++ # Add an endpoint "threshold" below zero and above one for either
++ # threshold method to account for floating point imprecisions.
++ self.thresholds = [0.0 - K.epsilon()] + thresholds + [1.0 + K.epsilon()]
++
++ if isinstance(curve, metrics_utils.AUCCurve):
++ self.curve = curve
++ else:
++ self.curve = metrics_utils.AUCCurve.from_str(curve)
++ if isinstance(summation_method, metrics_utils.AUCSummationMethod):
++ self.summation_method = summation_method
++ else:
++ self.summation_method = metrics_utils.AUCSummationMethod.from_str(
++ summation_method)
++ super(AUC, self).__init__(name=name, dtype=dtype)
++
++ # Create metric variables
++ self.true_positives = self.add_weight(
++ 'true_positives',
++ shape=(self.num_thresholds,),
++ initializer='zeros')
++ self.true_negatives = self.add_weight(
++ 'true_negatives',
++ shape=(self.num_thresholds,),
++ initializer='zeros')
++ self.false_positives = self.add_weight(
++ 'false_positives',
++ shape=(self.num_thresholds,),
++ initializer='zeros')
++ self.false_negatives = self.add_weight(
++ 'false_negatives',
++ shape=(self.num_thresholds,),
++ initializer='zeros')
++
++ def update_state(self, y_true, y_pred, sample_weight=None):
++ return metrics_utils.update_confusion_matrix_variables({
++ metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
++ metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives,
++ metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,
++ metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,
++ }, y_true, y_pred, self.thresholds, sample_weight=sample_weight)
++
++ def interpolate_pr_auc(self):
++ """Interpolation formula inspired by section 4 of Davis & Goadrich 2006.
++
++ https://www.biostat.wisc.edu/~page/rocpr.pdf
++
++ Note here we derive & use a closed formula not present in the paper
++ as follows:
++
++ Precision = TP / (TP + FP) = TP / P
++
++ Modeling all of TP (true positive), FP (false positive) and their sum
++ P = TP + FP (predicted positive) as varying linearly within each interval
++ [A, B] between successive thresholds, we get
++
++ Precision slope = dTP / dP
++ = (TP_B - TP_A) / (P_B - P_A)
++ = (TP - TP_A) / (P - P_A)
++ Precision = (TP_A + slope * (P - P_A)) / P
++
++ The area within the interval is (slope / total_pos_weight) times
++
++ int_A^B{Precision.dP} = int_A^B{(TP_A + slope * (P - P_A)) * dP / P}
++ int_A^B{Precision.dP} = int_A^B{slope * dP + intercept * dP / P}
++
++ where intercept = TP_A - slope * P_A = TP_B - slope * P_B, resulting in
++
++ int_A^B{Precision.dP} = TP_B - TP_A + intercept * log(P_B / P_A)
++
++ Bringing back the factor (slope / total_pos_weight) we'd put aside, we get
++
++ slope * [dTP + intercept * log(P_B / P_A)] / total_pos_weight
++
++ where dTP == TP_B - TP_A.
++
++ Note that when P_A == 0 the above calculation simplifies into
++
++ int_A^B{Precision.dTP} = int_A^B{slope * dTP} = slope * (TP_B - TP_A)
++
++ which is really equivalent to imputing constant precision throughout the
++ first bucket having >0 true positives.
++
++ # Returns
++ pr_auc: an approximation of the area under the P-R curve.
++ """
++ dtp = self.true_positives[:self.num_thresholds -
++ 1] - self.true_positives[1:]
++ p = self.true_positives + self.false_positives
++ dp = p[:self.num_thresholds - 1] - p[1:]
++
++ prec_slope = dtp / K.maximum(dp, 0)
++ intercept = self.true_positives[1:] - (prec_slope * p[1:])
++
++ # Logical and
++ pMin = K.expand_dims(p[:self.num_thresholds - 1] > 0, 0)
++ pMax = K.expand_dims(p[1:] > 0, 0)
++ are_different = K.concatenate([pMin, pMax], axis=0)
++ switch_condition = K.all(are_different, axis=0)
++
++ safe_p_ratio = K.switch(
++ switch_condition,
++ p[:self.num_thresholds - 1] / K.maximum(p[1:], 0),
++ K.ones_like(p[1:]))
++
++ numer = prec_slope * (dtp + intercept * K.log(safe_p_ratio))
++ denom = K.maximum(self.true_positives[1:] + self.false_negatives[1:], 0)
++ return K.sum((numer / denom))
++
++ def result(self):
++ if (self.curve == metrics_utils.AUCCurve.PR and
++ (self.summation_method ==
++ metrics_utils.AUCSummationMethod.INTERPOLATION)):
++ # This use case is different and is handled separately.
++ return self.interpolate_pr_auc()
++
++ # Set `x` and `y` values for the curves based on `curve` config.
++ recall = K.switch(
++ K.greater((self.true_positives), 0),
++ (self.true_positives /
++ (self.true_positives + self.false_negatives)),
++ K.zeros_like(self.true_positives))
++ if self.curve == metrics_utils.AUCCurve.ROC:
++ fp_rate = K.switch(
++ K.greater((self.false_positives), 0),
++ (self.false_positives /
++ (self.false_positives + self.true_negatives)),
++ K.zeros_like(self.false_positives))
++ x = fp_rate
++ y = recall
++ else: # curve == 'PR'.
++ precision = K.switch(
++ K.greater((self.true_positives), 0),
++ (self.true_positives / (self.true_positives + self.false_positives)),
++ K.zeros_like(self.true_positives))
++ x = recall
++ y = precision
++
++ # Find the rectangle heights based on `summation_method`.
++ if self.summation_method == metrics_utils.AUCSummationMethod.INTERPOLATION:
++ # Note: the case ('PR', 'interpolation') has been handled above.
++ heights = (y[:self.num_thresholds - 1] + y[1:]) / 2.
++ elif self.summation_method == metrics_utils.AUCSummationMethod.MINORING:
++ heights = K.minimum(y[:self.num_thresholds - 1], y[1:])
++ else: # self.summation_method = metrics_utils.AUCSummationMethod.MAJORING:
++ heights = K.maximum(y[:self.num_thresholds - 1], y[1:])
++
++ # Sum up the areas of all the rectangles.
++ return K.sum((x[:self.num_thresholds - 1] - x[1:]) * heights)
++
++ def reset_states(self):
++ K.batch_set_value(
++ [(v, np.zeros((self.num_thresholds,))) for v in self.variables])
++
++ def get_config(self):
++ config = {
++ 'num_thresholds': self.num_thresholds,
++ 'curve': self.curve.value,
++ 'summation_method': self.summation_method.value,
++ # We remove the endpoint thresholds as an inverse of how the thresholds
++ # were initialized. This ensures that a metric initialized from this
++ # config has the same thresholds.
++ 'thresholds': self.thresholds[1:-1],
++ }
++ base_config = super(AUC, self).get_config()
++ return dict(list(base_config.items()) + list(config.items()))
++
+
+ # Aliases
+
+diff --git a/keras/utils/__init__.py b/keras/utils/__init__.py
+index 8cc39d5..65af329 100644
+--- a/keras/utils/__init__.py
++++ b/keras/utils/__init__.py
+@@ -4,6 +4,8 @@ from . import generic_utils
+ from . import data_utils
+ from . import io_utils
+ from . import conv_utils
++from . import losses_utils
++from . import metrics_utils
+
+ # Globally-importable utils.
+ from .io_utils import HDF5Matrix
+diff --git a/keras/utils/losses_utils.py b/keras/utils/losses_utils.py
+new file mode 100644
+index 0000000..617ebb7
+--- /dev/null
++++ b/keras/utils/losses_utils.py
+@@ -0,0 +1,177 @@
++"""Utilities related to losses."""
++from __future__ import absolute_import
++from __future__ import division
++from __future__ import print_function
++
++import numpy as np
++
++from .. import backend as K
++
++
++class Reduction(object):
++ """Types of loss reduction.
++
++ Contains the following values:
++
++ * `NONE`: Un-reduced weighted losses with the same shape as input. When this
++ reduction type used with built-in Keras training loops like
++ `fit`/`evaluate`, the unreduced vector loss is passed to the optimizer but
++ the reported loss will be a scalar value.
++ * `SUM`: Scalar sum of weighted losses.
++ * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
++ """
++
++ NONE = 'none'
++ SUM = 'sum'
++ SUM_OVER_BATCH_SIZE = 'sum_over_batch_size'
++
++ @classmethod
++ def all(cls):
++ return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE)
++
++ @classmethod
++ def validate(cls, key):
++ if key not in cls.all():
++ raise ValueError('Invalid Reduction Key %s.' % key)
++
++
++def squeeze_or_expand_dimensions(y_pred, y_true=None, sample_weight=None):
++ """Squeeze or expand last dimension if needed.
++
++ 1. Squeezes last dim of `y_pred` or `y_true` if their rank differs by 1.
++ 2. Squeezes or expands last dim of `sample_weight` if its rank differs by 1
++ from the new rank of `y_pred`.
++ If `sample_weight` is scalar, it is kept scalar.
++
++ # Arguments
++ y_pred: Predicted values, a `Tensor` of arbitrary dimensions.
++ y_true: Optional label `Tensor` whose dimensions match `y_pred`.
++ sample_weight: Optional weight scalar or `Tensor` whose dimensions match
++ `y_pred`.
++
++ # Returns
++ Tuple of `y_pred`, `y_true` and `sample_weight`. Each of them possibly has
++ the last dimension squeezed, `sample_weight` could be extended by one
++ dimension.
++ """
++ if y_true is not None:
++ y_pred_rank = K.ndim(y_pred)
++ y_pred_shape = K.int_shape(y_pred)
++ y_true_rank = K.ndim(y_true)
++ y_true_shape = K.int_shape(y_true)
++
++ if (y_pred_rank - y_true_rank == 1) and (y_pred_shape[-1] == 1):
++ y_pred = K.squeeze(y_pred, -1)
++ elif (y_true_rank - y_pred_rank == 1) and (y_true_shape[-1] == 1):
++ y_true = K.squeeze(y_true, -1)
++
++ if sample_weight is None:
++ return y_pred, y_true
++
++ y_pred_rank = K.ndim(y_pred)
++ weights_rank = K.ndim(sample_weight)
++ if weights_rank != 0:
++ if weights_rank - y_pred_rank == 1:
++ sample_weight = K.squeeze(sample_weight, -1)
++ elif y_pred_rank - weights_rank == 1:
++ sample_weight = K.expand_dims(sample_weight, -1)
++ return y_pred, y_true, sample_weight
++
++
++def _num_elements(losses):
++ """Computes the number of elements in `losses` tensor."""
++ with K.name_scope('num_elements') as scope:
++ return K.cast(K.size(losses, name=scope), losses.dtype)
++
++
++def reduce_weighted_loss(weighted_losses, reduction=Reduction.SUM_OVER_BATCH_SIZE):
++ """Reduces the individual weighted loss measurements."""
++ if reduction == Reduction.NONE:
++ loss = weighted_losses
++ else:
++ loss = K.sum(weighted_losses)
++ if reduction == Reduction.SUM_OVER_BATCH_SIZE:
++ loss = loss / _num_elements(weighted_losses)
++ return loss
++
++
++def broadcast_weights(values, sample_weight):
++ # Broadcast weights if possible.
++ weights_shape = K.int_shape(sample_weight)
++ values_shape = K.int_shape(values)
++
++ if values_shape != weights_shape:
++ weights_rank = K.ndim(sample_weight)
++ values_rank = K.ndim(values)
++
++ # Raise error if ndim of weights is > values.
++ if weights_rank > values_rank:
++ raise ValueError(
++ 'Incompatible shapes: `values` {} vs `sample_weight` {}'.format(
++ values_shape, weights_shape))
++
++ # Expand dim of weights to match ndim of values, if required.
++ for i in range(weights_rank, values_rank):
++ sample_weight = K.expand_dims(sample_weight, axis=i)
++
++ if weights_shape is not None and values_shape is not None:
++ for i in range(weights_rank):
++ if (weights_shape[i] is not None and
++ values_shape[i] is not None and
++ weights_shape[i] != values_shape[i]):
++ # Cannot be broadcasted.
++ if weights_shape[i] != 1:
++ raise ValueError(
++ 'Incompatible shapes: `values` {} vs '
++ '`sample_weight` {}'.format(
++ values_shape, weights_shape))
++ sample_weight = K.repeat_elements(
++ sample_weight, values_shape[i], axis=i)
++ return sample_weight
++
++
++def compute_weighted_loss(losses,
++ sample_weight=None,
++ reduction=Reduction.SUM_OVER_BATCH_SIZE,
++ name=None):
++ """Computes the weighted loss.
++
++ # Arguments
++ losses: `Tensor` of shape `[batch_size, d1, ... dN]`.
++ sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as
++ ` losses`, or be broadcastable to `losses`.
++ reduction: (Optional) Type of Reduction to apply to loss.
++ Default value is `SUM_OVER_BATCH_SIZE`.
++ name: Optional name for the op.
++
++ # Raises
++ ValueError: If the shape of `sample_weight` is not compatible with `losses`.
++
++ # Returns
++ Weighted loss `Tensor` of the same type as `losses`. If `reduction` is
++ `NONE`, this has the same shape as `losses`; otherwise, it is scalar.
++ """
++ Reduction.validate(reduction)
++ if sample_weight is None:
++ sample_weight = 1.0
++ with K.name_scope(name or 'weighted_loss'):
++ input_dtype = K.dtype(losses)
++ losses = K.cast(losses, K.floatx())
++ sample_weight = K.cast(sample_weight, K.floatx())
++
++ # Update dimensions of `sample_weight` to match with `losses` if possible.
++ losses, _, sample_weight = squeeze_or_expand_dimensions(
++ losses, None, sample_weight)
++
++ # Broadcast weights if possible.
++ sample_weight = broadcast_weights(losses, sample_weight)
++
++ # Apply weights to losses.
++ weighted_losses = sample_weight * losses
++
++ # Apply reduction function to the individual weighted losses.
++ loss = reduce_weighted_loss(weighted_losses, reduction)
++ # Convert the result back to the input type.
++ loss = K.cast(loss, input_dtype)
++ return loss
++
+diff --git a/keras/utils/metrics_utils.py b/keras/utils/metrics_utils.py
+new file mode 100644
+index 0000000..e6a5bb0
+--- /dev/null
++++ b/keras/utils/metrics_utils.py
+@@ -0,0 +1,278 @@
++"""Utilities related to metrics."""
++from __future__ import absolute_import
++from __future__ import division
++from __future__ import print_function
++
++from enum import Enum
++
++from .. import backend as K
++from . import losses_utils
++
++NEG_INF = -1e10
++
++class Reduction(object):
++ """Types of metrics reduction.
++ Contains the following values:
++ * `SUM`: Scalar sum of weighted values.
++ * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` of weighted values divided by
++ number of elements in values.
++ * `WEIGHTED_MEAN`: Scalar sum of weighted values divided by sum of weights.
++ """
++
++ SUM = 'sum'
++ SUM_OVER_BATCH_SIZE = 'sum_over_batch_size'
++ WEIGHTED_MEAN = 'weighted_mean'
++
++
++def update_state_wrapper(update_state_fn):
++ """Decorator to wrap metric `update_state()` with `add_update()`.
++ # Arguments
++ update_state_fn: function that accumulates metric statistics.
++ # Returns
++ Decorated function that wraps `update_state_fn()` with `add_update()`.
++ """
++ def decorated(metric_obj, *args, **kwargs):
++ """Decorated function with `add_update()`."""
++
++ update_op = update_state_fn(*args, **kwargs)
++ metric_obj.add_update(update_op)
++ return update_op
++
++ return decorated
++
++def result_wrapper(result_fn):
++ """Decorator to wrap metric `result()` with identity op.
++ Wrapping result in identity so that control dependency between
++ update_op from `update_state` and result works in case result returns
++ a tensor.
++ # Arguments
++ result_fn: function that computes the metric result.
++ # Returns
++ Decorated function that wraps `result()` with identity op.
++ """
++ def decorated(metric_obj, *args, **kwargs):
++ result_t = K.identity(result_fn(*args, **kwargs))
++ metric_obj._call_result = result_t
++ result_t._is_metric = True
++ return result_t
++ return decorated
++
++
++def to_list(x):
++ if isinstance(x, list):
++ return x
++ return [x]
++
++
++def assert_thresholds_range(thresholds):
++ if thresholds is not None:
++ invalid_thresholds = [t for t in thresholds if t is None or t < 0 or t > 1]
++ if invalid_thresholds:
++ raise ValueError(
++ 'Threshold values must be in [0, 1]. Invalid values: {}'.format(
++ invalid_thresholds))
++
++
++def parse_init_thresholds(thresholds, default_threshold=0.5):
++ if thresholds is not None:
++ assert_thresholds_range(to_list(thresholds))
++ thresholds = to_list(default_threshold if thresholds is None else thresholds)
++ return thresholds
++
++class ConfusionMatrix(Enum):
++ TRUE_POSITIVES = 'tp'
++ FALSE_POSITIVES = 'fp'
++ TRUE_NEGATIVES = 'tn'
++ FALSE_NEGATIVES = 'fn'
++
++class AUCCurve(Enum):
++ """Type of AUC Curve (ROC or PR)."""
++ ROC = 'ROC'
++ PR = 'PR'
++
++ @staticmethod
++ def from_str(key):
++ if key in ('pr', 'PR'):
++ return AUCCurve.PR
++ elif key in ('roc', 'ROC'):
++ return AUCCurve.ROC
++ else:
++ raise ValueError('Invalid AUC curve value "%s".' % key)
++
++
++class AUCSummationMethod(Enum):
++ """Type of AUC summation method.
++
++ https://en.wikipedia.org/wiki/Riemann_sum)
++
++ Contains the following values:
++ * 'interpolation': Applies mid-point summation scheme for `ROC` curve. For
++ `PR` curve, interpolates (true/false) positives but not the ratio that is
++ precision (see Davis & Goadrich 2006 for details).
++ * 'minoring': Applies left summation for increasing intervals and right
++ summation for decreasing intervals.
++ * 'majoring': Applies right summation for increasing intervals and left
++ summation for decreasing intervals.
++ """
++ INTERPOLATION = 'interpolation'
++ MAJORING = 'majoring'
++ MINORING = 'minoring'
++
++ @staticmethod
++ def from_str(key):
++ if key in ('interpolation', 'Interpolation'):
++ return AUCSummationMethod.INTERPOLATION
++ elif key in ('majoring', 'Majoring'):
++ return AUCSummationMethod.MAJORING
++ elif key in ('minoring', 'Minoring'):
++ return AUCSummationMethod.MINORING
++ else:
++ raise ValueError('Invalid AUC summation method value "%s".' % key)
++
++def weighted_assign_add(label, pred, weights, var):
++ # Logical and
++ label = K.expand_dims(label, 0)
++ pred = K.expand_dims(pred, 0)
++ are_different = K.concatenate([label, pred], axis=0)
++ label_and_pred = K.all(are_different, axis=0)
++ label_and_pred = K.cast(label_and_pred, dtype=K.floatx())
++ if weights is not None:
++ label_and_pred *= weights
++ return var.assign_add(K.sum(label_and_pred, 1))
++
++def update_confusion_matrix_variables(variables_to_update,
++ y_true,
++ y_pred,
++ thresholds,
++ top_k=None,
++ class_id=None,
++ sample_weight=None):
++ """Returns op to update the given confusion matrix variables.
++ For every pair of values in y_true and y_pred:
++ true_positive: y_true == True and y_pred > thresholds
++ false_negatives: y_true == True and y_pred <= thresholds
++ true_negatives: y_true == False and y_pred <= thresholds
++ false_positive: y_true == False and y_pred > thresholds
++ The results will be weighted and added together. When multiple thresholds are
++ provided, we will repeat the same for every threshold.
++ For estimation of these metrics over a stream of data, the function creates an
++ `update_op` operation that updates the given variables.
++ If `sample_weight` is `None`, weights default to 1.
++ Use weights of 0 to mask values.
++ # Arguments
++ variables_to_update: Dictionary with 'tp', 'fn', 'tn', 'fp' as valid keys
++ and corresponding variables to update as values.
++ y_true: A `Tensor` whose shape matches `y_pred`. Will be cast to `bool`.
++ y_pred: A floating point `Tensor` of arbitrary shape and whose values are in
++ the range `[0, 1]`.
++ thresholds: A float value or a python list or tuple of float thresholds in
++ `[0, 1]`, or NEG_INF (used when top_k is set).
++ top_k: Optional int, indicates that the positive labels should be limited to
++ the top k predictions.
++ class_id: Optional int, limits the prediction and labels to the class
++ specified by this argument.
++ sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as
++ `y_true`, and must be broadcastable to `y_true` (i.e., all dimensions must
++ be either `1`, or the same as the corresponding `y_true` dimension).
++ # Returns
++ Update ops.
++ # Raises
++ ValueError: If `y_pred` and `y_true` have mismatched shapes, or if
++ `sample_weight` is not `None` and its shape doesn't match `y_pred`, or if
++ `variables_to_update` contains invalid keys.
++ """
++ if variables_to_update is None:
++ return
++ y_true = K.cast(y_true, dtype=K.floatx())
++ y_pred = K.cast(y_pred, dtype=K.floatx())
++ if sample_weight is not None:
++ sample_weight = K.cast(sample_weight, dtype=K.floatx())
++
++ if not any(key
++ for key in variables_to_update
++ if key in list(ConfusionMatrix)):
++ raise ValueError(
++ 'Please provide at least one valid confusion matrix '
++ 'variable to update. Valid variable key options are: "{}". '
++ 'Received: "{}"'.format(
++ list(ConfusionMatrix), variables_to_update.keys()))
++
++ invalid_keys = [
++ key for key in variables_to_update if key not in list(ConfusionMatrix)
++ ]
++ if invalid_keys:
++ raise ValueError(
++ 'Invalid keys: {}. Valid variable key options are: "{}"'.format(
++ invalid_keys, list(ConfusionMatrix)))
++
++ if sample_weight is None:
++ y_pred, y_true = losses_utils.squeeze_or_expand_dimensions(
++ y_pred, y_true=y_true)
++ else:
++ y_pred, y_true, sample_weight = (
++ losses_utils.squeeze_or_expand_dimensions(
++ y_pred, y_true=y_true, sample_weight=sample_weight))
++
++ if top_k is not None:
++ y_pred = _filter_top_k(y_pred, top_k)
++ if class_id is not None:
++ y_true = y_true[..., class_id]
++ y_pred = y_pred[..., class_id]
++
++ thresholds = to_list(thresholds)
++ num_thresholds = len(thresholds)
++ num_predictions = K.size(y_pred)
++
++ # Reshape predictions and labels.
++ predictions_2d = K.reshape(y_pred, [1, -1])
++ labels_2d = K.reshape(
++ K.cast(y_true, dtype='bool'), [1, -1])
++
++ # Tile the thresholds for every prediction.
++ thresh_tiled = K.tile(
++ K.expand_dims(K.constant(thresholds), 1),
++ K.stack([1, num_predictions]))
++
++ # Tile the predictions for every threshold.
++ preds_tiled = K.tile(predictions_2d, [num_thresholds, 1])
++
++ # Compare predictions and threshold.
++ pred_is_pos = K.greater(preds_tiled, thresh_tiled)
++ pred_is_neg = K.greater(thresh_tiled, preds_tiled)
++
++ # Tile labels by number of thresholds
++ label_is_pos = K.tile(labels_2d, [num_thresholds, 1])
++
++ if sample_weight is not None:
++ weights = losses_utils.broadcast_weights(
++ y_pred, K.cast(sample_weight, dtype=K.floatx()))
++ weights_tiled = K.tile(
++ K.reshape(weights, [1, -1]), [num_thresholds, 1])
++ else:
++ weights_tiled = None
++
++ update_ops = []
++ loop_vars = {
++ ConfusionMatrix.TRUE_POSITIVES: (label_is_pos, pred_is_pos),
++ }
++ update_tn = ConfusionMatrix.TRUE_NEGATIVES in variables_to_update
++ update_fp = ConfusionMatrix.FALSE_POSITIVES in variables_to_update
++ update_fn = ConfusionMatrix.FALSE_NEGATIVES in variables_to_update
++
++ if update_fn or update_tn:
++ loop_vars[ConfusionMatrix.FALSE_NEGATIVES] = (label_is_pos, pred_is_neg)
++
++ if update_fp or update_tn:
++ label_is_neg = K.equal(
++ label_is_pos, K.zeros_like(label_is_pos, dtype=label_is_pos.dtype))
++ loop_vars[ConfusionMatrix.FALSE_POSITIVES] = (label_is_neg, pred_is_pos)
++ if update_tn:
++ loop_vars[ConfusionMatrix.TRUE_NEGATIVES] = (label_is_neg, pred_is_neg)
++
++ for matrix_cond, (label, pred) in loop_vars.items():
++ if matrix_cond in variables_to_update:
++ update_ops.append(
++ weighted_assign_add(label, pred, weights_tiled,
++ variables_to_update[matrix_cond]))
++ return update_ops
++
+--
+2.26.2
+
generated by cgit v1.2.3 (git 2.44.0) at 2024-11-05 23:07:16 +0000