diff options
| -rw-r--r-- | gn/packages/bioinformatics.scm | 317 | ||||
| -rw-r--r-- | gn/packages/bnw.scm | 140 | ||||
| -rw-r--r-- | gn/packages/cran.scm | 29 | ||||
| -rw-r--r-- | gn/packages/cwl.scm | 77 | ||||
| -rw-r--r-- | gn/packages/graphviz.scm | 9 | ||||
| -rw-r--r-- | gn/packages/javascript.scm | 24 | ||||
| -rw-r--r-- | gn/packages/julia.scm | 198 | ||||
| -rw-r--r-- | gn/packages/jupyterhub.scm | 1 | ||||
| -rw-r--r-- | gn/packages/maths.scm | 1 | ||||
| -rw-r--r-- | gn/packages/python.scm | 193 | ||||
| -rw-r--r-- | gn/packages/ratspub.scm | 213 | ||||
| -rw-r--r-- | gn/services/archive-pubmed.service | 13 | ||||
| -rw-r--r-- | gn/services/archive-pubmed.timer | 9 | ||||
| -rw-r--r-- | gn/services/bnw-README | 4 | ||||
| -rw-r--r-- | gn/services/bnw-container.scm | 4 | ||||
| -rw-r--r-- | gn/services/bnw.service | 2 | ||||
| -rw-r--r-- | gn/services/genenetwork.scm | 26 | ||||
| -rw-r--r-- | keras-auc-optimizer.patch | 1133 |
18 files changed, 2062 insertions, 331 deletions
diff --git a/gn/packages/bioinformatics.scm b/gn/packages/bioinformatics.scm index 8897752..056bc3f 100644 --- a/gn/packages/bioinformatics.scm +++ b/gn/packages/bioinformatics.scm @@ -14,6 +14,8 @@ #:use-module (guix build-system trivial) #:use-module (guix build-system waf) #:use-module (gnu packages) + #:use-module (gn packages python) + #:use-module (gnu packages bioconductor) #:use-module (gnu packages bioinformatics) #:use-module (gnu packages boost) #:use-module (gnu packages check) @@ -22,6 +24,7 @@ #:use-module (gnu packages datastructures) #:use-module (gnu packages fontutils) #:use-module (gnu packages gcc) + #:use-module (gnu packages graphviz) #:use-module (gnu packages imagemagick) #:use-module (gnu packages jemalloc) #:use-module (gnu packages maths) @@ -31,9 +34,13 @@ #:use-module (gnu packages protobuf) #:use-module (gnu packages python) #:use-module (gnu packages python-science) + #:use-module (gnu packages python-web) #:use-module (gnu packages python-xyz) + #:use-module (gnu packages rdf) #:use-module (gnu packages readline) - #:use-module (gnu packages statistics)) + #:use-module (gnu packages serialization) + #:use-module (gnu packages statistics) + #:use-module (gnu packages time)) (define-public contra (package @@ -415,92 +422,7 @@ reads.") (license license:non-copyleft))) (define-public edirect-gn - (package - (inherit edirect) - (name "edirect-gn") - (arguments - (substitute-keyword-arguments (package-arguments edirect) - ((#:phases phases) - `(modify-phases ,phases - ; (replace 'build - ; (lambda* (#:key inputs #:allow-other-keys) - ; (let ((go (string-append (assoc-ref inputs "go") "/bin/go"))) - ; (invoke go "build" "xtract.go")))) - (add-after 'unpack 'patch-programs - (lambda* (#:key inputs #:allow-other-keys) - (let ((gzip (assoc-ref inputs "gzip"))) - (substitute* '("index-bioc" - "pm-index" - "pm-invert" - "pm-stash" - "rchive.go" - "run-ncbi-converter") - (("gunzip") (string-append gzip "/bin/gunzip"))) - (substitute* (find-files "." "^e") - (("exec perl") "exec")) - (substitute* '("xtract" "rchive") - ;; or add current directory to PATH - ((".*PATH.*") ""))) - #t)) - (replace 'install - (lambda* (#:key inputs outputs #:allow-other-keys) - (let ((bin (string-append (assoc-ref outputs "out") "/bin")) - (xtract.linux (assoc-ref inputs "xtract.Linux")) - (rchive.linux (assoc-ref inputs "rchive.Linux"))) - (for-each - (lambda (file) - (install-file file bin)) - '("archive-pubmed" "asp-cp" "asp-ls" "download-pubmed" - "edirect.pl" "efetch" "epost" "esearch" "fetch-pubmed" - "ftp-cp" "ftp-ls" "has-asp" "pm-prepare" "pm-refresh" - "pm-stash" "rchive" "xtract")) - (copy-file xtract.linux (string-append bin "/xtract.Linux")) - (copy-file rchive.linux (string-append bin "/rchive.Linux")) - (chmod (string-append bin "/xtract.Linux") #o555) - (chmod (string-append bin "/rchive.Linux") #o555)) - #t)) - (replace 'wrap-program - (lambda* (#:key outputs #:allow-other-keys) - ;; Make sure 'edirect.pl' finds all perl inputs at runtime. - (let ((out (assoc-ref outputs "out")) - (path (getenv "PERL5LIB"))) - (for-each - (lambda (file) - (wrap-program (string-append out "/bin/" file) - `("PERL5LIB" ":" prefix (,path)))) - '("edirect.pl" "asp-ls" "ftp-cp" "ftp-ls"))) - #t)))))) - (inputs - `(("gzip" ,gzip) - ,@(package-inputs edirect))) - (native-inputs - `( - ;("go" ,go) - ("xtract.Linux" - ,(origin - (method url-fetch) - (uri (string-append "ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/" - "/xtract.Linux")) ;; March 10, 2016 - (sha256 - (base32 - "15yhhh8kfipk12rhzabap81ys8wgj0khn0mp8p7zwqhq028fwj0l")))) - ("rchive.Linux" - ,(origin - (method url-fetch) - (uri (string-append "ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/" - "/rchive.Linux")) ;; November 14, 2017 - (sha256 - (base32 - "0hl8zj1md9xbmaj0pv99rjyisw8w74rirw97xwqk47dz8v8ml338")))))) - (native-search-paths - ;; Ideally this should be set for LWP somewhere. - (list (search-path-specification - (variable "PERL_LWP_SSL_CA_FILE") - (file-type 'regular) - (separator #f) - (files '("/etc/ssl/certs/ca-certificates.crt"))))) - ;; Due to the precompiled binaries we download: - (supported-systems '("x86_64-linux")))) + (deprecated-package "edirect-gn" edirect)) ;; TODO: Unbundle zlib, bamtools, tclap (define-public sniffles @@ -856,19 +778,19 @@ interest, and this app can provide values and figures for applicants to use.") (license license:gpl3)))) (define-public singlecellrshiny - (let ((commit "8061dcb477ba355de77d3e4fd3a15cf3267b56af") - (revision "1")) + (let ((commit "bdca74f4819d11e8fe7b15d9ab91b853f6542f7a") + (revision "3")) (package (name "singlecellrshiny") (version (git-version "0.0.0" revision commit)) (source (origin (method git-fetch) (uri (git-reference - (url "https://github.com/syousefi/singleCellRshiny.git") + (url "https://github.com/genenetwork/singleCellRshiny") (commit commit))) (file-name (git-file-name name version)) (sha256 - (base32 "1pd8a9jx6ijjggsifvq66madx31h29rah5pmz4kdzfzb4fskpqz1")))) + (base32 "1rxj933s9p9r7358vnp15f7ag6c0j65r4hgr8kyirfhmp1i8xdlw")))) (build-system trivial-build-system) (arguments `(#:modules ((guix build utils)) @@ -880,16 +802,20 @@ interest, and this app can provide values and figures for applicants to use.") (app (string-append out "/bin/" ,name)) (Rbin (string-append (assoc-ref %build-inputs "r-min") "/bin/Rscript")) + (top1001 (assoc-ref %build-inputs "RobTop1001.csv")) + (celltypes (assoc-ref %build-inputs "CellTypes_RGC_Master_08Dec2018.csv")) + (800-H1 (assoc-ref %build-inputs "800-H1-H20-RNA-Seq.csv")) (source (assoc-ref %build-inputs "source"))) (copy-recursively source targetdir) + (substitute* (string-append targetdir "/app.R") + ;; As seen in https://github.com/genenetwork/singleCellRshiny/commit/6b2a344dd0d02f65228ad8c350bac0ced5850d05.patch + (("library\\(DT\\)") "library(DT)\nlibrary(multtest)")) (substitute* (string-append targetdir "/global.R") - (("800-H1-H20-RNA-Seq-SingleCell-Retina-OMRF-03-29-19_FPKM_v2_SiamakPlay.csv") - "shinyRappToyDataset_SiamakPlay.csv") - ;; Comment out the two unreferenced files for now - (("^rgc.*") "") - ;(("CellTypes_RGC_Master_08Dec2018.csv") "") - ;(("RobTop1001.csv") "") - ) + (("800-H1-H20-RNA-Seq-SingleCell-Retina-OMRF-03-29-19_FPKM_v2_SiamakPlay.csv") 800-H1) + (("CellTypes_RGC_Master_08Dec2018.csv") celltypes) + (("RobTop1001.csv") top1001) + ;; As seen in https://github.com/genenetwork/singleCellRshiny/commit/6b2a344dd0d02f65228ad8c350bac0ced5850d05.patch + (("dim\\(sc.object.1") "dim(sc.object")) (mkdir-p (string-append out "/bin")) (call-with-output-file app (lambda (port) @@ -901,20 +827,41 @@ runApp(launch.browser=0, port=4208)~%\n" Rbin targetdir))) (chmod app #o555) #t)))) - (native-inputs `(("source" ,source))) (inputs - `(("r-min" ,r-minimal))) + `(("r-min" ,r-minimal) + ("RobTop1001.csv" + ,(origin + (method url-fetch) + (uri "https://archive.org/download/celltypesrgcmaster08dec2018/RobTop1001.csv") + (file-name "RobTop1001.csv") + (sha256 + (base32 "0pa73kc1p8417sjvvvhp9xsbh2h8g7h85pnmm16mnv4wjalhq0gn")))) + ("CellTypes_RGC_Master_08Dec2018.csv" + ,(origin + (method url-fetch) + (uri "https://archive.org/download/celltypesrgcmaster08dec2018/CellTypes_RGC_Master_08Dec2018.csv") + (file-name "CellTypes_RGC_Master_08Dec2018.csv") + (sha256 + (base32 "0y411968np1f5g21iym9xc9yj5c1jsn94rpkwkxh9pw2z43gvghn")))) + ("800-H1-H20-RNA-Seq.csv" + ,(origin + (method url-fetch) + (uri "https://archive.org/download/celltypesrgcmaster08dec2018/800-H1-H20-RNA-Seq-SingleCell-Retina-OMRF-03-29-19_FPKM_v2_SiamakPlay.csv") + (file-name "800-H1-H20-RNA-Seq-SingleCell-Retina-OMRF-03-29-19_FPKM_v2_SiamakPlay.csv") + (sha256 + (base32 "1b1y4lfs8drypm04i1rypbmk67rdqgs27nfh05pwnv3sja2nanam")))))) (propagated-inputs `(("r" ,r) ("r-dt" ,r-dt) + ("r-multtest" ,r-multtest) ("r-seurat" ,r-seurat) ("r-shiny" ,r-shiny))) - (home-page "http://rn6err.opar.io/") + (home-page "http://singlecell.opar.io/") (synopsis "RNA sequencing data analysis") (description "This is the R-Shiny programs to run some basic single cell RNA sequencing (scRNA-seq) data analysis.") - (license license:gpl3)))) + (license license:agpl3)))) (define-public seqwish (package @@ -1193,3 +1140,169 @@ here}.") (synopsis "Efficient sequence alignment of full genomes") (description "MUMmer is a versatil alignment tool for DNA and protein sequences.") (license license:artistic2.0))) + +(define-public grocsvs + (let ((commit "ecd956a65093a0b2c41849050e4512d46fecea5d") + (revision "1")) + (package + (name "grocsvs") + (version (git-version "0.2.6.1" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/grocsvs/grocsvs") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 "14505725gr7qxc17cxxf0k6lzcwmgi64pija4mwf29aw70qn35cc")))) + (build-system python-build-system) + (arguments + `(#:python ,python-2)) ; Only python-2 supported. + (inputs + `(("python2-admiral" ,python2-admiral) + ("python2-h5py" ,python2-h5py) + ("python2-ipython-cluster-helper" ,python2-ipython-cluster-helper) + ("python2-networkx" ,python2-networkx) + ("python2-psutil" ,python2-psutil) + ("python2-pandas" ,python2-pandas) + ("python2-pybedtools" ,python2-pybedtools) + ("python2-pyfaidx" ,python2-pyfaidx) + ("python2-pygraphviz" ,python2-pygraphviz) + ("python2-pysam" ,python2-pysam) + ("python2-scipy" ,python2-scipy))) + (home-page "https://github.com/grocsvs/grocsvs") + (synopsis "Genome-wide reconstruction of complex structural variants") + (description + "@dfn{Genome-wide Reconstruction of Complex Structural Variants} +(GROC-SVs), is a software pipeline for identifying large-scale structural +variants, performing sequence assembly at the breakpoints, and reconstructing +the complex structural variants using the long-fragment information from the +10x Genomics platform.") + (license license:expat)))) + +(define-public diagnostic-slider + (let ((commit "514d65d4982133e4869e578c5553fced4c6d506c") + (revision "1")) + (package + (name "diagnostic-slider") + (version (git-version "0.0.0" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/sens/diagnostic-slider") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 "04g8if32g8syg6v0bd3jjn05i3d394nx8i3ccl0883p8mlmdvlmx")))) + (build-system trivial-build-system) + (arguments + `(#:modules ((guix build utils)) + #:builder + (begin + (use-modules (guix build utils)) + (let* ((out (assoc-ref %outputs "out")) + (targetdir (string-append out "/share/" ,name)) + (app (string-append out "/bin/" ,name)) + (Rbin (string-append (assoc-ref %build-inputs "r-min") + "/bin/Rscript")) + (source (assoc-ref %build-inputs "source"))) + (copy-recursively source targetdir) + (mkdir-p (string-append out "/bin")) + (call-with-output-file app + (lambda (port) + (format port +"#!~a +library(shiny) +setwd(\"~a\") +runApp(launch.browser=0, port=4206)~%\n" + Rbin targetdir))) + (chmod app #o555) + #t)))) + (native-inputs + `(("source" ,source))) + (inputs + `(("r-min" ,r-minimal))) + (propagated-inputs + `(("r" ,r) + ("r-shiny" ,r-shiny))) + (home-page "https://github.com/sens/diagnostic-slider") + (synopsis "") + (description + "") + (license #f)))) + +(define-public bh20-seq-resource + (let ((commit "bbca5ac9b2538e410efe3e09651f87e5573145de") + (revision "2")) + (package + (name "bh20-seq-resource") + (version (git-version "1.0" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/arvados/bh20-seq-resource") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 "1kkysjgmhp7kfb17470ik821p9djsidyqmkbjvv37jx2w9pvw31z")))) + (build-system python-build-system) + (inputs + `(("python-arvados-python-client" ,python-arvados-python-client) + ("python-flask" ,python-flask) + ("python-magic" ,python-magic) + ("python-pyyaml" ,python-pyyaml) + ("python-schema-salad" ,python-schema-salad))) + (native-inputs + `(("git" ,(@ (gnu packages version-control) git)) + ("python-oauth2client" ,python-oauth2client) + ("python-pytest" ,python-pytest) + ("python-pytest-runner" ,python-pytest-runner) + ("python-uritemplate" ,python-uritemplate))) + (home-page "https://github.com/arvados/bh20-seq-resource") + (synopsis + "Tool to upload SARS-CoV-19 sequences and service to kick off analysis") + (description "This repository provides a sequence uploader for the +COVID-19 Virtual Biohackathon's Public Sequence Resource project. You can use +it to upload the genomes of SARS-CoV-2 samples to make them publicly and freely +available to other researchers.") + (license license:asl2.0)))) + +(define-public python-scanpy-git + (let ((commit "590d42309f9ed6550d7b887039990edfc1ac7648") ; April 22, 2020 + (revision "1")) + (package + (inherit python-scanpy) + (name "python-scanpy-git") + (version (git-version "1.4.6" revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/theislab/scanpy") + (commit commit))) + (file-name (git-file-name "python-scanpy" version)) + (sha256 + (base32 "0z3pk9vh4b7fqq7fs262i6z0pm1dnn6bf49a4r7r73k6gjj6namd")))) + (arguments + (substitute-keyword-arguments (package-arguments python-scanpy) + ((#:phases phases) + `(modify-phases ,phases + (add-before 'build 'fix-build + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out")) + (pyv (python-version (assoc-ref inputs "python")))) + (substitute* "setup.py" + (("use_scm_version=True") "use_scm_version=False")) + (substitute* "scanpy/__init__.py" + (("__version__.*") + (string-append "__version__ = '" ,version "'\n"))) + (mkdir-p + (string-append out "/lib/python" pyv "/site-packages")) + (setenv "PYTHONPATH" + (string-append out + "/lib/python" pyv "/site-packages/:" + (getenv "PYTHONPATH")))) + ;; These tests fail on this git revision + (delete-file "scanpy/tests/test_neighbors_key_added.py") + (delete-file "scanpy/tests/test_pca.py") + #t))))))))) diff --git a/gn/packages/bnw.scm b/gn/packages/bnw.scm index 5c97eab..043077d 100644 --- a/gn/packages/bnw.scm +++ b/gn/packages/bnw.scm @@ -14,7 +14,7 @@ (define-public bnw (let ((commit "eb6b002b924694808384f1a8d7c6d1121806ae04") - (revision "5")) + (revision "6")) (package (name "bnw") (version (git-version "1.22" revision commit)) ; June 28, 2019 @@ -86,59 +86,101 @@ (("rmdir ") (string-append (which "rmdir") " ")) (("wc ") (string-append (which "wc") " "))) #t))) - ;(add-after 'patch-source-shebangs 'replace-javascript - ; (lambda* (#:key inputs #:allow-other-keys) - ; (let ( - ; (jquery (assoc-ref inputs "jquery")) - ; (awesome (assoc-ref inputs "awesome")) - ; (cyto (assoc-ref inputs "cytoscape")) - ; (cyto2 (assoc-ref inputs "cytoscape-2")) - ; (cs-dagre (assoc-ref inputs "cyto-dagre")) - ; (d3js (assoc-ref inputs "d3js")) - ; (d3js-multi (assoc-ref inputs "d3js-multi")) - ; (dagre (assoc-ref inputs "dagre")) - ; (lodash (assoc-ref inputs "lodash")) - ; (canvas-toblob (assoc-ref inputs "canvas-toblob")) - ; (filesaver (assoc-ref inputs "filesaver")) - ; (panzoom (assoc-ref inputs "panzoom")) - ; (js-path "/share/genenetwork2/javascript/") - ; ) - ;(substitute* "sourcecodes/layout_cyto.php" - ; (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape/3.7.1/cytoscape.min.js") - ; (string-append cyto js-path "cytoscape/cytoscape.min.js")) - ; (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape/2.7.29/cytoscape.min.js") - ; (string-append cyto2 js-path "cytoscape/cytoscape.min.js")) - ; (("http://spades.bioinf.spbau.ru/~alla/graph_viewer/js/cytoscape-dagre.js") - ; (string-append cs-dagre js-path "cytoscape-dagre/cytoscape-dagre.js")) - ; (("https://unpkg.com/dagre@0.7.4/dist/dagre.js") - ; (string-append dagre js-path "dagre/dagre.js")) - ; (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape-panzoom/2.5.3/cytoscape.js-panzoom.css") - ; (string-append panzoom js-path "cytoscape-panzoom/cytoscape.js-panzoom.css")) - ; (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape-panzoom/2.5.3/cytoscape-panzoom.js") - ; (string-append panzoom js-path "cytoscape-panzoom/cytoscape-panzoom.js")) - ; (("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.0.3/css/font-awesome.css") - ; (string-append awesome "/share/web/font-awesome/css/font-awesome.css")) - ; (("https://ajax.googleapis.com/ajax/libs/jquery/3.1.0/jquery.min.js") - ; (string-append jquery "/share/web/jquery/jquery.min.js")) - ; (("https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.10/lodash.js") - ; (string-append lodash js-path "lodash/lodash.js"))) - ;(substitute* '("sourcecodes/layout_svg_wt.php" - ; "sourcecodes/layout_svg_no.php") - ; (("http://d3js.org/d3.v4.min.js") - ; (string-append d3js js-path "d3js/d3.min.js")) - ; (("http://d3js.org/d3-selection-multi.v1.js") - ; (string-append d3js-multi js-path "d3js-multi/d3-selection-multi.js")) - ; (("https://cdn.rawgit.com/eligrey/canvas-toBlob.js/f1a01896135ab378aa5c0118eadd81da55e698d8/canvas-toBlob.js") - ; (string-append canvas-toblob js-path "canvas-toBlob/canvas-toBlob.js")) - ; (("https://cdn.rawgit.com/eligrey/FileSaver.js/e9d941381475b5df8b7d7691013401e171014e89/FileSaver.min.js") - ; (string-append filesaver js-path "filesaver/filesaver.js"))) - ; ) - ; #t)) + (add-after 'patch-source-shebangs 'replace-javascript + (lambda* (#:key inputs #:allow-other-keys) + (let ((jquery (assoc-ref inputs "jquery")) + (awesome (assoc-ref inputs "awesome")) + (cyto (assoc-ref inputs "cytoscape")) + (cyto2 (assoc-ref inputs "cytoscape-2")) + (cs-dagre (assoc-ref inputs "cyto-dagre")) + (d3js (assoc-ref inputs "d3js")) + (d3js-multi (assoc-ref inputs "d3js-multi")) + (dagre (assoc-ref inputs "dagre")) + (lodash (assoc-ref inputs "lodash")) + (canvas-toblob (assoc-ref inputs "canvas-toblob")) + (filesaver (assoc-ref inputs "filesaver")) + (panzoom (assoc-ref inputs "panzoom")) + (js-path "/share/genenetwork2/javascript/")) + (substitute* "sourcecodes/layout_cyto.php" + (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape/3.7.1/cytoscape.min.js") + "/javascript/cytoscape.min.js") + (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape/2.7.29/cytoscape.min.js") + "/javascript/cytoscape2.min.js") + (("http://spades.bioinf.spbau.ru/~alla/graph_viewer/js/cytoscape-dagre.js") + "/javascript/cytoscape-dagre.js") + (("https://unpkg.com/dagre@0.7.4/dist/dagre.js") + "/javascript/dagre.js") + (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape-panzoom/2.5.3/cytoscape.js-panzoom.css") + "/javascript/cytoscape.js-panzoom.css") + (("https://cdnjs.cloudflare.com/ajax/libs/cytoscape-panzoom/2.5.3/cytoscape-panzoom.js") + "/javascript/cytoscape-panzoom.js") + (("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.0.3/css/font-awesome.css") + "/javascript/font-awesome.css") + (("https://ajax.googleapis.com/ajax/libs/jquery/3.1.0/jquery.min.js") + "/javascript/jquery.min.js") + (("https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.10/lodash.js") + "/javascript/lodash.js")) + (substitute* '("sourcecodes/layout_svg_wt.php" + "sourcecodes/layout_svg_no.php") + (("http://d3js.org/d3.v4.min.js") + "/javascript/d3.min.js") + (("http://d3js.org/d3-selection-multi.v1.js") + "/javascript/d3-selection-multi.js") + (("https://cdn.rawgit.com/eligrey/canvas-toBlob.js/f1a01896135ab378aa5c0118eadd81da55e698d8/canvas-toBlob.js") + "/javascript/canvas-toBlob.js") + (("https://cdn.rawgit.com/eligrey/FileSaver.js/e9d941381475b5df8b7d7691013401e171014e89/FileSaver.min.js") + "/javascript/filesaver.js"))) + #t)) (replace 'install (lambda* (#:key outputs #:allow-other-keys) (let ((out (assoc-ref outputs "out"))) (copy-recursively "." out)) #t)) + (add-after 'install 'install-javascript-libraries + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out")) + (jquery (assoc-ref inputs "jquery")) + (awesome (assoc-ref inputs "awesome")) + (cyto (assoc-ref inputs "cytoscape")) + (cyto2 (assoc-ref inputs "cytoscape-2")) + (cs-dagre (assoc-ref inputs "cyto-dagre")) + (d3js (assoc-ref inputs "d3js")) + (d3js-multi (assoc-ref inputs "d3js-multi")) + (dagre (assoc-ref inputs "dagre")) + (lodash (assoc-ref inputs "lodash")) + (canvas-toblob (assoc-ref inputs "canvas-toblob")) + (filesaver (assoc-ref inputs "filesaver")) + (panzoom (assoc-ref inputs "panzoom")) + (js-path "/share/genenetwork/javascript/") + (js-path2 "/share/genenetwork2/javascript/")) + (mkdir-p (string-append out "/javascript")) + (symlink (string-append (string-append cyto2 js-path2 "cytoscape/cytoscape.min.js")) + (string-append out "/javascript/cytoscape.min.js")) + (symlink (string-append (string-append cyto js-path2 "cytoscape/cytoscape.min.js")) + (string-append out "/javascript/cytoscape2.min.js")) + (symlink (string-append cs-dagre js-path2 "cytoscape-dagre/cytoscape-dagre.js") + (string-append out "/javascript/cytoscape-dagre.js")) + (symlink (string-append dagre js-path2 "dagre/dagre.js") + (string-append out "/javascript/dagre.js")) + (symlink (string-append panzoom js-path2 "cytoscape-panzoom/cytoscape.js-panzoom.css") + (string-append out "/javascript/cytoscape.js-panzoom.css")) + (symlink (string-append panzoom js-path2 "cytoscape-panzoom/cytoscape-panzoom.js") + (string-append out "/javascript/cytoscape-panzoom.js")) + (symlink (string-append awesome "/share/web/font-awesomecss/font-awesome.css") + (string-append out "/javascript/font-awesome.css")) + (symlink (string-append jquery "/share/web/jquery/jquery.min.js") + (string-append out "/javascript/jquery.min.js")) + (symlink (string-append lodash js-path2 "lodash/lodash.js") + (string-append out "/javascript/lodash.js")) + (symlink (string-append d3js js-path "d3js/d3.min.js") + (string-append out "/javascript/d3.min.js")) + (symlink (string-append d3js-multi js-path "d3js-multi/d3-selection-multi.js") + (string-append out "/javascript/d3-selection-multi.js")) + (symlink (string-append canvas-toblob js-path "canvas-toblob/canvas-toBlob.js") + (string-append out "/javascript/canvas-toBlob.js")) + (symlink (string-append filesaver js-path2 "filesaver/FileSaver.js") + (string-append out "/javascript/filesaver.js")) + #t))) (add-after 'install 'make-files-executable (lambda* (#:key outputs #:allow-other-keys) (let ((out (assoc-ref outputs "out"))) diff --git a/gn/packages/cran.scm b/gn/packages/cran.scm new file mode 100644 index 0000000..d92f9f3 --- /dev/null +++ b/gn/packages/cran.scm @@ -0,0 +1,29 @@ +(define-module (gn packages cran) + #:use-module ((guix licenses) #:prefix license:) + #:use-module (guix packages) + #:use-module (guix download) + #:use-module (guix build-system r) + #:use-module (gnu packages)) + +(define-public r-tictoc + (package + (name "r-tictoc") + (version "1.0") + (source + (origin + (method url-fetch) + (uri (cran-uri "tictoc" version)) + (sha256 + (base32 + "1zp2n8k2ax2jjw89dsri268asmm5ry3ijf32wbca5ji231y0knj7")))) + (build-system r-build-system) + (home-page "http://github.com/collectivemedia/tictoc") + (synopsis "Functions for timing R scripts") + (description + "This package provides the timing functions @code{tic} and @code{toc} that +can be nested. One can record all timings while a complex script is running, +and examine the values later. It is also possible to instrument the timing call +with custom callbacks. In addition, this package provides class 'Stack', +implemented as a vector, and class 'List', implemented as a list, both of whic +support operations 'push', 'pop', 'first', 'last' and 'clear'.") + (license license:asl2.0))) diff --git a/gn/packages/cwl.scm b/gn/packages/cwl.scm index 6e0c909..be531f6 100644 --- a/gn/packages/cwl.scm +++ b/gn/packages/cwl.scm @@ -61,7 +61,7 @@ ("python-mock" ,python-mock) ("python-subprocess32" ,python-subprocess32) ("python-ruamel.yaml" ,python-ruamel.yaml) - ("python-cachecontrol" ,python-cachecontrol) + ("python-cachecontrol" ,python-cachecontrol-0.11) ("python-lxml" ,python-lxml) ("python-mypy-extensions" ,python-mypy-extensions) ("python-mistune" ,python-mistune) @@ -89,78 +89,3 @@ (description "Common workflow language reference implementation") (license license:asl2.0)))) - -(define-public python-cachecontrol - (package - (name "python-cachecontrol") - (version "0.11.7") - (source - (origin - (method url-fetch) - ;; Pypi does not have tests. - (uri (string-append - "https://github.com/ionrock/cachecontrol/archive/v" - version ".tar.gz")) - (file-name (string-append name "-" version ".tar.gz")) - (sha256 - (base32 - "1yfhwihx1b1xjsx0r19va2m0r2s91im03x4d7pwzp87368f2lkkp")))) - (build-system python-build-system) - (arguments - `(#:tests? #f)) ;; Recent version breaks on cherrypy - (native-inputs - `(("python-pytest" ,python-pytest) - ("python-redis" ,python-redis) - ("python-webtest" ,python-webtest) - ("python-mock" ,python-mock))) - (propagated-inputs - `(("python-requests" ,python-requests) - ("python-lockfile" ,python-lockfile))) - (home-page "https://github.com/ionrock/cachecontrol") - (synopsis "The httplib2 caching algorithms for use with requests") - (description "CacheControl is a port of the caching algorithms in -@code{httplib2} for use with @code{requests} session objects.") - (license license:asl2.0))) - - -(define-public python-schema-salad - (let ((commit "eb85c3d49b99b7643e8a12248e2dc05504910c1e")) - (package - (name "python-schema-salad") - (version "3.0.20181129082112") - (source - (origin - ; (method url-fetch) - ; (uri (pypi-uri "schema-salad" version)) - (method git-fetch) - (uri (git-reference - (url "https://github.com/genenetwork/schema_salad.git") ;; my repo for Python3.7 - (commit commit))) - (file-name (git-file-name name (string-append version "-" (string-take commit 7)))) - (sha256 - (base32 - "174f224zzjr0nbjlq3ypciyfijnibasysrgjswvx8yhan2dizlhr")))) - (build-system python-build-system) - (arguments `(#:tests? #f)) ;; CWL includes no tests. - (inputs - `(("python-cython" ,python-cython) - ("python-setuptools" ,python-setuptools) - ("python-rdflib-jsonld" ,python-rdflib-jsonld) - ("python-mistune" ,python-mistune))) - (propagated-inputs - `(("python-rdflib" ,python-rdflib) - ("python-avro" ,python-avro) - ("python-pyyaml" ,python-pyyaml) - ("python-requests" ,python-requests) - ("python-shellescape" ,python-shellescape) - )) - (home-page - "https://github.com/common-workflow-language/common-workflow-language") - (synopsis - "Schema Annotations for Linked Avro Data (SALAD)") - (description - "Schema Annotations for Linked Avro Data (SALAD)") - (license license:asl2.0)))) - -; (define-public python2-schema-salad -; (package-with-python2 python-schema-salad)) diff --git a/gn/packages/graphviz.scm b/gn/packages/graphviz.scm index 81e745c..33b15a1 100644 --- a/gn/packages/graphviz.scm +++ b/gn/packages/graphviz.scm @@ -3,7 +3,6 @@ #:use-module (guix packages) #:use-module (guix download) #:use-module (guix utils) - #:use-module (gn packages python24) #:use-module (gnu packages gl) #:use-module (gnu packages graphviz) #:use-module (gnu packages gtk) @@ -28,18 +27,16 @@ "18bzyg17ni0lpcd2g5dhan8fjv3vzkjym38jq8vm42did5p9j47l")))) ;; TODO: unbundle libraries? (arguments - `(#:configure-flags '("--enable-python24=yes") + `(#:configure-flags '("--enable-swig=no") ,@(substitute-keyword-arguments (package-arguments graphviz) ((#:phases phases) `(modify-phases ,phases (delete 'move-docs) ; one output (delete 'move-guile-bindings)))))) (inputs - ;; TODO: Add(?) perl, guile@1.8, gtk@2, lua5.1, tcl8.[3-6], rsvg + ;; TODO(?): Add perl, guile@1.8, gtk@2, lua5.1, tcl8.[3-6], rsvg, python-2.4 `(("gdk-pixbuf" ,gdk-pixbuf) - ;("ghostscript" ,ghostscript) ("freeglut" ,freeglut) - ("python2.4" ,python-2.4) ,@(fold alist-delete (package-inputs graphviz) - '("libjpeg" "guile")))) + '("libjpeg" "guile" "swig")))) (license license:cpl1.0))) diff --git a/gn/packages/javascript.scm b/gn/packages/javascript.scm index 79a8060..95d4154 100644 --- a/gn/packages/javascript.scm +++ b/gn/packages/javascript.scm @@ -1121,3 +1121,27 @@ widgets, and themes built on top of the jQuery JavaScript Library.") (name "js-jquery-ui") (arguments `(#:javascript-files '("ui/jquery-ui.js"))) (build-system minify-build-system))) + +(define-public js-popper + (package + (name "js-popper") + (version "2.0.6") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/popperjs/popper-core") + (commit (string-append "v" version)))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0lzy981p9nja2l3xa2zvals6q31v3bzpxxa85yn9pm7wkj3vglf2")))) + (build-system minify-build-system) + (arguments + `(#:javascript-files '("src/popper.js"))) + (home-page "https://popper.js.org/") + (synopsis "Tooltip and popover positioning engine") + (description + "Given an element, such as a button, and a tooltip element describing it, +Popper will automatically put the tooltip in the right place near the button.") + (license license:expat))) diff --git a/gn/packages/julia.scm b/gn/packages/julia.scm new file mode 100644 index 0000000..b55efde --- /dev/null +++ b/gn/packages/julia.scm @@ -0,0 +1,198 @@ +(define-module (gn packages julia) + #:use-module ((guix licenses) #:prefix license:) + #:use-module (guix utils) + #:use-module (guix packages) + #:use-module (guix download) + #:use-module (guix git-download) + #:use-module (guix build-system julia) + #:use-module (gn packages cran) + #:use-module (gnu packages bioinformatics) + #:use-module (gnu packages compression) + #:use-module (gnu packages cran) + #:use-module (gnu packages statistics) + #:use-module (ice-9 match)) + +(define-public julia-lmgpu + (let ((commit "e9e95b5fa46f1905ca1ff32a3684a2616a7e482c") + (revision "1")) + (package + (name "julia-lmgpu") + (version (git-version "0.1.1" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/ChelseaTrotter/LMGPU.jl") + (commit commit))) + ;(file-name (git-file-name name version)) + (file-name "LMGPU") + (sha256 + (base32 + "1ddx2np1lakw1l2dclpcaihxd0fcj6bjxsvaxr6g5brxjqk5j7b1")))) + (build-system julia-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + ;; This is a super ugly hack. Some JULIA environment variable should + ;; be tuned so it can find the artifact directory. + (add-after 'unpack 'symlink-zlib-into-artifact-directory + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((julia-dir (string-append (assoc-ref outputs "out") + "/share/julia"))) + (mkdir-p julia-dir) + (symlink + (string-append (assoc-ref inputs "julia-zlib-jll") + "/share/julia/artifacts") + (string-append julia-dir "/artifacts"))) + #t)) + (add-after 'precompile 'check + (lambda* (#:key outputs #:allow-other-keys) + (let* ((out (assoc-ref outputs "out")) + (builddir (string-append out "/share/julia/"))) + (setenv "JULIA_LOAD_PATH" + (string-append builddir "packages/" ":" + (or (getenv "JULIA_LOAD_PATH") + ""))) + (setenv "HOME" (getcwd)) + (invoke "julia" "test/runtests.jl"))))))) + (native-inputs + `(("r" ,r-minimal) + ("r-mice" ,r-mice) + ("r-qtl2" ,r-qtl2) + ("r-tictoc" ,r-tictoc) + ("r-tidyverse" ,r-tidyverse))) + (propagated-inputs + `(("julia-zipfile" ,julia-zipfile))) + (home-page "https://github.com/ChelseaTrotter/LMGPU.jl") + (synopsis "") + (description "") + (license license:expat)))) + +(define-public julia-lmgpu-myapp + (package + (inherit julia-lmgpu) + (name "julia-lmgpu-myapp") + (source + (origin (inherit (package-source julia-lmgpu)) + (file-name "MyApp"))) + (arguments + (substitute-keyword-arguments (package-arguments julia-lmgpu) + ((#:phases phases) + `(modify-phases ,phases + (add-after 'unpack 'change-directory + (lambda _ + (chdir "bin/MyApp") #t)))))) + (propagated-inputs + `(("julia-lmgpu" ,julia-lmgpu) + ,@(package-propagated-inputs julia-lmgpu))) + (native-inputs + `(("julia-packagecompiler" ,julia-packagecompiler))))) + +(define-public julia-zipfile + (package + (name "julia-zipfile") + (version "0.9.1") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/fhs/ZipFile.jl") + (commit (string-append "v" version)))) + ;(file-name (git-file-name name version)) + (file-name "ZipFile") + (sha256 + (base32 + "1fpvlhfqg5kgq5vchlf8dyc73r6dzki0dz7plddc3bnr0ld00rlw")))) + (build-system julia-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + ;; This is a super ugly hack. Some JULIA environment variable should + ;; be tuned so it can find the artifact directory. + (add-after 'unpack 'symlink-zlib-into-artifact-directory + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((julia-dir (string-append (assoc-ref outputs "out") + "/share/julia"))) + (mkdir-p julia-dir) + (symlink + (string-append (assoc-ref inputs "julia-zlib-jll") + "/share/julia/artifacts") + (string-append julia-dir "/artifacts"))) + #t))))) + (propagated-inputs + `(("julia-zlib-jll" ,julia-zlib-jll))) + (home-page "https://github.com/fhs/ZipFile.jl") + (synopsis "Read/Write ZIP archives in Julia") + (description "This module provides support for reading and writing ZIP +archives in Julia.") + (license license:expat))) + +(define-public julia-zlib-jll + (package + (name "julia-zlib-jll") + (version "1.2.11+9") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/JuliaBinaryWrappers/Zlib_jll.jl") + (commit (string-append "Zlib-v" version)))) + ;(file-name (git-file-name name version)) + (file-name "Zlib_jll") + (sha256 + (base32 + "0m9n8dp4bwhkyjag1szmhz02k0bxzm4ka2ia2jh8crnd1qi8w9dz")))) + (build-system julia-build-system) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'symlink-zlib-into-artifact-directory + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((artifacts (string-append (assoc-ref outputs "out") + "/share/julia/artifacts"))) + (mkdir-p artifacts) + (symlink + (assoc-ref inputs "zlib") + ;; from git-tree-sha1 in Artifacts.toml + (string-append + artifacts + ,(match (%current-system) + ("x86_64-linux" "/7846a2956a213715c2c76632f3461cef87d9d545") + ("i686-linux" "/c8456cbd00982236828623bbc63f21b9b7b03821") + ("armhf-linux" "/748c38025b5596a5005a87ac2b9476603cf8615b") + ("aarch64-linux" "/3dd0c7cd5424c8746a1a32034ba1b10458f20b3b") + (_ "/UNSUPPORTED"))))) + #t))))) + (native-inputs + `(("zlib" ,zlib))) + (home-page "https://github.com/JuliaBinaryWrappers/Zlib_jll.jl") + (synopsis "Autogenerated package constructed using BinaryBuilder.jl") + (description "This is an autogenerated package constructed using +@url{https://github.com/JuliaPackaging/BinaryBuilder.jl, BinaryBuilder.jl}.") + (license license:expat))) + +(define-public julia-packagecompiler + (package + (name "julia-packagecompiler") + (version "1.1.1") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/JuliaLang/PackageCompiler.jl") + (commit (string-append "v" version)))) + ;(file-name (git-file-name name version)) + (file-name "PackageCompiler") + (sha256 + (base32 + "1s9xc17i308fdpyvkz1w6qb1h7yncdr2jgk1szfvygxd6yzkv1b4")))) + (build-system julia-build-system) + (home-page "https://github.com/JuliaLang/PackageCompiler.jl") + (synopsis "Compile your Julia Package") + (description "PackageCompiler is a Julia package with two main purposes: +@itemize +@item Creating custom sysimages for reduced latency when working locally with +packages that has a high startup time. +@item Creating \"apps\" which are a bundle of files including an executable that +can be sent and run on other machines without Julia being installed on that machine. +@end itemize") + (license license:expat))) diff --git a/gn/packages/jupyterhub.scm b/gn/packages/jupyterhub.scm index 5b69b5b..43da8ec 100644 --- a/gn/packages/jupyterhub.scm +++ b/gn/packages/jupyterhub.scm @@ -15,6 +15,7 @@ #:use-module (gnu packages python-crypto) #:use-module (gnu packages python-web) #:use-module (gnu packages python-xyz) + #:use-module (gnu packages rpc) #:use-module (gnu packages serialization) #:use-module (gnu packages time) #:use-module (gn packages node)) diff --git a/gn/packages/maths.scm b/gn/packages/maths.scm index 9e15400..1c1c1ae 100644 --- a/gn/packages/maths.scm +++ b/gn/packages/maths.scm @@ -30,6 +30,7 @@ ((#:configure-flags cf) `(cons "--enable-docs=no" ; docs fail to build ,cf)) + ((#:tests? _ #f) #f) ; tests hang ((#:phases phases) `(modify-phases ,phases (add-after 'unpack 'patch-configure-script diff --git a/gn/packages/python.scm b/gn/packages/python.scm index 60558a2..7e4786f 100644 --- a/gn/packages/python.scm +++ b/gn/packages/python.scm @@ -3,6 +3,7 @@ #:use-module (gnu packages) #:use-module (gnu packages attr) #:use-module (gnu packages base) + #:use-module (gnu packages bioinformatics) #:use-module (gnu packages check) #:use-module (gnu packages compression) #:use-module (gnu packages databases) @@ -30,6 +31,7 @@ #:use-module (gnu packages python-xyz) #:use-module (gnu packages rdf) #:use-module (gnu packages readline) + #:use-module (gnu packages serialization) #:use-module (gnu packages statistics) #:use-module (gnu packages tcl) #:use-module (gnu packages tex) @@ -228,25 +230,6 @@ until a value is returned.") "A library for W3C Provenance Data Model supporting PROV-JSON, PROV-XML and PROV-O (RDF)") (license license:expat))) -(define-public python-typing-extensions; guix candidate - (package - (name "python-typing-extensions") - (version "3.6.6") - (source - (origin - (method url-fetch) - (uri (pypi-uri "typing_extensions" version)) - (sha256 - (base32 - "07vhddjnd3mhdijyc3s0mwi9jgfjp3rr056nxqiavydbvkrvgrsi")))) - (build-system python-build-system) - (home-page "https://pypi.python.org/pypi/typing_extensions") - (synopsis "Python typing_extensions.") - (description - "Python typing_extensions.") - (license license:gpl2)) -) - (define-public python-subprocess32 ; guix candidate (package (name "python-subprocess32") @@ -322,39 +305,6 @@ until a value is returned.") (define-public python2-xlsxwriter (package-with-python2 python-xlsxwriter)) -(define-public python-rdflib-jsonld ; guix ready - (package - (name "python-rdflib-jsonld") - (version "0.4.0") - (source - (origin - (method url-fetch) - (uri (pypi-uri "rdflib-jsonld" version)) - (sha256 - (base32 - "0bdw2pbjmpy1l4p6slsjn54bqy6crk5hk4san84xxirgd9w78iql")))) - (build-system python-build-system) - (inputs - `(("python-setuptools" ,python-setuptools))) - (propagated-inputs - `(("python-rdflib" ,python-rdflib) - ("python-isodate" ,python-isodate) - ("python-pyparsing" ,python-pyparsing) - ("python-html5lib" ,python-html5lib) - ("python-nose" ,python-nose) -)) - (home-page - "https://github.com/RDFLib/rdflib-jsonld") - (synopsis - "rdflib extension adding JSON-LD parser and serializer") - (description - "rdflib extension adding JSON-LD parser and serializer") - (license license:bsd-3))) - -(define-public python2-rdflib-jsonld - (package-with-python2 python-rdflib-jsonld)) - - (define-public python-rserve (package (name "python-rserve") @@ -1094,3 +1044,142 @@ spreadsheets without the need for COM objects.") (synopsis "") (description "") (license license:bsd-4))) + +(define-public python-admiral + (package + (name "python-admiral") + (version "0.2") + (source + (origin + (method url-fetch) + (uri (pypi-uri "admiral" version)) + (sha256 + (base32 + "1b2zjgyz94ld5wr7s4cm4x5sxijx3w0dmd7r2cq1s8iqjzz6rd1x")))) + (build-system python-build-system) + (arguments '(#:tests? #f)) ; No tests + (propagated-inputs + `(("python-humanfriendly" ,python-humanfriendly))) + (home-page "https://github.com/nspies/admiral") + (synopsis + "Simple python high-performance computing cluster batch submission") + (description + "Simple python high-performance computing cluster batch submission.") + (license #f))) ; No license in repository. + +(define-public python2-admiral + (package-with-python2 python-admiral)) + +(define-public python-cachecontrol-0.11 + (package + (inherit python-cachecontrol) + (name "python-cachecontrol") + (version "0.11.7") + (source + (origin + (method url-fetch) + (uri (pypi-uri "CacheControl" version)) + (sha256 + (base32 + "07jsfhlbcwgqg6ayz8nznzaqg5rmxqblbzxz1qvg5wc44pcjjy4g")))))) + +(define-public python-ruamel.yaml-0.15 + (package + (inherit python-ruamel.yaml) + (name "python-ruamel.yaml") + (version "0.15.77") + (source + (origin + (method url-fetch) + (uri (pypi-uri "ruamel.yaml" version)) + (sha256 + (base32 + "1mhzxkkiv3xmr9izrgk78x7f1r5gi8kd5ac7z3vn7j00q1ydn6da")))))) + +(define-public python-pbr-1.6.0 + (package + (inherit python-pbr) + (name "python-pbr") + (version "1.6.0") + (source + (origin + (method url-fetch) + (uri (pypi-uri "pbr" version)) + (sha256 + (base32 + "1lg1klrczvzfan89y3bl9ykrknl3nb01vvai37fkww24apzyibjf")))))) + +(define-public python-arvados-python-client + (package + (name "python-arvados-python-client") + (version "2.0.2") + (source + (origin + (method url-fetch) + (uri (pypi-uri "arvados-python-client" version)) + (sha256 + (base32 + "19l4w6m5426x5k2kick630dh2jx26j16ycs2nhbfgr4cd43d29y4")))) + (build-system python-build-system) + (arguments + `(#:tests? #f)) ; tests not included? + (propagated-inputs + `(("python-ciso8601" ,python-ciso8601) + ("python-future" ,python-future) + ;("python-google-api-python-client" ,python-google-api-python-client) + ("python-google-api-client" ,python-google-api-client) + ("python-httplib2" ,python-httplib2) + ("python-pycurl" ,python-pycurl) + ("python-ruamel.yaml" ,python-ruamel.yaml-0.15) + ("python-setuptools" ,python-setuptools) + ("python-ws4py" ,python-ws4py))) + (native-inputs + `(("python-mock" ,python-mock) + ("python-oauth2client" ,python-oauth2client) + ("python-pbr" ,python-pbr-1.6.0) + ("python-pyyaml" ,python-pyyaml) + ("python-uritemplate" ,python-uritemplate))) + (home-page "https://arvados.org") + (synopsis "Arvados client library") + (description "This package provides the arvados module, an API client for +Arvados. It also includes higher-level functions to help you write Crunch +scripts, and command-line tools to store and retrieve data in the Keep storage +server.") + (license license:asl2.0))) + +(define-public python-schema-salad + (package + (name "python-schema-salad") + (version "5.0.20200416112825") + (source + (origin + (method url-fetch) + (uri (pypi-uri "schema-salad" version)) + (sha256 + (base32 + "1pm6q266qrw4r0w0vnzhsvqgk5j8b3q61hxg99awhgpjsmcvkmsz")))) + (build-system python-build-system) + (propagated-inputs + `(("python-cachecontrol" ,python-cachecontrol-0.11) + ("python-lockfile" ,python-lockfile) + ("python-mistune" ,python-mistune) + ("python-rdflib" ,python-rdflib) + ("python-rdflib-jsonld" ,python-rdflib-jsonld) + ("python-requests" ,python-requests) + ("python-ruamel.yaml" ,python-ruamel.yaml) + ("python-setuptools" ,python-setuptools) + ("python-typing-extensions" ,python-typing-extensions))) + (native-inputs + `(("python-pytest" ,python-pytest) + ("python-pytest-runner" ,python-pytest-runner))) + (home-page "https://github.com/common-workflow-language/schema_salad") + (synopsis "Schema Annotations for Linked Avro Data (SALAD)") + (description + "Salad is a schema language for describing JSON or YAML structured linked +data documents. Salad schema describes rules for preprocessing, structural +validation, and hyperlink checking for documents described by a Salad schema. +Salad supports rich data modeling with inheritance, template specialization, +object identifiers, object references, documentation generation, code +generation, and transformation to RDF. Salad provides a bridge between document +and record oriented data modeling and the Semantic Web.") + (license license:asl2.0))) diff --git a/gn/packages/ratspub.scm b/gn/packages/ratspub.scm index 29e1313..b664799 100644 --- a/gn/packages/ratspub.scm +++ b/gn/packages/ratspub.scm @@ -1,11 +1,16 @@ (define-module (gn packages ratspub) #:use-module ((guix licenses) #:prefix license:) + #:use-module (guix utils) + #:use-module (gnu packages) #:use-module (guix packages) #:use-module (guix git-download) #:use-module (guix build-system python) #:use-module (gnu packages admin) - #:use-module (gn packages bioinformatics) + #:use-module (gnu packages bioinformatics) #:use-module (gn packages javascript) + #:use-module (gnu packages machine-learning) + #:use-module (gnu packages python) + #:use-module (gnu packages python-crypto) #:use-module (gnu packages python-web) #:use-module (gnu packages python-xyz) #:use-module (gn packages web)) @@ -13,19 +18,27 @@ (define-public ratspub (package (name "ratspub") - (version "0.1") + (version "0.3.1") (source (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/chen42/ratspub.git") - (commit (string-append "v" version)))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "0cm9g38fxpa52458mdmhzhghj5c7b8l3k1b764zs9hdrki5s7wi7")))) + (method git-fetch) + (uri (git-reference + (url "https://github.com/chen42/ratspub.git") + (commit (string-append "v" version)))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1ii3721mqd3dbpjkhqi7yqjd9bqcf0g19kdbb8265pmbfjjsg164")) + (modules '((guix build utils))) + (snippet + '(begin (substitute* "server.py" + ;; Keep the service running on port 4200 + (("4201") "4200") + ;; Backport to python-keras-2.2.4 + (("learning_rate") "lr") ) + #t)))) (build-system python-build-system) (arguments - `(#:tests? #f ; no test suite + `(#:tests? #f ; no test suite #:phases (modify-phases %standard-phases (delete 'configure) @@ -34,12 +47,24 @@ (lambda* (#:key inputs outputs #:allow-other-keys) (let ((out (assoc-ref outputs "out")) (inetutils (assoc-ref inputs "inetutils"))) - (substitute* "templates/cytoscape.html" + (substitute* '("templates/cytoscape.html" + "templates/tableview.html" + "templates/tableview0.html" + "templates/userarchive.html") (("script src=.*") "script src=\"/static/cytoscape.min.js\"></script>\n")) (substitute* "templates/layout.html" - (("https://stackpath.bootstrapcdn.com/bootstrap/.*") - "/static/bootstrap.min.css\">\n")) + (("https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css.*") + "/static/bootstrap.min.css\">\n") + (("https://.*.bootstrapcdn.com/bootstrap/4.*/js/bootstrap.min.js.*") + "/static/bootstrap.min.js\"></script>\n") + (("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css") + "/static/font-awesome.min.css") + (("https://code.jquery.com/jquery-3.2.1.slim.min.js.*") + "/static/jquery.slim.min.js\"></script>\n") + ;(("https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js.*") + ; "/static/popper.min.js\"></script>\n") + ) (substitute* "ratspub.py" (("hostname") (string-append inetutils "/bin/hostname")))) #t)) @@ -51,31 +76,58 @@ (add-after 'install 'install-javascript (lambda* (#:key inputs outputs #:allow-other-keys) (let ((out (assoc-ref outputs "out")) + (awesome (assoc-ref inputs "font-awesome")) + (bootstrap (assoc-ref inputs "bootstrap")) (cytoscape (assoc-ref inputs "cytoscape")) - (bootstrap (assoc-ref inputs "bootstrap"))) + (jquery (assoc-ref inputs "jquery")) + ;(js-popper (assoc-ref inputs "js-popper")) + ) + (symlink (string-append awesome + "/share/web/font-awesomecss/font-awesome.min.css") + (string-append out "/static/font-awesome.min.css")) + (symlink (string-append bootstrap + "/share/web/bootstrap/css/bootstrap.min.css") + (string-append out "/static/bootstrap.min.css")) + (symlink (string-append bootstrap + "/share/web/bootstrap/js/bootstrap.min.js") + (string-append out "/static/bootstrap.min.js")) (symlink (string-append cytoscape "/share/genenetwork2/javascript/cytoscape/cytoscape.min.js") (string-append out "/static/cytoscape.min.js")) - (symlink (string-append bootstrap - "/share/web/bootstrap/css/bootstrap.min.css") - (string-append out "/static/bootstrap.min.css"))) + (symlink (string-append jquery + "/share/web/jquery/jquery.slim.min.js") + (string-append out "/static/jquery.slim.min.js")) + ;(symlink (string-append js-popper + ; "/share/web/popper/popper.min.js") + ; (string-append out "/static/popper.min.js")) + ) #t)) (add-after 'install 'wrap-executable (lambda* (#:key inputs outputs #:allow-other-keys) (let ((out (assoc-ref outputs "out")) (path (getenv "PYTHONPATH"))) (wrap-program (string-append out "/server.py") - `("PATH" ":" prefix (,(dirname (which "edirect.pl")))) - `("PYTHONPATH" ":" prefix (,path)))) + `("PATH" ":" prefix (,(dirname (which "edirect.pl")) + ,(dirname (which "dirname")) + ,(dirname (which "grep")) + ,(dirname (which "sed")))) + `("PYTHONPATH" ":" prefix (,path)))) #t))))) (inputs - `(("edirect" ,edirect-gn) + `(("edirect" ,edirect) ("inetutils" ,inetutils) - ("python-flask" ,python-flask) - ("python-nltk" ,python-nltk))) + ("python-bcrypt" ,python-bcrypt) + ("python-flask-sqlalchemy" ,python-flask-sqlalchemy) + ("python-keras" ,python-keras-for-ratspub) + ("python-nltk" ,python-nltk) + ("tensorflow" ,tensorflow))) (native-inputs - `(("cytoscape" ,javascript-cytoscape) - ("bootstrap" ,web-bootstrap))) + `(("bootstrap" ,web-bootstrap) + ("cytoscape" ,javascript-cytoscape) + ("font-awesome" ,web-font-awesome) + ("jquery" ,web-jquery) + ;("js-popper" ,js-popper) ; empty output + )) (home-page "http://rats.pub/") (synopsis "Relationship with Addiction Through Searches of PubMed") (description @@ -86,4 +138,113 @@ the question \"What do we know about these genes and addiction?\". Data from @acronym{EBI GWAS, European Bioinformatics Institute Genome-Wide Association Studies} catalog are also included in the search to better answer this question.") - (license #f))) + (license license:expat))) + +;; We want a copy of python-keras with the AUC optimizer backported. +;; We skip the tests because we "test in production". +;; That's a lie. The test suite just takes a long time to run. +(define-public python-keras-for-ratspub + (hidden-package + (package + (inherit python-keras) + (source + (origin + (inherit (package-source python-keras)) + (patches (search-patches "keras-auc-optimizer.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments python-keras) + ((#:phases phases) + `(modify-phases ,phases + (delete 'check)))))))) + +(define-public hrdp-project + (package + (name "hrdp-project") + (version "0.1") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/noderboarder/hrdp-project") + (commit (string-append "v" version)))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1ag7jm43p35yh0cqcn53wg4iw7sgfypw10mxq5klxvhgj3r6cf7i")))) + (build-system python-build-system) + (arguments + `(#:tests? #f ; no test suite + #:phases + (modify-phases %standard-phases + (delete 'configure) + (delete 'build) + (add-after 'unpack 'patch-sources + (lambda _ + (substitute* "./app/templates/layout.html" + (("https://.*.bootstrapcdn.com/bootstrap/4.*/css/bootstrap.min.css.*") + "/static/bootstrap.min.css\">\n") + (("https://.*.bootstrapcdn.com/bootstrap/4.*/js/bootstrap.min.js.*") + "/static/bootstrap.min.js\"></script>\n") + (("https://code.jquery.com/jquery-3.*.slim.min.js.*") + "/static/jquery.slim.min.js\"></script>\n") + ;(("https://cdn.jsdelivr.net/npm/popper.js@1.16.0/dist/umd/popper.min.js.*") + ; "/static/popper.min.js\"></script>\n") + ) + #t)) + (replace 'install + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out")) + (python (assoc-ref inputs "python"))) + (delete-file "main.py") + (with-output-to-file "main.py" + (lambda () + (format #t "#!~a/bin/python +from app import create_app + +app = create_app() + +if __name__ == '__main__': + app.run(debug=True, port=4222)~%" + python))) + (chmod "main.py" #o555) + (copy-recursively "." out)) + #t)) + (add-after 'install 'install-javascript + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out")) + (bootstrap (assoc-ref inputs "bootstrap")) + (jquery (assoc-ref inputs "jquery")) + ;(js-popper (assoc-ref inputs "js-popper")) + ) + (symlink (string-append bootstrap + "/share/web/bootstrap/css/bootstrap.min.css") + (string-append out "/app/static/bootstrap.min.css")) + (symlink (string-append bootstrap + "/share/web/bootstrap/js/bootstrap.min.js") + (string-append out "/app/static/bootstrap.min.js")) + (symlink (string-append jquery + "/share/web/jquery/jquery.slim.min.js") + (string-append out "/app/static/jquery.slim.min.js")) + ;(symlink (string-append js-popper + ; "/share/web/popper/popper.min.js") + ; (string-append out "/static/popper.min.js")) + ) + #t)) + (add-after 'install 'wrap-executable + (lambda* (#:key inputs outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out")) + (path (getenv "PYTHONPATH"))) + (wrap-program (string-append out "/main.py") + `("PYTHONPATH" ":" prefix (,path)))) + #t))))) + (inputs + `(("python" ,python) + ("python-flask-sqlalchemy" ,python-flask-sqlalchemy))) + (native-inputs + `(("bootstrap" ,web-bootstrap) + ("jquery" ,web-jquery) + ;("js-popper" ,js-popper) ; empty output + )) + (home-page "https://github.com/noderboarder/hrdp-project") + (synopsis "") + (description "") + (license license:expat))) diff --git a/gn/services/archive-pubmed.service b/gn/services/archive-pubmed.service new file mode 100644 index 0000000..fd00b71 --- /dev/null +++ b/gn/services/archive-pubmed.service @@ -0,0 +1,13 @@ +[Unit] +Description=Download PubMed Archives +After=network-online.target +Wants=network-online.target +AssertPathExists=/export2/PubMed + +[Service] +ExecStart=/usr/local/guix-profiles/ratspub/bin/archive-pubmed +Environment="EDIRECT_PUBMED_MASTER=/export2/PubMed" "PERL_LWP_SSL_CA_FILE=/etc/ssl/certs/ca-certificates.crt" "PATH=/usr/local/guix-profiles/ratspub/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/usr/sbin" +User=hchen + +[Install] +WantedBy=multi-user.target diff --git a/gn/services/archive-pubmed.timer b/gn/services/archive-pubmed.timer new file mode 100644 index 0000000..4bb27ba --- /dev/null +++ b/gn/services/archive-pubmed.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily PubMed downloads + +[Timer] +OnBootSec=5h +OnUnitInactiveSec=15h + +[Install] +WantedBy=default.target diff --git a/gn/services/bnw-README b/gn/services/bnw-README index ab7de53..d70b7fc 100644 --- a/gn/services/bnw-README +++ b/gn/services/bnw-README @@ -6,8 +6,8 @@ $ cd guix-bioinformatics && git pull # systemctl enable --now bnw.service for testing changes to the container, you'll want to do the following (or thereabouts): -$ guix system container gn/services/GN1-container.scm --share=/home/bnw/server=/srv/http --network -$ sudo -E $(guix system container gn/services/GN1-container.scm --share=/home/bnw/server=/srv/http --network) +$ guix system container gn/servicebnwN1-container.scm --network +$ sudo -E $(guix system container gn/services/bnw-container.scm --network) for running the service: see included bnw.service diff --git a/gn/services/bnw-container.scm b/gn/services/bnw-container.scm index 8463a75..39da38d 100644 --- a/gn/services/bnw-container.scm +++ b/gn/services/bnw-container.scm @@ -21,10 +21,6 @@ (match-lambda (($ <bnw-configuration> package deploy-directory port) #~(begin - (use-modules (guix build utils)) - (when (directory-exists? #$deploy-directory) - ;; Not 'delete-file-recursively' because the directory might be empty. - (system* "rm" "-r" #$(string-append deploy-directory "/*"))) (mkdir-p #$deploy-directory) (copy-recursively #$package #$deploy-directory) (invoke #$(file-append coreutils "/bin/chmod") "a+w" diff --git a/gn/services/bnw.service b/gn/services/bnw.service index 5ba8f3f..6ac99a1 100644 --- a/gn/services/bnw.service +++ b/gn/services/bnw.service @@ -3,7 +3,7 @@ Description=BNW web server Wants=guix-daemon.service [Service] -Environment="BNW_COMMAND=$(/bin/su -l bnw -c '/var/guix/profiles/per-user/bnw/current-guix/bin/guix system container /home/bnw/guix-bioinformatics/gn/services/bnw-container.scm --share=/home/bnw/server=/srv/http --share=/home/bnw/server/var-log=/var/log --network')" +Environment="BNW_COMMAND=$(/bin/su -l bnw -c '/var/guix/profiles/per-user/bnw/current-guix/bin/guix system container /home/bnw/guix-bioinformatics/gn/services/bnw-container.scm --share=/home/bnw/server/var-log=/var/log --network')" ExecStart=/bin/bash -c '${BNW_COMMAND}' [Install] diff --git a/gn/services/genenetwork.scm b/gn/services/genenetwork.scm index 0978cd2..4693152 100644 --- a/gn/services/genenetwork.scm +++ b/gn/services/genenetwork.scm @@ -2,13 +2,13 @@ (use-modules (gnu) (gn packages genenetwork) - (gn packages python) + (gn packages python24) (gn packages web)) (use-service-modules web) (use-package-modules python) (define %mod-python-path - (file-append mod-python "/lib/python2.7/site-packages")) + (file-append mod-python "/lib/python2.4/site-packages")) (operating-system (host-name "genenetwork") @@ -22,15 +22,15 @@ ;; No firmware for VMs (firmware '()) - (packages (cons* python-2 + (packages (cons* python-2.4 mod-python - python2-qtlreaper - python2-htmlgen-gn - python2-json-GN1 - python2-piddle - python2-pyx-GN1 - python2-pyxlwriter - python2-svg-GN1 + python24-qtlreaper + ;python24-htmlgen-gn + python24-json-GN1 + python24-piddle + python24-pyx-GN1 + python24-pyxlwriter + python24-svg-GN1 %base-packages)) (services (list (service httpd-service-type @@ -38,7 +38,7 @@ (config (httpd-config-file (server-name "www.genenetwork.org") - (document-root (file-append genenetwork "/web")) + (document-root (file-append genenetwork1 "/web")) (listen '("8811")) (modules (cons* (httpd-module @@ -46,8 +46,8 @@ (file (file-append mod-python "/modules/mod_python.so"))) %default-httpd-modules)) (extra-config (list "\ -PythonPath \"sys.path+['" %mod-python-path "', '" (file-append genenetwork "/web/webqtl") "']\" -<Directory " (file-append genenetwork "/web/webqtl") "> +PythonPath \"sys.path+['" %mod-python-path "', '" (file-append genenetwork1 "/web/webqtl") "']\" +<Directory " (file-append genenetwork1 "/web/webqtl") "> SetHandler python-program PythonHandler mod_python.publisher PythonAutoReload Off diff --git a/keras-auc-optimizer.patch b/keras-auc-optimizer.patch new file mode 100644 index 0000000..bbc6924 --- /dev/null +++ b/keras-auc-optimizer.patch @@ -0,0 +1,1133 @@ +From 901159da45695da24a5206125910f02fc50169ce Mon Sep 17 00:00:00 2001 +From: Efraim Flashner <efraim@flashner.co.il> +Date: Thu, 23 Apr 2020 15:50:37 +0300 +Subject: [PATCH] add keras metrics + +--- + keras/backend/tensorflow_backend.py | 12 + + keras/metrics.py | 584 ++++++++++++++++++++++++++++ + keras/utils/__init__.py | 2 + + keras/utils/losses_utils.py | 177 +++++++++ + keras/utils/metrics_utils.py | 278 +++++++++++++ + 5 files changed, 1053 insertions(+) + create mode 100644 keras/utils/losses_utils.py + create mode 100644 keras/utils/metrics_utils.py + +diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py +index bcb8be0..a2870f5 100644 +--- a/keras/backend/tensorflow_backend.py ++++ b/keras/backend/tensorflow_backend.py +@@ -4453,3 +4453,15 @@ def local_conv2d(inputs, kernel, kernel_size, strides, output_shape, data_format + else: + output = permute_dimensions(output, (2, 0, 1, 3)) + return output ++ ++#get_graph = tf_keras_backend.get_graph ++ ++#def is_symbolic(x): ++# return isinstance(x, tf.Tensor) and hasattr(x, 'op') ++ ++def size(x, name=None): ++# if is_symbolic(x): ++# with get_graph().as_default(): ++# return tf.size(x) ++ return tf.size(x, name=name) ++ +diff --git a/keras/metrics.py b/keras/metrics.py +index 8e3df1f..8f57910 100644 +--- a/keras/metrics.py ++++ b/keras/metrics.py +@@ -4,8 +4,12 @@ from __future__ import absolute_import + from __future__ import division + from __future__ import print_function + ++import abc + import six ++import types ++ + from . import backend as K ++from .engine.base_layer import Layer + from .losses import mean_squared_error + from .losses import mean_absolute_error + from .losses import mean_absolute_percentage_error +@@ -19,10 +23,201 @@ from .losses import binary_crossentropy + from .losses import kullback_leibler_divergence + from .losses import poisson + from .losses import cosine_proximity ++from .utils import losses_utils ++from .utils import metrics_utils + from .utils.generic_utils import deserialize_keras_object + from .utils.generic_utils import serialize_keras_object + + ++@six.add_metaclass(abc.ABCMeta) ++class Metric(Layer): ++ """Encapsulates metric logic and state. ++ ++ Standalone usage: ++ ```python ++ m = SomeMetric(...) ++ for input in ...: ++ m.update_state(input) ++ m.result() ++ ``` ++ ++ Usage with the `compile` API: ++ ```python ++ model.compile(optimizer='rmsprop', ++ loss=keras.losses.categorical_crossentropy, ++ metrics=[keras.metrics.CategoricalAccuracy()]) ++ ``` ++ ++ To be implemented by subclasses: ++ * `__init__()`: All state variables should be created in this method by ++ calling `self.add_weight()` like: `self.var = self.add_weight(...)` ++ * `update_state()`: Has all updates to the state variables like: ++ self.var.assign_add(...). ++ * `result()`: Computes and returns a value for the metric ++ from the state variables. ++ """ ++ ++ def __init__(self, name=None, dtype=None, **kwargs): ++ super(Metric, self).__init__(name=name, dtype=dtype, **kwargs) ++ self.stateful = True # All metric layers are stateful. ++ self.built = True ++ self.dtype = K.floatx() if dtype is None else dtype ++ ++ def __new__(cls, *args, **kwargs): ++ obj = super(Metric, cls).__new__(cls) ++ update_state_fn = obj.update_state ++ ++ obj.update_state = types.MethodType( ++ metrics_utils.update_state_wrapper(update_state_fn), obj) ++ return obj ++ ++ def __call__(self, *args, **kwargs): ++ """Accumulates statistics and then computes metric result value.""" ++ update_op = self.update_state(*args, **kwargs) ++ return self.result() ++ ++ def get_config(self): ++ """Returns the serializable config of the metric.""" ++ return {'name': self.name, 'dtype': self.dtype} ++ ++ def reset_states(self): ++ """Resets all of the metric state variables. ++ This function is called between epochs/steps, ++ when a metric is evaluated during training. ++ """ ++ K.batch_set_value([(v, 0) for v in self.weights]) ++ ++ @abc.abstractmethod ++ def update_state(self, *args, **kwargs): ++ """Accumulates statistics for the metric. """ ++ raise NotImplementedError('Must be implemented in subclasses.') ++ ++ @abc.abstractmethod ++ def result(self): ++ """Computes and returns the metric value tensor. ++ Result computation is an idempotent operation that simply calculates the ++ metric value using the state variables. ++ """ ++ raise NotImplementedError('Must be implemented in subclasses.') ++ ++ # For use by subclasses # ++ def add_weight(self, ++ name, ++ shape=(), ++ initializer=None, ++ dtype=None): ++ """Adds state variable. Only for use by subclasses.""" ++ return super(Metric, self).add_weight( ++ name=name, ++ shape=shape, ++ dtype=self.dtype if dtype is None else dtype, ++ trainable=False, ++ initializer=initializer) ++ ++ # End: For use by subclasses ### ++ ++ ++class Reduce(Metric): ++ """Encapsulates metrics that perform a reduce operation on the values.""" ++ ++ def __init__(self, reduction, name, dtype=None): ++ """Creates a `Reduce` instance. ++ # Arguments ++ reduction: a metrics `Reduction` enum value. ++ name: string name of the metric instance. ++ dtype: (Optional) data type of the metric result. ++ """ ++ super(Reduce, self).__init__(name=name, dtype=dtype) ++ self.reduction = reduction ++ self.total = self.add_weight('total', initializer='zeros') ++ if reduction in [metrics_utils.Reduction.SUM_OVER_BATCH_SIZE, ++ metrics_utils.Reduction.WEIGHTED_MEAN]: ++ self.count = self.add_weight('count', initializer='zeros') ++ ++ def update_state(self, values, sample_weight=None): ++ """Accumulates statistics for computing the reduction metric. ++ For example, if `values` is [1, 3, 5, 7] and reduction=SUM_OVER_BATCH_SIZE, ++ then the value of `result()` is 4. If the `sample_weight` is specified as ++ [1, 1, 0, 0] then value of `result()` would be 2. ++ # Arguments ++ values: Per-example value. ++ sample_weight: Optional weighting of each example. Defaults to 1. ++ """ ++ values = K.cast(values, self.dtype) ++ if sample_weight is not None: ++ sample_weight = K.cast(sample_weight, self.dtype) ++ # Update dimensions of weights to match with values if possible. ++ values, _, sample_weight = losses_utils.squeeze_or_expand_dimensions( ++ values, sample_weight=sample_weight) ++ ++ # Broadcast weights if possible. ++ sample_weight = losses_utils.broadcast_weights(sample_weight, values) ++ values = values * sample_weight ++ ++ value_sum = K.sum(values) ++ update_total_op = K.update_add(self.total, value_sum) ++ ++ # Exit early if the reduction doesn't have a denominator. ++ if self.reduction == metrics_utils.Reduction.SUM: ++ return update_total_op ++ ++ # Update `count` for reductions that require a denominator. ++ if self.reduction == metrics_utils.Reduction.SUM_OVER_BATCH_SIZE: ++ num_values = K.cast(K.size(values), self.dtype) ++ elif self.reduction == metrics_utils.Reduction.WEIGHTED_MEAN: ++ if sample_weight is None: ++ num_values = K.cast(K.size(values), self.dtype) ++ else: ++ num_values = K.sum(sample_weight) ++ else: ++ raise NotImplementedError( ++ 'reduction [%s] not implemented' % self.reduction) ++ ++ with K.control_dependencies([update_total_op]): ++ return K.update_add(self.count, num_values) ++ ++ def result(self): ++ if self.reduction == metrics_utils.Reduction.SUM: ++ return self.total ++ elif self.reduction in [ ++ metrics_utils.Reduction.WEIGHTED_MEAN, ++ metrics_utils.Reduction.SUM_OVER_BATCH_SIZE ++ ]: ++ return self.total / self.count ++ else: ++ raise NotImplementedError( ++ 'reduction [%s] not implemented' % self.reduction) ++ ++ ++class Sum(Reduce): ++ """Computes the (weighted) sum of the given values. ++ ++ For example, if values is [1, 3, 5, 7] then the sum is 16. ++ If the weights were specified as [1, 1, 0, 0] then the sum would be 4. ++ ++ This metric creates one variable, `total`, that is used to compute the sum of ++ `values`. This is ultimately returned as `sum`. ++ If `sample_weight` is `None`, weights default to 1. Use `sample_weight` of 0 ++ to mask values. ++ ++ Standalone usage: ++ ```python ++ m = keras.metrics.Sum() ++ m.update_state([1, 3, 5, 7]) ++ m.result() ++ ``` ++ """ ++ ++ def __init__(self, name='sum', dtype=None): ++ """Creates a `Sum` instance. ++ # Arguments ++ name: (Optional) string name of the metric instance. ++ dtype: (Optional) data type of the metric result. ++ """ ++ super(Sum, self).__init__(reduction=metrics_utils.Reduction.SUM, ++ name=name, dtype=dtype) ++ ++ + def binary_accuracy(y_true, y_pred): + return K.mean(K.equal(y_true, K.round(y_pred)), axis=-1) + +@@ -49,6 +244,395 @@ def sparse_top_k_categorical_accuracy(y_true, y_pred, k=5): + return K.mean(K.in_top_k(y_pred, K.cast(K.flatten(y_true), 'int32'), k), + axis=-1) + ++class SensitivitySpecificityBase(Metric): ++ """Abstract base class for computing sensitivity and specificity. ++ ++ For additional information about specificity and sensitivity, see the ++ following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity ++ """ ++ ++ def __init__(self, value, num_thresholds=200, name=None, dtype=None): ++ super(SensitivitySpecificityBase, self).__init__(name=name, dtype=dtype) ++ if num_thresholds <= 0: ++ raise ValueError('`num_thresholds` must be > 0.') ++ self.value = value ++ self.true_positives = self.add_weight( ++ 'true_positives', ++ shape=(num_thresholds,), ++ initializer='zeros') ++ self.true_negatives = self.add_weight( ++ 'true_negatives', ++ shape=(num_thresholds,), ++ initializer='zeros') ++ self.false_positives = self.add_weight( ++ 'false_positives', ++ shape=(num_thresholds,), ++ initializer='zeros') ++ self.false_negatives = self.add_weight( ++ 'false_negatives', ++ shape=(num_thresholds,), ++ initializer='zeros') ++ ++ # Compute `num_thresholds` thresholds in [0, 1] ++ if num_thresholds == 1: ++ self.thresholds = [0.5] ++ else: ++ thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) ++ for i in range(num_thresholds - 2)] ++ self.thresholds = [0.0] + thresholds + [1.0] ++ ++ def update_state(self, y_true, y_pred, sample_weight=None): ++ return metrics_utils.update_confusion_matrix_variables( ++ { ++ metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives, ++ metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives, ++ metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives, ++ metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives, ++ }, ++ y_true, ++ y_pred, ++ thresholds=self.thresholds, ++ sample_weight=sample_weight) ++ ++ def reset_states(self): ++ num_thresholds = len(self.thresholds) ++ K.batch_set_value( ++ [(v, np.zeros((num_thresholds,))) for v in self.variables]) ++ ++ ++class SensitivityAtSpecificity(SensitivitySpecificityBase): ++ """Computes the sensitivity at a given specificity. ++ ++ `Sensitivity` measures the proportion of actual positives that are correctly ++ identified as such (tp / (tp + fn)). ++ `Specificity` measures the proportion of actual negatives that are correctly ++ identified as such (tn / (tn + fp)). ++ ++ This metric creates four local variables, `true_positives`, `true_negatives`, ++ `false_positives` and `false_negatives` that are used to compute the ++ sensitivity at the given specificity. The threshold for the given specificity ++ value is computed and used to evaluate the corresponding sensitivity. ++ ++ If `sample_weight` is `None`, weights default to 1. ++ Use `sample_weight` of 0 to mask values. ++ ++ For additional information about specificity and sensitivity, see the ++ following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity ++ ++ Usage with the compile API: ++ ++ ```python ++ model = keras.Model(inputs, outputs) ++ model.compile( ++ 'sgd', ++ loss='mse', ++ metrics=[keras.metrics.SensitivityAtSpecificity()]) ++ ``` ++ ++ # Arguments ++ specificity: A scalar value in range `[0, 1]`. ++ num_thresholds: (Optional) Defaults to 200. The number of thresholds to ++ use for matching the given specificity. ++ name: (Optional) string name of the metric instance. ++ dtype: (Optional) data type of the metric result. ++ """ ++ ++ def __init__(self, specificity, num_thresholds=200, name=None, dtype=None): ++ if specificity < 0 or specificity > 1: ++ raise ValueError('`specificity` must be in the range [0, 1].') ++ self.specificity = specificity ++ self.num_thresholds = num_thresholds ++ super(SensitivityAtSpecificity, self).__init__( ++ specificity, num_thresholds=num_thresholds, name=name, dtype=dtype) ++ ++ def result(self): ++ # Calculate specificities at all the thresholds. ++ specificities = K.switch( ++ K.greater(self.true_negatives + self.false_positives, 0), ++ (self.true_negatives / (self.true_negatives + self.false_positives)), ++ K.zeros_like(self.thresholds)) ++ ++ # Find the index of the threshold where the specificity is closest to the ++ # given specificity. ++ min_index = K.argmin( ++ K.abs(specificities - self.value), axis=0) ++ min_index = K.cast(min_index, 'int32') ++ ++ # Compute sensitivity at that index. ++ return K.switch( ++ K.greater((self.true_positives[min_index] + ++ self.false_negatives[min_index]), 0), ++ (self.true_positives[min_index] / ++ (self.true_positives[min_index] + self.false_negatives[min_index])), ++ K.zeros_like(self.true_positives[min_index])) ++ ++ def get_config(self): ++ config = { ++ 'num_thresholds': self.num_thresholds, ++ 'specificity': self.specificity ++ } ++ base_config = super(SensitivityAtSpecificity, self).get_config() ++ return dict(list(base_config.items()) + list(config.items())) ++ ++ ++class AUC(Metric): ++ """Computes the approximate AUC (Area under the curve) via a Riemann sum. ++ ++ This metric creates four local variables, `true_positives`, `true_negatives`, ++ `false_positives` and `false_negatives` that are used to compute the AUC. ++ To discretize the AUC curve, a linearly spaced set of thresholds is used to ++ compute pairs of recall and precision values. The area under the ROC-curve is ++ therefore computed using the height of the recall values by the false positive ++ rate, while the area under the PR-curve is the computed using the height of ++ the precision values by the recall. ++ ++ This value is ultimately returned as `auc`, an idempotent operation that ++ computes the area under a discretized curve of precision versus recall values ++ (computed using the aforementioned variables). The `num_thresholds` variable ++ controls the degree of discretization with larger numbers of thresholds more ++ closely approximating the true AUC. The quality of the approximation may vary ++ dramatically depending on `num_thresholds`. The `thresholds` parameter can be ++ used to manually specify thresholds which split the predictions more evenly. ++ ++ For best results, `predictions` should be distributed approximately uniformly ++ in the range [0, 1] and not peaked around 0 or 1. The quality of the AUC ++ approximation may be poor if this is not the case. Setting `summation_method` ++ to 'minoring' or 'majoring' can help quantify the error in the approximation ++ by providing lower or upper bound estimate of the AUC. ++ ++ If `sample_weight` is `None`, weights default to 1. ++ Use `sample_weight` of 0 to mask values. ++ ++ Usage with the compile API: ++ ++ ```python ++ model = keras.Model(inputs, outputs) ++ model.compile('sgd', loss='mse', metrics=[keras.metrics.AUC()]) ++ ``` ++ ++ # Arguments ++ num_thresholds: (Optional) Defaults to 200. The number of thresholds to ++ use when discretizing the roc curve. Values must be > 1. ++ curve: (Optional) Specifies the name of the curve to be computed, 'ROC' ++ [default] or 'PR' for the Precision-Recall-curve. ++ summation_method: (Optional) Specifies the Riemann summation method used ++ (https://en.wikipedia.org/wiki/Riemann_sum): 'interpolation' [default], ++ applies mid-point summation scheme for `ROC`. For PR-AUC, interpolates ++ (true/false) positives but not the ratio that is precision (see Davis ++ & Goadrich 2006 for details); 'minoring' that applies left summation ++ for increasing intervals and right summation for decreasing intervals; ++ 'majoring' that does the opposite. ++ name: (Optional) string name of the metric instance. ++ dtype: (Optional) data type of the metric result. ++ thresholds: (Optional) A list of floating point values to use as the ++ thresholds for discretizing the curve. If set, the `num_thresholds` ++ parameter is ignored. Values should be in [0, 1]. Endpoint thresholds ++ equal to {-epsilon, 1+epsilon} for a small positive epsilon value will ++ be automatically included with these to correctly handle predictions ++ equal to exactly 0 or 1. ++ """ ++ ++ def __init__(self, ++ num_thresholds=200, ++ curve='ROC', ++ summation_method='interpolation', ++ name=None, ++ dtype=None, ++ thresholds=None): ++ # Validate configurations. ++ if (isinstance(curve, metrics_utils.AUCCurve) and ++ curve not in list(metrics_utils.AUCCurve)): ++ raise ValueError('Invalid curve: "{}". Valid options are: "{}"'.format( ++ curve, list(metrics_utils.AUCCurve))) ++ if isinstance( ++ summation_method, ++ metrics_utils.AUCSummationMethod) and summation_method not in list( ++ metrics_utils.AUCSummationMethod): ++ raise ValueError( ++ 'Invalid summation method: "{}". Valid options are: "{}"'.format( ++ summation_method, list(metrics_utils.AUCSummationMethod))) ++ ++ # Update properties. ++ if thresholds is not None: ++ # If specified, use the supplied thresholds. ++ self.num_thresholds = len(thresholds) + 2 ++ thresholds = sorted(thresholds) ++ else: ++ if num_thresholds <= 1: ++ raise ValueError('`num_thresholds` must be > 1.') ++ ++ # Otherwise, linearly interpolate (num_thresholds - 2) thresholds in ++ # (0, 1). ++ self.num_thresholds = num_thresholds ++ thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) ++ for i in range(num_thresholds - 2)] ++ ++ # Add an endpoint "threshold" below zero and above one for either ++ # threshold method to account for floating point imprecisions. ++ self.thresholds = [0.0 - K.epsilon()] + thresholds + [1.0 + K.epsilon()] ++ ++ if isinstance(curve, metrics_utils.AUCCurve): ++ self.curve = curve ++ else: ++ self.curve = metrics_utils.AUCCurve.from_str(curve) ++ if isinstance(summation_method, metrics_utils.AUCSummationMethod): ++ self.summation_method = summation_method ++ else: ++ self.summation_method = metrics_utils.AUCSummationMethod.from_str( ++ summation_method) ++ super(AUC, self).__init__(name=name, dtype=dtype) ++ ++ # Create metric variables ++ self.true_positives = self.add_weight( ++ 'true_positives', ++ shape=(self.num_thresholds,), ++ initializer='zeros') ++ self.true_negatives = self.add_weight( ++ 'true_negatives', ++ shape=(self.num_thresholds,), ++ initializer='zeros') ++ self.false_positives = self.add_weight( ++ 'false_positives', ++ shape=(self.num_thresholds,), ++ initializer='zeros') ++ self.false_negatives = self.add_weight( ++ 'false_negatives', ++ shape=(self.num_thresholds,), ++ initializer='zeros') ++ ++ def update_state(self, y_true, y_pred, sample_weight=None): ++ return metrics_utils.update_confusion_matrix_variables({ ++ metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives, ++ metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives, ++ metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives, ++ metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives, ++ }, y_true, y_pred, self.thresholds, sample_weight=sample_weight) ++ ++ def interpolate_pr_auc(self): ++ """Interpolation formula inspired by section 4 of Davis & Goadrich 2006. ++ ++ https://www.biostat.wisc.edu/~page/rocpr.pdf ++ ++ Note here we derive & use a closed formula not present in the paper ++ as follows: ++ ++ Precision = TP / (TP + FP) = TP / P ++ ++ Modeling all of TP (true positive), FP (false positive) and their sum ++ P = TP + FP (predicted positive) as varying linearly within each interval ++ [A, B] between successive thresholds, we get ++ ++ Precision slope = dTP / dP ++ = (TP_B - TP_A) / (P_B - P_A) ++ = (TP - TP_A) / (P - P_A) ++ Precision = (TP_A + slope * (P - P_A)) / P ++ ++ The area within the interval is (slope / total_pos_weight) times ++ ++ int_A^B{Precision.dP} = int_A^B{(TP_A + slope * (P - P_A)) * dP / P} ++ int_A^B{Precision.dP} = int_A^B{slope * dP + intercept * dP / P} ++ ++ where intercept = TP_A - slope * P_A = TP_B - slope * P_B, resulting in ++ ++ int_A^B{Precision.dP} = TP_B - TP_A + intercept * log(P_B / P_A) ++ ++ Bringing back the factor (slope / total_pos_weight) we'd put aside, we get ++ ++ slope * [dTP + intercept * log(P_B / P_A)] / total_pos_weight ++ ++ where dTP == TP_B - TP_A. ++ ++ Note that when P_A == 0 the above calculation simplifies into ++ ++ int_A^B{Precision.dTP} = int_A^B{slope * dTP} = slope * (TP_B - TP_A) ++ ++ which is really equivalent to imputing constant precision throughout the ++ first bucket having >0 true positives. ++ ++ # Returns ++ pr_auc: an approximation of the area under the P-R curve. ++ """ ++ dtp = self.true_positives[:self.num_thresholds - ++ 1] - self.true_positives[1:] ++ p = self.true_positives + self.false_positives ++ dp = p[:self.num_thresholds - 1] - p[1:] ++ ++ prec_slope = dtp / K.maximum(dp, 0) ++ intercept = self.true_positives[1:] - (prec_slope * p[1:]) ++ ++ # Logical and ++ pMin = K.expand_dims(p[:self.num_thresholds - 1] > 0, 0) ++ pMax = K.expand_dims(p[1:] > 0, 0) ++ are_different = K.concatenate([pMin, pMax], axis=0) ++ switch_condition = K.all(are_different, axis=0) ++ ++ safe_p_ratio = K.switch( ++ switch_condition, ++ p[:self.num_thresholds - 1] / K.maximum(p[1:], 0), ++ K.ones_like(p[1:])) ++ ++ numer = prec_slope * (dtp + intercept * K.log(safe_p_ratio)) ++ denom = K.maximum(self.true_positives[1:] + self.false_negatives[1:], 0) ++ return K.sum((numer / denom)) ++ ++ def result(self): ++ if (self.curve == metrics_utils.AUCCurve.PR and ++ (self.summation_method == ++ metrics_utils.AUCSummationMethod.INTERPOLATION)): ++ # This use case is different and is handled separately. ++ return self.interpolate_pr_auc() ++ ++ # Set `x` and `y` values for the curves based on `curve` config. ++ recall = K.switch( ++ K.greater((self.true_positives), 0), ++ (self.true_positives / ++ (self.true_positives + self.false_negatives)), ++ K.zeros_like(self.true_positives)) ++ if self.curve == metrics_utils.AUCCurve.ROC: ++ fp_rate = K.switch( ++ K.greater((self.false_positives), 0), ++ (self.false_positives / ++ (self.false_positives + self.true_negatives)), ++ K.zeros_like(self.false_positives)) ++ x = fp_rate ++ y = recall ++ else: # curve == 'PR'. ++ precision = K.switch( ++ K.greater((self.true_positives), 0), ++ (self.true_positives / (self.true_positives + self.false_positives)), ++ K.zeros_like(self.true_positives)) ++ x = recall ++ y = precision ++ ++ # Find the rectangle heights based on `summation_method`. ++ if self.summation_method == metrics_utils.AUCSummationMethod.INTERPOLATION: ++ # Note: the case ('PR', 'interpolation') has been handled above. ++ heights = (y[:self.num_thresholds - 1] + y[1:]) / 2. ++ elif self.summation_method == metrics_utils.AUCSummationMethod.MINORING: ++ heights = K.minimum(y[:self.num_thresholds - 1], y[1:]) ++ else: # self.summation_method = metrics_utils.AUCSummationMethod.MAJORING: ++ heights = K.maximum(y[:self.num_thresholds - 1], y[1:]) ++ ++ # Sum up the areas of all the rectangles. ++ return K.sum((x[:self.num_thresholds - 1] - x[1:]) * heights) ++ ++ def reset_states(self): ++ K.batch_set_value( ++ [(v, np.zeros((self.num_thresholds,))) for v in self.variables]) ++ ++ def get_config(self): ++ config = { ++ 'num_thresholds': self.num_thresholds, ++ 'curve': self.curve.value, ++ 'summation_method': self.summation_method.value, ++ # We remove the endpoint thresholds as an inverse of how the thresholds ++ # were initialized. This ensures that a metric initialized from this ++ # config has the same thresholds. ++ 'thresholds': self.thresholds[1:-1], ++ } ++ base_config = super(AUC, self).get_config() ++ return dict(list(base_config.items()) + list(config.items())) ++ + + # Aliases + +diff --git a/keras/utils/__init__.py b/keras/utils/__init__.py +index 8cc39d5..65af329 100644 +--- a/keras/utils/__init__.py ++++ b/keras/utils/__init__.py +@@ -4,6 +4,8 @@ from . import generic_utils + from . import data_utils + from . import io_utils + from . import conv_utils ++from . import losses_utils ++from . import metrics_utils + + # Globally-importable utils. + from .io_utils import HDF5Matrix +diff --git a/keras/utils/losses_utils.py b/keras/utils/losses_utils.py +new file mode 100644 +index 0000000..617ebb7 +--- /dev/null ++++ b/keras/utils/losses_utils.py +@@ -0,0 +1,177 @@ ++"""Utilities related to losses.""" ++from __future__ import absolute_import ++from __future__ import division ++from __future__ import print_function ++ ++import numpy as np ++ ++from .. import backend as K ++ ++ ++class Reduction(object): ++ """Types of loss reduction. ++ ++ Contains the following values: ++ ++ * `NONE`: Un-reduced weighted losses with the same shape as input. When this ++ reduction type used with built-in Keras training loops like ++ `fit`/`evaluate`, the unreduced vector loss is passed to the optimizer but ++ the reported loss will be a scalar value. ++ * `SUM`: Scalar sum of weighted losses. ++ * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses. ++ """ ++ ++ NONE = 'none' ++ SUM = 'sum' ++ SUM_OVER_BATCH_SIZE = 'sum_over_batch_size' ++ ++ @classmethod ++ def all(cls): ++ return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE) ++ ++ @classmethod ++ def validate(cls, key): ++ if key not in cls.all(): ++ raise ValueError('Invalid Reduction Key %s.' % key) ++ ++ ++def squeeze_or_expand_dimensions(y_pred, y_true=None, sample_weight=None): ++ """Squeeze or expand last dimension if needed. ++ ++ 1. Squeezes last dim of `y_pred` or `y_true` if their rank differs by 1. ++ 2. Squeezes or expands last dim of `sample_weight` if its rank differs by 1 ++ from the new rank of `y_pred`. ++ If `sample_weight` is scalar, it is kept scalar. ++ ++ # Arguments ++ y_pred: Predicted values, a `Tensor` of arbitrary dimensions. ++ y_true: Optional label `Tensor` whose dimensions match `y_pred`. ++ sample_weight: Optional weight scalar or `Tensor` whose dimensions match ++ `y_pred`. ++ ++ # Returns ++ Tuple of `y_pred`, `y_true` and `sample_weight`. Each of them possibly has ++ the last dimension squeezed, `sample_weight` could be extended by one ++ dimension. ++ """ ++ if y_true is not None: ++ y_pred_rank = K.ndim(y_pred) ++ y_pred_shape = K.int_shape(y_pred) ++ y_true_rank = K.ndim(y_true) ++ y_true_shape = K.int_shape(y_true) ++ ++ if (y_pred_rank - y_true_rank == 1) and (y_pred_shape[-1] == 1): ++ y_pred = K.squeeze(y_pred, -1) ++ elif (y_true_rank - y_pred_rank == 1) and (y_true_shape[-1] == 1): ++ y_true = K.squeeze(y_true, -1) ++ ++ if sample_weight is None: ++ return y_pred, y_true ++ ++ y_pred_rank = K.ndim(y_pred) ++ weights_rank = K.ndim(sample_weight) ++ if weights_rank != 0: ++ if weights_rank - y_pred_rank == 1: ++ sample_weight = K.squeeze(sample_weight, -1) ++ elif y_pred_rank - weights_rank == 1: ++ sample_weight = K.expand_dims(sample_weight, -1) ++ return y_pred, y_true, sample_weight ++ ++ ++def _num_elements(losses): ++ """Computes the number of elements in `losses` tensor.""" ++ with K.name_scope('num_elements') as scope: ++ return K.cast(K.size(losses, name=scope), losses.dtype) ++ ++ ++def reduce_weighted_loss(weighted_losses, reduction=Reduction.SUM_OVER_BATCH_SIZE): ++ """Reduces the individual weighted loss measurements.""" ++ if reduction == Reduction.NONE: ++ loss = weighted_losses ++ else: ++ loss = K.sum(weighted_losses) ++ if reduction == Reduction.SUM_OVER_BATCH_SIZE: ++ loss = loss / _num_elements(weighted_losses) ++ return loss ++ ++ ++def broadcast_weights(values, sample_weight): ++ # Broadcast weights if possible. ++ weights_shape = K.int_shape(sample_weight) ++ values_shape = K.int_shape(values) ++ ++ if values_shape != weights_shape: ++ weights_rank = K.ndim(sample_weight) ++ values_rank = K.ndim(values) ++ ++ # Raise error if ndim of weights is > values. ++ if weights_rank > values_rank: ++ raise ValueError( ++ 'Incompatible shapes: `values` {} vs `sample_weight` {}'.format( ++ values_shape, weights_shape)) ++ ++ # Expand dim of weights to match ndim of values, if required. ++ for i in range(weights_rank, values_rank): ++ sample_weight = K.expand_dims(sample_weight, axis=i) ++ ++ if weights_shape is not None and values_shape is not None: ++ for i in range(weights_rank): ++ if (weights_shape[i] is not None and ++ values_shape[i] is not None and ++ weights_shape[i] != values_shape[i]): ++ # Cannot be broadcasted. ++ if weights_shape[i] != 1: ++ raise ValueError( ++ 'Incompatible shapes: `values` {} vs ' ++ '`sample_weight` {}'.format( ++ values_shape, weights_shape)) ++ sample_weight = K.repeat_elements( ++ sample_weight, values_shape[i], axis=i) ++ return sample_weight ++ ++ ++def compute_weighted_loss(losses, ++ sample_weight=None, ++ reduction=Reduction.SUM_OVER_BATCH_SIZE, ++ name=None): ++ """Computes the weighted loss. ++ ++ # Arguments ++ losses: `Tensor` of shape `[batch_size, d1, ... dN]`. ++ sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as ++ ` losses`, or be broadcastable to `losses`. ++ reduction: (Optional) Type of Reduction to apply to loss. ++ Default value is `SUM_OVER_BATCH_SIZE`. ++ name: Optional name for the op. ++ ++ # Raises ++ ValueError: If the shape of `sample_weight` is not compatible with `losses`. ++ ++ # Returns ++ Weighted loss `Tensor` of the same type as `losses`. If `reduction` is ++ `NONE`, this has the same shape as `losses`; otherwise, it is scalar. ++ """ ++ Reduction.validate(reduction) ++ if sample_weight is None: ++ sample_weight = 1.0 ++ with K.name_scope(name or 'weighted_loss'): ++ input_dtype = K.dtype(losses) ++ losses = K.cast(losses, K.floatx()) ++ sample_weight = K.cast(sample_weight, K.floatx()) ++ ++ # Update dimensions of `sample_weight` to match with `losses` if possible. ++ losses, _, sample_weight = squeeze_or_expand_dimensions( ++ losses, None, sample_weight) ++ ++ # Broadcast weights if possible. ++ sample_weight = broadcast_weights(losses, sample_weight) ++ ++ # Apply weights to losses. ++ weighted_losses = sample_weight * losses ++ ++ # Apply reduction function to the individual weighted losses. ++ loss = reduce_weighted_loss(weighted_losses, reduction) ++ # Convert the result back to the input type. ++ loss = K.cast(loss, input_dtype) ++ return loss ++ +diff --git a/keras/utils/metrics_utils.py b/keras/utils/metrics_utils.py +new file mode 100644 +index 0000000..e6a5bb0 +--- /dev/null ++++ b/keras/utils/metrics_utils.py +@@ -0,0 +1,278 @@ ++"""Utilities related to metrics.""" ++from __future__ import absolute_import ++from __future__ import division ++from __future__ import print_function ++ ++from enum import Enum ++ ++from .. import backend as K ++from . import losses_utils ++ ++NEG_INF = -1e10 ++ ++class Reduction(object): ++ """Types of metrics reduction. ++ Contains the following values: ++ * `SUM`: Scalar sum of weighted values. ++ * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` of weighted values divided by ++ number of elements in values. ++ * `WEIGHTED_MEAN`: Scalar sum of weighted values divided by sum of weights. ++ """ ++ ++ SUM = 'sum' ++ SUM_OVER_BATCH_SIZE = 'sum_over_batch_size' ++ WEIGHTED_MEAN = 'weighted_mean' ++ ++ ++def update_state_wrapper(update_state_fn): ++ """Decorator to wrap metric `update_state()` with `add_update()`. ++ # Arguments ++ update_state_fn: function that accumulates metric statistics. ++ # Returns ++ Decorated function that wraps `update_state_fn()` with `add_update()`. ++ """ ++ def decorated(metric_obj, *args, **kwargs): ++ """Decorated function with `add_update()`.""" ++ ++ update_op = update_state_fn(*args, **kwargs) ++ metric_obj.add_update(update_op) ++ return update_op ++ ++ return decorated ++ ++def result_wrapper(result_fn): ++ """Decorator to wrap metric `result()` with identity op. ++ Wrapping result in identity so that control dependency between ++ update_op from `update_state` and result works in case result returns ++ a tensor. ++ # Arguments ++ result_fn: function that computes the metric result. ++ # Returns ++ Decorated function that wraps `result()` with identity op. ++ """ ++ def decorated(metric_obj, *args, **kwargs): ++ result_t = K.identity(result_fn(*args, **kwargs)) ++ metric_obj._call_result = result_t ++ result_t._is_metric = True ++ return result_t ++ return decorated ++ ++ ++def to_list(x): ++ if isinstance(x, list): ++ return x ++ return [x] ++ ++ ++def assert_thresholds_range(thresholds): ++ if thresholds is not None: ++ invalid_thresholds = [t for t in thresholds if t is None or t < 0 or t > 1] ++ if invalid_thresholds: ++ raise ValueError( ++ 'Threshold values must be in [0, 1]. Invalid values: {}'.format( ++ invalid_thresholds)) ++ ++ ++def parse_init_thresholds(thresholds, default_threshold=0.5): ++ if thresholds is not None: ++ assert_thresholds_range(to_list(thresholds)) ++ thresholds = to_list(default_threshold if thresholds is None else thresholds) ++ return thresholds ++ ++class ConfusionMatrix(Enum): ++ TRUE_POSITIVES = 'tp' ++ FALSE_POSITIVES = 'fp' ++ TRUE_NEGATIVES = 'tn' ++ FALSE_NEGATIVES = 'fn' ++ ++class AUCCurve(Enum): ++ """Type of AUC Curve (ROC or PR).""" ++ ROC = 'ROC' ++ PR = 'PR' ++ ++ @staticmethod ++ def from_str(key): ++ if key in ('pr', 'PR'): ++ return AUCCurve.PR ++ elif key in ('roc', 'ROC'): ++ return AUCCurve.ROC ++ else: ++ raise ValueError('Invalid AUC curve value "%s".' % key) ++ ++ ++class AUCSummationMethod(Enum): ++ """Type of AUC summation method. ++ ++ https://en.wikipedia.org/wiki/Riemann_sum) ++ ++ Contains the following values: ++ * 'interpolation': Applies mid-point summation scheme for `ROC` curve. For ++ `PR` curve, interpolates (true/false) positives but not the ratio that is ++ precision (see Davis & Goadrich 2006 for details). ++ * 'minoring': Applies left summation for increasing intervals and right ++ summation for decreasing intervals. ++ * 'majoring': Applies right summation for increasing intervals and left ++ summation for decreasing intervals. ++ """ ++ INTERPOLATION = 'interpolation' ++ MAJORING = 'majoring' ++ MINORING = 'minoring' ++ ++ @staticmethod ++ def from_str(key): ++ if key in ('interpolation', 'Interpolation'): ++ return AUCSummationMethod.INTERPOLATION ++ elif key in ('majoring', 'Majoring'): ++ return AUCSummationMethod.MAJORING ++ elif key in ('minoring', 'Minoring'): ++ return AUCSummationMethod.MINORING ++ else: ++ raise ValueError('Invalid AUC summation method value "%s".' % key) ++ ++def weighted_assign_add(label, pred, weights, var): ++ # Logical and ++ label = K.expand_dims(label, 0) ++ pred = K.expand_dims(pred, 0) ++ are_different = K.concatenate([label, pred], axis=0) ++ label_and_pred = K.all(are_different, axis=0) ++ label_and_pred = K.cast(label_and_pred, dtype=K.floatx()) ++ if weights is not None: ++ label_and_pred *= weights ++ return var.assign_add(K.sum(label_and_pred, 1)) ++ ++def update_confusion_matrix_variables(variables_to_update, ++ y_true, ++ y_pred, ++ thresholds, ++ top_k=None, ++ class_id=None, ++ sample_weight=None): ++ """Returns op to update the given confusion matrix variables. ++ For every pair of values in y_true and y_pred: ++ true_positive: y_true == True and y_pred > thresholds ++ false_negatives: y_true == True and y_pred <= thresholds ++ true_negatives: y_true == False and y_pred <= thresholds ++ false_positive: y_true == False and y_pred > thresholds ++ The results will be weighted and added together. When multiple thresholds are ++ provided, we will repeat the same for every threshold. ++ For estimation of these metrics over a stream of data, the function creates an ++ `update_op` operation that updates the given variables. ++ If `sample_weight` is `None`, weights default to 1. ++ Use weights of 0 to mask values. ++ # Arguments ++ variables_to_update: Dictionary with 'tp', 'fn', 'tn', 'fp' as valid keys ++ and corresponding variables to update as values. ++ y_true: A `Tensor` whose shape matches `y_pred`. Will be cast to `bool`. ++ y_pred: A floating point `Tensor` of arbitrary shape and whose values are in ++ the range `[0, 1]`. ++ thresholds: A float value or a python list or tuple of float thresholds in ++ `[0, 1]`, or NEG_INF (used when top_k is set). ++ top_k: Optional int, indicates that the positive labels should be limited to ++ the top k predictions. ++ class_id: Optional int, limits the prediction and labels to the class ++ specified by this argument. ++ sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as ++ `y_true`, and must be broadcastable to `y_true` (i.e., all dimensions must ++ be either `1`, or the same as the corresponding `y_true` dimension). ++ # Returns ++ Update ops. ++ # Raises ++ ValueError: If `y_pred` and `y_true` have mismatched shapes, or if ++ `sample_weight` is not `None` and its shape doesn't match `y_pred`, or if ++ `variables_to_update` contains invalid keys. ++ """ ++ if variables_to_update is None: ++ return ++ y_true = K.cast(y_true, dtype=K.floatx()) ++ y_pred = K.cast(y_pred, dtype=K.floatx()) ++ if sample_weight is not None: ++ sample_weight = K.cast(sample_weight, dtype=K.floatx()) ++ ++ if not any(key ++ for key in variables_to_update ++ if key in list(ConfusionMatrix)): ++ raise ValueError( ++ 'Please provide at least one valid confusion matrix ' ++ 'variable to update. Valid variable key options are: "{}". ' ++ 'Received: "{}"'.format( ++ list(ConfusionMatrix), variables_to_update.keys())) ++ ++ invalid_keys = [ ++ key for key in variables_to_update if key not in list(ConfusionMatrix) ++ ] ++ if invalid_keys: ++ raise ValueError( ++ 'Invalid keys: {}. Valid variable key options are: "{}"'.format( ++ invalid_keys, list(ConfusionMatrix))) ++ ++ if sample_weight is None: ++ y_pred, y_true = losses_utils.squeeze_or_expand_dimensions( ++ y_pred, y_true=y_true) ++ else: ++ y_pred, y_true, sample_weight = ( ++ losses_utils.squeeze_or_expand_dimensions( ++ y_pred, y_true=y_true, sample_weight=sample_weight)) ++ ++ if top_k is not None: ++ y_pred = _filter_top_k(y_pred, top_k) ++ if class_id is not None: ++ y_true = y_true[..., class_id] ++ y_pred = y_pred[..., class_id] ++ ++ thresholds = to_list(thresholds) ++ num_thresholds = len(thresholds) ++ num_predictions = K.size(y_pred) ++ ++ # Reshape predictions and labels. ++ predictions_2d = K.reshape(y_pred, [1, -1]) ++ labels_2d = K.reshape( ++ K.cast(y_true, dtype='bool'), [1, -1]) ++ ++ # Tile the thresholds for every prediction. ++ thresh_tiled = K.tile( ++ K.expand_dims(K.constant(thresholds), 1), ++ K.stack([1, num_predictions])) ++ ++ # Tile the predictions for every threshold. ++ preds_tiled = K.tile(predictions_2d, [num_thresholds, 1]) ++ ++ # Compare predictions and threshold. ++ pred_is_pos = K.greater(preds_tiled, thresh_tiled) ++ pred_is_neg = K.greater(thresh_tiled, preds_tiled) ++ ++ # Tile labels by number of thresholds ++ label_is_pos = K.tile(labels_2d, [num_thresholds, 1]) ++ ++ if sample_weight is not None: ++ weights = losses_utils.broadcast_weights( ++ y_pred, K.cast(sample_weight, dtype=K.floatx())) ++ weights_tiled = K.tile( ++ K.reshape(weights, [1, -1]), [num_thresholds, 1]) ++ else: ++ weights_tiled = None ++ ++ update_ops = [] ++ loop_vars = { ++ ConfusionMatrix.TRUE_POSITIVES: (label_is_pos, pred_is_pos), ++ } ++ update_tn = ConfusionMatrix.TRUE_NEGATIVES in variables_to_update ++ update_fp = ConfusionMatrix.FALSE_POSITIVES in variables_to_update ++ update_fn = ConfusionMatrix.FALSE_NEGATIVES in variables_to_update ++ ++ if update_fn or update_tn: ++ loop_vars[ConfusionMatrix.FALSE_NEGATIVES] = (label_is_pos, pred_is_neg) ++ ++ if update_fp or update_tn: ++ label_is_neg = K.equal( ++ label_is_pos, K.zeros_like(label_is_pos, dtype=label_is_pos.dtype)) ++ loop_vars[ConfusionMatrix.FALSE_POSITIVES] = (label_is_neg, pred_is_pos) ++ if update_tn: ++ loop_vars[ConfusionMatrix.TRUE_NEGATIVES] = (label_is_neg, pred_is_neg) ++ ++ for matrix_cond, (label, pred) in loop_vars.items(): ++ if matrix_cond in variables_to_update: ++ update_ops.append( ++ weighted_assign_add(label, pred, weights_tiled, ++ variables_to_update[matrix_cond])) ++ return update_ops ++ +-- +2.26.2 + |