diff options
| -rw-r--r-- | gn/packages/ratspub.scm | 152 | ||||
| -rw-r--r-- | gn/services/ratspub-container.scm | 129 | ||||
| -rw-r--r-- | keras-auc-optimizer.patch | 1133 |
3 files changed, 0 insertions, 1414 deletions
diff --git a/gn/packages/ratspub.scm b/gn/packages/ratspub.scm index d797e8a..c530979 100644 --- a/gn/packages/ratspub.scm +++ b/gn/packages/ratspub.scm @@ -20,135 +20,6 @@ #:use-module (gn packages python) #:use-module (gn packages web)) -(define-public ratspub - (package - (name "ratspub") - (version "0.4.5") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/chen42/ratspub") - (commit (string-append "v" version)))) - (file-name (git-file-name name version)) - ;; Keep the service running on port 4200. - (modules '((guix build utils))) - (snippet - '(begin (substitute* "server.py" - (("4201") "4200")))) - (sha256 - (base32 - "1y89rkqdxcnl2jjsy1wfp9p8qkgh6nzqs1r37wyhc8y7r3dva7kf")))) - (build-system python-build-system) - (arguments - `(#:tests? #f ; no test suite - #:phases - (modify-phases %standard-phases - (delete 'configure) - (delete 'build) - (add-after 'unpack 'make-files-writable - (lambda _ - (for-each make-file-writable (find-files ".")))) - (add-after 'unpack 'patch-datadir - (lambda _ - (substitute* "server.py" - (("^datadir.*") "datadir = \"/export/ratspub/\"\n")))) - (add-after 'unpack 'patch-sources - (lambda* (#:key inputs outputs #:allow-other-keys) - (let ((out (assoc-ref outputs "out")) - (inetutils (assoc-ref inputs "inetutils"))) - (substitute* '("templates/cytoscape.html" - "templates/tableview.html" - "templates/tableview0.html" - "templates/userarchive.html") - (("https.*FileSaver.js.*\\\">") "/static/FileSaver.js\">") - (("https.*cytoscape-svg.js.*\\\">") "/static/cytoscape-svg.js\">") - (("https.*cytoscape.min.js.*\\\">") "/static/cytoscape.min.js\">")) - (substitute* "templates/layout.html" - (("https.*bootstrap.min.css.*\\\">") "/static/bootstrap.min.css\">") - (("https.*4.*bootstrap.min.js.*\\\">") "/static/bootstrap.min.js\">") - (("https.*4.7.0/css/font-awesome.min.css") "/static/font-awesome.min.css") - (("https.*jquery-3.2.1.slim.min.js.*\\\">") "/static/jquery.slim.min.js\">") - (("https.*1.12.9/umd/popper.min.js.*\\\">") "/static/popper.min.js\">")) - (substitute* "ratspub.py" - (("hostname") (string-append inetutils "/bin/hostname")))))) - (replace 'install - (lambda* (#:key outputs #:allow-other-keys) - (let ((out (assoc-ref outputs "out"))) - (copy-recursively "." out)))) - (add-after 'install 'install-javascript - (lambda* (#:key inputs outputs #:allow-other-keys) - (let ((out (assoc-ref outputs "out")) - (awesome (assoc-ref inputs "font-awesome")) - (bootstrap (assoc-ref inputs "bootstrap")) - (cytoscape (assoc-ref inputs "cytoscape")) - (cytoscape-svg (assoc-ref inputs "cytoscape-svg")) - (jquery (assoc-ref inputs "jquery")) - (js-filesaver (assoc-ref inputs "js-filesaver")) - (js-popper (assoc-ref inputs "js-popper"))) - (symlink (string-append awesome - "/share/web/font-awesomecss/font-awesome.min.css") - (string-append out "/static/font-awesome.min.css")) - (symlink (string-append bootstrap - "/share/web/bootstrap/css/bootstrap.min.css") - (string-append out "/static/bootstrap.min.css")) - (symlink (string-append bootstrap - "/share/web/bootstrap/js/bootstrap.min.js") - (string-append out "/static/bootstrap.min.js")) - (symlink (string-append cytoscape - "/share/genenetwork2/javascript/cytoscape/cytoscape.min.js") - (string-append out "/static/cytoscape.min.js")) - (symlink (string-append cytoscape-svg - "/share/javascript/cytoscape-svg.js") - (string-append out "/static/cytoscape-svg.js")) - (symlink (string-append jquery - "/share/web/jquery/jquery.slim.min.js") - (string-append out "/static/jquery.slim.min.js")) - (symlink (string-append js-filesaver - "/share/javascript/FileSaver.js") - (string-append out "/static/FileSaver.js")) - (symlink (string-append js-popper - "/share/javascript/popper.min.js") - (string-append out "/static/popper.min.js"))))) - (add-after 'install 'wrap-executable - (lambda* (#:key inputs outputs #:allow-other-keys) - (let ((out (assoc-ref outputs "out")) - (path (getenv "GUIX_PYTHONPATH"))) - (wrap-program (string-append out "/server.py") - `("PATH" ":" prefix (,(dirname (which "edirect.pl")) - ,(dirname (which "dirname")) - ,(dirname (which "grep")) - ,(dirname (which "sed")))) - `("GUIX_PYTHONPATH" ":" prefix (,path))))))))) - (inputs - `(("edirect" ,edirect) - ("inetutils" ,inetutils) - ("python-bcrypt" ,python-bcrypt) - ("python-flask-sqlalchemy" ,python-flask-sqlalchemy) - ("python-keras" ,python-keras-for-ratspub) - ("python-nltk" ,python-nltk) - ("tensorflow" ,tensorflow))) - (native-inputs - `(("bootstrap" ,web-bootstrap) - ("cytoscape" ,javascript-cytoscape-3.17) - ;("cytoscape-svg" ,js-cytoscape-svg-0.3.1) ; TODO - ("cytoscape-svg" ,js-cytoscape-svg-vendor-0.3.1) - ("font-awesome" ,web-font-awesome) - ("jquery" ,web-jquery) - ("js-filesaver" ,js-filesaver-1.3.2) - ("js-popper" ,js-popper-1.12.9))) - (home-page "https://rats.pub/") - (synopsis "Relationship with Addiction Through Searches of PubMed") - (description - "RatsPub is a tool to efficiently and comprehensively answer the question -\"What do we know about these genes and addiction?\". RatsPub answers this -question by searching PubMed to find sentences containing the query terms (i.e., -gene symbols) and over 300 drug addiction-related keywords that are organized -into six categories. Data from @url{https://www.ebi.ac.uk/gwas/, -@acronym{NHGRI-EBI GWAS, European Bioinformatics Institute Genome-Wide -Association Studies}} catalog are also included in the search. These -gene-keyword relationships are presented as an interactive graph and a table.") - (license license:expat))) - (define use-corrected-inputs (package-input-rewriting/spec ;; Tensorflow-native provides much improved speeds. python-h5py@2 provides @@ -156,29 +27,6 @@ gene-keyword relationships are presented as an interactive graph and a table.") `(("tensorflow" . ,(const tensorflow-native)) ("python-h5py" . ,(const python-h5py-2))))) -(define-public ratspub-with-tensorflow-native - (package - (inherit - (tensowflow-native-instead-of-tensorflow ratspub)) - (name "ratspub-with-tensorflow-native"))) - -;; We want a copy of python-keras with the AUC optimizer backported. -;; We skip the tests because we "test in production". -;; That's a lie. The test suite just takes a long time to run. -(define-public python-keras-for-ratspub - (hidden-package - (package - (inherit python-keras) - (source - (origin - (inherit (package-source python-keras)) - (patches (search-patches "keras-auc-optimizer.patch")))) - (arguments - (substitute-keyword-arguments (package-arguments python-keras) - ((#:phases phases) - `(modify-phases ,phases - (delete 'check)))))))) - (define-public python-keras-no-tests (hidden-package (package diff --git a/gn/services/ratspub-container.scm b/gn/services/ratspub-container.scm deleted file mode 100644 index 1bf9fc6..0000000 --- a/gn/services/ratspub-container.scm +++ /dev/null @@ -1,129 +0,0 @@ -(define-module (gn services ratspub-container)) - -(use-modules (gnu) - (gn packages ratspub) - (guix download) - (guix modules) - (guix packages) - (guix records) - (ice-9 match)) -(use-service-modules shepherd) -(use-package-modules certs compression) - -(define-record-type* <ratspub-configuration> - ratspub-configuration - make-ratspub-configuration - ratspub-configuration? - (package ratspub-configuration-package ; package - (default ratspub))) - -(define %punkt.zip - (origin - (method url-fetch) - (uri "https://github.com/nltk/nltk_data/raw/b63a469d2f83a3cc9a2efcfe36915839d4e11d42/packages/tokenizers/punkt.zip") - (sha256 - (base32 "0i01c5qzn1p8dxyrpx4hry2n6x6b8rgcq1sck091n0jp036f6x4s")))) - -(define ratspub-activation - (match-lambda - (($ <ratspub-configuration> package) - #~(begin - (let ((nltk_data "/var/cache/nltk_data/tokenizers") - (data_dir "/export/ratspub")) - (unless (file-exists? "/export2/PubMed") - (mkdir-p "/export2/PubMed")) - (unless (file-exists? nltk_data) - (begin - ;; The correct way would be to use python-nltk to download the data - ;; python3 -m nltk.downloader -d /var/cache/nltk_data punkt - (mkdir-p nltk_data) - (chdir nltk_data) - (invoke #$(file-append unzip "/bin/unzip") "-q" #$%punkt.zip))) - (unless (file-exists? (string-append data_dir "/userspub.sqlite")) - (begin - (install-file #$(file-append package "/userspub.sqlite") data_dir) - (chmod (string-append data_dir "/userspub.sqlite") #o554)))))))) - -(define ratspub-shepherd-service - (match-lambda - (($ <ratspub-configuration> package) - (with-imported-modules (source-module-closure - '((gnu build shepherd) - (gnu system file-systems))) - (list (shepherd-service - (provision '(ratspub)) - (requirement '(networking)) - (modules '((gnu build shepherd) - (gnu system file-systems))) - (start #~(make-forkexec-constructor/container - (list #$(file-append package "/server.py")) - ;; Needs to run from the directory it is located in. - #:directory #$package - #:log-file "/var/log/ratspub.log" - ;; We don't need to set TMPDIR because we're inside a container. - #:environment-variables - '("EDIRECT_PUBMED_MASTER=/export2/PubMed" - "NLTK_DATA=/var/cache/nltk_data" - "PERL_LWP_SSL_CA_FILE=/etc/ssl/certs/ca-certificates.crt") - #:mappings (list (file-system-mapping - (source "/export2/PubMed") - (target source) - (writable? #t)) - (file-system-mapping - (source "/export/ratspub") - (target source) - (writable? #t)) - (file-system-mapping - (source "/var/cache/nltk_data") - (target source)) - (file-system-mapping - (source "/etc/ssl/certs") - (target source))))) - (stop #~(make-kill-destructor)))))))) - -(define ratspub-service-type - (service-type - (name 'ratspub) - (extensions - (list - (service-extension shepherd-root-service-type - ratspub-shepherd-service) - (service-extension activation-service-type - ratspub-activation) - ;; Make sure we get all the dependencies of RatsPub. - (service-extension profile-service-type - (compose list ratspub-configuration-package)))) - (default-value (ratspub-configuration)) - (description - "Run a RatsPub Webserver."))) - -(operating-system - (host-name "ratspub") - (timezone "Etc/UTC") - (locale "en_US.utf8") - - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets '("does-not-matter")))) - (file-systems (list (file-system - (device "does-not-matter") - (mount-point "/") - (type "does-not-matter")))) - ;; TODO: A more minimal kernel for use in a docker image - ;; (kernel linux-libre-vm) - ;; No firmware for VMs. - (firmware '()) - (packages (list nss-certs)) - - (services (list (service ratspub-service-type - (ratspub-configuration - ;; ratspub for docker, ratspub-with-tensorflow-native for architecture specific speed optimizations. - ;(package ratspub)))))) - (package ratspub-with-tensorflow-native)))))) - -;; guix system container -L /path/to/guix-bioinformatics/ -L /path/to/guix-past/modules/ /path/to/guix-bioinformatics/gn/services/ratspub-container.scm --network --share=/export2/PubMed=/export2/PubMed --share=/export/ratspub=/export/ratspub -;; For docker it isn't necessary to list the shared folders at build time. -;; guix system docker-image -L /path/to/guix-bioinformatics/ -L /path/to/guix-past/modules/ /path/to/guix-bioinformatics/gn/services/ratspub-container.scm --network -;; Docker instructions: -;; docker load --input ratspub-docker-image.tar.gz -;; docker run -d --privileged --net=host --name ratspub --volume /path/to/PubMed:/export2/PubMed guix diff --git a/keras-auc-optimizer.patch b/keras-auc-optimizer.patch deleted file mode 100644 index bbc6924..0000000 --- a/keras-auc-optimizer.patch +++ /dev/null @@ -1,1133 +0,0 @@ -From 901159da45695da24a5206125910f02fc50169ce Mon Sep 17 00:00:00 2001 -From: Efraim Flashner <efraim@flashner.co.il> -Date: Thu, 23 Apr 2020 15:50:37 +0300 -Subject: [PATCH] add keras metrics - ---- - keras/backend/tensorflow_backend.py | 12 + - keras/metrics.py | 584 ++++++++++++++++++++++++++++ - keras/utils/__init__.py | 2 + - keras/utils/losses_utils.py | 177 +++++++++ - keras/utils/metrics_utils.py | 278 +++++++++++++ - 5 files changed, 1053 insertions(+) - create mode 100644 keras/utils/losses_utils.py - create mode 100644 keras/utils/metrics_utils.py - -diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py -index bcb8be0..a2870f5 100644 ---- a/keras/backend/tensorflow_backend.py -+++ b/keras/backend/tensorflow_backend.py -@@ -4453,3 +4453,15 @@ def local_conv2d(inputs, kernel, kernel_size, strides, output_shape, data_format - else: - output = permute_dimensions(output, (2, 0, 1, 3)) - return output -+ -+#get_graph = tf_keras_backend.get_graph -+ -+#def is_symbolic(x): -+# return isinstance(x, tf.Tensor) and hasattr(x, 'op') -+ -+def size(x, name=None): -+# if is_symbolic(x): -+# with get_graph().as_default(): -+# return tf.size(x) -+ return tf.size(x, name=name) -+ -diff --git a/keras/metrics.py b/keras/metrics.py -index 8e3df1f..8f57910 100644 ---- a/keras/metrics.py -+++ b/keras/metrics.py -@@ -4,8 +4,12 @@ from __future__ import absolute_import - from __future__ import division - from __future__ import print_function - -+import abc - import six -+import types -+ - from . import backend as K -+from .engine.base_layer import Layer - from .losses import mean_squared_error - from .losses import mean_absolute_error - from .losses import mean_absolute_percentage_error -@@ -19,10 +23,201 @@ from .losses import binary_crossentropy - from .losses import kullback_leibler_divergence - from .losses import poisson - from .losses import cosine_proximity -+from .utils import losses_utils -+from .utils import metrics_utils - from .utils.generic_utils import deserialize_keras_object - from .utils.generic_utils import serialize_keras_object - - -+@six.add_metaclass(abc.ABCMeta) -+class Metric(Layer): -+ """Encapsulates metric logic and state. -+ -+ Standalone usage: -+ ```python -+ m = SomeMetric(...) -+ for input in ...: -+ m.update_state(input) -+ m.result() -+ ``` -+ -+ Usage with the `compile` API: -+ ```python -+ model.compile(optimizer='rmsprop', -+ loss=keras.losses.categorical_crossentropy, -+ metrics=[keras.metrics.CategoricalAccuracy()]) -+ ``` -+ -+ To be implemented by subclasses: -+ * `__init__()`: All state variables should be created in this method by -+ calling `self.add_weight()` like: `self.var = self.add_weight(...)` -+ * `update_state()`: Has all updates to the state variables like: -+ self.var.assign_add(...). -+ * `result()`: Computes and returns a value for the metric -+ from the state variables. -+ """ -+ -+ def __init__(self, name=None, dtype=None, **kwargs): -+ super(Metric, self).__init__(name=name, dtype=dtype, **kwargs) -+ self.stateful = True # All metric layers are stateful. -+ self.built = True -+ self.dtype = K.floatx() if dtype is None else dtype -+ -+ def __new__(cls, *args, **kwargs): -+ obj = super(Metric, cls).__new__(cls) -+ update_state_fn = obj.update_state -+ -+ obj.update_state = types.MethodType( -+ metrics_utils.update_state_wrapper(update_state_fn), obj) -+ return obj -+ -+ def __call__(self, *args, **kwargs): -+ """Accumulates statistics and then computes metric result value.""" -+ update_op = self.update_state(*args, **kwargs) -+ return self.result() -+ -+ def get_config(self): -+ """Returns the serializable config of the metric.""" -+ return {'name': self.name, 'dtype': self.dtype} -+ -+ def reset_states(self): -+ """Resets all of the metric state variables. -+ This function is called between epochs/steps, -+ when a metric is evaluated during training. -+ """ -+ K.batch_set_value([(v, 0) for v in self.weights]) -+ -+ @abc.abstractmethod -+ def update_state(self, *args, **kwargs): -+ """Accumulates statistics for the metric. """ -+ raise NotImplementedError('Must be implemented in subclasses.') -+ -+ @abc.abstractmethod -+ def result(self): -+ """Computes and returns the metric value tensor. -+ Result computation is an idempotent operation that simply calculates the -+ metric value using the state variables. -+ """ -+ raise NotImplementedError('Must be implemented in subclasses.') -+ -+ # For use by subclasses # -+ def add_weight(self, -+ name, -+ shape=(), -+ initializer=None, -+ dtype=None): -+ """Adds state variable. Only for use by subclasses.""" -+ return super(Metric, self).add_weight( -+ name=name, -+ shape=shape, -+ dtype=self.dtype if dtype is None else dtype, -+ trainable=False, -+ initializer=initializer) -+ -+ # End: For use by subclasses ### -+ -+ -+class Reduce(Metric): -+ """Encapsulates metrics that perform a reduce operation on the values.""" -+ -+ def __init__(self, reduction, name, dtype=None): -+ """Creates a `Reduce` instance. -+ # Arguments -+ reduction: a metrics `Reduction` enum value. -+ name: string name of the metric instance. -+ dtype: (Optional) data type of the metric result. -+ """ -+ super(Reduce, self).__init__(name=name, dtype=dtype) -+ self.reduction = reduction -+ self.total = self.add_weight('total', initializer='zeros') -+ if reduction in [metrics_utils.Reduction.SUM_OVER_BATCH_SIZE, -+ metrics_utils.Reduction.WEIGHTED_MEAN]: -+ self.count = self.add_weight('count', initializer='zeros') -+ -+ def update_state(self, values, sample_weight=None): -+ """Accumulates statistics for computing the reduction metric. -+ For example, if `values` is [1, 3, 5, 7] and reduction=SUM_OVER_BATCH_SIZE, -+ then the value of `result()` is 4. If the `sample_weight` is specified as -+ [1, 1, 0, 0] then value of `result()` would be 2. -+ # Arguments -+ values: Per-example value. -+ sample_weight: Optional weighting of each example. Defaults to 1. -+ """ -+ values = K.cast(values, self.dtype) -+ if sample_weight is not None: -+ sample_weight = K.cast(sample_weight, self.dtype) -+ # Update dimensions of weights to match with values if possible. -+ values, _, sample_weight = losses_utils.squeeze_or_expand_dimensions( -+ values, sample_weight=sample_weight) -+ -+ # Broadcast weights if possible. -+ sample_weight = losses_utils.broadcast_weights(sample_weight, values) -+ values = values * sample_weight -+ -+ value_sum = K.sum(values) -+ update_total_op = K.update_add(self.total, value_sum) -+ -+ # Exit early if the reduction doesn't have a denominator. -+ if self.reduction == metrics_utils.Reduction.SUM: -+ return update_total_op -+ -+ # Update `count` for reductions that require a denominator. -+ if self.reduction == metrics_utils.Reduction.SUM_OVER_BATCH_SIZE: -+ num_values = K.cast(K.size(values), self.dtype) -+ elif self.reduction == metrics_utils.Reduction.WEIGHTED_MEAN: -+ if sample_weight is None: -+ num_values = K.cast(K.size(values), self.dtype) -+ else: -+ num_values = K.sum(sample_weight) -+ else: -+ raise NotImplementedError( -+ 'reduction [%s] not implemented' % self.reduction) -+ -+ with K.control_dependencies([update_total_op]): -+ return K.update_add(self.count, num_values) -+ -+ def result(self): -+ if self.reduction == metrics_utils.Reduction.SUM: -+ return self.total -+ elif self.reduction in [ -+ metrics_utils.Reduction.WEIGHTED_MEAN, -+ metrics_utils.Reduction.SUM_OVER_BATCH_SIZE -+ ]: -+ return self.total / self.count -+ else: -+ raise NotImplementedError( -+ 'reduction [%s] not implemented' % self.reduction) -+ -+ -+class Sum(Reduce): -+ """Computes the (weighted) sum of the given values. -+ -+ For example, if values is [1, 3, 5, 7] then the sum is 16. -+ If the weights were specified as [1, 1, 0, 0] then the sum would be 4. -+ -+ This metric creates one variable, `total`, that is used to compute the sum of -+ `values`. This is ultimately returned as `sum`. -+ If `sample_weight` is `None`, weights default to 1. Use `sample_weight` of 0 -+ to mask values. -+ -+ Standalone usage: -+ ```python -+ m = keras.metrics.Sum() -+ m.update_state([1, 3, 5, 7]) -+ m.result() -+ ``` -+ """ -+ -+ def __init__(self, name='sum', dtype=None): -+ """Creates a `Sum` instance. -+ # Arguments -+ name: (Optional) string name of the metric instance. -+ dtype: (Optional) data type of the metric result. -+ """ -+ super(Sum, self).__init__(reduction=metrics_utils.Reduction.SUM, -+ name=name, dtype=dtype) -+ -+ - def binary_accuracy(y_true, y_pred): - return K.mean(K.equal(y_true, K.round(y_pred)), axis=-1) - -@@ -49,6 +244,395 @@ def sparse_top_k_categorical_accuracy(y_true, y_pred, k=5): - return K.mean(K.in_top_k(y_pred, K.cast(K.flatten(y_true), 'int32'), k), - axis=-1) - -+class SensitivitySpecificityBase(Metric): -+ """Abstract base class for computing sensitivity and specificity. -+ -+ For additional information about specificity and sensitivity, see the -+ following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity -+ """ -+ -+ def __init__(self, value, num_thresholds=200, name=None, dtype=None): -+ super(SensitivitySpecificityBase, self).__init__(name=name, dtype=dtype) -+ if num_thresholds <= 0: -+ raise ValueError('`num_thresholds` must be > 0.') -+ self.value = value -+ self.true_positives = self.add_weight( -+ 'true_positives', -+ shape=(num_thresholds,), -+ initializer='zeros') -+ self.true_negatives = self.add_weight( -+ 'true_negatives', -+ shape=(num_thresholds,), -+ initializer='zeros') -+ self.false_positives = self.add_weight( -+ 'false_positives', -+ shape=(num_thresholds,), -+ initializer='zeros') -+ self.false_negatives = self.add_weight( -+ 'false_negatives', -+ shape=(num_thresholds,), -+ initializer='zeros') -+ -+ # Compute `num_thresholds` thresholds in [0, 1] -+ if num_thresholds == 1: -+ self.thresholds = [0.5] -+ else: -+ thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) -+ for i in range(num_thresholds - 2)] -+ self.thresholds = [0.0] + thresholds + [1.0] -+ -+ def update_state(self, y_true, y_pred, sample_weight=None): -+ return metrics_utils.update_confusion_matrix_variables( -+ { -+ metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives, -+ metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives, -+ metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives, -+ metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives, -+ }, -+ y_true, -+ y_pred, -+ thresholds=self.thresholds, -+ sample_weight=sample_weight) -+ -+ def reset_states(self): -+ num_thresholds = len(self.thresholds) -+ K.batch_set_value( -+ [(v, np.zeros((num_thresholds,))) for v in self.variables]) -+ -+ -+class SensitivityAtSpecificity(SensitivitySpecificityBase): -+ """Computes the sensitivity at a given specificity. -+ -+ `Sensitivity` measures the proportion of actual positives that are correctly -+ identified as such (tp / (tp + fn)). -+ `Specificity` measures the proportion of actual negatives that are correctly -+ identified as such (tn / (tn + fp)). -+ -+ This metric creates four local variables, `true_positives`, `true_negatives`, -+ `false_positives` and `false_negatives` that are used to compute the -+ sensitivity at the given specificity. The threshold for the given specificity -+ value is computed and used to evaluate the corresponding sensitivity. -+ -+ If `sample_weight` is `None`, weights default to 1. -+ Use `sample_weight` of 0 to mask values. -+ -+ For additional information about specificity and sensitivity, see the -+ following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity -+ -+ Usage with the compile API: -+ -+ ```python -+ model = keras.Model(inputs, outputs) -+ model.compile( -+ 'sgd', -+ loss='mse', -+ metrics=[keras.metrics.SensitivityAtSpecificity()]) -+ ``` -+ -+ # Arguments -+ specificity: A scalar value in range `[0, 1]`. -+ num_thresholds: (Optional) Defaults to 200. The number of thresholds to -+ use for matching the given specificity. -+ name: (Optional) string name of the metric instance. -+ dtype: (Optional) data type of the metric result. -+ """ -+ -+ def __init__(self, specificity, num_thresholds=200, name=None, dtype=None): -+ if specificity < 0 or specificity > 1: -+ raise ValueError('`specificity` must be in the range [0, 1].') -+ self.specificity = specificity -+ self.num_thresholds = num_thresholds -+ super(SensitivityAtSpecificity, self).__init__( -+ specificity, num_thresholds=num_thresholds, name=name, dtype=dtype) -+ -+ def result(self): -+ # Calculate specificities at all the thresholds. -+ specificities = K.switch( -+ K.greater(self.true_negatives + self.false_positives, 0), -+ (self.true_negatives / (self.true_negatives + self.false_positives)), -+ K.zeros_like(self.thresholds)) -+ -+ # Find the index of the threshold where the specificity is closest to the -+ # given specificity. -+ min_index = K.argmin( -+ K.abs(specificities - self.value), axis=0) -+ min_index = K.cast(min_index, 'int32') -+ -+ # Compute sensitivity at that index. -+ return K.switch( -+ K.greater((self.true_positives[min_index] + -+ self.false_negatives[min_index]), 0), -+ (self.true_positives[min_index] / -+ (self.true_positives[min_index] + self.false_negatives[min_index])), -+ K.zeros_like(self.true_positives[min_index])) -+ -+ def get_config(self): -+ config = { -+ 'num_thresholds': self.num_thresholds, -+ 'specificity': self.specificity -+ } -+ base_config = super(SensitivityAtSpecificity, self).get_config() -+ return dict(list(base_config.items()) + list(config.items())) -+ -+ -+class AUC(Metric): -+ """Computes the approximate AUC (Area under the curve) via a Riemann sum. -+ -+ This metric creates four local variables, `true_positives`, `true_negatives`, -+ `false_positives` and `false_negatives` that are used to compute the AUC. -+ To discretize the AUC curve, a linearly spaced set of thresholds is used to -+ compute pairs of recall and precision values. The area under the ROC-curve is -+ therefore computed using the height of the recall values by the false positive -+ rate, while the area under the PR-curve is the computed using the height of -+ the precision values by the recall. -+ -+ This value is ultimately returned as `auc`, an idempotent operation that -+ computes the area under a discretized curve of precision versus recall values -+ (computed using the aforementioned variables). The `num_thresholds` variable -+ controls the degree of discretization with larger numbers of thresholds more -+ closely approximating the true AUC. The quality of the approximation may vary -+ dramatically depending on `num_thresholds`. The `thresholds` parameter can be -+ used to manually specify thresholds which split the predictions more evenly. -+ -+ For best results, `predictions` should be distributed approximately uniformly -+ in the range [0, 1] and not peaked around 0 or 1. The quality of the AUC -+ approximation may be poor if this is not the case. Setting `summation_method` -+ to 'minoring' or 'majoring' can help quantify the error in the approximation -+ by providing lower or upper bound estimate of the AUC. -+ -+ If `sample_weight` is `None`, weights default to 1. -+ Use `sample_weight` of 0 to mask values. -+ -+ Usage with the compile API: -+ -+ ```python -+ model = keras.Model(inputs, outputs) -+ model.compile('sgd', loss='mse', metrics=[keras.metrics.AUC()]) -+ ``` -+ -+ # Arguments -+ num_thresholds: (Optional) Defaults to 200. The number of thresholds to -+ use when discretizing the roc curve. Values must be > 1. -+ curve: (Optional) Specifies the name of the curve to be computed, 'ROC' -+ [default] or 'PR' for the Precision-Recall-curve. -+ summation_method: (Optional) Specifies the Riemann summation method used -+ (https://en.wikipedia.org/wiki/Riemann_sum): 'interpolation' [default], -+ applies mid-point summation scheme for `ROC`. For PR-AUC, interpolates -+ (true/false) positives but not the ratio that is precision (see Davis -+ & Goadrich 2006 for details); 'minoring' that applies left summation -+ for increasing intervals and right summation for decreasing intervals; -+ 'majoring' that does the opposite. -+ name: (Optional) string name of the metric instance. -+ dtype: (Optional) data type of the metric result. -+ thresholds: (Optional) A list of floating point values to use as the -+ thresholds for discretizing the curve. If set, the `num_thresholds` -+ parameter is ignored. Values should be in [0, 1]. Endpoint thresholds -+ equal to {-epsilon, 1+epsilon} for a small positive epsilon value will -+ be automatically included with these to correctly handle predictions -+ equal to exactly 0 or 1. -+ """ -+ -+ def __init__(self, -+ num_thresholds=200, -+ curve='ROC', -+ summation_method='interpolation', -+ name=None, -+ dtype=None, -+ thresholds=None): -+ # Validate configurations. -+ if (isinstance(curve, metrics_utils.AUCCurve) and -+ curve not in list(metrics_utils.AUCCurve)): -+ raise ValueError('Invalid curve: "{}". Valid options are: "{}"'.format( -+ curve, list(metrics_utils.AUCCurve))) -+ if isinstance( -+ summation_method, -+ metrics_utils.AUCSummationMethod) and summation_method not in list( -+ metrics_utils.AUCSummationMethod): -+ raise ValueError( -+ 'Invalid summation method: "{}". Valid options are: "{}"'.format( -+ summation_method, list(metrics_utils.AUCSummationMethod))) -+ -+ # Update properties. -+ if thresholds is not None: -+ # If specified, use the supplied thresholds. -+ self.num_thresholds = len(thresholds) + 2 -+ thresholds = sorted(thresholds) -+ else: -+ if num_thresholds <= 1: -+ raise ValueError('`num_thresholds` must be > 1.') -+ -+ # Otherwise, linearly interpolate (num_thresholds - 2) thresholds in -+ # (0, 1). -+ self.num_thresholds = num_thresholds -+ thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) -+ for i in range(num_thresholds - 2)] -+ -+ # Add an endpoint "threshold" below zero and above one for either -+ # threshold method to account for floating point imprecisions. -+ self.thresholds = [0.0 - K.epsilon()] + thresholds + [1.0 + K.epsilon()] -+ -+ if isinstance(curve, metrics_utils.AUCCurve): -+ self.curve = curve -+ else: -+ self.curve = metrics_utils.AUCCurve.from_str(curve) -+ if isinstance(summation_method, metrics_utils.AUCSummationMethod): -+ self.summation_method = summation_method -+ else: -+ self.summation_method = metrics_utils.AUCSummationMethod.from_str( -+ summation_method) -+ super(AUC, self).__init__(name=name, dtype=dtype) -+ -+ # Create metric variables -+ self.true_positives = self.add_weight( -+ 'true_positives', -+ shape=(self.num_thresholds,), -+ initializer='zeros') -+ self.true_negatives = self.add_weight( -+ 'true_negatives', -+ shape=(self.num_thresholds,), -+ initializer='zeros') -+ self.false_positives = self.add_weight( -+ 'false_positives', -+ shape=(self.num_thresholds,), -+ initializer='zeros') -+ self.false_negatives = self.add_weight( -+ 'false_negatives', -+ shape=(self.num_thresholds,), -+ initializer='zeros') -+ -+ def update_state(self, y_true, y_pred, sample_weight=None): -+ return metrics_utils.update_confusion_matrix_variables({ -+ metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives, -+ metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives, -+ metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives, -+ metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives, -+ }, y_true, y_pred, self.thresholds, sample_weight=sample_weight) -+ -+ def interpolate_pr_auc(self): -+ """Interpolation formula inspired by section 4 of Davis & Goadrich 2006. -+ -+ https://www.biostat.wisc.edu/~page/rocpr.pdf -+ -+ Note here we derive & use a closed formula not present in the paper -+ as follows: -+ -+ Precision = TP / (TP + FP) = TP / P -+ -+ Modeling all of TP (true positive), FP (false positive) and their sum -+ P = TP + FP (predicted positive) as varying linearly within each interval -+ [A, B] between successive thresholds, we get -+ -+ Precision slope = dTP / dP -+ = (TP_B - TP_A) / (P_B - P_A) -+ = (TP - TP_A) / (P - P_A) -+ Precision = (TP_A + slope * (P - P_A)) / P -+ -+ The area within the interval is (slope / total_pos_weight) times -+ -+ int_A^B{Precision.dP} = int_A^B{(TP_A + slope * (P - P_A)) * dP / P} -+ int_A^B{Precision.dP} = int_A^B{slope * dP + intercept * dP / P} -+ -+ where intercept = TP_A - slope * P_A = TP_B - slope * P_B, resulting in -+ -+ int_A^B{Precision.dP} = TP_B - TP_A + intercept * log(P_B / P_A) -+ -+ Bringing back the factor (slope / total_pos_weight) we'd put aside, we get -+ -+ slope * [dTP + intercept * log(P_B / P_A)] / total_pos_weight -+ -+ where dTP == TP_B - TP_A. -+ -+ Note that when P_A == 0 the above calculation simplifies into -+ -+ int_A^B{Precision.dTP} = int_A^B{slope * dTP} = slope * (TP_B - TP_A) -+ -+ which is really equivalent to imputing constant precision throughout the -+ first bucket having >0 true positives. -+ -+ # Returns -+ pr_auc: an approximation of the area under the P-R curve. -+ """ -+ dtp = self.true_positives[:self.num_thresholds - -+ 1] - self.true_positives[1:] -+ p = self.true_positives + self.false_positives -+ dp = p[:self.num_thresholds - 1] - p[1:] -+ -+ prec_slope = dtp / K.maximum(dp, 0) -+ intercept = self.true_positives[1:] - (prec_slope * p[1:]) -+ -+ # Logical and -+ pMin = K.expand_dims(p[:self.num_thresholds - 1] > 0, 0) -+ pMax = K.expand_dims(p[1:] > 0, 0) -+ are_different = K.concatenate([pMin, pMax], axis=0) -+ switch_condition = K.all(are_different, axis=0) -+ -+ safe_p_ratio = K.switch( -+ switch_condition, -+ p[:self.num_thresholds - 1] / K.maximum(p[1:], 0), -+ K.ones_like(p[1:])) -+ -+ numer = prec_slope * (dtp + intercept * K.log(safe_p_ratio)) -+ denom = K.maximum(self.true_positives[1:] + self.false_negatives[1:], 0) -+ return K.sum((numer / denom)) -+ -+ def result(self): -+ if (self.curve == metrics_utils.AUCCurve.PR and -+ (self.summation_method == -+ metrics_utils.AUCSummationMethod.INTERPOLATION)): -+ # This use case is different and is handled separately. -+ return self.interpolate_pr_auc() -+ -+ # Set `x` and `y` values for the curves based on `curve` config. -+ recall = K.switch( -+ K.greater((self.true_positives), 0), -+ (self.true_positives / -+ (self.true_positives + self.false_negatives)), -+ K.zeros_like(self.true_positives)) -+ if self.curve == metrics_utils.AUCCurve.ROC: -+ fp_rate = K.switch( -+ K.greater((self.false_positives), 0), -+ (self.false_positives / -+ (self.false_positives + self.true_negatives)), -+ K.zeros_like(self.false_positives)) -+ x = fp_rate -+ y = recall -+ else: # curve == 'PR'. -+ precision = K.switch( -+ K.greater((self.true_positives), 0), -+ (self.true_positives / (self.true_positives + self.false_positives)), -+ K.zeros_like(self.true_positives)) -+ x = recall -+ y = precision -+ -+ # Find the rectangle heights based on `summation_method`. -+ if self.summation_method == metrics_utils.AUCSummationMethod.INTERPOLATION: -+ # Note: the case ('PR', 'interpolation') has been handled above. -+ heights = (y[:self.num_thresholds - 1] + y[1:]) / 2. -+ elif self.summation_method == metrics_utils.AUCSummationMethod.MINORING: -+ heights = K.minimum(y[:self.num_thresholds - 1], y[1:]) -+ else: # self.summation_method = metrics_utils.AUCSummationMethod.MAJORING: -+ heights = K.maximum(y[:self.num_thresholds - 1], y[1:]) -+ -+ # Sum up the areas of all the rectangles. -+ return K.sum((x[:self.num_thresholds - 1] - x[1:]) * heights) -+ -+ def reset_states(self): -+ K.batch_set_value( -+ [(v, np.zeros((self.num_thresholds,))) for v in self.variables]) -+ -+ def get_config(self): -+ config = { -+ 'num_thresholds': self.num_thresholds, -+ 'curve': self.curve.value, -+ 'summation_method': self.summation_method.value, -+ # We remove the endpoint thresholds as an inverse of how the thresholds -+ # were initialized. This ensures that a metric initialized from this -+ # config has the same thresholds. -+ 'thresholds': self.thresholds[1:-1], -+ } -+ base_config = super(AUC, self).get_config() -+ return dict(list(base_config.items()) + list(config.items())) -+ - - # Aliases - -diff --git a/keras/utils/__init__.py b/keras/utils/__init__.py -index 8cc39d5..65af329 100644 ---- a/keras/utils/__init__.py -+++ b/keras/utils/__init__.py -@@ -4,6 +4,8 @@ from . import generic_utils - from . import data_utils - from . import io_utils - from . import conv_utils -+from . import losses_utils -+from . import metrics_utils - - # Globally-importable utils. - from .io_utils import HDF5Matrix -diff --git a/keras/utils/losses_utils.py b/keras/utils/losses_utils.py -new file mode 100644 -index 0000000..617ebb7 ---- /dev/null -+++ b/keras/utils/losses_utils.py -@@ -0,0 +1,177 @@ -+"""Utilities related to losses.""" -+from __future__ import absolute_import -+from __future__ import division -+from __future__ import print_function -+ -+import numpy as np -+ -+from .. import backend as K -+ -+ -+class Reduction(object): -+ """Types of loss reduction. -+ -+ Contains the following values: -+ -+ * `NONE`: Un-reduced weighted losses with the same shape as input. When this -+ reduction type used with built-in Keras training loops like -+ `fit`/`evaluate`, the unreduced vector loss is passed to the optimizer but -+ the reported loss will be a scalar value. -+ * `SUM`: Scalar sum of weighted losses. -+ * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses. -+ """ -+ -+ NONE = 'none' -+ SUM = 'sum' -+ SUM_OVER_BATCH_SIZE = 'sum_over_batch_size' -+ -+ @classmethod -+ def all(cls): -+ return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE) -+ -+ @classmethod -+ def validate(cls, key): -+ if key not in cls.all(): -+ raise ValueError('Invalid Reduction Key %s.' % key) -+ -+ -+def squeeze_or_expand_dimensions(y_pred, y_true=None, sample_weight=None): -+ """Squeeze or expand last dimension if needed. -+ -+ 1. Squeezes last dim of `y_pred` or `y_true` if their rank differs by 1. -+ 2. Squeezes or expands last dim of `sample_weight` if its rank differs by 1 -+ from the new rank of `y_pred`. -+ If `sample_weight` is scalar, it is kept scalar. -+ -+ # Arguments -+ y_pred: Predicted values, a `Tensor` of arbitrary dimensions. -+ y_true: Optional label `Tensor` whose dimensions match `y_pred`. -+ sample_weight: Optional weight scalar or `Tensor` whose dimensions match -+ `y_pred`. -+ -+ # Returns -+ Tuple of `y_pred`, `y_true` and `sample_weight`. Each of them possibly has -+ the last dimension squeezed, `sample_weight` could be extended by one -+ dimension. -+ """ -+ if y_true is not None: -+ y_pred_rank = K.ndim(y_pred) -+ y_pred_shape = K.int_shape(y_pred) -+ y_true_rank = K.ndim(y_true) -+ y_true_shape = K.int_shape(y_true) -+ -+ if (y_pred_rank - y_true_rank == 1) and (y_pred_shape[-1] == 1): -+ y_pred = K.squeeze(y_pred, -1) -+ elif (y_true_rank - y_pred_rank == 1) and (y_true_shape[-1] == 1): -+ y_true = K.squeeze(y_true, -1) -+ -+ if sample_weight is None: -+ return y_pred, y_true -+ -+ y_pred_rank = K.ndim(y_pred) -+ weights_rank = K.ndim(sample_weight) -+ if weights_rank != 0: -+ if weights_rank - y_pred_rank == 1: -+ sample_weight = K.squeeze(sample_weight, -1) -+ elif y_pred_rank - weights_rank == 1: -+ sample_weight = K.expand_dims(sample_weight, -1) -+ return y_pred, y_true, sample_weight -+ -+ -+def _num_elements(losses): -+ """Computes the number of elements in `losses` tensor.""" -+ with K.name_scope('num_elements') as scope: -+ return K.cast(K.size(losses, name=scope), losses.dtype) -+ -+ -+def reduce_weighted_loss(weighted_losses, reduction=Reduction.SUM_OVER_BATCH_SIZE): -+ """Reduces the individual weighted loss measurements.""" -+ if reduction == Reduction.NONE: -+ loss = weighted_losses -+ else: -+ loss = K.sum(weighted_losses) -+ if reduction == Reduction.SUM_OVER_BATCH_SIZE: -+ loss = loss / _num_elements(weighted_losses) -+ return loss -+ -+ -+def broadcast_weights(values, sample_weight): -+ # Broadcast weights if possible. -+ weights_shape = K.int_shape(sample_weight) -+ values_shape = K.int_shape(values) -+ -+ if values_shape != weights_shape: -+ weights_rank = K.ndim(sample_weight) -+ values_rank = K.ndim(values) -+ -+ # Raise error if ndim of weights is > values. -+ if weights_rank > values_rank: -+ raise ValueError( -+ 'Incompatible shapes: `values` {} vs `sample_weight` {}'.format( -+ values_shape, weights_shape)) -+ -+ # Expand dim of weights to match ndim of values, if required. -+ for i in range(weights_rank, values_rank): -+ sample_weight = K.expand_dims(sample_weight, axis=i) -+ -+ if weights_shape is not None and values_shape is not None: -+ for i in range(weights_rank): -+ if (weights_shape[i] is not None and -+ values_shape[i] is not None and -+ weights_shape[i] != values_shape[i]): -+ # Cannot be broadcasted. -+ if weights_shape[i] != 1: -+ raise ValueError( -+ 'Incompatible shapes: `values` {} vs ' -+ '`sample_weight` {}'.format( -+ values_shape, weights_shape)) -+ sample_weight = K.repeat_elements( -+ sample_weight, values_shape[i], axis=i) -+ return sample_weight -+ -+ -+def compute_weighted_loss(losses, -+ sample_weight=None, -+ reduction=Reduction.SUM_OVER_BATCH_SIZE, -+ name=None): -+ """Computes the weighted loss. -+ -+ # Arguments -+ losses: `Tensor` of shape `[batch_size, d1, ... dN]`. -+ sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as -+ ` losses`, or be broadcastable to `losses`. -+ reduction: (Optional) Type of Reduction to apply to loss. -+ Default value is `SUM_OVER_BATCH_SIZE`. -+ name: Optional name for the op. -+ -+ # Raises -+ ValueError: If the shape of `sample_weight` is not compatible with `losses`. -+ -+ # Returns -+ Weighted loss `Tensor` of the same type as `losses`. If `reduction` is -+ `NONE`, this has the same shape as `losses`; otherwise, it is scalar. -+ """ -+ Reduction.validate(reduction) -+ if sample_weight is None: -+ sample_weight = 1.0 -+ with K.name_scope(name or 'weighted_loss'): -+ input_dtype = K.dtype(losses) -+ losses = K.cast(losses, K.floatx()) -+ sample_weight = K.cast(sample_weight, K.floatx()) -+ -+ # Update dimensions of `sample_weight` to match with `losses` if possible. -+ losses, _, sample_weight = squeeze_or_expand_dimensions( -+ losses, None, sample_weight) -+ -+ # Broadcast weights if possible. -+ sample_weight = broadcast_weights(losses, sample_weight) -+ -+ # Apply weights to losses. -+ weighted_losses = sample_weight * losses -+ -+ # Apply reduction function to the individual weighted losses. -+ loss = reduce_weighted_loss(weighted_losses, reduction) -+ # Convert the result back to the input type. -+ loss = K.cast(loss, input_dtype) -+ return loss -+ -diff --git a/keras/utils/metrics_utils.py b/keras/utils/metrics_utils.py -new file mode 100644 -index 0000000..e6a5bb0 ---- /dev/null -+++ b/keras/utils/metrics_utils.py -@@ -0,0 +1,278 @@ -+"""Utilities related to metrics.""" -+from __future__ import absolute_import -+from __future__ import division -+from __future__ import print_function -+ -+from enum import Enum -+ -+from .. import backend as K -+from . import losses_utils -+ -+NEG_INF = -1e10 -+ -+class Reduction(object): -+ """Types of metrics reduction. -+ Contains the following values: -+ * `SUM`: Scalar sum of weighted values. -+ * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` of weighted values divided by -+ number of elements in values. -+ * `WEIGHTED_MEAN`: Scalar sum of weighted values divided by sum of weights. -+ """ -+ -+ SUM = 'sum' -+ SUM_OVER_BATCH_SIZE = 'sum_over_batch_size' -+ WEIGHTED_MEAN = 'weighted_mean' -+ -+ -+def update_state_wrapper(update_state_fn): -+ """Decorator to wrap metric `update_state()` with `add_update()`. -+ # Arguments -+ update_state_fn: function that accumulates metric statistics. -+ # Returns -+ Decorated function that wraps `update_state_fn()` with `add_update()`. -+ """ -+ def decorated(metric_obj, *args, **kwargs): -+ """Decorated function with `add_update()`.""" -+ -+ update_op = update_state_fn(*args, **kwargs) -+ metric_obj.add_update(update_op) -+ return update_op -+ -+ return decorated -+ -+def result_wrapper(result_fn): -+ """Decorator to wrap metric `result()` with identity op. -+ Wrapping result in identity so that control dependency between -+ update_op from `update_state` and result works in case result returns -+ a tensor. -+ # Arguments -+ result_fn: function that computes the metric result. -+ # Returns -+ Decorated function that wraps `result()` with identity op. -+ """ -+ def decorated(metric_obj, *args, **kwargs): -+ result_t = K.identity(result_fn(*args, **kwargs)) -+ metric_obj._call_result = result_t -+ result_t._is_metric = True -+ return result_t -+ return decorated -+ -+ -+def to_list(x): -+ if isinstance(x, list): -+ return x -+ return [x] -+ -+ -+def assert_thresholds_range(thresholds): -+ if thresholds is not None: -+ invalid_thresholds = [t for t in thresholds if t is None or t < 0 or t > 1] -+ if invalid_thresholds: -+ raise ValueError( -+ 'Threshold values must be in [0, 1]. Invalid values: {}'.format( -+ invalid_thresholds)) -+ -+ -+def parse_init_thresholds(thresholds, default_threshold=0.5): -+ if thresholds is not None: -+ assert_thresholds_range(to_list(thresholds)) -+ thresholds = to_list(default_threshold if thresholds is None else thresholds) -+ return thresholds -+ -+class ConfusionMatrix(Enum): -+ TRUE_POSITIVES = 'tp' -+ FALSE_POSITIVES = 'fp' -+ TRUE_NEGATIVES = 'tn' -+ FALSE_NEGATIVES = 'fn' -+ -+class AUCCurve(Enum): -+ """Type of AUC Curve (ROC or PR).""" -+ ROC = 'ROC' -+ PR = 'PR' -+ -+ @staticmethod -+ def from_str(key): -+ if key in ('pr', 'PR'): -+ return AUCCurve.PR -+ elif key in ('roc', 'ROC'): -+ return AUCCurve.ROC -+ else: -+ raise ValueError('Invalid AUC curve value "%s".' % key) -+ -+ -+class AUCSummationMethod(Enum): -+ """Type of AUC summation method. -+ -+ https://en.wikipedia.org/wiki/Riemann_sum) -+ -+ Contains the following values: -+ * 'interpolation': Applies mid-point summation scheme for `ROC` curve. For -+ `PR` curve, interpolates (true/false) positives but not the ratio that is -+ precision (see Davis & Goadrich 2006 for details). -+ * 'minoring': Applies left summation for increasing intervals and right -+ summation for decreasing intervals. -+ * 'majoring': Applies right summation for increasing intervals and left -+ summation for decreasing intervals. -+ """ -+ INTERPOLATION = 'interpolation' -+ MAJORING = 'majoring' -+ MINORING = 'minoring' -+ -+ @staticmethod -+ def from_str(key): -+ if key in ('interpolation', 'Interpolation'): -+ return AUCSummationMethod.INTERPOLATION -+ elif key in ('majoring', 'Majoring'): -+ return AUCSummationMethod.MAJORING -+ elif key in ('minoring', 'Minoring'): -+ return AUCSummationMethod.MINORING -+ else: -+ raise ValueError('Invalid AUC summation method value "%s".' % key) -+ -+def weighted_assign_add(label, pred, weights, var): -+ # Logical and -+ label = K.expand_dims(label, 0) -+ pred = K.expand_dims(pred, 0) -+ are_different = K.concatenate([label, pred], axis=0) -+ label_and_pred = K.all(are_different, axis=0) -+ label_and_pred = K.cast(label_and_pred, dtype=K.floatx()) -+ if weights is not None: -+ label_and_pred *= weights -+ return var.assign_add(K.sum(label_and_pred, 1)) -+ -+def update_confusion_matrix_variables(variables_to_update, -+ y_true, -+ y_pred, -+ thresholds, -+ top_k=None, -+ class_id=None, -+ sample_weight=None): -+ """Returns op to update the given confusion matrix variables. -+ For every pair of values in y_true and y_pred: -+ true_positive: y_true == True and y_pred > thresholds -+ false_negatives: y_true == True and y_pred <= thresholds -+ true_negatives: y_true == False and y_pred <= thresholds -+ false_positive: y_true == False and y_pred > thresholds -+ The results will be weighted and added together. When multiple thresholds are -+ provided, we will repeat the same for every threshold. -+ For estimation of these metrics over a stream of data, the function creates an -+ `update_op` operation that updates the given variables. -+ If `sample_weight` is `None`, weights default to 1. -+ Use weights of 0 to mask values. -+ # Arguments -+ variables_to_update: Dictionary with 'tp', 'fn', 'tn', 'fp' as valid keys -+ and corresponding variables to update as values. -+ y_true: A `Tensor` whose shape matches `y_pred`. Will be cast to `bool`. -+ y_pred: A floating point `Tensor` of arbitrary shape and whose values are in -+ the range `[0, 1]`. -+ thresholds: A float value or a python list or tuple of float thresholds in -+ `[0, 1]`, or NEG_INF (used when top_k is set). -+ top_k: Optional int, indicates that the positive labels should be limited to -+ the top k predictions. -+ class_id: Optional int, limits the prediction and labels to the class -+ specified by this argument. -+ sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as -+ `y_true`, and must be broadcastable to `y_true` (i.e., all dimensions must -+ be either `1`, or the same as the corresponding `y_true` dimension). -+ # Returns -+ Update ops. -+ # Raises -+ ValueError: If `y_pred` and `y_true` have mismatched shapes, or if -+ `sample_weight` is not `None` and its shape doesn't match `y_pred`, or if -+ `variables_to_update` contains invalid keys. -+ """ -+ if variables_to_update is None: -+ return -+ y_true = K.cast(y_true, dtype=K.floatx()) -+ y_pred = K.cast(y_pred, dtype=K.floatx()) -+ if sample_weight is not None: -+ sample_weight = K.cast(sample_weight, dtype=K.floatx()) -+ -+ if not any(key -+ for key in variables_to_update -+ if key in list(ConfusionMatrix)): -+ raise ValueError( -+ 'Please provide at least one valid confusion matrix ' -+ 'variable to update. Valid variable key options are: "{}". ' -+ 'Received: "{}"'.format( -+ list(ConfusionMatrix), variables_to_update.keys())) -+ -+ invalid_keys = [ -+ key for key in variables_to_update if key not in list(ConfusionMatrix) -+ ] -+ if invalid_keys: -+ raise ValueError( -+ 'Invalid keys: {}. Valid variable key options are: "{}"'.format( -+ invalid_keys, list(ConfusionMatrix))) -+ -+ if sample_weight is None: -+ y_pred, y_true = losses_utils.squeeze_or_expand_dimensions( -+ y_pred, y_true=y_true) -+ else: -+ y_pred, y_true, sample_weight = ( -+ losses_utils.squeeze_or_expand_dimensions( -+ y_pred, y_true=y_true, sample_weight=sample_weight)) -+ -+ if top_k is not None: -+ y_pred = _filter_top_k(y_pred, top_k) -+ if class_id is not None: -+ y_true = y_true[..., class_id] -+ y_pred = y_pred[..., class_id] -+ -+ thresholds = to_list(thresholds) -+ num_thresholds = len(thresholds) -+ num_predictions = K.size(y_pred) -+ -+ # Reshape predictions and labels. -+ predictions_2d = K.reshape(y_pred, [1, -1]) -+ labels_2d = K.reshape( -+ K.cast(y_true, dtype='bool'), [1, -1]) -+ -+ # Tile the thresholds for every prediction. -+ thresh_tiled = K.tile( -+ K.expand_dims(K.constant(thresholds), 1), -+ K.stack([1, num_predictions])) -+ -+ # Tile the predictions for every threshold. -+ preds_tiled = K.tile(predictions_2d, [num_thresholds, 1]) -+ -+ # Compare predictions and threshold. -+ pred_is_pos = K.greater(preds_tiled, thresh_tiled) -+ pred_is_neg = K.greater(thresh_tiled, preds_tiled) -+ -+ # Tile labels by number of thresholds -+ label_is_pos = K.tile(labels_2d, [num_thresholds, 1]) -+ -+ if sample_weight is not None: -+ weights = losses_utils.broadcast_weights( -+ y_pred, K.cast(sample_weight, dtype=K.floatx())) -+ weights_tiled = K.tile( -+ K.reshape(weights, [1, -1]), [num_thresholds, 1]) -+ else: -+ weights_tiled = None -+ -+ update_ops = [] -+ loop_vars = { -+ ConfusionMatrix.TRUE_POSITIVES: (label_is_pos, pred_is_pos), -+ } -+ update_tn = ConfusionMatrix.TRUE_NEGATIVES in variables_to_update -+ update_fp = ConfusionMatrix.FALSE_POSITIVES in variables_to_update -+ update_fn = ConfusionMatrix.FALSE_NEGATIVES in variables_to_update -+ -+ if update_fn or update_tn: -+ loop_vars[ConfusionMatrix.FALSE_NEGATIVES] = (label_is_pos, pred_is_neg) -+ -+ if update_fp or update_tn: -+ label_is_neg = K.equal( -+ label_is_pos, K.zeros_like(label_is_pos, dtype=label_is_pos.dtype)) -+ loop_vars[ConfusionMatrix.FALSE_POSITIVES] = (label_is_neg, pred_is_pos) -+ if update_tn: -+ loop_vars[ConfusionMatrix.TRUE_NEGATIVES] = (label_is_neg, pred_is_neg) -+ -+ for matrix_cond, (label, pred) in loop_vars.items(): -+ if matrix_cond in variables_to_update: -+ update_ops.append( -+ weighted_assign_add(label, pred, weights_tiled, -+ variables_to_update[matrix_cond])) -+ return update_ops -+ --- -2.26.2 - |