Independent metrics (#31)

This patch separates metrics by the task they target to measure.
So now, there are tree types of metrics:
 - binary metrics
 - categorical metrics
 - sparse categorical metrics

To preserve backward compatibility with the older code, old names
"precision", "recall", etc. refer to the "binary_precision",
"binary_recall", etc. respectively.

These changes require to bump major version of the package.

Author: ybubnov

README.md

@@ -18,35 +18,73 @@ pip install keras-metrics
 The usage of the package is simple:
 ```py
 import keras
-import keras_metrics
+import keras_metrics as km
 
 model = models.Sequential()
 model.add(keras.layers.Dense(1, activation="sigmoid", input_dim=2))
 model.add(keras.layers.Dense(1, activation="softmax"))
 
 model.compile(optimizer="sgd",
               loss="binary_crossentropy",
-              metrics=[keras_metrics.precision(), keras_metrics.recall()])
+              metrics=[km.binary_precision(), km.binary_recall()])
 ```
 
 Similar configuration for multi-label binary crossentropy:
 ```py
 import keras
-import keras_metrics
+import keras_metrics as km
 
 model = models.Sequential()
 model.add(keras.layers.Dense(1, activation="sigmoid", input_dim=2))
 model.add(keras.layers.Dense(2, activation="softmax"))
 
 # Calculate precision for the second label.
-precision = keras_metrics.precision(label=1)
+precision = km.binary_precision(label=1)
 
 # Calculate recall for the first label.
-recall = keras_metrics.recall(label=0)
+recall = km.recall(label=0)
 
 model.compile(optimizer="sgd",
               loss="binary_crossentropy",
               metrics=[precision, recall])
 ```
 
+Keras metrics package also supports metrics for categorical crossentropy and
+sparse categorical crossentropy:
+```py
+import keras_metrics as km
+
+c_precision = km.categorical_precision()
+sc_precision = km.sparse_categorical_precision()
+
+# ...
+```
+
+## Tensorflow Keras
+
+Tensorflow library provides the ```keras``` package as parts of its API, in
+order to use ```keras_metrics``` with Tensorflow Keras, you are advised to
+perform model training with initialized global variables:
+```py
+import numpy as np
+import keras_metrics as km
+import tensorflow as tf
+import tensorflow.keras as keras
+
+model = keras.Sequential()
+model.add(keras.layers.Dense(1, activation="softmax"))
+model.compile(optimizer="sgd",
+              loss="binary_crossentropy",
+              metrics=[km.binary_true_positive()])
+
+x = np.array([[0], [1], [0], [1]])
+y = np.array([1, 0, 1, 0]
+
+# Wrap model.fit into the session with global
+# variables initialization.
+with tf.Session() as s:
+    s.run(tf.global_variables_initializer())
+    model.fit(x=x, y=y)
+```
+
 [BuildStatus]: https://travis-ci.org/netrack/keras-metrics.svg?branch=master

keras_metrics/__init__.py

@@ -1,4 +1,63 @@
+from functools import partial
+from keras_metrics import metrics as m
+from keras_metrics import casts
+
+
 __version__ = "0.0.7"
 
 
-from keras_metrics.metrics import *
+def metric_fn(cls, cast_strategy):
+    def fn(label=0, **kwargs):
+        metric = cls(label=label, cast_strategy=cast_strategy, **kwargs)
+        metric.__name__ = "%s_%s" % (cast_strategy.__name__, cls.__name__)
+        return metric
+    return fn
+
+
+binary_metric = partial(
+    metric_fn, cast_strategy=casts.binary)
+
+
+categorical_metric = partial(
+    metric_fn, cast_strategy=casts.categorical)
+
+
+sparse_categorical_metric = partial(
+    metric_fn, cast_strategy=casts.sparse_categorical)
+
+
+binary_true_positive = binary_metric(m.true_positive)
+binary_true_negative = binary_metric(m.true_negative)
+binary_false_positive = binary_metric(m.false_positive)
+binary_false_negative = binary_metric(m.false_negative)
+binary_precision = binary_metric(m.precision)
+binary_recall = binary_metric(m.recall)
+binary_f1_score = binary_metric(m.f1_score)
+
+
+categorical_true_positive = categorical_metric(m.true_positive)
+categorical_true_negative = categorical_metric(m.true_negative)
+categorical_false_positive = categorical_metric(m.false_positive)
+categorical_false_negative = categorical_metric(m.false_negative)
+categorical_precision = categorical_metric(m.precision)
+categorical_recall = categorical_metric(m.recall)
+categorical_f1_score = categorical_metric(m.f1_score)
+
+
+sparse_categorical_true_positive = sparse_categorical_metric(m.true_positive)
+sparse_categorical_true_negative = sparse_categorical_metric(m.true_negative)
+sparse_categorical_false_positive = sparse_categorical_metric(m.false_positive)
+sparse_categorical_false_negative = sparse_categorical_metric(m.false_negative)
+sparse_categorical_precision = sparse_categorical_metric(m.precision)
+sparse_categorical_recall = sparse_categorical_metric(m.recall)
+sparse_categorical_f1_score = sparse_categorical_metric(m.f1_score)
+
+
+# For backward compatibility.
+true_positive = binary_true_positive
+true_negative = binary_true_negative
+false_positive = binary_false_positive
+false_negative = binary_false_negative
+precision = binary_precision
+recall = binary_recall
+f1_score = binary_f1_score

keras_metrics/casts.py

@@ -0,0 +1,26 @@
+from keras import backend as K
+
+
+def binary(y_true, y_pred, dtype="int32", label=0):
+    return categorical(y_true, y_pred, dtype, label)
+
+
+def categorical(y_true, y_pred, dtype="int32", label=0):
+    column = slice(label, label+1)
+
+    y_true = y_true[..., column]
+    y_pred = y_pred[..., column]
+
+    y_true = K.cast(K.round(y_true), dtype)
+    y_pred = K.cast(K.round(y_pred), dtype)
+
+    return y_true, y_pred
+
+
+def sparse_categorical(y_true, y_pred, dtype="int32", label=0):
+    y_true = K.cast(K.equal(y_true, label), dtype=dtype)
+
+    y_pred = y_pred[..., slice(label, label+1)]
+    y_pred = K.cast(K.round(y_pred), dtype)
+
+    return y_true, y_pred

keras_metrics/metrics.py

@@ -1,76 +1,40 @@
-from functools import partial
 from keras import backend as K
 from keras.layers import Layer
 from operator import truediv
 
 
 class layer(Layer):
 
-    def __init__(self, label=None, name=None, sparse=False, **kwargs):
+    def __init__(self, name=None, label=0, cast_strategy=None, **kwargs):
         super(layer, self).__init__(name=name, **kwargs)
+
         self.stateful = True
+        self.label = label
+        self.cast_strategy = cast_strategy
         self.epsilon = K.constant(K.epsilon(), dtype="float64")
-        self.__name__ = name
-        self.sparse = sparse
-
-        # If layer metric is explicitly created to evaluate specified class,
-        # then use a binary transformation of the output arrays, otherwise
-        # calculate an "overall" metric.
-        if label:
-            self.cast_strategy = partial(self._binary, label=label)
-        else:
-            self.cast_strategy = self._categorical
 
     def cast(self, y_true, y_pred, dtype="int32"):
         """Convert the specified true and predicted output to the specified
         destination type (int32 by default).
         """
-        return self.cast_strategy(y_true, y_pred, dtype=dtype)
-
-    def _binary(self, y_true, y_pred, dtype, label=0):
-        column = slice(label, label+1)
-
-        # Slice a column of the output array.
-        if self.sparse:
-            y_true = K.cast(K.equal(y_true, label), dtype=dtype)
-        else:
-            y_true = y_true[...,column]
-        y_pred = y_pred[...,column]
-        return self._categorical(y_true, y_pred, dtype)
-
-    def _categorical(self, y_true, y_pred, dtype):
-        # In case when user did not specify the label, and the shape
-        # of the output vector has exactly two elements, we can choose
-        # the label automatically.
-        #
-        # When the shape had dimension 3 and more and the label is
-        # not specified, we should throw an error as long as calculated
-        # metric is incorrect.
-        labels = y_pred.shape[-1]
-        if labels == 2:
-            return self._binary(y_true, y_pred, dtype, label=1)
-        elif labels > 2:
-            raise ValueError("With 2 and more output classes a "
-                             "metric label must be specified")
-
-        y_true = K.cast(K.round(y_true), dtype)
-        y_pred = K.cast(K.round(y_pred), dtype)
-        return y_true, y_pred
+        return self.cast_strategy(
+            y_true, y_pred, dtype=dtype, label=self.label)
 
     def __getattribute__(self, name):
         if name == "get_config":
             raise AttributeError
         return object.__getattribute__(self, name)
 
+
 class true_positive(layer):
     """Create a metric for model's true positives amount calculation.
 
     A true positive is an outcome where the model correctly predicts the
     positive class.
     """
 
-    def __init__(self, name="true_positive", label=None, sparse=False, **kwargs):
-        super(true_positive, self).__init__(name=name, label=label, sparse=sparse, **kwargs)
+    def __init__(self, name="true_positive", **kwargs):
+        super(true_positive, self).__init__(name=name, **kwargs)
         self.tp = K.variable(0, dtype="int32")
 
     def reset_states(self):
@@ -96,8 +60,8 @@ class true_negative(layer):
     negative class.
     """
 
-    def __init__(self, name="true_negative", label=None, sparse=False, **kwargs):
-        super(true_negative, self).__init__(name=name, label=label, sparse=sparse, **kwargs)
+    def __init__(self, name="true_negative", **kwargs):
+        super(true_negative, self).__init__(name=name, **kwargs)
         self.tn = K.variable(0, dtype="int32")
 
     def reset_states(self):
@@ -126,8 +90,8 @@ class false_negative(layer):
     negative class.
     """
 
-    def __init__(self, name="false_negative", label=None, sparse=False, **kwargs):
-        super(false_negative, self).__init__(name=name, label=label, sparse=sparse, **kwargs)
+    def __init__(self, name="false_negative", **kwargs):
+        super(false_negative, self).__init__(name=name, **kwargs)
         self.fn = K.variable(0, dtype="int32")
 
     def reset_states(self):
@@ -154,8 +118,8 @@ class false_positive(layer):
     positive class.
     """
 
-    def __init__(self, name="false_positive", label=None, sparse=False, **kwargs):
-        super(false_positive, self).__init__(name=name, label=label, sparse=sparse, **kwargs)
+    def __init__(self, name="false_positive", **kwargs):
+        super(false_positive, self).__init__(name=name, **kwargs)
         self.fp = K.variable(0, dtype="int32")
 
     def reset_states(self):
@@ -178,14 +142,15 @@ def __call__(self, y_true, y_pred):
 class recall(layer):
     """Create a metric for model's recall calculation.
 
-    Recall measures proportion of actual positives that was identified correctly.
+    Recall measures proportion of actual positives that was identified
+    correctly.
     """
 
-    def __init__(self, name="recall", label=None, sparse=False, **kwargs):
-        super(recall, self).__init__(name=name, label=label, sparse=sparse, **kwargs)
+    def __init__(self, name="recall", **kwargs):
+        super(recall, self).__init__(name=name, **kwargs)
 
-        self.tp = true_positive(label=label, sparse=sparse)
-        self.fn = false_negative(label=label, sparse=sparse)
+        self.tp = true_positive(**kwargs)
+        self.fn = false_negative(**kwargs)
 
     def reset_states(self):
         """Reset the state of the metrics."""
@@ -212,11 +177,11 @@ class precision(layer):
     actually correct.
     """
 
-    def __init__(self, name="precision", label=None, sparse=False, **kwargs):
-        super(precision, self).__init__(name=name, label=label, sparse=sparse, **kwargs)
+    def __init__(self, name="precision", **kwargs):
+        super(precision, self).__init__(name=name, **kwargs)
 
-        self.tp = true_positive(label=label, sparse=sparse)
-        self.fp = false_positive(label=label, sparse=sparse)
+        self.tp = true_positive(**kwargs)
+        self.fp = false_positive(**kwargs)
 
     def reset_states(self):
         """Reset the state of the metrics."""
@@ -242,11 +207,11 @@ class f1_score(layer):
     The F1 score is the harmonic mean of precision and recall.
     """
 
-    def __init__(self, name="f1_score", label=None, sparse=False, **kwargs):
-        super(f1_score, self).__init__(name=name, label=label, sparse=sparse, **kwargs)
+    def __init__(self, name="f1_score", **kwargs):
+        super(f1_score, self).__init__(name=name, **kwargs)
 
-        self.precision = precision(label=label, sparse=sparse)
-        self.recall = recall(label=label, sparse=sparse)
+        self.precision = precision(**kwargs)
+        self.recall = recall(**kwargs)
 
     def reset_states(self):
         """Reset the state of the metrics."""