Merge remote-tracking branch 'upstream/master' into ci/docs/add-mkdocs

Author: smokestacklightnin

README.md

@@ -209,7 +209,9 @@ combinations may also work.
 
 |tensorflow-model-analysis                                                            |apache-beam[gcp]|pyarrow   |tensorflow         |tensorflow-metadata |tfx-bsl   |
 |------------------------------------------------------------------------------------ |----------------|----------|-------------------|--------------------|----------|
-|[GitHub master](https://github.com/tensorflow/model-analysis/blob/master/RELEASE.md) | 2.47.0         | 10.0.0   | nightly (2.x)     | 1.15.0             | 1.15.1   |
+|[GitHub master](https://github.com/tensorflow/model-analysis/blob/master/RELEASE.md) | 2.60.0         | 10.0.1   | nightly (2.x)     | 1.16.1             | 1.16.1   |
+|[0.47.1](https://github.com/tensorflow/model-analysis/blob/v0.47.1/RELEASE.md)       | 2.60.0         | 10.0.1   | 2.16              | 1.16.1             | 1.16.1   |
+|[0.47.0](https://github.com/tensorflow/model-analysis/blob/v0.47.0/RELEASE.md)       | 2.60.0         | 10.0.1   | 2.16              | 1.16.1             | 1.16.1   |
 |[0.46.0](https://github.com/tensorflow/model-analysis/blob/v0.46.0/RELEASE.md)       | 2.47.0         | 10.0.0   | 2.15              | 1.15.0             | 1.15.1   |
 |[0.45.0](https://github.com/tensorflow/model-analysis/blob/v0.45.0/RELEASE.md)       | 2.47.0         | 10.0.0   | 2.13              | 1.14.0             | 1.14.0   |
 |[0.44.0](https://github.com/tensorflow/model-analysis/blob/v0.44.0/RELEASE.md)       | 2.40.0         | 6.0.0    | 2.12              | 1.13.1             | 1.13.0   |

RELEASE.md

@@ -4,6 +4,38 @@
 
 ## Major Features and Improvements
 
+## Bug fixes and other Changes
+
+## Breaking Changes
+
+*   Depends on `tensorflow>=2.17,<2.18`.
+*   Depends on `protobuf>=4.25.2,<5` for Python 3.11 and on
+    `protobuf>=4.21.6,<6.0.0` for 3.9 and 3.10.
+
+## Deprecations
+
+# Version 0.47.1
+
+## Major Features and Improvements
+
+*   N/A
+
+## Bug fixes and other Changes
+
+*   Removing addons from __init__.py as it's deprecated with Eval Saved Model.
+
+## Breaking Changes
+
+*   N/A
+
+## Deprecations
+
+*   N/A
+
+# Version 0.47.0
+
+## Major Features and Improvements
+
 *   Adds `False{Negative,Positive}FeatureSampler` metrics.
 *   Adds `RecallAtFalsePositiveRate` metrics.
 *   Adds 'NegToNegFlipRate', 'NegToPosFlipRate', 'PosToNegFlipRate',
@@ -20,6 +52,8 @@
 *   Remove keras_util_test.py which is based on estimator models.
 *   Remove dependency on eval_saved_model encodings.
 *   Downloads `punkt_tab` in Rouge metric.
+*   Depends on `tensorflow 2.16`
+*   Relax dependency on Protobuf to include version 5.x
 
 ## Breaking Changes
 

docs/install.md

@@ -152,7 +152,9 @@ combinations may also work.
 
 tensorflow-model-analysis                                                            | apache-beam[gcp] | pyarrow | tensorflow    | tensorflow-metadata | tfx-bsl
 ------------------------------------------------------------------------------------ | ---------------- | ------- | ------------- | ------------------- | -------
-[GitHub master](https://github.com/tensorflow/model-analysis/blob/master/RELEASE.md) | 2.47.0           | 10.0.0  | nightly (2.x) | 1.15.0              | 1.15.1
+[GitHub master](https://github.com/tensorflow/model-analysis/blob/master/RELEASE.md) | 2.60.0           | 10.0.1  | nightly (2.x) | 1.16.1              | 1.16.1
+[0.47.1](https://github.com/tensorflow/model-analysis/blob/v0.47.1/RELEASE.md)       | 2.60.0           | 10.0.1  | 2.16          | 1.16.1              | 1.16.1
+[0.47.0](https://github.com/tensorflow/model-analysis/blob/v0.47.0/RELEASE.md)       | 2.60.0           | 10.0.1  | 2.16          | 1.16.1              | 1.16.1
 [0.46.0](https://github.com/tensorflow/model-analysis/blob/v0.46.0/RELEASE.md)       | 2.47.0           | 10.0.0  | 2.15          | 1.15.0              | 1.15.1
 [0.45.0](https://github.com/tensorflow/model-analysis/blob/v0.45.0/RELEASE.md)       | 2.47.0           | 10.0.0  | 2.13          | 1.14.0              | 1.14.0
 [0.44.0](https://github.com/tensorflow/model-analysis/blob/v0.44.0/RELEASE.md)       | 2.40.0           | 6.0.0   | 2.12          | 1.13.1              | 1.13.0

setup.py

@@ -263,7 +263,7 @@ def _make_docs_packages():
 def _make_extra_packages_tfjs():
   # Packages needed for tfjs.
   return [
-      'tensorflowjs>=4.5.0,<5',
+      'tensorflowjs>=4.22.0,<5',
   ]
 
 
@@ -314,30 +314,25 @@ def select_constraint(default, nightly=None, git_master=None):
         'numpy>=1.23.5',
         'pandas>=1.0,<2',
         'pillow>=9.4.0',
-        'protobuf>=4.25.2,<5;python_version>="3.11"',
-        'protobuf>=3.20.3,<5;python_version<"3.11"',
+        'protobuf>=4.25.2,<6.0.0;python_version>="3.11"',
+        'protobuf>=4.21.6,<6.0.0;python_version<"3.11"',
         'pyarrow>=10,<11',
         'rouge-score>=0.1.2,<2',
         'sacrebleu>=2.3,<4',
         'scipy>=1.4.1,<2',
         'six>=1.12,<2',
-        'tensorflow'
-        + select_constraint(
-            default='>=2.15,<2.16',
-            nightly='>=2.16.0.dev',
-            git_master='@git+https://github.com/tensorflow/tensorflow@master',
-        ),
+        'tensorflow>=2.17,<2.18',
         'tensorflow-estimator>=2.10',
         'tensorflow-metadata'
         + select_constraint(
-            default='>=1.15.0,<1.16.0',
-            nightly='>=1.16.0.dev',
+            default='>=1.16.1,<1.17.0',
+            nightly='>=1.17.0.dev',
             git_master='@git+https://github.com/tensorflow/metadata@master',
         ),
         'tfx-bsl'
         + select_constraint(
-            default='>=1.15.1,<1.16.0',
-            nightly='>=1.16.0.dev',
+            default='>=1.16.1,<1.17.0',
+            nightly='>=1.17.0.dev',
             git_master='@git+https://github.com/tensorflow/tfx-bsl@master',
         ),
     ],

tensorflow_model_analysis/__init__.py

@@ -63,7 +63,6 @@
   # TODO(b/73882264): The orders should be kept in order to make benchmark on
   # DataFlow work. We need to look into why the import orders matters for the
   # DataFlow benchmark.
-  from tensorflow_model_analysis import addons
   from tensorflow_model_analysis import extractors
   from tensorflow_model_analysis import slicer
   from tensorflow_model_analysis import validators

tensorflow_model_analysis/metrics/metric_util.py

@@ -909,7 +909,7 @@ def lookup(label):
   return np.array([lookup(l) for l in labels.flatten()]).reshape(labels.shape)
 
 
-def select_class_id(
+def select_class_id(  # pytype: disable=annotation-type-mismatch
     class_id: int,
     labels: Any,
     predictions: Any,
@@ -983,9 +983,11 @@ def lookup(arr, target):
           predictions.reshape(predictions_out_shape))
 
 
-def _verify_sparse_labels(labels: np.ndarray,
-                          predictions: np.ndarray,
-                          sparse_labels: bool = None) -> bool:
+def _verify_sparse_labels(
+    labels: np.ndarray,  # pytype: disable=annotation-type-mismatch
+    predictions: np.ndarray,
+    sparse_labels: bool = None,
+) -> bool:
   """Checks if labels are sparse or not.
 
   Args:

tensorflow_model_analysis/notebook/jupyter/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tensorflow_model_analysis",
-  "version": "0.47.0.dev",
+  "version": "0.48.0.dev",
   "homepage": "https://github.com/tensorflow/model-analysis",
   "bugs": "https://github.com/tensorflow/model-analysis/issues",
   "license": "Apache-2.0",

tensorflow_model_analysis/proto/config.proto

@@ -70,13 +70,20 @@ message ModelSpec {
   map<string, string> label_keys = 6;  // oneof not allowed with maps
   // Optional prediction key (single_output model). The prediction key is used
   // to distinguish between different values when the output from the predict
-  // call is a dict instead of a single tensor as is the case with
-  // tf.Estimators. If not set and the prediction is a dict, the keys 'scores',
-  // 'logistic', 'predictions', or 'probabilities' are tried (in that order).
+  // call is a dict instead of a single tensor. For estimator models, this is
+  // always the case and the prediction key is automatically inferred -- the
+  // keys 'scores', 'logistic', 'predictions', and 'probabilities' are tried
+  // (in that order). For Keras models, outputs are typically not dicts but if
+  // they are, then prediction key is not inferred and so MUST be specified.
+  //
+  // Note: For multi-class predictions, a prediction key needs to be specified
+  // so that the metrics can be computed correctly per class.
+  // TODO(b/399156775): Remove once the bug is fixed.
+  //
   // The prediction key is also used in cases where the predictions are
-  // pre-calculated and stored along side the features. In this case the
-  // prediction key refers to a key in the features dictionary. Use one of
-  // prediction_key or prediction_keys.
+  // pre-calculated and stored along side the features (a model is not used).
+  // In this case the prediction key refering to a key in the features
+  // dictionary must be provided. Use one of prediction_key or prediction_keys.
   //
   // Note that prediction_key is NOT the same as the output_name used in the
   // MetricsSpec. The output_name refers to the name of an output for a
@@ -105,11 +112,6 @@ message ModelSpec {
   // Here 'head1' or 'head2' would be the output_name, whereas 'probabilities'
   // would be the prediction_key.
   //
-  // If a model is not being used (i.e. the predictions are stored in the
-  // inputs), then a prediction_key is required. However, if a model is being
-  // used to compute the predictions a prediction_key typically is not needed
-  // since all the default tf.Estimator values for the keys are handled by TFMA
-  // and keras does not return multiple tensors for a given output.
   string prediction_key = 7;
   // Optional prediction keys (multi-output model) keyed by output_name. Use one
   // of prediction_key or prediction_keys. See comment under prediction_key on

tensorflow_model_analysis/utils/example_keras_model.py

@@ -60,6 +60,13 @@ def call(self, inputs):
     return tf.reshape(inputs, (1, 32))
 
 
+class Caster(keras.layers.Layer):
+  """A Keras layer that reshapes the input."""
+
+  def call(self, inputs):
+    return tf.cast(inputs, tf.float32)
+
+
 def get_example_classifier_model(input_feature_key: str = LANGUAGE):
   """Returns a Keras model for testing."""
   parser = ExampleCLassifierParser(input_feature_key)
@@ -85,6 +92,8 @@ def get_example_classifier_model(input_feature_key: str = LANGUAGE):
   parsed_example = parser(inputs)
   text_vector = text_vectorization(parsed_example)
   text_vector = Reshaper()(text_vector)
+  # Cast to float32 so that data type is consistent for the dense layers.
+  text_vector = Caster()(text_vector)
   output1 = dense1(text_vector)
   output2 = dense2(output1)
   return tf.keras.Model(inputs=inputs, outputs=output2)

tensorflow_model_analysis/utils/model_util.py

@@ -278,9 +278,8 @@ def get_feature_values_for_model_spec_field(
         batched_extracts[constants.TRANSFORMED_FEATURES_KEY]):
       transformed_features = batched_extracts[
           constants.TRANSFORMED_FEATURES_KEY]
-      if len(model_specs) > 1 and transformed_features:
-        if spec.name in transformed_features:
-          transformed_features = transformed_features[spec.name]
+      if transformed_features and spec.name in transformed_features:
+        transformed_features = transformed_features[spec.name]
       transformed_features = transformed_features or {}
     else:
       transformed_features = {}

tensorflow_model_analysis/version.py

@@ -15,4 +15,4 @@
 
 # Version string for this release of TFMA.
 # Note that setup.py reads and uses this version.
-VERSION = '0.47.0.dev'
+VERSION = '0.48.0.dev'