rouge metric integration

Author: pmittaldev

examples/how_to_questions/metrics/data_integrity/dataset_rouge_summary_metric.ipynb

@@ -0,0 +1,101 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Evidently Dataset ROUGE Summary Metric"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from evidently.report import Report\n",
+    "from evidently.metrics import ROUGESummaryMetric"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "current_data = {\n",
+    "    \"summary\": [\"hello there\", \"general kenobi\"],\n",
+    "}\n",
+    "\n",
+    "current_df = pd.DataFrame(current_data)\n",
+    "\n",
+    "reference_data = {\n",
+    "    \"summary\": [\"hello there\", \"no de\"]\n",
+    "}\n",
+    "\n",
+    "current_df = pd.DataFrame(current_data)\n",
+    "reference_df = pd.DataFrame(reference_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "report = Report(metrics=[\n",
+    "    ROUGESummaryMetric(column_name=\"summary\", rouge_n=1)\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "report.run(current_data=current_df, reference_data=reference_df)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "report.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "report.as_dict()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "evidently",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

examples/how_to_questions/metrics/data_integrity/dataset_summary_metric.ipynb

@@ -116,7 +116,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.13"
+   "version": "3.8.19"
   }
  },
  "nbformat": 4,

requirements.min.txt

@@ -31,4 +31,5 @@ openai==1.16.2
 evaluate==0.4.1
 transformers[torch]==4.39.3
 sentence-transformers==2.7.0
+rouge-score==0.1.2
 chromadb==0.4.0

setup.py

@@ -92,7 +92,7 @@
             "types-python-dateutil==2.8.19",
             "types-ujson>=5.4.0",
             "pillow==10.3.0",
-            "httpx==0.27.0",
+            "httpx==0.24.1",
             "ruff==0.3.7",
             "pre-commit==3.5.0",
             "pytest-asyncio==0.23.7",
@@ -102,6 +102,7 @@
             "evaluate>=0.4.1",
             "transformers[torch]>=4.39.3",
             "sentence-transformers>=2.7.0",
+            "rouge-score>=0.1.2",
             "chromadb>=0.4.0",
         ],
         "spark": ["pyspark>=3.4.0"],

src/evidently/metrics/__init__.py

@@ -32,6 +32,7 @@
 from .data_integrity.column_summary_metric import ColumnSummaryMetric
 from .data_integrity.dataset_missing_values_metric import DatasetMissingValuesMetric
 from .data_integrity.dataset_summary_metric import DatasetSummaryMetric
+from .data_integrity.rouge_summary_metric import ROUGESummaryMetric
 from .data_quality.column_category_metric import ColumnCategoryMetric
 from .data_quality.column_correlations_metric import ColumnCorrelationsMetric
 from .data_quality.column_distribution_metric import ColumnDistributionMetric
@@ -99,6 +100,7 @@
     "ColumnSummaryMetric",
     "DatasetMissingValuesMetric",
     "DatasetSummaryMetric",
+    "ROUGESummaryMetric",
     "ColumnCategoryMetric",
     "ColumnCorrelationsMetric",
     "ColumnDistributionMetric",

src/evidently/metrics/data_integrity/rouge_summary_metric.py

@@ -0,0 +1,85 @@
+from typing import List
+from typing import Union
+
+import evaluate
+import pandas as pd
+
+from evidently.base_metric import ColumnName
+from evidently.base_metric import InputData
+from evidently.base_metric import Metric
+from evidently.base_metric import MetricResult
+from evidently.core import IncludeTags
+from evidently.model.widget import BaseWidgetInfo
+from evidently.renderers.base_renderer import MetricRenderer
+from evidently.renderers.base_renderer import default_renderer
+from evidently.renderers.html_widgets import header_text
+from evidently.renderers.html_widgets import table_data
+
+
+class ROUGESummaryMetricResult(MetricResult):
+    class Config:
+        type_alias = "evidently:metric_result:ROUGESummaryMetricResult"
+        field_tags = {
+            "rouge_type": {IncludeTags.Parameter},
+            "value": {IncludeTags.Parameter},
+        }
+
+    rouge_type: str
+    score: dict
+
+
+class ROUGESummaryMetric(Metric[ROUGESummaryMetricResult]):
+    class Config:
+        type_alias = "evidently:metric:ROUGESummaryMetric"
+        arbitrary_types_allowed = True
+
+    column_name: str
+    rouge_n: int
+
+    def __init__(self, column_name: Union[str, ColumnName], rouge_n: int):
+        self.column_name = column_name
+        self.rouge_n = rouge_n
+        super().__init__()
+
+    def _calculate_summary_rouge(self, current_data: pd.Series, reference_data: pd.Series):
+        rouge_evaluator = evaluate.load("rouge")
+
+        predictions = current_data.astype(str).tolist()
+        references = reference_data.astype(str).tolist()
+
+        rouge_scores = rouge_evaluator.compute(
+            rouge_types=[f"rouge{self.rouge_n}"], predictions=predictions, references=references, use_aggregator=False
+        )
+
+        per_row_rouge_scores = rouge_scores[f"rouge{self.rouge_n}"]
+
+        summary_rouge_score = sum(per_row_rouge_scores) / len(per_row_rouge_scores)
+
+        return per_row_rouge_scores, summary_rouge_score
+
+    def calculate(self, data: InputData) -> MetricResult:
+        if len(data.current_data[self.column_name]) == 0 or len(data.reference_data[self.column_name]) == 0:
+            raise ValueError("The current data or the reference data is empty.")
+
+        per_row_rouge_scores, summary_rouge_score = self._calculate_summary_rouge(
+            data.current_data[self.column_name], data.reference_data[self.column_name]
+        )
+
+        result = ROUGESummaryMetricResult(
+            rouge_type=f"ROUGE-{self.rouge_n}",
+            score={"per_row_scores": per_row_rouge_scores, "summary_score": summary_rouge_score},
+        )
+        return result
+
+
+@default_renderer(wrap_type=ROUGESummaryMetric)
+class ROUGESummaryMetricRenderer(MetricRenderer):
+    @staticmethod
+    def _get_table(metric, n: int = 2) -> BaseWidgetInfo:
+        column_names = ["Metric", "Value"]
+        rows = ([metric.rouge_type, metric.score],)
+        return table_data(title="", column_names=column_names, data=rows)
+
+    def render_html(self, obj: ROUGESummaryMetricResult) -> List[BaseWidgetInfo]:
+        metric = obj.get_result()
+        return [header_text(label="ROUGE Metric"), self._get_table(metric)]

tests/metrics/data_interity/test_dataset_rouge_summary_metric.py

@@ -0,0 +1,69 @@
+import json
+
+import pandas as pd
+import pytest
+
+from evidently.metrics.data_integrity.rouge_summary_metric import ROUGESummaryMetric
+from evidently.metrics.data_integrity.rouge_summary_metric import ROUGESummaryMetricResult
+from evidently.report.report import Report
+from tests.conftest import smart_assert_equal
+
+
+@pytest.mark.parametrize(
+    "current_df, reference_df, metric, expected_result",
+    (
+        (
+            pd.DataFrame(
+                {
+                    "summary": ["hello there", "general kenobi"],
+                }
+            ),
+            pd.DataFrame({"summary": ["hello there", "no de"]}),
+            ROUGESummaryMetric(column_name="summary", rouge_n=1),
+            ROUGESummaryMetricResult(rouge_type="ROUGE-1", score={"per_row_scores": [1.0, 0.0], "summary_score": 0.5}),
+        ),
+    ),
+)
+def test_rouge_summary_metric_success(
+    current_df: pd.DataFrame,
+    reference_df: pd.DataFrame,
+    metric,
+    expected_result: ROUGESummaryMetricResult,
+) -> None:
+    report = Report(metrics=[metric])
+
+    report.run(current_data=current_df, reference_data=reference_df)
+
+    smart_assert_equal(metric.get_result(), expected_result)
+
+
+@pytest.mark.parametrize(
+    "current_df, reference_df, metric, expected_json",
+    (
+        (
+            pd.DataFrame(
+                {
+                    "summary": ["hello there", "general kenobi"],
+                }
+            ),
+            pd.DataFrame({"summary": ["hello there", "no de"]}),
+            ROUGESummaryMetric(column_name="summary", rouge_n=1),
+            {"rouge_type": "ROUGE-1", "score": {"per_row_scores": [1.0, 0.0], "summary_score": 0.5}},
+        ),
+    ),
+)
+def test_rouge_summary_metric_with_report(
+    current_df: pd.DataFrame,
+    reference_df: pd.DataFrame,
+    metric,
+    expected_json: dict,
+) -> None:
+    report = Report(metrics=[metric])
+
+    report.run(current_data=current_df, reference_data=reference_df)
+
+    assert report.show()
+    json_result = report.json()
+    assert len(json_result) > 0
+    result = json.loads(json_result)
+    assert result["metrics"][0]["result"] == expected_json