feat: add metric classes for 2 pairwise metrics for rapid evaluation SDK.

PiperOrigin-RevId: 644127034

Author: jsondai

tests/unit/vertexai/test_evaluation.py

@@ -332,7 +332,7 @@ def test_compute_pairwise_metrics_with_model_inference(self, api_transport):
         mock_candidate_model._model_name = "publishers/google/model/gemini-pro"
         test_metrics = [
             evaluation.PairwiseMetric(
-                metric="summarization_quality",
+                metric="pairwise_summarization_quality",
                 baseline_model=mock_baseline_model,
                 use_reference=False,
             )
@@ -609,11 +609,11 @@ def test_evaluate_pairwise_metrics_with_multiple_baseline_models(self):
         mock_candidate_model._model_name = "publishers/google/model/gemini-1.0-ultra"
         test_metrics = [
             evaluation.PairwiseMetric(
-                metric="summarization_quality",
+                metric="pairwise_summarization_quality",
                 baseline_model=mock_baseline_model_1,
             ),
             evaluation.PairwiseMetric(
-                metric="summarization_quality",
+                metric="pairwise_summarization_quality",
                 baseline_model=mock_baseline_model_2,
             ),
         ]

vertexai/preview/evaluation/metrics/_base.py

@@ -17,6 +17,7 @@
 
 from typing import Any, Callable, Dict, Literal, Optional, Union
 from vertexai import generative_models
+from vertexai.preview.evaluation import constants
 
 
 class PairwiseMetric:
@@ -85,9 +86,12 @@ class PairwiseMetric:
     def __init__(
         self,
         *,
-        metric: Literal["summarization_quality", "question_answering_quality"],
-        baseline_model: Union[
-            generative_models.GenerativeModel, Callable[[str], str]
+        metric: Literal[
+            constants.Metric.PAIRWISE_SUMMARIZATION_QUALITY,
+            constants.Metric.PAIRWISE_QUESTION_ANSWERING_QUALITY,
+        ],
+        baseline_model: Optional[
+            Union[generative_models.GenerativeModel, Callable[[str], str]]
         ] = None,
         use_reference: bool = False,
         version: Optional[int] = None,
@@ -97,10 +101,14 @@ def __init__(
         Args:
           metric: The Side-by-side(SxS) pairwise evaluation metric name.
           baseline_model: The baseline model for the Side-by-side(SxS) comparison.
+            If not specified, `baseline_model_response` column is required in the dataset.
           use_reference: Whether to use reference to compute the metric. If
             specified, the reference column is required in the dataset.
           version: The metric version to use for evaluation.
         """
+        # TODO(b/311221071): Remove the legacy metric names for GA.
+        if metric in ("summarization_quality", "question_answering_quality"):
+            metric = f"pairwise_{metric}"
         self._metric = metric
         self._baseline_model = baseline_model
         self._use_reference = use_reference
@@ -111,7 +119,7 @@ def __str__(self):
 
     @property
     def pairwise_metric_name(self) -> str:
-        return f"pairwise_{self._metric}"
+        return self._metric
 
     @property
     def baseline_model(

vertexai/preview/evaluation/metrics/_pairwise_question_answering_quality.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import Callable, Optional, Union
+from vertexai.generative_models import _generative_models
+from vertexai.preview.evaluation import constants
+from vertexai.preview.evaluation.metrics import _base
+
+
+class PairwiseQuestionAnsweringQuality(_base.PairwiseMetric):
+    """The Side-by-side(SxS) Pairwise Metric for Question Answering Quality."""
+
+    _metric_name = constants.Metric.PAIRWISE_QUESTION_ANSWERING_QUALITY
+
+    def __init__(
+        self,
+        *,
+        baseline_model: Optional[
+            Union[_generative_models.GenerativeModel, Callable[[str], str]]
+        ] = None,
+        use_reference: bool = False,
+        version: Optional[int] = None
+    ):
+        super().__init__(
+            metric=PairwiseQuestionAnsweringQuality._metric_name,
+            baseline_model=baseline_model,
+            use_reference=use_reference,
+            version=version,
+        )

vertexai/preview/evaluation/metrics/_pairwise_summarization_quality.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import Callable, Optional, Union
+from vertexai.generative_models import _generative_models
+from vertexai.preview.evaluation import constants
+from vertexai.preview.evaluation.metrics import _base
+
+
+class PairwiseSummarizationQuality(_base.PairwiseMetric):
+    """The Side-by-side(SxS) Pairwise Metric for summarization quality."""
+
+    _metric_name = constants.Metric.PAIRWISE_SUMMARIZATION_QUALITY
+
+    def __init__(
+        self,
+        *,
+        baseline_model: Optional[
+            Union[_generative_models.GenerativeModel, Callable[[str], str]]
+        ] = None,
+        use_reference: bool = False,
+        version: Optional[int] = None
+    ):
+        super().__init__(
+            metric=PairwiseSummarizationQuality._metric_name,
+            baseline_model=baseline_model,
+            use_reference=use_reference,
+            version=version,
+        )