Skip to content

Commit cc2d36e

Browse files
jsondaicopybara-github
jsondai
authored and
copybara-github
committed
feat: Eval SDK: Modify Gemini scraping to log errors during response generation and return a consistent error message
PiperOrigin-RevId: 732308644
1 parent acf8a80 commit cc2d36e

File tree

4 files changed

+112
-72
lines changed

4 files changed

+112
-72
lines changed

‎vertexai/evaluation/_evaluation.py

Lines changed: 55 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -303,52 +303,71 @@ def _aggregate_summary_metrics(
303303

304304

305305
def _generate_content_text_response(
306-
model: generative_models.GenerativeModel, prompt: str
306+
model: generative_models.GenerativeModel, prompt: str, max_retries: int = 3
307307
) -> str:
308-
"""Generates a text response from Gemini model from a text prompt.
308+
"""Generates a text response from Gemini model from a text prompt with retries.
309309
310310
Args:
311311
model: The Gemini model instance.
312312
prompt: The prompt to send to the model.
313+
max_retries: Maximum number of retries for response generation.
313314
314315
Returns:
315316
The text response from the model.
316-
317-
Raises:
318-
RuntimeError if the prompt or the response for the prompt is blocked for
319-
safety reasons.
317+
Returns constants.RESPONSE_ERROR if there is an error after all retries.
320318
"""
321-
response = model.generate_content(prompt)
322-
try:
323-
if not response.candidates:
324-
raise RuntimeError(
325-
f"The model response was blocked due to"
326-
f" {response._raw_response.prompt_feedback.block_reason.name}.\n"
327-
f"Blocked reason message:"
328-
f" {response._raw_response.prompt_feedback.block_reason_message}.\n"
329-
"The input prompt may be blocked for safety reasons.",
330-
f"Prompt: {prompt}.",
331-
)
332-
else:
333-
candidate = response.candidates[0]
334-
if candidate.finish_reason not in _SUCCESSFUL_FINISH_REASONS:
335-
raise RuntimeError(
336-
"The model response did not finish"
337-
" successfully.\n"
338-
f"Finish reason: {candidate.finish_reason}.\n"
339-
f"Finish message: {candidate.finish_message}.\n"
340-
f"Safety ratings: {candidate.safety_ratings}.\n"
341-
"Please adjust the model safety_settings, or"
342-
" try a different prompt."
319+
for retry_attempt in range(max_retries):
320+
try:
321+
response = model.generate_content(prompt)
322+
if not response.candidates:
323+
error_message = (
324+
f"The model response was blocked due to"
325+
f" {response._raw_response.prompt_feedback.block_reason.name}.\n"
326+
f"Blocked reason message:"
327+
f" {response._raw_response.prompt_feedback.block_reason_message}.\n"
328+
"The input prompt may be blocked for safety reasons.\n"
329+
f"Prompt: {prompt}.\n"
330+
f"Retry attempt: {retry_attempt + 1}/{max_retries}"
343331
)
344-
return response.candidates[0].content.parts[0].text
345-
except Exception:
346-
raise RuntimeError(
347-
f"Failed to generate response candidates from Gemini model"
348-
f" {model._model_name}.\n"
349-
f"Response: {response}.\n"
350-
f"Prompt: {prompt}."
351-
)
332+
_LOGGER.warning(error_message)
333+
break
334+
else:
335+
candidate = response.candidates[0]
336+
if candidate.finish_reason not in _SUCCESSFUL_FINISH_REASONS:
337+
error_message = (
338+
"The model response did not finish"
339+
" successfully.\n"
340+
f"Finish reason: {candidate.finish_reason}.\n"
341+
f"Finish message: {candidate.finish_message}.\n"
342+
f"Safety ratings: {candidate.safety_ratings}.\n"
343+
"Please adjust the model safety_settings, or"
344+
" try a different prompt.\n"
345+
f"Retry attempt: {retry_attempt + 1}/{max_retries}"
346+
)
347+
_LOGGER.warning(error_message)
348+
else:
349+
return response.candidates[0].content.parts[0].text
350+
except Exception as e:
351+
error_message = (
352+
f"Failed to generate response candidates from Gemini model"
353+
f" {model._model_name}.\n"
354+
f"Error: {e}.\n"
355+
f"Prompt: {prompt}.\n"
356+
f"Retry attempt: {retry_attempt + 1}/{max_retries}"
357+
)
358+
_LOGGER.warning(error_message)
359+
if retry_attempt < max_retries - 1:
360+
_LOGGER.info(
361+
f"Retrying response generation for prompt: {prompt}, attempt"
362+
f" {retry_attempt + 1}/{max_retries}..."
363+
)
364+
365+
final_error_message = (
366+
f"Failed to generate response from Gemini model {model._model_name}.\n"
367+
f"Prompt: {prompt}."
368+
)
369+
_LOGGER.warning(final_error_message)
370+
return constants.RESPONSE_ERROR
352371

353372

354373
def _generate_responses_from_gemini_model(

‎vertexai/evaluation/constants.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
# The number of concurrent workers to use for making model inference and
2121
# evaluation requests.
2222
MAX_WORKERS = 100
23+
RESPONSE_ERROR = "Error"
2324

2425

2526
@dataclasses.dataclass(frozen=True)

‎vertexai/preview/evaluation/_evaluation.py

Lines changed: 55 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -340,52 +340,71 @@ def _aggregate_summary_metrics(
340340

341341

342342
def _generate_content_text_response(
343-
model: generative_models.GenerativeModel, prompt: str
343+
model: generative_models.GenerativeModel, prompt: str, max_retries: int = 3
344344
) -> str:
345-
"""Generates a text response from Gemini model from a text prompt.
345+
"""Generates a text response from Gemini model from a text prompt with retries.
346346
347347
Args:
348348
model: The Gemini model instance.
349349
prompt: The prompt to send to the model.
350+
max_retries: Maximum number of retries for response generation.
350351
351352
Returns:
352353
The text response from the model.
353-
354-
Raises:
355-
RuntimeError if the prompt or the response for the prompt is blocked for
356-
safety reasons.
354+
Returns constants.RESPONSE_ERROR if there is an error after all retries.
357355
"""
358-
response = model.generate_content(prompt)
359-
try:
360-
if not response.candidates:
361-
raise RuntimeError(
362-
f"The model response was blocked due to"
363-
f" {response._raw_response.prompt_feedback.block_reason.name}.\n." # pylint: disable=protected-access
364-
f"Blocked reason message:"
365-
f" {response._raw_response.prompt_feedback.block_reason_message}.\n." # pylint: disable=protected-access
366-
"The input prompt may be blocked for safety reasons.",
367-
f"Prompt: {prompt}.",
368-
)
369-
else:
370-
candidate = response.candidates[0]
371-
if candidate.finish_reason not in _SUCCESSFUL_FINISH_REASONS:
372-
raise RuntimeError(
373-
"The model response did not finish"
374-
" successfully.\n"
375-
f"Finish reason: {candidate.finish_reason}.\n"
376-
f"Finish message: {candidate.finish_message}.\n"
377-
f"Safety ratings: {candidate.safety_ratings}.\n"
378-
"Please adjust the model safety_settings, or"
379-
" try a different prompt."
356+
for retry_attempt in range(max_retries):
357+
try:
358+
response = model.generate_content(prompt)
359+
if not response.candidates:
360+
error_message = (
361+
f"The model response was blocked due to"
362+
f" {response._raw_response.prompt_feedback.block_reason.name}.\n"
363+
f"Blocked reason message:"
364+
f" {response._raw_response.prompt_feedback.block_reason_message}.\n"
365+
"The input prompt may be blocked for safety reasons.\n"
366+
f"Prompt: {prompt}.\n"
367+
f"Retry attempt: {retry_attempt + 1}/{max_retries}"
380368
)
381-
return response.candidates[0].content.parts[0].text
382-
except Exception:
383-
raise RuntimeError(
384-
f"Failed to generate response candidates from Gemini model"
385-
f" {model._model_name}.\n" # pylint: disable=protected-access
386-
f"Response: {response}.\n"
387-
f"Prompt: {prompt}."
388-
)
369+
_LOGGER.warning(error_message)
370+
break
371+
else:
372+
candidate = response.candidates[0]
373+
if candidate.finish_reason not in _SUCCESSFUL_FINISH_REASONS:
374+
error_message = (
375+
"The model response did not finish"
376+
" successfully.\n"
377+
f"Finish reason: {candidate.finish_reason}.\n"
378+
f"Finish message: {candidate.finish_message}.\n"
379+
f"Safety ratings: {candidate.safety_ratings}.\n"
380+
"Please adjust the model safety_settings, or"
381+
" try a different prompt.\n"
382+
f"Retry attempt: {retry_attempt + 1}/{max_retries}"
383+
)
384+
_LOGGER.warning(error_message)
385+
else:
386+
return response.candidates[0].content.parts[0].text
387+
except Exception as e:
388+
error_message = (
389+
f"Failed to generate response candidates from Gemini model"
390+
f" {model._model_name}.\n"
391+
f"Error: {e}.\n"
392+
f"Prompt: {prompt}.\n"
393+
f"Retry attempt: {retry_attempt + 1}/{max_retries}"
394+
)
395+
_LOGGER.warning(error_message)
396+
if retry_attempt < max_retries - 1:
397+
_LOGGER.info(
398+
f"Retrying response generation for prompt: {prompt}, attempt"
399+
f" {retry_attempt + 1}/{max_retries}..."
400+
)
401+
402+
final_error_message = (
403+
f"Failed to generate response from Gemini model {model._model_name}.\n"
404+
f"Prompt: {prompt}."
405+
)
406+
_LOGGER.warning(final_error_message)
407+
return constants.RESPONSE_ERROR
389408

390409

391410
def _generate_responses_from_gemini_model(

‎vertexai/preview/evaluation/constants.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
# The number of concurrent workers to use for making model inference and
2121
# evaluation requests.
2222
MAX_WORKERS = 100
23+
RESPONSE_ERROR = "Error"
2324

2425

2526
@dataclasses.dataclass(frozen=True)

0 commit comments

Comments
 (0)