Merge pull request #512 from guardrails-ai/exception-handling

Exception handling

Author: zsimjee

docs/examples/input_validation.ipynb

@@ -18,7 +18,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {
     "is_executing": true
    },
@@ -47,23 +47,34 @@
    "source": [
     "When `fix` is specified as the on-fail handler, the prompt will automatically be amended before calling the LLM.\n",
     "\n",
-    "In any other case (for example, `exception`), a `ValidationException` will be returned in the outcome."
+    "In any other case (for example, `exception`), a `ValidatorError` will be returned in the outcome."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {
     "is_executing": true
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Validation failed for field with errors: must be exactly two words\n"
+     ]
+    }
+   ],
    "source": [
     "import openai\n",
+    "from guardrails.errors import ValidatorError\n",
     "\n",
-    "outcome = guard(\n",
-    "    openai.ChatCompletion.create,\n",
-    ")\n",
-    "outcome.error"
+    "try:\n",
+    "    guard(\n",
+    "        openai.ChatCompletion.create,\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)"
    ]
   },
   {
@@ -75,9 +86,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Validation failed for field with errors: must be exactly two words\n"
+     ]
+    }
+   ],
    "source": [
     "from guardrails.validators import TwoWords\n",
     "from pydantic import BaseModel\n",
@@ -91,11 +110,13 @@
     "guard = Guard.from_pydantic(Pet)\n",
     "guard.with_prompt_validation([TwoWords(on_fail=\"exception\")])\n",
     "\n",
-    "outcome = guard(\n",
-    "    openai.ChatCompletion.create,\n",
-    "    prompt=\"This is not two words\",\n",
-    ")\n",
-    "outcome.error"
+    "try:\n",
+    "    guard(\n",
+    "        openai.ChatCompletion.create,\n",
+    "        prompt=\"This is not two words\",\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)"
    ]
   }
  ],
@@ -115,7 +136,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.0"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

docs/examples/response_is_on_topic.ipynb

@@ -37,7 +37,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -54,7 +54,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -76,7 +76,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -115,7 +115,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -129,6 +129,7 @@
    "source": [
     "import guardrails as gd\n",
     "from guardrails.validators import OnTopic\n",
+    "from guardrails.errors import ValidatorError\n",
     "\n",
     "# Create the Guard with the OnTopic Validator\n",
     "guard = gd.Guard.from_string(\n",
@@ -146,11 +147,12 @@
     ")\n",
     "\n",
     "# Test with a given text\n",
-    "output = guard.parse(\n",
-    "    llm_output=text,\n",
-    ")\n",
-    "\n",
-    "print(output.error)"
+    "try:\n",
+    "    guard.parse(\n",
+    "        llm_output=text,\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)\n"
    ]
   },
   {
@@ -164,7 +166,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -191,9 +193,12 @@
     ")\n",
     "\n",
     "# Test with a given text\n",
-    "output = guard.parse(llm_output=text)\n",
-    "\n",
-    "print(output.error)"
+    "try:\n",
+    "    guard.parse(\n",
+    "        llm_output=text,\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)"
    ]
   },
   {
@@ -207,7 +212,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -234,11 +239,12 @@
     ")\n",
     "\n",
     "# Test with a given text\n",
-    "output = guard.parse(\n",
-    "    llm_output=text\n",
-    ")\n",
-    "\n",
-    "print(output.error)"
+    "try:\n",
+    "    guard.parse(\n",
+    "        llm_output=text,\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)"
    ]
   }
  ],
@@ -258,7 +264,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

guardrails/classes/history/call.py

@@ -1,6 +1,6 @@
 from typing import Dict, Optional, Union
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 from rich.panel import Panel
 from rich.pretty import pretty_repr
 from rich.tree import Tree
@@ -30,19 +30,26 @@ class Call(ArbitraryModel):
     inputs: CallInputs = Field(
         description="The inputs as passed in to Guard.__call__ or Guard.parse"
     )
+    _exception: Optional[Exception] = PrivateAttr()
 
     # Prevent Pydantic from changing our types
     # Without this, Pydantic casts iterations to a list
     def __init__(
         self,
         iterations: Optional[Stack[Iteration]] = None,
         inputs: Optional[CallInputs] = None,
+        exception: Optional[Exception] = None,
     ):
         iterations = iterations or Stack()
         inputs = inputs or CallInputs()
-        super().__init__(iterations=iterations, inputs=inputs)  # type: ignore
+        super().__init__(
+            iterations=iterations,  # type: ignore
+            inputs=inputs,  # type: ignore
+            _exception=exception,  # type: ignore
+        )
         self.iterations = iterations
         self.inputs = inputs
+        self._exception = exception
 
     @property
     def prompt(self) -> Optional[str]:
@@ -278,14 +285,18 @@ def validator_logs(self) -> Stack[ValidatorLogs]:
     def error(self) -> Optional[str]:
         """The error message from any exception that raised and interrupted the
         run."""
-        if self.iterations.empty():
+        if self._exception:
+            return str(self._exception)
+        elif self.iterations.empty():
             return None
         return self.iterations.last.error  # type: ignore
 
     @property
     def exception(self) -> Optional[Exception]:
         """The exception that interrupted the run."""
-        if self.iterations.empty():
+        if self._exception:
+            return self._exception
+        elif self.iterations.empty():
             return None
         return self.iterations.last.exception  # type: ignore
 

guardrails/classes/validation_outcome.py

@@ -2,7 +2,7 @@
 
 from pydantic import Field
 
-from guardrails.classes.history import Call
+from guardrails.classes.history import Call, Iteration
 from guardrails.classes.output_type import OT
 from guardrails.constants import pass_status
 from guardrails.utils.logs_utils import ArbitraryModel
@@ -32,15 +32,12 @@ class ValidationOutcome(Generic[OT], ArbitraryModel):
     error: Optional[str] = Field(default=None)
 
     @classmethod
-    def from_guard_history(cls, call: Call, error_message: Optional[str]):
-        last_output = (
-            call.iterations.last.validation_output
-            if not call.iterations.empty() and call.iterations.last is not None
-            else None
-        )
+    def from_guard_history(cls, call: Call):
+        last_iteration = call.iterations.last or Iteration()
+        last_output = last_iteration.validation_output or last_iteration.parsed_output
         validation_passed = call.status == pass_status
         reask = last_output if isinstance(last_output, ReAsk) else None
-        error = call.error or error_message
+        error = call.error
         output = cast(OT, call.validated_output)
         return cls(
             raw_llm_output=call.raw_outputs.last,

guardrails/datatypes.py

@@ -48,7 +48,9 @@ def verify_metadata_requirements(
             metadata, vars(datatype.children).values()
         )
         missing_keys.update(nested_missing_keys)
-    return list(missing_keys)
+    missing_keys = list(missing_keys)
+    missing_keys.sort()
+    return missing_keys
 
 
 class DataType:

guardrails/errors/__init__.py

@@ -0,0 +1,3 @@
+from guardrails.validator_base import ValidatorError
+
+__all__ = ["ValidatorError"]

guardrails/guard.py

@@ -422,10 +422,8 @@ def _call_sync(
                     base_model=self.base_model,
                     full_schema_reask=full_schema_reask,
                 )
-                call, error_message = runner(
-                    call_log=call_log, prompt_params=prompt_params
-                )
-                return ValidationOutcome[OT].from_guard_history(call, error_message)
+                call = runner(call_log=call_log, prompt_params=prompt_params)
+                return ValidationOutcome[OT].from_guard_history(call)
 
     async def _call_async(
         self,
@@ -483,10 +481,10 @@ async def _call_async(
                 base_model=self.base_model,
                 full_schema_reask=full_schema_reask,
             )
-            call, error_message = await runner.async_run(
+            call = await runner.async_run(
                 call_log=call_log, prompt_params=prompt_params
             )
-            return ValidationOutcome[OT].from_guard_history(call, error_message)
+            return ValidationOutcome[OT].from_guard_history(call)
 
     def __repr__(self):
         return f"Guard(RAIL={self.rail})"
@@ -662,9 +660,9 @@ def _sync_parse(
                 base_model=self.base_model,
                 full_schema_reask=full_schema_reask,
             )
-            call, error_message = runner(call_log=call_log, prompt_params=prompt_params)
+            call = runner(call_log=call_log, prompt_params=prompt_params)
 
-            return ValidationOutcome[OT].from_guard_history(call, error_message)
+            return ValidationOutcome[OT].from_guard_history(call)
 
     async def _async_parse(
         self,
@@ -704,11 +702,11 @@ async def _async_parse(
                 base_model=self.base_model,
                 full_schema_reask=full_schema_reask,
             )
-            call, error_message = await runner.async_run(
+            call = await runner.async_run(
                 call_log=call_log, prompt_params=prompt_params
             )
 
-            return ValidationOutcome[OT].from_guard_history(call, error_message)
+            return ValidationOutcome[OT].from_guard_history(call)
 
     def with_prompt_validation(
         self,