Support empty response for Completions and ChatCompletions API

lm_eval/models/openai_completions.py

@@ -8,7 +8,6 @@
 from lm_eval.models.api_models import TemplateAPI
 from lm_eval.models.utils import handle_stop_sequences
 
-
 eval_logger = logging.getLogger(__name__)
 
 
@@ -95,7 +94,10 @@ def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]:
         for out in outputs:
             tmp = [None] * len(out["choices"])
             for choices in out["choices"]:
-                tmp[choices["index"]] = choices["text"]
+                x = ""
+                if choices["text"] is not None:
+                    x = choices["text"]
+                tmp[choices["index"]] = x
             res = res + tmp
         return res
 
@@ -167,7 +169,10 @@ def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]:
         for out in outputs:
             tmp = [None] * len(out["choices"])
             for choices in out["choices"]:
-                tmp[choices["index"]] = choices["message"]["content"]
+                x = ""
+                if choices["message"]["content"] is not None:
+                    x = choices["message"]["content"]
+                tmp[choices["index"]] = x
             res = res + tmp
         return res
 

tests/models/test_api.py

@@ -3,6 +3,7 @@
 
 import pytest
 
+from lm_eval.api.instance import Instance
 from lm_eval.models.openai_completions import LocalCompletionsAPI
 
 
@@ -161,6 +162,29 @@ def test_model_tokenized_call_usage(
         assert result == {"result": "success"}
 
 
+def test_generate_until_with_null_message_content(api):
+    with patch("requests.post") as mock_post:
+        mock_response = MagicMock()
+        mock_response.json.return_value = {
+            "choices": [
+                {
+                    "index": 0,
+                    "text": None,
+                }
+            ]
+        }
+        mock_response.ok = True
+        mock_post.return_value = mock_response
+        request = Instance(
+            request_type="generate_until",
+            doc={},
+            arguments=("Test prompt", {"max_gen_toks": 10}),
+            idx=0,
+        )
+
+        _ = api.generate_until([request])
+
+
 class DummyAsyncContextManager:
     def __init__(self, result):
         self.result = result