From 06413e094af39cba0c988a2b4708ff24e9ee99a6 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 7 Nov 2025 09:49:06 -0500 Subject: [PATCH] [Bugfix] Parse gpt-oss refusals w/ newer openai-harmony The output generated by gpt-oss models does not always strictly follow its expected harmony chat template format. This commonly - but not exclusively - happens when gpt-oss-120b generates refusals for content that violates its built-in safety guidelines. To fix this, a non-strict mode was added to the openai-harmony library to allow attempted recovery of malformed message headers in the model output, such as a missing `<|message|>` special token before the assistant text. This will resolve some cases where the error `openai_harmony.HarmonyError: unexpected tokens remaining in message header` was previously thrown. It will not resolve all of those, as not every malformed message output can be recovered. Other ongoing work around using structured output for the Harmony format can help prevent these kinds of things in the first place, once that work lands and in the cases where the user and/or server decide to enable it. I believe it should be safe to enable this non-strict mode by default in vLLM, as the code paths that enables in the openai-harmony library only gets triggered once it's already detected malformed output. So, there shouldn't be any performance penalty in the common case. And, in the event that the malformed content cannot be properly recovered, the openai-harmony library will still end up throwing an error. This is related to vllm-project/vllm#23567 as well as openai/harmony#80. Signed-off-by: Ben Browning --- requirements/common.txt | 2 +- requirements/test.txt | 2 +- tests/entrypoints/test_harmony_utils.py | 25 +++++++++++++++++++++++++ vllm/entrypoints/harmony_utils.py | 2 +- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/requirements/common.txt b/requirements/common.txt index ce5607b7fbf2..a03d4a6411d9 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -47,5 +47,5 @@ ninja # Required for xgrammar, rocm, tpu, xpu pybase64 # fast base64 implementation cbor2 # Required for cross-language serialization of hashable objects setproctitle # Used to set process names for better debugging and monitoring -openai-harmony >= 0.0.3 # Required for gpt-oss +openai-harmony >= 0.0.8 # Required for gpt-oss anthropic == 0.71.0 diff --git a/requirements/test.txt b/requirements/test.txt index 9d13fa424115..85ae104c89ba 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -620,7 +620,7 @@ omegaconf==2.3.0 # lightning open-clip-torch==2.32.0 # via -r requirements/test.in -openai-harmony==0.0.4 +openai-harmony==0.0.8 # via gpt-oss opencensus==0.11.4 # via ray diff --git a/tests/entrypoints/test_harmony_utils.py b/tests/entrypoints/test_harmony_utils.py index 6fa051a678d6..60f9ae1d0627 100644 --- a/tests/entrypoints/test_harmony_utils.py +++ b/tests/entrypoints/test_harmony_utils.py @@ -4,6 +4,8 @@ from openai_harmony import Role from vllm.entrypoints.harmony_utils import ( + get_encoding, + get_streamable_parser_for_assistant, has_custom_tools, parse_input_to_harmony_message, ) @@ -264,3 +266,26 @@ def test_has_custom_tools() -> None: assert has_custom_tools( {"web_search_preview", "code_interpreter", "container", "others"} ) + + +def test_malformed_refusal_message() -> None: + """Test parsing a malformed refusal message sometimes generated by gpt-oss""" + output_text = ( + "...\n\nAccording to policy, we must refuse.<|end|>" + "<|start|>assistant<|channel|>analysis<|message|>We must refuse.<|end|>" + "<|start|>assistant<|channel|>final<|message|>I can't help with that.<|end|>" + ) + output_tokens = get_encoding().encode(output_text, allowed_special="all") + parser = get_streamable_parser_for_assistant() + for token in output_tokens: + parser.process(token) + assert len(parser.messages) == 3 + assert parser.messages[0].author.role == Role.ASSISTANT + # using "in" here instead of "==" to allow for whitespace variances + assert "According to policy, we must refuse." in parser.messages[0].content[0].text + assert parser.messages[1].author.role == Role.ASSISTANT + assert parser.messages[1].channel == "analysis" + assert parser.messages[1].content[0].text == "We must refuse." + assert parser.messages[2].author.role == Role.ASSISTANT + assert parser.messages[2].channel == "final" + assert parser.messages[2].content[0].text == "I can't help with that." diff --git a/vllm/entrypoints/harmony_utils.py b/vllm/entrypoints/harmony_utils.py index 7958d0317739..950e97012998 100644 --- a/vllm/entrypoints/harmony_utils.py +++ b/vllm/entrypoints/harmony_utils.py @@ -503,7 +503,7 @@ def get_stop_tokens_for_assistant_actions() -> list[int]: def get_streamable_parser_for_assistant() -> StreamableParser: - return StreamableParser(get_encoding(), role=Role.ASSISTANT) + return StreamableParser(get_encoding(), role=Role.ASSISTANT, strict=False) def parse_output_into_messages(token_ids: Iterable[int]) -> StreamableParser: