From 2b8df0b26bed3bb625f8336532f6ed8b88e36ea7 Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Fri, 7 Nov 2025 19:54:08 +0000 Subject: [PATCH 1/2] Add additional expected outputs due to flakiness Signed-off-by: Varun Sundar Rabindranath --- tests/lora/test_mixtral.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/lora/test_mixtral.py b/tests/lora/test_mixtral.py index 868ca51b3331..fae7deaa3423 100644 --- a/tests/lora/test_mixtral.py +++ b/tests/lora/test_mixtral.py @@ -56,15 +56,20 @@ def test_mixtral_lora(mixtral_lora_files, tp_size): ) expected_lora_output = [ - "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501 - "give_opinion(name[SpellForce 3], developer[Grimlore Games], release_year[2017], rating[poor])", # noqa: E501 - "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])", # noqa: E501 + [ + "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])" # noqa: E501 + ], + [ + "give_opinion(name[SpellForce 3], developer[Grimlore Games], release_year[2017], rating[poor])", # noqa: E501 + "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501 + ], + [ + "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])" # noqa: E501 + ], ] - assert ( - do_sample(llm, mixtral_lora_files, lora_id=1, prompts=prompts) - == expected_lora_output - ) - assert ( - do_sample(llm, mixtral_lora_files, lora_id=2, prompts=prompts) - == expected_lora_output - ) + + lora_output = do_sample(llm, mixtral_lora_files, lora_id=1, prompts=prompts) + assert all([lora_output[i] in expected_lora_output[i] for i in range(len(prompts))]) + + lora_output = do_sample(llm, mixtral_lora_files, lora_id=2, prompts=prompts) + assert all([lora_output[i] in expected_lora_output[i] for i in range(len(prompts))]) From ae3f13666233dfeda1eeb718bc75ca625cf95624 Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Fri, 7 Nov 2025 20:18:56 +0000 Subject: [PATCH 2/2] better check Signed-off-by: Varun Sundar Rabindranath --- tests/lora/test_mixtral.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/lora/test_mixtral.py b/tests/lora/test_mixtral.py index fae7deaa3423..12c73f2d79f7 100644 --- a/tests/lora/test_mixtral.py +++ b/tests/lora/test_mixtral.py @@ -68,8 +68,10 @@ def test_mixtral_lora(mixtral_lora_files, tp_size): ], ] - lora_output = do_sample(llm, mixtral_lora_files, lora_id=1, prompts=prompts) - assert all([lora_output[i] in expected_lora_output[i] for i in range(len(prompts))]) + def check_outputs(generated: list[str]): + assert len(generated) == len(expected_lora_output) + for gen, gt_choices in zip(generated, expected_lora_output): + assert gen in gt_choices - lora_output = do_sample(llm, mixtral_lora_files, lora_id=2, prompts=prompts) - assert all([lora_output[i] in expected_lora_output[i] for i in range(len(prompts))]) + check_outputs(do_sample(llm, mixtral_lora_files, lora_id=1, prompts=prompts)) + check_outputs(do_sample(llm, mixtral_lora_files, lora_id=2, prompts=prompts))