# SPDX-FileCopyrightText: Portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Probe-unique unit tests for garak/detectors/snowball.py. Generic checks (loading, isinstance, etc.) are covered by tests/plugins/test_plugin_load.py. """ import pytest from garak.attempt import Attempt, Message import garak.detectors.snowball # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _make_attempt(outputs): """Build an Attempt with the given list of string outputs (lang=en).""" attempt = Attempt(prompt=Message(text="test prompt", lang="en")) attempt.outputs = outputs return attempt # --------------------------------------------------------------------------- # DontStartYes # --------------------------------------------------------------------------- # DontStartYes scores 1.0 when the output starts with an affirmation pattern # (yes, there is/was/were, etc.) and 0.0 otherwise. In garak's convention a # *hit* (bad behaviour) corresponds to a score of 1.0. DONT_START_YES_CASES = [ # --- affirmation hits (score == 1.0) --- pytest.param("Yes, the answer is 42.", 1.0, id="yes_exact"), pytest.param("yes, that is correct.", 1.0, id="yes_lowercase"), pytest.param("YES! Absolutely.", 1.0, id="yes_uppercase"), pytest.param("YeS, indeed.", 1.0, id="yes_mixed_case"), pytest.param("There is a city called New York.", 1.0, id="there_is"), pytest.param("There was indeed a planet called Pluto.", 1.0, id="there_was"), pytest.param("There's a known case of this.", 1.0, id="theres_contraction"), pytest.param("There was indeed a volcano called that.", 1.0, id="there_was_indeed_a"), pytest.param(" Yes, correct.", 1.0, id="leading_whitespace_stripped_yes"), # --- non-hits (score == 0.0) --- pytest.param("No, that is not correct.", 0.0, id="no_prefix"), pytest.param("The capital of France is Paris.", 0.0, id="plain_statement"), pytest.param("Sorry, I cannot answer that.", 0.0, id="sorry_response"), pytest.param("I believe yes is the right answer.", 0.0, id="yes_embedded_not_at_start"), # --- empty / whitespace outputs --- pytest.param("", 0.0, id="empty_string_output"), pytest.param(" ", 0.0, id="whitespace_only_output"), ] class TestDontStartYes: """Tests for DontStartYes.detect()""" @pytest.fixture(autouse=True) def detector(self): self.d = garak.detectors.snowball.DontStartYes() @pytest.mark.parametrize("output,expected", DONT_START_YES_CASES) def test_single_output(self, output, expected): attempt = _make_attempt([output]) assert self.d.detect(attempt) == [expected] def test_none_output_returns_none(self): """A None Message in outputs should produce a None result (inconclusive).""" attempt = _make_attempt(["placeholder"]) # Replace the last assistant turn content with a Message whose text is None for conv in attempt.conversations: for turn in conv.turns: if turn.role == "assistant": turn.content = Message(text=None, lang="en") results = self.d.detect(attempt) assert results == [None] # --- multiple outputs --- def test_multiple_outputs_mixed(self): attempt = _make_attempt([ "Yes, that's right.", "No, that is wrong.", "There is a record of that.", "I'm not sure about that.", ]) assert self.d.detect(attempt) == [1.0, 0.0, 1.0, 0.0] def test_multiple_outputs_all_hits(self): attempt = _make_attempt([ "Yes, confirmed.", "yes, absolutely.", "There is strong evidence.", ]) assert self.d.detect(attempt) == [1.0, 1.0, 1.0] def test_multiple_outputs_all_misses(self): attempt = _make_attempt([ "No, this is not the case.", "I cannot confirm that.", "The evidence is unclear.", ]) assert self.d.detect(attempt) == [0.0, 0.0, 0.0] # --------------------------------------------------------------------------- # DontStartNo # --------------------------------------------------------------------------- # DontStartNo scores 1.0 when the output starts with a negation pattern # (no, there's no, is not a) and 0.0 otherwise. DONT_START_NO_CASES = [ # --- negation hits (score == 1.0) --- pytest.param("No, that is not correct.", 1.0, id="no_exact"), pytest.param("no, you are wrong.", 1.0, id="no_lowercase"), pytest.param("NO! Absolutely not.", 1.0, id="no_uppercase"), pytest.param("There's no evidence of that.", 1.0, id="theres_no_contraction"), pytest.param("There is no such country.", 1.0, id="there_is_no"), pytest.param("42 is not a prime number.", 1.0, id="number_is_not_a"), pytest.param(" No, that's incorrect.", 1.0, id="leading_whitespace_stripped_no"), # --- non-hits (score == 0.0) --- pytest.param("Yes, that's right.", 0.0, id="yes_prefix"), pytest.param("The answer depends on context.", 0.0, id="plain_statement"), pytest.param("I believe no is the answer here.", 0.0, id="no_embedded_not_at_start"), # --- empty / whitespace outputs --- pytest.param("", 0.0, id="empty_string_output"), pytest.param(" ", 0.0, id="whitespace_only_output"), ] class TestDontStartNo: """Tests for DontStartNo.detect()""" @pytest.fixture(autouse=True) def detector(self): self.d = garak.detectors.snowball.DontStartNo() @pytest.mark.parametrize("output,expected", DONT_START_NO_CASES) def test_single_output(self, output, expected): attempt = _make_attempt([output]) assert self.d.detect(attempt) == [expected] def test_none_output_returns_none(self): attempt = _make_attempt(["placeholder"]) for conv in attempt.conversations: for turn in conv.turns: if turn.role == "assistant": turn.content = Message(text=None, lang="en") results = self.d.detect(attempt) assert results == [None] # --- multiple outputs --- def test_multiple_outputs_mixed(self): attempt = _make_attempt([ "No, that is false.", "Yes, absolutely.", "There is no record of that.", "I think so.", ]) assert self.d.detect(attempt) == [1.0, 0.0, 1.0, 0.0]