diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index dd04591e41..9fb9d8add3 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -292,16 +292,24 @@ async def chat_completion(self, model: str, system: str, user: str, temperature: thinking_kwargs_gpt5 = None if model.startswith('gpt-5'): - if model.endswith('_thinking'): - thinking_kwargs_gpt5 = { - "reasoning_effort": 'low', - "allowed_openai_params": ["reasoning_effort"], - } - else: - thinking_kwargs_gpt5 = { - "reasoning_effort": 'minimal', - "allowed_openai_params": ["reasoning_effort"], - } + # Use configured reasoning_effort or default to MEDIUM + config_effort = get_settings().config.reasoning_effort + try: + ReasoningEffort(config_effort) + effort = config_effort + except (ValueError, TypeError): + effort = ReasoningEffort.MEDIUM.value + if config_effort is not None: + get_logger().warning( + f"Invalid reasoning_effort '{config_effort}' in config. " + f"Using default '{effort}'. Valid values: {[e.value for e in ReasoningEffort]}" + ) + + thinking_kwargs_gpt5 = { + "reasoning_effort": effort, + "allowed_openai_params": ["reasoning_effort"], + } + get_logger().info(f"Using reasoning_effort='{effort}' for GPT-5 model") model = 'openai/'+model.replace('_thinking', '') # remove _thinking suffix @@ -338,9 +346,19 @@ async def chat_completion(self, model: str, system: str, user: str, temperature: del kwargs['temperature'] # Add reasoning_effort if model supports it - if (model in self.support_reasoning_models): - supported_reasoning_efforts = [ReasoningEffort.HIGH.value, ReasoningEffort.MEDIUM.value, ReasoningEffort.LOW.value] - reasoning_effort = get_settings().config.reasoning_effort if (get_settings().config.reasoning_effort in supported_reasoning_efforts) else ReasoningEffort.MEDIUM.value + if model in self.support_reasoning_models: + config_effort = get_settings().config.reasoning_effort + try: + ReasoningEffort(config_effort) + reasoning_effort = config_effort + except (ValueError, TypeError): + reasoning_effort = ReasoningEffort.MEDIUM.value + if config_effort is not None: + get_logger().warning( + f"Invalid reasoning_effort '{config_effort}' in config. " + f"Using default '{reasoning_effort}'. Valid values: {[e.value for e in ReasoningEffort]}" + ) + get_logger().info(f"Adding reasoning_effort with value {reasoning_effort} to model {model}.") kwargs["reasoning_effort"] = reasoning_effort diff --git a/pr_agent/algo/utils.py b/pr_agent/algo/utils.py index 72a0624c51..74ac1272a5 100644 --- a/pr_agent/algo/utils.py +++ b/pr_agent/algo/utils.py @@ -64,9 +64,12 @@ class PRReviewHeader(str, Enum): class ReasoningEffort(str, Enum): + XHIGH = "xhigh" HIGH = "high" MEDIUM = "medium" LOW = "low" + MINIMAL = "minimal" + NONE = "none" class PRDescriptionHeader(str, Enum): diff --git a/tests/unittest/test_litellm_reasoning_effort.py b/tests/unittest/test_litellm_reasoning_effort.py new file mode 100644 index 0000000000..f27cbe229a --- /dev/null +++ b/tests/unittest/test_litellm_reasoning_effort.py @@ -0,0 +1,668 @@ +import pytest +from unittest.mock import patch, MagicMock, AsyncMock, call +from pr_agent.algo.ai_handlers.litellm_ai_handler import LiteLLMAIHandler +import pr_agent.algo.ai_handlers.litellm_ai_handler as litellm_handler + + +def create_mock_settings(reasoning_effort_value): + """Create a fake settings object with configurable reasoning_effort.""" + return type('', (), { + 'config': type('', (), { + 'reasoning_effort': reasoning_effort_value, + 'ai_timeout': 120, + 'custom_reasoning_model': False, + 'max_model_tokens': 32000, + 'verbosity_level': 0, + 'get': lambda self, key, default=None: default + })(), + 'litellm': type('', (), { + 'get': lambda self, key, default=None: default + })(), + 'get': lambda self, key, default=None: default + })() + + +def create_mock_acompletion_response(): + """Create a properly structured mock response for acompletion.""" + mock_response = MagicMock() + mock_response.__getitem__ = lambda self, key: { + "choices": [{"message": {"content": "test"}, "finish_reason": "stop"}] + }[key] + mock_response.dict.return_value = {"choices": [{"message": {"content": "test"}, "finish_reason": "stop"}]} + return mock_response + + +@pytest.fixture +def mock_logger(): + """Mock logger to capture info and warning calls.""" + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.get_logger') as mock_log: + mock_log_instance = MagicMock() + mock_log.return_value = mock_log_instance + yield mock_log_instance + + +class TestLiteLLMReasoningEffort: + """ + Comprehensive test suite for GPT-5 reasoning_effort configuration handling. + + Tests cover: + - Valid reasoning_effort values for GPT-5 models + - Invalid reasoning_effort values with warning logging + - Model detection (GPT-5 vs non-GPT-5) + - Model suffix handling (_thinking vs regular) + - Default fallback logic + - Logging behavior (info and warning messages) + - thinking_kwargs_gpt5 structure validation + """ + + # ========== Group 1: Valid Configuration Tests ========== + + @pytest.mark.asyncio + async def test_gpt5_valid_reasoning_effort_none(self, monkeypatch, mock_logger): + """Test GPT-5 with valid reasoning_effort='none' from config.""" + fake_settings = create_mock_settings("none") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + # Mock acompletion to capture kwargs + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Verify the call was made with correct reasoning_effort + assert mock_completion.called + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "none" + assert "reasoning_effort" in call_kwargs["allowed_openai_params"] + + # Verify info log + mock_logger.info.assert_any_call("Using reasoning_effort='none' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_valid_reasoning_effort_low(self, monkeypatch, mock_logger): + """Test GPT-5 with valid reasoning_effort='low' from config.""" + fake_settings = create_mock_settings("low") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "low" + assert "reasoning_effort" in call_kwargs["allowed_openai_params"] + mock_logger.info.assert_any_call("Using reasoning_effort='low' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_valid_reasoning_effort_medium(self, monkeypatch, mock_logger): + """Test GPT-5 with valid reasoning_effort='medium' from config.""" + fake_settings = create_mock_settings("medium") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + assert "reasoning_effort" in call_kwargs["allowed_openai_params"] + mock_logger.info.assert_any_call("Using reasoning_effort='medium' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_valid_reasoning_effort_high(self, monkeypatch, mock_logger): + """Test GPT-5 with valid reasoning_effort='high' from config.""" + fake_settings = create_mock_settings("high") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "high" + assert "reasoning_effort" in call_kwargs["allowed_openai_params"] + mock_logger.info.assert_any_call("Using reasoning_effort='high' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_valid_reasoning_effort_xhigh(self, monkeypatch, mock_logger): + """Test GPT-5 with valid reasoning_effort='xhigh' from config.""" + fake_settings = create_mock_settings("xhigh") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5.2", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "xhigh" + assert "reasoning_effort" in call_kwargs["allowed_openai_params"] + mock_logger.info.assert_any_call("Using reasoning_effort='xhigh' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_valid_reasoning_effort_minimal(self, monkeypatch, mock_logger): + """Test GPT-5 with valid reasoning_effort='minimal' from config.""" + fake_settings = create_mock_settings("minimal") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "minimal" + assert "reasoning_effort" in call_kwargs["allowed_openai_params"] + mock_logger.info.assert_any_call("Using reasoning_effort='minimal' for GPT-5 model") + + # ========== Group 2: Invalid Configuration Tests ========== + + @pytest.mark.asyncio + async def test_gpt5_invalid_reasoning_effort_with_warning(self, monkeypatch, mock_logger): + """Test GPT-5 with invalid reasoning_effort logs warning and uses default.""" + fake_settings = create_mock_settings("extreme") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Should default to 'medium' + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + + # Verify warning logged + mock_logger.warning.assert_called_once() + warning_call = mock_logger.warning.call_args[0][0] + assert "Invalid reasoning_effort 'extreme' in config" in warning_call + assert "Valid values:" in warning_call + + # Verify info log + mock_logger.info.assert_any_call("Using reasoning_effort='medium' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_invalid_reasoning_effort_thinking_model(self, monkeypatch, mock_logger): + """Test GPT-5 _thinking model with invalid reasoning_effort defaults to 'medium'.""" + fake_settings = create_mock_settings("invalid_value") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07_thinking", + system="test system", + user="test user" + ) + + # Should default to 'medium' (no special handling for _thinking models) + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + + # Verify warning logged + mock_logger.warning.assert_called_once() + + # Verify info log + mock_logger.info.assert_any_call("Using reasoning_effort='medium' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_none_config_defaults_to_medium(self, monkeypatch, mock_logger): + """Test GPT-5 with None config defaults to 'medium' without warning.""" + fake_settings = create_mock_settings(None) + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Should default to 'medium' + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + + # No warning should be logged + mock_logger.warning.assert_not_called() + + # Info log should show effort + mock_logger.info.assert_any_call("Using reasoning_effort='medium' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_none_config_thinking_model_defaults_to_medium(self, monkeypatch, mock_logger): + """Test GPT-5 _thinking model with None config defaults to 'medium' without warning.""" + fake_settings = create_mock_settings(None) + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07_thinking", + system="test system", + user="test user" + ) + + # Should default to 'medium' (no special handling for _thinking models) + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + + # No warning should be logged + mock_logger.warning.assert_not_called() + + # Info log + mock_logger.info.assert_any_call("Using reasoning_effort='medium' for GPT-5 model") + + # ========== Group 3: Model Detection Tests ========== + + @pytest.mark.asyncio + async def test_gpt5_model_detection_various_versions(self, monkeypatch, mock_logger): + """Test various GPT-5 model version strings trigger the reasoning_effort logic.""" + fake_settings = create_mock_settings("medium") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + gpt5_models = ["gpt-5-2025-08-07", "gpt-5.1", "gpt-5-turbo", "gpt-5.1-codex"] + + for model in gpt5_models: + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model=model, + system="test system", + user="test user" + ) + + # All should trigger GPT-5 logic + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + assert "reasoning_effort" in call_kwargs["allowed_openai_params"] + + @pytest.mark.asyncio + async def test_non_gpt5_model_no_thinking_kwargs(self, monkeypatch, mock_logger): + """Test non-GPT-5 models do not trigger reasoning_effort logic.""" + fake_settings = create_mock_settings("high") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + non_gpt5_models = ["gpt-4o", "gpt-4-turbo", "claude-3-5-sonnet"] + + for model in non_gpt5_models: + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model=model, + system="test system", + user="test user" + ) + + # Should not have reasoning_effort in kwargs + call_kwargs = mock_completion.call_args[1] + assert "reasoning_effort" not in call_kwargs + + @pytest.mark.asyncio + async def test_gpt5_suffix_removal(self, monkeypatch, mock_logger): + """Test that _thinking suffix is properly removed from model name.""" + fake_settings = create_mock_settings("low") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5_thinking", + system="test system", + user="test user" + ) + + # Model should be transformed to openai/gpt-5 + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["model"] == "openai/gpt-5" + + # ========== Group 4: Model Suffix Handling Tests ========== + + @pytest.mark.asyncio + async def test_gpt5_thinking_suffix_default_medium(self, monkeypatch, mock_logger): + """Test _thinking suffix models default to 'medium' when config is None.""" + fake_settings = create_mock_settings(None) + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07_thinking", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + mock_logger.info.assert_any_call("Using reasoning_effort='medium' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_regular_suffix_default_medium(self, monkeypatch, mock_logger): + """Test regular GPT-5 models default to 'medium' when config is None.""" + fake_settings = create_mock_settings(None) + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + mock_logger.info.assert_any_call("Using reasoning_effort='medium' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_thinking_suffix_config_overrides_default(self, monkeypatch, mock_logger): + """Test that config overrides the default for _thinking models.""" + fake_settings = create_mock_settings("high") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07_thinking", + system="test system", + user="test user" + ) + + # Should use 'high' from config, not 'medium' default + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "high" + mock_logger.info.assert_any_call("Using reasoning_effort='high' for GPT-5 model") + + # ========== Group 5: Logging Behavior Tests ========== + + @pytest.mark.asyncio + async def test_gpt5_info_logging_configured_value(self, monkeypatch, mock_logger): + """Test info log when using configured value.""" + fake_settings = create_mock_settings("low") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Verify log + mock_logger.info.assert_any_call("Using reasoning_effort='low' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_info_logging_default_value(self, monkeypatch, mock_logger): + """Test info log when using default value.""" + fake_settings = create_mock_settings(None) + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Verify log + mock_logger.info.assert_any_call("Using reasoning_effort='medium' for GPT-5 model") + + @pytest.mark.asyncio + async def test_gpt5_warning_only_for_invalid_non_none(self, monkeypatch, mock_logger): + """Test warning logged only for invalid non-None values.""" + # Test None - should not warn + fake_settings = create_mock_settings(None) + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # No warning for None + mock_logger.warning.assert_not_called() + + # Reset mock + mock_logger.reset_mock() + + # Test invalid string - should warn + fake_settings = create_mock_settings("ultra") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Warning should be logged for invalid value + mock_logger.warning.assert_called_once() + + # ========== Group 6: Structure Validation Tests ========== + + @pytest.mark.asyncio + async def test_thinking_kwargs_gpt5_structure(self, monkeypatch, mock_logger): + """Test that thinking_kwargs_gpt5 has correct structure.""" + fake_settings = create_mock_settings("medium") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + + # Verify structure + assert "reasoning_effort" in call_kwargs + assert call_kwargs["reasoning_effort"] == "medium" + assert "allowed_openai_params" in call_kwargs + assert isinstance(call_kwargs["allowed_openai_params"], list) + assert "reasoning_effort" in call_kwargs["allowed_openai_params"] + + @pytest.mark.asyncio + async def test_thinking_kwargs_not_created_for_non_gpt5(self, monkeypatch, mock_logger): + """Test that thinking_kwargs_gpt5 is not created for non-GPT-5 models.""" + fake_settings = create_mock_settings("high") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-4o", + system="test system", + user="test user" + ) + + call_kwargs = mock_completion.call_args[1] + + # Should not have reasoning_effort keys + assert "reasoning_effort" not in call_kwargs + assert call_kwargs.get("allowed_openai_params") is None or "reasoning_effort" not in call_kwargs.get("allowed_openai_params", []) + + # ========== Group 7: Edge Cases ========== + + @pytest.mark.asyncio + async def test_empty_string_reasoning_effort(self, monkeypatch, mock_logger): + """Test empty string reasoning_effort is treated as invalid.""" + fake_settings = create_mock_settings("") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Should default to 'medium' and log warning + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + mock_logger.warning.assert_called_once() + + @pytest.mark.asyncio + async def test_case_sensitive_reasoning_effort(self, monkeypatch, mock_logger): + """Test that reasoning_effort validation is case-sensitive.""" + fake_settings = create_mock_settings("LOW") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Should treat uppercase as invalid and default to 'medium' + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + mock_logger.warning.assert_called_once() + + @pytest.mark.asyncio + async def test_whitespace_reasoning_effort(self, monkeypatch, mock_logger): + """Test that reasoning_effort with whitespace is treated as invalid.""" + fake_settings = create_mock_settings(" low ") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5-2025-08-07", + system="test system", + user="test user" + ) + + # Should treat value with whitespace as invalid + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium" + mock_logger.warning.assert_called_once() + + @pytest.mark.asyncio + async def test_gpt5_prefix_match_only(self, monkeypatch, mock_logger): + """Test that model.startswith('gpt-5') matching behavior. + + Note: The current logic uses startswith('gpt-5'), which means + models like 'gpt-50' will also match (since 'gpt-50'.startswith('gpt-5') is True). + This test documents the current behavior. + """ + fake_settings = create_mock_settings("medium") + monkeypatch.setattr(litellm_handler, "get_settings", lambda: fake_settings) + + # Test gpt-50 (will match due to startswith logic) + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-50", + system="test system", + user="test user" + ) + + # Due to startswith('gpt-5'), gpt-50 will match and have reasoning_effort + call_kwargs = mock_completion.call_args[1] + assert "reasoning_effort" in call_kwargs + + # Reset mock + mock_logger.reset_mock() + + # Test gpt-5 (should match) + with patch('pr_agent.algo.ai_handlers.litellm_ai_handler.acompletion', new_callable=AsyncMock) as mock_completion: + mock_completion.return_value = create_mock_acompletion_response() + + handler = LiteLLMAIHandler() + await handler.chat_completion( + model="gpt-5", + system="test system", + user="test user" + ) + + # Should have reasoning_effort + call_kwargs = mock_completion.call_args[1] + assert call_kwargs["reasoning_effort"] == "medium"