test(infrastructure): add comprehensive tests for circuit breaker system

- Test all circuit breaker states (CLOSED, OPEN, HALF_OPEN)
- Verify state transitions based on failure/success thresholds
- Test automatic recovery mechanism with configurable timeout
- Add tests for concurrent call handling
- Test both sync and async function protection
- Verify timeout handling and custom exceptions
- Test CircuitBreakerManager for multi-service management
- Add decorator tests for convenient usage
- Test health status monitoring across services
- Cover edge cases like race conditions and rapid state changes
- Verify statistics collection and reset functionality

tests/unit/infrastructure/test_circuit_breaker.py

@@ -0,0 +1,555 @@
+"""
+Unit tests for circuit breaker implementation.
+Tests failure detection, state transitions, and recovery mechanisms.
+"""
+
+import pytest
+import asyncio
+import time
+from unittest.mock import Mock, AsyncMock, patch
+from datetime import datetime, timedelta
+
+from src.infrastructure.resilience.circuit_breaker import (
+    CircuitBreaker,
+    CircuitBreakerConfig,
+    CircuitBreakerManager,
+    CircuitState,
+    CircuitBreakerOpenException,
+    CircuitBreakerTimeoutException,
+    circuit_breaker,
+    circuit_breaker_manager
+)
+
+
+class MockException(Exception):
+    """Mock exception for testing."""
+    pass
+
+
+async def async_success_function(value: int = 42) -> int:
+    """Async function that always succeeds."""
+    await asyncio.sleep(0.01)
+    return value
+
+
+async def async_failure_function():
+    """Async function that always fails."""
+    await asyncio.sleep(0.01)
+    raise MockException("Intentional failure")
+
+
+def sync_success_function(value: int = 42) -> int:
+    """Sync function that always succeeds."""
+    return value
+
+
+def sync_failure_function():
+    """Sync function that always fails."""
+    raise MockException("Intentional failure")
+
+
+async def async_slow_function(delay: float = 2.0) -> str:
+    """Async function that is slow."""
+    await asyncio.sleep(delay)
+    return "completed"
+
+
+@pytest.fixture
+def circuit_config():
+    """Create test circuit breaker config."""
+    return CircuitBreakerConfig(
+        failure_threshold=3,
+        recovery_timeout=1.0,  # Short for tests
+        success_threshold=2,
+        timeout=0.5,  # Short timeout for tests
+        expected_exception=MockException
+    )
+
+
+@pytest.fixture
+def circuit(circuit_config):
+    """Create test circuit breaker."""
+    return CircuitBreaker("test_service", circuit_config)
+
+
+class TestCircuitBreakerConfig:
+    """Test CircuitBreakerConfig class."""
+    
+    @pytest.mark.unit
+    def test_default_config(self):
+        """Test default configuration values."""
+        config = CircuitBreakerConfig()
+        
+        assert config.failure_threshold == 5
+        assert config.recovery_timeout == 60.0
+        assert config.success_threshold == 3
+        assert config.timeout == 30.0
+        assert config.expected_exception == Exception
+    
+    @pytest.mark.unit
+    def test_custom_config(self):
+        """Test custom configuration values."""
+        config = CircuitBreakerConfig(
+            failure_threshold=10,
+            recovery_timeout=120.0,
+            success_threshold=5,
+            timeout=60.0,
+            expected_exception=ValueError
+        )
+        
+        assert config.failure_threshold == 10
+        assert config.recovery_timeout == 120.0
+        assert config.success_threshold == 5
+        assert config.timeout == 60.0
+        assert config.expected_exception == ValueError
+
+
+class TestCircuitBreaker:
+    """Test CircuitBreaker class."""
+    
+    @pytest.mark.unit
+    def test_initialization(self, circuit_config):
+        """Test circuit breaker initialization."""
+        breaker = CircuitBreaker("test_service", circuit_config)
+        
+        assert breaker.name == "test_service"
+        assert breaker.config == circuit_config
+        assert breaker.state == CircuitState.CLOSED
+        assert breaker.stats.total_requests == 0
+        assert breaker.stats.successful_requests == 0
+        assert breaker.stats.failed_requests == 0
+    
+    @pytest.mark.unit
+    async def test_successful_async_call(self, circuit):
+        """Test successful async function call."""
+        result = await circuit.call(async_success_function, 100)
+        
+        assert result == 100
+        assert circuit.state == CircuitState.CLOSED
+        assert circuit.stats.total_requests == 1
+        assert circuit.stats.successful_requests == 1
+        assert circuit.stats.failed_requests == 0
+    
+    @pytest.mark.unit
+    async def test_successful_sync_call(self, circuit):
+        """Test successful sync function call."""
+        result = await circuit.call(sync_success_function, 100)
+        
+        assert result == 100
+        assert circuit.state == CircuitState.CLOSED
+        assert circuit.stats.total_requests == 1
+        assert circuit.stats.successful_requests == 1
+        assert circuit.stats.failed_requests == 0
+    
+    @pytest.mark.unit
+    async def test_failed_call(self, circuit):
+        """Test failed function call."""
+        with pytest.raises(MockException):
+            await circuit.call(async_failure_function)
+        
+        assert circuit.state == CircuitState.CLOSED  # Not open yet
+        assert circuit.stats.total_requests == 1
+        assert circuit.stats.successful_requests == 0
+        assert circuit.stats.failed_requests == 1
+        assert circuit.stats.current_consecutive_failures == 1
+    
+    @pytest.mark.unit
+    async def test_circuit_opens_after_threshold(self, circuit):
+        """Test circuit opens after failure threshold."""
+        # Fail 3 times to reach threshold
+        for _ in range(3):
+            with pytest.raises(MockException):
+                await circuit.call(async_failure_function)
+        
+        assert circuit.state == CircuitState.OPEN
+        assert circuit.stats.failed_requests == 3
+        assert circuit.stats.current_consecutive_failures == 3
+        assert circuit.stats.state_changes == 1
+    
+    @pytest.mark.unit
+    async def test_open_circuit_rejects_calls(self, circuit):
+        """Test open circuit rejects calls."""
+        # Open the circuit
+        for _ in range(3):
+            with pytest.raises(MockException):
+                await circuit.call(async_failure_function)
+        
+        assert circuit.state == CircuitState.OPEN
+        
+        # Try another call - should be rejected
+        with pytest.raises(CircuitBreakerOpenException) as exc_info:
+            await circuit.call(async_success_function)
+        
+        assert "Circuit breaker 'test_service' is open" in str(exc_info.value)
+        assert circuit.stats.rejected_requests == 1
+        assert circuit.stats.total_requests == 4  # 3 failures + 1 rejected
+    
+    @pytest.mark.unit
+    async def test_half_open_transition(self, circuit):
+        """Test transition to half-open state."""
+        # Open the circuit
+        for _ in range(3):
+            with pytest.raises(MockException):
+                await circuit.call(async_failure_function)
+        
+        assert circuit.state == CircuitState.OPEN
+        
+        # Wait for recovery timeout
+        await asyncio.sleep(1.1)  # Recovery timeout is 1.0
+        
+        # Next call should go through (half-open)
+        result = await circuit.call(async_success_function)
+        
+        assert result == 42
+        assert circuit.state == CircuitState.HALF_OPEN
+        assert circuit.stats.current_consecutive_successes == 1
+    
+    @pytest.mark.unit
+    async def test_half_open_to_closed(self, circuit):
+        """Test successful recovery from half-open to closed."""
+        # Open the circuit
+        for _ in range(3):
+            with pytest.raises(MockException):
+                await circuit.call(async_failure_function)
+        
+        # Wait for recovery
+        await asyncio.sleep(1.1)
+        
+        # Two successful calls to close circuit (success_threshold=2)
+        await circuit.call(async_success_function)
+        assert circuit.state == CircuitState.HALF_OPEN
+        
+        await circuit.call(async_success_function)
+        assert circuit.state == CircuitState.CLOSED
+        assert circuit.stats.state_changes == 2  # CLOSED->OPEN->CLOSED
+    
+    @pytest.mark.unit
+    async def test_half_open_to_open(self, circuit):
+        """Test failure in half-open state reopens circuit."""
+        # Open the circuit
+        for _ in range(3):
+            with pytest.raises(MockException):
+                await circuit.call(async_failure_function)
+        
+        # Wait for recovery
+        await asyncio.sleep(1.1)
+        
+        # Successful call puts in half-open
+        await circuit.call(async_success_function)
+        assert circuit.state == CircuitState.HALF_OPEN
+        
+        # Failed call reopens circuit
+        with pytest.raises(MockException):
+            await circuit.call(async_failure_function)
+        
+        assert circuit.state == CircuitState.OPEN
+    
+    @pytest.mark.unit
+    async def test_timeout_handling(self, circuit):
+        """Test request timeout handling."""
+        with pytest.raises(CircuitBreakerTimeoutException) as exc_info:
+            await circuit.call(async_slow_function, 2.0)  # 2s delay, 0.5s timeout
+        
+        assert "timed out after 0.5s" in str(exc_info.value)
+        assert circuit.stats.failed_requests == 1
+        assert circuit.state == CircuitState.CLOSED  # One failure, not at threshold
+    
+    @pytest.mark.unit
+    async def test_unexpected_exception_passthrough(self, circuit):
+        """Test unexpected exceptions pass through."""
+        async def unexpected_error():
+            raise ValueError("Unexpected error")
+        
+        # ValueError is not the expected exception type
+        with pytest.raises(ValueError):
+            await circuit.call(unexpected_error)
+        
+        # Should not count as circuit breaker failure
+        assert circuit.stats.failed_requests == 0
+        assert circuit.stats.total_requests == 1
+    
+    @pytest.mark.unit
+    async def test_reset_circuit(self, circuit):
+        """Test manual circuit reset."""
+        # Open the circuit
+        for _ in range(3):
+            with pytest.raises(MockException):
+                await circuit.call(async_failure_function)
+        
+        assert circuit.state == CircuitState.OPEN
+        
+        # Reset circuit
+        await circuit.reset()
+        
+        assert circuit.state == CircuitState.CLOSED
+        assert circuit.stats.current_consecutive_failures == 0
+        assert circuit.stats.current_consecutive_successes == 0
+    
+    @pytest.mark.unit
+    async def test_force_open(self, circuit):
+        """Test forcing circuit to open state."""
+        await circuit.force_open()
+        
+        assert circuit.state == CircuitState.OPEN
+        
+        # Should reject calls
+        with pytest.raises(CircuitBreakerOpenException):
+            await circuit.call(async_success_function)
+    
+    @pytest.mark.unit
+    def test_get_stats(self, circuit):
+        """Test getting circuit breaker statistics."""
+        stats = circuit.get_stats()
+        
+        assert stats["name"] == "test_service"
+        assert stats["state"] == "closed"
+        assert stats["config"]["failure_threshold"] == 3
+        assert stats["stats"]["total_requests"] == 0
+        assert stats["stats"]["success_rate"] == 0
+    
+    @pytest.mark.unit
+    async def test_concurrent_calls(self, circuit):
+        """Test concurrent calls through circuit breaker."""
+        # Run multiple concurrent successful calls
+        results = await asyncio.gather(
+            circuit.call(async_success_function, 1),
+            circuit.call(async_success_function, 2),
+            circuit.call(async_success_function, 3)
+        )
+        
+        assert results == [1, 2, 3]
+        assert circuit.stats.total_requests == 3
+        assert circuit.stats.successful_requests == 3
+        assert circuit.state == CircuitState.CLOSED
+
+
+class TestCircuitBreakerManager:
+    """Test CircuitBreakerManager class."""
+    
+    @pytest.mark.unit
+    def test_manager_initialization(self):
+        """Test circuit breaker manager initialization."""
+        manager = CircuitBreakerManager()
+        
+        assert len(manager._breakers) == 0
+        assert len(manager._default_configs) == 0
+    
+    @pytest.mark.unit
+    def test_register_default_config(self):
+        """Test registering default configuration."""
+        manager = CircuitBreakerManager()
+        config = CircuitBreakerConfig(failure_threshold=10)
+        
+        manager.register_default_config("test_service", config)
+        
+        assert "test_service" in manager._default_configs
+        assert manager._default_configs["test_service"] == config
+    
+    @pytest.mark.unit
+    def test_get_circuit_breaker(self):
+        """Test getting circuit breaker."""
+        manager = CircuitBreakerManager()
+        
+        breaker1 = manager.get_circuit_breaker("service1")
+        breaker2 = manager.get_circuit_breaker("service1")
+        breaker3 = manager.get_circuit_breaker("service2")
+        
+        assert breaker1 is breaker2  # Same instance
+        assert breaker1 is not breaker3  # Different services
+        assert len(manager._breakers) == 2
+    
+    @pytest.mark.unit
+    def test_get_circuit_breaker_with_default_config(self):
+        """Test getting circuit breaker with default config."""
+        manager = CircuitBreakerManager()
+        config = CircuitBreakerConfig(failure_threshold=10)
+        
+        manager.register_default_config("test_service", config)
+        breaker = manager.get_circuit_breaker("test_service")
+        
+        assert breaker.config.failure_threshold == 10
+    
+    @pytest.mark.unit
+    async def test_call_service(self):
+        """Test calling service through manager."""
+        manager = CircuitBreakerManager()
+        
+        result = await manager.call_service(
+            "test_service",
+            async_success_function,
+            100
+        )
+        
+        assert result == 100
+        assert "test_service" in manager._breakers
+    
+    @pytest.mark.unit
+    def test_get_all_stats(self):
+        """Test getting all circuit breaker stats."""
+        manager = CircuitBreakerManager()
+        
+        manager.get_circuit_breaker("service1")
+        manager.get_circuit_breaker("service2")
+        
+        stats = manager.get_all_stats()
+        
+        assert len(stats) == 2
+        assert "service1" in stats
+        assert "service2" in stats
+        assert stats["service1"]["name"] == "service1"
+    
+    @pytest.mark.unit
+    async def test_reset_all(self):
+        """Test resetting all circuit breakers."""
+        manager = CircuitBreakerManager()
+        
+        # Create and open multiple breakers
+        for service in ["service1", "service2"]:
+            breaker = manager.get_circuit_breaker(service)
+            await breaker.force_open()
+        
+        # Reset all
+        await manager.reset_all()
+        
+        # Check all are closed
+        for breaker in manager._breakers.values():
+            assert breaker.state == CircuitState.CLOSED
+    
+    @pytest.mark.unit
+    async def test_get_health_status(self):
+        """Test getting health status of all services."""
+        manager = CircuitBreakerManager()
+        
+        # Create breakers in different states
+        breaker1 = manager.get_circuit_breaker("healthy_service")
+        
+        breaker2 = manager.get_circuit_breaker("degraded_service")
+        breaker2.state = CircuitState.HALF_OPEN
+        
+        breaker3 = manager.get_circuit_breaker("failed_service")
+        await breaker3.force_open()
+        
+        health = manager.get_health_status()
+        
+        assert health["overall_health"] == "degraded"
+        assert health["total_services"] == 3
+        assert "healthy_service" in health["healthy_services"]
+        assert "degraded_service" in health["degraded_services"]
+        assert "failed_service" in health["failed_services"]
+        assert health["health_score"] == pytest.approx(1/3)
+
+
+class TestCircuitBreakerDecorator:
+    """Test circuit breaker decorator."""
+    
+    @pytest.mark.unit
+    async def test_decorator_basic(self):
+        """Test basic decorator usage."""
+        @circuit_breaker("decorated_service")
+        async def decorated_function(value: int) -> int:
+            return value * 2
+        
+        result = await decorated_function(5)
+        
+        assert result == 10
+        
+        # Check breaker was created
+        stats = circuit_breaker_manager.get_all_stats()
+        assert "decorated_service" in stats
+    
+    @pytest.mark.unit
+    async def test_decorator_with_config(self):
+        """Test decorator with custom config."""
+        config = CircuitBreakerConfig(failure_threshold=2)
+        
+        @circuit_breaker("custom_service", config)
+        async def failing_function():
+            raise MockException("Fail")
+        
+        # Fail twice to open circuit
+        for _ in range(2):
+            with pytest.raises(MockException):
+                await failing_function()
+        
+        # Check circuit is open
+        breaker = circuit_breaker_manager.get_circuit_breaker("custom_service")
+        assert breaker.state == CircuitState.OPEN
+
+
+class TestGlobalCircuitBreakerManager:
+    """Test global circuit breaker manager instance."""
+    
+    @pytest.mark.unit
+    def test_default_configurations(self):
+        """Test default configurations are registered."""
+        # Check some default services are configured
+        assert "transparency_api" in circuit_breaker_manager._default_configs
+        assert "llm_service" in circuit_breaker_manager._default_configs
+        assert "database" in circuit_breaker_manager._default_configs
+        assert "redis" in circuit_breaker_manager._default_configs
+        
+        # Check configuration values
+        transparency_config = circuit_breaker_manager._default_configs["transparency_api"]
+        assert transparency_config.failure_threshold == 3
+        assert transparency_config.timeout == 15.0
+
+
+class TestCircuitBreakerEdgeCases:
+    """Test edge cases and error scenarios."""
+    
+    @pytest.mark.unit
+    async def test_multiple_concurrent_failures(self, circuit):
+        """Test handling multiple concurrent failures."""
+        # Create concurrent failing calls
+        tasks = []
+        for _ in range(5):
+            tasks.append(circuit.call(async_failure_function))
+        
+        # All should fail
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        
+        assert all(isinstance(r, MockException) for r in results)
+        assert circuit.state == CircuitState.OPEN
+        assert circuit.stats.failed_requests >= 3  # At least threshold
+    
+    @pytest.mark.unit
+    async def test_race_condition_state_change(self, circuit):
+        """Test race condition during state change."""
+        # Bring circuit to edge of opening (2 failures, threshold is 3)
+        for _ in range(2):
+            with pytest.raises(MockException):
+                await circuit.call(async_failure_function)
+        
+        # Concurrent calls - one fails, one succeeds
+        results = await asyncio.gather(
+            circuit.call(async_failure_function),
+            circuit.call(async_success_function),
+            return_exceptions=True
+        )
+        
+        # Circuit state should be consistent
+        assert circuit.state in [CircuitState.CLOSED, CircuitState.OPEN]
+        assert circuit.stats.total_requests == 4
+    
+    @pytest.mark.unit
+    async def test_very_short_recovery_timeout(self):
+        """Test very short recovery timeout."""
+        config = CircuitBreakerConfig(
+            failure_threshold=1,
+            recovery_timeout=0.01,  # Very short
+            success_threshold=1
+        )
+        breaker = CircuitBreaker("fast_recovery", config)
+        
+        # Open circuit
+        with pytest.raises(MockException):
+            await breaker.call(async_failure_function)
+        
+        assert breaker.state == CircuitState.OPEN
+        
+        # Wait for recovery
+        await asyncio.sleep(0.02)
+        
+        # Should be able to try again
+        result = await breaker.call(async_success_function)
+        assert result == 42