diff --git a/tests/test_testing.py b/tests/test_testing.py new file mode 100644 index 0000000..3441ca0 --- /dev/null +++ b/tests/test_testing.py @@ -0,0 +1,74 @@ +import pytest +import sys +from pathlib import Path +from unittest.mock import Mock, AsyncMock + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from promptforge.testing.validator import Validator +from promptforge.testing.metrics import MetricsCollector, ComparisonResult +from promptforge.testing.ab_test import ABTest, ABTestConfig + + +class TestValidator: + def test_valid_json_response(self): + validator = Validator.from_prompt_rules([ + {"type": "json"} + ]) + assert validator.is_valid('{"key": "value"}') + + def test_invalid_json_response(self): + validator = Validator.from_prompt_rules([ + {"type": "json"} + ]) + assert not validator.is_valid("not json") + + def test_regex_validation(self): + validator = Validator.from_prompt_rules([ + {"type": "regex", "pattern": "^\\d+\\. .+", "message": "Must be numbered list"} + ]) + assert validator.is_valid("1. First item") + assert not validator.is_valid("Not numbered") + + +class TestMetricsCollector: + def test_compare_metrics(self): + from promptforge.testing.metrics import TestMetrics + + collector = MetricsCollector() + metrics1 = TestMetrics( + test_id="1", + prompt_name="Test", + provider="openai", + model="gpt-4", + latency_ms=100.0, + success=True, + tokens_used=50 + ) + metrics2 = TestMetrics( + test_id="2", + prompt_name="Test", + provider="openai", + model="gpt-4", + latency_ms=200.0, + success=True, + tokens_used=60 + ) + + result = collector.compare("Test", [metrics1, metrics2]) + assert result.total_runs == 2 + assert result.successful_runs == 2 + assert result.avg_latency_ms == 150.0 + + +class TestABTest: + def test_config_defaults(self): + config = ABTestConfig() + assert config.iterations == 3 + assert config.parallel == False + + def test_ab_test_init(self): + mock_provider = Mock() + config = ABTestConfig(iterations=5) + ab_test = ABTest(mock_provider, config) + assert ab_test.config.iterations == 5 \ No newline at end of file