74 lines
2.2 KiB
Python
74 lines
2.2 KiB
Python
import pytest
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import Mock, AsyncMock
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
|
|
|
from promptforge.testing.validator import Validator
|
|
from promptforge.testing.metrics import MetricsCollector, ComparisonResult
|
|
from promptforge.testing.ab_test import ABTest, ABTestConfig
|
|
|
|
|
|
class TestValidator:
|
|
def test_valid_json_response(self):
|
|
validator = Validator.from_prompt_rules([
|
|
{"type": "json"}
|
|
])
|
|
assert validator.is_valid('{"key": "value"}')
|
|
|
|
def test_invalid_json_response(self):
|
|
validator = Validator.from_prompt_rules([
|
|
{"type": "json"}
|
|
])
|
|
assert not validator.is_valid("not json")
|
|
|
|
def test_regex_validation(self):
|
|
validator = Validator.from_prompt_rules([
|
|
{"type": "regex", "pattern": "^\\d+\\. .+", "message": "Must be numbered list"}
|
|
])
|
|
assert validator.is_valid("1. First item")
|
|
assert not validator.is_valid("Not numbered")
|
|
|
|
|
|
class TestMetricsCollector:
|
|
def test_compare_metrics(self):
|
|
from promptforge.testing.metrics import TestMetrics
|
|
|
|
collector = MetricsCollector()
|
|
metrics1 = TestMetrics(
|
|
test_id="1",
|
|
prompt_name="Test",
|
|
provider="openai",
|
|
model="gpt-4",
|
|
latency_ms=100.0,
|
|
success=True,
|
|
tokens_used=50
|
|
)
|
|
metrics2 = TestMetrics(
|
|
test_id="2",
|
|
prompt_name="Test",
|
|
provider="openai",
|
|
model="gpt-4",
|
|
latency_ms=200.0,
|
|
success=True,
|
|
tokens_used=60
|
|
)
|
|
|
|
result = collector.compare("Test", [metrics1, metrics2])
|
|
assert result.total_runs == 2
|
|
assert result.successful_runs == 2
|
|
assert result.avg_latency_ms == 150.0
|
|
|
|
|
|
class TestABTest:
|
|
def test_config_defaults(self):
|
|
config = ABTestConfig()
|
|
assert config.iterations == 3
|
|
assert config.parallel == False
|
|
|
|
def test_ab_test_init(self):
|
|
mock_provider = Mock()
|
|
config = ABTestConfig(iterations=5)
|
|
ab_test = ABTest(mock_provider, config)
|
|
assert ab_test.config.iterations == 5 |