package cc.unitmesh.xuiper.eval.evaluator import cc.unitmesh.xuiper.model.EvaluationCriterion import cc.unitmesh.xuiper.model.TestCase /** * Evaluator - Interface for DSL evaluation strategies * * Pluggable evaluation system that allows: * 1. Adding new evaluation criteria * 2. Customizing evaluation logic per criterion * 3. Combining multiple evaluators * * When LLM capabilities change: * - Add new evaluators for new criteria * - Adjust scoring logic for existing evaluators * - Create composite evaluators with different weights */ interface Evaluator { /** The criterion this evaluator handles */ val criterion: EvaluationCriterion /** Human-readable name */ val name: String /** Description of what this evaluator measures */ val description: String /** * Evaluate the actual output against expected * * @param testCase The test case with expected output * @param actualDsl The actual DSL generated by LLM * @return Score from 0.0 to 1.0 */ fun evaluate(testCase: TestCase, actualDsl: String): Float } /** * Evaluator registry - manages all available evaluators */ object EvaluatorRegistry { private val evaluators = mutableMapOf() init { // Register default evaluators register(SyntaxEvaluator) register(ComponentMatchEvaluator) register(StructureEvaluator) register(PropsEvaluator) register(StateBindingEvaluator) register(ActionEvaluator) register(FormattingEvaluator) register(RedundancyEvaluator) } fun register(evaluator: Evaluator) { evaluators[evaluator.criterion] = evaluator } fun get(criterion: EvaluationCriterion): Evaluator? { return evaluators[criterion] } fun getAll(): List { return evaluators.values.toList() } /** * Evaluate a test case with all applicable evaluators */ fun evaluateAll(testCase: TestCase, actualDsl: String): Map { return testCase.criteria.associateWith { criterion -> evaluators[criterion]?.evaluate(testCase, actualDsl) ?: 0f } } } // ==================== Default Evaluator Implementations ==================== object SyntaxEvaluator : Evaluator { override val criterion = EvaluationCriterion.SYNTAX_VALID override val name = "Syntax Validator" override val description = "Checks if the DSL syntax is valid" override fun evaluate(testCase: TestCase, actualDsl: String): Float { if (actualDsl.isBlank()) return 0f val hasComponent = actualDsl.contains("component ") || actualDsl.contains("Card") || actualDsl.contains("VStack") val hasIndentation = actualDsl.lines().any { it.startsWith(" ") || it.startsWith("\t") } return when { hasComponent && hasIndentation -> 1.0f hasComponent || hasIndentation -> 0.5f else -> 0.2f } } } object ComponentMatchEvaluator : Evaluator { override val criterion = EvaluationCriterion.COMPONENT_MATCH override val name = "Component Matcher" override val description = "Checks if the correct components are used" private val componentPattern = Regex("(VStack|HStack|Card|Text|Button|Image|Input|Badge|Checkbox)\\s*[(:.]") override fun evaluate(testCase: TestCase, actualDsl: String): Float { val expected = componentPattern.findAll(testCase.expectedDsl).map { it.groupValues[1] }.toSet() val actual = componentPattern.findAll(actualDsl).map { it.groupValues[1] }.toSet() if (expected.isEmpty()) return 1.0f return actual.intersect(expected).size.toFloat() / expected.size } } object StructureEvaluator : Evaluator { override val criterion = EvaluationCriterion.STRUCTURE_SIMILAR override val name = "Structure Analyzer" override val description = "Checks if the nesting structure is similar" override fun evaluate(testCase: TestCase, actualDsl: String): Float { val expectedDepth = maxIndentDepth(testCase.expectedDsl) val actualDepth = maxIndentDepth(actualDsl) val diff = kotlin.math.abs(expectedDepth - actualDepth) return when { diff == 0 -> 1.0f diff <= 1 -> 0.8f diff <= 2 -> 0.5f else -> 0.2f } } private fun maxIndentDepth(dsl: String): Int { return dsl.lines().maxOfOrNull { line -> line.takeWhile { it == ' ' }.length / 4 } ?: 0 } } object PropsEvaluator : Evaluator { override val criterion = EvaluationCriterion.PROPS_CORRECT override val name = "Props Checker" override val description = "Checks if properties are correctly specified" private val propPattern = Regex("(padding|shadow|spacing|align|justify|style|intent|radius)\\s*[=:]") override fun evaluate(testCase: TestCase, actualDsl: String): Float { val expected = propPattern.findAll(testCase.expectedDsl).map { it.groupValues[1] }.toSet() val actual = propPattern.findAll(actualDsl).map { it.groupValues[1] }.toSet() if (expected.isEmpty()) return 1.0f return actual.intersect(expected).size.toFloat() / expected.size } } object StateBindingEvaluator : Evaluator { override val criterion = EvaluationCriterion.STATE_BINDINGS override val name = "State Binding Checker" override val description = "Checks if state bindings are correctly used" override fun evaluate(testCase: TestCase, actualDsl: String): Float { val expectedHas = testCase.expectedDsl.contains("state:") || testCase.expectedDsl.contains("<<") || testCase.expectedDsl.contains(":=") val actualHas = actualDsl.contains("state:") || actualDsl.contains("<<") || actualDsl.contains(":=") return if (expectedHas == actualHas) 1.0f else 0.0f } } object ActionEvaluator : Evaluator { override val criterion = EvaluationCriterion.ACTIONS_CORRECT override val name = "Action Checker" override val description = "Checks if actions are correctly specified" override fun evaluate(testCase: TestCase, actualDsl: String): Float { val expectedHas = testCase.expectedDsl.contains("on_click") || testCase.expectedDsl.contains("Navigate") || testCase.expectedDsl.contains("Fetch") val actualHas = actualDsl.contains("on_click") || actualDsl.contains("Navigate") || actualDsl.contains("Fetch") return if (expectedHas == actualHas) 1.0f else 0.0f } } object FormattingEvaluator : Evaluator { override val criterion = EvaluationCriterion.FORMATTING override val name = "Formatting Checker" override val description = "Checks if indentation is consistent" override fun evaluate(testCase: TestCase, actualDsl: String): Float { val lines = actualDsl.lines() val properlyIndented = lines.all { line -> line.isBlank() || !line.startsWith(" ") || line.takeWhile { it == ' ' }.length % 4 == 0 } return if (properlyIndented) 1.0f else 0.5f } } object RedundancyEvaluator : Evaluator { override val criterion = EvaluationCriterion.NO_REDUNDANCY override val name = "Redundancy Checker" override val description = "Checks for unnecessary code" override fun evaluate(testCase: TestCase, actualDsl: String): Float { val expectedLines = testCase.expectedDsl.lines().count { it.isNotBlank() } val actualLines = actualDsl.lines().count { it.isNotBlank() } if (expectedLines == 0) return 1.0f val ratio = actualLines.toFloat() / expectedLines return when { ratio in 0.8f..1.2f -> 1.0f ratio in 0.6f..1.5f -> 0.7f else -> 0.4f } } }