package cc.unitmesh.agent.subagent
import cc.unitmesh.agent.core.SubAgent
import cc.unitmesh.agent.webagent.executor.*
import cc.unitmesh.agent.webagent.model.*
import cc.unitmesh.agent.webagent.perception.PageStateExtractor
import cc.unitmesh.agent.webagent.perception.PageStateExtractorConfig
import cc.unitmesh.agent.webagent.perception.createPageStateExtractor
import cc.unitmesh.agent.webagent.planner.TestActionPlanner
import cc.unitmesh.agent.webagent.prompt.WebAgentPrompts
import cc.unitmesh.agent.model.AgentDefinition
import cc.unitmesh.agent.model.PromptConfig
import cc.unitmesh.agent.tool.ToolResult
import cc.unitmesh.llm.LLMService
import cc.unitmesh.llm.ModelConfig
import kotlinx.coroutines.delay
import kotlinx.serialization.Serializable
/**
* Web Agent - AI-driven web automation with visual understanding.
*
* Features:
* - Natural language scenario generation for testing and RPA
* - Multi-modal perception (DOM + Accessibility Tree + Vision)
* - Self-healing locators with two-level strategy
* - Deterministic execution with AI planning
*
* @see Issue #532
*/
class WebAgent(
private val llmService: LLMService,
private val config: cc.unitmesh.agent.webagent.WebAgentConfig = _root_ide_package_.cc.unitmesh.agent.webagent.WebAgentConfig()
) : SubAgent(
AgentDefinition(
name = "WebAgent",
displayName = "Web Automation Agent",
description = "AI-driven web automation with visual understanding and self-healing locators for testing and RPA",
promptConfig = PromptConfig(
systemPrompt = WebAgentPrompts.systemPrompt,
queryTemplate = null,
initialMessages = emptyList()
),
modelConfig = ModelConfig.default(),
runConfig = cc.unitmesh.agent.model.RunConfig(
maxTurns = 50,
maxTimeMinutes = 10,
terminateOnError = false
)
)
) {
private var pageStateExtractor: PageStateExtractor? = null
private var browserExecutor: DriverBasedBrowserActionExecutor? = null
private var planner: TestActionPlanner? = null
private var selfHealingLocator: SelfHealingLocator? = null
/**
* Check if the agent is available on this platform
*/
override val isAvailable: Boolean
get() = pageStateExtractor?.isAvailable == true && browserExecutor?.isAvailable == true
/**
* Initialize the agent with browser components using platform factory functions.
*
* Note: This method only initializes the PageStateExtractor and planner.
* The browserExecutor will be null - use [initializeWithDriver]
* to provide a BrowserDriver for full functionality.
*/
fun initialize() {
val extractorConfig = PageStateExtractorConfig(
viewportWidth = config.viewportWidth,
viewportHeight = config.viewportHeight
)
pageStateExtractor = createPageStateExtractor(extractorConfig)
// Note: browserExecutor is not set here because createBrowserActionExecutor
// returns BrowserActionExecutor? which is platform-specific.
// Use initializeWithDriver() to provide a BrowserDriver.
browserExecutor = null
selfHealingLocator = SelfHealingLocator(
llmService = if (config.enableLLMHealing) llmService else null,
config = SelfHealingConfig(threshold = config.healingThreshold)
)
planner = TestActionPlanner(llmService)
}
/**
* Initialize the agent with a BrowserDriver.
*
* This creates a DriverBasedBrowserActionExecutor from the driver.
* You still need to provide a PageStateExtractor.
*
* @param driver The BrowserDriver to use for browser operations
* @param extractor The PageStateExtractor to use for page state extraction
*/
fun initializeWithDriver(
driver: BrowserDriver,
extractor: PageStateExtractor
) {
pageStateExtractor = extractor
browserExecutor = DriverBasedBrowserActionExecutor(
driver = driver,
config = BrowserExecutorConfig(
headless = config.headless,
viewportWidth = config.viewportWidth,
viewportHeight = config.viewportHeight,
defaultTimeoutMs = config.defaultTimeoutMs,
slowMotionMs = config.slowMotionMs
)
)
selfHealingLocator = SelfHealingLocator(
llmService = if (config.enableLLMHealing) llmService else null,
config = SelfHealingConfig(threshold = config.healingThreshold)
)
planner = TestActionPlanner(llmService)
}
override fun validateInput(input: Map): WebAgentInput {
val scenario = input["scenario"] as? TestScenario
val naturalLanguage = input["naturalLanguage"] as? String
val startUrl = input["startUrl"] as? String
if (scenario == null && naturalLanguage == null) {
throw IllegalArgumentException("Either 'scenario' or 'naturalLanguage' is required")
}
return WebAgentInput(
scenario = scenario,
naturalLanguage = naturalLanguage,
startUrl = startUrl ?: "about:blank"
)
}
override suspend fun execute(
input: WebAgentInput,
onProgress: (String) -> Unit
): ToolResult.AgentResult {
if (!isAvailable) {
return ToolResult.AgentResult(
success = false,
content = "E2E Testing Agent is not available on this platform"
)
}
try {
// Navigate to start URL
onProgress("Navigating to ${input.startUrl}...")
browserExecutor?.navigateTo(input.startUrl)
delay(1000) // Wait for page load
// Get or generate scenario
val scenario = input.scenario ?: run {
onProgress("Generating test scenario from description...")
val pageState = pageStateExtractor?.extractPageState()
?: return ToolResult.AgentResult(
success = false,
content = "Failed to extract page state"
)
planner?.generateScenario(
input.naturalLanguage ?: "",
input.startUrl,
pageState
) ?: return ToolResult.AgentResult(
success = false,
content = "Failed to generate scenario"
)
}
// Execute scenario
onProgress("Executing scenario: ${scenario.name}")
val result = executeScenario(scenario, onProgress)
return ToolResult.AgentResult(
success = result.status == TestStatus.PASSED,
content = formatTestResult(result),
metadata = mapOf(
"scenarioId" to result.scenarioId,
"scenarioName" to result.scenarioName,
"passedSteps" to result.passedSteps.toString(),
"failedSteps" to result.failedSteps.toString(),
"selfHealedSteps" to result.selfHealedSteps.toString(),
"durationMs" to result.totalDurationMs.toString()
)
)
} catch (e: Exception) {
return ToolResult.AgentResult(
success = false,
content = e.message ?: "Unknown error"
)
}
}
/**
* Format test result for output
*/
private fun formatTestResult(result: E2ETestResult): String {
return buildString {
appendLine("Test ${if (result.status == TestStatus.PASSED) "PASSED" else "FAILED"}: ${result.scenarioName}")
appendLine("Steps: ${result.passedSteps}/${result.stepResults.size} passed")
appendLine("Duration: ${result.totalDurationMs}ms")
if (result.selfHealedSteps > 0) {
appendLine("Self-healed: ${result.selfHealedSteps} steps")
}
if (result.errorSummary != null) {
appendLine("Error: ${result.errorSummary}")
}
}
}
/**
* Execute a test scenario
*/
private suspend fun executeScenario(
scenario: TestScenario,
onProgress: (String) -> Unit
): E2ETestResult {
val startTime = currentTimeMillis()
val stepResults = mutableListOf()
var memory = _root_ide_package_.cc.unitmesh.agent.webagent.TestMemory.Companion.empty(config.maxMemorySize)
// Execute setup actions
scenario.setup.forEach { action ->
browserExecutor?.execute(action, ActionExecutionContext(emptyMap()))
}
// Execute each step
for ((index, step) in scenario.steps.withIndex()) {
onProgress("Step ${index + 1}/${scenario.steps.size}: ${step.description}")
val stepResult = executeStep(step, memory)
stepResults.add(stepResult)
// Update memory
memory = memory.withAction(
_root_ide_package_.cc.unitmesh.agent.webagent.ActionRecord(
actionType = step.action::class.simpleName ?: "Unknown",
targetId = getTargetId(step.action),
timestamp = currentTimeMillis(),
success = stepResult.success,
description = step.description
)
)
if (!stepResult.success && !step.continueOnFailure) {
onProgress("Step failed: ${stepResult.error}")
break
}
// Slow motion delay
if (config.slowMotionMs > 0) {
delay(config.slowMotionMs)
}
}
// Execute teardown actions
scenario.teardown.forEach { action ->
browserExecutor?.execute(action, ActionExecutionContext(emptyMap()))
}
val endTime = currentTimeMillis()
val passed = stepResults.count { it.success }
val failed = stepResults.count { !it.success }
val selfHealed = stepResults.count { it.selfHealed }
return E2ETestResult(
scenarioId = scenario.id,
scenarioName = scenario.name,
status = if (failed == 0) TestStatus.PASSED else TestStatus.FAILED,
stepResults = stepResults,
totalDurationMs = endTime - startTime,
passedSteps = passed,
failedSteps = failed,
skippedSteps = scenario.steps.size - stepResults.size,
selfHealedSteps = selfHealed,
errorSummary = if (failed > 0) "Failed $failed of ${stepResults.size} steps" else null,
finalUrl = pageStateExtractor?.getCurrentUrl(),
finalTitle = pageStateExtractor?.getPageTitle(),
startedAt = startTime,
completedAt = endTime
)
}
/**
* Execute a single test step with self-healing
*/
private suspend fun executeStep(step: TestStep, memory: cc.unitmesh.agent.webagent.TestMemory): StepResult {
val startTime = currentTimeMillis()
var retries = 0
var lastError: String? = null
var selfHealed = false
var healedSelector: String? = null
while (retries <= step.retryCount) {
try {
// Get current page state
val pageState = pageStateExtractor?.extractPageState()
?: return StepResult(
stepId = step.id,
success = false,
durationMs = currentTimeMillis() - startTime,
error = "Failed to extract page state"
)
// Build execution context
val tagMapping = pageState.actionableElements.associateBy { it.tagId }
val context = ActionExecutionContext(
tagMapping = tagMapping,
selfHealingLocator = selfHealingLocator,
timeoutMs = step.timeoutMs ?: config.defaultTimeoutMs
)
// Execute action
val result = browserExecutor?.execute(step.action, context)
?: return StepResult(
stepId = step.id,
success = false,
durationMs = currentTimeMillis() - startTime,
error = "Browser executor not available"
)
if (result.success) {
return StepResult(
stepId = step.id,
success = true,
durationMs = currentTimeMillis() - startTime,
selfHealed = result.selfHealed || selfHealed,
healedSelector = result.healedSelector ?: healedSelector,
retriesAttempted = retries
)
}
lastError = result.error
// Try self-healing if enabled
if (config.enableSelfHealing && step.elementFingerprint != null) {
val healingResult = selfHealingLocator?.healWithAlgorithm(
step.elementFingerprint.selector,
step.elementFingerprint,
pageState.actionableElements
)
if (healingResult != null) {
selfHealed = true
healedSelector = healingResult.healedSelector
// Retry with healed selector would happen on next iteration
}
}
} catch (e: Exception) {
lastError = e.message
}
retries++
if (retries <= step.retryCount) {
delay(500) // Brief delay before retry
}
}
return StepResult(
stepId = step.id,
success = false,
durationMs = currentTimeMillis() - startTime,
error = lastError ?: "Unknown error",
selfHealed = selfHealed,
healedSelector = healedSelector,
retriesAttempted = retries
)
}
/**
* Get target ID from action (if applicable)
*/
private fun getTargetId(action: TestAction): Int? {
return when (action) {
is TestAction.Click -> action.targetId
is TestAction.Type -> action.targetId
is TestAction.Hover -> action.targetId
is TestAction.Scroll -> action.targetId
is TestAction.Assert -> action.targetId
is TestAction.Select -> action.targetId
is TestAction.UploadFile -> action.targetId
else -> null
}
}
override fun formatOutput(output: ToolResult.AgentResult): String {
return output.content
}
/**
* Cleanup resources
*/
fun close() {
pageStateExtractor?.close()
browserExecutor?.close()
}
}
/**
* Input for Web Agent
*/
@Serializable
data class WebAgentInput(
/**
* Predefined test scenario (optional)
*/
val scenario: TestScenario? = null,
/**
* Natural language description to generate scenario (optional)
*/
val naturalLanguage: String? = null,
/**
* Starting URL for the automation
*/
val startUrl: String
)
/**
* Platform-specific time function
*/
fun currentTimeMillis(): Long {
return kotlinx.datetime.Clock.System.now().toEpochMilliseconds()
}