package cc.unitmesh.server.cli import cc.unitmesh.agent.config.McpToolConfigService import cc.unitmesh.agent.config.ToolConfigFile import cc.unitmesh.agent.document.DocumentAgent import cc.unitmesh.agent.document.DocumentAgentMode import cc.unitmesh.agent.document.DocumentTask import cc.unitmesh.agent.render.CodingAgentRenderer import cc.unitmesh.devins.db.DocumentIndexDatabaseRepository import cc.unitmesh.devins.db.DocumentIndexRecord import cc.unitmesh.devins.db.DocumentIndexRepository import cc.unitmesh.devins.document.* import cc.unitmesh.llm.LLMService import cc.unitmesh.llm.LLMProviderType import cc.unitmesh.llm.ModelConfig import cc.unitmesh.llm.compression.TokenInfo import com.charleskorn.kaml.Yaml import kotlinx.coroutines.runBlocking import kotlinx.serialization.Serializable import java.io.File import java.security.MessageDigest /** * JVM CLI for testing DocumentAgent with PPTX, DOCX, PDF files and Product Feature Tree generation * * Usage: * ```bash * # Document query mode (default) * ./gradlew :mpp-ui:runDocumentCli -PdocProjectPath=/path/to/docs -PdocQuery="What is this about?" [-PdocPath=specific.pptx] * * # Feature tree mode - generate product feature tree from source code * ./gradlew :mpp-ui:runDocumentCli -PdocProjectPath=/path/to/project -PdocMode=feature-tree [-PdocLanguage=ZH] * ``` */ object DocumentCli { @JvmStatic fun main(args: Array) { println("=".repeat(80)) println("AutoDev Document CLI (JVM - Tika Support)") println("=".repeat(80)) // Parse arguments val projectPath = System.getProperty("projectPath") ?: args.getOrNull(0) ?: run { System.err.println("Usage: -PdocProjectPath= -PdocQuery= [-PdocPath=] [-PdocMode=feature-tree] [-PdocLanguage=EN|ZH]") return } // Check mode - support both positional args and system properties val mode = System.getProperty("mode") ?: args.lastOrNull()?.takeIf { it.equals("feature-tree", ignoreCase = true) || it.equals("featuretree", ignoreCase = true) || it == "query" } ?: "query" val language = System.getProperty("language") ?: "EN" val isFeatureTreeMode = mode.equals("feature-tree", ignoreCase = true) || mode.equals("featuretree", ignoreCase = true) val query = if (isFeatureTreeMode) { "Generate product feature tree for this codebase" } else { System.getProperty("query") ?: args.getOrNull(1) ?: run { System.err.println("Usage: -PdocProjectPath= -PdocQuery= [-PdocPath=] [-PdocMode=feature-tree] [-PdocLanguage=EN|ZH]") return } } val documentPath = System.getProperty("documentPath") ?: args.getOrNull(2) println("šŸ“‚ Project Path: $projectPath") if (isFeatureTreeMode) { println("🌳 Mode: Feature Tree Generation") println("🌐 Language: $language") } else { println("ā“ Query: $query") if (documentPath != null) { println("šŸ“„ Document: $documentPath") } } println() runBlocking { try { // Initialize platform parsers println("šŸ”§ Initializing document parsers (Tika)...") DocumentRegistry.initializePlatformParsers() println("āœ… Parsers initialized") println() // Scan and register documents val projectDir = File(projectPath).absoluteFile if (!projectDir.exists()) { System.err.println("āŒ Project path does not exist: $projectPath") return@runBlocking } val startTime = System.currentTimeMillis() val documents = scanDocuments(projectDir) val scanTime = System.currentTimeMillis() - startTime println("šŸ“– Found ${documents.size} documents (${scanTime}ms)") documents.take(10).forEach { doc -> val relativePath = doc.relativeTo(projectDir).path val size = doc.length() / 1024 println(" • $relativePath (${size}KB)") } if (documents.size > 10) { println(" ... and ${documents.size - 10} more") } println() if (documents.isEmpty()) { println("āš ļø No documents found in: $projectPath") return@runBlocking } // Initialize DocumentIndexRepository val indexRepository = DocumentIndexDatabaseRepository.getInstance() println("šŸ“š Using document index: ~/.autodev/autodev.db") println() // Register documents with caching println("šŸ“ Registering documents...") var registeredCount = 0 var cachedCount = 0 val registerStartTime = System.currentTimeMillis() for (doc in documents) { val relativePath = doc.relativeTo(projectDir).path val formatType = DocumentParserFactory.detectFormat(doc.name) val result = registerDocumentWithCache(doc, relativePath, indexRepository) val typeSymbol = when (formatType) { DocumentFormatType.MARKDOWN -> "šŸ“" DocumentFormatType.PDF -> "šŸ“•" DocumentFormatType.DOCX -> "šŸ“˜" DocumentFormatType.PLAIN_TEXT -> "šŸ“„" DocumentFormatType.SOURCE_CODE -> "šŸ’»" else -> "šŸ“„" } when (result) { is RegisterResult.Success -> { println(" $typeSymbol $relativePath") registeredCount++ } is RegisterResult.FromCache -> { println(" $typeSymbol $relativePath (cached)") registeredCount++ cachedCount++ } is RegisterResult.Failed -> { println(" āœ— $relativePath (${result.reason})") } } } val registerTime = System.currentTimeMillis() - registerStartTime println("āœ… Registered $registeredCount/${documents.size} documents (${registerTime}ms)") if (cachedCount > 0) { println(" šŸ“¦ $cachedCount from cache, ${registeredCount - cachedCount} newly parsed") } println() // Show registered documents summary val registeredPaths = DocumentRegistry.getRegisteredPaths() println("šŸ“š Document Registry Summary:") println(" Total registered: ${registeredPaths.size}") // Count by type val byType = registeredPaths.groupBy { path -> val ext = File(path).extension.lowercase() when (ext) { "md" -> "Markdown" "pdf" -> "PDF" "docx", "doc" -> "Word" "pptx", "ppt" -> "PowerPoint" "java", "kt", "kts" -> "JVM Source" "js", "ts", "tsx" -> "JS/TS Source" "py" -> "Python Source" "go" -> "Go Source" "rs" -> "Rust Source" "cs" -> "C# Source" else -> "Other" } } byType.forEach { (type, paths) -> println(" - $type: ${paths.size}") } println() // Register index provider to bridge DocumentRegistry with DocumentIndexService println("šŸ”— Registering document index provider...") val provider = cc.unitmesh.devins.service.DocumentIndexServiceProvider(indexRepository) DocumentRegistry.setIndexProvider(provider) println("āœ… Index provider registered") println() // Create DocumentAgent println("🧠 Creating DocumentAgent...") // Load configuration from ~/.autodev/config.yaml val configFile = File(System.getProperty("user.home"), ".autodev/config.yaml") if (!configFile.exists()) { System.err.println("āŒ Configuration file not found: ${configFile.absolutePath}") System.err.println(" Please create ~/.autodev/config.yaml with your LLM configuration") return@runBlocking } val yamlContent = configFile.readText() val yaml = Yaml(configuration = com.charleskorn.kaml.YamlConfiguration(strictMode = false)) val config = yaml.decodeFromString(AutoDevConfig.serializer(), yamlContent) val activeName = config.active val activeConfig = config.configs.find { it.name == activeName } if (activeConfig == null) { System.err.println("āŒ Active configuration '$activeName' not found in config.yaml") System.err.println(" Available configs: ${config.configs.map { it.name }.joinToString(", ")}") return@runBlocking } println("šŸ“ Using config: ${activeConfig.name} (${activeConfig.provider}/${activeConfig.model})") // Convert provider string to LLMProviderType val providerType = when (activeConfig.provider.lowercase()) { "openai" -> LLMProviderType.OPENAI "anthropic" -> LLMProviderType.ANTHROPIC "google" -> LLMProviderType.GOOGLE "deepseek" -> LLMProviderType.DEEPSEEK "ollama" -> LLMProviderType.OLLAMA "openrouter" -> LLMProviderType.OPENROUTER "glm" -> LLMProviderType.GLM "qwen" -> LLMProviderType.QWEN "kimi" -> LLMProviderType.KIMI else -> LLMProviderType.CUSTOM_OPENAI_BASE } val llmService = LLMService( ModelConfig( provider = providerType, modelName = activeConfig.model, apiKey = activeConfig.apiKey, temperature = activeConfig.temperature ?: 0.7, maxTokens = activeConfig.maxTokens ?: 4096, baseUrl = activeConfig.baseUrl ?: "" ) ) val renderer = ConsoleRenderer() val mcpConfigService = McpToolConfigService(ToolConfigFile()) val dummyParser = DocumentParserFactory.createParserForFile("dummy.md")!! val agent = DocumentAgent( llmService = llmService, parserService = dummyParser, renderer = renderer, fileSystem = cc.unitmesh.agent.tool.filesystem.DefaultToolFileSystem(projectPath), shellExecutor = null, mcpToolConfigService = mcpConfigService, enableLLMStreaming = true ) println("āœ… Agent created") println() // Execute query if (isFeatureTreeMode) { println("🌳 Generating product feature tree...") } else { println("šŸ” Executing query...") } println() val queryStartTime = System.currentTimeMillis() val agentMode = if (isFeatureTreeMode) DocumentAgentMode.FEATURE_TREE else DocumentAgentMode.DOCUMENT_QUERY val result = agent.execute( DocumentTask( query = query, documentPath = documentPath, mode = agentMode, language = language ), onProgress = { } ) val queryTime = System.currentTimeMillis() - queryStartTime println() println("=".repeat(80)) println("šŸ“Š Result:") println("=".repeat(80)) println(result.content) println() if (result.success) { println("āœ… Query completed successfully") println("ā±ļø Query time: ${queryTime}ms") println("ā±ļø Total time: ${System.currentTimeMillis() - startTime}ms") } else { println("āŒ Query failed") } } catch (e: Exception) { System.err.println("āŒ Error: ${e.message}") e.printStackTrace() } } } /** * Scan directory for documents */ private fun scanDocuments( dir: File, extensions: List = listOf( ".md", ".pdf", ".docx", ".pptx", ".txt", // Documents ".java", ".kt", ".kts", // JVM source code ".js", ".ts", ".tsx", // JavaScript/TypeScript ".py", ".go", ".rs", ".cs" // Other languages ) ): List { val documents = mutableListOf() val skipDirs = setOf("node_modules", ".git", "build", "dist", "target", ".gradle", "bin", ".idea", "out", "libs", "generated") fun scanRecursive(current: File) { if (!current.canRead()) return if (current.isDirectory) { if (skipDirs.contains(current.name) || current.name.startsWith(".")) { return } current.listFiles()?.forEach { scanRecursive(it) } } else if (current.isFile) { val ext = "." + current.extension.lowercase() if (extensions.contains(ext)) { documents.add(current) } } } scanRecursive(dir) return documents } /** * Register result types */ private sealed class RegisterResult { data class Success(val fromCache: Boolean = false) : RegisterResult() data class FromCache(val record: DocumentIndexRecord) : RegisterResult() data class Failed(val reason: String) : RegisterResult() } /** * Register a document with caching support */ private suspend fun registerDocumentWithCache( file: File, relativePath: String, indexRepository: DocumentIndexRepository ): RegisterResult { return try { // Get parser for this file type val parser = DocumentParserFactory.createParserForFile(file.name) ?: return RegisterResult.Failed("no parser") // Calculate file hash val fileHash = calculateFileHash(file) val lastModified = file.lastModified() // Check cache - if hash matches and content exists, use cached content val cachedRecord = indexRepository.get(relativePath) if (cachedRecord != null && cachedRecord.hash == fileHash && cachedRecord.status == "success" && cachedRecord.content != null) { // File unchanged - restore from cached extracted content // This avoids re-parsing large binary files (PPTX, PDF) val formatType = DocumentParserFactory.detectFormat(file.name) ?: DocumentFormatType.PLAIN_TEXT val metadata = DocumentMetadata( lastModified = lastModified, fileSize = file.length(), formatType = formatType ) val documentFile = DocumentFile( name = file.name, path = relativePath, metadata = metadata ) // Parse from cached extracted content (not original binary!) val parsedDoc = parser.parse(documentFile, cachedRecord.content) DocumentRegistry.registerDocument(relativePath, parsedDoc, parser) // Return FromCache - file unchanged, restored from DB return RegisterResult.FromCache(cachedRecord) } // Not in cache or outdated, parse fresh val formatType = DocumentParserFactory.detectFormat(file.name) ?: DocumentFormatType.PLAIN_TEXT // Read file content val content = when (formatType) { DocumentFormatType.MARKDOWN, DocumentFormatType.PLAIN_TEXT, DocumentFormatType.SOURCE_CODE -> { file.readText() } else -> { // For binary formats (PDF, DOCX, PPTX), read as bytes and convert to ISO-8859-1 val bytes = file.readBytes() String(bytes, Charsets.ISO_8859_1) } } // Create DocumentFile val metadata = DocumentMetadata( lastModified = lastModified, fileSize = file.length(), formatType = formatType ) val documentFile = DocumentFile( name = file.name, path = relativePath, metadata = metadata ) // Parse document val parsedDoc = parser.parse(documentFile, content) // Register in registry DocumentRegistry.registerDocument(relativePath, parsedDoc, parser) // Save to cache: store extracted text (not original binary) // For PDF/PPTX: Tika extracts ~50KB text from 10MB binary // For Markdown: store original text val extractedContent = parser.getDocumentContent() ?: content val indexRecord = DocumentIndexRecord( path = relativePath, hash = fileHash, lastModified = lastModified, status = "success", content = extractedContent, // Store extracted text (not binary!) error = null, indexedAt = System.currentTimeMillis() ) indexRepository.save(indexRecord) RegisterResult.Success(fromCache = false) } catch (e: Exception) { // Save error to cache (to avoid re-parsing failed files) try { val fileHash = calculateFileHash(file) val indexRecord = DocumentIndexRecord( path = relativePath, hash = fileHash, lastModified = file.lastModified(), status = "failed", content = null, // No content on failure error = e.message, indexedAt = System.currentTimeMillis() ) indexRepository.save(indexRecord) } catch (cacheError: Exception) { // Ignore cache errors } RegisterResult.Failed(e.message ?: "unknown error") } } /** * Calculate SHA-256 hash of file */ private fun calculateFileHash(file: File): String { val digest = MessageDigest.getInstance("SHA-256") val bytes = file.readBytes() val hashBytes = digest.digest(bytes) return hashBytes.joinToString("") { "%02x".format(it) } } } /** * Simple config data classes for YAML parsing */ @Serializable data class AutoDevConfig( val active: String, val configs: List ) @Serializable data class LLMConfigEntry( val name: String, val provider: String, val apiKey: String, val model: String, val baseUrl: String? = null, val temperature: Double? = null, val maxTokens: Int? = null ) /** * Console renderer for CLI output */ class ConsoleRenderer : CodingAgentRenderer { // Thinking block state private var isInThinkBlock = false private val reasoningBuffer = StringBuilder() private var thinkingLineCount = 0 private val maxThinkingLines = 3 private val thinkingLines = mutableListOf() private var currentThinkingLine = StringBuilder() override fun renderIterationHeader(current: Int, max: Int) { println("\n━━━ Iteration $current/$max ━━━") } override fun renderLLMResponseStart() { println("šŸ’­ ") // Reset thinking state isInThinkBlock = false reasoningBuffer.clear() thinkingLines.clear() currentThinkingLine.clear() } override fun renderLLMResponseChunk(chunk: String) { reasoningBuffer.append(chunk) // Extract thinking content val extraction = extractThinkingContent(reasoningBuffer.toString()) // Render thinking content if present if (extraction.hasThinking) { val thinkContent = extraction.thinkingContent.toString() if (thinkContent.isNotEmpty()) { val wasInThinkBlock = isInThinkBlock isInThinkBlock = extraction.hasIncompleteThinkBlock renderThinkingChunk( thinkContent, isStart = !wasInThinkBlock && (extraction.hasCompleteThinkBlock || extraction.hasIncompleteThinkBlock), isEnd = extraction.hasCompleteThinkBlock && !extraction.hasIncompleteThinkBlock ) } } else if (isInThinkBlock && !extraction.hasIncompleteThinkBlock) { isInThinkBlock = false } // Output non-thinking content val cleanContent = extraction.contentWithoutThinking if (cleanContent.isNotEmpty()) { print(cleanContent) System.out.flush() } } override fun renderLLMResponseEnd() { println("\n") } override fun renderThinkingChunk(chunk: String, isStart: Boolean, isEnd: Boolean) { if (isStart) { thinkingLineCount = 0 thinkingLines.clear() currentThinkingLine.clear() print("\u001B[90m🧠 Thinking: ") } for (char in chunk) { if (char == '\n') { thinkingLines.add(currentThinkingLine.toString()) currentThinkingLine.clear() thinkingLineCount++ if (thinkingLines.size > maxThinkingLines) { thinkingLines.removeAt(0) } print("\r\u001B[K") print("\u001B[90m🧠 ${thinkingLines.lastOrNull() ?: ""}") } else { currentThinkingLine.append(char) print("\u001B[90m$char") } } System.out.flush() if (isEnd) { println("\u001B[0m") thinkingLines.clear() currentThinkingLine.clear() } } private fun extractThinkingContent(content: String): ThinkingExtractionResult { val result = ThinkingExtractionResult() var remaining = content val completeThinkPattern = Regex("([\\s\\S]*?)") val matches = completeThinkPattern.findAll(remaining) for (match in matches) { result.thinkingContent.append(match.groupValues[1]) result.hasCompleteThinkBlock = true } remaining = remaining.replace(completeThinkPattern, "") val openThinkIndex = remaining.lastIndexOf("") if (openThinkIndex != -1) { val closeThinkIndex = remaining.indexOf("", openThinkIndex) if (closeThinkIndex == -1) { val thinkContent = remaining.substring(openThinkIndex + 7) result.thinkingContent.append(thinkContent) result.hasIncompleteThinkBlock = true remaining = remaining.substring(0, openThinkIndex) } } val partialThinkPattern = Regex("<(?:t(?:h(?:i(?:n(?:k)?)?)?)?)?$| println(" āŽæ $line") } } } private fun formatCliParameters(params: String): String { val trimmed = params.trim() // Handle JSON format if (trimmed.startsWith("{") && trimmed.endsWith("}")) { val lines = mutableListOf() val jsonPattern = Regex(""""(\w+)"\s*:\s*("([^"]*)"|(\d+)|true|false|null)""") jsonPattern.findAll(trimmed).forEach { match -> val key = match.groups[1]?.value ?: "" val value = match.groups[3]?.value ?: match.groups[4]?.value ?: match.groups[2]?.value?.removeSurrounding("\"") ?: "" lines.add("$key = $value") } return if (lines.isNotEmpty()) lines.joinToString(", ") else params } return params } override fun renderToolResult( toolName: String, success: Boolean, output: String?, fullOutput: String?, metadata: Map ) { val statusSymbol = if (success) "āœ“" else "āœ—" val preview = (output ?: fullOutput ?: "").lines().take(3).joinToString(" ").take(100) println(" $statusSymbol ${if (preview.length < (output ?: fullOutput ?: "").length) "$preview..." else preview}") } override fun renderTaskComplete(executionTimeMs: Long, toolsUsedCount: Int) { val parts = mutableListOf() if (executionTimeMs > 0) { val seconds = executionTimeMs / 1000.0 val rounded = (seconds * 100).toLong() / 100.0 parts.add("${rounded}s") } if (toolsUsedCount > 0) { parts.add("$toolsUsedCount tools") } val info = if (parts.isNotEmpty()) " (${parts.joinToString(", ")})" else "" println("\nāœ“ Task marked as complete$info") } override fun renderFinalResult(success: Boolean, message: String, iterations: Int) { val symbol = if (success) "āœ…" else "āŒ" println("\n$symbol Final result after $iterations iterations:") println(message) } override fun renderError(message: String) { System.err.println("āŒ Error: $message") } override fun renderRepeatWarning(toolName: String, count: Int) { println("āš ļø Warning: Tool '$toolName' called $count times") } override fun renderRecoveryAdvice(recoveryAdvice: String) { println("šŸ’” Recovery advice: $recoveryAdvice") } override fun updateTokenInfo(tokenInfo: TokenInfo) { // Skip token info in CLI } override fun renderUserConfirmationRequest(toolName: String, params: Map) { println("ā“ Confirmation required for: $toolName") println(" Params: $params") } }