diff --git a/cfspider-browser/src/services/ai.ts b/cfspider-browser/src/services/ai.ts index e8e6bcb..d418f0c 100644 --- a/cfspider-browser/src/services/ai.ts +++ b/cfspider-browser/src/services/ai.ts @@ -1,4 +1,5 @@ import { useStore } from '../store' +import { matchSkill, updateSkillLearning, type Skill } from './skills' const isElectron = typeof window !== 'undefined' && (window as any).electronAPI !== undefined @@ -1242,10 +1243,32 @@ export const aiTools = [ type: 'function', function: { name: 'scan_interactive_elements', - description: 'Scan and list all interactive elements on the page (buttons, links, inputs). Use to discover available actions.', + description: 'Scan and list all interactive elements on the page (buttons, links, inputs). Use to discover available actions. Returns indexed list for use with click_by_index.', parameters: { type: 'object', properties: {} } } }, + { + type: 'function', + function: { + name: 'click_by_index', + description: '通过索引点击元素。先调用 scan_interactive_elements 获取元素列表，然后使用此工具通过索引点击。', + parameters: { + type: 'object', + properties: { + category: { + type: 'string', + description: '元素类别: inputs, buttons, links, selects', + enum: ['inputs', 'buttons', 'links', 'selects'] + }, + index: { + type: 'number', + description: '元素索引号（从1开始）' + } + }, + required: ['category', 'index'] + } + } + }, { type: 'function', function: { @@ -1286,6 +1309,27 @@ export const aiTools = [ required: ['selector'] } } + }, + // ==================== 视频分析工具 ==================== + { + type: 'function', + function: { + name: 'summarize_video', + description: '总结页面上正在播放的视频内容。通过抽取关键帧进行视觉分析，生成视频摘要。需要视觉模型支持。', + parameters: { + type: 'object', + properties: { + frame_count: { + type: 'number', + description: '要分析的帧数（默认10帧，最多40帧）' + }, + focus: { + type: 'string', + description: '分析重点，如 "人物"、"产品"、"教程步骤" 等（可选）' + } + } + } + } } ] @@ -1555,19 +1599,46 @@ async function executeToolCall(name: string, args: Record): Pro // Common website domain mappings var domainMap = { 'jd': ['jd.com'], + '京东': ['jd.com'], 'taobao': ['taobao.com', 'tmall.com'], + '淘宝': ['taobao.com'], 'tmall': ['tmall.com'], + '天猫': ['tmall.com'], 'github': ['github.com'], 'amazon': ['amazon.com', 'amazon.cn'], + '亚马逊': ['amazon.cn', 'amazon.com'], 'google': ['google.com'], + '谷歌': ['google.com'], 'baidu': ['baidu.com'], + '百度': ['baidu.com'], 'bing': ['bing.com'], + '必应': ['bing.com'], 'microsoft': ['microsoft.com'], + '微软': ['microsoft.com'], 'apple': ['apple.com'], + '苹果': ['apple.com'], 'facebook': ['facebook.com'], 'twitter': ['twitter.com', 'x.com'], 'youtube': ['youtube.com'], - 'bilibili': ['bilibili.com'] + '油管': ['youtube.com'], + 'bilibili': ['bilibili.com'], + 'b站': ['bilibili.com'], + '哔哩哔哩': ['bilibili.com'], + '爱奇艺': ['iqiyi.com'], + 'iqiyi': ['iqiyi.com'], + '优酷': ['youku.com'], + 'youku': ['youku.com'], + '腾讯视频': ['v.qq.com'], + '芒果tv': ['mgtv.com'], + '抖音': ['douyin.com'], + 'douyin': ['douyin.com'], + '知乎': ['zhihu.com'], + 'zhihu': ['zhihu.com'], + '微博': ['weibo.com'], + 'weibo': ['weibo.com'], + '网易': ['163.com'], + '新浪': ['sina.com.cn'], + '搜狐': ['sohu.com'] }; // Find matching domain patterns @@ -1679,13 +1750,17 @@ async function executeToolCall(name: string, args: Record): Pro var badSubdomainPrefixes = ['home.', 'my.', 'user.', 'account.', 'login.', 'passport.', 'member.', 'profile.', 'center.', 'i.', 'u.', 'sso.', 'auth.']; // Also check for these keywords in the result text var badKeywords = ['个人中心', '我的订单', '我的账户', '账户设置', '登录', 'home.', '/home', '个人信息']; - // Skip these UI elements (Copilot, navigation tabs, images, etc.) + // Skip these UI elements (Copilot, navigation tabs, images, sidebar cards, etc.) var badUIElements = [ 'copilot', 'copilotsearch', 'bingcopilot', 'b_sydConvTab', 'sydneyToggle', 'ai生成', 'ai搜索', 'bing ai', '全部', '视频', '图片', '地图', '资讯', '更多', 'b_scopeList', 'b_header', 'b_algo_group', 'b_rich', - '/images/', 'images/search', 'image.baidu', 'images.google' + '/images/', 'images/search', 'image.baidu', 'images.google', + // 右侧信息卡（Wikipedia, 知识图谱等） + 'b_entityTP', 'b_sideBleed', 'b_context', 'b_overlay', 'b_pag', 'b_footer', + 'wikipedia', 'wikidata', 'wikimedia', 'youtube.com', 'youtu.be', + 'knowledge-panel', 'kno-', 'side-panel', 'sidePanel' ]; // 山寨/镜像网站黑名单 @@ -1803,7 +1878,7 @@ async function executeToolCall(name: string, args: Record): Pro } } - // Helper to check if element is a bad UI element (Copilot, nav tabs, etc.) + // Helper to check if element is a bad UI element (Copilot, nav tabs, sidebar, etc.) function isBadUIElement(el) { var href = (el.href || '').toLowerCase(); var className = (el.className || '').toLowerCase(); @@ -1825,8 +1900,17 @@ async function executeToolCall(name: string, args: Record): Pro } } - // Skip elements in header area (navigation) var rect = el.getBoundingClientRect(); + + // Skip elements in right sidebar area (x > 60% of viewport width) + // This catches Wikipedia info cards, knowledge panels, etc. + var viewportWidth = window.innerWidth; + if (rect.left > viewportWidth * 0.6 && rect.right > viewportWidth * 0.65) { + console.log('CFSpider: Skipping element in right sidebar:', href || el.textContent?.slice(0, 30)); + return true; + } + + // Skip elements in header area (navigation) if (rect.top < 180 && rect.top > 0) { // Check if it looks like a navigation item if (className.indexOf('scope') !== -1 || className.indexOf('nav') !== -1 || @@ -2588,6 +2672,27 @@ async function executeToolCall(name: string, args: Record): Pro case 'read_full_page': { if (!webview) return 'Error: Cannot access page' + + // 检查是否有视觉模型可用 + const { aiConfig: pageCfg } = store + const useBuiltInPage = pageCfg.useBuiltIn !== false && (!pageCfg.endpoint || !pageCfg.apiKey) + const hasVisionPage = useBuiltInPage ? !!BUILT_IN_AI.visionModel : !!pageCfg.visionModel + + // 如果没有视觉模型，使用 DOM 提取文本 + if (!hasVisionPage) { + try { + const textContent = await webview.executeJavaScript(` + (function() { + var main = document.querySelector('main, article, .content, .main, #content, #main, body'); + return main ? main.innerText.slice(0, 10000) : document.body.innerText.slice(0, 10000); + })() + `) + return '页面文本内容（DOM提取，无视觉分析）：\n\n' + textContent + } catch (e) { + return 'Error reading page: ' + e + } + } + try { const maxScrolls = (args.max_scrolls as number) || 10 const allContents: string[] = [] @@ -2684,6 +2789,16 @@ async function executeToolCall(name: string, args: Record): Pro case 'solve_captcha': { if (!webview) return 'Error: Cannot access page' + + // 检查是否有视觉模型可用 + const { aiConfig } = store + const useBuiltIn = aiConfig.useBuiltIn !== false && (!aiConfig.endpoint || !aiConfig.apiKey) + const hasVisionModel = useBuiltIn ? !!BUILT_IN_AI.visionModel : !!aiConfig.visionModel + + if (!hasVisionModel) { + return '当前为单模型模式，无法使用验证码识别功能。请在设置中配置视觉模型，或使用内置 AI 服务。' + } + try { const captchaType = args.captcha_type as string store.setCurrentModelType('vision') @@ -2878,6 +2993,16 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click case 'visual_click': { if (!webview) return 'Error: Cannot access page' + + // 检查是否有视觉模型可用 + const { aiConfig: aiCfg } = store + const useBuiltInVision = aiCfg.useBuiltIn !== false && (!aiCfg.endpoint || !aiCfg.apiKey) + const hasVision = useBuiltInVision ? !!BUILT_IN_AI.visionModel : !!aiCfg.visionModel + + if (!hasVision) { + return '当前为单模型模式，无法使用视觉点击功能。请尝试使用 click_text 或 click_element 替代。' + } + try { const description = args.description as string @@ -4268,6 +4393,116 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click } } + case 'click_by_index': { + if (!webview) return 'Error: Cannot access page' + const category = args.category as string + const index = args.index as number + + if (!category || !index) { + return 'Error: 需要提供 category（元素类别）和 index（索引号）' + } + + try { + const result = await webview.executeJavaScript(` + (function() { + var category = '${category}'; + var index = ${index} - 1; // 转换为0-based索引 + var elements = []; + + if (category === 'inputs') { + document.querySelectorAll('input, textarea').forEach(function(el) { + if (el.offsetWidth > 0 && el.offsetHeight > 0) { + elements.push(el); + } + }); + } else if (category === 'buttons') { + document.querySelectorAll('button, input[type="submit"], input[type="button"], [role="button"]').forEach(function(el) { + if (el.offsetWidth > 0 && el.offsetHeight > 0) { + elements.push(el); + } + }); + } else if (category === 'links') { + var linkCount = 0; + document.querySelectorAll('a[href]').forEach(function(el) { + if (linkCount < 15 && el.offsetWidth > 0) { + var text = (el.textContent || '').trim(); + if (text.length > 2) { + elements.push(el); + linkCount++; + } + } + }); + } else if (category === 'selects') { + document.querySelectorAll('select').forEach(function(el) { + if (el.offsetWidth > 0) { + elements.push(el); + } + }); + } + + if (index < 0 || index >= elements.length) { + return { success: false, error: '索引超出范围，该类别共有 ' + elements.length + ' 个元素' }; + } + + var el = elements[index]; + var rect = el.getBoundingClientRect(); + var text = (el.textContent || el.value || el.placeholder || '').slice(0, 50).trim(); + + // 添加高亮 + var h = document.createElement('div'); + h.id = 'cfspider-agent-highlight'; + h.style.cssText = 'position:fixed;pointer-events:none;z-index:2147483647;border:4px solid #22c55e;background:rgba(34,197,94,0.2);border-radius:6px;box-shadow:0 0 20px rgba(34,197,94,0.5);'; + h.style.left = (rect.left - 4) + 'px'; + h.style.top = (rect.top - 4) + 'px'; + h.style.width = (rect.width + 8) + 'px'; + h.style.height = (rect.height + 8) + 'px'; + document.body.appendChild(h); + + // 点击元素 + el.click(); + if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') { + el.focus(); + } + + setTimeout(function() { + var hh = document.getElementById('cfspider-agent-highlight'); + if (hh) hh.remove(); + }, 1000); + + return { + success: true, + tag: el.tagName, + text: text, + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2 + }; + })() + `) + + if (!result.success) { + return `点击失败: ${result.error}` + } + + // 显示虚拟鼠标动画 + const container = document.getElementById('browser-container') + if (container) { + const containerRect = container.getBoundingClientRect() + const x = result.x + containerRect.left + const y = result.y + containerRect.top + store.showMouse() + store.moveMouse(x, y, 400) + await new Promise(resolve => setTimeout(resolve, 450)) + store.clickMouse() + } + + await new Promise(resolve => setTimeout(resolve, 500)) + + return `已点击 ${category} 类别的第 ${index} 个元素: <${result.tag}> "${result.text}"` + } catch (e) { + return `点击失败: ${e}` + } + } + case 'get_page_content': { if (!webview) return 'Error: Cannot access page' try { @@ -4399,6 +4634,266 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click } } + // ==================== 视频分析工具 ==================== + case 'summarize_video': { + if (!webview) return 'Error: Cannot access page' + + // 检查视觉模型是否可用 + const { aiConfig: videoConfig } = store + const useBuiltInVideo = videoConfig.useBuiltIn !== false && (!videoConfig.endpoint || !videoConfig.apiKey) + const hasVisionForVideo = useBuiltInVideo ? !!BUILT_IN_AI.visionModel : !!videoConfig.visionModel + + if (!hasVisionForVideo) { + return '视频总结需要视觉模型支持。请在设置中配置视觉模型，或使用内置 AI 服务。' + } + + try { + const frameCount = Math.min(Math.max((args.frame_count as number) || 10, 1), 40) + const focus = (args.focus as string) || '' + + store.setCurrentModelType('vision') + console.log('[CFSpider] 开始视频分析，计划抽取', frameCount, '帧') + + // 检测页面上的视频元素 + const videoInfo = await webview.executeJavaScript(` + (function() { + // 查找视频元素 + var video = document.querySelector('video'); + if (!video) { + // 尝试查找 iframe 中的视频（如 YouTube） + var iframes = document.querySelectorAll('iframe'); + for (var i = 0; i < iframes.length; i++) { + var src = iframes[i].src || ''; + if (src.includes('youtube') || src.includes('bilibili') || src.includes('youku') || src.includes('iqiyi')) { + return { + type: 'iframe', + platform: src.includes('youtube') ? 'YouTube' : + src.includes('bilibili') ? 'Bilibili' : + src.includes('youku') ? '优酷' : + src.includes('iqiyi') ? '爱奇艺' : '视频平台', + src: src + }; + } + } + return null; + } + + return { + type: 'video', + duration: video.duration || 0, + currentTime: video.currentTime || 0, + paused: video.paused, + width: video.videoWidth, + height: video.videoHeight, + src: video.src || video.currentSrc + }; + })() + `) + + if (!videoInfo) { + store.setCurrentModelType(null) + return '页面上未找到视频元素。请确保视频正在播放或页面包含视频播放器。' + } + + // 如果是 iframe 嵌入的视频（如 YouTube），使用页面截图分析 + if (videoInfo.type === 'iframe') { + console.log('[CFSpider] 检测到嵌入式视频平台:', videoInfo.platform) + + // 对嵌入视频，直接截图分析当前画面（最多分析10帧，因为无法控制进度） + const iframeFrameCount = Math.min(frameCount, 10) + const frameAnalyses: string[] = [] + + for (let i = 0; i < iframeFrameCount; i++) { + const image = await webview.capturePage() + if (!image) continue + + const base64Image = image.toDataURL().replace(/^data:image\/\w+;base64,/, '') + + const response = await (window as any).electronAPI.aiChat({ + endpoint: BUILT_IN_AI.endpoint, + apiKey: getBuiltInKey(), + model: BUILT_IN_AI.visionModel, + messages: [{ + role: 'user', + content: [ + { + type: 'text', + text: `这是一个${videoInfo.platform}视频的第${i + 1}帧截图。请分析视频画面内容： +${focus ? `重点关注：${focus}` : ''} + +请描述： +1. 画面中的主要内容 +2. 视频类型（教程、娱乐、新闻等） +3. 关键信息或字幕内容（如有） + +简洁回答，50字以内。` + }, + { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } } + ] + }] + }) + + if (response.content) { + frameAnalyses.push(`帧${i + 1}: ${response.content}`) + } + + // 等待一小段时间再截下一帧 + if (i < iframeFrameCount - 1) { + await new Promise(resolve => setTimeout(resolve, 1500)) + } + } + + store.setCurrentModelType(null) + + if (frameAnalyses.length === 0) { + return `检测到${videoInfo.platform}视频，但无法分析画面内容。` + } + + // 生成总结 + let summary = `[${videoInfo.platform}视频分析] 共分析${frameAnalyses.length}帧\n\n` + summary += frameAnalyses.join('\n\n') + + if (frameAnalyses.length > 1) { + // 请求生成整体总结 + const summaryResponse = await (window as any).electronAPI.aiChat({ + endpoint: useBuiltInVideo ? BUILT_IN_AI.endpoint : videoConfig.endpoint, + apiKey: useBuiltInVideo ? getBuiltInKey() : videoConfig.apiKey, + model: useBuiltInVideo ? BUILT_IN_AI.model : videoConfig.model, + messages: [{ + role: 'user', + content: `根据以下视频帧分析，生成一个简短的视频总结（150字以内）：\n\n${frameAnalyses.join('\n')}` + }] + }) + + if (summaryResponse.content) { + summary += `\n\n[总结]\n${summaryResponse.content}` + } + } + + return summary + } + + // 原生 video 元素 - 通过控制播放进度抽取帧 + const duration = videoInfo.duration + if (duration <= 0) { + store.setCurrentModelType(null) + return '视频时长未知或为直播流，无法进行帧分析。' + } + + console.log('[CFSpider] 视频时长:', duration, '秒，抽取', frameCount, '帧') + + // 计算抽帧时间点（均匀分布） + const timePoints: number[] = [] + for (let i = 0; i < frameCount; i++) { + timePoints.push((duration / (frameCount + 1)) * (i + 1)) + } + + const frameAnalyses: string[] = [] + + // 批量分析帧（每5帧为一组） + const batchSize = 5 + for (let batch = 0; batch < Math.ceil(timePoints.length / batchSize); batch++) { + const batchStart = batch * batchSize + const batchEnd = Math.min(batchStart + batchSize, timePoints.length) + + for (let i = batchStart; i < batchEnd; i++) { + const targetTime = timePoints[i] + + // 跳转到指定时间点 + await webview.executeJavaScript(` + (function() { + var video = document.querySelector('video'); + if (video) { + video.currentTime = ${targetTime}; + video.pause(); + } + })() + `) + + // 等待视频加载该帧 + await new Promise(resolve => setTimeout(resolve, 300)) + + // 截图 + const image = await webview.capturePage() + if (!image) continue + + const base64Image = image.toDataURL().replace(/^data:image\/\w+;base64,/, '') + const timeStr = `${Math.floor(targetTime / 60)}:${String(Math.floor(targetTime % 60)).padStart(2, '0')}` + + // 分析帧内容 + const response = await (window as any).electronAPI.aiChat({ + endpoint: BUILT_IN_AI.endpoint, + apiKey: getBuiltInKey(), + model: BUILT_IN_AI.visionModel, + messages: [{ + role: 'user', + content: [ + { + type: 'text', + text: `这是视频在 ${timeStr} 时刻的画面。请简短描述画面内容（30字以内）。${focus ? `重点关注：${focus}` : ''}` + }, + { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } } + ] + }] + }) + + if (response.content) { + frameAnalyses.push(`[${timeStr}] ${response.content}`) + } + } + + // 每批次之间短暂休息 + if (batch < Math.ceil(timePoints.length / batchSize) - 1) { + await new Promise(resolve => setTimeout(resolve, 200)) + } + } + + // 恢复视频播放 + await webview.executeJavaScript(` + (function() { + var video = document.querySelector('video'); + if (video) video.play(); + })() + `) + + store.setCurrentModelType(null) + + if (frameAnalyses.length === 0) { + return '视频帧分析失败，无法获取有效信息。' + } + + // 生成视频总结 + const durationMin = Math.floor(duration / 60) + const durationSec = Math.floor(duration % 60) + let summary = `[视频分析] 时长: ${durationMin}分${durationSec}秒 | 分析帧数: ${frameAnalyses.length}\n\n` + summary += '关键帧内容:\n' + summary += frameAnalyses.join('\n') + + // 生成整体总结 + if (frameAnalyses.length > 2) { + const summaryResponse = await (window as any).electronAPI.aiChat({ + endpoint: useBuiltInVideo ? BUILT_IN_AI.endpoint : videoConfig.endpoint, + apiKey: useBuiltInVideo ? getBuiltInKey() : videoConfig.apiKey, + model: useBuiltInVideo ? BUILT_IN_AI.model : videoConfig.model, + messages: [{ + role: 'user', + content: `根据以下视频帧分析，生成一个详细的视频总结（200字以内）：\n\n${frameAnalyses.join('\n')}` + }] + }) + + if (summaryResponse.content) { + summary += `\n\n[总结]\n${summaryResponse.content}` + } + } + + return summary + } catch (e) { + store.setCurrentModelType(null) + console.error('[CFSpider] 视频分析失败:', e) + return `视频分析失败: ${e}` + } + } + default: return 'Unknown tool: ' + name } @@ -4431,9 +4926,9 @@ const systemPrompt = `你是 CFspider 智能浏览器自动化助手，由 viole 对于这些情况，只需用中文自然回复，不要调用任何工具。 示例： -- 用户："你好" -> 回复："嗨~你好！我是 CFspider 智能浏览器助手，有什么可以帮你的吗？" -- 用户："你是谁" -> 回复："我是 CFspider 智能浏览器 AI 助手，由 violetteam 团队开发，来自 cfspider 项目。我可以帮你自动化浏览器操作，比如搜索、点击、导航网站等，交给我就行~" -- 用户："谢谢" -> 回复："不客气呀！有需要随时叫我~" +- 用户："你好" -> 回复："你好，我是 CFspider 智能浏览器助手，有什么可以帮你的吗？" +- 用户："你是谁" -> 回复："你好，我是 CFspider 智能浏览器 AI 助手。{{MODEL_INTRODUCTION}} CFspider 工具由 violetteam 团队开发。我可以帮你自动化浏览器操作，比如搜索、点击、导航网站等。" +- 用户："谢谢" -> 回复："不客气，有需要随时叫我。" ### 以下情况使用工具： - 打开网站："打开京东"、"去 GitHub" @@ -4843,10 +5338,54 @@ export async function sendAIMessage(content: string, useTools: boolean = true) { try { // 构建聊天历史，包含工具调用信息以便 AI 记住之前的操作 + + // 动态生成模型名介绍 + const toolModelName = useBuiltIn + ? BUILT_IN_AI.model.split('/').pop() || 'DeepSeek-V3' + : (effectiveConfig.model || '未配置') + const visionModelName = useBuiltIn + ? BUILT_IN_AI.visionModel?.split('/').pop() || '' + : (aiConfig.visionModel?.split('/').pop() || '') + + // 提取模型开发团队（从模型名推断） + const getModelTeam = (modelName: string): string => { + const name = modelName.toLowerCase() + if (name.includes('deepseek')) return 'DeepSeek' + if (name.includes('qwen')) return '阿里云通义千问' + if (name.includes('glm') || name.includes('chatglm')) return '智谱 AI' + if (name.includes('gpt')) return 'OpenAI' + if (name.includes('claude')) return 'Anthropic' + if (name.includes('gemini')) return 'Google' + if (name.includes('llama')) return 'Meta' + if (name.includes('mistral')) return 'Mistral AI' + if (name.includes('yi')) return '零一万物' + if (name.includes('baichuan')) return '百川智能' + if (name.includes('moonshot') || name.includes('kimi')) return 'Moonshot AI' + return '' + } + + const toolModelTeam = getModelTeam(toolModelName) + const visionModelTeam = visionModelName ? getModelTeam(visionModelName) : '' + + let modelIntroduction = '' + if (modelMode === 'dual' && visionModelName) { + const toolTeamStr = toolModelTeam ? `（由 ${toolModelTeam} 开发）` : '' + const visionTeamStr = visionModelTeam ? `（由 ${visionModelTeam} 开发）` : '' + modelIntroduction = `目前由两个模型共同驱动：${toolModelName}${toolTeamStr}负责文本理解和工具调用，${visionModelName}${visionTeamStr}负责页面视觉分析。` + } else { + const teamStr = toolModelTeam ? `（由 ${toolModelTeam} 开发）` : '' + modelIntroduction = `目前由 ${toolModelName} 模型${teamStr}驱动。` + } + + // 替换系统提示中的模型名占位符 + const dynamicSystemPrompt = systemPrompt + .replace(/\{\{TOOL_MODEL_NAME\}\}/g, toolModelName) + .replace(/\{\{MODEL_INTRODUCTION\}\}/g, modelIntroduction) + // 如果有 OCR 页面分析结果，添加到系统提示词中 const enhancedSystemPrompt = pageContext - ? `${systemPrompt}\n\n## 当前页面分析结果（由视觉模型提供）\n\n${pageContext}\n\n请根据以上页面分析结果来决定下一步操作。` - : systemPrompt + ? `${dynamicSystemPrompt}\n\n## 当前页面分析结果（由视觉模型提供）\n\n${pageContext}\n\n请根据以上页面分析结果来决定下一步操作。` + : dynamicSystemPrompt const chatHistory: Array<{ role: string; content?: string; tool_calls?: any[]; tool_call_id?: string; name?: string }> = [ { role: 'system', content: enhancedSystemPrompt } @@ -4883,6 +5422,28 @@ export async function sendAIMessage(content: string, useTools: boolean = true) { // 添加当前用户消息 chatHistory.push({ role: 'user', content }) + // 尝试匹配技能，提供给 AI 作为参考 + let matchedSkill: Skill | null = null + try { + const webview = document.querySelector('webview') as any + if (webview) { + const currentUrl = await webview.executeJavaScript('window.location.href') as string + const currentDomain = new URL(currentUrl).hostname + matchedSkill = await matchSkill(content, currentDomain) + if (matchedSkill) { + console.log('[CFSpider] 匹配到技能:', matchedSkill.name, '成功率:', matchedSkill.successRate) + // 将技能信息添加到聊天历史作为系统提示 + const skillHint = `[技能提示] 匹配到「${matchedSkill.name}」技能（成功率: ${matchedSkill.successRate}%） +触发词: ${matchedSkill.triggers.join(', ')} +操作步骤: ${matchedSkill.steps.map((s, i) => `${i+1}. ${s.action}${s.target ? `: ${s.target}` : ''}`).join(' -> ')} +${matchedSkill.learnedPatterns.length > 0 ? `学习到的模式: ${matchedSkill.learnedPatterns.slice(0, 3).map(p => p.pattern).join(', ')}` : ''}` + chatHistory.push({ role: 'system', content: skillHint }) + } + } + } catch (e) { + console.error('[CFSpider] 技能匹配失败:', e) + } + let iteration = 0 const maxIterations = 30 @@ -4971,7 +5532,8 @@ export async function sendAIMessage(content: string, useTools: boolean = true) { updateLastMessageWithToolCalls('', toolCallHistory) // 如果操作失败，触发紧张模式和语气词反应 - if (result.includes('Error') || result.includes('失败') || result.includes('not found') || result.includes('Cannot')) { + const operationFailed = result.includes('Error') || result.includes('失败') || result.includes('not found') || result.includes('Cannot') + if (operationFailed) { // 触发紧张乱动 store.panicMouse(1500) @@ -4984,6 +5546,23 @@ export async function sendAIMessage(content: string, useTools: boolean = true) { // 等待紧张动画完成 await new Promise(resolve => setTimeout(resolve, 1200)) } + + // 更新技能学习（如果匹配到了技能） + if (matchedSkill) { + try { + await updateSkillLearning(matchedSkill.id, !operationFailed, { + pattern: `${funcName}:${JSON.stringify(funcArgs).slice(0, 50)}`, + selector: funcArgs.selector || funcArgs.text || funcArgs.target, + confidence: operationFailed ? 30 : 70, + successCount: operationFailed ? 0 : 1, + failureCount: operationFailed ? 1 : 0, + lastUsed: Date.now(), + examples: operationFailed ? [] : [result.slice(0, 100)] + }) + } catch (e) { + console.error('[CFSpider] 技能学习更新失败:', e) + } + } chatHistory.push({ role: 'assistant', diff --git a/cfspider-browser/src/services/builtinSkills.ts b/cfspider-browser/src/services/builtinSkills.ts new file mode 100644 index 0000000..2e195cb --- /dev/null +++ b/cfspider-browser/src/services/builtinSkills.ts @@ -0,0 +1,272 @@ +/** + * 内置技能定义 + * 从 Markdown 文件解析而来 + */ +import { Skill } from './skills' + +export const BUILT_IN_SKILLS: Skill[] = [ + // 必应搜索技能 + { + id: 'bing-search', + name: '必应搜索', + description: '在必应搜索引擎上搜索关键词', + triggers: ['搜索', '查找', '找一下', 'search', '查', '百度一下'], + domains: ['bing.com', 'cn.bing.com'], + steps: [ + { + action: 'scan', + }, + { + action: 'input', + target: '#sb_form_q, input[name="q"], textarea[name="q"]', + value: '{query}', + fallbacks: ['input[type="search"]', '.search-box input'] + }, + { + action: 'click', + target: '#sb_form_go, #search_icon, button[type="submit"]', + fallbacks: ['按回车键', 'form button'], + optional: true + }, + { + action: 'wait', + value: '1500', + successIndicator: 'URL包含 /search?q=' + } + ], + successRate: 95, + usageCount: 0, + lastUsed: 0, + learnedPatterns: [], + isBuiltIn: true, + version: 1 + }, + + // 点击搜索结果技能 + { + id: 'click-search-result', + name: '点击搜索结果', + description: '在搜索引擎结果页面中点击指定的链接', + triggers: ['打开', '点击', '进入', '访问'], + domains: ['bing.com', 'cn.bing.com', 'baidu.com', 'google.com'], + steps: [ + { + action: 'scan', + }, + { + action: 'verify', + value: '识别目标链接，排除 Copilot、图片搜索、山寨网站、翻页按钮' + }, + { + action: 'click', + target: 'click_by_index(type="link", index={index})', + fallbacks: ['visual_click("{target_name}")', 'click_text("{target_domain}")'] + }, + { + action: 'wait', + value: '2000' + }, + { + action: 'verify', + successIndicator: 'URL 包含目标域名' + } + ], + successRate: 85, + usageCount: 0, + lastUsed: 0, + learnedPatterns: [], + isBuiltIn: true, + version: 1 + }, + + // 访问网站技能 + { + id: 'navigate-to-website', + name: '访问网站', + description: '通过搜索引擎搜索并访问目标网站官网', + triggers: ['打开', '进入', '访问', '去', '跳转到'], + domains: [], // 通用技能 + steps: [ + { + action: 'verify', + value: '检查是否已经在目标网站或搜索引擎' + }, + { + action: 'navigate', + target: 'https://cn.bing.com', + optional: true, + successIndicator: '如果当前不在搜索引擎' + }, + { + action: 'input', + target: '#sb_form_q', + value: '{site_name}' + }, + { + action: 'click', + target: '#sb_form_go' + }, + { + action: 'wait', + value: '1500' + }, + { + action: 'scan' + }, + { + action: 'click', + target: 'click_by_index(type="link", index={index})', + value: '验证链接 href 包含目标域名' + }, + { + action: 'wait', + value: '2000' + }, + { + action: 'verify', + successIndicator: 'URL 包含目标域名' + } + ], + successRate: 90, + usageCount: 0, + lastUsed: 0, + learnedPatterns: [], + isBuiltIn: true, + version: 1 + }, + + // 爱奇艺导航技能 + { + id: 'iqiyi-navigation', + name: '爱奇艺导航', + description: '在爱奇艺网站上导航和搜索视频', + triggers: ['搜索视频', '找电影', '看剧', '播放'], + domains: ['iqiyi.com'], + steps: [ + { + action: 'scan' + }, + { + action: 'input', + target: '.search-input, input[placeholder*="搜索"]', + value: '{query}' + }, + { + action: 'click', + target: '.search-btn, button[class*="search"]', + fallbacks: ['按回车键'] + }, + { + action: 'wait', + value: '2000', + successIndicator: 'URL 包含 /search/' + } + ], + successRate: 80, + usageCount: 0, + lastUsed: 0, + learnedPatterns: [], + isBuiltIn: true, + version: 1 + }, + + // 通用表单填写技能 + { + id: 'fill-form', + name: '填写表单', + description: '智能识别并填写网页表单', + triggers: ['填写', '输入信息', '注册', '登录'], + domains: [], // 通用 + steps: [ + { + action: 'scan' + }, + { + action: 'input', + target: '{detected_input}', + value: '{value}' + }, + { + action: 'verify', + value: '确认输入成功' + } + ], + successRate: 85, + usageCount: 0, + lastUsed: 0, + learnedPatterns: [], + isBuiltIn: true, + version: 1 + }, + + // 滚动查看内容技能 + { + id: 'scroll-and-read', + name: '滚动查看', + description: '滚动页面并阅读完整内容', + triggers: ['阅读', '查看', '总结', '看看'], + domains: [], // 通用 + steps: [ + { + action: 'verify', + value: '获取当前滚动位置' + }, + { + action: 'scroll', + target: 'down', + value: '500' + }, + { + action: 'wait', + value: '500' + }, + { + action: 'verify', + value: '检查是否到达页面底部' + } + ], + successRate: 95, + usageCount: 0, + lastUsed: 0, + learnedPatterns: [], + isBuiltIn: true, + version: 1 + }, + + // 智能登录技能 + { + id: 'auto-login', + name: '智能登录', + description: '检测登录需求，智能处理自动登录或手动登录', + triggers: ['登录', 'login', '注册', 'signup', '需要登录'], + domains: [], // 通用 + steps: [ + { + action: 'verify', + value: 'detect_login 检测是否需要登录' + }, + { + action: 'verify', + value: 'request_login_choice 询问用户选择' + }, + { + action: 'input', + value: 'auto_login 或等待手动登录' + }, + { + action: 'wait', + value: '2000' + }, + { + action: 'verify', + value: '验证登录成功' + } + ], + successRate: 70, + usageCount: 0, + lastUsed: 0, + learnedPatterns: [], + isBuiltIn: true, + version: 1 + } +] diff --git a/cfspider-browser/src/services/skills.ts b/cfspider-browser/src/services/skills.ts new file mode 100644 index 0000000..82be325 --- /dev/null +++ b/cfspider-browser/src/services/skills.ts @@ -0,0 +1,371 @@ +/** + * CFSpider 技能系统 + * 让 AI 针对特定网站和任务有预定义的操作流程 + */ + +// ========== 数据结构定义 ========== + +// 技能步骤动作类型 +export type SkillAction = 'click' | 'input' | 'scroll' | 'wait' | 'verify' | 'navigate' | 'scan' + +// 技能步骤 +export interface SkillStep { + action: SkillAction + target?: string // CSS 选择器或元素描述 + value?: string // 输入值或参数（支持 {query} 等占位符） + fallbacks?: string[] // 备选方案（选择器或操作描述） + successIndicator?: string // 成功标志（URL 变化、元素出现等） + timeout?: number // 超时时间（毫秒） + optional?: boolean // 是否可选（失败不中断） +} + +// 学习到的模式 +export interface LearnedPattern { + pattern: string // 模式描述 + selector?: string // 关联的选择器 + confidence: number // 置信度 0-100 + successCount: number // 成功次数 + failureCount: number // 失败次数 + lastUsed: number // 上次使用时间 + examples: string[] // 成功实例（最多5个） +} + +// 技能定义 +export interface Skill { + id: string // 唯一标识 + name: string // 技能名称，如"必应搜索" + description: string // 技能描述 + triggers: string[] // 触发词，如["搜索", "查找", "百度一下"] + domains: string[] // 适用域名，如["bing.com", "baidu.com"]，空数组表示通用 + steps: SkillStep[] // 操作步骤 + successRate: number // 成功率 0-100 + usageCount: number // 使用次数 + lastUsed: number // 上次使用时间 + learnedPatterns: LearnedPattern[] // 学习到的模式 + isBuiltIn: boolean // 是否为内置技能 + version: number // 版本号（用于更新） +} + +// 技能执行结果 +export interface SkillExecutionResult { + success: boolean + skillId: string + stepsCompleted: number + totalSteps: number + finalUrl?: string + finalTitle?: string + error?: string + executionLog: SkillExecutionLog[] +} + +// 执行日志 +export interface SkillExecutionLog { + step: number + action: SkillAction + target?: string + result: 'success' | 'failure' | 'skipped' + duration: number + error?: string + fallbackUsed?: string +} + +// ========== 技能缓存 ========== + +// 内存中的技能缓存 +let skillsCache: Skill[] = [] +let skillsLoaded = false + +// 检查是否在 Electron 环境 +const isElectron = typeof window !== 'undefined' && (window as any).electronAPI !== undefined + +// ========== 核心函数 ========== + +/** + * 加载技能（从持久化存储） + */ +export async function loadSkills(): Promise { + if (skillsLoaded) return skillsCache + + try { + if (isElectron && (window as any).electronAPI.loadSkills) { + const savedSkills = await (window as any).electronAPI.loadSkills() as Skill[] + if (Array.isArray(savedSkills)) { + // 合并内置技能和保存的技能 + let BUILT_IN_SKILLS: Skill[] = [] + try { + // @ts-ignore - 动态导入，运行时正常 + const builtinSkillsModule = await import('./builtinSkills') + BUILT_IN_SKILLS = builtinSkillsModule.BUILT_IN_SKILLS + } catch (e) { + console.error('[Skills] Failed to load built-in skills:', e) + BUILT_IN_SKILLS = [] + } + + // 内置技能始终使用最新版本 + const builtInMap = new Map(BUILT_IN_SKILLS.map(s => [s.id, s])) + + // 合并：保留用户技能的学习数据，但使用内置技能的步骤定义 + skillsCache = BUILT_IN_SKILLS.map((builtIn: Skill) => { + const saved_skill = savedSkills.find((s: Skill) => s.id === builtIn.id) + if (saved_skill) { + return { + ...builtIn, + successRate: saved_skill.successRate, + usageCount: saved_skill.usageCount, + lastUsed: saved_skill.lastUsed, + learnedPatterns: saved_skill.learnedPatterns || [] + } + } + return builtIn + }) + + // 添加用户自定义技能（包括从学习中提升的技能） + const userSkills = savedSkills.filter((s: Skill) => !builtInMap.has(s.id)) + skillsCache.push(...userSkills) + + console.log('[Skills] Loaded skills:', skillsCache.length, '(内置:', BUILT_IN_SKILLS.length, '学习:', userSkills.length, ')') + } + } + } catch (e) { + console.error('[Skills] Failed to load:', e) + } + + // 如果没有加载到技能，使用内置技能 + if (skillsCache.length === 0) { + try { + // @ts-ignore - 动态导入，运行时正常 + const builtinSkillsModule = await import('./builtinSkills') + skillsCache = [...builtinSkillsModule.BUILT_IN_SKILLS] + } catch (e) { + console.error('[Skills] Failed to load built-in skills:', e) + skillsCache = [] + } + } + + skillsLoaded = true + return skillsCache +} + +/** + * 保存技能（到持久化存储）- 永久保存，无容量限制 + */ +export async function saveSkills(): Promise { + if (!isElectron || !(window as any).electronAPI.saveSkills) return + + try { + // 保存所有有学习数据的技能（包括内置技能的学习数据和用户技能） + // 永久保存，不删除（像真人一辈子都会） + const toSave = skillsCache.filter(s => + s.usageCount > 0 || s.learnedPatterns.length > 0 || !s.isBuiltIn + ) + await (window as any).electronAPI.saveSkills(toSave) + console.log('[Skills] Permanently saved:', toSave.length, '技能（永久保存，无容量限制）') + } catch (e) { + console.error('[Skills] Failed to save:', e) + } +} + +/** + * 根据用户指令和当前域名匹配技能 + */ +export async function matchSkill(instruction: string, currentDomain: string): Promise { + await loadSkills() + + const instructionLower = instruction.toLowerCase() + + // 评分匹配 + let bestMatch: Skill | null = null + let bestScore = 0 + + for (const skill of skillsCache) { + let score = 0 + + // 检查域名匹配 + const domainMatch = skill.domains.length === 0 || + skill.domains.some(d => currentDomain.includes(d)) + if (!domainMatch) continue + + // 检查触发词匹配 + for (const trigger of skill.triggers) { + if (instructionLower.includes(trigger.toLowerCase())) { + score += 10 + // 完全匹配加分 + if (instructionLower.startsWith(trigger.toLowerCase())) { + score += 5 + } + } + } + + // 域名精确匹配加分 + if (skill.domains.some(d => currentDomain === d || currentDomain === 'www.' + d)) { + score += 5 + } + + // 使用频率加分（最多 +10） + score += Math.min(10, Math.floor(skill.usageCount / 10)) + + // 成功率加分（最多 +10） + score += Math.floor(skill.successRate / 10) + + if (score > bestScore) { + bestScore = score + bestMatch = skill + } + } + + // 至少需要 10 分才算匹配 + return bestScore >= 10 ? bestMatch : null +} + +/** + * 获取所有技能 + */ +export async function getAllSkills(): Promise { + await loadSkills() + return skillsCache +} + +/** + * 根据 ID 获取技能 + */ +export async function getSkillById(id: string): Promise { + await loadSkills() + return skillsCache.find(s => s.id === id) || null +} + +/** + * 更新技能学习数据 + */ +export async function updateSkillLearning( + skillId: string, + success: boolean, + pattern?: LearnedPattern +): Promise { + await loadSkills() + + const skill = skillsCache.find(s => s.id === skillId) + if (!skill) return + + // 更新使用统计 + skill.usageCount++ + skill.lastUsed = Date.now() + + // 更新成功率（移动平均） + const weight = Math.min(skill.usageCount, 20) // 最多考虑最近 20 次 + skill.successRate = Math.round( + (skill.successRate * (weight - 1) + (success ? 100 : 0)) / weight + ) + + // 添加学习模式 + if (pattern) { + const existing = skill.learnedPatterns.find(p => p.pattern === pattern.pattern) + if (existing) { + existing.confidence = Math.min(100, existing.confidence + (success ? 10 : -5)) + if (success) existing.successCount++ + else existing.failureCount++ + existing.lastUsed = Date.now() + if (success && pattern.examples.length > 0) { + existing.examples.push(...pattern.examples) + existing.examples = existing.examples.slice(-5) // 只保留最近 5 个 + } + } else { + skill.learnedPatterns.push(pattern) + } + + // 只保留置信度 > 10 的模式，最多 20 个 + skill.learnedPatterns = skill.learnedPatterns + .filter(p => p.confidence > 10) + .sort((a, b) => b.confidence - a.confidence) + .slice(0, 20) + } + + // 技能优化：从学习模式中提取最佳选择器 + if (skill.learnedPatterns.length > 0 && skill.usageCount >= 5) { + optimizeSkillSteps(skill) + } + + // 随机保存（模拟渐进学习） + if (Math.random() < 0.3) { + await saveSkills() + } +} + +/** + * 优化技能步骤（从学习模式中提取最佳实践） + */ +function optimizeSkillSteps(skill: Skill): void { + // 找出置信度最高的模式 + const bestPatterns = skill.learnedPatterns + .filter(p => p.confidence > 60 && p.successCount > 2) + .sort((a, b) => b.confidence - a.confidence) + .slice(0, 3) + + if (bestPatterns.length === 0) return + + // 将高置信度的选择器添加为备选方案 + for (const step of skill.steps) { + if (step.action === 'click' || step.action === 'input') { + for (const pattern of bestPatterns) { + if (pattern.selector && !step.fallbacks?.includes(pattern.selector)) { + step.fallbacks = step.fallbacks || [] + // 将学习到的选择器插入到前面（优先使用） + step.fallbacks.unshift(pattern.selector) + // 最多保留 5 个备选 + step.fallbacks = step.fallbacks.slice(0, 5) + console.log('[Skills] Added learned selector to step:', pattern.selector) + } + } + } + } +} + +/** + * 创建用户自定义技能 + */ +export async function createSkill( + name: string, + description: string, + triggers: string[], + domains: string[], + steps: SkillStep[] +): Promise { + await loadSkills() + + const skill: Skill = { + id: `user-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`, + name, + description, + triggers, + domains, + steps, + successRate: 50, // 初始 50% + usageCount: 0, + lastUsed: 0, + learnedPatterns: [], + isBuiltIn: false, + version: 1 + } + + skillsCache.push(skill) + await saveSkills() + + return skill +} + +/** + * 删除技能 + */ +export async function deleteSkill(skillId: string): Promise { + await loadSkills() + + const index = skillsCache.findIndex(s => s.id === skillId) + if (index === -1) return false + + // 不允许删除内置技能 + if (skillsCache[index].isBuiltIn) return false + + skillsCache.splice(index, 1) + await saveSkills() + + return true +}