feat: 娣诲姞鐪熶汉瀛︿範绯荤粺鍜岃棰戞€荤粨鍔熻兘(鏈€澶?0甯?

This commit is contained in:
陈海富
2026-01-30 00:12:46 +08:00
parent 40dc68dfb3
commit e0e93717e8
3 changed files with 1234 additions and 12 deletions

View File

@@ -1,4 +1,5 @@
import { useStore } from '../store'
import { matchSkill, updateSkillLearning, type Skill } from './skills'
const isElectron = typeof window !== 'undefined' && (window as any).electronAPI !== undefined
@@ -1242,10 +1243,32 @@ export const aiTools = [
type: 'function',
function: {
name: 'scan_interactive_elements',
description: 'Scan and list all interactive elements on the page (buttons, links, inputs). Use to discover available actions.',
description: 'Scan and list all interactive elements on the page (buttons, links, inputs). Use to discover available actions. Returns indexed list for use with click_by_index.',
parameters: { type: 'object', properties: {} }
}
},
{
type: 'function',
function: {
name: 'click_by_index',
description: '通过索引点击元素。先调用 scan_interactive_elements 获取元素列表,然后使用此工具通过索引点击。',
parameters: {
type: 'object',
properties: {
category: {
type: 'string',
description: '元素类别: inputs, buttons, links, selects',
enum: ['inputs', 'buttons', 'links', 'selects']
},
index: {
type: 'number',
description: '元素索引号从1开始'
}
},
required: ['category', 'index']
}
}
},
{
type: 'function',
function: {
@@ -1286,6 +1309,27 @@ export const aiTools = [
required: ['selector']
}
}
},
// ==================== 视频分析工具 ====================
{
type: 'function',
function: {
name: 'summarize_video',
description: '总结页面上正在播放的视频内容。通过抽取关键帧进行视觉分析,生成视频摘要。需要视觉模型支持。',
parameters: {
type: 'object',
properties: {
frame_count: {
type: 'number',
description: '要分析的帧数默认10帧最多40帧'
},
focus: {
type: 'string',
description: '分析重点,如 "人物"、"产品"、"教程步骤" 等(可选)'
}
}
}
}
}
]
@@ -1555,19 +1599,46 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
// Common website domain mappings
var domainMap = {
'jd': ['jd.com'],
'京东': ['jd.com'],
'taobao': ['taobao.com', 'tmall.com'],
'淘宝': ['taobao.com'],
'tmall': ['tmall.com'],
'天猫': ['tmall.com'],
'github': ['github.com'],
'amazon': ['amazon.com', 'amazon.cn'],
'亚马逊': ['amazon.cn', 'amazon.com'],
'google': ['google.com'],
'谷歌': ['google.com'],
'baidu': ['baidu.com'],
'百度': ['baidu.com'],
'bing': ['bing.com'],
'必应': ['bing.com'],
'microsoft': ['microsoft.com'],
'微软': ['microsoft.com'],
'apple': ['apple.com'],
'苹果': ['apple.com'],
'facebook': ['facebook.com'],
'twitter': ['twitter.com', 'x.com'],
'youtube': ['youtube.com'],
'bilibili': ['bilibili.com']
'油管': ['youtube.com'],
'bilibili': ['bilibili.com'],
'b站': ['bilibili.com'],
'哔哩哔哩': ['bilibili.com'],
'爱奇艺': ['iqiyi.com'],
'iqiyi': ['iqiyi.com'],
'优酷': ['youku.com'],
'youku': ['youku.com'],
'腾讯视频': ['v.qq.com'],
'芒果tv': ['mgtv.com'],
'抖音': ['douyin.com'],
'douyin': ['douyin.com'],
'知乎': ['zhihu.com'],
'zhihu': ['zhihu.com'],
'微博': ['weibo.com'],
'weibo': ['weibo.com'],
'网易': ['163.com'],
'新浪': ['sina.com.cn'],
'搜狐': ['sohu.com']
};
// Find matching domain patterns
@@ -1679,13 +1750,17 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
var badSubdomainPrefixes = ['home.', 'my.', 'user.', 'account.', 'login.', 'passport.', 'member.', 'profile.', 'center.', 'i.', 'u.', 'sso.', 'auth.'];
// Also check for these keywords in the result text
var badKeywords = ['个人中心', '我的订单', '我的账户', '账户设置', '登录', 'home.', '/home', '个人信息'];
// Skip these UI elements (Copilot, navigation tabs, images, etc.)
// Skip these UI elements (Copilot, navigation tabs, images, sidebar cards, etc.)
var badUIElements = [
'copilot', 'copilotsearch', 'bingcopilot', 'b_sydConvTab', 'sydneyToggle',
'ai生成', 'ai搜索', 'bing ai',
'全部', '视频', '图片', '地图', '资讯', '更多',
'b_scopeList', 'b_header', 'b_algo_group', 'b_rich',
'/images/', 'images/search', 'image.baidu', 'images.google'
'/images/', 'images/search', 'image.baidu', 'images.google',
// 右侧信息卡Wikipedia, 知识图谱等)
'b_entityTP', 'b_sideBleed', 'b_context', 'b_overlay', 'b_pag', 'b_footer',
'wikipedia', 'wikidata', 'wikimedia', 'youtube.com', 'youtu.be',
'knowledge-panel', 'kno-', 'side-panel', 'sidePanel'
];
// 山寨/镜像网站黑名单
@@ -1803,7 +1878,7 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
}
}
// Helper to check if element is a bad UI element (Copilot, nav tabs, etc.)
// Helper to check if element is a bad UI element (Copilot, nav tabs, sidebar, etc.)
function isBadUIElement(el) {
var href = (el.href || '').toLowerCase();
var className = (el.className || '').toLowerCase();
@@ -1825,8 +1900,17 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
}
}
// Skip elements in header area (navigation)
var rect = el.getBoundingClientRect();
// Skip elements in right sidebar area (x > 60% of viewport width)
// This catches Wikipedia info cards, knowledge panels, etc.
var viewportWidth = window.innerWidth;
if (rect.left > viewportWidth * 0.6 && rect.right > viewportWidth * 0.65) {
console.log('CFSpider: Skipping element in right sidebar:', href || el.textContent?.slice(0, 30));
return true;
}
// Skip elements in header area (navigation)
if (rect.top < 180 && rect.top > 0) {
// Check if it looks like a navigation item
if (className.indexOf('scope') !== -1 || className.indexOf('nav') !== -1 ||
@@ -2588,6 +2672,27 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
case 'read_full_page': {
if (!webview) return 'Error: Cannot access page'
// 检查是否有视觉模型可用
const { aiConfig: pageCfg } = store
const useBuiltInPage = pageCfg.useBuiltIn !== false && (!pageCfg.endpoint || !pageCfg.apiKey)
const hasVisionPage = useBuiltInPage ? !!BUILT_IN_AI.visionModel : !!pageCfg.visionModel
// 如果没有视觉模型,使用 DOM 提取文本
if (!hasVisionPage) {
try {
const textContent = await webview.executeJavaScript(`
(function() {
var main = document.querySelector('main, article, .content, .main, #content, #main, body');
return main ? main.innerText.slice(0, 10000) : document.body.innerText.slice(0, 10000);
})()
`)
return '页面文本内容DOM提取无视觉分析\n\n' + textContent
} catch (e) {
return 'Error reading page: ' + e
}
}
try {
const maxScrolls = (args.max_scrolls as number) || 10
const allContents: string[] = []
@@ -2684,6 +2789,16 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
case 'solve_captcha': {
if (!webview) return 'Error: Cannot access page'
// 检查是否有视觉模型可用
const { aiConfig } = store
const useBuiltIn = aiConfig.useBuiltIn !== false && (!aiConfig.endpoint || !aiConfig.apiKey)
const hasVisionModel = useBuiltIn ? !!BUILT_IN_AI.visionModel : !!aiConfig.visionModel
if (!hasVisionModel) {
return '当前为单模型模式,无法使用验证码识别功能。请在设置中配置视觉模型,或使用内置 AI 服务。'
}
try {
const captchaType = args.captcha_type as string
store.setCurrentModelType('vision')
@@ -2878,6 +2993,16 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click
case 'visual_click': {
if (!webview) return 'Error: Cannot access page'
// 检查是否有视觉模型可用
const { aiConfig: aiCfg } = store
const useBuiltInVision = aiCfg.useBuiltIn !== false && (!aiCfg.endpoint || !aiCfg.apiKey)
const hasVision = useBuiltInVision ? !!BUILT_IN_AI.visionModel : !!aiCfg.visionModel
if (!hasVision) {
return '当前为单模型模式,无法使用视觉点击功能。请尝试使用 click_text 或 click_element 替代。'
}
try {
const description = args.description as string
@@ -4268,6 +4393,116 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click
}
}
case 'click_by_index': {
if (!webview) return 'Error: Cannot access page'
const category = args.category as string
const index = args.index as number
if (!category || !index) {
return 'Error: 需要提供 category元素类别和 index索引号'
}
try {
const result = await webview.executeJavaScript(`
(function() {
var category = '${category}';
var index = ${index} - 1; // 转换为0-based索引
var elements = [];
if (category === 'inputs') {
document.querySelectorAll('input, textarea').forEach(function(el) {
if (el.offsetWidth > 0 && el.offsetHeight > 0) {
elements.push(el);
}
});
} else if (category === 'buttons') {
document.querySelectorAll('button, input[type="submit"], input[type="button"], [role="button"]').forEach(function(el) {
if (el.offsetWidth > 0 && el.offsetHeight > 0) {
elements.push(el);
}
});
} else if (category === 'links') {
var linkCount = 0;
document.querySelectorAll('a[href]').forEach(function(el) {
if (linkCount < 15 && el.offsetWidth > 0) {
var text = (el.textContent || '').trim();
if (text.length > 2) {
elements.push(el);
linkCount++;
}
}
});
} else if (category === 'selects') {
document.querySelectorAll('select').forEach(function(el) {
if (el.offsetWidth > 0) {
elements.push(el);
}
});
}
if (index < 0 || index >= elements.length) {
return { success: false, error: '索引超出范围,该类别共有 ' + elements.length + ' 个元素' };
}
var el = elements[index];
var rect = el.getBoundingClientRect();
var text = (el.textContent || el.value || el.placeholder || '').slice(0, 50).trim();
// 添加高亮
var h = document.createElement('div');
h.id = 'cfspider-agent-highlight';
h.style.cssText = 'position:fixed;pointer-events:none;z-index:2147483647;border:4px solid #22c55e;background:rgba(34,197,94,0.2);border-radius:6px;box-shadow:0 0 20px rgba(34,197,94,0.5);';
h.style.left = (rect.left - 4) + 'px';
h.style.top = (rect.top - 4) + 'px';
h.style.width = (rect.width + 8) + 'px';
h.style.height = (rect.height + 8) + 'px';
document.body.appendChild(h);
// 点击元素
el.click();
if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
el.focus();
}
setTimeout(function() {
var hh = document.getElementById('cfspider-agent-highlight');
if (hh) hh.remove();
}, 1000);
return {
success: true,
tag: el.tagName,
text: text,
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2
};
})()
`)
if (!result.success) {
return `点击失败: ${result.error}`
}
// 显示虚拟鼠标动画
const container = document.getElementById('browser-container')
if (container) {
const containerRect = container.getBoundingClientRect()
const x = result.x + containerRect.left
const y = result.y + containerRect.top
store.showMouse()
store.moveMouse(x, y, 400)
await new Promise(resolve => setTimeout(resolve, 450))
store.clickMouse()
}
await new Promise(resolve => setTimeout(resolve, 500))
return `已点击 ${category} 类别的第 ${index} 个元素: <${result.tag}> "${result.text}"`
} catch (e) {
return `点击失败: ${e}`
}
}
case 'get_page_content': {
if (!webview) return 'Error: Cannot access page'
try {
@@ -4399,6 +4634,266 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click
}
}
// ==================== 视频分析工具 ====================
case 'summarize_video': {
if (!webview) return 'Error: Cannot access page'
// 检查视觉模型是否可用
const { aiConfig: videoConfig } = store
const useBuiltInVideo = videoConfig.useBuiltIn !== false && (!videoConfig.endpoint || !videoConfig.apiKey)
const hasVisionForVideo = useBuiltInVideo ? !!BUILT_IN_AI.visionModel : !!videoConfig.visionModel
if (!hasVisionForVideo) {
return '视频总结需要视觉模型支持。请在设置中配置视觉模型,或使用内置 AI 服务。'
}
try {
const frameCount = Math.min(Math.max((args.frame_count as number) || 10, 1), 40)
const focus = (args.focus as string) || ''
store.setCurrentModelType('vision')
console.log('[CFSpider] 开始视频分析,计划抽取', frameCount, '帧')
// 检测页面上的视频元素
const videoInfo = await webview.executeJavaScript(`
(function() {
// 查找视频元素
var video = document.querySelector('video');
if (!video) {
// 尝试查找 iframe 中的视频(如 YouTube
var iframes = document.querySelectorAll('iframe');
for (var i = 0; i < iframes.length; i++) {
var src = iframes[i].src || '';
if (src.includes('youtube') || src.includes('bilibili') || src.includes('youku') || src.includes('iqiyi')) {
return {
type: 'iframe',
platform: src.includes('youtube') ? 'YouTube' :
src.includes('bilibili') ? 'Bilibili' :
src.includes('youku') ? '优酷' :
src.includes('iqiyi') ? '爱奇艺' : '视频平台',
src: src
};
}
}
return null;
}
return {
type: 'video',
duration: video.duration || 0,
currentTime: video.currentTime || 0,
paused: video.paused,
width: video.videoWidth,
height: video.videoHeight,
src: video.src || video.currentSrc
};
})()
`)
if (!videoInfo) {
store.setCurrentModelType(null)
return '页面上未找到视频元素。请确保视频正在播放或页面包含视频播放器。'
}
// 如果是 iframe 嵌入的视频(如 YouTube使用页面截图分析
if (videoInfo.type === 'iframe') {
console.log('[CFSpider] 检测到嵌入式视频平台:', videoInfo.platform)
// 对嵌入视频直接截图分析当前画面最多分析10帧因为无法控制进度
const iframeFrameCount = Math.min(frameCount, 10)
const frameAnalyses: string[] = []
for (let i = 0; i < iframeFrameCount; i++) {
const image = await webview.capturePage()
if (!image) continue
const base64Image = image.toDataURL().replace(/^data:image\/\w+;base64,/, '')
const response = await (window as any).electronAPI.aiChat({
endpoint: BUILT_IN_AI.endpoint,
apiKey: getBuiltInKey(),
model: BUILT_IN_AI.visionModel,
messages: [{
role: 'user',
content: [
{
type: 'text',
text: `这是一个${videoInfo.platform}视频的第${i + 1}帧截图。请分析视频画面内容:
${focus ? `重点关注:${focus}` : ''}
请描述:
1. 画面中的主要内容
2. 视频类型(教程、娱乐、新闻等)
3. 关键信息或字幕内容(如有)
简洁回答50字以内。`
},
{ type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } }
]
}]
})
if (response.content) {
frameAnalyses.push(`${i + 1}: ${response.content}`)
}
// 等待一小段时间再截下一帧
if (i < iframeFrameCount - 1) {
await new Promise(resolve => setTimeout(resolve, 1500))
}
}
store.setCurrentModelType(null)
if (frameAnalyses.length === 0) {
return `检测到${videoInfo.platform}视频,但无法分析画面内容。`
}
// 生成总结
let summary = `[${videoInfo.platform}视频分析] 共分析${frameAnalyses.length}\n\n`
summary += frameAnalyses.join('\n\n')
if (frameAnalyses.length > 1) {
// 请求生成整体总结
const summaryResponse = await (window as any).electronAPI.aiChat({
endpoint: useBuiltInVideo ? BUILT_IN_AI.endpoint : videoConfig.endpoint,
apiKey: useBuiltInVideo ? getBuiltInKey() : videoConfig.apiKey,
model: useBuiltInVideo ? BUILT_IN_AI.model : videoConfig.model,
messages: [{
role: 'user',
content: `根据以下视频帧分析生成一个简短的视频总结150字以内\n\n${frameAnalyses.join('\n')}`
}]
})
if (summaryResponse.content) {
summary += `\n\n[总结]\n${summaryResponse.content}`
}
}
return summary
}
// 原生 video 元素 - 通过控制播放进度抽取帧
const duration = videoInfo.duration
if (duration <= 0) {
store.setCurrentModelType(null)
return '视频时长未知或为直播流,无法进行帧分析。'
}
console.log('[CFSpider] 视频时长:', duration, '秒,抽取', frameCount, '帧')
// 计算抽帧时间点(均匀分布)
const timePoints: number[] = []
for (let i = 0; i < frameCount; i++) {
timePoints.push((duration / (frameCount + 1)) * (i + 1))
}
const frameAnalyses: string[] = []
// 批量分析帧每5帧为一组
const batchSize = 5
for (let batch = 0; batch < Math.ceil(timePoints.length / batchSize); batch++) {
const batchStart = batch * batchSize
const batchEnd = Math.min(batchStart + batchSize, timePoints.length)
for (let i = batchStart; i < batchEnd; i++) {
const targetTime = timePoints[i]
// 跳转到指定时间点
await webview.executeJavaScript(`
(function() {
var video = document.querySelector('video');
if (video) {
video.currentTime = ${targetTime};
video.pause();
}
})()
`)
// 等待视频加载该帧
await new Promise(resolve => setTimeout(resolve, 300))
// 截图
const image = await webview.capturePage()
if (!image) continue
const base64Image = image.toDataURL().replace(/^data:image\/\w+;base64,/, '')
const timeStr = `${Math.floor(targetTime / 60)}:${String(Math.floor(targetTime % 60)).padStart(2, '0')}`
// 分析帧内容
const response = await (window as any).electronAPI.aiChat({
endpoint: BUILT_IN_AI.endpoint,
apiKey: getBuiltInKey(),
model: BUILT_IN_AI.visionModel,
messages: [{
role: 'user',
content: [
{
type: 'text',
text: `这是视频在 ${timeStr} 时刻的画面。请简短描述画面内容30字以内${focus ? `重点关注:${focus}` : ''}`
},
{ type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } }
]
}]
})
if (response.content) {
frameAnalyses.push(`[${timeStr}] ${response.content}`)
}
}
// 每批次之间短暂休息
if (batch < Math.ceil(timePoints.length / batchSize) - 1) {
await new Promise(resolve => setTimeout(resolve, 200))
}
}
// 恢复视频播放
await webview.executeJavaScript(`
(function() {
var video = document.querySelector('video');
if (video) video.play();
})()
`)
store.setCurrentModelType(null)
if (frameAnalyses.length === 0) {
return '视频帧分析失败,无法获取有效信息。'
}
// 生成视频总结
const durationMin = Math.floor(duration / 60)
const durationSec = Math.floor(duration % 60)
let summary = `[视频分析] 时长: ${durationMin}${durationSec}秒 | 分析帧数: ${frameAnalyses.length}\n\n`
summary += '关键帧内容:\n'
summary += frameAnalyses.join('\n')
// 生成整体总结
if (frameAnalyses.length > 2) {
const summaryResponse = await (window as any).electronAPI.aiChat({
endpoint: useBuiltInVideo ? BUILT_IN_AI.endpoint : videoConfig.endpoint,
apiKey: useBuiltInVideo ? getBuiltInKey() : videoConfig.apiKey,
model: useBuiltInVideo ? BUILT_IN_AI.model : videoConfig.model,
messages: [{
role: 'user',
content: `根据以下视频帧分析生成一个详细的视频总结200字以内\n\n${frameAnalyses.join('\n')}`
}]
})
if (summaryResponse.content) {
summary += `\n\n[总结]\n${summaryResponse.content}`
}
}
return summary
} catch (e) {
store.setCurrentModelType(null)
console.error('[CFSpider] 视频分析失败:', e)
return `视频分析失败: ${e}`
}
}
default:
return 'Unknown tool: ' + name
}
@@ -4431,9 +4926,9 @@ const systemPrompt = `你是 CFspider 智能浏览器自动化助手,由 viole
对于这些情况,只需用中文自然回复,不要调用任何工具。
示例:
- 用户:"你好" -> 回复:"嗨~你好我是 CFspider 智能浏览器助手,有什么可以帮你的吗?"
- 用户:"你是谁" -> 回复:"我是 CFspider 智能浏览器 AI 助手由 violetteam 团队开发,来自 cfspider 项目。我可以帮你自动化浏览器操作,比如搜索、点击、导航网站等,交给我就行~"
- 用户:"谢谢" -> 回复:"不客气呀!有需要随时叫我~"
- 用户:"你好" -> 回复:"你好我是 CFspider 智能浏览器助手,有什么可以帮你的吗?"
- 用户:"你是谁" -> 回复:"你好,我是 CFspider 智能浏览器 AI 助手。{{MODEL_INTRODUCTION}} CFspider 工具由 violetteam 团队开发。我可以帮你自动化浏览器操作,比如搜索、点击、导航网站等"
- 用户:"谢谢" -> 回复:"不客气有需要随时叫我"
### 以下情况使用工具:
- 打开网站:"打开京东"、"去 GitHub"
@@ -4843,10 +5338,54 @@ export async function sendAIMessage(content: string, useTools: boolean = true) {
try {
// 构建聊天历史,包含工具调用信息以便 AI 记住之前的操作
// 动态生成模型名介绍
const toolModelName = useBuiltIn
? BUILT_IN_AI.model.split('/').pop() || 'DeepSeek-V3'
: (effectiveConfig.model || '未配置')
const visionModelName = useBuiltIn
? BUILT_IN_AI.visionModel?.split('/').pop() || ''
: (aiConfig.visionModel?.split('/').pop() || '')
// 提取模型开发团队(从模型名推断)
const getModelTeam = (modelName: string): string => {
const name = modelName.toLowerCase()
if (name.includes('deepseek')) return 'DeepSeek'
if (name.includes('qwen')) return '阿里云通义千问'
if (name.includes('glm') || name.includes('chatglm')) return '智谱 AI'
if (name.includes('gpt')) return 'OpenAI'
if (name.includes('claude')) return 'Anthropic'
if (name.includes('gemini')) return 'Google'
if (name.includes('llama')) return 'Meta'
if (name.includes('mistral')) return 'Mistral AI'
if (name.includes('yi')) return '零一万物'
if (name.includes('baichuan')) return '百川智能'
if (name.includes('moonshot') || name.includes('kimi')) return 'Moonshot AI'
return ''
}
const toolModelTeam = getModelTeam(toolModelName)
const visionModelTeam = visionModelName ? getModelTeam(visionModelName) : ''
let modelIntroduction = ''
if (modelMode === 'dual' && visionModelName) {
const toolTeamStr = toolModelTeam ? `(由 ${toolModelTeam} 开发)` : ''
const visionTeamStr = visionModelTeam ? `(由 ${visionModelTeam} 开发)` : ''
modelIntroduction = `目前由两个模型共同驱动:${toolModelName}${toolTeamStr}负责文本理解和工具调用,${visionModelName}${visionTeamStr}负责页面视觉分析。`
} else {
const teamStr = toolModelTeam ? `(由 ${toolModelTeam} 开发)` : ''
modelIntroduction = `目前由 ${toolModelName} 模型${teamStr}驱动。`
}
// 替换系统提示中的模型名占位符
const dynamicSystemPrompt = systemPrompt
.replace(/\{\{TOOL_MODEL_NAME\}\}/g, toolModelName)
.replace(/\{\{MODEL_INTRODUCTION\}\}/g, modelIntroduction)
// 如果有 OCR 页面分析结果,添加到系统提示词中
const enhancedSystemPrompt = pageContext
? `${systemPrompt}\n\n## 当前页面分析结果(由视觉模型提供)\n\n${pageContext}\n\n请根据以上页面分析结果来决定下一步操作。`
: systemPrompt
? `${dynamicSystemPrompt}\n\n## 当前页面分析结果(由视觉模型提供)\n\n${pageContext}\n\n请根据以上页面分析结果来决定下一步操作。`
: dynamicSystemPrompt
const chatHistory: Array<{ role: string; content?: string; tool_calls?: any[]; tool_call_id?: string; name?: string }> = [
{ role: 'system', content: enhancedSystemPrompt }
@@ -4883,6 +5422,28 @@ export async function sendAIMessage(content: string, useTools: boolean = true) {
// 添加当前用户消息
chatHistory.push({ role: 'user', content })
// 尝试匹配技能,提供给 AI 作为参考
let matchedSkill: Skill | null = null
try {
const webview = document.querySelector('webview') as any
if (webview) {
const currentUrl = await webview.executeJavaScript('window.location.href') as string
const currentDomain = new URL(currentUrl).hostname
matchedSkill = await matchSkill(content, currentDomain)
if (matchedSkill) {
console.log('[CFSpider] 匹配到技能:', matchedSkill.name, '成功率:', matchedSkill.successRate)
// 将技能信息添加到聊天历史作为系统提示
const skillHint = `[技能提示] 匹配到「${matchedSkill.name}」技能(成功率: ${matchedSkill.successRate}%
触发词: ${matchedSkill.triggers.join(', ')}
操作步骤: ${matchedSkill.steps.map((s, i) => `${i+1}. ${s.action}${s.target ? `: ${s.target}` : ''}`).join(' -> ')}
${matchedSkill.learnedPatterns.length > 0 ? `学习到的模式: ${matchedSkill.learnedPatterns.slice(0, 3).map(p => p.pattern).join(', ')}` : ''}`
chatHistory.push({ role: 'system', content: skillHint })
}
}
} catch (e) {
console.error('[CFSpider] 技能匹配失败:', e)
}
let iteration = 0
const maxIterations = 30
@@ -4971,7 +5532,8 @@ export async function sendAIMessage(content: string, useTools: boolean = true) {
updateLastMessageWithToolCalls('', toolCallHistory)
// 如果操作失败,触发紧张模式和语气词反应
if (result.includes('Error') || result.includes('失败') || result.includes('not found') || result.includes('Cannot')) {
const operationFailed = result.includes('Error') || result.includes('失败') || result.includes('not found') || result.includes('Cannot')
if (operationFailed) {
// 触发紧张乱动
store.panicMouse(1500)
@@ -4984,6 +5546,23 @@ export async function sendAIMessage(content: string, useTools: boolean = true) {
// 等待紧张动画完成
await new Promise(resolve => setTimeout(resolve, 1200))
}
// 更新技能学习(如果匹配到了技能)
if (matchedSkill) {
try {
await updateSkillLearning(matchedSkill.id, !operationFailed, {
pattern: `${funcName}:${JSON.stringify(funcArgs).slice(0, 50)}`,
selector: funcArgs.selector || funcArgs.text || funcArgs.target,
confidence: operationFailed ? 30 : 70,
successCount: operationFailed ? 0 : 1,
failureCount: operationFailed ? 1 : 0,
lastUsed: Date.now(),
examples: operationFailed ? [] : [result.slice(0, 100)]
})
} catch (e) {
console.error('[CFSpider] 技能学习更新失败:', e)
}
}
chatHistory.push({
role: 'assistant',

View File

@@ -0,0 +1,272 @@
/**
* 内置技能定义
* 从 Markdown 文件解析而来
*/
import { Skill } from './skills'
export const BUILT_IN_SKILLS: Skill[] = [
// 必应搜索技能
{
id: 'bing-search',
name: '必应搜索',
description: '在必应搜索引擎上搜索关键词',
triggers: ['搜索', '查找', '找一下', 'search', '查', '百度一下'],
domains: ['bing.com', 'cn.bing.com'],
steps: [
{
action: 'scan',
},
{
action: 'input',
target: '#sb_form_q, input[name="q"], textarea[name="q"]',
value: '{query}',
fallbacks: ['input[type="search"]', '.search-box input']
},
{
action: 'click',
target: '#sb_form_go, #search_icon, button[type="submit"]',
fallbacks: ['按回车键', 'form button'],
optional: true
},
{
action: 'wait',
value: '1500',
successIndicator: 'URL包含 /search?q='
}
],
successRate: 95,
usageCount: 0,
lastUsed: 0,
learnedPatterns: [],
isBuiltIn: true,
version: 1
},
// 点击搜索结果技能
{
id: 'click-search-result',
name: '点击搜索结果',
description: '在搜索引擎结果页面中点击指定的链接',
triggers: ['打开', '点击', '进入', '访问'],
domains: ['bing.com', 'cn.bing.com', 'baidu.com', 'google.com'],
steps: [
{
action: 'scan',
},
{
action: 'verify',
value: '识别目标链接,排除 Copilot、图片搜索、山寨网站、翻页按钮'
},
{
action: 'click',
target: 'click_by_index(type="link", index={index})',
fallbacks: ['visual_click("{target_name}")', 'click_text("{target_domain}")']
},
{
action: 'wait',
value: '2000'
},
{
action: 'verify',
successIndicator: 'URL 包含目标域名'
}
],
successRate: 85,
usageCount: 0,
lastUsed: 0,
learnedPatterns: [],
isBuiltIn: true,
version: 1
},
// 访问网站技能
{
id: 'navigate-to-website',
name: '访问网站',
description: '通过搜索引擎搜索并访问目标网站官网',
triggers: ['打开', '进入', '访问', '去', '跳转到'],
domains: [], // 通用技能
steps: [
{
action: 'verify',
value: '检查是否已经在目标网站或搜索引擎'
},
{
action: 'navigate',
target: 'https://cn.bing.com',
optional: true,
successIndicator: '如果当前不在搜索引擎'
},
{
action: 'input',
target: '#sb_form_q',
value: '{site_name}'
},
{
action: 'click',
target: '#sb_form_go'
},
{
action: 'wait',
value: '1500'
},
{
action: 'scan'
},
{
action: 'click',
target: 'click_by_index(type="link", index={index})',
value: '验证链接 href 包含目标域名'
},
{
action: 'wait',
value: '2000'
},
{
action: 'verify',
successIndicator: 'URL 包含目标域名'
}
],
successRate: 90,
usageCount: 0,
lastUsed: 0,
learnedPatterns: [],
isBuiltIn: true,
version: 1
},
// 爱奇艺导航技能
{
id: 'iqiyi-navigation',
name: '爱奇艺导航',
description: '在爱奇艺网站上导航和搜索视频',
triggers: ['搜索视频', '找电影', '看剧', '播放'],
domains: ['iqiyi.com'],
steps: [
{
action: 'scan'
},
{
action: 'input',
target: '.search-input, input[placeholder*="搜索"]',
value: '{query}'
},
{
action: 'click',
target: '.search-btn, button[class*="search"]',
fallbacks: ['按回车键']
},
{
action: 'wait',
value: '2000',
successIndicator: 'URL 包含 /search/'
}
],
successRate: 80,
usageCount: 0,
lastUsed: 0,
learnedPatterns: [],
isBuiltIn: true,
version: 1
},
// 通用表单填写技能
{
id: 'fill-form',
name: '填写表单',
description: '智能识别并填写网页表单',
triggers: ['填写', '输入信息', '注册', '登录'],
domains: [], // 通用
steps: [
{
action: 'scan'
},
{
action: 'input',
target: '{detected_input}',
value: '{value}'
},
{
action: 'verify',
value: '确认输入成功'
}
],
successRate: 85,
usageCount: 0,
lastUsed: 0,
learnedPatterns: [],
isBuiltIn: true,
version: 1
},
// 滚动查看内容技能
{
id: 'scroll-and-read',
name: '滚动查看',
description: '滚动页面并阅读完整内容',
triggers: ['阅读', '查看', '总结', '看看'],
domains: [], // 通用
steps: [
{
action: 'verify',
value: '获取当前滚动位置'
},
{
action: 'scroll',
target: 'down',
value: '500'
},
{
action: 'wait',
value: '500'
},
{
action: 'verify',
value: '检查是否到达页面底部'
}
],
successRate: 95,
usageCount: 0,
lastUsed: 0,
learnedPatterns: [],
isBuiltIn: true,
version: 1
},
// 智能登录技能
{
id: 'auto-login',
name: '智能登录',
description: '检测登录需求,智能处理自动登录或手动登录',
triggers: ['登录', 'login', '注册', 'signup', '需要登录'],
domains: [], // 通用
steps: [
{
action: 'verify',
value: 'detect_login 检测是否需要登录'
},
{
action: 'verify',
value: 'request_login_choice 询问用户选择'
},
{
action: 'input',
value: 'auto_login 或等待手动登录'
},
{
action: 'wait',
value: '2000'
},
{
action: 'verify',
value: '验证登录成功'
}
],
successRate: 70,
usageCount: 0,
lastUsed: 0,
learnedPatterns: [],
isBuiltIn: true,
version: 1
}
]

View File

@@ -0,0 +1,371 @@
/**
* CFSpider 技能系统
* 让 AI 针对特定网站和任务有预定义的操作流程
*/
// ========== 数据结构定义 ==========
// 技能步骤动作类型
export type SkillAction = 'click' | 'input' | 'scroll' | 'wait' | 'verify' | 'navigate' | 'scan'
// 技能步骤
export interface SkillStep {
action: SkillAction
target?: string // CSS 选择器或元素描述
value?: string // 输入值或参数(支持 {query} 等占位符)
fallbacks?: string[] // 备选方案(选择器或操作描述)
successIndicator?: string // 成功标志URL 变化、元素出现等)
timeout?: number // 超时时间(毫秒)
optional?: boolean // 是否可选(失败不中断)
}
// 学习到的模式
export interface LearnedPattern {
pattern: string // 模式描述
selector?: string // 关联的选择器
confidence: number // 置信度 0-100
successCount: number // 成功次数
failureCount: number // 失败次数
lastUsed: number // 上次使用时间
examples: string[] // 成功实例最多5个
}
// 技能定义
export interface Skill {
id: string // 唯一标识
name: string // 技能名称,如"必应搜索"
description: string // 技能描述
triggers: string[] // 触发词,如["搜索", "查找", "百度一下"]
domains: string[] // 适用域名,如["bing.com", "baidu.com"],空数组表示通用
steps: SkillStep[] // 操作步骤
successRate: number // 成功率 0-100
usageCount: number // 使用次数
lastUsed: number // 上次使用时间
learnedPatterns: LearnedPattern[] // 学习到的模式
isBuiltIn: boolean // 是否为内置技能
version: number // 版本号(用于更新)
}
// 技能执行结果
export interface SkillExecutionResult {
success: boolean
skillId: string
stepsCompleted: number
totalSteps: number
finalUrl?: string
finalTitle?: string
error?: string
executionLog: SkillExecutionLog[]
}
// 执行日志
export interface SkillExecutionLog {
step: number
action: SkillAction
target?: string
result: 'success' | 'failure' | 'skipped'
duration: number
error?: string
fallbackUsed?: string
}
// ========== 技能缓存 ==========
// 内存中的技能缓存
let skillsCache: Skill[] = []
let skillsLoaded = false
// 检查是否在 Electron 环境
const isElectron = typeof window !== 'undefined' && (window as any).electronAPI !== undefined
// ========== 核心函数 ==========
/**
* 加载技能(从持久化存储)
*/
export async function loadSkills(): Promise<Skill[]> {
if (skillsLoaded) return skillsCache
try {
if (isElectron && (window as any).electronAPI.loadSkills) {
const savedSkills = await (window as any).electronAPI.loadSkills() as Skill[]
if (Array.isArray(savedSkills)) {
// 合并内置技能和保存的技能
let BUILT_IN_SKILLS: Skill[] = []
try {
// @ts-ignore - 动态导入,运行时正常
const builtinSkillsModule = await import('./builtinSkills')
BUILT_IN_SKILLS = builtinSkillsModule.BUILT_IN_SKILLS
} catch (e) {
console.error('[Skills] Failed to load built-in skills:', e)
BUILT_IN_SKILLS = []
}
// 内置技能始终使用最新版本
const builtInMap = new Map(BUILT_IN_SKILLS.map(s => [s.id, s]))
// 合并:保留用户技能的学习数据,但使用内置技能的步骤定义
skillsCache = BUILT_IN_SKILLS.map((builtIn: Skill) => {
const saved_skill = savedSkills.find((s: Skill) => s.id === builtIn.id)
if (saved_skill) {
return {
...builtIn,
successRate: saved_skill.successRate,
usageCount: saved_skill.usageCount,
lastUsed: saved_skill.lastUsed,
learnedPatterns: saved_skill.learnedPatterns || []
}
}
return builtIn
})
// 添加用户自定义技能(包括从学习中提升的技能)
const userSkills = savedSkills.filter((s: Skill) => !builtInMap.has(s.id))
skillsCache.push(...userSkills)
console.log('[Skills] Loaded skills:', skillsCache.length, '(内置:', BUILT_IN_SKILLS.length, '学习:', userSkills.length, ')')
}
}
} catch (e) {
console.error('[Skills] Failed to load:', e)
}
// 如果没有加载到技能,使用内置技能
if (skillsCache.length === 0) {
try {
// @ts-ignore - 动态导入,运行时正常
const builtinSkillsModule = await import('./builtinSkills')
skillsCache = [...builtinSkillsModule.BUILT_IN_SKILLS]
} catch (e) {
console.error('[Skills] Failed to load built-in skills:', e)
skillsCache = []
}
}
skillsLoaded = true
return skillsCache
}
/**
* 保存技能(到持久化存储)- 永久保存,无容量限制
*/
export async function saveSkills(): Promise<void> {
if (!isElectron || !(window as any).electronAPI.saveSkills) return
try {
// 保存所有有学习数据的技能(包括内置技能的学习数据和用户技能)
// 永久保存,不删除(像真人一辈子都会)
const toSave = skillsCache.filter(s =>
s.usageCount > 0 || s.learnedPatterns.length > 0 || !s.isBuiltIn
)
await (window as any).electronAPI.saveSkills(toSave)
console.log('[Skills] Permanently saved:', toSave.length, '技能(永久保存,无容量限制)')
} catch (e) {
console.error('[Skills] Failed to save:', e)
}
}
/**
* 根据用户指令和当前域名匹配技能
*/
export async function matchSkill(instruction: string, currentDomain: string): Promise<Skill | null> {
await loadSkills()
const instructionLower = instruction.toLowerCase()
// 评分匹配
let bestMatch: Skill | null = null
let bestScore = 0
for (const skill of skillsCache) {
let score = 0
// 检查域名匹配
const domainMatch = skill.domains.length === 0 ||
skill.domains.some(d => currentDomain.includes(d))
if (!domainMatch) continue
// 检查触发词匹配
for (const trigger of skill.triggers) {
if (instructionLower.includes(trigger.toLowerCase())) {
score += 10
// 完全匹配加分
if (instructionLower.startsWith(trigger.toLowerCase())) {
score += 5
}
}
}
// 域名精确匹配加分
if (skill.domains.some(d => currentDomain === d || currentDomain === 'www.' + d)) {
score += 5
}
// 使用频率加分(最多 +10
score += Math.min(10, Math.floor(skill.usageCount / 10))
// 成功率加分(最多 +10
score += Math.floor(skill.successRate / 10)
if (score > bestScore) {
bestScore = score
bestMatch = skill
}
}
// 至少需要 10 分才算匹配
return bestScore >= 10 ? bestMatch : null
}
/**
* 获取所有技能
*/
export async function getAllSkills(): Promise<Skill[]> {
await loadSkills()
return skillsCache
}
/**
* 根据 ID 获取技能
*/
export async function getSkillById(id: string): Promise<Skill | null> {
await loadSkills()
return skillsCache.find(s => s.id === id) || null
}
/**
* 更新技能学习数据
*/
export async function updateSkillLearning(
skillId: string,
success: boolean,
pattern?: LearnedPattern
): Promise<void> {
await loadSkills()
const skill = skillsCache.find(s => s.id === skillId)
if (!skill) return
// 更新使用统计
skill.usageCount++
skill.lastUsed = Date.now()
// 更新成功率(移动平均)
const weight = Math.min(skill.usageCount, 20) // 最多考虑最近 20 次
skill.successRate = Math.round(
(skill.successRate * (weight - 1) + (success ? 100 : 0)) / weight
)
// 添加学习模式
if (pattern) {
const existing = skill.learnedPatterns.find(p => p.pattern === pattern.pattern)
if (existing) {
existing.confidence = Math.min(100, existing.confidence + (success ? 10 : -5))
if (success) existing.successCount++
else existing.failureCount++
existing.lastUsed = Date.now()
if (success && pattern.examples.length > 0) {
existing.examples.push(...pattern.examples)
existing.examples = existing.examples.slice(-5) // 只保留最近 5 个
}
} else {
skill.learnedPatterns.push(pattern)
}
// 只保留置信度 > 10 的模式,最多 20 个
skill.learnedPatterns = skill.learnedPatterns
.filter(p => p.confidence > 10)
.sort((a, b) => b.confidence - a.confidence)
.slice(0, 20)
}
// 技能优化:从学习模式中提取最佳选择器
if (skill.learnedPatterns.length > 0 && skill.usageCount >= 5) {
optimizeSkillSteps(skill)
}
// 随机保存(模拟渐进学习)
if (Math.random() < 0.3) {
await saveSkills()
}
}
/**
* 优化技能步骤(从学习模式中提取最佳实践)
*/
function optimizeSkillSteps(skill: Skill): void {
// 找出置信度最高的模式
const bestPatterns = skill.learnedPatterns
.filter(p => p.confidence > 60 && p.successCount > 2)
.sort((a, b) => b.confidence - a.confidence)
.slice(0, 3)
if (bestPatterns.length === 0) return
// 将高置信度的选择器添加为备选方案
for (const step of skill.steps) {
if (step.action === 'click' || step.action === 'input') {
for (const pattern of bestPatterns) {
if (pattern.selector && !step.fallbacks?.includes(pattern.selector)) {
step.fallbacks = step.fallbacks || []
// 将学习到的选择器插入到前面(优先使用)
step.fallbacks.unshift(pattern.selector)
// 最多保留 5 个备选
step.fallbacks = step.fallbacks.slice(0, 5)
console.log('[Skills] Added learned selector to step:', pattern.selector)
}
}
}
}
}
/**
* 创建用户自定义技能
*/
export async function createSkill(
name: string,
description: string,
triggers: string[],
domains: string[],
steps: SkillStep[]
): Promise<Skill> {
await loadSkills()
const skill: Skill = {
id: `user-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
name,
description,
triggers,
domains,
steps,
successRate: 50, // 初始 50%
usageCount: 0,
lastUsed: 0,
learnedPatterns: [],
isBuiltIn: false,
version: 1
}
skillsCache.push(skill)
await saveSkills()
return skill
}
/**
* 删除技能
*/
export async function deleteSkill(skillId: string): Promise<boolean> {
await loadSkills()
const index = skillsCache.findIndex(s => s.id === skillId)
if (index === -1) return false
// 不允许删除内置技能
if (skillsCache[index].isBuiltIn) return false
skillsCache.splice(index, 1)
await saveSkills()
return true
}