mirror of
https://github.com/violettoolssite/CFspider.git
synced 2026-04-05 03:09:01 +08:00
feat: 娣诲姞鐪熶汉瀛︿範绯荤粺鍜岃棰戞€荤粨鍔熻兘(鏈€澶?0甯?
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { useStore } from '../store'
|
||||
import { matchSkill, updateSkillLearning, type Skill } from './skills'
|
||||
|
||||
const isElectron = typeof window !== 'undefined' && (window as any).electronAPI !== undefined
|
||||
|
||||
@@ -1242,10 +1243,32 @@ export const aiTools = [
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'scan_interactive_elements',
|
||||
description: 'Scan and list all interactive elements on the page (buttons, links, inputs). Use to discover available actions.',
|
||||
description: 'Scan and list all interactive elements on the page (buttons, links, inputs). Use to discover available actions. Returns indexed list for use with click_by_index.',
|
||||
parameters: { type: 'object', properties: {} }
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'click_by_index',
|
||||
description: '通过索引点击元素。先调用 scan_interactive_elements 获取元素列表,然后使用此工具通过索引点击。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
category: {
|
||||
type: 'string',
|
||||
description: '元素类别: inputs, buttons, links, selects',
|
||||
enum: ['inputs', 'buttons', 'links', 'selects']
|
||||
},
|
||||
index: {
|
||||
type: 'number',
|
||||
description: '元素索引号(从1开始)'
|
||||
}
|
||||
},
|
||||
required: ['category', 'index']
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
@@ -1286,6 +1309,27 @@ export const aiTools = [
|
||||
required: ['selector']
|
||||
}
|
||||
}
|
||||
},
|
||||
// ==================== 视频分析工具 ====================
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'summarize_video',
|
||||
description: '总结页面上正在播放的视频内容。通过抽取关键帧进行视觉分析,生成视频摘要。需要视觉模型支持。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
frame_count: {
|
||||
type: 'number',
|
||||
description: '要分析的帧数(默认10帧,最多40帧)'
|
||||
},
|
||||
focus: {
|
||||
type: 'string',
|
||||
description: '分析重点,如 "人物"、"产品"、"教程步骤" 等(可选)'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1555,19 +1599,46 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
|
||||
// Common website domain mappings
|
||||
var domainMap = {
|
||||
'jd': ['jd.com'],
|
||||
'京东': ['jd.com'],
|
||||
'taobao': ['taobao.com', 'tmall.com'],
|
||||
'淘宝': ['taobao.com'],
|
||||
'tmall': ['tmall.com'],
|
||||
'天猫': ['tmall.com'],
|
||||
'github': ['github.com'],
|
||||
'amazon': ['amazon.com', 'amazon.cn'],
|
||||
'亚马逊': ['amazon.cn', 'amazon.com'],
|
||||
'google': ['google.com'],
|
||||
'谷歌': ['google.com'],
|
||||
'baidu': ['baidu.com'],
|
||||
'百度': ['baidu.com'],
|
||||
'bing': ['bing.com'],
|
||||
'必应': ['bing.com'],
|
||||
'microsoft': ['microsoft.com'],
|
||||
'微软': ['microsoft.com'],
|
||||
'apple': ['apple.com'],
|
||||
'苹果': ['apple.com'],
|
||||
'facebook': ['facebook.com'],
|
||||
'twitter': ['twitter.com', 'x.com'],
|
||||
'youtube': ['youtube.com'],
|
||||
'bilibili': ['bilibili.com']
|
||||
'油管': ['youtube.com'],
|
||||
'bilibili': ['bilibili.com'],
|
||||
'b站': ['bilibili.com'],
|
||||
'哔哩哔哩': ['bilibili.com'],
|
||||
'爱奇艺': ['iqiyi.com'],
|
||||
'iqiyi': ['iqiyi.com'],
|
||||
'优酷': ['youku.com'],
|
||||
'youku': ['youku.com'],
|
||||
'腾讯视频': ['v.qq.com'],
|
||||
'芒果tv': ['mgtv.com'],
|
||||
'抖音': ['douyin.com'],
|
||||
'douyin': ['douyin.com'],
|
||||
'知乎': ['zhihu.com'],
|
||||
'zhihu': ['zhihu.com'],
|
||||
'微博': ['weibo.com'],
|
||||
'weibo': ['weibo.com'],
|
||||
'网易': ['163.com'],
|
||||
'新浪': ['sina.com.cn'],
|
||||
'搜狐': ['sohu.com']
|
||||
};
|
||||
|
||||
// Find matching domain patterns
|
||||
@@ -1679,13 +1750,17 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
|
||||
var badSubdomainPrefixes = ['home.', 'my.', 'user.', 'account.', 'login.', 'passport.', 'member.', 'profile.', 'center.', 'i.', 'u.', 'sso.', 'auth.'];
|
||||
// Also check for these keywords in the result text
|
||||
var badKeywords = ['个人中心', '我的订单', '我的账户', '账户设置', '登录', 'home.', '/home', '个人信息'];
|
||||
// Skip these UI elements (Copilot, navigation tabs, images, etc.)
|
||||
// Skip these UI elements (Copilot, navigation tabs, images, sidebar cards, etc.)
|
||||
var badUIElements = [
|
||||
'copilot', 'copilotsearch', 'bingcopilot', 'b_sydConvTab', 'sydneyToggle',
|
||||
'ai生成', 'ai搜索', 'bing ai',
|
||||
'全部', '视频', '图片', '地图', '资讯', '更多',
|
||||
'b_scopeList', 'b_header', 'b_algo_group', 'b_rich',
|
||||
'/images/', 'images/search', 'image.baidu', 'images.google'
|
||||
'/images/', 'images/search', 'image.baidu', 'images.google',
|
||||
// 右侧信息卡(Wikipedia, 知识图谱等)
|
||||
'b_entityTP', 'b_sideBleed', 'b_context', 'b_overlay', 'b_pag', 'b_footer',
|
||||
'wikipedia', 'wikidata', 'wikimedia', 'youtube.com', 'youtu.be',
|
||||
'knowledge-panel', 'kno-', 'side-panel', 'sidePanel'
|
||||
];
|
||||
|
||||
// 山寨/镜像网站黑名单
|
||||
@@ -1803,7 +1878,7 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to check if element is a bad UI element (Copilot, nav tabs, etc.)
|
||||
// Helper to check if element is a bad UI element (Copilot, nav tabs, sidebar, etc.)
|
||||
function isBadUIElement(el) {
|
||||
var href = (el.href || '').toLowerCase();
|
||||
var className = (el.className || '').toLowerCase();
|
||||
@@ -1825,8 +1900,17 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
|
||||
}
|
||||
}
|
||||
|
||||
// Skip elements in header area (navigation)
|
||||
var rect = el.getBoundingClientRect();
|
||||
|
||||
// Skip elements in right sidebar area (x > 60% of viewport width)
|
||||
// This catches Wikipedia info cards, knowledge panels, etc.
|
||||
var viewportWidth = window.innerWidth;
|
||||
if (rect.left > viewportWidth * 0.6 && rect.right > viewportWidth * 0.65) {
|
||||
console.log('CFSpider: Skipping element in right sidebar:', href || el.textContent?.slice(0, 30));
|
||||
return true;
|
||||
}
|
||||
|
||||
// Skip elements in header area (navigation)
|
||||
if (rect.top < 180 && rect.top > 0) {
|
||||
// Check if it looks like a navigation item
|
||||
if (className.indexOf('scope') !== -1 || className.indexOf('nav') !== -1 ||
|
||||
@@ -2588,6 +2672,27 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
|
||||
|
||||
case 'read_full_page': {
|
||||
if (!webview) return 'Error: Cannot access page'
|
||||
|
||||
// 检查是否有视觉模型可用
|
||||
const { aiConfig: pageCfg } = store
|
||||
const useBuiltInPage = pageCfg.useBuiltIn !== false && (!pageCfg.endpoint || !pageCfg.apiKey)
|
||||
const hasVisionPage = useBuiltInPage ? !!BUILT_IN_AI.visionModel : !!pageCfg.visionModel
|
||||
|
||||
// 如果没有视觉模型,使用 DOM 提取文本
|
||||
if (!hasVisionPage) {
|
||||
try {
|
||||
const textContent = await webview.executeJavaScript(`
|
||||
(function() {
|
||||
var main = document.querySelector('main, article, .content, .main, #content, #main, body');
|
||||
return main ? main.innerText.slice(0, 10000) : document.body.innerText.slice(0, 10000);
|
||||
})()
|
||||
`)
|
||||
return '页面文本内容(DOM提取,无视觉分析):\n\n' + textContent
|
||||
} catch (e) {
|
||||
return 'Error reading page: ' + e
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const maxScrolls = (args.max_scrolls as number) || 10
|
||||
const allContents: string[] = []
|
||||
@@ -2684,6 +2789,16 @@ async function executeToolCall(name: string, args: Record<string, unknown>): Pro
|
||||
|
||||
case 'solve_captcha': {
|
||||
if (!webview) return 'Error: Cannot access page'
|
||||
|
||||
// 检查是否有视觉模型可用
|
||||
const { aiConfig } = store
|
||||
const useBuiltIn = aiConfig.useBuiltIn !== false && (!aiConfig.endpoint || !aiConfig.apiKey)
|
||||
const hasVisionModel = useBuiltIn ? !!BUILT_IN_AI.visionModel : !!aiConfig.visionModel
|
||||
|
||||
if (!hasVisionModel) {
|
||||
return '当前为单模型模式,无法使用验证码识别功能。请在设置中配置视觉模型,或使用内置 AI 服务。'
|
||||
}
|
||||
|
||||
try {
|
||||
const captchaType = args.captcha_type as string
|
||||
store.setCurrentModelType('vision')
|
||||
@@ -2878,6 +2993,16 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click
|
||||
|
||||
case 'visual_click': {
|
||||
if (!webview) return 'Error: Cannot access page'
|
||||
|
||||
// 检查是否有视觉模型可用
|
||||
const { aiConfig: aiCfg } = store
|
||||
const useBuiltInVision = aiCfg.useBuiltIn !== false && (!aiCfg.endpoint || !aiCfg.apiKey)
|
||||
const hasVision = useBuiltInVision ? !!BUILT_IN_AI.visionModel : !!aiCfg.visionModel
|
||||
|
||||
if (!hasVision) {
|
||||
return '当前为单模型模式,无法使用视觉点击功能。请尝试使用 click_text 或 click_element 替代。'
|
||||
}
|
||||
|
||||
try {
|
||||
const description = args.description as string
|
||||
|
||||
@@ -4268,6 +4393,116 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click
|
||||
}
|
||||
}
|
||||
|
||||
case 'click_by_index': {
|
||||
if (!webview) return 'Error: Cannot access page'
|
||||
const category = args.category as string
|
||||
const index = args.index as number
|
||||
|
||||
if (!category || !index) {
|
||||
return 'Error: 需要提供 category(元素类别)和 index(索引号)'
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await webview.executeJavaScript(`
|
||||
(function() {
|
||||
var category = '${category}';
|
||||
var index = ${index} - 1; // 转换为0-based索引
|
||||
var elements = [];
|
||||
|
||||
if (category === 'inputs') {
|
||||
document.querySelectorAll('input, textarea').forEach(function(el) {
|
||||
if (el.offsetWidth > 0 && el.offsetHeight > 0) {
|
||||
elements.push(el);
|
||||
}
|
||||
});
|
||||
} else if (category === 'buttons') {
|
||||
document.querySelectorAll('button, input[type="submit"], input[type="button"], [role="button"]').forEach(function(el) {
|
||||
if (el.offsetWidth > 0 && el.offsetHeight > 0) {
|
||||
elements.push(el);
|
||||
}
|
||||
});
|
||||
} else if (category === 'links') {
|
||||
var linkCount = 0;
|
||||
document.querySelectorAll('a[href]').forEach(function(el) {
|
||||
if (linkCount < 15 && el.offsetWidth > 0) {
|
||||
var text = (el.textContent || '').trim();
|
||||
if (text.length > 2) {
|
||||
elements.push(el);
|
||||
linkCount++;
|
||||
}
|
||||
}
|
||||
});
|
||||
} else if (category === 'selects') {
|
||||
document.querySelectorAll('select').forEach(function(el) {
|
||||
if (el.offsetWidth > 0) {
|
||||
elements.push(el);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (index < 0 || index >= elements.length) {
|
||||
return { success: false, error: '索引超出范围,该类别共有 ' + elements.length + ' 个元素' };
|
||||
}
|
||||
|
||||
var el = elements[index];
|
||||
var rect = el.getBoundingClientRect();
|
||||
var text = (el.textContent || el.value || el.placeholder || '').slice(0, 50).trim();
|
||||
|
||||
// 添加高亮
|
||||
var h = document.createElement('div');
|
||||
h.id = 'cfspider-agent-highlight';
|
||||
h.style.cssText = 'position:fixed;pointer-events:none;z-index:2147483647;border:4px solid #22c55e;background:rgba(34,197,94,0.2);border-radius:6px;box-shadow:0 0 20px rgba(34,197,94,0.5);';
|
||||
h.style.left = (rect.left - 4) + 'px';
|
||||
h.style.top = (rect.top - 4) + 'px';
|
||||
h.style.width = (rect.width + 8) + 'px';
|
||||
h.style.height = (rect.height + 8) + 'px';
|
||||
document.body.appendChild(h);
|
||||
|
||||
// 点击元素
|
||||
el.click();
|
||||
if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
|
||||
el.focus();
|
||||
}
|
||||
|
||||
setTimeout(function() {
|
||||
var hh = document.getElementById('cfspider-agent-highlight');
|
||||
if (hh) hh.remove();
|
||||
}, 1000);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
tag: el.tagName,
|
||||
text: text,
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2
|
||||
};
|
||||
})()
|
||||
`)
|
||||
|
||||
if (!result.success) {
|
||||
return `点击失败: ${result.error}`
|
||||
}
|
||||
|
||||
// 显示虚拟鼠标动画
|
||||
const container = document.getElementById('browser-container')
|
||||
if (container) {
|
||||
const containerRect = container.getBoundingClientRect()
|
||||
const x = result.x + containerRect.left
|
||||
const y = result.y + containerRect.top
|
||||
store.showMouse()
|
||||
store.moveMouse(x, y, 400)
|
||||
await new Promise(resolve => setTimeout(resolve, 450))
|
||||
store.clickMouse()
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 500))
|
||||
|
||||
return `已点击 ${category} 类别的第 ${index} 个元素: <${result.tag}> "${result.text}"`
|
||||
} catch (e) {
|
||||
return `点击失败: ${e}`
|
||||
}
|
||||
}
|
||||
|
||||
case 'get_page_content': {
|
||||
if (!webview) return 'Error: Cannot access page'
|
||||
try {
|
||||
@@ -4399,6 +4634,266 @@ ${detectedType === 'click' ? '1. 按顺序使用 click_element 或 visual_click
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 视频分析工具 ====================
|
||||
case 'summarize_video': {
|
||||
if (!webview) return 'Error: Cannot access page'
|
||||
|
||||
// 检查视觉模型是否可用
|
||||
const { aiConfig: videoConfig } = store
|
||||
const useBuiltInVideo = videoConfig.useBuiltIn !== false && (!videoConfig.endpoint || !videoConfig.apiKey)
|
||||
const hasVisionForVideo = useBuiltInVideo ? !!BUILT_IN_AI.visionModel : !!videoConfig.visionModel
|
||||
|
||||
if (!hasVisionForVideo) {
|
||||
return '视频总结需要视觉模型支持。请在设置中配置视觉模型,或使用内置 AI 服务。'
|
||||
}
|
||||
|
||||
try {
|
||||
const frameCount = Math.min(Math.max((args.frame_count as number) || 10, 1), 40)
|
||||
const focus = (args.focus as string) || ''
|
||||
|
||||
store.setCurrentModelType('vision')
|
||||
console.log('[CFSpider] 开始视频分析,计划抽取', frameCount, '帧')
|
||||
|
||||
// 检测页面上的视频元素
|
||||
const videoInfo = await webview.executeJavaScript(`
|
||||
(function() {
|
||||
// 查找视频元素
|
||||
var video = document.querySelector('video');
|
||||
if (!video) {
|
||||
// 尝试查找 iframe 中的视频(如 YouTube)
|
||||
var iframes = document.querySelectorAll('iframe');
|
||||
for (var i = 0; i < iframes.length; i++) {
|
||||
var src = iframes[i].src || '';
|
||||
if (src.includes('youtube') || src.includes('bilibili') || src.includes('youku') || src.includes('iqiyi')) {
|
||||
return {
|
||||
type: 'iframe',
|
||||
platform: src.includes('youtube') ? 'YouTube' :
|
||||
src.includes('bilibili') ? 'Bilibili' :
|
||||
src.includes('youku') ? '优酷' :
|
||||
src.includes('iqiyi') ? '爱奇艺' : '视频平台',
|
||||
src: src
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'video',
|
||||
duration: video.duration || 0,
|
||||
currentTime: video.currentTime || 0,
|
||||
paused: video.paused,
|
||||
width: video.videoWidth,
|
||||
height: video.videoHeight,
|
||||
src: video.src || video.currentSrc
|
||||
};
|
||||
})()
|
||||
`)
|
||||
|
||||
if (!videoInfo) {
|
||||
store.setCurrentModelType(null)
|
||||
return '页面上未找到视频元素。请确保视频正在播放或页面包含视频播放器。'
|
||||
}
|
||||
|
||||
// 如果是 iframe 嵌入的视频(如 YouTube),使用页面截图分析
|
||||
if (videoInfo.type === 'iframe') {
|
||||
console.log('[CFSpider] 检测到嵌入式视频平台:', videoInfo.platform)
|
||||
|
||||
// 对嵌入视频,直接截图分析当前画面(最多分析10帧,因为无法控制进度)
|
||||
const iframeFrameCount = Math.min(frameCount, 10)
|
||||
const frameAnalyses: string[] = []
|
||||
|
||||
for (let i = 0; i < iframeFrameCount; i++) {
|
||||
const image = await webview.capturePage()
|
||||
if (!image) continue
|
||||
|
||||
const base64Image = image.toDataURL().replace(/^data:image\/\w+;base64,/, '')
|
||||
|
||||
const response = await (window as any).electronAPI.aiChat({
|
||||
endpoint: BUILT_IN_AI.endpoint,
|
||||
apiKey: getBuiltInKey(),
|
||||
model: BUILT_IN_AI.visionModel,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `这是一个${videoInfo.platform}视频的第${i + 1}帧截图。请分析视频画面内容:
|
||||
${focus ? `重点关注:${focus}` : ''}
|
||||
|
||||
请描述:
|
||||
1. 画面中的主要内容
|
||||
2. 视频类型(教程、娱乐、新闻等)
|
||||
3. 关键信息或字幕内容(如有)
|
||||
|
||||
简洁回答,50字以内。`
|
||||
},
|
||||
{ type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } }
|
||||
]
|
||||
}]
|
||||
})
|
||||
|
||||
if (response.content) {
|
||||
frameAnalyses.push(`帧${i + 1}: ${response.content}`)
|
||||
}
|
||||
|
||||
// 等待一小段时间再截下一帧
|
||||
if (i < iframeFrameCount - 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, 1500))
|
||||
}
|
||||
}
|
||||
|
||||
store.setCurrentModelType(null)
|
||||
|
||||
if (frameAnalyses.length === 0) {
|
||||
return `检测到${videoInfo.platform}视频,但无法分析画面内容。`
|
||||
}
|
||||
|
||||
// 生成总结
|
||||
let summary = `[${videoInfo.platform}视频分析] 共分析${frameAnalyses.length}帧\n\n`
|
||||
summary += frameAnalyses.join('\n\n')
|
||||
|
||||
if (frameAnalyses.length > 1) {
|
||||
// 请求生成整体总结
|
||||
const summaryResponse = await (window as any).electronAPI.aiChat({
|
||||
endpoint: useBuiltInVideo ? BUILT_IN_AI.endpoint : videoConfig.endpoint,
|
||||
apiKey: useBuiltInVideo ? getBuiltInKey() : videoConfig.apiKey,
|
||||
model: useBuiltInVideo ? BUILT_IN_AI.model : videoConfig.model,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: `根据以下视频帧分析,生成一个简短的视频总结(150字以内):\n\n${frameAnalyses.join('\n')}`
|
||||
}]
|
||||
})
|
||||
|
||||
if (summaryResponse.content) {
|
||||
summary += `\n\n[总结]\n${summaryResponse.content}`
|
||||
}
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
// 原生 video 元素 - 通过控制播放进度抽取帧
|
||||
const duration = videoInfo.duration
|
||||
if (duration <= 0) {
|
||||
store.setCurrentModelType(null)
|
||||
return '视频时长未知或为直播流,无法进行帧分析。'
|
||||
}
|
||||
|
||||
console.log('[CFSpider] 视频时长:', duration, '秒,抽取', frameCount, '帧')
|
||||
|
||||
// 计算抽帧时间点(均匀分布)
|
||||
const timePoints: number[] = []
|
||||
for (let i = 0; i < frameCount; i++) {
|
||||
timePoints.push((duration / (frameCount + 1)) * (i + 1))
|
||||
}
|
||||
|
||||
const frameAnalyses: string[] = []
|
||||
|
||||
// 批量分析帧(每5帧为一组)
|
||||
const batchSize = 5
|
||||
for (let batch = 0; batch < Math.ceil(timePoints.length / batchSize); batch++) {
|
||||
const batchStart = batch * batchSize
|
||||
const batchEnd = Math.min(batchStart + batchSize, timePoints.length)
|
||||
|
||||
for (let i = batchStart; i < batchEnd; i++) {
|
||||
const targetTime = timePoints[i]
|
||||
|
||||
// 跳转到指定时间点
|
||||
await webview.executeJavaScript(`
|
||||
(function() {
|
||||
var video = document.querySelector('video');
|
||||
if (video) {
|
||||
video.currentTime = ${targetTime};
|
||||
video.pause();
|
||||
}
|
||||
})()
|
||||
`)
|
||||
|
||||
// 等待视频加载该帧
|
||||
await new Promise(resolve => setTimeout(resolve, 300))
|
||||
|
||||
// 截图
|
||||
const image = await webview.capturePage()
|
||||
if (!image) continue
|
||||
|
||||
const base64Image = image.toDataURL().replace(/^data:image\/\w+;base64,/, '')
|
||||
const timeStr = `${Math.floor(targetTime / 60)}:${String(Math.floor(targetTime % 60)).padStart(2, '0')}`
|
||||
|
||||
// 分析帧内容
|
||||
const response = await (window as any).electronAPI.aiChat({
|
||||
endpoint: BUILT_IN_AI.endpoint,
|
||||
apiKey: getBuiltInKey(),
|
||||
model: BUILT_IN_AI.visionModel,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `这是视频在 ${timeStr} 时刻的画面。请简短描述画面内容(30字以内)。${focus ? `重点关注:${focus}` : ''}`
|
||||
},
|
||||
{ type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } }
|
||||
]
|
||||
}]
|
||||
})
|
||||
|
||||
if (response.content) {
|
||||
frameAnalyses.push(`[${timeStr}] ${response.content}`)
|
||||
}
|
||||
}
|
||||
|
||||
// 每批次之间短暂休息
|
||||
if (batch < Math.ceil(timePoints.length / batchSize) - 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, 200))
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复视频播放
|
||||
await webview.executeJavaScript(`
|
||||
(function() {
|
||||
var video = document.querySelector('video');
|
||||
if (video) video.play();
|
||||
})()
|
||||
`)
|
||||
|
||||
store.setCurrentModelType(null)
|
||||
|
||||
if (frameAnalyses.length === 0) {
|
||||
return '视频帧分析失败,无法获取有效信息。'
|
||||
}
|
||||
|
||||
// 生成视频总结
|
||||
const durationMin = Math.floor(duration / 60)
|
||||
const durationSec = Math.floor(duration % 60)
|
||||
let summary = `[视频分析] 时长: ${durationMin}分${durationSec}秒 | 分析帧数: ${frameAnalyses.length}\n\n`
|
||||
summary += '关键帧内容:\n'
|
||||
summary += frameAnalyses.join('\n')
|
||||
|
||||
// 生成整体总结
|
||||
if (frameAnalyses.length > 2) {
|
||||
const summaryResponse = await (window as any).electronAPI.aiChat({
|
||||
endpoint: useBuiltInVideo ? BUILT_IN_AI.endpoint : videoConfig.endpoint,
|
||||
apiKey: useBuiltInVideo ? getBuiltInKey() : videoConfig.apiKey,
|
||||
model: useBuiltInVideo ? BUILT_IN_AI.model : videoConfig.model,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: `根据以下视频帧分析,生成一个详细的视频总结(200字以内):\n\n${frameAnalyses.join('\n')}`
|
||||
}]
|
||||
})
|
||||
|
||||
if (summaryResponse.content) {
|
||||
summary += `\n\n[总结]\n${summaryResponse.content}`
|
||||
}
|
||||
}
|
||||
|
||||
return summary
|
||||
} catch (e) {
|
||||
store.setCurrentModelType(null)
|
||||
console.error('[CFSpider] 视频分析失败:', e)
|
||||
return `视频分析失败: ${e}`
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
return 'Unknown tool: ' + name
|
||||
}
|
||||
@@ -4431,9 +4926,9 @@ const systemPrompt = `你是 CFspider 智能浏览器自动化助手,由 viole
|
||||
对于这些情况,只需用中文自然回复,不要调用任何工具。
|
||||
|
||||
示例:
|
||||
- 用户:"你好" -> 回复:"嗨~你好!我是 CFspider 智能浏览器助手,有什么可以帮你的吗?"
|
||||
- 用户:"你是谁" -> 回复:"我是 CFspider 智能浏览器 AI 助手,由 violetteam 团队开发,来自 cfspider 项目。我可以帮你自动化浏览器操作,比如搜索、点击、导航网站等,交给我就行~"
|
||||
- 用户:"谢谢" -> 回复:"不客气呀!有需要随时叫我~"
|
||||
- 用户:"你好" -> 回复:"你好,我是 CFspider 智能浏览器助手,有什么可以帮你的吗?"
|
||||
- 用户:"你是谁" -> 回复:"你好,我是 CFspider 智能浏览器 AI 助手。{{MODEL_INTRODUCTION}} CFspider 工具由 violetteam 团队开发。我可以帮你自动化浏览器操作,比如搜索、点击、导航网站等。"
|
||||
- 用户:"谢谢" -> 回复:"不客气,有需要随时叫我。"
|
||||
|
||||
### 以下情况使用工具:
|
||||
- 打开网站:"打开京东"、"去 GitHub"
|
||||
@@ -4843,10 +5338,54 @@ export async function sendAIMessage(content: string, useTools: boolean = true) {
|
||||
|
||||
try {
|
||||
// 构建聊天历史,包含工具调用信息以便 AI 记住之前的操作
|
||||
|
||||
// 动态生成模型名介绍
|
||||
const toolModelName = useBuiltIn
|
||||
? BUILT_IN_AI.model.split('/').pop() || 'DeepSeek-V3'
|
||||
: (effectiveConfig.model || '未配置')
|
||||
const visionModelName = useBuiltIn
|
||||
? BUILT_IN_AI.visionModel?.split('/').pop() || ''
|
||||
: (aiConfig.visionModel?.split('/').pop() || '')
|
||||
|
||||
// 提取模型开发团队(从模型名推断)
|
||||
const getModelTeam = (modelName: string): string => {
|
||||
const name = modelName.toLowerCase()
|
||||
if (name.includes('deepseek')) return 'DeepSeek'
|
||||
if (name.includes('qwen')) return '阿里云通义千问'
|
||||
if (name.includes('glm') || name.includes('chatglm')) return '智谱 AI'
|
||||
if (name.includes('gpt')) return 'OpenAI'
|
||||
if (name.includes('claude')) return 'Anthropic'
|
||||
if (name.includes('gemini')) return 'Google'
|
||||
if (name.includes('llama')) return 'Meta'
|
||||
if (name.includes('mistral')) return 'Mistral AI'
|
||||
if (name.includes('yi')) return '零一万物'
|
||||
if (name.includes('baichuan')) return '百川智能'
|
||||
if (name.includes('moonshot') || name.includes('kimi')) return 'Moonshot AI'
|
||||
return ''
|
||||
}
|
||||
|
||||
const toolModelTeam = getModelTeam(toolModelName)
|
||||
const visionModelTeam = visionModelName ? getModelTeam(visionModelName) : ''
|
||||
|
||||
let modelIntroduction = ''
|
||||
if (modelMode === 'dual' && visionModelName) {
|
||||
const toolTeamStr = toolModelTeam ? `(由 ${toolModelTeam} 开发)` : ''
|
||||
const visionTeamStr = visionModelTeam ? `(由 ${visionModelTeam} 开发)` : ''
|
||||
modelIntroduction = `目前由两个模型共同驱动:${toolModelName}${toolTeamStr}负责文本理解和工具调用,${visionModelName}${visionTeamStr}负责页面视觉分析。`
|
||||
} else {
|
||||
const teamStr = toolModelTeam ? `(由 ${toolModelTeam} 开发)` : ''
|
||||
modelIntroduction = `目前由 ${toolModelName} 模型${teamStr}驱动。`
|
||||
}
|
||||
|
||||
// 替换系统提示中的模型名占位符
|
||||
const dynamicSystemPrompt = systemPrompt
|
||||
.replace(/\{\{TOOL_MODEL_NAME\}\}/g, toolModelName)
|
||||
.replace(/\{\{MODEL_INTRODUCTION\}\}/g, modelIntroduction)
|
||||
|
||||
// 如果有 OCR 页面分析结果,添加到系统提示词中
|
||||
const enhancedSystemPrompt = pageContext
|
||||
? `${systemPrompt}\n\n## 当前页面分析结果(由视觉模型提供)\n\n${pageContext}\n\n请根据以上页面分析结果来决定下一步操作。`
|
||||
: systemPrompt
|
||||
? `${dynamicSystemPrompt}\n\n## 当前页面分析结果(由视觉模型提供)\n\n${pageContext}\n\n请根据以上页面分析结果来决定下一步操作。`
|
||||
: dynamicSystemPrompt
|
||||
|
||||
const chatHistory: Array<{ role: string; content?: string; tool_calls?: any[]; tool_call_id?: string; name?: string }> = [
|
||||
{ role: 'system', content: enhancedSystemPrompt }
|
||||
@@ -4883,6 +5422,28 @@ export async function sendAIMessage(content: string, useTools: boolean = true) {
|
||||
// 添加当前用户消息
|
||||
chatHistory.push({ role: 'user', content })
|
||||
|
||||
// 尝试匹配技能,提供给 AI 作为参考
|
||||
let matchedSkill: Skill | null = null
|
||||
try {
|
||||
const webview = document.querySelector('webview') as any
|
||||
if (webview) {
|
||||
const currentUrl = await webview.executeJavaScript('window.location.href') as string
|
||||
const currentDomain = new URL(currentUrl).hostname
|
||||
matchedSkill = await matchSkill(content, currentDomain)
|
||||
if (matchedSkill) {
|
||||
console.log('[CFSpider] 匹配到技能:', matchedSkill.name, '成功率:', matchedSkill.successRate)
|
||||
// 将技能信息添加到聊天历史作为系统提示
|
||||
const skillHint = `[技能提示] 匹配到「${matchedSkill.name}」技能(成功率: ${matchedSkill.successRate}%)
|
||||
触发词: ${matchedSkill.triggers.join(', ')}
|
||||
操作步骤: ${matchedSkill.steps.map((s, i) => `${i+1}. ${s.action}${s.target ? `: ${s.target}` : ''}`).join(' -> ')}
|
||||
${matchedSkill.learnedPatterns.length > 0 ? `学习到的模式: ${matchedSkill.learnedPatterns.slice(0, 3).map(p => p.pattern).join(', ')}` : ''}`
|
||||
chatHistory.push({ role: 'system', content: skillHint })
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[CFSpider] 技能匹配失败:', e)
|
||||
}
|
||||
|
||||
let iteration = 0
|
||||
const maxIterations = 30
|
||||
|
||||
@@ -4971,7 +5532,8 @@ export async function sendAIMessage(content: string, useTools: boolean = true) {
|
||||
updateLastMessageWithToolCalls('', toolCallHistory)
|
||||
|
||||
// 如果操作失败,触发紧张模式和语气词反应
|
||||
if (result.includes('Error') || result.includes('失败') || result.includes('not found') || result.includes('Cannot')) {
|
||||
const operationFailed = result.includes('Error') || result.includes('失败') || result.includes('not found') || result.includes('Cannot')
|
||||
if (operationFailed) {
|
||||
// 触发紧张乱动
|
||||
store.panicMouse(1500)
|
||||
|
||||
@@ -4984,6 +5546,23 @@ export async function sendAIMessage(content: string, useTools: boolean = true) {
|
||||
// 等待紧张动画完成
|
||||
await new Promise(resolve => setTimeout(resolve, 1200))
|
||||
}
|
||||
|
||||
// 更新技能学习(如果匹配到了技能)
|
||||
if (matchedSkill) {
|
||||
try {
|
||||
await updateSkillLearning(matchedSkill.id, !operationFailed, {
|
||||
pattern: `${funcName}:${JSON.stringify(funcArgs).slice(0, 50)}`,
|
||||
selector: funcArgs.selector || funcArgs.text || funcArgs.target,
|
||||
confidence: operationFailed ? 30 : 70,
|
||||
successCount: operationFailed ? 0 : 1,
|
||||
failureCount: operationFailed ? 1 : 0,
|
||||
lastUsed: Date.now(),
|
||||
examples: operationFailed ? [] : [result.slice(0, 100)]
|
||||
})
|
||||
} catch (e) {
|
||||
console.error('[CFSpider] 技能学习更新失败:', e)
|
||||
}
|
||||
}
|
||||
|
||||
chatHistory.push({
|
||||
role: 'assistant',
|
||||
|
||||
272
cfspider-browser/src/services/builtinSkills.ts
Normal file
272
cfspider-browser/src/services/builtinSkills.ts
Normal file
@@ -0,0 +1,272 @@
|
||||
/**
|
||||
* 内置技能定义
|
||||
* 从 Markdown 文件解析而来
|
||||
*/
|
||||
import { Skill } from './skills'
|
||||
|
||||
export const BUILT_IN_SKILLS: Skill[] = [
|
||||
// 必应搜索技能
|
||||
{
|
||||
id: 'bing-search',
|
||||
name: '必应搜索',
|
||||
description: '在必应搜索引擎上搜索关键词',
|
||||
triggers: ['搜索', '查找', '找一下', 'search', '查', '百度一下'],
|
||||
domains: ['bing.com', 'cn.bing.com'],
|
||||
steps: [
|
||||
{
|
||||
action: 'scan',
|
||||
},
|
||||
{
|
||||
action: 'input',
|
||||
target: '#sb_form_q, input[name="q"], textarea[name="q"]',
|
||||
value: '{query}',
|
||||
fallbacks: ['input[type="search"]', '.search-box input']
|
||||
},
|
||||
{
|
||||
action: 'click',
|
||||
target: '#sb_form_go, #search_icon, button[type="submit"]',
|
||||
fallbacks: ['按回车键', 'form button'],
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
action: 'wait',
|
||||
value: '1500',
|
||||
successIndicator: 'URL包含 /search?q='
|
||||
}
|
||||
],
|
||||
successRate: 95,
|
||||
usageCount: 0,
|
||||
lastUsed: 0,
|
||||
learnedPatterns: [],
|
||||
isBuiltIn: true,
|
||||
version: 1
|
||||
},
|
||||
|
||||
// 点击搜索结果技能
|
||||
{
|
||||
id: 'click-search-result',
|
||||
name: '点击搜索结果',
|
||||
description: '在搜索引擎结果页面中点击指定的链接',
|
||||
triggers: ['打开', '点击', '进入', '访问'],
|
||||
domains: ['bing.com', 'cn.bing.com', 'baidu.com', 'google.com'],
|
||||
steps: [
|
||||
{
|
||||
action: 'scan',
|
||||
},
|
||||
{
|
||||
action: 'verify',
|
||||
value: '识别目标链接,排除 Copilot、图片搜索、山寨网站、翻页按钮'
|
||||
},
|
||||
{
|
||||
action: 'click',
|
||||
target: 'click_by_index(type="link", index={index})',
|
||||
fallbacks: ['visual_click("{target_name}")', 'click_text("{target_domain}")']
|
||||
},
|
||||
{
|
||||
action: 'wait',
|
||||
value: '2000'
|
||||
},
|
||||
{
|
||||
action: 'verify',
|
||||
successIndicator: 'URL 包含目标域名'
|
||||
}
|
||||
],
|
||||
successRate: 85,
|
||||
usageCount: 0,
|
||||
lastUsed: 0,
|
||||
learnedPatterns: [],
|
||||
isBuiltIn: true,
|
||||
version: 1
|
||||
},
|
||||
|
||||
// 访问网站技能
|
||||
{
|
||||
id: 'navigate-to-website',
|
||||
name: '访问网站',
|
||||
description: '通过搜索引擎搜索并访问目标网站官网',
|
||||
triggers: ['打开', '进入', '访问', '去', '跳转到'],
|
||||
domains: [], // 通用技能
|
||||
steps: [
|
||||
{
|
||||
action: 'verify',
|
||||
value: '检查是否已经在目标网站或搜索引擎'
|
||||
},
|
||||
{
|
||||
action: 'navigate',
|
||||
target: 'https://cn.bing.com',
|
||||
optional: true,
|
||||
successIndicator: '如果当前不在搜索引擎'
|
||||
},
|
||||
{
|
||||
action: 'input',
|
||||
target: '#sb_form_q',
|
||||
value: '{site_name}'
|
||||
},
|
||||
{
|
||||
action: 'click',
|
||||
target: '#sb_form_go'
|
||||
},
|
||||
{
|
||||
action: 'wait',
|
||||
value: '1500'
|
||||
},
|
||||
{
|
||||
action: 'scan'
|
||||
},
|
||||
{
|
||||
action: 'click',
|
||||
target: 'click_by_index(type="link", index={index})',
|
||||
value: '验证链接 href 包含目标域名'
|
||||
},
|
||||
{
|
||||
action: 'wait',
|
||||
value: '2000'
|
||||
},
|
||||
{
|
||||
action: 'verify',
|
||||
successIndicator: 'URL 包含目标域名'
|
||||
}
|
||||
],
|
||||
successRate: 90,
|
||||
usageCount: 0,
|
||||
lastUsed: 0,
|
||||
learnedPatterns: [],
|
||||
isBuiltIn: true,
|
||||
version: 1
|
||||
},
|
||||
|
||||
// 爱奇艺导航技能
|
||||
{
|
||||
id: 'iqiyi-navigation',
|
||||
name: '爱奇艺导航',
|
||||
description: '在爱奇艺网站上导航和搜索视频',
|
||||
triggers: ['搜索视频', '找电影', '看剧', '播放'],
|
||||
domains: ['iqiyi.com'],
|
||||
steps: [
|
||||
{
|
||||
action: 'scan'
|
||||
},
|
||||
{
|
||||
action: 'input',
|
||||
target: '.search-input, input[placeholder*="搜索"]',
|
||||
value: '{query}'
|
||||
},
|
||||
{
|
||||
action: 'click',
|
||||
target: '.search-btn, button[class*="search"]',
|
||||
fallbacks: ['按回车键']
|
||||
},
|
||||
{
|
||||
action: 'wait',
|
||||
value: '2000',
|
||||
successIndicator: 'URL 包含 /search/'
|
||||
}
|
||||
],
|
||||
successRate: 80,
|
||||
usageCount: 0,
|
||||
lastUsed: 0,
|
||||
learnedPatterns: [],
|
||||
isBuiltIn: true,
|
||||
version: 1
|
||||
},
|
||||
|
||||
// 通用表单填写技能
|
||||
{
|
||||
id: 'fill-form',
|
||||
name: '填写表单',
|
||||
description: '智能识别并填写网页表单',
|
||||
triggers: ['填写', '输入信息', '注册', '登录'],
|
||||
domains: [], // 通用
|
||||
steps: [
|
||||
{
|
||||
action: 'scan'
|
||||
},
|
||||
{
|
||||
action: 'input',
|
||||
target: '{detected_input}',
|
||||
value: '{value}'
|
||||
},
|
||||
{
|
||||
action: 'verify',
|
||||
value: '确认输入成功'
|
||||
}
|
||||
],
|
||||
successRate: 85,
|
||||
usageCount: 0,
|
||||
lastUsed: 0,
|
||||
learnedPatterns: [],
|
||||
isBuiltIn: true,
|
||||
version: 1
|
||||
},
|
||||
|
||||
// 滚动查看内容技能
|
||||
{
|
||||
id: 'scroll-and-read',
|
||||
name: '滚动查看',
|
||||
description: '滚动页面并阅读完整内容',
|
||||
triggers: ['阅读', '查看', '总结', '看看'],
|
||||
domains: [], // 通用
|
||||
steps: [
|
||||
{
|
||||
action: 'verify',
|
||||
value: '获取当前滚动位置'
|
||||
},
|
||||
{
|
||||
action: 'scroll',
|
||||
target: 'down',
|
||||
value: '500'
|
||||
},
|
||||
{
|
||||
action: 'wait',
|
||||
value: '500'
|
||||
},
|
||||
{
|
||||
action: 'verify',
|
||||
value: '检查是否到达页面底部'
|
||||
}
|
||||
],
|
||||
successRate: 95,
|
||||
usageCount: 0,
|
||||
lastUsed: 0,
|
||||
learnedPatterns: [],
|
||||
isBuiltIn: true,
|
||||
version: 1
|
||||
},
|
||||
|
||||
// 智能登录技能
|
||||
{
|
||||
id: 'auto-login',
|
||||
name: '智能登录',
|
||||
description: '检测登录需求,智能处理自动登录或手动登录',
|
||||
triggers: ['登录', 'login', '注册', 'signup', '需要登录'],
|
||||
domains: [], // 通用
|
||||
steps: [
|
||||
{
|
||||
action: 'verify',
|
||||
value: 'detect_login 检测是否需要登录'
|
||||
},
|
||||
{
|
||||
action: 'verify',
|
||||
value: 'request_login_choice 询问用户选择'
|
||||
},
|
||||
{
|
||||
action: 'input',
|
||||
value: 'auto_login 或等待手动登录'
|
||||
},
|
||||
{
|
||||
action: 'wait',
|
||||
value: '2000'
|
||||
},
|
||||
{
|
||||
action: 'verify',
|
||||
value: '验证登录成功'
|
||||
}
|
||||
],
|
||||
successRate: 70,
|
||||
usageCount: 0,
|
||||
lastUsed: 0,
|
||||
learnedPatterns: [],
|
||||
isBuiltIn: true,
|
||||
version: 1
|
||||
}
|
||||
]
|
||||
371
cfspider-browser/src/services/skills.ts
Normal file
371
cfspider-browser/src/services/skills.ts
Normal file
@@ -0,0 +1,371 @@
|
||||
/**
|
||||
* CFSpider 技能系统
|
||||
* 让 AI 针对特定网站和任务有预定义的操作流程
|
||||
*/
|
||||
|
||||
// ========== 数据结构定义 ==========
|
||||
|
||||
// 技能步骤动作类型
|
||||
export type SkillAction = 'click' | 'input' | 'scroll' | 'wait' | 'verify' | 'navigate' | 'scan'
|
||||
|
||||
// 技能步骤
|
||||
export interface SkillStep {
|
||||
action: SkillAction
|
||||
target?: string // CSS 选择器或元素描述
|
||||
value?: string // 输入值或参数(支持 {query} 等占位符)
|
||||
fallbacks?: string[] // 备选方案(选择器或操作描述)
|
||||
successIndicator?: string // 成功标志(URL 变化、元素出现等)
|
||||
timeout?: number // 超时时间(毫秒)
|
||||
optional?: boolean // 是否可选(失败不中断)
|
||||
}
|
||||
|
||||
// 学习到的模式
|
||||
export interface LearnedPattern {
|
||||
pattern: string // 模式描述
|
||||
selector?: string // 关联的选择器
|
||||
confidence: number // 置信度 0-100
|
||||
successCount: number // 成功次数
|
||||
failureCount: number // 失败次数
|
||||
lastUsed: number // 上次使用时间
|
||||
examples: string[] // 成功实例(最多5个)
|
||||
}
|
||||
|
||||
// 技能定义
|
||||
export interface Skill {
|
||||
id: string // 唯一标识
|
||||
name: string // 技能名称,如"必应搜索"
|
||||
description: string // 技能描述
|
||||
triggers: string[] // 触发词,如["搜索", "查找", "百度一下"]
|
||||
domains: string[] // 适用域名,如["bing.com", "baidu.com"],空数组表示通用
|
||||
steps: SkillStep[] // 操作步骤
|
||||
successRate: number // 成功率 0-100
|
||||
usageCount: number // 使用次数
|
||||
lastUsed: number // 上次使用时间
|
||||
learnedPatterns: LearnedPattern[] // 学习到的模式
|
||||
isBuiltIn: boolean // 是否为内置技能
|
||||
version: number // 版本号(用于更新)
|
||||
}
|
||||
|
||||
// 技能执行结果
|
||||
export interface SkillExecutionResult {
|
||||
success: boolean
|
||||
skillId: string
|
||||
stepsCompleted: number
|
||||
totalSteps: number
|
||||
finalUrl?: string
|
||||
finalTitle?: string
|
||||
error?: string
|
||||
executionLog: SkillExecutionLog[]
|
||||
}
|
||||
|
||||
// 执行日志
|
||||
export interface SkillExecutionLog {
|
||||
step: number
|
||||
action: SkillAction
|
||||
target?: string
|
||||
result: 'success' | 'failure' | 'skipped'
|
||||
duration: number
|
||||
error?: string
|
||||
fallbackUsed?: string
|
||||
}
|
||||
|
||||
// ========== 技能缓存 ==========
|
||||
|
||||
// 内存中的技能缓存
|
||||
let skillsCache: Skill[] = []
|
||||
let skillsLoaded = false
|
||||
|
||||
// 检查是否在 Electron 环境
|
||||
const isElectron = typeof window !== 'undefined' && (window as any).electronAPI !== undefined
|
||||
|
||||
// ========== 核心函数 ==========
|
||||
|
||||
/**
|
||||
* 加载技能(从持久化存储)
|
||||
*/
|
||||
export async function loadSkills(): Promise<Skill[]> {
|
||||
if (skillsLoaded) return skillsCache
|
||||
|
||||
try {
|
||||
if (isElectron && (window as any).electronAPI.loadSkills) {
|
||||
const savedSkills = await (window as any).electronAPI.loadSkills() as Skill[]
|
||||
if (Array.isArray(savedSkills)) {
|
||||
// 合并内置技能和保存的技能
|
||||
let BUILT_IN_SKILLS: Skill[] = []
|
||||
try {
|
||||
// @ts-ignore - 动态导入,运行时正常
|
||||
const builtinSkillsModule = await import('./builtinSkills')
|
||||
BUILT_IN_SKILLS = builtinSkillsModule.BUILT_IN_SKILLS
|
||||
} catch (e) {
|
||||
console.error('[Skills] Failed to load built-in skills:', e)
|
||||
BUILT_IN_SKILLS = []
|
||||
}
|
||||
|
||||
// 内置技能始终使用最新版本
|
||||
const builtInMap = new Map(BUILT_IN_SKILLS.map(s => [s.id, s]))
|
||||
|
||||
// 合并:保留用户技能的学习数据,但使用内置技能的步骤定义
|
||||
skillsCache = BUILT_IN_SKILLS.map((builtIn: Skill) => {
|
||||
const saved_skill = savedSkills.find((s: Skill) => s.id === builtIn.id)
|
||||
if (saved_skill) {
|
||||
return {
|
||||
...builtIn,
|
||||
successRate: saved_skill.successRate,
|
||||
usageCount: saved_skill.usageCount,
|
||||
lastUsed: saved_skill.lastUsed,
|
||||
learnedPatterns: saved_skill.learnedPatterns || []
|
||||
}
|
||||
}
|
||||
return builtIn
|
||||
})
|
||||
|
||||
// 添加用户自定义技能(包括从学习中提升的技能)
|
||||
const userSkills = savedSkills.filter((s: Skill) => !builtInMap.has(s.id))
|
||||
skillsCache.push(...userSkills)
|
||||
|
||||
console.log('[Skills] Loaded skills:', skillsCache.length, '(内置:', BUILT_IN_SKILLS.length, '学习:', userSkills.length, ')')
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[Skills] Failed to load:', e)
|
||||
}
|
||||
|
||||
// 如果没有加载到技能,使用内置技能
|
||||
if (skillsCache.length === 0) {
|
||||
try {
|
||||
// @ts-ignore - 动态导入,运行时正常
|
||||
const builtinSkillsModule = await import('./builtinSkills')
|
||||
skillsCache = [...builtinSkillsModule.BUILT_IN_SKILLS]
|
||||
} catch (e) {
|
||||
console.error('[Skills] Failed to load built-in skills:', e)
|
||||
skillsCache = []
|
||||
}
|
||||
}
|
||||
|
||||
skillsLoaded = true
|
||||
return skillsCache
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存技能(到持久化存储)- 永久保存,无容量限制
|
||||
*/
|
||||
export async function saveSkills(): Promise<void> {
|
||||
if (!isElectron || !(window as any).electronAPI.saveSkills) return
|
||||
|
||||
try {
|
||||
// 保存所有有学习数据的技能(包括内置技能的学习数据和用户技能)
|
||||
// 永久保存,不删除(像真人一辈子都会)
|
||||
const toSave = skillsCache.filter(s =>
|
||||
s.usageCount > 0 || s.learnedPatterns.length > 0 || !s.isBuiltIn
|
||||
)
|
||||
await (window as any).electronAPI.saveSkills(toSave)
|
||||
console.log('[Skills] Permanently saved:', toSave.length, '技能(永久保存,无容量限制)')
|
||||
} catch (e) {
|
||||
console.error('[Skills] Failed to save:', e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据用户指令和当前域名匹配技能
|
||||
*/
|
||||
export async function matchSkill(instruction: string, currentDomain: string): Promise<Skill | null> {
|
||||
await loadSkills()
|
||||
|
||||
const instructionLower = instruction.toLowerCase()
|
||||
|
||||
// 评分匹配
|
||||
let bestMatch: Skill | null = null
|
||||
let bestScore = 0
|
||||
|
||||
for (const skill of skillsCache) {
|
||||
let score = 0
|
||||
|
||||
// 检查域名匹配
|
||||
const domainMatch = skill.domains.length === 0 ||
|
||||
skill.domains.some(d => currentDomain.includes(d))
|
||||
if (!domainMatch) continue
|
||||
|
||||
// 检查触发词匹配
|
||||
for (const trigger of skill.triggers) {
|
||||
if (instructionLower.includes(trigger.toLowerCase())) {
|
||||
score += 10
|
||||
// 完全匹配加分
|
||||
if (instructionLower.startsWith(trigger.toLowerCase())) {
|
||||
score += 5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 域名精确匹配加分
|
||||
if (skill.domains.some(d => currentDomain === d || currentDomain === 'www.' + d)) {
|
||||
score += 5
|
||||
}
|
||||
|
||||
// 使用频率加分(最多 +10)
|
||||
score += Math.min(10, Math.floor(skill.usageCount / 10))
|
||||
|
||||
// 成功率加分(最多 +10)
|
||||
score += Math.floor(skill.successRate / 10)
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score
|
||||
bestMatch = skill
|
||||
}
|
||||
}
|
||||
|
||||
// 至少需要 10 分才算匹配
|
||||
return bestScore >= 10 ? bestMatch : null
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有技能
|
||||
*/
|
||||
export async function getAllSkills(): Promise<Skill[]> {
|
||||
await loadSkills()
|
||||
return skillsCache
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据 ID 获取技能
|
||||
*/
|
||||
export async function getSkillById(id: string): Promise<Skill | null> {
|
||||
await loadSkills()
|
||||
return skillsCache.find(s => s.id === id) || null
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新技能学习数据
|
||||
*/
|
||||
export async function updateSkillLearning(
|
||||
skillId: string,
|
||||
success: boolean,
|
||||
pattern?: LearnedPattern
|
||||
): Promise<void> {
|
||||
await loadSkills()
|
||||
|
||||
const skill = skillsCache.find(s => s.id === skillId)
|
||||
if (!skill) return
|
||||
|
||||
// 更新使用统计
|
||||
skill.usageCount++
|
||||
skill.lastUsed = Date.now()
|
||||
|
||||
// 更新成功率(移动平均)
|
||||
const weight = Math.min(skill.usageCount, 20) // 最多考虑最近 20 次
|
||||
skill.successRate = Math.round(
|
||||
(skill.successRate * (weight - 1) + (success ? 100 : 0)) / weight
|
||||
)
|
||||
|
||||
// 添加学习模式
|
||||
if (pattern) {
|
||||
const existing = skill.learnedPatterns.find(p => p.pattern === pattern.pattern)
|
||||
if (existing) {
|
||||
existing.confidence = Math.min(100, existing.confidence + (success ? 10 : -5))
|
||||
if (success) existing.successCount++
|
||||
else existing.failureCount++
|
||||
existing.lastUsed = Date.now()
|
||||
if (success && pattern.examples.length > 0) {
|
||||
existing.examples.push(...pattern.examples)
|
||||
existing.examples = existing.examples.slice(-5) // 只保留最近 5 个
|
||||
}
|
||||
} else {
|
||||
skill.learnedPatterns.push(pattern)
|
||||
}
|
||||
|
||||
// 只保留置信度 > 10 的模式,最多 20 个
|
||||
skill.learnedPatterns = skill.learnedPatterns
|
||||
.filter(p => p.confidence > 10)
|
||||
.sort((a, b) => b.confidence - a.confidence)
|
||||
.slice(0, 20)
|
||||
}
|
||||
|
||||
// 技能优化:从学习模式中提取最佳选择器
|
||||
if (skill.learnedPatterns.length > 0 && skill.usageCount >= 5) {
|
||||
optimizeSkillSteps(skill)
|
||||
}
|
||||
|
||||
// 随机保存(模拟渐进学习)
|
||||
if (Math.random() < 0.3) {
|
||||
await saveSkills()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 优化技能步骤(从学习模式中提取最佳实践)
|
||||
*/
|
||||
function optimizeSkillSteps(skill: Skill): void {
|
||||
// 找出置信度最高的模式
|
||||
const bestPatterns = skill.learnedPatterns
|
||||
.filter(p => p.confidence > 60 && p.successCount > 2)
|
||||
.sort((a, b) => b.confidence - a.confidence)
|
||||
.slice(0, 3)
|
||||
|
||||
if (bestPatterns.length === 0) return
|
||||
|
||||
// 将高置信度的选择器添加为备选方案
|
||||
for (const step of skill.steps) {
|
||||
if (step.action === 'click' || step.action === 'input') {
|
||||
for (const pattern of bestPatterns) {
|
||||
if (pattern.selector && !step.fallbacks?.includes(pattern.selector)) {
|
||||
step.fallbacks = step.fallbacks || []
|
||||
// 将学习到的选择器插入到前面(优先使用)
|
||||
step.fallbacks.unshift(pattern.selector)
|
||||
// 最多保留 5 个备选
|
||||
step.fallbacks = step.fallbacks.slice(0, 5)
|
||||
console.log('[Skills] Added learned selector to step:', pattern.selector)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建用户自定义技能
|
||||
*/
|
||||
export async function createSkill(
|
||||
name: string,
|
||||
description: string,
|
||||
triggers: string[],
|
||||
domains: string[],
|
||||
steps: SkillStep[]
|
||||
): Promise<Skill> {
|
||||
await loadSkills()
|
||||
|
||||
const skill: Skill = {
|
||||
id: `user-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
|
||||
name,
|
||||
description,
|
||||
triggers,
|
||||
domains,
|
||||
steps,
|
||||
successRate: 50, // 初始 50%
|
||||
usageCount: 0,
|
||||
lastUsed: 0,
|
||||
learnedPatterns: [],
|
||||
isBuiltIn: false,
|
||||
version: 1
|
||||
}
|
||||
|
||||
skillsCache.push(skill)
|
||||
await saveSkills()
|
||||
|
||||
return skill
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除技能
|
||||
*/
|
||||
export async function deleteSkill(skillId: string): Promise<boolean> {
|
||||
await loadSkills()
|
||||
|
||||
const index = skillsCache.findIndex(s => s.id === skillId)
|
||||
if (index === -1) return false
|
||||
|
||||
// 不允许删除内置技能
|
||||
if (skillsCache[index].isBuiltIn) return false
|
||||
|
||||
skillsCache.splice(index, 1)
|
||||
await saveSkills()
|
||||
|
||||
return true
|
||||
}
|
||||
Reference in New Issue
Block a user