252 lines
8.0 KiB
JavaScript
252 lines
8.0 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* AINewsCollector - AI 新闻采集核心脚本
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const https = require('https');
|
|
const http = require('http');
|
|
|
|
// 配置
|
|
const CONFIG_PATH = path.join(__dirname, '../../config.json');
|
|
const CACHE_PATH = path.join(__dirname, '../../cache/seen_urls.json');
|
|
const DAILY_DIR = path.join(__dirname, '../../daily');
|
|
|
|
// HTTP 请求封装
|
|
function fetch(url) {
|
|
return new Promise((resolve, reject) => {
|
|
const client = url.startsWith('https') ? https : http;
|
|
const req = client.get(url, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (compatible; AINewsCollector/1.0)',
|
|
'Accept': 'application/json, text/html, */*'
|
|
},
|
|
timeout: 30000
|
|
}, (res) => {
|
|
let data = '';
|
|
res.on('data', chunk => data += chunk);
|
|
res.on('end', () => resolve({ data, status: res.statusCode }));
|
|
});
|
|
req.on('error', reject);
|
|
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
|
|
});
|
|
}
|
|
|
|
// ============ 数据采集器 ============
|
|
|
|
async function collectArxiv(config) {
|
|
const items = [];
|
|
const categories = config.categories || ['cs.AI', 'cs.CL', 'cs.LG'];
|
|
|
|
for (const cat of categories) {
|
|
try {
|
|
const url = `http://export.arxiv.org/api/query?search_query=cat:${cat}&sortBy=lastUpdatedDate&sortOrder=descending&maxResults=${config.maxResults || 50}`;
|
|
const { data } = await fetch(url);
|
|
|
|
const entries = data.split('<entry>').slice(1);
|
|
|
|
for (const entry of entries) {
|
|
const title = entry.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim().replace(/\n/g, ' ');
|
|
const summary = entry.match(/<summary>([\s\S]*?)<\/summary>/)?.[1]?.trim().replace(/\n/g, ' ').slice(0, 200);
|
|
const link = entry.match(/<id>([\s\S]*?)<\/id>/)?.[1]?.trim();
|
|
|
|
if (title && link) {
|
|
items.push({
|
|
title,
|
|
summary: summary || '',
|
|
url: link,
|
|
source: 'arXiv',
|
|
sourceWeight: config.weight || 6,
|
|
date: new Date().toISOString().split('T')[0]
|
|
});
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error(`[arXiv] Error:`, err.message);
|
|
}
|
|
}
|
|
return items;
|
|
}
|
|
|
|
async function collectHuggingFace(config) {
|
|
const items = [];
|
|
try {
|
|
const { data } = await fetch(`https://huggingface.co/api/daily-papers?limit=${config.maxResults || 30}`);
|
|
const papers = JSON.parse(data);
|
|
|
|
for (const paper of papers) {
|
|
items.push({
|
|
title: paper.title || paper.paper?.title || 'Untitled',
|
|
summary: (paper.summary || '').slice(0, 200),
|
|
url: `https://huggingface.co/papers/${paper.paper?.id || paper.id}`,
|
|
source: 'Hugging Face',
|
|
sourceWeight: config.weight || 8,
|
|
date: new Date().toISOString().split('T')[0]
|
|
});
|
|
}
|
|
} catch (err) {
|
|
console.error('[HuggingFace] Error:', err.message);
|
|
}
|
|
return items;
|
|
}
|
|
|
|
async function collectGitHub(config) {
|
|
const items = [];
|
|
try {
|
|
const { data } = await fetch('https://github.com/trending?since=daily');
|
|
const repoMatches = data.match(/<article class="Box-row">[\s\S]*?<\/article>/g) || [];
|
|
|
|
for (const repo of repoMatches.slice(0, config.maxResults || 30)) {
|
|
const href = repo.match(/href="\/([^"]+)"/)?.[1];
|
|
const desc = repo.match(/<p class="col-9[^"]*">([\s\S]*?)<\/p>/)?.[1]?.trim().replace(/<[^>]+>/g, '');
|
|
const stars = repo.match(/([\d,]+)\s*stars today/)?.[1] || '0';
|
|
|
|
if (href) {
|
|
items.push({
|
|
title: href,
|
|
summary: (desc || '').slice(0, 200),
|
|
url: `https://github.com/${href}`,
|
|
source: 'GitHub Trending',
|
|
sourceWeight: config.weight || 7,
|
|
stars: parseInt(stars.replace(/,/g, '')) || 0,
|
|
date: new Date().toISOString().split('T')[0]
|
|
});
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error('[GitHub] Error:', err.message);
|
|
}
|
|
return items;
|
|
}
|
|
|
|
// ============ 处理模块 ============
|
|
|
|
function calculateKeywordScore(item, topics) {
|
|
const text = `${item.title} ${item.summary}`.toLowerCase();
|
|
let score = 0;
|
|
for (const topic of topics) {
|
|
for (const keyword of topic.keywords) {
|
|
if (text.includes(keyword.toLowerCase())) score += 2;
|
|
}
|
|
}
|
|
return score;
|
|
}
|
|
|
|
function deduplicate(items, seenUrls) {
|
|
return items.filter(item => {
|
|
const urlKey = item.url.toLowerCase().replace(/\/$/, '');
|
|
if (seenUrls.has(urlKey)) return false;
|
|
seenUrls.add(urlKey);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
function sortItems(items, topics) {
|
|
return items.map(item => ({
|
|
...item,
|
|
finalScore: item.sourceWeight + calculateKeywordScore(item, topics) + (item.stars > 100 ? 2 : 0)
|
|
})).sort((a, b) => b.finalScore - a.finalScore);
|
|
}
|
|
|
|
// ============ 输出模块 ============
|
|
|
|
function generateMarkdown(items, topCount, topics) {
|
|
const date = new Date().toLocaleDateString('zh-CN', {
|
|
year: 'numeric', month: '2-digit', day: '2-digit', timeZone: 'Asia/Shanghai'
|
|
}).replace(/\//g, '-');
|
|
|
|
const top10 = items.slice(0, topCount);
|
|
|
|
let md = `# AI Daily Brief - ${date}\n\n`;
|
|
md += `> 采集时间: ${new Date().toLocaleString('zh-CN', { timeZone: 'Asia/Shanghai' })}\n`;
|
|
md += `> 总条目: ${items.length}\n\n`;
|
|
|
|
md += `## 🔥 Top ${topCount} 重要消息\n\n`;
|
|
for (let i = 0; i < top10.length; i++) {
|
|
const item = top10[i];
|
|
md += `${i + 1}. [${item.title}](${item.url}) - **${item.source}**\n`;
|
|
if (item.summary) md += ` > ${item.summary.slice(0, 150)}${item.summary.length > 150 ? '...' : ''}\n`;
|
|
md += '\n';
|
|
}
|
|
|
|
md += `## 📂 分类汇总\n\n`;
|
|
|
|
for (const topic of topics) {
|
|
const topicItems = items.filter(item => {
|
|
const text = `${item.title} ${item.summary}`.toLowerCase();
|
|
return topic.keywords.some(k => text.includes(k.toLowerCase()));
|
|
}).filter(item => !top10.includes(item));
|
|
|
|
if (topicItems.length > 0) {
|
|
md += `### ${topic.name}\n\n`;
|
|
for (const item of topicItems.slice(0, 10)) {
|
|
md += `- [${item.title}](${item.url}) - ${item.source}\n`;
|
|
}
|
|
md += '\n';
|
|
}
|
|
}
|
|
|
|
md += `---\n*Generated by AINewsCollector*\n`;
|
|
|
|
return { md, date };
|
|
}
|
|
|
|
// ============ 主流程 ============
|
|
|
|
async function main() {
|
|
console.log('🚀 AINewsCollector 开始运行...\n');
|
|
|
|
const config = JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8'));
|
|
|
|
let seenUrls = new Set();
|
|
if (fs.existsSync(CACHE_PATH)) {
|
|
const cache = JSON.parse(fs.readFileSync(CACHE_PATH, 'utf8'));
|
|
seenUrls = new Set(cache.urls || []);
|
|
}
|
|
|
|
let allItems = [];
|
|
|
|
console.log('📡 采集 arXiv...');
|
|
const arxivItems = await collectArxiv(config.sources.arxiv);
|
|
allItems.push(...arxivItems);
|
|
console.log(` 获取 ${arxivItems.length} 条\n`);
|
|
|
|
console.log('📡 采集 Hugging Face Papers...');
|
|
const hfItems = await collectHuggingFace(config.sources.huggingface);
|
|
allItems.push(...hfItems);
|
|
console.log(` 获取 ${hfItems.length} 条\n`);
|
|
|
|
console.log('📡 采集 GitHub Trending...');
|
|
const ghItems = await collectGitHub(config.sources.github);
|
|
allItems.push(...ghItems);
|
|
console.log(` 获取 ${ghItems.length} 条\n`);
|
|
|
|
const originalCount = allItems.length;
|
|
allItems = deduplicate(allItems, seenUrls);
|
|
console.log(`🔄 去重: ${originalCount} → ${allItems.length}\n`);
|
|
|
|
allItems = sortItems(allItems, config.topics);
|
|
|
|
const { md, date } = generateMarkdown(allItems, config.output.topCount, config.topics);
|
|
|
|
if (!fs.existsSync(DAILY_DIR)) fs.mkdirSync(DAILY_DIR, { recursive: true });
|
|
const outputPath = path.join(DAILY_DIR, `${date}.md`);
|
|
fs.writeFileSync(outputPath, md);
|
|
console.log(`📝 简报已保存: ${outputPath}\n`);
|
|
|
|
const cacheData = { lastUpdate: new Date().toISOString(), urls: Array.from(seenUrls).slice(-5000) };
|
|
fs.writeFileSync(CACHE_PATH, JSON.stringify(cacheData, null, 2));
|
|
|
|
console.log('✅ 采集完成!\n');
|
|
|
|
return { success: true, outputPath, date, itemCount: allItems.length, content: md };
|
|
}
|
|
|
|
if (require.main === module) {
|
|
main().catch(err => { console.error('❌ Error:', err); process.exit(1); });
|
|
}
|
|
|
|
module.exports = { main };
|