Add logging functionality and new log-related endpoints

- Integrated a logging service to redirect console outputs. - Added new endpoints for managing logs: - Retrieve log file list - Read specific log file content - Fetch recent logs - Stream logs via Server-Sent Events - Search logs by keyword - Cleanup old log files - Updated application metadata to include new log management features.
2025-10-31 09:23:05 +08:00
parent 76683fb519
commit f96a69e846
5 changed files with 1179 additions and 2 deletions
--- a/src/app.js
+++ b/src/app.js
@@ -1,3 +1,6 @@
+// 首先加载日志服务，这样所有console输出都会被重定向
+const logger = require('./services/logger');
+
 const express = require('express');
 const path = require('path');
 const Database = require('./database/db');
@@ -76,11 +79,12 @@ class ProxyApp {
        name: 'Proxy IP Service',
        version: '1.0.0',
        description: '代理IP抓取、验证和管理服务',
-        web_interface: {
+          web_interface: {
          dashboard: '/index.html',
          proxies: '/proxies.html',
          history: '/history.html',
-          monitoring: '/monitoring.html'
+          monitoring: '/monitoring.html',
+          logs: '/logs.html'
        },
        endpoints: {
          'GET /api/health': '健康检查',
@@ -100,6 +104,10 @@ class ProxyApp {
          'DELETE /api/proxies/cleanup': '清理无效代理',
          'GET /api/history': '执行历史',
          'GET /api/history/logs/system': '系统日志',
+          'GET /api/history/logs/files': '获取日志文件列表',
+          'GET /api/history/logs/file/:filename': '读取日志文件',
+          'GET /api/history/logs/recent': '获取最近日志',
+          'GET /api/history/logs/stream': '实时日志流(SSE)',
          'GET /api/history/stats': '历史统计'
        },
        scheduler: {
--- a/src/routes/history.js
+++ b/src/routes/history.js
@@ -2,6 +2,7 @@ const express = require('express');
 const router = express.Router();
 const HistoryModel = require('../database/models/history');
 const LogsModel = require('../database/models/logs');
+const logger = require('../services/logger');

 // 获取执行历史列表
 router.get('/', async (req, res) => {
@@ -334,4 +335,218 @@ router.get('/export', async (req, res) => {
  }
 });

+// ==================== 文件日志相关接口 ====================
+
+// 获取日志文件列表
+router.get('/logs/files', (req, res) => {
+  try {
+    const days = parseInt(req.query.days) || 7;
+    const files = logger.getLogFiles(days);
+    
+    res.json({
+      success: true,
+      data: files
+    });
+  } catch (error) {
+    console.error('获取日志文件列表失败:', error);
+    res.status(500).json({
+      success: false,
+      error: '获取日志文件列表失败',
+      message: error.message
+    });
+  }
+});
+
+// 读取日志文件内容
+router.get('/logs/file/:filename', (req, res) => {
+  try {
+    const filename = req.params.filename;
+    const limit = parseInt(req.query.limit) || 1000;
+    const offset = parseInt(req.query.offset) || 0;
+    
+    // 安全检查：确保文件名只包含允许的字符
+    if (!/^app-\d{4}-\d{2}-\d{2}\.log$/.test(filename)) {
+      return res.status(400).json({
+        success: false,
+        error: '无效的日志文件名格式'
+      });
+    }
+    
+    const result = logger.readLogFile(filename, limit, offset);
+    
+    res.json({
+      success: true,
+      data: result.logs,
+      pagination: {
+        total: result.total,
+        limit: result.limit,
+        offset: result.offset,
+        hasMore: result.offset + result.limit < result.total
+      }
+    });
+  } catch (error) {
+    console.error('读取日志文件失败:', error);
+    res.status(500).json({
+      success: false,
+      error: '读取日志文件失败',
+      message: error.message
+    });
+  }
+});
+
+// 获取最近的实时日志
+router.get('/logs/recent', (req, res) => {
+  try {
+    const limit = Math.min(parseInt(req.query.limit) || 100, 500);
+    const logs = logger.getRecentLogs(limit);
+    
+    res.json({
+      success: true,
+      data: logs,
+      count: logs.length
+    });
+  } catch (error) {
+    console.error('获取最近日志失败:', error);
+    res.status(500).json({
+      success: false,
+      error: '获取最近日志失败',
+      message: error.message
+    });
+  }
+});
+
+// 实时日志流（Server-Sent Events）
+router.get('/logs/stream', (req, res) => {
+  try {
+    // 设置SSE响应头
+    res.setHeader('Content-Type', 'text/event-stream');
+    res.setHeader('Cache-Control', 'no-cache');
+    res.setHeader('Connection', 'keep-alive');
+    res.setHeader('X-Accel-Buffering', 'no'); // 禁用Nginx缓冲
+    
+    // 发送初始连接确认
+    res.write('data: {"type":"connected","message":"实时日志流已连接"}\n\n');
+    
+    // 设置日志事件监听器
+    const emitLog = (logEntry) => {
+      try {
+        const data = JSON.stringify({
+          type: 'log',
+          data: logEntry
+        });
+        res.write(`data: ${data}\n\n`);
+      } catch (error) {
+        console.error('发送日志事件失败:', error);
+      }
+    };
+    
+    logger.addLogEmitter(emitLog);
+    
+    // 发送最近的日志
+    const recentLogs = logger.getRecentLogs(50);
+    recentLogs.forEach(logEntry => {
+      const data = JSON.stringify({
+        type: 'log',
+        data: logEntry
+      });
+      res.write(`data: ${data}\n\n`);
+    });
+    
+    // 保持连接：定期发送心跳
+    const heartbeat = setInterval(() => {
+      try {
+        res.write('data: {"type":"heartbeat","time":' + Date.now() + '}\n\n');
+      } catch (error) {
+        clearInterval(heartbeat);
+      }
+    }, 30000); // 每30秒发送一次心跳
+    
+    // 客户端断开连接时清理
+    req.on('close', () => {
+      clearInterval(heartbeat);
+      logger.removeLogEmitter(emitLog);
+      res.end();
+    });
+    
+    // 防止连接被意外关闭
+    req.on('error', () => {
+      clearInterval(heartbeat);
+      logger.removeLogEmitter(emitLog);
+    });
+    
+  } catch (error) {
+    console.error('建立日志流失败:', error);
+    res.status(500).json({
+      success: false,
+      error: '建立日志流失败',
+      message: error.message
+    });
+  }
+});
+
+// 搜索文件日志
+router.get('/logs/search-file', (req, res) => {
+  try {
+    const keyword = req.query.keyword;
+    const date = req.query.date || null;
+    const limit = Math.min(parseInt(req.query.limit) || 500, 1000);
+    
+    if (!keyword || keyword.trim().length === 0) {
+      return res.status(400).json({
+        success: false,
+        error: '搜索关键词不能为空'
+      });
+    }
+    
+    const result = logger.searchLogs(keyword.trim(), date, limit);
+    
+    res.json({
+      success: true,
+      data: result.logs,
+      count: result.total,
+      keyword: keyword.trim(),
+      date: date
+    });
+  } catch (error) {
+    console.error('搜索文件日志失败:', error);
+    res.status(500).json({
+      success: false,
+      error: '搜索文件日志失败',
+      message: error.message
+    });
+  }
+});
+
+// 清理旧日志文件
+router.delete('/logs/cleanup-files', (req, res) => {
+  try {
+    const daysToKeep = parseInt(req.body.days) || 30;
+    
+    if (daysToKeep < 1 || daysToKeep > 365) {
+      return res.status(400).json({
+        success: false,
+        error: '保留天数必须在1-365之间'
+      });
+    }
+    
+    const deletedCount = logger.cleanupOldLogs(daysToKeep);
+    
+    res.json({
+      success: true,
+      message: `清理完成，删除了 ${deletedCount} 个旧日志文件`,
+      data: {
+        deleted_count: deletedCount,
+        days_to_keep: daysToKeep
+      }
+    });
+  } catch (error) {
+    console.error('清理旧日志文件失败:', error);
+    res.status(500).json({
+      success: false,
+      error: '清理旧日志文件失败',
+      message: error.message
+    });
+  }
+});
+
 module.exports = router;
--- a/src/services/logger.js
+++ b/src/services/logger.js
@@ -0,0 +1,353 @@
+const fs = require('fs');
+const path = require('path');
+
+class Logger {
+  constructor() {
+    this.logsDir = path.join(__dirname, '../../logs');
+    this.currentLogFile = null;
+    this.currentDate = null;
+    this.originalConsole = {
+      log: console.log,
+      error: console.error,
+      warn: console.warn,
+      info: console.info,
+      debug: console.debug
+    };
+    this.logBuffer = []; // 用于实时日志推送
+    this.maxBufferSize = 1000; // 最多缓存1000条日志
+    this.logEmitters = new Set(); // 支持多个日志监听器
+    
+    // 确保logs目录存在
+    this.ensureLogsDirectory();
+    
+    // 初始化今天的日志文件
+    this.initializeTodayLog();
+    
+    // 重定向console
+    this.redirectConsole();
+  }
+
+  ensureLogsDirectory() {
+    if (!fs.existsSync(this.logsDir)) {
+      fs.mkdirSync(this.logsDir, { recursive: true });
+      // 使用原始console，避免循环调用
+      this.originalConsole.log(`创建日志目录: ${this.logsDir}`);
+    }
+  }
+
+  getLogFileName(date = null) {
+    const logDate = date || new Date();
+    const dateStr = logDate.toISOString().slice(0, 10); // YYYY-MM-DD
+    return path.join(this.logsDir, `app-${dateStr}.log`);
+  }
+
+  initializeTodayLog() {
+    const today = new Date().toISOString().slice(0, 10);
+    
+    if (this.currentDate !== today) {
+      this.currentDate = today;
+      this.currentLogFile = this.getLogFileName();
+      
+      // 确保文件存在
+      if (!fs.existsSync(this.currentLogFile)) {
+        fs.writeFileSync(this.currentLogFile, '');
+      }
+    }
+  }
+
+  formatTimestamp() {
+    const now = new Date();
+    return now.toISOString().replace('T', ' ').slice(0, 19);
+  }
+
+  writeToFile(level, message) {
+    try {
+      // 检查是否需要切换到新的一天的日志文件
+      const today = new Date().toISOString().slice(0, 10);
+      if (this.currentDate !== today) {
+        this.initializeTodayLog();
+      }
+
+      const timestamp = this.formatTimestamp();
+      const logLine = `[${timestamp}] [${level.toUpperCase()}] ${message}\n`;
+      
+      // 写入文件
+      fs.appendFileSync(this.currentLogFile, logLine, 'utf8');
+      
+      // 添加到缓冲区用于实时推送
+      const logEntry = {
+        timestamp,
+        level,
+        message,
+        time: Date.now()
+      };
+      
+      this.logBuffer.push(logEntry);
+      
+      // 限制缓冲区大小
+      if (this.logBuffer.length > this.maxBufferSize) {
+        this.logBuffer.shift();
+      }
+      
+      // 触发实时日志事件（如果有监听器）
+      if (this.logEmitters.size > 0) {
+        this.logEmitters.forEach(emitFn => {
+          try {
+            emitFn(logEntry);
+          } catch (error) {
+            // 如果某个监听器出错，移除它
+            this.logEmitters.delete(emitFn);
+          }
+        });
+      }
+      
+    } catch (error) {
+      // 如果文件写入失败，至少输出到原始console
+      this.originalConsole.error('写入日志文件失败:', error);
+      this.originalConsole[level](message);
+    }
+  }
+
+  redirectConsole() {
+    // 重定向 console.log
+    console.log = (...args) => {
+      const message = args.map(arg => 
+        typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
+      ).join(' ');
+      
+      this.originalConsole.log(...args);
+      this.writeToFile('info', message);
+    };
+
+    // 重定向 console.error
+    console.error = (...args) => {
+      const message = args.map(arg => 
+        typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
+      ).join(' ');
+      
+      this.originalConsole.error(...args);
+      this.writeToFile('error', message);
+    };
+
+    // 重定向 console.warn
+    console.warn = (...args) => {
+      const message = args.map(arg => 
+        typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
+      ).join(' ');
+      
+      this.originalConsole.warn(...args);
+      this.writeToFile('warn', message);
+    };
+
+    // 重定向 console.info
+    console.info = (...args) => {
+      const message = args.map(arg => 
+        typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
+      ).join(' ');
+      
+      this.originalConsole.info(...args);
+      this.writeToFile('info', message);
+    };
+
+    // 重定向 console.debug（如果有的话）
+    if (console.debug) {
+      console.debug = (...args) => {
+        const message = args.map(arg => 
+          typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
+        ).join(' ');
+        
+        this.originalConsole.debug(...args);
+        this.writeToFile('debug', message);
+      };
+    }
+  }
+
+  // 获取日志文件列表
+  getLogFiles(days = 7) {
+    try {
+      const files = fs.readdirSync(this.logsDir);
+      const logFiles = files
+        .filter(file => file.startsWith('app-') && file.endsWith('.log'))
+        .map(file => {
+          const filePath = path.join(this.logsDir, file);
+          const stats = fs.statSync(filePath);
+          return {
+            filename: file,
+            date: file.replace('app-', '').replace('.log', ''),
+            size: stats.size,
+            modified: stats.mtime.toISOString()
+          };
+        })
+        .sort((a, b) => b.date.localeCompare(a.date))
+        .slice(0, days);
+      
+      return logFiles;
+    } catch (error) {
+      console.error('获取日志文件列表失败:', error);
+      return [];
+    }
+  }
+
+  // 读取日志文件内容
+  readLogFile(filename, limit = 1000, offset = 0) {
+    try {
+      const filePath = path.join(this.logsDir, filename);
+      
+      if (!fs.existsSync(filePath)) {
+        throw new Error('日志文件不存在');
+      }
+
+      // 读取文件内容
+      const content = fs.readFileSync(filePath, 'utf8');
+      const lines = content.split('\n').filter(line => line.trim());
+      
+      // 反转数组以获取最新的日志（最后的行）
+      const reversedLines = lines.reverse();
+      
+      // 分页
+      const paginatedLines = reversedLines.slice(offset, offset + limit);
+      
+      // 解析日志行
+      const logs = paginatedLines
+        .map(line => {
+          // 解析格式: [YYYY-MM-DD HH:mm:ss] [LEVEL] message
+          const match = line.match(/^\[([^\]]+)\] \[([^\]]+)\] (.+)$/);
+          if (match) {
+            return {
+              timestamp: match[1],
+              level: match[2].toLowerCase(),
+              message: match[3],
+              raw: line
+            };
+          }
+          return {
+            timestamp: null,
+            level: 'info',
+            message: line,
+            raw: line
+          };
+        })
+        .reverse(); // 再次反转，以时间正序显示
+      
+      return {
+        logs,
+        total: lines.length,
+        limit,
+        offset
+      };
+    } catch (error) {
+      console.error('读取日志文件失败:', error);
+      throw error;
+    }
+  }
+
+  // 获取实时日志流（从缓冲区）
+  getRecentLogs(limit = 100) {
+    return this.logBuffer.slice(-limit);
+  }
+
+  // 添加日志事件监听器（用于SSE）
+  addLogEmitter(emitFn) {
+    if (emitFn && typeof emitFn === 'function') {
+      this.logEmitters.add(emitFn);
+    }
+  }
+
+  // 移除日志事件监听器
+  removeLogEmitter(emitFn) {
+    this.logEmitters.delete(emitFn);
+  }
+
+  // 兼容旧API
+  setLogEmitter(emitFn) {
+    if (emitFn) {
+      this.addLogEmitter(emitFn);
+    } else {
+      // 如果传入null，清除所有监听器（为了兼容旧代码）
+      this.logEmitters.clear();
+    }
+  }
+
+  // 搜索日志
+  searchLogs(keyword, date = null, limit = 500) {
+    try {
+      const filename = date ? `app-${date}.log` : this.getLogFileName().split(path.sep).pop();
+      const filePath = path.join(this.logsDir, filename);
+      
+      if (!fs.existsSync(filePath)) {
+        return { logs: [], total: 0 };
+      }
+
+      const content = fs.readFileSync(filePath, 'utf8');
+      const lines = content.split('\n');
+      
+      const matchedLines = lines
+        .map((line, index) => {
+          if (line.toLowerCase().includes(keyword.toLowerCase())) {
+            const match = line.match(/^\[([^\]]+)\] \[([^\]]+)\] (.+)$/);
+            if (match) {
+              return {
+                timestamp: match[1],
+                level: match[2].toLowerCase(),
+                message: match[3],
+                raw: line,
+                lineNumber: index + 1
+              };
+            }
+            return {
+              timestamp: null,
+              level: 'info',
+              message: line,
+              raw: line,
+              lineNumber: index + 1
+            };
+          }
+          return null;
+        })
+        .filter(line => line !== null)
+        .slice(-limit); // 只返回最新的匹配结果
+      
+      return {
+        logs: matchedLines,
+        total: matchedLines.length
+      };
+    } catch (error) {
+      console.error('搜索日志失败:', error);
+      throw error;
+    }
+  }
+
+  // 清理旧日志文件（保留指定天数）
+  cleanupOldLogs(daysToKeep = 30) {
+    try {
+      const files = fs.readdirSync(this.logsDir);
+      const cutoffDate = new Date();
+      cutoffDate.setDate(cutoffDate.getDate() - daysToKeep);
+      const cutoffDateStr = cutoffDate.toISOString().slice(0, 10);
+
+      let deletedCount = 0;
+      
+      files.forEach(file => {
+        if (file.startsWith('app-') && file.endsWith('.log')) {
+          const dateStr = file.replace('app-', '').replace('.log', '');
+          if (dateStr < cutoffDateStr) {
+            const filePath = path.join(this.logsDir, file);
+            fs.unlinkSync(filePath);
+            deletedCount++;
+          }
+        }
+      });
+
+      return deletedCount;
+    } catch (error) {
+      console.error('清理旧日志失败:', error);
+      throw error;
+    }
+  }
+}
+
+// 创建单例
+const logger = new Logger();
+
+module.exports = logger;
+