Compare commits

..

14 Commits

Author SHA1 Message Date
6abae17fea Implement dynamic database path configuration in database.js, enhancing compatibility with Docker and local development environments. Update db.js to log the database connection path. Remove response data truncation in validator.js for improved readability. 2025-10-31 10:32:44 +08:00
da140d5415 Remove response data truncation logic in ProxyManager for improved readability of displayed content. 2025-10-31 10:28:50 +08:00
4cefbbbcd6 Refactor ProxyScraper to limit proxy attempts to 3 and improve error handling. Added a flag to switch to direct connection if proxy fails. Enhanced logging for proxy testing outcomes. 2025-10-31 10:03:40 +08:00
a4127509af Enhance ProxyManager and ProxyValidator to display detailed response information, including status codes, response data, and headers. Implement response data truncation for better readability. Update validation logic to capture and return response details in case of errors. 2025-10-31 09:56:31 +08:00
6c3b3928f9 Add "实时日志" link to navigation in history, index, monitoring, and proxies pages 2025-10-31 09:44:44 +08:00
54bc260453 Restore directory creation in Dockerfile for data and logs, removing previous comment-out for improved setup consistency. 2025-10-31 09:40:50 +08:00
d48e0f2d9f Merge branch 'master' of https://git.theluyuan.com/theluyuan/dailiip 2025-10-31 09:39:57 +08:00
9bdc1bd9b3 Comment out directory creation and user switch in Dockerfile for potential troubleshooting purposes. 2025-10-31 09:39:53 +08:00
8cb074660a Update DOCKER_README.md to address log file permission issues, providing troubleshooting steps for Linux/Mac and Windows users. Clarify the importance of ensuring the logs directory has the correct permissions for the container's node user. 2025-10-31 09:36:08 +08:00
804f996c69 Merge branch 'master' of https://git.theluyuan.com/theluyuan/dailiip 2025-10-31 09:35:24 +08:00
735d3e0677 proxy 2025-10-31 09:34:05 +08:00
c9ccb3435d Update Dockerfile to create both data and logs directories; enhance logger to handle directory and file creation errors gracefully. Log warnings for permission issues without blocking application startup. 2025-10-31 09:33:05 +08:00
f96a69e846 Add logging functionality and new log-related endpoints
- Integrated a logging service to redirect console outputs.
- Added new endpoints for managing logs:
  - Retrieve log file list
  - Read specific log file content
  - Fetch recent logs
  - Stream logs via Server-Sent Events
  - Search logs by keyword
  - Cleanup old log files
- Updated application metadata to include new log management features.
2025-10-31 09:23:05 +08:00
76683fb519 Enhance ProxyValidator to support three validation attempts with improved logging. Update retryCount default to 2 and clarify retry logic in comments. 2025-10-31 08:50:45 +08:00
16 changed files with 1490 additions and 83 deletions

View File

@@ -159,7 +159,27 @@ mkdir -p data logs
sudo chown -R 1000:1000 data logs sudo chown -R 1000:1000 data logs
``` ```
### 3. 性能优化 ### 3. 日志文件权限问题
如果遇到 `EACCES: permission denied` 错误:
```bash
# Linux/Mac
mkdir -p logs
chmod 777 logs # 或者 chmod 755 logs
# Windows PowerShell (如果使用Docker Desktop)
New-Item -ItemType Directory -Path logs -Force
# 在Docker Desktop设置中确保目录有正确权限
# 或者删除logs目录让容器自动创建
rm -rf logs
# 然后重新启动容器
docker-compose up -d
```
**注意**:容器内使用 `node` 用户UID 1000运行确保挂载的logs目录对该用户有写入权限。
### 4. 性能优化
```bash ```bash
# 限制内存使用 # 限制内存使用
docker run -d --memory=512m proxy-ip-manager docker run -d --memory=512m proxy-ip-manager

View File

@@ -21,22 +21,23 @@ RUN apk add --no-cache \
# 配置npm使用国内镜像源 # 配置npm使用国内镜像源
# RUN npm config set registry https://registry.npmmirror.com # RUN npm config set registry https://registry.npmmirror.com
RUN npm -g i pnpm
# 复制package.json和package-lock.json如果存在 # 复制package.json和package-lock.json如果存在
COPY package*.json ./ COPY package*.json ./
# 安装项目依赖 # 安装项目依赖
RUN pnpm i #RUN npm config set proyx http://192.168.3.135:1084
RUN npm config set https-proxy http://192.168.3.135:1084
RUN npm install
# 复制项目文件 # 复制项目文件
COPY . . COPY . .
# 创建数据目录并设置权限 # 创建数据目录和日志目录并设置权限
RUN mkdir -p /app/data && \ RUN mkdir -p /app/data /app/logs
chown -R node:node /app # chown -R node:node /app
# 切换到非root用户 # 切换到非root用户
USER node # USER node
# 暴露端口 # 暴露端口
EXPOSE 3000 EXPOSE 3000

View File

@@ -1,9 +1,38 @@
const path = require('path'); const path = require('path');
const fs = require('fs');
// 数据库文件路径配置
// 在Docker环境中使用 /app/data本地开发使用项目根目录的data文件夹
let dbPath;
if (process.env.DB_PATH) {
// 如果设置了环境变量,使用环境变量的路径
dbPath = process.env.DB_PATH;
} else if (fs.existsSync('/app/data')) {
// Docker容器内使用/app/data目录
dbPath = '/app/data/proxies.db';
} else {
// 本地开发环境使用项目根目录的data文件夹
const dataDir = path.join(__dirname, '../data');
dbPath = path.join(dataDir, 'proxies.db');
// 确保data目录存在
if (!fs.existsSync(dataDir)) {
try {
fs.mkdirSync(dataDir, { recursive: true });
} catch (error) {
console.warn('无法创建data目录将使用项目根目录:', error.message);
// 如果创建失败,回退到项目根目录
dbPath = path.join(__dirname, '../proxies.db');
}
}
}
console.log(`数据库文件路径: ${dbPath}`);
const dbConfig = { const dbConfig = {
development: { development: {
dialect: 'sqlite', dialect: 'sqlite',
storage: path.join(__dirname, '../proxies.db') storage: dbPath
} }
}; };

View File

@@ -43,6 +43,11 @@
<i class="bi bi-activity"></i> 系统监控 <i class="bi bi-activity"></i> 系统监控
</a> </a>
</li> </li>
<li class="nav-item">
<a class="nav-link" href="logs.html">
<i class="bi bi-journal-text"></i> 实时日志
</a>
</li>
</ul> </ul>
<div class="navbar-nav"> <div class="navbar-nav">

View File

@@ -43,6 +43,11 @@
<i class="bi bi-activity"></i> 系统监控 <i class="bi bi-activity"></i> 系统监控
</a> </a>
</li> </li>
<li class="nav-item">
<a class="nav-link" href="logs.html">
<i class="bi bi-journal-text"></i> 实时日志
</a>
</li>
</ul> </ul>
<div class="navbar-nav"> <div class="navbar-nav">

367
public/js/logs.js Normal file
View File

@@ -0,0 +1,367 @@
// 实时日志查看页面JavaScript
class LogViewer {
constructor() {
this.eventSource = null;
this.logs = [];
this.maxLogs = 1000; // 最多保留1000条日志
this.autoScroll = true;
this.currentFilter = {
level: '',
keyword: ''
};
this.init();
}
async init() {
console.log('初始化日志查看器...');
// 绑定事件
this.bindEvents();
// 加载日志文件列表
await this.loadLogFiles();
// 加载最近日志
await this.loadRecentLogs();
console.log('日志查看器初始化完成');
}
bindEvents() {
// 开始/停止实时日志流
document.getElementById('startStreamBtn').addEventListener('click', () => {
this.startStream();
});
document.getElementById('stopStreamBtn').addEventListener('click', () => {
this.stopStream();
});
// 清空日志
document.getElementById('clearLogsBtn').addEventListener('click', () => {
this.clearLogs();
});
// 自动滚动切换
document.getElementById('autoScrollCheck').addEventListener('change', (e) => {
this.autoScroll = e.target.checked;
if (this.autoScroll) {
this.scrollToBottom();
}
});
// 日志级别过滤
document.getElementById('logLevelFilter').addEventListener('change', (e) => {
this.currentFilter.level = e.target.value;
this.renderLogs();
});
// 搜索
document.getElementById('searchBtn').addEventListener('click', () => {
this.searchLogs();
});
document.getElementById('searchKeyword').addEventListener('keypress', (e) => {
if (e.key === 'Enter') {
this.searchLogs();
}
});
// 日志文件选择
document.getElementById('logFileSelect').addEventListener('change', async (e) => {
const filename = e.target.value;
if (filename) {
await this.loadLogFile(filename);
} else {
await this.loadRecentLogs();
}
});
}
async loadLogFiles() {
try {
const response = await fetch('/api/history/logs/files?days=7');
const result = await response.json();
if (result.success) {
const select = document.getElementById('logFileSelect');
// 保留"今日日志"选项
select.innerHTML = '<option value="">今日日志</option>';
result.data.forEach(file => {
const option = document.createElement('option');
option.value = file.filename;
option.textContent = `${file.date} (${this.formatFileSize(file.size)})`;
select.appendChild(option);
});
}
} catch (error) {
console.error('加载日志文件列表失败:', error);
this.showAlert('加载日志文件列表失败', 'danger');
}
}
async loadRecentLogs() {
try {
const response = await fetch('/api/history/logs/recent?limit=100');
const result = await response.json();
if (result.success) {
this.logs = result.data.map(log => ({
timestamp: log.timestamp,
level: log.level.toUpperCase(),
message: log.message
}));
this.renderLogs();
this.updateLogCount();
}
} catch (error) {
console.error('加载最近日志失败:', error);
this.showAlert('加载日志失败', 'danger');
}
}
async loadLogFile(filename) {
try {
const response = await fetch(`/api/history/logs/file/${filename}?limit=500`);
const result = await response.json();
if (result.success) {
this.logs = result.data.map(log => ({
timestamp: log.timestamp,
level: log.level.toUpperCase(),
message: log.message
}));
this.renderLogs();
this.updateLogCount();
}
} catch (error) {
console.error('加载日志文件失败:', error);
this.showAlert('加载日志文件失败', 'danger');
}
}
startStream() {
if (this.eventSource) {
return; // 已经在运行
}
try {
this.eventSource = new EventSource('/api/history/logs/stream');
this.eventSource.onopen = () => {
this.updateConnectionStatus(true);
document.getElementById('startStreamBtn').disabled = true;
document.getElementById('stopStreamBtn').disabled = false;
this.showAlert('实时日志流已连接', 'success');
};
this.eventSource.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
if (data.type === 'connected') {
console.log('实时日志流连接成功');
} else if (data.type === 'log') {
this.addLog(data.data);
} else if (data.type === 'heartbeat') {
// 心跳包,保持连接
}
} catch (error) {
console.error('解析日志数据失败:', error);
}
};
this.eventSource.onerror = (error) => {
console.error('实时日志流错误:', error);
this.updateConnectionStatus(false);
this.stopStream();
this.showAlert('实时日志流连接断开', 'warning');
};
} catch (error) {
console.error('启动实时日志流失败:', error);
this.showAlert('启动实时日志流失败: ' + error.message, 'danger');
}
}
stopStream() {
if (this.eventSource) {
this.eventSource.close();
this.eventSource = null;
this.updateConnectionStatus(false);
document.getElementById('startStreamBtn').disabled = false;
document.getElementById('stopStreamBtn').disabled = true;
this.showAlert('实时日志流已停止', 'info');
}
}
addLog(logEntry) {
// 添加新日志
this.logs.push({
timestamp: logEntry.timestamp,
level: logEntry.level ? logEntry.level.toUpperCase() : 'INFO',
message: logEntry.message
});
// 限制日志数量
if (this.logs.length > this.maxLogs) {
this.logs.shift();
}
// 更新显示
this.renderLogs();
this.updateLogCount();
// 自动滚动
if (this.autoScroll) {
setTimeout(() => this.scrollToBottom(), 10);
}
}
renderLogs() {
const container = document.getElementById('logContainer');
// 过滤日志
let filteredLogs = this.logs;
if (this.currentFilter.level) {
filteredLogs = filteredLogs.filter(log => log.level === this.currentFilter.level);
}
if (this.currentFilter.keyword) {
const keyword = this.currentFilter.keyword.toLowerCase();
filteredLogs = filteredLogs.filter(log =>
log.message.toLowerCase().includes(keyword) ||
log.timestamp.toLowerCase().includes(keyword)
);
}
// 渲染日志
container.innerHTML = filteredLogs.map(log => {
const levelClass = log.level || 'INFO';
const messageClass = log.level ? log.level.toLowerCase() : 'info';
return `
<div class="log-entry ${messageClass}">
<span class="log-timestamp">${log.timestamp || ''}</span>
<span class="log-level ${levelClass}">${levelClass}</span>
<span class="log-message">${this.escapeHtml(log.message)}</span>
</div>
`;
}).join('');
}
clearLogs() {
if (confirm('确定要清空所有日志吗?')) {
this.logs = [];
this.renderLogs();
this.updateLogCount();
this.showAlert('日志已清空', 'info');
}
}
searchLogs() {
const keyword = document.getElementById('searchKeyword').value.trim();
this.currentFilter.keyword = keyword;
this.renderLogs();
}
scrollToBottom() {
const container = document.getElementById('logContainer');
container.scrollTop = container.scrollHeight;
// 显示自动滚动指示器
const indicator = document.getElementById('autoScrollIndicator');
if (this.autoScroll) {
indicator.classList.add('active');
setTimeout(() => {
indicator.classList.remove('active');
}, 2000);
}
}
updateLogCount() {
const count = this.logs.length;
document.getElementById('logCount').textContent = count;
}
updateConnectionStatus(connected) {
const indicator = document.getElementById('connectionStatus');
const text = document.getElementById('connectionStatusText');
if (connected) {
indicator.className = 'status-indicator online';
text.textContent = '已连接';
} else {
indicator.className = 'status-indicator offline';
text.textContent = '未连接';
}
}
formatFileSize(bytes) {
if (bytes < 1024) return bytes + ' B';
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(2) + ' KB';
return (bytes / (1024 * 1024)).toFixed(2) + ' MB';
}
escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
showAlert(message, type = 'info') {
const alertContainer = document.getElementById('alertContainer');
if (!alertContainer) return;
const alertId = 'alert-' + Date.now();
const alertHtml = `
<div class="alert alert-${type} alert-dismissible fade show" role="alert" id="${alertId}">
<i class="bi bi-${this.getAlertIcon(type)}"></i>
${message}
<button type="button" class="btn-close" data-bs-dismiss="alert"></button>
</div>
`;
alertContainer.insertAdjacentHTML('beforeend', alertHtml);
// 自动移除提示
setTimeout(() => {
const alertElement = document.getElementById(alertId);
if (alertElement) {
alertElement.remove();
}
}, 3000);
}
getAlertIcon(type) {
const icons = {
'success': 'check-circle-fill',
'danger': 'exclamation-triangle-fill',
'warning': 'exclamation-triangle-fill',
'info': 'info-circle-fill'
};
return icons[type] || 'info-circle-fill';
}
}
// 全局变量
let logViewer;
// 页面加载完成后初始化
document.addEventListener('DOMContentLoaded', () => {
logViewer = new LogViewer();
});
// 页面卸载时清理
window.addEventListener('beforeunload', () => {
if (logViewer) {
logViewer.stopStream();
}
});

View File

@@ -333,6 +333,42 @@ class ProxyManager {
document.getElementById('validationStatus').textContent = '验证完成'; document.getElementById('validationStatus').textContent = '验证完成';
// 显示结果 // 显示结果
let responseResultHtml = '';
if (result.data.responseStatus !== null && result.data.responseStatus !== undefined) {
responseResultHtml = `
<hr>
<h6>响应结果:</h6>
<p><strong>状态码:</strong> <span class="badge ${result.data.responseStatus >= 200 && result.data.responseStatus < 300 ? 'bg-success' : 'bg-warning'}">${result.data.responseStatus}</span></p>
`;
if (result.data.responseData) {
// 限制显示长度,避免内容过长
let displayData = result.data.responseData;
// const maxLength = 300;
// if (displayData.length > maxLength) {
// displayData = displayData.substring(0, maxLength) + '... (内容已截断,完整内容请查看日志)';
// }
responseResultHtml += `
<p><strong>响应内容:</strong></p>
<pre class="bg-light p-2 border rounded" style="max-height: 200px; overflow-y: auto; font-size: 12px; white-space: pre-wrap; word-wrap: break-word;">${this.escapeHtml(displayData)}</pre>
`;
}
if (result.data.testUrl) {
responseResultHtml += `
<p class="text-muted small"><strong>测试URL:</strong> ${result.data.testUrl}</p>
`;
}
} else if (result.data.error) {
responseResultHtml = `
<hr>
<h6>响应结果:</h6>
<p class="text-danger"><strong>错误:</strong> ${result.data.error}</p>
`;
}
document.getElementById('validationResults').innerHTML = ` document.getElementById('validationResults').innerHTML = `
<div class="alert ${result.data.isValid ? 'alert-success' : 'alert-danger'}"> <div class="alert ${result.data.isValid ? 'alert-success' : 'alert-danger'}">
<h6>验证结果: ${result.data.isValid ? '成功' : '失败'}</h6> <h6>验证结果: ${result.data.isValid ? '成功' : '失败'}</h6>
@@ -340,6 +376,7 @@ class ProxyManager {
<p><strong>响应时间:</strong> ${result.data.responseTime}ms</p> <p><strong>响应时间:</strong> ${result.data.responseTime}ms</p>
${result.data.error ? `<p><strong>错误信息:</strong> ${result.data.error}</p>` : ''} ${result.data.error ? `<p><strong>错误信息:</strong> ${result.data.error}</p>` : ''}
</div> </div>
${responseResultHtml}
`; `;
// 延迟刷新列表 // 延迟刷新列表
@@ -581,6 +618,13 @@ class ProxyManager {
return icons[type] || 'info-circle-fill'; return icons[type] || 'info-circle-fill';
} }
escapeHtml(text) {
if (!text) return '';
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
convertToCSV(data) { convertToCSV(data) {
if (!data || data.length === 0) return ''; if (!data || data.length === 0) return '';

234
public/logs.html Normal file
View File

@@ -0,0 +1,234 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>实时日志查看 - 代理IP管理系统</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" rel="stylesheet">
<link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.7.2/font/bootstrap-icons.css" rel="stylesheet">
<link href="css/dashboard.css" rel="stylesheet">
<style>
#logContainer {
font-family: 'Consolas', 'Monaco', 'Courier New', monospace;
font-size: 13px;
line-height: 1.6;
max-height: calc(100vh - 300px);
overflow-y: auto;
background: #1e1e1e;
color: #d4d4d4;
padding: 15px;
border-radius: 4px;
}
.log-entry {
margin-bottom: 4px;
padding: 2px 0;
word-wrap: break-word;
}
.log-timestamp {
color: #858585;
margin-right: 8px;
}
.log-level {
padding: 2px 6px;
border-radius: 3px;
font-size: 11px;
font-weight: bold;
margin-right: 8px;
}
.log-level.ERROR {
background: #f14c4c;
color: white;
}
.log-level.WARN {
background: #cca700;
color: white;
}
.log-level.INFO {
background: #007acc;
color: white;
}
.log-level.DEBUG {
background: #808080;
color: white;
}
.log-message {
color: #d4d4d4;
}
.log-entry.error .log-message {
color: #f48771;
}
.log-entry.warn .log-message {
color: #dcdcaa;
}
.log-entry.info .log-message {
color: #4ec9b0;
}
.log-entry.debug .log-message {
color: #808080;
}
.auto-scroll-indicator {
position: fixed;
bottom: 20px;
right: 20px;
background: rgba(0, 0, 0, 0.8);
color: white;
padding: 10px 15px;
border-radius: 20px;
font-size: 12px;
z-index: 1000;
display: none;
}
.auto-scroll-indicator.active {
display: block;
}
</style>
</head>
<body>
<!-- 导航栏 -->
<nav class="navbar navbar-expand-lg navbar-dark bg-primary">
<div class="container-fluid">
<a class="navbar-brand" href="index.html">
<i class="bi bi-shield-check"></i>
代理IP管理系统
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarNav">
<ul class="navbar-nav me-auto">
<li class="nav-item">
<a class="nav-link" href="index.html">
<i class="bi bi-speedometer2"></i> 仪表板
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="proxies.html">
<i class="bi bi-list-ul"></i> 代理管理
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="history.html">
<i class="bi bi-clock-history"></i> 执行历史
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="monitoring.html">
<i class="bi bi-activity"></i> 系统监控
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="logs.html">
<i class="bi bi-journal-text"></i> 实时日志
</a>
</li>
</ul>
<div class="navbar-nav">
<div class="nav-link">
<span class="status-indicator" id="connectionStatus"></span>
<span id="connectionStatusText">连接中...</span>
</div>
</div>
</div>
</div>
</nav>
<!-- 主要内容 -->
<div class="container-fluid mt-4">
<!-- 控制面板 -->
<div class="card mb-3">
<div class="card-body">
<div class="row align-items-center">
<div class="col-md-6">
<div class="btn-group" role="group">
<button type="button" class="btn btn-primary" id="startStreamBtn">
<i class="bi bi-play-fill"></i> 开始实时日志
</button>
<button type="button" class="btn btn-secondary" id="stopStreamBtn" disabled>
<i class="bi bi-stop-fill"></i> 停止
</button>
<button type="button" class="btn btn-outline-secondary" id="clearLogsBtn">
<i class="bi bi-x-circle"></i> 清空
</button>
</div>
<div class="form-check form-switch mt-2">
<input class="form-check-input" type="checkbox" id="autoScrollCheck" checked>
<label class="form-check-label" for="autoScrollCheck">
自动滚动
</label>
</div>
</div>
<div class="col-md-6 text-end">
<div class="input-group" style="max-width: 400px; margin-left: auto;">
<input type="text" class="form-control" id="searchKeyword" placeholder="搜索日志...">
<button class="btn btn-outline-secondary" type="button" id="searchBtn">
<i class="bi bi-search"></i> 搜索
</button>
</div>
<div class="mt-2">
<select class="form-select form-select-sm d-inline-block" id="logLevelFilter" style="width: auto;">
<option value="">所有级别</option>
<option value="ERROR">ERROR</option>
<option value="WARN">WARN</option>
<option value="INFO">INFO</option>
<option value="DEBUG">DEBUG</option>
</select>
<span class="ms-2 text-muted small">
日志数: <span id="logCount">0</span>
</span>
</div>
</div>
</div>
</div>
</div>
<!-- 日志显示区域 -->
<div class="card">
<div class="card-header d-flex justify-content-between align-items-center">
<h6 class="mb-0">
<i class="bi bi-terminal"></i> 实时日志流
</h6>
<div>
<select class="form-select form-select-sm d-inline-block" id="logFileSelect" style="width: auto;">
<option value="">今日日志</option>
</select>
</div>
</div>
<div class="card-body p-0">
<div id="logContainer"></div>
</div>
</div>
</div>
<!-- 自动滚动指示器 -->
<div class="auto-scroll-indicator" id="autoScrollIndicator">
<i class="bi bi-arrow-down-circle"></i> 自动滚动中...
</div>
<!-- 提示消息容器 -->
<div class="position-fixed top-0 end-0 p-3" style="z-index: 1050">
<div id="alertContainer"></div>
</div>
<!-- JavaScript -->
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js"></script>
<script src="js/logs.js"></script>
</body>
</html>

View File

@@ -43,6 +43,11 @@
<i class="bi bi-activity"></i> 系统监控 <i class="bi bi-activity"></i> 系统监控
</a> </a>
</li> </li>
<li class="nav-item">
<a class="nav-link" href="logs.html">
<i class="bi bi-journal-text"></i> 实时日志
</a>
</li>
</ul> </ul>
<div class="navbar-nav"> <div class="navbar-nav">

View File

@@ -43,6 +43,11 @@
<i class="bi bi-activity"></i> 系统监控 <i class="bi bi-activity"></i> 系统监控
</a> </a>
</li> </li>
<li class="nav-item">
<a class="nav-link" href="logs.html">
<i class="bi bi-journal-text"></i> 实时日志
</a>
</li>
</ul> </ul>
<div class="navbar-nav"> <div class="navbar-nav">

View File

@@ -1,3 +1,6 @@
// 首先加载日志服务这样所有console输出都会被重定向
const logger = require('./services/logger');
const express = require('express'); const express = require('express');
const path = require('path'); const path = require('path');
const Database = require('./database/db'); const Database = require('./database/db');
@@ -80,7 +83,8 @@ class ProxyApp {
dashboard: '/index.html', dashboard: '/index.html',
proxies: '/proxies.html', proxies: '/proxies.html',
history: '/history.html', history: '/history.html',
monitoring: '/monitoring.html' monitoring: '/monitoring.html',
logs: '/logs.html'
}, },
endpoints: { endpoints: {
'GET /api/health': '健康检查', 'GET /api/health': '健康检查',
@@ -100,6 +104,10 @@ class ProxyApp {
'DELETE /api/proxies/cleanup': '清理无效代理', 'DELETE /api/proxies/cleanup': '清理无效代理',
'GET /api/history': '执行历史', 'GET /api/history': '执行历史',
'GET /api/history/logs/system': '系统日志', 'GET /api/history/logs/system': '系统日志',
'GET /api/history/logs/files': '获取日志文件列表',
'GET /api/history/logs/file/:filename': '读取日志文件',
'GET /api/history/logs/recent': '获取最近日志',
'GET /api/history/logs/stream': '实时日志流(SSE)',
'GET /api/history/stats': '历史统计' 'GET /api/history/stats': '历史统计'
}, },
scheduler: { scheduler: {

View File

@@ -9,15 +9,19 @@ class Database {
connect() { connect() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const dbPath = config.development.storage;
console.log(`正在连接数据库: ${dbPath}`);
this.db = new sqlite3.Database( this.db = new sqlite3.Database(
config.development.storage, dbPath,
sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE,
(err) => { (err) => {
if (err) { if (err) {
console.error('数据库连接失败:', err.message); console.error('数据库连接失败:', err.message);
console.error('数据库路径:', dbPath);
reject(err); reject(err);
} else { } else {
console.log('已连接到 SQLite 数据库'); console.log(`已连接到 SQLite 数据库: ${dbPath}`);
resolve(); resolve();
} }
} }

View File

@@ -2,6 +2,7 @@ const express = require('express');
const router = express.Router(); const router = express.Router();
const HistoryModel = require('../database/models/history'); const HistoryModel = require('../database/models/history');
const LogsModel = require('../database/models/logs'); const LogsModel = require('../database/models/logs');
const logger = require('../services/logger');
// 获取执行历史列表 // 获取执行历史列表
router.get('/', async (req, res) => { router.get('/', async (req, res) => {
@@ -334,4 +335,218 @@ router.get('/export', async (req, res) => {
} }
}); });
// ==================== 文件日志相关接口 ====================
// 获取日志文件列表
router.get('/logs/files', (req, res) => {
try {
const days = parseInt(req.query.days) || 7;
const files = logger.getLogFiles(days);
res.json({
success: true,
data: files
});
} catch (error) {
console.error('获取日志文件列表失败:', error);
res.status(500).json({
success: false,
error: '获取日志文件列表失败',
message: error.message
});
}
});
// 读取日志文件内容
router.get('/logs/file/:filename', (req, res) => {
try {
const filename = req.params.filename;
const limit = parseInt(req.query.limit) || 1000;
const offset = parseInt(req.query.offset) || 0;
// 安全检查:确保文件名只包含允许的字符
if (!/^app-\d{4}-\d{2}-\d{2}\.log$/.test(filename)) {
return res.status(400).json({
success: false,
error: '无效的日志文件名格式'
});
}
const result = logger.readLogFile(filename, limit, offset);
res.json({
success: true,
data: result.logs,
pagination: {
total: result.total,
limit: result.limit,
offset: result.offset,
hasMore: result.offset + result.limit < result.total
}
});
} catch (error) {
console.error('读取日志文件失败:', error);
res.status(500).json({
success: false,
error: '读取日志文件失败',
message: error.message
});
}
});
// 获取最近的实时日志
router.get('/logs/recent', (req, res) => {
try {
const limit = Math.min(parseInt(req.query.limit) || 100, 500);
const logs = logger.getRecentLogs(limit);
res.json({
success: true,
data: logs,
count: logs.length
});
} catch (error) {
console.error('获取最近日志失败:', error);
res.status(500).json({
success: false,
error: '获取最近日志失败',
message: error.message
});
}
});
// 实时日志流Server-Sent Events
router.get('/logs/stream', (req, res) => {
try {
// 设置SSE响应头
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Accel-Buffering', 'no'); // 禁用Nginx缓冲
// 发送初始连接确认
res.write('data: {"type":"connected","message":"实时日志流已连接"}\n\n');
// 设置日志事件监听器
const emitLog = (logEntry) => {
try {
const data = JSON.stringify({
type: 'log',
data: logEntry
});
res.write(`data: ${data}\n\n`);
} catch (error) {
console.error('发送日志事件失败:', error);
}
};
logger.addLogEmitter(emitLog);
// 发送最近的日志
const recentLogs = logger.getRecentLogs(50);
recentLogs.forEach(logEntry => {
const data = JSON.stringify({
type: 'log',
data: logEntry
});
res.write(`data: ${data}\n\n`);
});
// 保持连接:定期发送心跳
const heartbeat = setInterval(() => {
try {
res.write('data: {"type":"heartbeat","time":' + Date.now() + '}\n\n');
} catch (error) {
clearInterval(heartbeat);
}
}, 30000); // 每30秒发送一次心跳
// 客户端断开连接时清理
req.on('close', () => {
clearInterval(heartbeat);
logger.removeLogEmitter(emitLog);
res.end();
});
// 防止连接被意外关闭
req.on('error', () => {
clearInterval(heartbeat);
logger.removeLogEmitter(emitLog);
});
} catch (error) {
console.error('建立日志流失败:', error);
res.status(500).json({
success: false,
error: '建立日志流失败',
message: error.message
});
}
});
// 搜索文件日志
router.get('/logs/search-file', (req, res) => {
try {
const keyword = req.query.keyword;
const date = req.query.date || null;
const limit = Math.min(parseInt(req.query.limit) || 500, 1000);
if (!keyword || keyword.trim().length === 0) {
return res.status(400).json({
success: false,
error: '搜索关键词不能为空'
});
}
const result = logger.searchLogs(keyword.trim(), date, limit);
res.json({
success: true,
data: result.logs,
count: result.total,
keyword: keyword.trim(),
date: date
});
} catch (error) {
console.error('搜索文件日志失败:', error);
res.status(500).json({
success: false,
error: '搜索文件日志失败',
message: error.message
});
}
});
// 清理旧日志文件
router.delete('/logs/cleanup-files', (req, res) => {
try {
const daysToKeep = parseInt(req.body.days) || 30;
if (daysToKeep < 1 || daysToKeep > 365) {
return res.status(400).json({
success: false,
error: '保留天数必须在1-365之间'
});
}
const deletedCount = logger.cleanupOldLogs(daysToKeep);
res.json({
success: true,
message: `清理完成,删除了 ${deletedCount} 个旧日志文件`,
data: {
deleted_count: deletedCount,
days_to_keep: daysToKeep
}
});
} catch (error) {
console.error('清理旧日志文件失败:', error);
res.status(500).json({
success: false,
error: '清理旧日志文件失败',
message: error.message
});
}
});
module.exports = router; module.exports = router;

375
src/services/logger.js Normal file
View File

@@ -0,0 +1,375 @@
const fs = require('fs');
const path = require('path');
class Logger {
constructor() {
this.logsDir = path.join(__dirname, '../../logs');
this.currentLogFile = null;
this.currentDate = null;
this.originalConsole = {
log: console.log,
error: console.error,
warn: console.warn,
info: console.info,
debug: console.debug
};
this.logBuffer = []; // 用于实时日志推送
this.maxBufferSize = 1000; // 最多缓存1000条日志
this.logEmitters = new Set(); // 支持多个日志监听器
// 确保logs目录存在
this.ensureLogsDirectory();
// 初始化今天的日志文件
this.initializeTodayLog();
// 重定向console
this.redirectConsole();
}
ensureLogsDirectory() {
try {
if (!fs.existsSync(this.logsDir)) {
fs.mkdirSync(this.logsDir, { recursive: true });
// 使用原始console避免循环调用
this.originalConsole.log(`创建日志目录: ${this.logsDir}`);
}
} catch (error) {
// 如果创建目录失败(比如权限问题),警告但不阻止应用启动
this.originalConsole.warn(`无法创建日志目录 ${this.logsDir}:`, error.message);
this.originalConsole.warn('日志将仅输出到控制台,不会保存到文件');
}
}
getLogFileName(date = null) {
const logDate = date || new Date();
const dateStr = logDate.toISOString().slice(0, 10); // YYYY-MM-DD
return path.join(this.logsDir, `app-${dateStr}.log`);
}
initializeTodayLog() {
const today = new Date().toISOString().slice(0, 10);
if (this.currentDate !== today) {
this.currentDate = today;
this.currentLogFile = this.getLogFileName();
// 确保文件存在 - 使用try-catch处理权限问题
if (!fs.existsSync(this.currentLogFile)) {
try {
// 尝试创建空文件,如果失败则会在下次写入时自动创建
fs.writeFileSync(this.currentLogFile, '', { flag: 'a' });
} catch (error) {
// 如果创建文件失败(比如权限问题),不抛出错误,会在后续写入时处理
this.originalConsole.warn(`无法创建日志文件 ${this.currentLogFile}:`, error.message);
}
}
}
}
formatTimestamp() {
const now = new Date();
return now.toISOString().replace('T', ' ').slice(0, 19);
}
writeToFile(level, message) {
try {
// 检查是否需要切换到新的一天的日志文件
const today = new Date().toISOString().slice(0, 10);
if (this.currentDate !== today) {
this.initializeTodayLog();
}
const timestamp = this.formatTimestamp();
const logLine = `[${timestamp}] [${level.toUpperCase()}] ${message}\n`;
// 写入文件 - 使用flag: 'a'确保文件不存在时自动创建
try {
fs.appendFileSync(this.currentLogFile, logLine, { encoding: 'utf8', flag: 'a' });
} catch (error) {
// 如果写入失败比如权限问题输出到原始console
this.originalConsole.error('写入日志文件失败:', error.message);
this.originalConsole[level](message);
return; // 不继续处理
}
// 添加到缓冲区用于实时推送
const logEntry = {
timestamp,
level,
message,
time: Date.now()
};
this.logBuffer.push(logEntry);
// 限制缓冲区大小
if (this.logBuffer.length > this.maxBufferSize) {
this.logBuffer.shift();
}
// 触发实时日志事件(如果有监听器)
if (this.logEmitters.size > 0) {
this.logEmitters.forEach(emitFn => {
try {
emitFn(logEntry);
} catch (error) {
// 如果某个监听器出错,移除它
this.logEmitters.delete(emitFn);
}
});
}
} catch (error) {
// 如果文件写入失败(已经在上面处理了),这里只是防止其他错误
// 如果上面的appendFileSync已经捕获了错误这里不会执行
// 这里主要是处理其他可能的错误
if (error.code !== 'EACCES' && !error.message.includes('permission')) {
this.originalConsole.error('日志处理失败:', error);
}
}
}
redirectConsole() {
// 重定向 console.log
console.log = (...args) => {
const message = args.map(arg =>
typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
).join(' ');
this.originalConsole.log(...args);
this.writeToFile('info', message);
};
// 重定向 console.error
console.error = (...args) => {
const message = args.map(arg =>
typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
).join(' ');
this.originalConsole.error(...args);
this.writeToFile('error', message);
};
// 重定向 console.warn
console.warn = (...args) => {
const message = args.map(arg =>
typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
).join(' ');
this.originalConsole.warn(...args);
this.writeToFile('warn', message);
};
// 重定向 console.info
console.info = (...args) => {
const message = args.map(arg =>
typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
).join(' ');
this.originalConsole.info(...args);
this.writeToFile('info', message);
};
// 重定向 console.debug如果有的话
if (console.debug) {
console.debug = (...args) => {
const message = args.map(arg =>
typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)
).join(' ');
this.originalConsole.debug(...args);
this.writeToFile('debug', message);
};
}
}
// 获取日志文件列表
getLogFiles(days = 7) {
try {
const files = fs.readdirSync(this.logsDir);
const logFiles = files
.filter(file => file.startsWith('app-') && file.endsWith('.log'))
.map(file => {
const filePath = path.join(this.logsDir, file);
const stats = fs.statSync(filePath);
return {
filename: file,
date: file.replace('app-', '').replace('.log', ''),
size: stats.size,
modified: stats.mtime.toISOString()
};
})
.sort((a, b) => b.date.localeCompare(a.date))
.slice(0, days);
return logFiles;
} catch (error) {
console.error('获取日志文件列表失败:', error);
return [];
}
}
// 读取日志文件内容
readLogFile(filename, limit = 1000, offset = 0) {
try {
const filePath = path.join(this.logsDir, filename);
if (!fs.existsSync(filePath)) {
throw new Error('日志文件不存在');
}
// 读取文件内容
const content = fs.readFileSync(filePath, 'utf8');
const lines = content.split('\n').filter(line => line.trim());
// 反转数组以获取最新的日志(最后的行)
const reversedLines = lines.reverse();
// 分页
const paginatedLines = reversedLines.slice(offset, offset + limit);
// 解析日志行
const logs = paginatedLines
.map(line => {
// 解析格式: [YYYY-MM-DD HH:mm:ss] [LEVEL] message
const match = line.match(/^\[([^\]]+)\] \[([^\]]+)\] (.+)$/);
if (match) {
return {
timestamp: match[1],
level: match[2].toLowerCase(),
message: match[3],
raw: line
};
}
return {
timestamp: null,
level: 'info',
message: line,
raw: line
};
})
.reverse(); // 再次反转,以时间正序显示
return {
logs,
total: lines.length,
limit,
offset
};
} catch (error) {
console.error('读取日志文件失败:', error);
throw error;
}
}
// 获取实时日志流(从缓冲区)
getRecentLogs(limit = 100) {
return this.logBuffer.slice(-limit);
}
// 添加日志事件监听器用于SSE
addLogEmitter(emitFn) {
if (emitFn && typeof emitFn === 'function') {
this.logEmitters.add(emitFn);
}
}
// 移除日志事件监听器
removeLogEmitter(emitFn) {
this.logEmitters.delete(emitFn);
}
// 兼容旧API
setLogEmitter(emitFn) {
if (emitFn) {
this.addLogEmitter(emitFn);
} else {
// 如果传入null清除所有监听器为了兼容旧代码
this.logEmitters.clear();
}
}
// 搜索日志
searchLogs(keyword, date = null, limit = 500) {
try {
const filename = date ? `app-${date}.log` : this.getLogFileName().split(path.sep).pop();
const filePath = path.join(this.logsDir, filename);
if (!fs.existsSync(filePath)) {
return { logs: [], total: 0 };
}
const content = fs.readFileSync(filePath, 'utf8');
const lines = content.split('\n');
const matchedLines = lines
.map((line, index) => {
if (line.toLowerCase().includes(keyword.toLowerCase())) {
const match = line.match(/^\[([^\]]+)\] \[([^\]]+)\] (.+)$/);
if (match) {
return {
timestamp: match[1],
level: match[2].toLowerCase(),
message: match[3],
raw: line,
lineNumber: index + 1
};
}
return {
timestamp: null,
level: 'info',
message: line,
raw: line,
lineNumber: index + 1
};
}
return null;
})
.filter(line => line !== null)
.slice(-limit); // 只返回最新的匹配结果
return {
logs: matchedLines,
total: matchedLines.length
};
} catch (error) {
console.error('搜索日志失败:', error);
throw error;
}
}
// 清理旧日志文件(保留指定天数)
cleanupOldLogs(daysToKeep = 30) {
try {
const files = fs.readdirSync(this.logsDir);
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - daysToKeep);
const cutoffDateStr = cutoffDate.toISOString().slice(0, 10);
let deletedCount = 0;
files.forEach(file => {
if (file.startsWith('app-') && file.endsWith('.log')) {
const dateStr = file.replace('app-', '').replace('.log', '');
if (dateStr < cutoffDateStr) {
const filePath = path.join(this.logsDir, file);
fs.unlinkSync(filePath);
deletedCount++;
}
}
});
return deletedCount;
} catch (error) {
console.error('清理旧日志失败:', error);
throw error;
}
}
}
// 创建单例
const logger = new Logger();
module.exports = logger;

View File

@@ -91,19 +91,25 @@ class ProxyScraper {
return null; // 无本地代理,使用直连 return null; // 无本地代理,使用直连
} }
// 尝试几个代理,找到可用的 // 尝试几个代理,找到可用的最多尝试3个避免耗时过长
for (let i = 0; i < Math.min(5, this.localProxies.length); i++) { const maxAttempts = Math.min(3, this.localProxies.length);
for (let i = 0; i < maxAttempts; i++) {
const proxyConfig = this.getNextLocalProxy(); const proxyConfig = this.getNextLocalProxy();
try {
if (await this.testProxyForScraping(proxyConfig)) { if (await this.testProxyForScraping(proxyConfig)) {
console.log(`✓ 代理 ${proxyConfig.host}:${proxyConfig.port} 可用`); console.log(`✓ 代理 ${proxyConfig.host}:${proxyConfig.port} 可用`);
return proxyConfig; return proxyConfig;
} else { } else {
console.log(`✗ 代理 ${proxyConfig.host}:${proxyConfig.port} 不可用,尝试下一个`); console.log(`✗ 代理 ${proxyConfig.host}:${proxyConfig.port} 不可用,尝试下一个`);
} }
} catch (error) {
console.log(`✗ 代理 ${proxyConfig.host}:${proxyConfig.port} 测试出错: ${error.message}`);
// 继续尝试下一个代理
}
} }
console.log('测试的本地代理都不可用,使用直连'); console.log(`测试了 ${maxAttempts}本地代理都不可用,使用直连`);
return null; // 所有测试的代理都不可用,使用直连 return null; // 所有测试的代理都不可用,使用直连
} }
@@ -181,6 +187,8 @@ class ProxyScraper {
console.log(`正在抓取第 ${pageNum} 页: ${url}`); console.log(`正在抓取第 ${pageNum} 页: ${url}`);
let useDirectConnection = false; // 标志:是否应该直接使用直连
for (let attempt = 1; attempt <= retryCount; attempt++) { for (let attempt = 1; attempt <= retryCount; attempt++) {
let proxyConfig = null; let proxyConfig = null;
let proxyUsed = ''; let proxyUsed = '';
@@ -188,8 +196,14 @@ class ProxyScraper {
try { try {
const userAgent = this.getRandomUserAgent(); const userAgent = this.getRandomUserAgent();
// 获取可用代理配置(每次请求都尝试不同的代理 // 如果之前使用代理失败过,或者标记为使用直连,则跳过代理
if (!useDirectConnection && this.localProxies.length > 0) {
// 尝试获取可用代理配置
proxyConfig = await this.getWorkingProxy(); proxyConfig = await this.getWorkingProxy();
} else {
proxyConfig = null;
console.log('跳过代理,直接使用直连');
}
const requestConfig = { const requestConfig = {
headers: { headers: {
@@ -242,6 +256,12 @@ class ProxyScraper {
} catch (error) { } catch (error) {
console.error(`${attempt} 次尝试抓取第 ${pageNum} 页失败 (${proxyUsed}):`, error.message); console.error(`${attempt} 次尝试抓取第 ${pageNum} 页失败 (${proxyUsed}):`, error.message);
// 如果使用代理失败,下次重试时使用直连
if (proxyConfig) {
console.log(`代理 ${proxyConfig.host}:${proxyConfig.port} 抓取失败,下次重试将使用直连`);
useDirectConnection = true; // 标记为使用直连
}
if (attempt === retryCount) { if (attempt === retryCount) {
throw new Error(`抓取第 ${pageNum} 页失败,已重试 ${retryCount} 次: ${error.message}`); throw new Error(`抓取第 ${pageNum} 页失败,已重试 ${retryCount} 次: ${error.message}`);
} }

View File

@@ -10,6 +10,7 @@ class ProxyValidator {
/** /**
* 核心统一验证代理方法 - 所有验证代理的代码都统一使用此方法 * 核心统一验证代理方法 - 所有验证代理的代码都统一使用此方法
* 验证会尝试3次1次初始 + 2次重试只要有一次成功就算通过
* @param {string} ip - 代理IP地址 * @param {string} ip - 代理IP地址
* @param {number|string} port - 代理端口 * @param {number|string} port - 代理端口
* @param {object} options - 可选配置参数 * @param {object} options - 可选配置参数
@@ -18,7 +19,7 @@ class ProxyValidator {
* @param {string} options.userAgent - User-Agent默认使用类中定义的 * @param {string} options.userAgent - User-Agent默认使用类中定义的
* @param {boolean} options.updateDatabase - 是否更新数据库默认true * @param {boolean} options.updateDatabase - 是否更新数据库默认true
* @param {boolean} options.logResult - 是否打印日志默认true * @param {boolean} options.logResult - 是否打印日志默认true
* @param {number} options.retryCount - 重试次数,默认0不重试 * @param {number} options.retryCount - 重试次数,默认2总共尝试3次
* @param {number} options.retryDelay - 重试延迟毫秒默认500 * @param {number} options.retryDelay - 重试延迟毫秒默认500
* @returns {Promise<object>} 验证结果 {ip, port, isValid, responseTime, error, testUrl} * @returns {Promise<object>} 验证结果 {ip, port, isValid, responseTime, error, testUrl}
*/ */
@@ -29,7 +30,7 @@ class ProxyValidator {
userAgent = this.userAgent, userAgent = this.userAgent,
updateDatabase = true, updateDatabase = true,
logResult = true, logResult = true,
retryCount = 0, retryCount = 2, // 默认重试2次总共尝试3次
retryDelay = 500 retryDelay = 500
} = options; } = options;
@@ -41,16 +42,18 @@ class ProxyValidator {
}; };
if (logResult) { if (logResult) {
console.log(`正在验证代理 ${ip}:${port}`); console.log(`正在验证代理 ${ip}:${port}(将尝试 ${retryCount + 1} 次)`);
} }
// 支持重试机制 // 支持重试机制:总共尝试 retryCount + 1 次,只要有一次成功就返回成功
let lastError = null; let lastError = null;
let lastResult = null; let lastResult = null;
for (let attempt = 0; attempt <= retryCount; attempt++) { for (let attempt = 0; attempt <= retryCount; attempt++) {
const attemptNumber = attempt + 1;
if (attempt > 0 && logResult) { if (attempt > 0 && logResult) {
console.log(`代理 ${ip}:${port}${attempt + 1}试验证...`); console.log(`代理 ${ip}:${port}${attemptNumber}试验证...`);
await this.sleep(retryDelay); await this.sleep(retryDelay);
} }
@@ -73,39 +76,57 @@ class ProxyValidator {
// 检查响应内容 - 根据不同的测试URL使用不同的验证逻辑 // 检查响应内容 - 根据不同的测试URL使用不同的验证逻辑
if (response.status >= 200 && response.status < 300) { if (response.status >= 200 && response.status < 300) {
if (testUrl.includes('baidu.com')) { // if (testUrl.includes('baidu.com')) {
isValid = response.data && response.data.includes('百度'); // isValid = response.data && response.data.includes('baidu');
} else if (testUrl.includes('httpbin.org')) { // } else if (testUrl.includes('httpbin.org')) {
isValid = response.data && (response.data.includes('origin') || response.data.includes('ip')); // isValid = response.data && (response.data.includes('origin') || response.data.includes('ip'));
} else if (testUrl.includes('google.com')) { // } else if (testUrl.includes('google.com')) {
isValid = response.data && response.data.toLowerCase().includes('google'); // isValid = response.data && response.data.toLowerCase().includes('google');
} else { // } else {
// 对于其他URL只要能连接就认为有效 // 对于其他URL只要能连接就认为有效
isValid = true; isValid = true;
} // }
} }
if (logResult) { // 只要有一次验证成功,立即返回成功结果
if (isValid) { if (isValid) {
console.log(`✓ 代理 ${ip}:${port} 验证成功,响应时间: ${responseTime}ms`); if (logResult) {
} else { console.log(`✓ 代理 ${ip}:${port}${attemptNumber} 次尝试验证成功,响应时间: ${responseTime}ms`);
console.log(`✗ 代理 ${ip}:${port} 验证失败,响应不正确`);
} }
// 提取响应信息用于显示
let responseData = '';
let responseStatus = response.status;
try {
// 只取响应数据的前500个字符避免数据过大
const dataStr = typeof response.data === 'string'
? response.data
: JSON.stringify(response.data);
// responseData = dataStr.substring(0, 500);
responseData = dataStr;
// if (dataStr.length > 500) {
// responseData += '... (已截断)';
// }
} catch (e) {
responseData = '无法解析响应内容';
} }
const result = { const result = {
ip: ip, ip: ip,
port: parseInt(port), port: parseInt(port),
isValid: isValid, isValid: true,
responseTime: responseTime, responseTime: responseTime,
error: null, error: null,
testUrl: testUrl testUrl: testUrl,
responseStatus: responseStatus,
responseData: responseData,
responseHeaders: response.headers || {}
}; };
// 更新数据库中的验证结果(如果需要) // 更新数据库中的验证结果(如果需要)
if (updateDatabase) { if (updateDatabase) {
try { try {
await ProxyModel.updateValidity(ip, port, isValid ? 1 : 0, responseTime); await ProxyModel.updateValidity(ip, port, 1, responseTime);
} catch (dbError) { } catch (dbError) {
// 如果代理不在数据库中,忽略更新错误 // 如果代理不在数据库中,忽略更新错误
if (!dbError.message.includes('not found')) { if (!dbError.message.includes('not found')) {
@@ -115,6 +136,48 @@ class ProxyValidator {
} }
return result; return result;
}
// 请求成功但验证失败(响应内容不正确),记录错误并继续重试
if (logResult) {
console.log(`✗ 代理 ${ip}:${port}${attemptNumber} 次尝试验证失败,响应内容不正确`);
}
lastError = new Error('响应内容验证失败');
// 提取响应信息
let responseData = '';
let responseStatus = response ? response.status : null;
try {
if (response && response.data) {
const dataStr = typeof response.data === 'string'
? response.data
: JSON.stringify(response.data);
responseData = dataStr;
// responseData = dataStr.substring(0, 500);
// if (dataStr.length > 500) {
// responseData += '... (已截断)';
// }
}
} catch (e) {
responseData = '无法解析响应内容';
}
lastResult = {
ip: ip,
port: parseInt(port),
isValid: false,
responseTime: responseTime,
error: '响应内容验证失败',
testUrl: testUrl,
responseStatus: responseStatus,
responseData: responseData,
responseHeaders: response ? (response.headers || {}) : {}
};
// 如果还有重试机会,继续尝试
if (attempt < retryCount) {
continue;
}
} catch (error) { } catch (error) {
const responseTime = Date.now() - startTime; const responseTime = Date.now() - startTime;
@@ -125,23 +188,32 @@ class ProxyValidator {
isValid: false, isValid: false,
responseTime: responseTime, responseTime: responseTime,
error: error.message, error: error.message,
testUrl: testUrl testUrl: testUrl,
responseStatus: null,
responseData: null,
responseHeaders: {}
}; };
// 如果不是最后一次尝试,继续重试 if (logResult) {
console.log(`✗ 代理 ${ip}:${port}${attemptNumber} 次尝试验证失败: ${error.message}`);
}
// 如果还有重试机会,继续尝试
if (attempt < retryCount) { if (attempt < retryCount) {
continue; continue;
} }
}
}
// 最后一次尝试失败 // 所有尝试失败了,返回最后一次的结果
if (logResult) { if (logResult) {
console.log(`✗ 代理 ${ip}:${port} 验证失败: ${error.message}`); console.log(`✗ 代理 ${ip}:${port} 所有 ${retryCount + 1} 次尝试都失败`);
} }
// 更新数据库中的验证结果(如果需要) // 更新数据库中的验证结果(如果需要)
if (updateDatabase) { if (updateDatabase && lastResult) {
try { try {
await ProxyModel.updateValidity(ip, port, 0, responseTime); await ProxyModel.updateValidity(ip, port, 0, lastResult.responseTime);
} catch (dbError) { } catch (dbError) {
// 如果代理不在数据库中,忽略更新错误 // 如果代理不在数据库中,忽略更新错误
if (!dbError.message.includes('not found')) { if (!dbError.message.includes('not found')) {
@@ -150,18 +222,16 @@ class ProxyValidator {
} }
} }
return lastResult;
}
}
// 所有重试都失败了
return lastResult || { return lastResult || {
ip: ip, ip: ip,
port: parseInt(port), port: parseInt(port),
isValid: false, isValid: false,
responseTime: Date.now() - startTime, responseTime: Date.now() - startTime,
error: lastError ? lastError.message : '验证失败', error: lastError ? lastError.message : '所有验证尝试都失败',
testUrl: testUrl testUrl: testUrl,
responseStatus: null,
responseData: null,
responseHeaders: {}
}; };
} }