Implement dynamic database path configuration in database.js, enhancing compatibility with Docker and local development environments. Update db.js to log the database connection path. Remove response data truncation in validator.js for improved readability.

Dockerfile

@@ -33,7 +33,7 @@ RUN npm install
 COPY . .
 
 # 创建数据目录和日志目录并设置权限
-# RUN mkdir -p /app/data /app/logs && \
+RUN mkdir -p /app/data /app/logs 
 #     chown -R node:node /app
 
 # 切换到非root用户

config/database.js

@@ -1,9 +1,38 @@
 const path = require('path');
+const fs = require('fs');
+
+// 数据库文件路径配置
+// 在Docker环境中使用 /app/data，本地开发使用项目根目录的data文件夹
+let dbPath;
+if (process.env.DB_PATH) {
+  // 如果设置了环境变量，使用环境变量的路径
+  dbPath = process.env.DB_PATH;
+} else if (fs.existsSync('/app/data')) {
+  // Docker容器内，使用/app/data目录
+  dbPath = '/app/data/proxies.db';
+} else {
+  // 本地开发环境，使用项目根目录的data文件夹
+  const dataDir = path.join(__dirname, '../data');
+  dbPath = path.join(dataDir, 'proxies.db');
+  
+  // 确保data目录存在
+  if (!fs.existsSync(dataDir)) {
+    try {
+      fs.mkdirSync(dataDir, { recursive: true });
+    } catch (error) {
+      console.warn('无法创建data目录，将使用项目根目录:', error.message);
+      // 如果创建失败，回退到项目根目录
+      dbPath = path.join(__dirname, '../proxies.db');
+    }
+  }
+}
+
+console.log(`数据库文件路径: ${dbPath}`);
 
 const dbConfig = {
   development: {
     dialect: 'sqlite',
-    storage: path.join(__dirname, '../proxies.db')
+    storage: dbPath
   }
 };
 

public/history.html

@@ -43,6 +43,11 @@
                             <i class="bi bi-activity"></i> 系统监控
                         </a>
                     </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="logs.html">
+                            <i class="bi bi-journal-text"></i> 实时日志
+                        </a>
+                    </li>
                 </ul>
 
                 <div class="navbar-nav">

public/index.html

@@ -43,6 +43,11 @@
                             <i class="bi bi-activity"></i> 系统监控
                         </a>
                     </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="logs.html">
+                            <i class="bi bi-journal-text"></i> 实时日志
+                        </a>
+                    </li>
                 </ul>
 
                 <div class="navbar-nav">

public/js/proxies.js

@@ -333,6 +333,42 @@ class ProxyManager {
             document.getElementById('validationStatus').textContent = '验证完成';
 
             // 显示结果
+            let responseResultHtml = '';
+            
+            if (result.data.responseStatus !== null && result.data.responseStatus !== undefined) {
+                responseResultHtml = `
+                    <hr>
+                    <h6>响应结果:</h6>
+                    <p><strong>状态码:</strong> <span class="badge ${result.data.responseStatus >= 200 && result.data.responseStatus < 300 ? 'bg-success' : 'bg-warning'}">${result.data.responseStatus}</span></p>
+                `;
+                
+                if (result.data.responseData) {
+                    // 限制显示长度，避免内容过长
+                    let displayData = result.data.responseData;
+                    // const maxLength = 300;
+                    // if (displayData.length > maxLength) {
+                    //     displayData = displayData.substring(0, maxLength) + '... (内容已截断，完整内容请查看日志)';
+                    // }
+                    
+                    responseResultHtml += `
+                        <p><strong>响应内容:</strong></p>
+                        <pre class="bg-light p-2 border rounded" style="max-height: 200px; overflow-y: auto; font-size: 12px; white-space: pre-wrap; word-wrap: break-word;">${this.escapeHtml(displayData)}</pre>
+                    `;
+                }
+                
+                if (result.data.testUrl) {
+                    responseResultHtml += `
+                        <p class="text-muted small"><strong>测试URL:</strong> ${result.data.testUrl}</p>
+                    `;
+                }
+            } else if (result.data.error) {
+                responseResultHtml = `
+                    <hr>
+                    <h6>响应结果:</h6>
+                    <p class="text-danger"><strong>错误:</strong> ${result.data.error}</p>
+                `;
+            }
+            
             document.getElementById('validationResults').innerHTML = `
                 <div class="alert ${result.data.isValid ? 'alert-success' : 'alert-danger'}">
                     <h6>验证结果: ${result.data.isValid ? '成功' : '失败'}</h6>
@@ -340,6 +376,7 @@ class ProxyManager {
                     <p><strong>响应时间:</strong> ${result.data.responseTime}ms</p>
                     ${result.data.error ? `<p><strong>错误信息:</strong> ${result.data.error}</p>` : ''}
                 </div>
+                ${responseResultHtml}
             `;
 
             // 延迟刷新列表
@@ -581,6 +618,13 @@ class ProxyManager {
         return icons[type] || 'info-circle-fill';
     }
 
+    escapeHtml(text) {
+        if (!text) return '';
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+
     convertToCSV(data) {
         if (!data || data.length === 0) return '';
 

public/monitoring.html

@@ -43,6 +43,11 @@
                             <i class="bi bi-activity"></i> 系统监控
                         </a>
                     </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="logs.html">
+                            <i class="bi bi-journal-text"></i> 实时日志
+                        </a>
+                    </li>
                 </ul>
 
                 <div class="navbar-nav">

public/proxies.html

@@ -43,6 +43,11 @@
                             <i class="bi bi-activity"></i> 系统监控
                         </a>
                     </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="logs.html">
+                            <i class="bi bi-journal-text"></i> 实时日志
+                        </a>
+                    </li>
                 </ul>
 
                 <div class="navbar-nav">

src/database/db.js

@@ -9,15 +9,19 @@ class Database {
 
   connect() {
     return new Promise((resolve, reject) => {
+      const dbPath = config.development.storage;
+      console.log(`正在连接数据库: ${dbPath}`);
+      
       this.db = new sqlite3.Database(
-        config.development.storage,
+        dbPath,
         sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE,
         (err) => {
           if (err) {
             console.error('数据库连接失败:', err.message);
+            console.error('数据库路径:', dbPath);
             reject(err);
           } else {
-            console.log('已连接到 SQLite 数据库');
+            console.log(`已连接到 SQLite 数据库: ${dbPath}`);
             resolve();
           }
         }

src/services/scraper.js

@@ -91,19 +91,25 @@ class ProxyScraper {
       return null; // 无本地代理，使用直连
     }
 
-    // 尝试几个代理，找到可用的
+    // 尝试几个代理，找到可用的（最多尝试3个，避免耗时过长）
-    for (let i = 0; i < Math.min(5, this.localProxies.length); i++) {
+    const maxAttempts = Math.min(3, this.localProxies.length);
+    for (let i = 0; i < maxAttempts; i++) {
       const proxyConfig = this.getNextLocalProxy();
 
-      if (await this.testProxyForScraping(proxyConfig)) {
+      try {
-        console.log(`✓ 代理 ${proxyConfig.host}:${proxyConfig.port} 可用`);
+        if (await this.testProxyForScraping(proxyConfig)) {
-        return proxyConfig;
+          console.log(`✓ 代理 ${proxyConfig.host}:${proxyConfig.port} 可用`);
-      } else {
+          return proxyConfig;
-        console.log(`✗ 代理 ${proxyConfig.host}:${proxyConfig.port} 不可用，尝试下一个`);
+        } else {
+          console.log(`✗ 代理 ${proxyConfig.host}:${proxyConfig.port} 不可用，尝试下一个`);
+        }
+      } catch (error) {
+        console.log(`✗ 代理 ${proxyConfig.host}:${proxyConfig.port} 测试出错: ${error.message}`);
+        // 继续尝试下一个代理
       }
     }
 
-    console.log('测试的本地代理都不可用，使用直连');
+    console.log(`测试了 ${maxAttempts} 个本地代理都不可用，将使用直连`);
     return null; // 所有测试的代理都不可用，使用直连
   }
 
@@ -181,6 +187,8 @@ class ProxyScraper {
 
     console.log(`正在抓取第 ${pageNum} 页: ${url}`);
 
+    let useDirectConnection = false; // 标志：是否应该直接使用直连
+
     for (let attempt = 1; attempt <= retryCount; attempt++) {
       let proxyConfig = null;
       let proxyUsed = '';
@@ -188,8 +196,14 @@ class ProxyScraper {
       try {
         const userAgent = this.getRandomUserAgent();
 
-        // 获取可用代理配置（每次请求都尝试不同的代理）
+        // 如果之前使用代理失败过，或者标记为使用直连，则跳过代理
-        proxyConfig = await this.getWorkingProxy();
+        if (!useDirectConnection && this.localProxies.length > 0) {
+          // 尝试获取可用代理配置
+          proxyConfig = await this.getWorkingProxy();
+        } else {
+          proxyConfig = null;
+          console.log('跳过代理，直接使用直连');
+        }
 
         const requestConfig = {
           headers: {
@@ -242,6 +256,12 @@ class ProxyScraper {
       } catch (error) {
         console.error(`第 ${attempt} 次尝试抓取第 ${pageNum} 页失败 (${proxyUsed}):`, error.message);
 
+        // 如果使用代理失败，下次重试时使用直连
+        if (proxyConfig) {
+          console.log(`代理 ${proxyConfig.host}:${proxyConfig.port} 抓取失败，下次重试将使用直连`);
+          useDirectConnection = true; // 标记为使用直连
+        }
+
         if (attempt === retryCount) {
           throw new Error(`抓取第 ${pageNum} 页失败，已重试 ${retryCount} 次: ${error.message}`);
         }

src/services/validator.js

@@ -76,16 +76,16 @@ class ProxyValidator {
 
         // 检查响应内容 - 根据不同的测试URL使用不同的验证逻辑
         if (response.status >= 200 && response.status < 300) {
-          if (testUrl.includes('baidu.com')) {
+          // if (testUrl.includes('baidu.com')) {
-            isValid = response.data && response.data.includes('百度');
+          //   isValid = response.data && response.data.includes('baidu');
-          } else if (testUrl.includes('httpbin.org')) {
+          // } else if (testUrl.includes('httpbin.org')) {
-            isValid = response.data && (response.data.includes('origin') || response.data.includes('ip'));
+          //   isValid = response.data && (response.data.includes('origin') || response.data.includes('ip'));
-          } else if (testUrl.includes('google.com')) {
+          // } else if (testUrl.includes('google.com')) {
-            isValid = response.data && response.data.toLowerCase().includes('google');
+          //   isValid = response.data && response.data.toLowerCase().includes('google');
-          } else {
+          // } else {
             // 对于其他URL，只要能连接就认为有效
             isValid = true;
-          }
+          // }
         }
 
         // 只要有一次验证成功，立即返回成功结果
@@ -94,13 +94,33 @@ class ProxyValidator {
             console.log(`✓ 代理 ${ip}:${port} 第 ${attemptNumber} 次尝试验证成功，响应时间: ${responseTime}ms`);
           }
 
+          // 提取响应信息用于显示
+          let responseData = '';
+          let responseStatus = response.status;
+          try {
+            // 只取响应数据的前500个字符，避免数据过大
+            const dataStr = typeof response.data === 'string' 
+              ? response.data 
+              : JSON.stringify(response.data);
+            // responseData = dataStr.substring(0, 500);
+            responseData = dataStr;
+            // if (dataStr.length > 500) {
+            //   responseData += '... (已截断)';
+            // }
+          } catch (e) {
+            responseData = '无法解析响应内容';
+          }
+
           const result = {
             ip: ip,
             port: parseInt(port),
             isValid: true,
             responseTime: responseTime,
             error: null,
-            testUrl: testUrl
+            testUrl: testUrl,
+            responseStatus: responseStatus,
+            responseData: responseData,
+            responseHeaders: response.headers || {}
           };
 
           // 更新数据库中的验证结果（如果需要）
@@ -124,13 +144,34 @@ class ProxyValidator {
         }
         
         lastError = new Error('响应内容验证失败');
+        // 提取响应信息
+        let responseData = '';
+        let responseStatus = response ? response.status : null;
+        try {
+          if (response && response.data) {
+            const dataStr = typeof response.data === 'string' 
+              ? response.data 
+              : JSON.stringify(response.data);
+            responseData = dataStr;
+            // responseData = dataStr.substring(0, 500);
+            // if (dataStr.length > 500) {
+            //   responseData += '... (已截断)';
+            // }
+          }
+        } catch (e) {
+          responseData = '无法解析响应内容';
+        }
+
         lastResult = {
           ip: ip,
           port: parseInt(port),
           isValid: false,
           responseTime: responseTime,
           error: '响应内容验证失败',
-          testUrl: testUrl
+          testUrl: testUrl,
+          responseStatus: responseStatus,
+          responseData: responseData,
+          responseHeaders: response ? (response.headers || {}) : {}
         };
 
         // 如果还有重试机会，继续尝试
@@ -147,7 +188,10 @@ class ProxyValidator {
           isValid: false,
           responseTime: responseTime,
           error: error.message,
-          testUrl: testUrl
+          testUrl: testUrl,
+          responseStatus: null,
+          responseData: null,
+          responseHeaders: {}
         };
 
         if (logResult) {
@@ -184,7 +228,10 @@ class ProxyValidator {
       isValid: false,
       responseTime: Date.now() - startTime,
       error: lastError ? lastError.message : '所有验证尝试都失败',
-      testUrl: testUrl
+      testUrl: testUrl,
+      responseStatus: null,
+      responseData: null,
+      responseHeaders: {}
     };
   }