Implement dynamic database path configuration in database.js, enhancing compatibility with Docker and local development environments. Update db.js to log the database connection path. Remove response data truncation in validator.js for improved readability.

Remove response data truncation logic in ProxyManager for improved readability of displayed content.
Refactor ProxyScraper to limit proxy attempts to 3 and improve error handling. Added a flag to switch to direct connection if proxy fails. Enhanced logging for proxy testing outcomes.
2025-10-31 10:32:44 +08:00 · 2025-10-31 10:28:50 +08:00 · 2025-10-31 10:03:40 +08:00 · 2025-10-31 09:56:31 +08:00 · 2025-10-31 09:44:44 +08:00 · 2025-10-31 09:40:50 +08:00
11 changed files with 217 additions and 32 deletions
--- a/DOCKER_README.md
+++ b/DOCKER_README.md
@@ -159,7 +159,27 @@ mkdir -p data logs
 sudo chown -R 1000:1000 data logs
 ```

-### 3. 性能优化
+### 3. 日志文件权限问题
+如果遇到 `EACCES: permission denied` 错误：
+
+```bash
+# Linux/Mac
+mkdir -p logs
+chmod 777 logs  # 或者 chmod 755 logs
+
+# Windows PowerShell (如果使用Docker Desktop)
+New-Item -ItemType Directory -Path logs -Force
+# 在Docker Desktop设置中，确保目录有正确权限
+
+# 或者删除logs目录，让容器自动创建
+rm -rf logs
+# 然后重新启动容器
+docker-compose up -d
+```
+
+**注意**：容器内使用 `node` 用户（UID 1000）运行，确保挂载的logs目录对该用户有写入权限。
+
+### 4. 性能优化
 ```bash
 # 限制内存使用
 docker run -d --memory=512m proxy-ip-manager
--- a/11
+++ b/11
@@ -21,22 +21,23 @@ RUN apk add --no-cache \

 # 配置npm使用国内镜像源
 # RUN npm config set registry https://registry.npmmirror.com
-RUN npm -g i pnpm
 # 复制package.json和package-lock.json（如果存在）
 COPY package*.json ./

 # 安装项目依赖
-RUN pnpm i
+#RUN npm config set proyx http://192.168.3.135:1084
+RUN npm config set https-proxy http://192.168.3.135:1084
+RUN npm install

 # 复制项目文件
 COPY . .

 # 创建数据目录和日志目录并设置权限
-RUN mkdir -p /app/data /app/logs && \
-    chown -R node:node /app
+RUN mkdir -p /app/data /app/logs 
+#     chown -R node:node /app

 # 切换到非root用户
-USER node
+# USER node

 # 暴露端口
 EXPOSE 3000
--- a/config/database.js
+++ b/config/database.js
@@ -1,9 +1,38 @@
 const path = require('path');
+const fs = require('fs');
+
+// 数据库文件路径配置
+// 在Docker环境中使用 /app/data，本地开发使用项目根目录的data文件夹
+let dbPath;
+if (process.env.DB_PATH) {
+  // 如果设置了环境变量，使用环境变量的路径
+  dbPath = process.env.DB_PATH;
+} else if (fs.existsSync('/app/data')) {
+  // Docker容器内，使用/app/data目录
+  dbPath = '/app/data/proxies.db';
+} else {
+  // 本地开发环境，使用项目根目录的data文件夹
+  const dataDir = path.join(__dirname, '../data');
+  dbPath = path.join(dataDir, 'proxies.db');
+  
+  // 确保data目录存在
+  if (!fs.existsSync(dataDir)) {
+    try {
+      fs.mkdirSync(dataDir, { recursive: true });
+    } catch (error) {
+      console.warn('无法创建data目录，将使用项目根目录:', error.message);
+      // 如果创建失败，回退到项目根目录
+      dbPath = path.join(__dirname, '../proxies.db');
+    }
+  }
+}
+
+console.log(`数据库文件路径: ${dbPath}`);

 const dbConfig = {
  development: {
    dialect: 'sqlite',
-    storage: path.join(__dirname, '../proxies.db')
+    storage: dbPath
  }
 };

--- a/public/history.html
+++ b/public/history.html
@@ -43,6 +43,11 @@
                            <i class="bi bi-activity"></i> 系统监控
                        </a>
                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="logs.html">
+                            <i class="bi bi-journal-text"></i> 实时日志
+                        </a>
+                    </li>
                </ul>

                <div class="navbar-nav">
--- a/public/index.html
+++ b/public/index.html
@@ -43,6 +43,11 @@
                            <i class="bi bi-activity"></i> 系统监控
                        </a>
                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="logs.html">
+                            <i class="bi bi-journal-text"></i> 实时日志
+                        </a>
+                    </li>
                </ul>

                <div class="navbar-nav">
--- a/public/js/proxies.js
+++ b/public/js/proxies.js
@@ -333,6 +333,42 @@ class ProxyManager {
            document.getElementById('validationStatus').textContent = '验证完成';

            // 显示结果
+            let responseResultHtml = '';
+            
+            if (result.data.responseStatus !== null && result.data.responseStatus !== undefined) {
+                responseResultHtml = `
+                    <hr>
+                    <h6>响应结果:</h6>
+                    <p><strong>状态码:</strong> <span class="badge ${result.data.responseStatus >= 200 && result.data.responseStatus < 300 ? 'bg-success' : 'bg-warning'}">${result.data.responseStatus}</span></p>
+                `;
+                
+                if (result.data.responseData) {
+                    // 限制显示长度，避免内容过长
+                    let displayData = result.data.responseData;
+                    // const maxLength = 300;
+                    // if (displayData.length > maxLength) {
+                    //     displayData = displayData.substring(0, maxLength) + '... (内容已截断，完整内容请查看日志)';
+                    // }
+                    
+                    responseResultHtml += `
+                        <p><strong>响应内容:</strong></p>
+                        <pre class="bg-light p-2 border rounded" style="max-height: 200px; overflow-y: auto; font-size: 12px; white-space: pre-wrap; word-wrap: break-word;">${this.escapeHtml(displayData)}</pre>
+                    `;
+                }
+                
+                if (result.data.testUrl) {
+                    responseResultHtml += `
+                        <p class="text-muted small"><strong>测试URL:</strong> ${result.data.testUrl}</p>
+                    `;
+                }
+            } else if (result.data.error) {
+                responseResultHtml = `
+                    <hr>
+                    <h6>响应结果:</h6>
+                    <p class="text-danger"><strong>错误:</strong> ${result.data.error}</p>
+                `;
+            }
+            
            document.getElementById('validationResults').innerHTML = `
                <div class="alert ${result.data.isValid ? 'alert-success' : 'alert-danger'}">
                    <h6>验证结果: ${result.data.isValid ? '成功' : '失败'}</h6>
@@ -340,6 +376,7 @@ class ProxyManager {
                    <p><strong>响应时间:</strong> ${result.data.responseTime}ms</p>
                    ${result.data.error ? `<p><strong>错误信息:</strong> ${result.data.error}</p>` : ''}
                </div>
+                ${responseResultHtml}
            `;

            // 延迟刷新列表
@@ -581,6 +618,13 @@ class ProxyManager {
        return icons[type] || 'info-circle-fill';
    }

+    escapeHtml(text) {
+        if (!text) return '';
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+
    convertToCSV(data) {
        if (!data || data.length === 0) return '';

--- a/public/monitoring.html
+++ b/public/monitoring.html
@@ -43,6 +43,11 @@
                            <i class="bi bi-activity"></i> 系统监控
                        </a>
                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="logs.html">
+                            <i class="bi bi-journal-text"></i> 实时日志
+                        </a>
+                    </li>
                </ul>

                <div class="navbar-nav">
--- a/public/proxies.html
+++ b/public/proxies.html
@@ -43,6 +43,11 @@
                            <i class="bi bi-activity"></i> 系统监控
                        </a>
                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="logs.html">
+                            <i class="bi bi-journal-text"></i> 实时日志
+                        </a>
+                    </li>
                </ul>

                <div class="navbar-nav">
--- a/src/database/db.js
+++ b/src/database/db.js
@@ -9,15 +9,19 @@ class Database {

  connect() {
    return new Promise((resolve, reject) => {
+      const dbPath = config.development.storage;
+      console.log(`正在连接数据库: ${dbPath}`);
+      
      this.db = new sqlite3.Database(
-        config.development.storage,
+        dbPath,
        sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE,
        (err) => {
          if (err) {
            console.error('数据库连接失败:', err.message);
+            console.error('数据库路径:', dbPath);
            reject(err);
          } else {
-            console.log('已连接到 SQLite 数据库');
+            console.log(`已连接到 SQLite 数据库: ${dbPath}`);
            resolve();
          }
        }
--- a/src/services/scraper.js
+++ b/src/services/scraper.js
@@ -91,19 +91,25 @@ class ProxyScraper {
      return null; // 无本地代理，使用直连
    }

-    // 尝试几个代理，找到可用的
-    for (let i = 0; i < Math.min(5, this.localProxies.length); i++) {
+    // 尝试几个代理，找到可用的（最多尝试3个，避免耗时过长）
+    const maxAttempts = Math.min(3, this.localProxies.length);
+    for (let i = 0; i < maxAttempts; i++) {
      const proxyConfig = this.getNextLocalProxy();

+      try {
        if (await this.testProxyForScraping(proxyConfig)) {
          console.log(`✓ 代理 ${proxyConfig.host}:${proxyConfig.port} 可用`);
          return proxyConfig;
        } else {
          console.log(`✗ 代理 ${proxyConfig.host}:${proxyConfig.port} 不可用，尝试下一个`);
        }
+      } catch (error) {
+        console.log(`✗ 代理 ${proxyConfig.host}:${proxyConfig.port} 测试出错: ${error.message}`);
+        // 继续尝试下一个代理
+      }
    }

-    console.log('测试的本地代理都不可用，使用直连');
+    console.log(`测试了 ${maxAttempts} 个本地代理都不可用，将使用直连`);
    return null; // 所有测试的代理都不可用，使用直连
  }

@@ -181,6 +187,8 @@ class ProxyScraper {

    console.log(`正在抓取第 ${pageNum} 页: ${url}`);

+    let useDirectConnection = false; // 标志：是否应该直接使用直连
+
    for (let attempt = 1; attempt <= retryCount; attempt++) {
      let proxyConfig = null;
      let proxyUsed = '';
@@ -188,8 +196,14 @@ class ProxyScraper {
      try {
        const userAgent = this.getRandomUserAgent();

-        // 获取可用代理配置（每次请求都尝试不同的代理）
+        // 如果之前使用代理失败过，或者标记为使用直连，则跳过代理
+        if (!useDirectConnection && this.localProxies.length > 0) {
+          // 尝试获取可用代理配置
          proxyConfig = await this.getWorkingProxy();
+        } else {
+          proxyConfig = null;
+          console.log('跳过代理，直接使用直连');
+        }

        const requestConfig = {
          headers: {
@@ -242,6 +256,12 @@ class ProxyScraper {
      } catch (error) {
        console.error(`第 ${attempt} 次尝试抓取第 ${pageNum} 页失败 (${proxyUsed}):`, error.message);

+        // 如果使用代理失败，下次重试时使用直连
+        if (proxyConfig) {
+          console.log(`代理 ${proxyConfig.host}:${proxyConfig.port} 抓取失败，下次重试将使用直连`);
+          useDirectConnection = true; // 标记为使用直连
+        }
+
        if (attempt === retryCount) {
          throw new Error(`抓取第 ${pageNum} 页失败，已重试 ${retryCount} 次: ${error.message}`);
        }
--- a/src/services/validator.js
+++ b/src/services/validator.js
@@ -76,16 +76,16 @@ class ProxyValidator {

        // 检查响应内容 - 根据不同的测试URL使用不同的验证逻辑
        if (response.status >= 200 && response.status < 300) {
-          if (testUrl.includes('baidu.com')) {
-            isValid = response.data && response.data.includes('百度');
-          } else if (testUrl.includes('httpbin.org')) {
-            isValid = response.data && (response.data.includes('origin') || response.data.includes('ip'));
-          } else if (testUrl.includes('google.com')) {
-            isValid = response.data && response.data.toLowerCase().includes('google');
-          } else {
+          // if (testUrl.includes('baidu.com')) {
+          //   isValid = response.data && response.data.includes('baidu');
+          // } else if (testUrl.includes('httpbin.org')) {
+          //   isValid = response.data && (response.data.includes('origin') || response.data.includes('ip'));
+          // } else if (testUrl.includes('google.com')) {
+          //   isValid = response.data && response.data.toLowerCase().includes('google');
+          // } else {
            // 对于其他URL，只要能连接就认为有效
            isValid = true;
-          }
+          // }
        }

        // 只要有一次验证成功，立即返回成功结果
@@ -94,13 +94,33 @@ class ProxyValidator {
            console.log(`✓ 代理 ${ip}:${port} 第 ${attemptNumber} 次尝试验证成功，响应时间: ${responseTime}ms`);
          }

+          // 提取响应信息用于显示
+          let responseData = '';
+          let responseStatus = response.status;
+          try {
+            // 只取响应数据的前500个字符，避免数据过大
+            const dataStr = typeof response.data === 'string' 
+              ? response.data 
+              : JSON.stringify(response.data);
+            // responseData = dataStr.substring(0, 500);
+            responseData = dataStr;
+            // if (dataStr.length > 500) {
+            //   responseData += '... (已截断)';
+            // }
+          } catch (e) {
+            responseData = '无法解析响应内容';
+          }
+
          const result = {
            ip: ip,
            port: parseInt(port),
            isValid: true,
            responseTime: responseTime,
            error: null,
-            testUrl: testUrl
+            testUrl: testUrl,
+            responseStatus: responseStatus,
+            responseData: responseData,
+            responseHeaders: response.headers || {}
          };

          // 更新数据库中的验证结果（如果需要）
@@ -124,13 +144,34 @@ class ProxyValidator {
        }
        
        lastError = new Error('响应内容验证失败');
+        // 提取响应信息
+        let responseData = '';
+        let responseStatus = response ? response.status : null;
+        try {
+          if (response && response.data) {
+            const dataStr = typeof response.data === 'string' 
+              ? response.data 
+              : JSON.stringify(response.data);
+            responseData = dataStr;
+            // responseData = dataStr.substring(0, 500);
+            // if (dataStr.length > 500) {
+            //   responseData += '... (已截断)';
+            // }
+          }
+        } catch (e) {
+          responseData = '无法解析响应内容';
+        }
+
        lastResult = {
          ip: ip,
          port: parseInt(port),
          isValid: false,
          responseTime: responseTime,
          error: '响应内容验证失败',
-          testUrl: testUrl
+          testUrl: testUrl,
+          responseStatus: responseStatus,
+          responseData: responseData,
+          responseHeaders: response ? (response.headers || {}) : {}
        };

        // 如果还有重试机会，继续尝试
@@ -147,7 +188,10 @@ class ProxyValidator {
          isValid: false,
          responseTime: responseTime,
          error: error.message,
-          testUrl: testUrl
+          testUrl: testUrl,
+          responseStatus: null,
+          responseData: null,
+          responseHeaders: {}
        };

        if (logResult) {
@@ -184,7 +228,10 @@ class ProxyValidator {
      isValid: false,
      responseTime: Date.now() - startTime,
      error: lastError ? lastError.message : '所有验证尝试都失败',
-      testUrl: testUrl
+      testUrl: testUrl,
+      responseStatus: null,
+      responseData: null,
+      responseHeaders: {}
    };
  }
Author	SHA1	Message	Date
theluyuan	6abae17fea	Implement dynamic database path configuration in database.js, enhancing compatibility with Docker and local development environments. Update db.js to log the database connection path. Remove response data truncation in validator.js for improved readability.	2025-10-31 10:32:44 +08:00
theluyuan	da140d5415	Remove response data truncation logic in ProxyManager for improved readability of displayed content.	2025-10-31 10:28:50 +08:00
theluyuan	4cefbbbcd6	Refactor ProxyScraper to limit proxy attempts to 3 and improve error handling. Added a flag to switch to direct connection if proxy fails. Enhanced logging for proxy testing outcomes.	2025-10-31 10:03:40 +08:00
theluyuan	a4127509af	Enhance ProxyManager and ProxyValidator to display detailed response information, including status codes, response data, and headers. Implement response data truncation for better readability. Update validation logic to capture and return response details in case of errors.	2025-10-31 09:56:31 +08:00
theluyuan	6c3b3928f9	Add "实时日志" link to navigation in history, index, monitoring, and proxies pages	2025-10-31 09:44:44 +08:00
theluyuan	54bc260453	Restore directory creation in Dockerfile for data and logs, removing previous comment-out for improved setup consistency.	2025-10-31 09:40:50 +08:00
theluyuan	d48e0f2d9f	Merge branch 'master' of https://git.theluyuan.com/theluyuan/dailiip	2025-10-31 09:39:57 +08:00
theluyuan	9bdc1bd9b3	Comment out directory creation and user switch in Dockerfile for potential troubleshooting purposes.	2025-10-31 09:39:53 +08:00
theluyuan	8cb074660a	Update DOCKER_README.md to address log file permission issues, providing troubleshooting steps for Linux/Mac and Windows users. Clarify the importance of ensuring the logs directory has the correct permissions for the container's node user.	2025-10-31 09:36:08 +08:00
theluyuan	804f996c69	Merge branch 'master' of https://git.theluyuan.com/theluyuan/dailiip	2025-10-31 09:35:24 +08:00
theluyuan	735d3e0677	proxy	2025-10-31 09:34:05 +08:00