mirror of
https://github.com/dunwu/linux-tutorial.git
synced 2024-04-15 19:55:24 +08:00
167 lines
63 KiB
HTML
167 lines
63 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-US">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<title>Elastic 技术栈之 Logstash 基础 | LINUX-TUTORIAL</title>
|
||
<meta name="generator" content="VuePress 1.8.2">
|
||
<link rel="icon" href="/linux-tutorial/favicon.ico">
|
||
<meta name="description" content="数据库教程">
|
||
|
||
<link rel="preload" href="/linux-tutorial/assets/css/0.styles.45d9d031.css" as="style"><link rel="preload" href="/linux-tutorial/assets/js/app.79a38eea.js" as="script"><link rel="preload" href="/linux-tutorial/assets/js/4.fb6e0f89.js" as="script"><link rel="preload" href="/linux-tutorial/assets/js/53.76541550.js" as="script"><link rel="preload" href="/linux-tutorial/assets/js/5.cb43ecfb.js" as="script"><link rel="prefetch" href="/linux-tutorial/assets/js/10.7933187b.js"><link rel="prefetch" href="/linux-tutorial/assets/js/11.b9b41530.js"><link rel="prefetch" href="/linux-tutorial/assets/js/12.70a5dba8.js"><link rel="prefetch" href="/linux-tutorial/assets/js/13.857dcc43.js"><link rel="prefetch" href="/linux-tutorial/assets/js/14.5a603a55.js"><link rel="prefetch" href="/linux-tutorial/assets/js/15.d217acb7.js"><link rel="prefetch" href="/linux-tutorial/assets/js/16.ad565eae.js"><link rel="prefetch" href="/linux-tutorial/assets/js/17.d43e9f56.js"><link rel="prefetch" href="/linux-tutorial/assets/js/18.aa00ff43.js"><link rel="prefetch" href="/linux-tutorial/assets/js/19.43ce44b3.js"><link rel="prefetch" href="/linux-tutorial/assets/js/20.5618e1ff.js"><link rel="prefetch" href="/linux-tutorial/assets/js/21.1c5a41d7.js"><link rel="prefetch" href="/linux-tutorial/assets/js/22.fbe9fdf1.js"><link rel="prefetch" href="/linux-tutorial/assets/js/23.a4fb0e74.js"><link rel="prefetch" href="/linux-tutorial/assets/js/24.e3a23b69.js"><link rel="prefetch" href="/linux-tutorial/assets/js/25.9896afe9.js"><link rel="prefetch" href="/linux-tutorial/assets/js/26.96164082.js"><link rel="prefetch" href="/linux-tutorial/assets/js/27.391033bb.js"><link rel="prefetch" href="/linux-tutorial/assets/js/28.703f74c2.js"><link rel="prefetch" href="/linux-tutorial/assets/js/29.02a952cb.js"><link rel="prefetch" href="/linux-tutorial/assets/js/30.7e13628f.js"><link rel="prefetch" href="/linux-tutorial/assets/js/31.c4652f75.js"><link rel="prefetch" href="/linux-tutorial/assets/js/32.05d2cbec.js"><link rel="prefetch" href="/linux-tutorial/assets/js/33.3b265df8.js"><link rel="prefetch" href="/linux-tutorial/assets/js/34.26330a03.js"><link rel="prefetch" href="/linux-tutorial/assets/js/35.417d706d.js"><link rel="prefetch" href="/linux-tutorial/assets/js/36.0ed775e0.js"><link rel="prefetch" href="/linux-tutorial/assets/js/37.34430c74.js"><link rel="prefetch" href="/linux-tutorial/assets/js/38.87d5e0ff.js"><link rel="prefetch" href="/linux-tutorial/assets/js/39.7b648b3e.js"><link rel="prefetch" href="/linux-tutorial/assets/js/40.3b7a219e.js"><link rel="prefetch" href="/linux-tutorial/assets/js/41.e727eee9.js"><link rel="prefetch" href="/linux-tutorial/assets/js/42.0134c187.js"><link rel="prefetch" href="/linux-tutorial/assets/js/43.175e982f.js"><link rel="prefetch" href="/linux-tutorial/assets/js/44.72d90888.js"><link rel="prefetch" href="/linux-tutorial/assets/js/45.d49955bd.js"><link rel="prefetch" href="/linux-tutorial/assets/js/46.a9c290ec.js"><link rel="prefetch" href="/linux-tutorial/assets/js/47.cc639f04.js"><link rel="prefetch" href="/linux-tutorial/assets/js/48.98c78321.js"><link rel="prefetch" href="/linux-tutorial/assets/js/49.a7c3afed.js"><link rel="prefetch" href="/linux-tutorial/assets/js/50.22d8c542.js"><link rel="prefetch" href="/linux-tutorial/assets/js/51.28055fcd.js"><link rel="prefetch" href="/linux-tutorial/assets/js/52.f8103df5.js"><link rel="prefetch" href="/linux-tutorial/assets/js/54.e78d2776.js"><link rel="prefetch" href="/linux-tutorial/assets/js/55.3ce3079c.js"><link rel="prefetch" href="/linux-tutorial/assets/js/56.832958c9.js"><link rel="prefetch" href="/linux-tutorial/assets/js/57.961ce896.js"><link rel="prefetch" href="/linux-tutorial/assets/js/58.6d6fbc82.js"><link rel="prefetch" href="/linux-tutorial/assets/js/59.d5e48112.js"><link rel="prefetch" href="/linux-tutorial/assets/js/6.c8f4721c.js"><link rel="prefetch" href="/linux-tutorial/assets/js/60.7927b23b.js"><link rel="prefetch" href="/linux-tutorial/assets/js/61.ee233f24.js"><link rel="prefetch" href="/linux-tutorial/assets/js/62.6ba50cc7.js"><link rel="prefetch" href="/linux-tutorial/assets/js/63.9cbf9f2b.js"><link rel="prefetch" href="/linux-tutorial/assets/js/64.0be148a4.js"><link rel="prefetch" href="/linux-tutorial/assets/js/65.c520257e.js"><link rel="prefetch" href="/linux-tutorial/assets/js/66.f2335390.js"><link rel="prefetch" href="/linux-tutorial/assets/js/67.e5737218.js"><link rel="prefetch" href="/linux-tutorial/assets/js/68.46427a01.js"><link rel="prefetch" href="/linux-tutorial/assets/js/69.450417bb.js"><link rel="prefetch" href="/linux-tutorial/assets/js/7.046e5a1b.js"><link rel="prefetch" href="/linux-tutorial/assets/js/70.072034d2.js"><link rel="prefetch" href="/linux-tutorial/assets/js/8.77fb8967.js"><link rel="prefetch" href="/linux-tutorial/assets/js/9.ebfa537e.js"><link rel="prefetch" href="/linux-tutorial/assets/js/vendors~flowchart.20a64d45.js"><link rel="prefetch" href="/linux-tutorial/assets/js/vendors~notification.ea176280.js">
|
||
<link rel="stylesheet" href="/linux-tutorial/assets/css/0.styles.45d9d031.css">
|
||
</head>
|
||
<body>
|
||
<div id="app" data-server-rendered="true"><div class="theme-container"><header class="navbar"><div class="sidebar-button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="img" viewBox="0 0 448 512" class="icon"><path fill="currentColor" d="M436 124H12c-6.627 0-12-5.373-12-12V80c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12z"></path></svg></div> <a href="/linux-tutorial/" class="home-link router-link-active"><img src="images/dunwu-logo-100.png" alt="LINUX-TUTORIAL" class="logo"> <span class="site-name can-hide">LINUX-TUTORIAL</span></a> <div class="links"><div class="search-box"><input aria-label="Search" autocomplete="off" spellcheck="false" value=""> <!----></div> <nav class="nav-links can-hide"><div class="nav-item"><a href="/linux-tutorial/linux/cli/" class="nav-link">
|
||
Linux 命令
|
||
</a></div><div class="nav-item"><a href="/linux-tutorial/linux/ops/" class="nav-link">
|
||
Linux 运维
|
||
</a></div><div class="nav-item"><a href="/linux-tutorial/linux/soft/" class="nav-link router-link-active">
|
||
Linux 软件运维
|
||
</a></div><div class="nav-item"><a href="/linux-tutorial/docker/" class="nav-link">
|
||
Docker 教程
|
||
</a></div><div class="nav-item"><a href="https://github.com/dunwu/blog" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||
🎯 博客
|
||
<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></div> <a href="https://github.com/dunwu/linux-tutorial" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||
Github
|
||
<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></nav></div></header> <div class="sidebar-mask"></div> <aside class="sidebar"><nav class="nav-links"><div class="nav-item"><a href="/linux-tutorial/linux/cli/" class="nav-link">
|
||
Linux 命令
|
||
</a></div><div class="nav-item"><a href="/linux-tutorial/linux/ops/" class="nav-link">
|
||
Linux 运维
|
||
</a></div><div class="nav-item"><a href="/linux-tutorial/linux/soft/" class="nav-link router-link-active">
|
||
Linux 软件运维
|
||
</a></div><div class="nav-item"><a href="/linux-tutorial/docker/" class="nav-link">
|
||
Docker 教程
|
||
</a></div><div class="nav-item"><a href="https://github.com/dunwu/blog" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
||
🎯 博客
|
||
<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></div> <a href="https://github.com/dunwu/linux-tutorial" target="_blank" rel="noopener noreferrer" class="repo-link">
|
||
Github
|
||
<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></nav> <ul class="sidebar-links"><li><section class="sidebar-group depth-0"><p class="sidebar-heading open"><span>Elastic 技术栈之 Logstash 基础</span> <!----></p> <ul class="sidebar-links sidebar-group-items"><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#简介" class="sidebar-link">简介</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#功能" class="sidebar-link">功能</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#工作原理" class="sidebar-link">工作原理</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#设置" class="sidebar-link">设置</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#设置文件" class="sidebar-link">设置文件</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#logstash-yml-设置项" class="sidebar-link">logstash.yml 设置项</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#启动" class="sidebar-link">启动</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#命令行" class="sidebar-link">命令行</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#配置文件" class="sidebar-link">配置文件</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#插件" class="sidebar-link">插件</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#input" class="sidebar-link">input</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#filter" class="sidebar-link">filter</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#output" class="sidebar-link">output</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#codec" class="sidebar-link">codec</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#实战" class="sidebar-link">实战</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#传输控制台数据" class="sidebar-link">传输控制台数据</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#传输-logback-日志" class="sidebar-link">传输 logback 日志</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#传输文件" class="sidebar-link">传输文件</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#小技巧" class="sidebar-link">小技巧</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#启动、终止应用" class="sidebar-link">启动、终止应用</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#资料" class="sidebar-link">资料</a><ul class="sidebar-sub-headers"></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#推荐阅读" class="sidebar-link">推荐阅读</a><ul class="sidebar-sub-headers"></ul></li></ul></section></li></ul> </aside> <main class="page"> <div class="theme-default-content content__default"><h1 id="elastic-技术栈之-logstash-基础"><a href="#elastic-技术栈之-logstash-基础" class="header-anchor">#</a> Elastic 技术栈之 Logstash 基础</h1> <blockquote><p>本文是 Elastic 技术栈(ELK)的 Logstash 应用。</p> <p>如果不了解 Elastic 的安装、配置、部署,可以参考:<a href="https://github.com/dunwu/JavaStack/blob/master/docs/javatool/elastic/elastic-quickstart.md" target="_blank" rel="noopener noreferrer">Elastic 技术栈之快速入门<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p></blockquote> <h2 id="简介"><a href="#简介" class="header-anchor">#</a> 简介</h2> <p>Logstash 可以传输和处理你的日志、事务或其他数据。</p> <h3 id="功能"><a href="#功能" class="header-anchor">#</a> 功能</h3> <p>Logstash 是 Elasticsearch 的最佳数据管道。</p> <p>Logstash 是插件式管理模式,在输入、过滤、输出以及编码过程中都可以使用插件进行定制。Logstash 社区有超过 200 种可用插件。</p> <h3 id="工作原理"><a href="#工作原理" class="header-anchor">#</a> 工作原理</h3> <p>Logstash 有两个必要元素:<code>input</code> 和 <code>output</code> ,一个可选元素:<code>filter</code>。</p> <p>这三个元素,分别代表 Logstash 事件处理的三个阶段:输入 > 过滤器 > 输出。</p> <p><img src="https://www.elastic.co/guide/en/logstash/current/static/images/basic_logstash_pipeline.png" alt="img"></p> <ul><li>input 负责从数据源采集数据。</li> <li>filter 将数据修改为你指定的格式或内容。</li> <li>output 将数据传输到目的地。</li></ul> <p>在实际应用场景中,通常输入、输出、过滤器不止一个。Logstash 的这三个元素都使用插件式管理方式,用户可以根据应用需要,灵活的选用各阶段需要的插件,并组合使用。</p> <p>后面将对插件展开讲解,暂且不表。</p> <h2 id="设置"><a href="#设置" class="header-anchor">#</a> 设置</h2> <h3 id="设置文件"><a href="#设置文件" class="header-anchor">#</a> 设置文件</h3> <ul><li><strong><code>logstash.yml</code></strong>:logstash 的默认启动配置文件</li> <li><strong><code>jvm.options</code></strong>:logstash 的 JVM 配置文件。</li> <li><strong><code>startup.options</code></strong> (Linux):包含系统安装脚本在 <code>/usr/share/logstash/bin</code> 中使用的选项为您的系统构建适当的启动脚本。安装 Logstash 软件包时,系统安装脚本将在安装过程结束时执行,并使用 <code>startup.options</code> 中指定的设置来设置用户,组,服务名称和服务描述等选项。</li></ul> <h3 id="logstash-yml-设置项"><a href="#logstash-yml-设置项" class="header-anchor">#</a> logstash.yml 设置项</h3> <p>节选部分设置项,更多项请参考:https://www.elastic.co/guide/en/logstash/current/logstash-settings-file.html</p> <table><thead><tr><th>参数</th> <th>描述</th> <th>默认值</th></tr></thead> <tbody><tr><td><code>node.name</code></td> <td>节点名</td> <td>机器的主机名</td></tr> <tr><td><code>path.data</code></td> <td>Logstash及其插件用于任何持久性需求的目录。</td> <td><code>LOGSTASH_HOME/data</code></td></tr> <tr><td><code>pipeline.workers</code></td> <td>同时执行管道的过滤器和输出阶段的工作任务数量。如果发现事件正在备份,或CPU未饱和,请考虑增加此数字以更好地利用机器处理能力。</td> <td>Number of the host’s CPU cores</td></tr> <tr><td><code>pipeline.batch.size</code></td> <td>尝试执行过滤器和输出之前,单个工作线程从输入收集的最大事件数量。较大的批量处理大小一般来说效率更高,但是以增加的内存开销为代价。您可能必须通过设置 <code>LS_HEAP_SIZE</code> 变量来有效使用该选项来增加JVM堆大小。</td> <td><code>125</code></td></tr> <tr><td><code>pipeline.batch.delay</code></td> <td>创建管道事件批处理时,在将一个尺寸过小的批次发送给管道工作任务之前,等待每个事件需要多长时间(毫秒)。</td> <td><code>5</code></td></tr> <tr><td><code>pipeline.unsafe_shutdown</code></td> <td>如果设置为true,则即使在内存中仍存在inflight事件时,也会强制Logstash在关闭期间退出。默认情况下,Logstash将拒绝退出,直到所有接收到的事件都被推送到输出。启用此选项可能会导致关机期间数据丢失。</td> <td><code>false</code></td></tr> <tr><td><code>path.config</code></td> <td>主管道的Logstash配置路径。如果您指定一个目录或通配符,配置文件将按字母顺序从目录中读取。</td> <td>Platform-specific. See [<a href="https://github.com/elastic/logstash/blob/6.1/docs/static/settings-file.asciidoc#dir-layout" target="_blank" rel="noopener noreferrer">dir-layout]<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a>.</td></tr> <tr><td><code>config.string</code></td> <td>包含用于主管道的管道配置的字符串。使用与配置文件相同的语法。</td> <td>None</td></tr> <tr><td><code>config.test_and_exit</code></td> <td>设置为true时,检查配置是否有效,然后退出。请注意,使用此设置不会检查grok模式的正确性。 Logstash可以从目录中读取多个配置文件。如果将此设置与log.level:debug结合使用,则Logstash将记录组合的配置文件,并注掉其源文件的配置块。</td> <td><code>false</code></td></tr> <tr><td><code>config.reload.automatic</code></td> <td>设置为true时,定期检查配置是否已更改,并在配置更改时重新加载配置。这也可以通过SIGHUP信号手动触发。</td> <td><code>false</code></td></tr> <tr><td><code>config.reload.interval</code></td> <td>Logstash 检查配置文件更改的时间间隔。</td> <td><code>3s</code></td></tr> <tr><td><code>config.debug</code></td> <td>设置为true时,将完全编译的配置显示为调试日志消息。您还必须设置<code>log.level:debug</code>。警告:日志消息将包括任何传递给插件配置作为明文的“密码”选项,并可能导致明文密码出现在您的日志!</td> <td><code>false</code></td></tr> <tr><td><code>config.support_escapes</code></td> <td>当设置为true时,带引号的字符串将处理转义字符。</td> <td><code>false</code></td></tr> <tr><td><code>modules</code></td> <td>配置时,模块必须处于上表所述的嵌套YAML结构中。</td> <td>None</td></tr> <tr><td><code>http.host</code></td> <td>绑定地址</td> <td><code>"127.0.0.1"</code></td></tr> <tr><td><code>http.port</code></td> <td>绑定端口</td> <td><code>9600</code></td></tr> <tr><td><code>log.level</code></td> <td>日志级别。有效选项:fatal > error > warn > info > debug > trace</td> <td><code>info</code></td></tr> <tr><td><code>log.format</code></td> <td>日志格式。json (JSON 格式)或 plain (原对象)</td> <td><code>plain</code></td></tr> <tr><td><code>path.logs</code></td> <td>Logstash 自身日志的存储路径</td> <td><code>LOGSTASH_HOME/logs</code></td></tr> <tr><td><code>path.plugins</code></td> <td>在哪里可以找到自定义的插件。您可以多次指定此设置以包含多个路径。</td> <td></td></tr></tbody></table> <h2 id="启动"><a href="#启动" class="header-anchor">#</a> 启动</h2> <h3 id="命令行"><a href="#命令行" class="header-anchor">#</a> 命令行</h3> <p>通过命令行启动 logstash 的方式如下:</p> <div class="language- extra-class"><pre class="language-text"><code>bin/logstash [options]
|
||
</code></pre></div><p>其中 [options] 是您可以指定用于控制 Logstash 执行的命令行标志。</p> <p>在命令行上设置的任何标志都会覆盖 Logstash 设置文件(<code>logstash.yml</code>)中的相应设置,但设置文件本身不会更改。</p> <blockquote><p><strong>注</strong></p> <p>虽然可以通过指定命令行参数的方式,来控制 logstash 的运行方式,但显然这么做很麻烦。</p> <p>建议通过指定配置文件的方式,来控制 logstash 运行,启动命令如下:</p> <div class="language- extra-class"><pre class="language-text"><code>bin/logstash -f logstash.conf
|
||
</code></pre></div><p>若想了解更多的命令行参数细节,请参考:https://www.elastic.co/guide/en/logstash/current/running-logstash-command-line.html</p></blockquote> <h3 id="配置文件"><a href="#配置文件" class="header-anchor">#</a> 配置文件</h3> <p>上节,我们了解到,logstash 可以执行 <code>bin/logstash -f logstash.conf</code> ,按照配置文件中的参数去覆盖默认设置文件(<code>logstash.yml</code>)中的设置。</p> <p>这节,我们就来学习一下这个配置文件如何配置参数。</p> <h4 id="配置文件结构"><a href="#配置文件结构" class="header-anchor">#</a> 配置文件结构</h4> <p>在工作原理一节中,我们已经知道了 Logstash 主要有三个工作阶段 input 、filter、output。而 logstash 配置文件文件结构也与之相对应:</p> <div class="language- extra-class"><pre class="language-text"><code>input {}
|
||
|
||
filter {}
|
||
|
||
output {}
|
||
</code></pre></div><blockquote><p>每个部分都包含一个或多个插件的配置选项。如果指定了多个过滤器,则会按照它们在配置文件中的显示顺序应用它们。</p></blockquote> <h4 id="插件配置"><a href="#插件配置" class="header-anchor">#</a> 插件配置</h4> <p>插件的配置由插件名称和插件的一个设置块组成。</p> <p>下面的例子中配置了两个输入文件配置:</p> <div class="language- extra-class"><pre class="language-text"><code>input {
|
||
file {
|
||
path => "/var/log/messages"
|
||
type => "syslog"
|
||
}
|
||
|
||
file {
|
||
path => "/var/log/apache/access.log"
|
||
type => "apache"
|
||
}
|
||
}
|
||
</code></pre></div><p>您可以配置的设置因插件类型而异。你可以参考: <a href="https://www.elastic.co/guide/en/logstash/current/input-plugins.html" target="_blank" rel="noopener noreferrer">Input Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a>, <a href="https://www.elastic.co/guide/en/logstash/current/output-plugins.html" target="_blank" rel="noopener noreferrer">Output Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a>, <a href="https://www.elastic.co/guide/en/logstash/current/filter-plugins.html" target="_blank" rel="noopener noreferrer">Filter Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a>, 和 <a href="https://www.elastic.co/guide/en/logstash/current/codec-plugins.html" target="_blank" rel="noopener noreferrer">Codec Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。</p> <h4 id="值类型"><a href="#值类型" class="header-anchor">#</a> 值类型</h4> <p>一个插件可以要求设置的值是一个特定的类型,比如布尔值,列表或哈希值。以下值类型受支持。</p> <ul><li>Array</li></ul> <div class="language- extra-class"><pre class="language-text"><code> users => [ {id => 1, name => bob}, {id => 2, name => jane} ]
|
||
</code></pre></div><ul><li>Lists</li></ul> <div class="language- extra-class"><pre class="language-text"><code> path => [ "/var/log/messages", "/var/log/*.log" ]
|
||
uris => [ "http://elastic.co", "http://example.net" ]
|
||
</code></pre></div><ul><li>Boolean</li></ul> <div class="language- extra-class"><pre class="language-text"><code> ssl_enable => true
|
||
</code></pre></div><ul><li>Bytes</li></ul> <div class="language- extra-class"><pre class="language-text"><code> my_bytes => "1113" # 1113 bytes
|
||
my_bytes => "10MiB" # 10485760 bytes
|
||
my_bytes => "100kib" # 102400 bytes
|
||
my_bytes => "180 mb" # 180000000 bytes
|
||
</code></pre></div><ul><li>Codec</li></ul> <div class="language- extra-class"><pre class="language-text"><code> codec => "json"
|
||
</code></pre></div><ul><li>Hash</li></ul> <div class="language- extra-class"><pre class="language-text"><code>match => {
|
||
"field1" => "value1"
|
||
"field2" => "value2"
|
||
...
|
||
}
|
||
</code></pre></div><ul><li>Number</li></ul> <div class="language- extra-class"><pre class="language-text"><code> port => 33
|
||
</code></pre></div><ul><li>Password</li></ul> <div class="language- extra-class"><pre class="language-text"><code> my_password => "password"
|
||
</code></pre></div><ul><li>URI</li></ul> <div class="language- extra-class"><pre class="language-text"><code> my_uri => "http://foo:bar@example.net"
|
||
</code></pre></div><ul><li>Path</li></ul> <div class="language- extra-class"><pre class="language-text"><code> my_path => "/tmp/logstash"
|
||
</code></pre></div><ul><li><p>String</p></li> <li><p>转义字符</p></li></ul> <h2 id="插件"><a href="#插件" class="header-anchor">#</a> 插件</h2> <h3 id="input"><a href="#input" class="header-anchor">#</a> input</h3> <blockquote><p>Logstash 支持各种输入选择 ,可以在同一时间从众多常用来源捕捉事件。能够以连续的流式传输方式,轻松地从您的日志、指标、Web 应用、数据存储以及各种 AWS 服务采集数据。</p></blockquote> <h4 id="常用-input-插件"><a href="#常用-input-插件" class="header-anchor">#</a> 常用 input 插件</h4> <ul><li><strong>file</strong>:从文件系统上的文件读取,就像UNIX命令 <code>tail -0F</code> 一样</li> <li>**syslog:**在众所周知的端口514上侦听系统日志消息,并根据RFC3164格式进行解析</li> <li>**redis:**从redis服务器读取,使用redis通道和redis列表。 Redis经常用作集中式Logstash安装中的“代理”,它将来自远程Logstash“托运人”的Logstash事件排队。</li> <li>**beats:**处理由Filebeat发送的事件。</li></ul> <p>更多详情请见:<a href="https://www.elastic.co/guide/en/logstash/current/input-plugins.html" target="_blank" rel="noopener noreferrer">Input Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h3 id="filter"><a href="#filter" class="header-anchor">#</a> filter</h3> <blockquote><p>过滤器是Logstash管道中的中间处理设备。如果符合特定条件,您可以将条件过滤器组合在一起,对事件执行操作。</p></blockquote> <h4 id="常用-filter-插件"><a href="#常用-filter-插件" class="header-anchor">#</a> 常用 filter 插件</h4> <ul><li><p>**grok:**解析和结构任意文本。 Grok目前是Logstash中将非结构化日志数据解析为结构化和可查询的最佳方法。</p></li> <li><p>**mutate:**对事件字段执行一般转换。您可以重命名,删除,替换和修改事件中的字段。</p></li> <li><p>**drop:**完全放弃一个事件,例如调试事件。</p></li> <li><p>**clone:**制作一个事件的副本,可能会添加或删除字段。</p></li> <li><p>**geoip:**添加有关IP地址的地理位置的信息(也可以在Kibana中显示惊人的图表!)</p></li></ul> <p>更多详情请见:<a href="https://www.elastic.co/guide/en/logstash/current/filter-plugins.html" target="_blank" rel="noopener noreferrer">Filter Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h3 id="output"><a href="#output" class="header-anchor">#</a> output</h3> <blockquote><p>输出是Logstash管道的最后阶段。一个事件可以通过多个输出,但是一旦所有输出处理完成,事件就完成了执行。</p></blockquote> <h4 id="常用-output-插件"><a href="#常用-output-插件" class="header-anchor">#</a> 常用 output 插件</h4> <ul><li>**elasticsearch:**将事件数据发送给 Elasticsearch(推荐模式)。</li> <li>**file:**将事件数据写入文件或磁盘。</li> <li>**graphite:**将事件数据发送给 graphite(一个流行的开源工具,存储和绘制指标。 http://graphite.readthedocs.io/en/latest/)。</li> <li>**statsd:**将事件数据发送到 statsd (这是一种侦听统计数据的服务,如计数器和定时器,通过UDP发送并将聚合发送到一个或多个可插入的后端服务)。</li></ul> <p>更多详情请见:<a href="https://www.elastic.co/guide/en/logstash/current/output-plugins.html" target="_blank" rel="noopener noreferrer">Output Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h3 id="codec"><a href="#codec" class="header-anchor">#</a> codec</h3> <p>用于格式化对应的内容。</p> <h4 id="常用-codec-插件"><a href="#常用-codec-插件" class="header-anchor">#</a> 常用 codec 插件</h4> <ul><li>**json:**以JSON格式对数据进行编码或解码。</li> <li>**multiline:**将多行文本事件(如java异常和堆栈跟踪消息)合并为单个事件。</li></ul> <p>更多插件请见:<a href="https://www.elastic.co/guide/en/logstash/current/codec-plugins.html" target="_blank" rel="noopener noreferrer">Codec Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h2 id="实战"><a href="#实战" class="header-anchor">#</a> 实战</h2> <p>前面的内容都是对 Logstash 的介绍和原理说明。接下来,我们来实战一些常见的应用场景。</p> <h3 id="传输控制台数据"><a href="#传输控制台数据" class="header-anchor">#</a> 传输控制台数据</h3> <blockquote><p>stdin input 插件从标准输入读取事件。这是最简单的 input 插件,一般用于测试场景。</p></blockquote> <p><strong>应用</strong></p> <p>(1)创建 <code>logstash-input-stdin.conf</code> :</p> <div class="language- extra-class"><pre class="language-text"><code>input { stdin { } }
|
||
output {
|
||
elasticsearch { hosts => ["localhost:9200"] }
|
||
stdout { codec => rubydebug }
|
||
}
|
||
</code></pre></div><p>更多配置项可以参考:https://www.elastic.co/guide/en/logstash/current/plugins-inputs-stdin.html</p> <p>(2)执行 logstash,使用 <code>-f</code> 来指定你的配置文件:</p> <div class="language- extra-class"><pre class="language-text"><code>bin/logstash -f logstash-input-stdin.conf
|
||
</code></pre></div><h3 id="传输-logback-日志"><a href="#传输-logback-日志" class="header-anchor">#</a> 传输 logback 日志</h3> <blockquote><p>elk 默认使用的 Java 日志工具是 log4j2 ,并不支持 logback 和 log4j。</p> <p>想使用 logback + logstash ,可以使用 <a href="https://github.com/logstash/logstash-logback-encoder" target="_blank" rel="noopener noreferrer">logstash-logback-encoder<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。<a href="https://github.com/logstash/logstash-logback-encoder" target="_blank" rel="noopener noreferrer">logstash-logback-encoder<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 提供了 UDP / TCP / 异步方式来传输日志数据到 logstash。</p> <p>如果你使用的是 log4j ,也不是不可以用这种方式,只要引入桥接 jar 包即可。如果你对 log4j 、logback ,或是桥接 jar 包不太了解,可以参考我的这篇博文:<a href="https://github.com/dunwu/JavaStack/blob/master/docs/javalib/java-log.md" target="_blank" rel="noopener noreferrer">细说 Java 主流日志工具库<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。</p></blockquote> <h4 id="tcp-应用"><a href="#tcp-应用" class="header-anchor">#</a> TCP 应用</h4> <ol><li><p>logstash 配置</p> <p>(1)创建 <code>logstash-input-tcp.conf</code> :</p></li></ol> <div class="language- extra-class"><pre class="language-text"><code>input {
|
||
tcp {
|
||
port => 9251
|
||
codec => json_lines
|
||
mode => server
|
||
}
|
||
}
|
||
output {
|
||
elasticsearch { hosts => ["localhost:9200"] }
|
||
stdout { codec => rubydebug }
|
||
}
|
||
</code></pre></div><p>更多配置项可以参考:https://www.elastic.co/guide/en/logstash/current/plugins-inputs-tcp.html</p> <p>(2)执行 logstash,使用 <code>-f</code> 来指定你的配置文件:<code>bin/logstash -f logstash-input-udp.conf</code></p> <ol start="2"><li><p>java 应用配置</p> <p>(1)在 Java 应用的 pom.xml 中引入 jar 包:</p></li></ol> <div class="language-xml extra-class"><pre class="language-xml"><code><span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>net.logstash.logback<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>logstash-logback-encoder<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>4.11<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span>
|
||
|
||
<span class="token comment"><!-- logback 依赖包 --></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>ch.qos.logback<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>logback-core<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>1.2.3<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>ch.qos.logback<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>logback-classic<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>1.2.3<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>ch.qos.logback<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>logback-access<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>1.2.3<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span>
|
||
</code></pre></div><p>(2)接着,在 logback.xml 中添加 appender</p> <div class="language-xml extra-class"><pre class="language-xml"><code><span class="token tag"><span class="token tag"><span class="token punctuation"><</span>appender</span> <span class="token attr-name">name</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>ELK-TCP<span class="token punctuation">"</span></span> <span class="token attr-name">class</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>net.logstash.logback.appender.LogstashTcpSocketAppender<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
|
||
<span class="token comment"><!--
|
||
destination 是 logstash 服务的 host:port,
|
||
相当于和 logstash 建立了管道,将日志数据定向传输到 logstash
|
||
--></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>destination</span><span class="token punctuation">></span></span>192.168.28.32:9251<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>destination</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>encoder</span> <span class="token attr-name">charset</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>UTF-8<span class="token punctuation">"</span></span> <span class="token attr-name">class</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>net.logstash.logback.encoder.LogstashEncoder<span class="token punctuation">"</span></span><span class="token punctuation">/></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>appender</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>logger</span> <span class="token attr-name">name</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>io.github.dunwu.spring<span class="token punctuation">"</span></span> <span class="token attr-name">level</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>TRACE<span class="token punctuation">"</span></span> <span class="token attr-name">additivity</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>false<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>appender-ref</span> <span class="token attr-name">ref</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>ELK-TCP<span class="token punctuation">"</span></span> <span class="token punctuation">/></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>logger</span><span class="token punctuation">></span></span>
|
||
</code></pre></div><p>(3)接下来,就是 logback 的具体使用 ,如果对此不了解,不妨参考一下我的这篇博文:<a href="https://github.com/dunwu/JavaStack/blob/master/docs/javalib/java-log.md" target="_blank" rel="noopener noreferrer">细说 Java 主流日志工具库<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。</p> <p><strong>实例:</strong><a href="https://github.com/dunwu/JavaStack/blob/master/codes/javatool/src/main/resources/logback.xml" target="_blank" rel="noopener noreferrer">我的logback.xml<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h4 id="udp-应用"><a href="#udp-应用" class="header-anchor">#</a> UDP 应用</h4> <p>UDP 和 TCP 的使用方式大同小异。</p> <ol><li><p>logstash 配置</p> <p>(1)创建 <code>logstash-input-udp.conf</code> :</p></li></ol> <div class="language- extra-class"><pre class="language-text"><code>input {
|
||
udp {
|
||
port => 9250
|
||
codec => json
|
||
}
|
||
}
|
||
output {
|
||
elasticsearch { hosts => ["localhost:9200"] }
|
||
stdout { codec => rubydebug }
|
||
}
|
||
</code></pre></div><p>更多配置项可以参考:https://www.elastic.co/guide/en/logstash/current/plugins-inputs-udp.html</p> <p>(2)执行 logstash,使用 <code>-f</code> 来指定你的配置文件:<code>bin/logstash -f logstash-input-udp.conf</code></p> <ol start="2"><li><p>java 应用配置</p> <p>(1)在 Java 应用的 pom.xml 中引入 jar 包:</p> <p>与 <strong>TCP 应用</strong> 一节中的引入依赖包完全相同。</p> <p>(2)接着,在 logback.xml 中添加 appender</p></li></ol> <div class="language-xml extra-class"><pre class="language-xml"><code><span class="token tag"><span class="token tag"><span class="token punctuation"><</span>appender</span> <span class="token attr-name">name</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>ELK-UDP<span class="token punctuation">"</span></span> <span class="token attr-name">class</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>net.logstash.logback.appender.LogstashSocketAppender<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>host</span><span class="token punctuation">></span></span>192.168.28.32<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>host</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>port</span><span class="token punctuation">></span></span>9250<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>port</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>appender</span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>logger</span> <span class="token attr-name">name</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>io.github.dunwu.spring<span class="token punctuation">"</span></span> <span class="token attr-name">level</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>TRACE<span class="token punctuation">"</span></span> <span class="token attr-name">additivity</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>false<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>appender-ref</span> <span class="token attr-name">ref</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>ELK-UDP<span class="token punctuation">"</span></span> <span class="token punctuation">/></span></span>
|
||
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>logger</span><span class="token punctuation">></span></span>
|
||
</code></pre></div><p>(3)接下来,就是 logback 的具体使用 ,如果对此不了解,不妨参考一下我的这篇博文:<a href="https://github.com/dunwu/JavaStack/blob/master/docs/javalib/java-log.md" target="_blank" rel="noopener noreferrer">细说 Java 主流日志工具库<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。</p> <p><strong>实例:</strong><a href="https://github.com/dunwu/JavaStack/blob/master/codes/javatool/src/main/resources/logback.xml" target="_blank" rel="noopener noreferrer">我的logback.xml<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h3 id="传输文件"><a href="#传输文件" class="header-anchor">#</a> 传输文件</h3> <blockquote><p>在 Java Web 领域,需要用到一些重要的工具,例如 Tomcat 、Nginx 、Mysql 等。这些不属于业务应用,但是它们的日志数据对于定位问题、分析统计同样很重要。这时无法使用 logback 方式将它们的日志传输到 logstash。</p> <p>如何采集这些日志文件呢?别急,你可以使用 logstash 的 file input 插件。</p> <p>需要注意的是,传输文件这种方式,必须在日志所在的机器上部署 logstash 。</p></blockquote> <p><strong>应用</strong></p> <p>logstash 配置</p> <p>(1)创建 <code>logstash-input-file.conf</code> :</p> <div class="language- extra-class"><pre class="language-text"><code>input {
|
||
file {
|
||
path => ["/var/log/nginx/access.log"]
|
||
type => "nginx-access-log"
|
||
start_position => "beginning"
|
||
}
|
||
}
|
||
|
||
output {
|
||
if [type] == "nginx-access-log" {
|
||
elasticsearch {
|
||
hosts => ["localhost:9200"]
|
||
index => "nginx-access-log"
|
||
}
|
||
}
|
||
}
|
||
</code></pre></div><p>(2)执行 logstash,使用 <code>-f</code> 来指定你的配置文件:<code>bin/logstash -f logstash-input-file.conf</code></p> <p>更多配置项可以参考:https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html</p> <h2 id="小技巧"><a href="#小技巧" class="header-anchor">#</a> 小技巧</h2> <h3 id="启动、终止应用"><a href="#启动、终止应用" class="header-anchor">#</a> 启动、终止应用</h3> <p>如果你的 logstash 每次都是通过指定配置文件方式启动。不妨建立一个启动脚本。</p> <div class="language- extra-class"><pre class="language-text"><code># cd xxx 进入 logstash 安装目录下的 bin 目录
|
||
logstash -f logstash.conf
|
||
</code></pre></div><p>如果你的 logstash 运行在 linux 系统下,不妨使用 nohup 来启动一个守护进程。这样做的好处在于,即使关闭终端,应用仍会运行。</p> <p><strong>创建 startup.sh</strong></p> <div class="language- extra-class"><pre class="language-text"><code>nohup ./logstash -f logstash.conf >> nohup.out 2>&1 &
|
||
</code></pre></div><p>终止应用没有什么好方法,你只能使用 ps -ef | grep logstash ,查出进程,将其kill 。不过,我们可以写一个脚本来干这件事:</p> <p><strong>创建 shutdown.sh</strong></p> <p>脚本不多解释,请自行领会作用。</p> <div class="language- extra-class"><pre class="language-text"><code>PID=`ps -ef | grep logstash | awk '{ print $2}' | head -n 1`
|
||
kill -9 ${PID}
|
||
</code></pre></div><h2 id="资料"><a href="#资料" class="header-anchor">#</a> 资料</h2> <ul><li><a href="https://www.elastic.co/guide/en/logstash/current/index.html" target="_blank" rel="noopener noreferrer">Logstash 官方文档<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li> <li><a href="https://github.com/logstash/logstash-logback-encoder" target="_blank" rel="noopener noreferrer">logstash-logback-encoder<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li> <li><a href="https://github.com/chenryn/logstash-best-practice-cn" target="_blank" rel="noopener noreferrer">ELK Stack权威指南<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li> <li><a href="https://github.com/judasn/Linux-Tutorial/blob/master/ELK-Install-And-Settings.md" target="_blank" rel="noopener noreferrer">ELK(Elasticsearch、Logstash、Kibana)安装和配置<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li></ul> <h2 id="推荐阅读"><a href="#推荐阅读" class="header-anchor">#</a> 推荐阅读</h2> <ul><li><a href="https://github.com/dunwu/JavaStack/blob/master/docs/javatool/elastic/README.md" target="_blank" rel="noopener noreferrer">Elastic 技术栈<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li> <li><a href="https://github.com/dunwu/JavaStack" target="_blank" rel="noopener noreferrer">JavaStack<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li></ul></div> <footer class="page-edit"><div class="edit-link"><a href="https://github.com/dunwu/linux-tutorial/edit/master/docs/linux/soft/elastic/elastic-logstash.md" target="_blank" rel="noopener noreferrer">帮助我们改善此页面!</a> <span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></div> <div class="last-updated"><span class="prefix">上次更新:</span> <span class="time">a year ago</span></div></footer> <!----> </main></div><div class="global-ui"><!----><!----></div></div>
|
||
<script src="/linux-tutorial/assets/js/app.79a38eea.js" defer></script><script src="/linux-tutorial/assets/js/4.fb6e0f89.js" defer></script><script src="/linux-tutorial/assets/js/53.76541550.js" defer></script><script src="/linux-tutorial/assets/js/5.cb43ecfb.js" defer></script>
|
||
</body>
|
||
</html>
|